目录
前言
环境：centos 7.9 k8s 1.22.17 ceph集群
安装ceph集群
首先得有ceph集群，ceph集群的安装详情参考https://blog.csdn.net/MssGuo/article/details/122280657，这里仅简要给出ceph的安装步骤：
注意：这里的安装ceph集群是使用ceph-deploy工具安装的，官方已经不建议使用该工具安装，请参考ceph官网。
#准备3台服务器，配置主机名本地解析
vim /etc/hosts
192.168.118.128 ceph1
192.168.118.129 ceph2
192.168.118.130 ceph3
#关闭防火墙等基本操作
systemctl stop firewalld
systemctl disable firewalld
vim /etc/selinux/config
setenforce 0
yum install ntp
systemctl enable ntpd
systemctl start ntpd;
#安装epel源和ceph源
yum install epel-release -y #安装配置epel源
vim /etc/yum.repos.d/ceph.repo #配置ceph源
[ceph]
name=ceph
baseurl=http://mirrors.aliyun.com/ceph/rpm-mimic/el7/x86_64/
enabled=1
gpgcheck=0
priority=1
[ceph-noarch]
name=cephnoarch
baseurl=http://mirrors.aliyun.com/ceph/rpm-mimic/el7/noarch/
enabled=1
gpgcheck=0
priority=1
[ceph-source]
name=Ceph source packages
baseurl=http://mirrors.aliyun.com/ceph/rpm-mimic/el7/SRPMS
enabled=1
gpgcheck=0
priority=1
#ssh免密登录
ssh-keygen -t rsa
ssh-copy-id -i /root/.ssh/id_rsa.pub root@ceph1
ssh-copy-id -i /root/.ssh/id_rsa.pub root@ceph2
ssh-copy-id -i /root/.ssh/id_rsa.pub root@ceph3
ssh root@ceph1
ssh root@ceph2
ssh root@ceph3
#ceph1节点安装部署工具
yum install ceph-deploy -y
mkdir /etc/ceph && cd /etc/ceph #在创建一个目录，用于保存ceph-deploy生成的配置文件
yum install -y python-setuptools #先安装python-setuptools依赖，防止报错
ceph-deploy new node1 #创建一个集群，node1是主机名，不是集群名
yum install ceph ceph-radosgw -y #在node1、node2、node3上安装软件
#在client客户端服务器（如有）安装
yum -y install ceph-common
cd /etc/ceph/ #以下操作的目录均在集群的配置目录下面进行操作
vim ceph.conf
public_network = 192.168.118.0/24 #monitor网络,写网段即可
ceph-deploy mon create-initial #创建初始化monitor监控
ceph-deploy admin node1 node2 node3 #将配置文件信息同步到所有节点
ceph-deploy mon add node2 #加多一个mon
ceph-deploy mon add node3 #再加多一个mon
ceph-deploy mgr create node1 #创建一个mgr,node1是主机名
ceph-deploy mgr create node2 #同理再创建一个node2
ceph-deploy mgr create node3 #再创建一个node3
#列表所有node节点的磁盘,都有sda和sdb两个盘,sdb为我们要加入分布式存储的盘
ceph-deploy disk list node1 #列出node1节点服务器的磁盘
ceph-deploy disk list node2 #列出node2节点服务器的磁盘
ceph-deploy disk list node3 #列出node3节点服务器的磁盘
#zap表示干掉磁盘上的数据,相当于格式化
ceph-deploy disk zap node1 /dev/sdb #格式化node1上的sdb磁盘
ceph-deploy disk zap node2 /dev/sdb #格式化node2上的sdb磁盘
ceph-deploy disk zap node3 /dev/sdb #格式化node3上的sdb磁盘
ceph-deploy osd create --data /dev/sdb node1 #将node1上的sdb磁盘创建为osd
ceph-deploy osd create --data /dev/sdb node2 #继续将node2上的sdb磁盘创建为osd
ceph-deploy osd create --data /dev/sdb node3 #继续将node3上的sdb磁盘创建为osd
ceph集群创建rbd块存储
我们要模拟的是k8s静态pv，所以要使用ceph的rbd块存储，首先ceph集群得有rbd块存储，下面将在ceph集群管理节点ceph1上演示创建rbd块存储。
#首先的有pool池，创建pool池
ceph osd pool create k8s-pool 16 #创建了一个pool池，名称叫做k8s-pool
rbd create k8s --pool k8s-pool --size 1024 #创建了一个名称叫做k8s的rbd块存储，大小为1G
rbd feature disable k8s-pool/k8s object-map fast-diff deep-flatten
#不要执行rbd map k8s-pool/k8s 映射成为设备，否则k8s的pod会报已经使用
rbd块存储不支持ReadWriteMany
在官网，，如下，我们知道rbd块存储不支持ReadWriteMany挂载。
https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes
k8s配置rbd块存储（静态供给）
创建secret
#在ceph集群上执行这个命令就可以得到keying
[root@node1 ceph]# ceph auth get-key client.admin
AQAgv4ZkabOqHBAAq+8Eh/Q/8raOcRLW/atLxA==
[root@node1 ceph]# cat /etc/ceph/ceph.mon.keyring #或者查看ceph集群配置目录的ceph.mon.keyring 文件也可以
[mon.]
key = AQA8vYZkAAAAABAAhuMfp97xZYf8JgkWlHZsCA==
caps mon = allow *
[root@node1 ceph]#
#上面我们知道ceph集群的keyring了，即client.admin的keying
#客户端要挂载rbd块设备就必须知道这个keying，所以我们要创建secret保存它
echo -n 'AQAgv4ZkabOqHBAAq+8Eh/Q/8raOcRLW/atLxA==' | base64 #对字符串进行加密，由于echo 默认字符串后面换行，所以-n参数很重要，可以去掉换行符
QVFBZ3Y0WmthYk9xSEJBQXErOEVoL1EvOHJhT2NSTFcvYXRMeEE9PQ== #得到秘文
[root@master ceph]# echo 'QVFBZ3Y0WmthYk9xSEJBQXErOEVoL1EvOHJhT2NSTFcvYXRMeEE9PQ==' | base64 --decode #解密验证看看对不对
AQAgv4ZkabOqHBAAq+8Eh/Q/8raOcRLW/atLxA==[root@master ceph]# #没有换行符，正确的
或者直接在ceph集群中进行base64加密亦可：
ceph auth get-key client.admin |base64 #得到的秘文和上面的秘文是一样的
#编写secret文件
vim ceph-secret.yaml
apiVersion: v1
data:
key: QVFBZ3Y0WmthYk9xSEJBQXErOEVoL1EvOHJhT2NSTFcvYXRMeEE9PQ== #上面的秘文
kind: Secret
metadata:
name: ceph-secret
namespace: default
type: kubernetes.io/rbd #这个类型k8s内置的rbd类型
kubectl apply -f ceph-secret.yaml #创建secret
创建pv
#我们先查看pv的rbd块存储的字段有哪些
[root@master ceph]# kubectl explain pv.spec.rbd
FIELDS:
fsType 文件系统类型，"ext4", "xfs"，默认ext4，
image The rados image name.必须参数
keyring RBDUser的Keyring文件. 默认是/etc/ceph/keyring.
monitors Ceph monitors，即监视器，必须参数
pool The rados pool name. 不写默认是rbd. More info:
readOnly 是否只读挂载，默认是false. More info:
secretRef 包含RBDUser认证的秘钥，如果提供在覆盖keyring.
user The rados user name. Default is admin。
#编写pv资源清单
vim rbd-pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: rdb-pv
namespace: default
spec:
accessModes: #rbd块存储只支持ReadWriteOnce、ReadOnlyMany
- ReadWriteOnce
capacity:
storage: 200M
rbd:
monitors: #ceph集群的monitor的IP+端口，可以写多个，多个可以提供高可用
- '192.168.158.142:6789' #ceph集群的monitor端口就是6789
- '192.168.158.143:6789'
- '192.168.158.144:6789'
pool: k8s-pool #rbd块存储所在的pool，即上面在ceph创建的k8s-pool 池
image: k8s #image其实就是rbd块存储的名称，即上面在ceph集群创建的k8s 块存储，这里只是在ceph集群中叫法不一样而已
fsType: xfs #rbd块设备挂载到pod里面的挂载点文件系统
readOnly: false
user: admin #ceph集群中的rados用户名，默认是admin，我们ceph集群默认就是这个用户名
secretRef:
name: ceph-secret #保存了admin用户的keying 秘钥
persistentVolumeReclaimPolicy: Delete
kubectl apply -f rbd-pv.yaml
创建pvc
[root@master ceph]# cat rbd-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: rbd-pvc
namespace: default
spec:
accessModes:
- ReadWriteOnce #rbd块存储只支持ReadWriteOnce、ReadOnlyMany
resources:
requests:
storage: 200M
storageClassName: "" #写空字符串，表示不使用存储类
kubectl apply -f rbd-pvc.yaml
k8s节点安装客户端依赖包
#由于不知道pod将会调度到哪个节点，所以每个k8s集群都要安装ceph-common
#这个依赖包里面有相应的命令，kubelet会使用到相应的命令进行rbd存储挂载
yum install ceph-common -y
部署pod
vim nginx-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx
spec:
replicas: 1 #先设定1个副本
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:1.18
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
name: web
volumeMounts:
- name: www
mountPath: /usr/share/nginx/html
volumes:
- name: www
persistentVolumeClaim:
claimName: rbd-pvc
kubectl apply -f nginx-deployment.yaml
查看pod
[root@master ceph]# kubectl get pod
NAME READY STATUS RESTARTS AGE
nginx-77cbdf8dc8-sqwt2 1/1 Running 0 9m22s
验证是否持久化
[root@master ceph]# kubectl exec -it nginx-77cbdf8dc8-sqwt2 -- bash #进入pod里面
root@nginx-77cbdf8dc8-sqwt2:/# cd /usr/share/nginx/html
root@nginx-77cbdf8dc8-sqwt2://usr/share/nginx/html# echo "good" >index.html #创建一个首页文件并写点内容
root@nginx-77cbdf8dc8-sqwt2://usr/share/nginx/html# curl localhost:80
good
root@nginx-77cbdf8dc8-sqwt2://usr/share/nginx/html# exit
[root@master ceph]# kubectl get pod -owide
NAME READY STATUS RESTARTS AGE IP NODE
nginx-77cbdf8dc8-sqwt2 1/1 Running 0 12m 10.244.1.45 node1
[root@master ceph]# curl 10.244.1.45:80 #正常访问
good
#删除容器
[root@master ceph]# kubectl delete pod nginx-77cbdf8dc8-sqwt2 --grace-period=0 --force
[root@master ceph]# kubectl get pod -owide
NAME READY STATUS RESTARTS AGE IP NODE
nginx-77cbdf8dc8-fkxrc 1/1 Running 0 20s 10.244.2.45 node2
[root@master ceph]# curl 10.244.2.45:80 #正常访问，说明持久化成功
good
[root@master ceph]#
#pod扩容副本为2个，验证是否正常
[root@master ceph]# kubectl scale deployment nginx --replicas=2
[root@master ceph]# kubectl get pod -owide
NAME READY STATUS RESTARTS AGE IP NODE
nginx-77cbdf8dc8-564pv 0/1 ContainerCreating 0 9s <none> node1
nginx-77cbdf8dc8-fkxrc 1/1 Running 0 118s 10.244.2.45 node2
[root@master ceph]# kubectl describe pod nginx-77cbdf8dc8-564pv
.......
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled 17s default-scheduler Successfully assigned default/nginx-77cbdf8dc8-564pv to node1
Warning FailedAttachVolume 17s attachdetach-controller Multi-Attach error for volume "rdb-pv" Volume is already used by pod(s) nginx-77cbdf8dc8-fkxrc
[root@master ceph]#
以上验证说明，只能有一个pod挂载rbd块存储，如果是两个pod或多个，就会报错，原因很简单，官方也说不支持多个pod挂载rbd块存储。
k8s配置rbd块存储（动态供给）
先将上面静态验证的资源全部删除干净。
查看官网
#通过查看官网，如下，我们知道，k8s内置了rbd的制备器（Provisioner），所以我们不需要手动创建Provisioner。
https://kubernetes.io/zh-cn/docs/concepts/storage/storage-classes/
ceph集群创建pool
默认你已经有ceph集群了，这里我们要在k8s中的存储类使用ceph的rbd块存储，存储类会动态的创建rbd块，所以我们只需要事先在ceph集群中创建pool池即可：
#在ceph集群中创建pool池
ceph osd pool create k8s-pool 16 #创建了一个pool池，名称叫做k8s-pool
创建secret
#我们要知道ceph集群的keyring，即client.admin的keying
#客户端要挂载rbd块设备就必须知道这个keying，所以我们要创建secret保存它
ceph auth get-key client.admin |base64 #在ceph集群管理节点执行
QVFBZ3Y0WmthYk9xSEJBQXErOEVoL1EvOHJhT2NSTFcvYXRMeEE9PQ== #得到秘文
#编写secret文件
vim ceph-secret.yaml
apiVersion: v1
data:
key: QVFBZ3Y0WmthYk9xSEJBQXErOEVoL1EvOHJhT2NSTFcvYXRMeEE9PQ== #上面的秘文
kind: Secret
metadata:
name: ceph-secret
namespace: default
type: kubernetes.io/rbd #这个类型k8s内置的rbd类型
kubectl apply -f ceph-secret.yaml #创建secret
创建rbd存储类
#官网例子：https://kubernetes.io/zh-cn/docs/concepts/storage/storage-classes/#ceph-rbd
vim rbd-storageclass.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: ceph-rbd-storageclass
provisioner: kubernetes.io/rbd #k8s内置的rbd的sc制备器
allowVolumeExpansion: true #允许自动扩容
parameters: #monitor写多个，写为一行，道号分隔
monitors: 192.168.158.142:6789,192.168.158.143:6789,192.168.158.144:6789
adminId: admin #ceph集群的admin用户
adminSecretName: ceph-secret #秘钥
adminSecretNamespace: default
pool: k8s-pool #ceph中创建好的pool池
userId: admin #这里应该是有两种用户，user这种应该应该是ceph集群创建的普通用户，这里默认是admin
userSecretName: ceph-secret
userSecretNamespace: default
fsType: ext4
imageFormat: "2"
imageFeatures: "layering"
创建pvc
vim rbd-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: rbd-pvc
namespace: default
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 200M
storageClassName: "ceph-rbd-storageclass"
kubectl apply -f rbd-pvc.yaml
pvc一直Pending
[root@master ceph]# kubectl describe pvc rbd-pvc
Warning ProvisioningFailed 9s (x2 over 18s) persistentvolume-controller Failed to provision volume with StorageClass "ceph-rbd-storageclass": failed to create rbd image: executable file not found in $PATH, command output:
#排查发现，该问题早在多年前就已经出现了，而根本原因在于，k8s内置的rbd provisioner存在问题，通过查看controller-manager日志可以看到
[root@master ceph]# kubectl logs kube-controller-manager-master -n kube-system
E0613 03:26:28.072518 1 rbd.go:706] rbd: create volume failed, err: failed to create rbd image: executable file not found in $PATH, command output:
E0613 03:26:28.072599 1 goroutinemap.go:150] Operation for "provision-default/rbd-pvc[2759f972-7d36-44eb-bbdf-d35c049f4f9d]" failed. No retries permitted until 2023-06-13 03:28:30.072573469 +0000 UTC m=+6526.000817530 (durationBeforeRetry 2m2s). Error: failed to create rbd image: executable file not found in $PATH, command output:
I0613 03:26:28.097211 1 event.go:291] "Event occurred" object="default/rbd-pvc" kind="PersistentVolumeClaim" apiVersion="v1" type="Warning" reason="ProvisioningFailed" message="Failed to provision volume with StorageClass \"ceph-rbd-storageclass\": failed to create rbd image: executable file not found in $PATH, command output: "
#解决办法就是创建新的provisioner，不在使用k8s内置的rbd provisioner。
创建存储制备器provisioner
官方文档：https://github.com/kubernetes-retired/external-storage/tree/master/ceph/rbd/deploy
官方提供两种安装provisioner的方法，一种是没有rbac，一种是由rbac，任选一种即可
方法一（no-rbac）:
vim deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: rbd-provisioner
spec:
replicas: 1
selector:
matchLabels:
app: rbd-provisioner
strategy:
type: Recreate
template:
metadata:
labels:
app: rbd-provisioner
spec:
containers:
- name: rbd-provisioner
image: "quay.io/external_storage/rbd-provisioner:latest"
env:
- name: PROVISIONER_NAME
value: ceph.com/rbd #记住这个值，这个是provisioner制备器的名称
kubectl apply -f deployment.yaml
方法二（rbac）:
[root@master rbac]# vim clusterrole.yaml
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-provisioner
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
- apiGroups: [""]
resources: ["services"]
resourceNames: ["kube-dns","coredns"]
verbs: ["list", "get"]
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
[root@master rbac]#
[root@master rbac]# vim clusterrolebinding.yaml
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-provisioner
subjects:
- kind: ServiceAccount
name: rbd-provisioner
namespace: default #命名空间可以自行修改
roleRef:
kind: ClusterRole
name: rbd-provisioner
apiGroup: rbac.authorization.k8s.io
[root@master rbac]#
[root@master rbac]# vim role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: rbd-provisioner
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get"]
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
[root@master rbac]#
[root@master rbac]# cat rolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: rbd-provisioner
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rbd-provisioner
subjects:
- kind: ServiceAccount
name: rbd-provisioner
namespace: default #命名空间可以自行修改
[root@master rbac]#
[root@master rbac]# vim serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: rbd-provisioner
[root@master rbac]#
[root@master rbac]# vim deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: rbd-provisioner
spec:
replicas: 1
selector:
matchLabels:
app: rbd-provisioner
strategy:
type: Recreate
template:
metadata:
labels:
app: rbd-provisioner
spec:
containers:
- name: rbd-provisioner
image: "quay.io/external_storage/rbd-provisioner:latest"
env:
- name: PROVISIONER_NAME
value: ceph.com/rbd #记住这个值，这个是provisioner制备器的名称
serviceAccount: rbd-provisioner
[root@master rbac]#
kubectl apply -f ./rabc
重存修改storageclass的provisioner
[root@master ceph]# cat rbd-storageclass.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: ceph-rbd-storageclass
provisioner: ceph.com/rbd #改为制备器的名称，注意不是deployment的名称
allowVolumeExpansion: true
parameters:
monitors: 192.168.158.142:6789,192.168.158.143:6789,192.168.158.144:6789
adminId: admin
adminSecretName: ceph-secret
adminSecretNamespace: default
pool: k8s-pool
userId: admin
userSecretName: ceph-secret
userSecretNamespace: default
fsType: ext4
imageFormat: "2"
imageFeatures: "layering"
[root@master ceph]#
kubectl delete rbd-storageclass.yaml;
kubectl apply -f rbd-storageclass.yaml;
#然后创建pvc，查看正常，已经马上创建了pv
继续创建pvc
kubectl delete-f rbd-pvc.yaml
kubectl apply -f rbd-pvc.yaml
[root@master ceph]# kubectl get -f rbd-pvc.yaml #可以看到存储类已经正常创建了pv
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
rbd-pvc Bound pvc-ab63babe-c743-4ea4-8d59-9944dc9ac000 191Mi RWO ceph-rbd-storageclass 8m44s
[root@master ceph]#
#我们回到ceph集群的管理节点看看
[root@node1 ~]# rbd ls k8s-pool #可以看到，已经创建了image，即rbd块
kubernetes-dynamic-pvc-d2ce2db1-09a7-11ee-a9cd-72b4f5f91329 #（名字不用管，查看rbd-provisioner的日志可以看到创建rbd image的日志信息）
[root@node1 ~]#
创建deployment验证
[root@master ceph]# cat nginx-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx
spec:
replicas: 1
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:1.18
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
name: web
volumeMounts:
- name: www
mountPath: /usr/share/nginx/html
volumes:
- name: www
persistentVolumeClaim:
claimName: rbd-pvc
[root@master ceph]#
[root@master ceph]# kubectl apply -f nginx-deployment.yaml
创建验证
[root@master ceph]# kubectl get pod
NAME READY STATUS RESTARTS AGE
nginx-77cbdf8dc8-nsdwn 1/1 Running 0 2m20s
[root@master ceph]# kubectl exec -it nginx-77cbdf8dc8-nsdwn -- bash
root@nginx-77cbdf8dc8-nsdwn:/# df -Th
Filesystem Type Size Used Avail Use% Mounted on
/dev/rbd0 ext4 181M 1.6M 176M 1% /usr/share/nginx/html
root@nginx-77cbdf8dc8-nsdwn:/# echo "good" >>/usr/share/nginx/html/index.html
root@nginx-77cbdf8dc8-nsdwn:/# curl localhost:80
good
root@nginx-77cbdf8dc8-nsdwn:/# exit
[root@master ceph]# kubectl describe pod nginx-77cbdf8dc8-nsdwn | grep -i ip
IP: 10.244.1.47
[root@master ceph]# curl 10.244.1.47:80
good
[root@master ceph]# kubectl delete pod nginx-77cbdf8dc8-nsdwn
[root@master ceph]# kubectl describe pod nginx-77cbdf8dc8-dczhq | grep -i ip
IP: 10.244.2.48
[root@master ceph]# curl 10.244.2.48:80
good
[root@master ceph]#
#验证成功，rbd不能有2个或多个pod同时读写挂载这里就不在验证了
总结
1、首先得有ceph集群；
2、k8s集群节点安装yum -y install cepe-common；
3、静态供给的话，首先得在ceph集群创建pool和rbd块存储（也称image），然后创建secret，secret主要是保存keying；
4、创建pv、pvc、pod；
5、动态供给的话，首先得在ceph集群创建pool（不用创建rbd块存储即image，存储类会自动创建），同理创建secret，secret主要是保存keying；
6、创建存储制备器，由于k8s内置的存储制备器有点问题，所以需要根据官网手动创建一个rbd存储制备器；
7、创建存储类，使用上面创建的的存储制备器和secret；
8、创建pvc，存储类就会自动创建pv，在ceph集群管理节点上查看rbd 的image，如命令rbd ls k8s-pool，就能看到自动创建的image了。
9、pod挂载pvc使用；
10、由于rbd不能使用ReadWriteMany的pv访问模式，所以rbd块存储不适合多个pod对一个pvc同读同写的应用场景，rbd更合适StatefulSet创建的pod。因为StatefulSet创建的pod，每个pod都独占一个pv，这正好合适rbd存储。