目录
一.实验环境分析
| 主机名 | IP地址 | 内存 | 硬盘 | 系统 |
| ceph01 | NAT：192.168.43.101/24 VM1：192.168.100.11/24 |
2G | 系统盘：20G OSD盘：500G |
centos7.4（1708） |
| ceph02 | NAT：192.168.43.102/24 VM1：192.168.100.12/24 |
2G | 系统盘：20G OSD盘：500G |
centos7.4（1708） |
| ceph03 | NAT：192.168.43.103/24 VM1：192.168.43.13/24 |
2G | 系统盘：20G OSD盘：500G |
centos7.4（1708） |
二.集群部署操作
基础环境配置（三个节点都操作）
- 配置主机名
hostnamectl set-hostname ceph01
hostnamectl set-hostname ceph02
hostnamectl set-hostname ceph03- 配置hosts映射文件
vi /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.100.11 ceph01
192.168.100.12 ceph02
192.168.100.13 ceph03
- 关闭防火墙和核心防护
systemctl stop firewalld
setenforce 0- 配置网络（以ceph02为例）
[root@ceph02 network-scripts]# cat ifcfg-ens36
TYPE="Ethernet"
PROXY_METHOD="none"
BROWSER_ONLY="no"
BOOTPROTO="static"
DEFROUTE="yes"
IPV4_FAILURE_FATAL="no"
IPV6INIT="yes"
IPV6_AUTOCONF="yes"
IPV6_DEFROUTE="yes"
IPV6_FAILURE_FATAL="no"
IPV6_ADDR_GEN_MODE="stable-privacy"
NAME="ens36"
DEVICE="ens36"
ONBOOT="yes"
IPADDR="192.168.100.12"
PREFIX="24"
GATEWAY="192.168.100.1"
IPV6_PRIVACY="no"
[root@ceph02 network-scripts]#
##VM1网卡是和物理机相匹配
##重启网卡之后确保IP地址能够正常使用
##且能够ping通 www.baidu.com- 配置ssh密钥免交互
ssh-keygen（执行此条命令，一直回车下去）
ssh-copy-id root@ceph01
ssh-copy-id root@ceph02
ssh-copy-id root@ceph03
##操作完成之后最好测试一下配置YUM源（三个节点都操作）
- 开启yum缓存（最后制作镜像包）
vi /etc/yum.conf
keepcache=1- 备份本机的源
cd /etc/yum.repos.d
mkdir backup
mv C* backup- 下载centos源和epel源
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
##注意如果是最小化安装需要yum一下安装工具，再备份之前
yum install wget curl -y- 编辑ceph的源
cd /etc/yum.repos.d
vi ceph.repo
[ceph]
name=Ceph packages for
baseurl=https://mirrors.aliyun.com/ceph/rpm-mimic/el7/$basearch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://mirrors.aliyun.com/ceph/keys/release.asc
priority=1
[ceph-noarch]
name=Ceph noarch packages
baseurl=https://mirrors.aliyun.com/ceph/rpm-mimic/el7/noarch/
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://mirrors.aliyun.com/ceph/keys/release.asc
priority=1
[ceph-source]
name=Ceph source packages
baseurl=https://mirrors.aliyun.com/ceph/rpm-mimic/el7/SRPMS/
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://mirrors.aliyun.com/ceph/keys/release.asc
priority=1
- 更新一下源（时间花费较长，需要保持网络通畅）
yum update -y配置时钟同步
- ceph01配置时间服务器
[root@ceph01 ~]# yum install -y ntpdate ntp
[root@ceph01 ~]# ntpdate ntp1.aliyun.com
30 Mar 20:54:47 ntpdate[56630]: adjust time server 120.25.115.20 offset 0.012804 sec
[root@ceph01 ~]#
[root@ceph01 ~]# clock -w ##把当前系统时间保存到CMOS中
[root@ceph01 ~]# vi /etc/ntp.conf
[root@ceph01 ~]# cat /etc/ntp.conf ##清空原有数据
driftfile /var/lib/ntp/drift
restrict default nomodify
restrict 127.0.0.1
restrict ::1
restrict 192.168.100.0 mask 255.255.255.0 nomodify notrap fudge 127.127.1.0 stratum 10
server 127.127.1.0
includefile /etc/ntp/crypto/pw
keys /etc/ntp/keys
disable monitor
[root@ceph01 ~]# systemctl start ntpd
- ceph02配置同步客户端
[root@ceph02 ~]# yum install -y ntpdate
[root@ceph02 ~]# ntpdate ceph01 ##同步ceph01的时间
30 Mar 21:05:41 ntpdate[88782]: adjust time server 192.168.100.11 offset 0.026245 sec
[root@ceph02 ~]# crontab -e
no crontab for root - using an empty one
crontab: installing new crontab
[root@ceph02 ~]# crontab -l ##定制周期性计划表
*/5 * * * * /usr/bin/ntpdate 192.168.100.11 >> /var/log/ntpdate.log
[root@ceph02 ~]# systemctl restart crond
[root@ceph02 ~]#
- ceph03配置同步客户端
[root@ceph03 ~]# yum install ntpdate -y
[root@ceph03 ~]# ntpdate ceph01
30 Mar 21:08:47 ntpdate[88537]: adjust time server 192.168.100.11 offset 0.003575 sec
[root@ceph03 ~]# crontab -e
no crontab for root - using an empty one
crontab: installing new crontab
[root@ceph03 ~]# crontab -l
**/5 * * * * /usr/bin/ntpdate 192.168.100.11 >> /var/log/ntpdate.log
[root@ceph03 ~]# systemctl restart crond
[root@ceph03 ~]#
CEPH集群搭建
- 在ceph01上安装管理工具和ceph软件包
mkdir /etc/ceph
yum install -y python-setuptools ceph-deploy ceph
- ceph02安装ceph软件包
[root@ceph02 ~]# mkdir /etc/ceph
[root@ceph02 ~]# yum install -y ceph- ceph03安装ceph软件包
[root@ceph03 ~]# mkdir /etc/ceph
[root@ceph03 ~]# yum install -y ceph
-
在ceph01上创建mon监控（暂时监控ceph01和ceph02，ceph03留作扩容）
[root@ceph01 ~]# cd /etc/ceph/
[root@ceph01 ceph]# ls
rbdmap
[root@ceph01 ceph]# ceph-deploy new ceph01 ceph02
[root@ceph01 ceph]# ls
ceph.conf ceph-deploy-ceph.log ceph.mon.keyring rbdmap
[root@ceph01 ceph]# ceph-deploy mon create-initial ##初始化mon，并且收取密钥
[root@ceph01 ceph]# ls
ceph.bootstrap-mds.keyring ceph.bootstrap-rgw.keyring ceph-deploy-ceph.log
ceph.bootstrap-mgr.keyring ceph.client.admin.keyring ceph.mon.keyring
ceph.bootstrap-osd.keyring ceph.conf rbdmap
[root@ceph01 ceph]#
-
在ceph01上创建osd，（暂时制作ceph01和ceph02，ceph03留作osd扩容）
[root@ceph01 ~]# cd /etc/ceph/ ##必须进入这个目录，调用ceph.conf文件
[root@ceph01 ceph]# ceph-deploy osd create --data /dev/sdb ceph01
[root@ceph01 ceph]# ceph-deploy osd create --data /dev/sdb ceph02
- 下发密钥到ceph01、ceph02，且添加权限
##在ceph01上
[root@ceph01 ~]# cd /etc/ceph/
[root@ceph01 ceph]# ceph-deploy admin ceph01 ceph02
[root@ceph01 ceph]# chmod +x ceph.client.admin.keyring
##在ceph02上
[root@ceph02 ~]# chmod +x /etc/ceph/ceph.client.admin.keyring
[root@ceph02 ~]#
三.CEPH集群的扩容操作以及数据恢复
OSD的扩容
- 登录ceph01，将ceph03上的磁盘，添加到ceph集群中作为osd
[root@ceph01 ~]# cd /etc/ceph/
[root@ceph01 ceph]# ceph-deploy osd create --data /dev/sdb ceph03
MON的扩容（在ceph03上添加mon）
- 登录ceph01做内部通信设置，并且下发给ceph02、ceph03
[root@ceph01 ~]# cd /etc/ceph/
[root@ceph01 ceph]# vi ceph.conf
[root@ceph01 ceph]# echo "public network=192.168.100.0/24" >> ceph.conf
[root@ceph01 ceph]# cat ceph.conf
[global]
fsid = 21278c49-7b43-4f60-b460-6075d17c946a
mon_initial_members = ceph01, ceph02
mon_host = 192.168.100.11,192.168.100.12
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
public network=192.168.100.0/24
[root@ceph01 ceph]# ceph-deploy --overwrite-conf admin ceph02 ceph03 ##将文件下发- 登录ceph03，添加密钥权限
[root@ceph03 ceph]# ls
ceph.client.admin.keyring ceph.conf rbdmap tmprfRxWN tmpsm6tdW
[root@ceph03 ceph]# chmod +x ceph.client.admin.keyring - 登录ceph01，将ceph03 mon 加入集群
[root@ceph01 ~]# cd /etc/ceph/
[root@ceph01 ceph]# ceph-deploy --overwrite-conf admin ceph02 ceph03
[root@ceph01 ceph]# 
模拟企业生产环境中OSD数据恢复
- 登录ceph01模拟故障，删除ceph03的osd
[root@ceph01 ceph]# ceph osd out osd.2 ##移除osd.2
[root@ceph01 ceph]# ceph osd crush remove osd.2 ##删除osd.2
[root@ceph01 ceph]# ceph auth del osd.2 ##删除osd.2的认证（权重）
[root@ceph01 ceph]# ceph osd down osd.2 ##停止osd.2d的使用
marked down osd.2.
[root@ceph01 ceph]# ceph -s
cluster:
id: 21278c49-7b43-4f60-b460-6075d17c946a
health: HEALTH_WARN
no active mgr
services:
mon: 3 daemons, quorum ceph01,ceph02,ceph03
mgr: no daemons active
osd: 3 osds: 2 up, 2 in
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs:
[root@ceph01 ceph]# ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 0.97458 root default
-3 0.48729 host ceph01
0 hdd 0.48729 osd.0 up 1.00000 1.00000
-5 0.48729 host ceph02
1 hdd 0.48729 osd.1 up 1.00000 1.00000
-7 0 host ceph03
2 0 osd.2 down 0 1.00000
[root@ceph01 ceph]# ceph osd rm osd.2 ##彻底删除osd.2
removed osd.2
[root@ceph01 ceph]# ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 0.97458 root default
-3 0.48729 host ceph01
0 hdd 0.48729 osd.0 up 1.00000 1.00000
-5 0.48729 host ceph02
1 hdd 0.48729 osd.1 up 1.00000 1.00000
-7 0 host ceph03
[root@ceph01 ceph]#
- 恢复osd到集群中,登录到ceph03中
[root@ceph03 ~]# cd /var/lib/ceph/osd/ceph-2/
[root@ceph03 ceph-2]# ls
block ceph_fsid fsid keyring ready require_osd_release type whoami
[root@ceph03 ceph-2]# cat fsid ##osd.2的uuid
b4585b33-391b-4f83-a578-e1ee2d429033
[root@ceph03 ceph-2]# ceph osd create b4585b33-391b-4f83-a578-e1ee2d429033 #创建UUID
2
[root@ceph03 ceph-2]# ceph auth add osd.2 osd 'allow *' mon 'allow rwx' -i /var/lib/ceph/osd/ceph-2/keyring ##增加权限
added key for osd.2
[root@ceph03 ceph-2]# ceph osd crush add 2 0.48729 host=ceph03 #0.48729为权重，参照之前的osd.2的权重
set item id 2 name 'osd.2' weight 0.48729 at location {host=ceph03}: no change
[root@ceph03 ceph-2]# ceph osd in osd.2
marked in osd.2.
[root@ceph03 ceph-2]# ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 1.46187 root default
-3 0.48729 host ceph01
0 hdd 0.48729 osd.0 up 1.00000 1.00000
-5 0.48729 host ceph02
1 hdd 0.48729 osd.1 up 1.00000 1.00000
-7 0.48729 host ceph03
2 0.48729 osd.2 down 1.00000 1.00000
[root@ceph03 ceph-2]# systemctl restart ceph-osd.target ##启动osd.2
[root@ceph03 ceph-2]# ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 1.46187 root default
-3 0.48729 host ceph01
0 hdd 0.48729 osd.0 up 1.00000 1.00000
-5 0.48729 host ceph02
1 hdd 0.48729 osd.1 up 1.00000 1.00000
-7 0.48729 host ceph03
2 hdd 0.48729 osd.2 up 1.00000 1.00000
[root@ceph03 ceph-2]#
由于ceph是分布式存储，具有容错性
四.CEPH的常规维护命令
创建mgr服务
- 登录ceph01
[root@ceph01 ceph]# ceph -s
cluster:
id: 21278c49-7b43-4f60-b460-6075d17c946a
health: HEALTH_WARN
no active mgr
services:
mon: 3 daemons, quorum ceph01,ceph02,ceph03
mgr: no daemons active
osd: 3 osds: 3 up, 3 in
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 0 B used, 0 B / 0 B avail
pgs:
[root@ceph01 ceph]# ceph-deploy mgr create ceph01 ceph02 ceph03
[root@ceph01 ceph]# ceph -s
cluster:
id: 21278c49-7b43-4f60-b460-6075d17c946a
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph01,ceph02,ceph03
mgr: ceph01(active), standbys: ceph02, ceph03
osd: 3 osds: 3 up, 3 in
data:
pools: 0 pools, 0 pgs
objects: 0 objects, 0 B
usage: 3.0 GiB used, 1.5 TiB / 1.5 TiB avail
pgs:
[root@ceph01 ceph]#
pool的基本操作
- 创建pool
[root@ceph01 ~]# ceph osd pool ls
[root@ceph01 ~]# ceph osd pool create cinder 64
pool 'cinder' created
[root@ceph01 ~]# ceph osd pool create glance 64
pool 'glance' created
[root@ceph01 ~]# ceph osd pool ls
cinder
glance
- 删除pool
1.设置删除权限，下发文件
[root@ceph01 ~]# cd /etc/ceph/
[root@ceph01 ceph]# cat ceph.conf
[global]
fsid = 21278c49-7b43-4f60-b460-6075d17c946a
mon_initial_members = ceph01, ceph02
mon_host = 192.168.100.11,192.168.100.12
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
public network= 192.168.100.0/24
mon_allow_pool_delete = true
[root@ceph01 ceph]# ceph-deploy --overwrite-conf admin ceph01 ceph02 ceph03
systemctl restart ceph-mon.target ##重启三个节点的mon服务
2.删除pool
[root@ceph01 ceph]# ceph osd pool rm glance glance --yes-i-really-really-mean-it
pool 'glance' removed
[root@ceph01 ceph]# ceph osd pool ls
cinder
[root@ceph01 ceph]#
- 重命名pool
[root@ceph01 ceph]# ceph osd pool ls
cinder
[root@ceph01 ceph]# ceph osd pool rename cinder CINDER
pool 'cinder' renamed to 'CINDER'
[root@ceph01 ceph]# ceph osd pool ls
CINDER
[root@ceph01 ceph]#
注：一般的下发命令都必须在/etc/ceph这个目录下进行
