16.04快速安装Ceph集群
准备工作
-
假设集群: 选一台作管理机
注意: ceph集群引用hostname,而非ip.172.17.4.16 linuxidc16 #hostname必须是linuxidc16 172.17.4.17 linuxidc17 #hostname必须是linuxidc17 172.17.4.18 linuxidc18 #hostname必须是linuxidc18 172.17.4.19 linuxidc19 #hostname必须是linuxidc19, 选作管理机 172.17.4.20 linuxidc20 #hostname必须是linuxidc20 172.17.4.21 linuxidc21 #hostname必须是linuxidc21 172.17.4.22 linuxidc22 #hostname必须是linuxidc22
-
安装用户
注意: 建议不要使用"ceph",因为"apt install ceph"需要使用ceph. 用root最省事 ^_^root@linuxidc19:~# ssh-keygen #创建安装用rsa,安装完记得删除 root@linuxidc19:~# less .ssh/id_rsa.pub ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDZAQ8ikono13kGzRLpyx1BCRfzICieEKg27WABW+2eFkYXmbvzLSu8XkmC3a0x7M0ADd2aW2FGv8ZtxPgHMRFzyaeoXwGZUFU5zmd9CASglHibKlWzdUgl5sW/DSdSxFYs+ooQvZcKSMmbFYlo70Ndv5kHja2FEMKnLexJpox4fFb9Rv1KEez0p1j28iICc3nHXLAZigezKkaFiv/Wfk/g59nk6ZCNAr1UnDvTdQOnVTIu6Q6UkpoiDx3+R5oeWRynI2l6T/bhrMpubO5PaluFKvas0DF3eGZLf2XQPR8sYzt7PkOGC1QANi7D11UxTqurZF8Meb0LdH0KEqE1kH7z root@cadmin # 在各台机执行,authorized_keys必须是600 mkdir /root/.ssh echo -n 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDZAQ8ikono13kGzRLpyx1BCRfzICieEKg27WABW+2eFkYXmbvzLSu8XkmC3a0x7M0ADd2aW2FGv8ZtxPgHMRFzyaeoXwGZUFU5zmd9CASglHibKlWzdUgl5sW/DSdSxFYs+ooQvZcKSMmbFYlo70Ndv5kHja2FEMKnLexJpox4fFb9Rv1KEez0p1j28iICc3nHXLAZigezKkaFiv/Wfk/g59nk6ZCNAr1UnDvTdQOnVTIu6Q6UkpoiDx3+R5oeWRynI2l6T/bhrMpubO5PaluFKvas0DF3eGZLf2XQPR8sYzt7PkOGC1QANi7D11UxTqurZF8Meb0LdH0KEqE1kH7z' > /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys
测试用户
#在linuxidc19无密ssh for node in 16 17 18 19 20 21 22; do ssh linuxidc$node echo 123; done; #确认yes The authenticity of host 'linuxidc22 (172.17.4.22)' can't be established. ECDSA key fingerprint is SHA256:6S9mp0m8bV15bKX9RugP2CflINUvxORw82yCwJqVyrE. Are you sure you want to continue connecting (yes/no)? yes
-
更新源
root@linuxidc19: for node in 16 17 18 19 20 21 22; do ssh linuxidc$node \ wget -q -O- https://download.ceph.com/keys/release.asc | sudo apt-key add -;\ echo deb https://download.ceph.com/debian-jewel/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list;\ apt update && apt -y dist-upgrade && apt -y autoremove; done #apt使用-y默认Yes
-
安装ceph-deploy
root@linuxidc19: apt install ceph-deploy
注意: 16.04下这二个版本都有bug.
- 1.5.32在执行ceph-deploy mon create-initial命令后生成的ceph.client.admin.keyring文件是空的!
-
1.5.34则在重装时不会替换/var/lib/ceph/bootstrap-osd/ceph.keyring, 造成ceph-deploy osd activate失败.
root@linuxidc19:~# apt-cache policy ceph-deploy ceph-deploy: Installed: 1.5.34 Candidate: 1.5.34 Version table: *** 1.5.34 999 999 https://download.ceph.com/debian-jewel xenial/main amd64 Packages 999 https://download.ceph.com/debian-jewel xenial/main i386 Packages 100 /var/lib/dpkg/status 1.5.32-0ubuntu1 500 500 http://hk.archive.ubuntu.com/ubuntu xenial/universe amd64 Packages 500 http://hk.archive.ubuntu.com/ubuntu xenial/universe i386 Packages
安装过程
-
创建管理目录
root@linuxidc19:~# mkdir ceph-admin root@linuxidc19:~# cd ceph-admin/
-
初始配置
root@linuxidc19:~/ceph-admin# ceph-deploy new linuxidc16 test17 drwxr-xr-x 2 root root 4096 Sep 14 10:28 ./ drwx------ 6 root root 4096 Sep 14 10:28 ../ -rw-r--r-- 1 root root 215 Sep 14 10:28 ceph.conf -rw-r--r-- 1 root root 4080 Sep 14 10:28 ceph-deploy-ceph.log -rw------- 1 root root 73 Sep 14 10:28 ceph.mon.keyring root@linuxidc19:~/ceph-admin# vi ceph.conf [global] fsid = c96ee478-4df7-4100-9792-c078cf44b3c4 mon_initial_members = linuxidc16, test17 mon_host = 172.17.4.16,172.17.4.17 auth_cluster_required = cephx auth_service_required = cephx auth_client_required = cephx public_network = 172.17.4.0/24 cluster_network = 172.17.4.0/24 # 以下是针对ext4的bug osd_max_object_name_len = 256 osd_max_object_namespace_len = 64 osd_check_max_object_name_len_on_startup = false
需要注意: jewel 10.2.1对EXT4格式有个文件名长度BUG!
-
安装ceph
root@linuxidc19:~/ceph-admin# for node in 16 17 18 19 20 21 22; do ceph-deploy install linuxidc$node; done;
-
创建初始mon结点
root@linuxidc19:~/ceph-admin# ceph-deploy mon create-initial
-
创建激活osd结点, 去掉16,17
root@linuxidc19:~/ceph-admin# for node in 18 19 20 21 22; do ssh linuxidc$node 'mkdir -R /data/ceph/osd; chown ceph:ceph -R /data/ceph; rm -rf /data/ceph/osd/*'; ceph-deploy osd prepare linuxidc$node:/data/ceph/osd; ceph-deploy osd activate linuxidc$node:/data/ceph/osd; done;
-
分发管理配置
root@linuxidc19:~/ceph-admin# for node in 16 17 18 19 20 21 22; do ceph-deploy admin linuxidc$node; ssh linuxidc$node 'chmod a+r /etc/ceph/ceph.client.admin.keyring'; done;
-
测试集群是否安装成功
root@linuxidc19:~/ceph-admin# ceph -s root@linuxidc19:~/ceph-admin# ceph osd tree root@linuxidc19:~/ceph-admin# ceph osd perf