Moosefs分布式文件系统
1. MFS简介
2. MFS部署
2.1 主master的部署
[root@demo1 ~]# curl "http://ppa.moosefs.com/MooseFS-3-el7.repo" > /etc/yum.repos.d/MooseFS.repo #下载repo源
[root@demo1 yum.repos.d]# vim MooseFS.repo
[root@demo1 yum.repos.d]# cat MooseFS.repo
[MooseFS]
name=MooseFS $releasever - $basearch
baseurl=http://ppa.moosefs.com/moosefs-3/yum/el7
gpgcheck=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-MooseFS
enabled=1
[root@demo1 yum.repos.d]# yum install -y moosefs-master ##安装MFS master
[root@demo1 yum.repos.d]# yum install moosefs-cgi moosefs-cgiserv moosefs-cli -y ##安装主服务器
[root@demo1 yum.repos.d]# cd /etc/mfs/
[root@demo1 mfs]# ls
mfsexports.cfg mfsmaster.cfg mfstopology.cfg
mfsexports.cfg.sample mfsmaster.cfg.sample mfstopology.cfg.sample
[root@demo1 mfs]# systemctl start moosefs-master
[root@demo1 mfs]# vim /etc/hosts ##解析必须做
172.25.13.1 demo1 mfsmaster
172.25.13.2 demo2
172.25.13.3 demo3
[root@demo1 mfs]# systemctl start moosefs-cgiserv.service ##网页监控
[root@demo1 mfs]# netstat -antlp
##9419-9421端口 Master-Servers
##9422 chunnkserver连接端口
##9425 CGI Server
2.2 slave端的部署
2.2.1 demo2
[root@demo1 ~]# cd /etc/yum.repos.d/
[root@demo1 yum.repos.d]# ls
MooseFS.repo redhat.repo rhel7.6.repo
[root@demo1 yum.repos.d]# scp MooseFS.repo server2:/etc/yum.repos.d/
ssh: Could not resolve hostname server2: Name or service not known
lost connection
[root@demo1 yum.repos.d]# scp MooseFS.repo demo2:/etc/yum.repos.d/
[root@demo1 yum.repos.d]# scp MooseFS.repo demo3:/etc/yum.repos.d/
[root@demo2 ~]# yum install moosefs-chunkserver ##安装冷备份，两个slave端
[root@demo3 ~]# yum install moosefs-chunkserver ##安装冷备份
[root@demo3 ~]# vim /etc/hosts ##做解析
##server2添加一块虚拟磁盘，当作分布式存储
[root@demo2 mfs]# fdisk /dev/vdb ##默认分一块就可以
[root@demo2 mfs]# mkfs.xfs /dev/vdb1 ##格式化
[root@demo2 mfs]# mkdir /mnt/chunk1
[root@demo2 mfs]# mount /dev/vdb1 /mnt/chunk1/ ##挂载
[root@demo2 mfs]# blkid
/dev/vdb1: UUID="e94e65fa-35cd-4a40-9ef8-fa6ef42ef128" TYPE="xfs"
[root@demo2 mfs]# vim /etc/fstab
[root@demo2 mfs]# cat /etc/fstab ##永久挂载
UUID="e94e65fa-35cd-4a40-9ef8-fa6ef42ef128" /mnt/chunk1 xfs defaults 0 0
[root@demo2 mfs]# mount -a ##刷新
[root@demo2 mfs]# id mfs ##设置权限
uid=997(mfs) gid=995(mfs) groups=995(mfs)
[root@demo2 mfs]# chown mfs.mfs /mnt/chunk1/
##启动服务启动服务
[root@demo2 mfs]# systemctl start moosefs-chunkserver
##查看demo2是否链接到demo1这个master上
[root@demo2 ~]# cd /etc/mfs/
[root@demo2 mfs]# ls
mfschunkserver.cfg mfschunkserver.cfg.sample mfshdd.cfg mfshdd.cfg.sample
[root@demo2 mfs]# vim mfshdd.cfg ##设置挂载目录
[root@demo2 mfs]# grep -v ^# mfshdd.cfg
/mnt/chunk1
[root@demo2 mfs]# systemctl restart moosefs-chunkserver ##重启服务
2.2.2 demo3
[root@demo3 ~]# cd /etc/mfs/
[root@demo3 mfs]# ls
mfschunkserver.cfg mfschunkserver.cfg.sample mfshdd.cfg mfshdd.cfg.sample
[root@demo3 mfs]# mkdir /mnt/chunk2
[root@demo3 mfs]# vim mfshdd.cfg
[root@demo3 mfs]# grep -v ^# mfshdd.cfg
/mnt/chunk2
[root@demo3 mfs]# chown mfs.mfs /mnt/chunk2
[root@demo3 mfs]# df -h /
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/rhel-root 17G 1.2G 16G 7% /
[root@demo3 mfs]# systemctl start moosefs-chunkserver
master端的cpu压力来自于分布式系统文件大小
内存压力来自于文件数量
2.3 真机作为客户端
[root@westos ~]# cd /etc/yum.repos.d/
[root@westos yum.repos.d]# ls
aliyun.repo google-chrome.repo redhat.repo rhel8.2.repo
[root@westos yum.repos.d]# curl "http://ppa.moosefs.com/MooseFS-3-el8.repo" > /etc/yum.repos.d/MooseFS.repo
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 168 100 168 0 0 93 0 0:00:01 0:00:01 --:--:-- 93
[root@westos yum.repos.d]# vim MooseFS.repo
[root@westos yum.repos.d]# cat MooseFS.repo
[MooseFS]
name=MooseFS $releasever - $basearch
baseurl=http://ppa.moosefs.com/moosefs-3/yum/el8
gpgcheck=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-MooseFS
enabled=1
[root@westos yum.repos.d]# yum install moosefs-client
[root@westos ~]# cd /etc/mfs/
[root@westos mfs]# ls
mfsmount.cfg mfsmount.cfg.sample
[root@westos mfs]# vim mfsmount.cfg
[root@westos mfs]# grep -v ^# mfsmount.cfg
/mnt/mfs #添加挂载的目录，前提是这个挂载目录必须是空的
[root@westos mfs]# vim /etc/hosts ##做解析
[root@westos ~]# mkdir /mnt/mfs
[root@westos ~]# mfsmount
mfsmaster accepted connection with parameters: read-write,restricted_ip,admin ; root mapped to root:root
[root@westos ~]# ll /etc/mfs/mfsmount.cfg
-rw-r--r--. 1 root root 400 Mar 19 11:29 /etc/mfs/mfsmount.cfg
[root@westos ~]# df
Filesystem 1K-blocks Used Available Use% Mounted on
devtmpfs 7772476 0 7772476 0% /dev
tmpfs 7801380 191128 7610252 3% /dev/shm
tmpfs 7801380 18548 7782832 1% /run
tmpfs 7801380 0 7801380 0% /sys/fs/cgroup
/dev/mapper/rhel-root 148006288 95042268 52964020 65% /
/dev/nvme0n1p8 1038336 228892 809444 23% /boot
/dev/loop1 4391278 4391278 0 100% /var/www/html/rhel7.6
/dev/loop0 8238560 8238560 0 100% /var/www/html/rhel8.2
/dev/nvme0n1p1 98304 67874 30430 70% /boot/efi
tmpfs 1560276 20 1560256 1% /run/user/42
tmpfs 1560276 40 1560236 1% /run/user/0
/dev/sda1 976727036 447430532 529296504 46% /run/media/root/My Passport
mfs#mfsmaster:9421 28285952 1749568 26536384 7% /mnt/mfs ##挂载成功
[root@westos mfs]# ls
dir1 dir2
[root@westos mfs]# mfsgetgoal dir1
dir1: 2
[root@westos mfs]# mfsgetgoal dir2
dir2: 2
[root@westos mfs]# mfssetgoal -r 1 dir1
dir1:
inodes with goal changed: 1
inodes with goal not changed: 0
inodes with permission denied: 0
[root@westos mfs]# mfsgetgoal dir1
dir1: 1
[root@westos mfs]# cd dir1/
[root@westos dir1]# cp /etc/passwd .
[root@westos dir1]# mfsfileinfo passwd
passwd:
chunk 0: 0000000000000001_00000001 / (id:1 ver:1)
copy 1: 172.25.13.3:9422 (status:VALID)
[root@westos dir1]# cd ../dir2/
[root@westos dir2]# cp /etc/fstab .
[root@westos dir2]# ls
fstab
[root@westos dir2]# mfsfileinfo fstab
fstab:
chunk 0: 0000000000000002_00000001 / (id:2 ver:1)
copy 1: 172.25.13.2:9422 (status:VALID)
copy 2: 172.25.13.3:9422 (status:VALID)
3. 存储类storage class
3.1 添加一个server4主机slave，一个server5
##server4
[root@server4 ~]# vim /etc/hosts ##做解析
[root@server4 ~]# yum install -y moosefs-chunkserver ##安装软件
[root@server4 ~]# mkdir /mnt/chunk3
[root@server4 ~]# cd /etc/mfs/
[root@server4 mfs]# ls
mfschunkserver.cfg mfschunkserver.cfg.sample mfshdd.cfg mfshdd.cfg.sample
[root@server4 mfs]# vim mfshdd.cfg
[root@server4 mfs]# grep -v ^# mfshdd.cfg
/mnt/chunk3
[root@server4 mfs]# chown mfs.mfs /mnt/chunk3
[root@server4 mfs]# systemctl start moosefs-chunkserver
##server5,和server4一样
3.2 指定存储类标签存放数据
3.2.1 设置标签
[root@westos dir2]# ls
fstab
[root@westos dir2]# dd if=/dev/zero of=bigfile bs=1M count=100
100+0 records in
100+0 records out
104857600 bytes (105 MB, 100 MiB) copied, 0.228773 s, 458 MB/s
[root@westos dir2]# ls
bigfile fstab
[root@westos dir2]# mfsfileinfo bigfile ##查看分布信息
##server2设置标签
[root@server2 mfs]# pwd
/etc/mfs
[root@server2 mfs]# ls
mfschunkserver.cfg mfschunkserver.cfg.sample mfshdd.cfg mfshdd.cfg.sample
[root@server2 mfs]# vim mfschunkserver.cfg ##设置标签A
[root@server2 mfs]# systemctl reload moosefs-chunkserver.service
## server3设置标签B
[root@serve3 ~]# cd /etc/mfs/
[root@serve3 mfs]# ls
mfschunkserver.cfg mfschunkserver.cfg.sample mfshdd.cfg mfshdd.cfg.sample
[root@serve3 mfs]# vim mfschunkserver.cfg
[root@serve3 mfs]# systemctl reload moosefs-chunkserver.service
##server4设置标签为A
[root@server4 ~]# cd /etc/mfs/
[root@server4 mfs]# ls
mfschunkserver.cfg mfschunkserver.cfg.sample mfshdd.cfg mfshdd.cfg.sample
[root@server4 mfs]# vim mfschunkserver.cfg
[root@server4 mfs]# systemctl reload moosefs-chunkserver.service
3.2.2 使用标签
[root@westos ~]# cd /mnt/mfs
[root@westos mfs]# mfsscadmin create 2A sclass1 ##创建标签
[root@westos mfs]# pwd
/mnt/mfs
[root@westos mfs]# ls
dir1 dir2
[root@westos mfs]# cd dir2/
[root@westos dir2]# ls
bigfile fstab
[root@westos dir2]# mfssetsclass sclass1 fstab ##设置标签
fstab: storage class: 'sclass1'
[root@westos dir2]# mfsfileinfo fstab
fstab:
chunk 0: 0000000000000002_00000001 / (id:2 ver:1)
copy 1: 172.25.13.2:9422 (status:VALID)
copy 2: 172.25.13.4:9422 (status:VALID)
3.3 CKA设置
##server2主机设置标签A S H(S表示固态存储，H表示机械硬盘存储)
##server3主机设置标签B S H
##server4主机设置标签B S
##server5主机设置标签B H
[root@westos dir2]# mfsscadmin create -C 2AS -K AS,BS -A AH,BH -d 30 sclass2 ##C创建，K保持，A打包，d表示打包之后的过期时间
storage class make sclass2: ok
[root@westos dir2]# mfssetsclass sclass2 fstab
fstab: storage class: 'sclass2'
[root@westos dir2]# mfsfileinfo fstab
fstab:
chunk 0: 0000000000000002_00000001 / (id:2 ver:1)
copy 1: 172.25.13.2:9422 (status:VALID)
copy 2: 172.25.13.4:9422 (status:VALID)
3.4 恢复数据
[root@westos ~]# cd /mnt/mfs/
[root@westos mfs]# ls
dir1 dir2
[root@westos dir2]# mfsgettrashtime fstab ##查看删除之后可以在多久的时间之内恢复
fstab: 86400
[root@westos mfs]# cd dir1/
[root@westos dir1]# ls
passwd
[root@westos dir1]# rm -fr passwd
[root@westos dir1]# ls
[root@westos dir1]# mfsgettrashtime passwd
passwd: realpath error on (/mnt/mfs/dir1/passwd): ENOENT (No such file or directory)
[root@westos ~]# mkdir /mnt/mfsmeta
[root@westos ~]# mfsmount -m /mnt/mfsmeta
[root@westos mnt]# cd /mnt/mfsmeta/
[root@westos mfsmeta]# ls
sustained trash
[root@westos mfsmeta]# cd trash/
[root@westos trash]# find -name *passwd*
./004/00000004|dir1|passwd
[root@westos trash]# cd 004/
[root@westos 004]# ls
'00000004|dir1|passwd' undel
[root@westos 004]# mv 00000004\|dir1\|passwd undel/
[root@westos 004]# ls
undel
[root@westos 004]# cd /mnt/mfs
[root@westos mfs]# ls
dir1 dir2
[root@westos mfs]# cd dir1/
[root@westos dir1]# ls
passwd
[root@westos dir1]# mfsfileinfo passwd
passwd:
chunk 0: 0000000000000001_00000001 / (id:1 ver:1)
copy 1: 172.25.13.2:9422 (status:VALID)
