MFS分布式文件系统

目录

 集群部署

Master Servers

​Chunkservers

​编辑Clients

Storage Classes

LABEL 

mfs高可用

pacemaker高可用

​编辑ISCSI

添加集群资源

---

主机	ip	角色
server1	192.168.81.11	Master Servers
server2	192.168.81.12	Chunkservers
server3	192.168.81.13	Chunkservers
server4	192.168.81.14	Chunkservers
server5	192.168.81.15	Clients

 集群部署

Download current version of MooseFS | MooseFS Distributed File System

所有节点添加密钥和软件源库

[root@server1 ~]# curl "https://repository.moosefs.com/RPM-GPG-KEY-MooseFS" > /etc/pki/rpm-gpg/RPM-GPG-KEY-MooseFS
[root@server1 ~]# curl "http://repository.moosefs.com/MooseFS-3-el7.repo" > /etc/yum.repos.d/MooseFS.repo

Master Servers

[root@server1 ~]#yum install moosefs-master moosefs-cgi moosefs-cgiserv moosefs-cli
[root@server1 ~]#systemctl enable --now moosefs-master.service
[root@server1 ~]#systemctl enable --now moosefs-cgiserv.service

访问http://192.168.81.11:9425/

 添加解析

Chunkservers

server2、server3、server4

[root@server2 ~]# yum install moosefs-chunkserver

[root@server2 ~]# cd /etc/mfs/
[root@server2 mfs]# vim mfshdd.cfg
/mnt/chunk1                //指定挂载点
[root@server2 ~]# cd /mnt
[root@server2 mnt]# mkdir chunk1[root@server2 mnt]# chown -R mfs.mfs chunk1/
[root@server2 ~]# systemctl enable --now moosefs-chunkserver

Clients

[root@server5 ~]# yum install moosefs-client

[root@server5 ~]# vim /etc/mfs/mfsmount.cfg
/mnt/mfs
[root@server5 ~]# mkdir /mnt/mfs
[root@server5 ~]# mfsmount

[root@server5 ~]# cd /mnt/mfs/
[root@server5 mfs]# mkdir dir1
[root@server5 mfs]# mkdir dir2
[root@server5 mfs]# cd dir1/
[root@server5 dir1]# vim file1
[root@server5 dir1]# mfsfileinfo file1

[root@server5 dir1]# cd -
/mnt/mfs
[root@server5 mfs]# mfsgetgoal dir2/
dir2/: 2
[root@server5 mfs]# mfssetgoal -r 1 dir2/
dir2/:inodes with goal changed:                       1inodes with goal not changed:                   0inodes with permission denied:                  0
[root@server5 mfs]# mfsgetgoal -r 1 dir2/
1: realpath error on (/mnt/mfs/1): ENOENT (No such file or directory)
dir2/:directories with goal          1 :          1
[root@server5 mfs]# cd dir2/
[root@server5 dir2]# vim file1
[root@server5 dir2]# mfsfileinfo file1
file1:chunk 0: 0000000000000004_00000001 / (id:4 ver:1)copy 1: 192.168.81.12:9422 (status:VALID)

Storage Classes

LABEL 

[root@server2 ~]# vim /etc/mfs/mfschunkserver.cfg

[root@server5 mfs]# mfsscadmin create 2S class_2s
storage class make class_2s: ok
[root@server5 mfs]# cd dir1/
[root@server5 dir1]# ls
file1
[root@server5 dir1]# mfssetsclass -r class_2s file1
file1:inodes with storage class changed:              1inodes with storage class not changed:          0inodes with permission denied:                  0
[root@server5 dir1]# mfsfile
mfsfileinfo    mfsfilepaths   mfsfilerepair
[root@server5 dir1]# mfsfileinfo file1
file1:chunk 0: 0000000000000002_00000001 / (id:2 ver:1)copy 1: 192.168.81.12:9422 (status:VALID)copy 2: 192.168.81.14:9422 (status:VALID)

[root@server5 dir1]# mfsscadmin create A,B class_ab
storage class make class_ab: ok
[root@server5 dir1]# mfssetsclass -r class_ab file1
file1:inodes with storage class changed:              1inodes with storage class not changed:          0inodes with permission denied:                  0
[root@server5 dir1]# mfsfileinfo file1
file1:chunk 0: 0000000000000002_00000001 / (id:2 ver:1)copy 1: 192.168.81.12:9422 (status:VALID)copy 2: 192.168.81.13:9422 (status:VALID)

[root@server5 mnt]# cd mfs/
[root@server5 mfs]# mkdir dir3
[root@server5 mfs]# mfsscadmin create S,H class_sh
storage class make class_sh: ok
[root@server5 mfs]# cd dir3/
[root@server5 dir3]# vim file1
[root@server5 dir3]# mfssetsclass -r class_sh file1
file1:inodes with storage class changed:              1inodes with storage class not changed:          0inodes with permission denied:                  0
[root@server5 dir3]# mfsfileinfo file1
file1:chunk 0: 0000000000000006_00000001 / (id:6 ver:1)copy 1: 192.168.81.12:9422 (status:VALID)copy 2: 192.168.81.13:9422 (status:VALID)

列出当前系统中所有的存储类 (class)

[root@server5 dir3]# mfsscadmin list

mfs高可用

新建虚机server6

[root@server6 ~]# curl "https://repository.moosefs.com/RPM-GPG-KEY-MooseFS" > /etc/pki/rpm-gpg/RPM-GPG-KEY-MooseFS
[root@server6 ~]# curl "http://repository.moosefs.com/MooseFS-3-el7.repo" > /etc/yum.repos.d/MooseFS.repo
[root@server6 ~]# yum install moosefs-master moosefs-cgi moosefs-cgiserv moosefs-cli
[root@server6 ~]# systemctl enable --now moosefs-master.service
[root@server6 ~]# systemctl enable --now moosefs-cgiserv.service

pacemaker高可用

[root@server1 ~]# vim /etc/yum.repos.d/dvd.repo
[HighAvailability]
name=rhel7.6 HighAvailability
baseurl=file:///media/addons/HighAvailability
gpgcheck=0[root@server1 ~]# yum install -y pacemaker pcs psmisc policycoreutils-python

server6同步 

[root@server1 ~]# scp /etc/yum.repos.d/dvd.repo server6:/etc/yum.repos.d/
[root@server1 ~]# ssh server6 yum install -y pacemaker pcs psmisc policycoreutils-python

节点认证

[root@server1 ~]# systemctl enable --now pcsd.service
[root@server1 ~]# ssh server6 systemctl enable --now pcsd.service[root@server1 ~]# echo westos | passwd --stdin hacluster
[root@server1 ~]# ssh server6 'echo westos | passwd --stdin hacluster'[root@server1 ~]# pcs cluster auth server1 server6
Username: hacluster
Password: westos

创建集群

[root@server1 ~]# pcs cluster setup --name mycluster server1 server6
[root@server1 ~]# pcs cluster start --all
[root@server1 ~]# pcs cluster enable --all

禁用stonith

[root@server1 ~]# pcs property set stonith-enabled=false

ISCSI

server4添加硬盘分区

[root@server5 ~]# yum install targetcli -y
[root@server5 ~]# systemctl enable --now target

[root@server5 ~]# targetcli
/> /backstores/block create westos:storage /dev/sdb1
/> /iscsi create iqn.2023-11.org.westos:storage
/> /iscsi/iqn.2023-11.org.westos:storage/tpg1/luns create /backstores/block/westos:storage
/> /iscsi/iqn.2023-11.org.westos:storage/tpg1/acls create iqn.2023-11.org.westos:westoskey1

客户端

[root@server1 ~]# yum install iscsi-initiator-utils.x86_64 -y
[root@server1 ~]# vim /etc/iscsi/initiatorname.iscsi
InitiatorName=iqn.2023-11.org.westos:westoskey1
[root@server1 ~]# systemctl restart iscsid 

[root@server1 ~]# iscsiadm -m discovery -t st -p 192.168.81.15
[root@server1 ~]# iscsiadm -m node -T iqn.2023-11.org.westos:storage -p 192.168.81.15 -l

创建分区

[root@server1 ~]# fdisk /dev/sdb
[root@server1 ~]# mkfs.xfs /dev/sdb1

[root@server1 ~]# mount /dev/sdb1 /mnt/
[root@server1 ~]# cd /var/lib/mfs/
[root@server1 mfs]# systemctl disable --now moosefs-master.service
[root@server1 mfs]# cp * /mnt/
[root@server1 mnt]# chown -R mfs.mfs /mnt/
[root@server1 mnt]# ll -d /mnt/
[root@server1 ~]# umount /mnt/
[root@server1 ~]# ll -d /mnt/

server6类推

[root@server6 mfs]# mount /dev/sdb1 /var/lib/mfs/
[root@server6 mfs]# chown -R mfs.mfs .
[root@server6 ~]# umount /var/lib/mfs/

 

添加集群资源

[root@server1 ~]# pcs resource create vip ocf:heartbeat:IPaddr2 ip=192.168.81.200 op monitor interval=30s
[root@server1 ~]# pcs resource create mfs ocf:heartbeat:Filesystem device=/dev/sdb1 directory=/var/lib/mfs fstype=xfs
[root@server1 ~]# pcs resource create mfs-master systemd:moosefs-master op monitor interval=60s
[root@server1 ~]# pcs resource group add groupmfs vip mfs mfs-master

测试

[root@server1 ~]# pcs node standby