综合实验nginx+nfs+kpa

综合实验

实验目的：

静态资源和动态资源分别存放在远端存储NFS上，NFS上数据实现实时备份，用户通过负载访问后端的web服务。实现ngixn负载高可用，当keepalived master宕机，vip能自动跳转到备用节点

实验环境：

六台服务器都是centos8.5系统

主机名	IP地址
master-kpa1	10.1.1.161
backup-kpa2	10.1.1.162
nginx-web1	10.1.1.121
nginx-web2	10.1.1.122
master-nfs	10.1.1.123
slave-nfs	10.1.1.124

实验步骤：

1NFS

1.1nfs共享和实时同步

#10.1.1.123master-nfs服务器端

#10.1.1.121master-nfs服务器端
(1)#在nfs服务器端安装nfs-utils和rpcbind包
[root@master-nfs ~]# yum install -y nfs-utils rpcbind
#nfs-utils:提供了NFS服务器程序和对应的管理工具 
#rpcbind:获取nfs服务器端的端口等信息
[root@master-nfs ~]# systemctl start rpcbind
[root@master-nfs ~]# yum -y install net-tools #此包中含有一些常用网络查看命令
[root@master-nfs ~]# netstat -tunlp | grep 111
(2)#创建/data/NFSdata目录，更改属主、属组
[root@master-nfs ~]# mkdir -p /data/NFSdata/web1
[root@master-nfs ~]# mkdir -p /data/NFSdata/web2
(3)#注意：此处不改权限，客户端没有创建文件权限
[root@master-nfs ~]# chown -R nobody:nobody /data
(4)#配置NFS服务的配置文件
[root@master-nfs ~]# vim /etc/exports
/data/NFSdata/web1 	#表示要共享文件的目录 
10.1.1.0/24 #表示所有允许访问的客户端IP网段
(rw,sync) 		#rw:表示读写权限，sync:表示数据同步写入内存硬盘
/data/NFSdata/web2 10.1.1.0/24(rw,sync)
/data/NFSdata/web1 10.1.1.0/24(rw,sync)
(5)#重启服务及实现开机自启动
[root@master-nfs ~]# systemctl start nfs-server.service
[root@master-nfs ~]# systemctl enable rpcbind.service 
[root@master-nfs ~]# systemctl enable nfs-server.service --now

1.2配置nfs客户端

2.1nginx-web1上10.1.1.121配置nfs客户端

(1)下载工具包nfs-utils
[root@nginx-web1 ~]# yum -y install nfs-utils
(2)#查看远程主机的NFS共享
[root@nginx-web1 ~]# showmount -e 10.1.1.123
Export list for 10.1.1.123:
/data/NFSdata/web1 10.1.1.0/24
/data/NFSdata/web2 10.1.1.0/24
(3)创建挂载目录
[root@nginx-web1 ~]# mkdir -p /data-web1/static
[root@nginx-web1 ~]# mkdir -p /data-web1/image
(4)永久挂载
[root@nginx-web1 ~]# vim /etc/fsstab/
10.1.1.123:/data/NFSdata/web1                 /data-web1                nfs        defaults        0 0
[root@nginx-web1 ~]# mount -a#挂载生效
(5)测试是否共享
[root@nginx-web1 static]# touch 1.txt
[root@nginx-web1 static]# ls
1.txt  index.html
[root@master-nfs web1]# cd /data/NFSdata/web1/static/
[root@master-nfs static]# ls
1.txt  index.html

2.2nginx-web2上10.1.1.122配置nfs客户端

(1)下载工具包nfs-utils
[root@nginx-web2 ~]# yum -y install nfs-utils
(2)#查看远程主机的NFS共享
[root@nginx-web2 ~]# showmount -e 10.1.1.123
Export list for 10.1.1.123:
/data/NFSdata/web1 10.1.1.0/24
/data/NFSdata/web2 10.1.1.0/24
(3)创建挂载目录
[root@nginx-web2 ~]# mkdir -p /data-web2/static
[root@nginx-web2 ~]# mkdir -p /data-web2/image
(4)永久挂载
[root@nginx-web2 ~]# vim /etc/fsstab/
10.1.1.123:/data/NFSdata/web2                /data-web1                nfs        defaults        0 0
[root@nginx-web2 ~]# mount -a#挂载生效
(5)测试是否共享
[root@nginx-web2 ~]# cd /data-web2/static
[root@nginx-web2 static]# ls
index.html
[root@nginx-web2 static]# touch 2.txt
[root@nginx-web2 static]# ls
2.txt  index.html
[root@master-nfs ~]# cd /data/NFSdata/web2/static/
[root@master-nfs static]# ls
2.txt  index.html

1.3部署Rsync服务

在10.1.1.124slave-nfs上部署Rsync服务端

(1)下载Rsync软件包
[root@slave-nfs ~]#yum -y install rsync
(2)新增vim /etc/rsyncd.conf配置文件
[root@slave-nfs ~]# vim /etc/rsyncd.conf
[root@slave-nfs ~]# cat /etc/rsyncd.conf 
uid = rsync
#组id
gid = rsync
#程序安全设置
use chroot = no
#客户端连接数
max connections = 200
#进程号文件位置
pid file = /var/run/rsyncd.pid
#进程锁文件位置
lock file = /var/run/rsync.lock
#日志文件位置
log file = /var/run/rsyncd.log
#连接超时时间
timeout = 300
#3.1版本以上要加这个
fake super = yes
#模块名称
[backup]
#同步数据的目录
path = /backup
#有错误时忽略
ignore errors
#只读模式（true为只读，false为可读可写）
read only = false
#阻止远程列表
list = false
#允许访问的IP
hosts allow = 10.1.1.0/24
#虚拟用户
auth users = rsync_backup
#存放用户和密码的文件
secrets file = /etc/rsync.password
(3)创建密码文件vi /etc/rsync.password
[root@slave-nfs ~]# vim /etc/rsync.password
rsync_backup:123456
(4)给/etc/rsync.password降权
[root@slave-nfs ~]# chmod 600 /etc/rsync.password
(5)创建程序用户rsync
[root@slave-nfs ~]# useradd -M -s /sbin/nologin rsync
(6)创建/backup目录并修改所有者所属组
[root@slave-nfs ~]# mkdir /backup
[root@slave-nfs ~]# chown rsync.rsync /backup
(7)守护进程启动rsync
[root@slave-nfs ~]# rsync --daemon

master-nfs10.1.1.123作为rsync客户端

(1)123客户端节点新增密码文件vim /etc/rsync.password
[root@master-nfs ~]# vim /etc/rsync.password
123456
(2)给/etc/rsync.password降权
[root@master-nfs ~]# chmod 600 /etc/rsync.password
(3)测试123节点传输文件到124上
[root@master-nfs ~]# touch 121.txt
[root@master-nfs ~]# rsync -zav 121.txt rsync_backup@10.1.1.124::backup --password-file=/etc/rsync.password
sending incremental file list
121.txt
sent 90 bytes  received 43 bytes  88.67 bytes/sec
total size is 0  speedup is 0.00
[root@slave-nfs backup]# ls
121.txt  2.txt  #-a：表示以归档模式同步文件，相当于 参数的缩写。这个选项会保留文件的元数据（如所有者、权限、时间戳等）以及其他有用的信息，例如符号链接和设备文件。-rlptgoD
-v：表示启用详细模式，输出同步过程中的详细信息。
-z：表示使用压缩算法进行传输，可以减少数据传输量。在网络较慢或传输大文件时特别有用。
rsync -avz 命令可以将文件或目录以归档模式进行同步，并在同步过程中输出详细信息，同时使用压缩算法减少传输量。

1.4部署inotify服务

NFS服务器上部署inotify服务,实现实时同步

(1)#下载epel源
[root@master-nfs ~]#  yum install -y https://mirrors.aliyun.com/epel/epel-release-latest-8.noarch.rpm
(2)#清除缓存
[root@master-nfs ~]# yum clean all
[root@master-nfs ~]# yum makecache
(3)下载inotify包
[root@master-nfs ~]# yum -y install inotify-tools
(4)编写脚本inotify脚本
[root@master-nfs ~]# vim inotify.sh
backupServer=10.1.1.124
path=/data/NFSdata/
inotifywait -mrq --format '%w%f' -e create,close_write,delete $path | while read line
doif [ -f $line ];thenrsync -za $line --delete rsync_backup@$backupServer::backup --password-file=/etc/rsync.passwordelsecd $pathrsync -za ./ --delete rsync_backup@$backupServer::backup --password-file=/etc/rsync.passwordfi
done
(5)#后台运行
[root@master-nfs ~]# sh inotify.sh &
[1] 350

2.实现nignx负载

2.1安装Nignx

四台服务器都安装nginx

yum -y install nginx
systemctl enable nginx.service --now
systemctl status nginx.service
cd /etc/nginx/
cp nginx.conf nginx.conf.bak

2.2配置负载

 [root@master-kpa1 ~]# vim /etc/nginx/nginx.confupstream myweb {server 10.1.1.121:80 weight=1;server 10.1.1.122:80 weight=1;}server {listen       80 default_server;listen       [::]:80 default_server;server_name  kpa1.sxh.com;location / {root         /usr/share/nginx/html;proxy_pass      http://myweb;proxy_set_header Host $http_host;proxy_set_header X-Real-IP $remote_addr;proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;}[root@backup-kpa2 ~]# vim /etc/nginx/nginx.confupstream myweb {server 10.1.1.121:80 weight=1;server 10.1.1.122:80 weight=1;}server {listen       80 default_server;listen       [::]:80 default_server;server_name  kpa1.sxh.com;location / {root         /usr/share/nginx/html;proxy_pass      http://myweb;proxy_set_header Host $http_host;proxy_set_header X-Real-IP $remote_addr;proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;}

2.3配置动静分离

 [root@nginx-web1 ~]# vim /etc/nginx/nginx.confserver {listen       80 default_server;listen       [::]:80 default_server;server_name kpa1.sxh.com;location / {root /data-web1/static;}#location /sxh1.jpg {#root /data-web1/image;#}location  ~* \.(gif|jpg|jpeg)$ {root /data-web1/image;}[root@nginx-web2 ~]# vim /etc/nginx/nginx.confserver {listen       80 default_server;listen       [::]:80 default_server;server_name  kpa1.sxh.com;location / {root /data/static;}#location /sxh2.jpg {#root /data/image;#}location  ~* \.(gif|jpg|jpeg)$ {root /data-web1/image;}

2.4测试反向代理

[root@slave-nfs ~]# curl kpa1.sxh.com
this is a papge web1 10.1.1.121
[root@slave-nfs ~]# curl kpa1.sxh.com
this a page web2 10.1.1.122

3.实现keepalived

3.1编译安装kpa

master-kpa1和backup-kpa2都需要编译安装keepalived

#下载依赖软件
[root@master-kpa1 ~]# yum install gcc curl openssl-devel libnl3-devel net-snmp-devel
#下载二进制文件
[root@master-kpa1 ~]# wget https://keepalived.org/software/keepalived-2.0.20.tar.gz
#解压到指定目录
[root@master-kpa1 ~]# tar xvf keepalived-2.0.20.tar.gz -C /usr/local/src
#选项--disable-fwmark 可用于禁用iptables规则,可访止VIP无法访问,无此选项默认会启用iptables规则
[root@master-kpa1 ~]# cd /usr/local/src/keepalived-2.0.20/
#配置文件路径
[root@master-kpa1 keepalived-2.0.20]# ./configure --prefix=/usr/local/keepalived --disable-fwmark
#编译并安装
[root@master-kpa1 keepalived-2.0.20]# make && make install
[root@master-kpa1 keepalived-2.0.20]# cd
[root@master-kpa1 ~]# /usr/local/keepalived/sbin/keepalived -v
#创建配置文件
[root@master-kpa1 ~]# mkdir /etc/keepalived #没有创建,则服务起不来
[root@master-kpa1 ~]# cp /usr/local/keepalived/etc/keepalived/keepalived.conf /etc/keepalived
[root@master-kpa1 ~]## systemctl enable --now keepalived.service 
#注意事项
#不进行下面配置，结果：重启不报错，但是status状态一直dead
[root@master-kpa1 ~]# vim /etc/keepalived/keepalived.conf
vrrp_instance VI_1 {state MASTERinterface ens160#根据自己网卡名设置virtual_router_id 51priority 100advert_int 1authentication {auth_type PASSauth_pass 1111}virtual_ipaddress {10.1.1.88 #改为vip}
}
[root@master-kpa1 ~]# systemctl restart keepalived.service 
[root@master-kpa1 ~]# systemctl stauts keepalived.service 

3.2实现kpa单主架构

master-kpa1配置

[root@master-kpa1 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived
global_defs {notification_email {[e-mail]}notification_email_from keepalived@localhostsmtp_server 127.0.0.1smtp_connect_timeout 30router_id master-kpa1vrrp_skip_check_adv_addrvrrp_garp_interval 0vrrp_gna_interval 0vrrp_mcast_group4 230.1.1.1
}
vrrp_script check_nginx {script "/usr/bin/killall -0 nginx"interval 3weight -50fail 3rise 1
}
vrrp_instance VI_1 {state MASTERinterface ens160virtual_router_id 51priority 100advert_int 1authentication {auth_type PASSauth_pass 1111}virtual_ipaddress {10.1.1.88 dev ens160 label ens160:1}notify_master "/usr/bin/systemctl restart nginx.service"notify_backup "/usr/bin/systemctl restart nginx.service"	
#	notify_master "/etc/keepalived/notify.sh master"
#	notify_backup "/etc/keepalived/notify.sh backup"
#	notify_fault "/etc/keepalived/notify.sh fault"track_script { check_nginx}
}

backup-kpa2配置

[root@backup-kpa2 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived
global_defs {notification_email {[e-mail]}notification_email_from keepalived@localhostsmtp_server 127.0.0.1   smtp_connect_timeout 30router_id back-kpa2vrrp_skip_check_adv_addrvrrp_garp_interval 0vrrp_gna_interval 0vrrp_mcast_group4 230.1.1.1
}
vrrp_script check_nginx {script "/usr/bin/killall -0 nginx"interval 3weight -50fail 3rise 1
}
vrrp_instance VI_1 {state BACKUPinterface ens160virtual_router_id 51priority 70advert_int 1authentication {auth_type PASSauth_pass 1111}virtual_ipaddress {10.1.1.88 dev ens160 label ens160:2 }track_interface {ens160}notify_master "/usr/bin/systemctl restart nginx.service"notify_backup "/usr/bin/systemctl restart nginx.service"#notify_master "/etc/keepalived/notify.sh master"#notify_backup "/etc/keepalived/notify.sh backup"#notify_fault "/etc/keepalived/notify.sh fault"track_script {check_nginx}}

抓包分析

[root@kpa1 ~]# tcpdump -i ens160 -nn src host 10.1.1.161 and dst 10.1.1.162
dropped privs to tcpdump
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on ens160, link-type EN10MB (Ethernet), capture size 262144 bytes
15:52:48.226434 IP 10.1.1.161 > 10.1.1.162: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20
15:52:49.227488 IP 10.1.1.161 > 10.1.1.162: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20
15:52:50.228497 IP 10.1.1.161 > 10.1.1.162: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20

3.3QQ邮箱设置

[root@kpa1 ~]# vim /etc/mail.rc
set from=[e-mail]
set smtp=smtp.qq.com
set smtp-auth-user=[e-mail]
set smtp-auth-password=zoboduhoqcqcdfhf
set smtp-auth=login
[root@kpa1 ~]# yum -y install mailx
#发送邮件测试
[root@kpa1 ~]# echo "Test Mail 30599555740" |mail -s warning [e-mail]

3.4创建通知脚本

[root@kpa1 ~]# cat /etc/keepalived/notify.sh 
#!/bin/bash
contact='[e-mail]'
notify() {mailsubject="(hostname) to be $1,vip floating"mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"echo "$mailbody" |mail -s "$mailsubject" $contact
}
case $1 in
master)notify master;;
backup)notify backup;;
fault)notify fault;;
*)echo "Usage: $(basename $0) {master|backup|fault}"exit 1;;
esac
[root@kpa1 ~]# chmod a+x /etc/keepalived/notify.sh 

3.5kpa测试宕机

[root@master-kpa1 ~]# killall keepalived 
[root@master-kpa1 ~]# systemctl restart keepalived.service