lvs DR模式调试

DS配置：

# cat /etc/keepalived_docker/keepalived.conf 
! Configuration File for keepalived
global_defs {router_id LVS_70          # 设置lvs的id，在一个网络内应该是唯一的
}vrrp_instance VI_70 {state MASTER            # 两个 DS，一个为 MASTER，一个为 BACKUPinterface ens33         # 当前 IP 对应的网络接口virtual_router_id 70    # 虚拟路由 ID(0-255)，在一个 VRRP 实例中主备服务器 ID 必须一样priority 100            # 优先级值设定：MASTER 要比 BACKUP 的值大advert_int 1            # 通告时间间隔：单位秒，主备要一致authentication {        # 认证机制，主从节点保持一致即可auth_type PASSauth_pass 1111}virtual_ipaddress {192.168.137.70        # VIP，可配置多个}
}# LB 配置，定义对外提供服务的LVS的VIP以及port
virtual_server 192.168.137.70 80 {delay_loop 3                    # 设置健康状态检查时间lb_algo wlc                     # 设置负载调度的算法，wrr/rr等lb_kind DR                      # 这里测试用了 Direct Route 模式persistence_timeout 600         # 持久连接超时时间即会话保持时间，这段时间内同一ip发起的请求将被转发到同一个 RSprotocol TCPvirtualhost LVSreal_server 192.168.137.3 80 {weight 1TCP_CHECK {connect_timeout 3retry 3                     # 旧版本为 nb_get_retrydelay_before_retry 2connect_port 80}}real_server 192.168.137.4 80 {weight 1TCP_CHECK {connect_timeout 3retry 3                     # 旧版本为 nb_get_retrydelay_before_retry 2connect_port 80}}
}

DS开通ip转发：

#vim /etc/sysctl.conf
net.ipv4.ip_forward = 1
sysctl -p

DS网卡信息：

[root@el7 lvs]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:af:ed:5f brd ff:ff:ff:ff:ff:ff
    inet 192.168.137.2/24 brd 192.168.137.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet 192.168.137.60/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet 192.168.137.70/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:feaf:ed5f/64 scope link
       valid_lft forever preferred_lft forever

RS脚本：

# cat lvs-rs.sh 
#!/bin/bash
VIP=192.168.137.70
#/etc/rc.d/init.d/functions
case "$1" instart)echo "Start Real Server ..."ifconfig lo:0 $VIP netmask 255.255.255.255 broadcast $VIP uproute add -host $VIP dev lo:0#如果多个vip继续类似添加：#ifconfig lo:2 $VIP2 netmask 255.255.255.255 broadcast $VIP2 up#route add -host $VIP2 dev lo:2echo "1" > /proc/sys/net/ipv4/conf/lo/arp_ignoreecho "2" > /proc/sys/net/ipv4/conf/lo/arp_announceecho "1" > /proc/sys/net/ipv4/conf/all/arp_ignoreecho "2" > /proc/sys/net/ipv4/conf/all/arp_announcesysctl -p > /dev/null 2>&1systemctl stop firewalldecho "Real Server Started.";;stop)echo "Stop Real Server ..."ifconfig lo:0 down#如果多个vip继续类似添加：#ifconfig lo:2 downroute del $VIP >/dev/null 2>&1echo "0" > /proc/sys/net/ipv4/conf/lo/arp_ignoreecho "0" > /proc/sys/net/ipv4/conf/lo/arp_announceecho "0" > /proc/sys/net/ipv4/conf/all/arp_ignoreecho "0" > /proc/sys/net/ipv4/conf/all/arp_announceecho "Real Server Stopped.";;*)echo "Usage: $0 {start|stop}"exit 1
esac
exit 0# 添加VIP，修改内核参数（实现对内可见、对外隐藏），注意掩码必须是255.255.255.255
# 因为VIP是绑在环回网卡上（回环网卡离内核更近，数据包优先匹配）的，如果掩码不是4个255，则数据包在返回的时候匹配上了回环网卡，匹配不上真实网关，数据包发送不出去
#RS、DS 有同样的vip，同一个网段中拥有两个vip，客户端在网关发送arp广播需找vip时需要让RS不接受响应。
#arp_ignore 设置为1，意味着当别人的arp请求过来的时候，如果接收的设备没有这个ip，就不做出响应(这个ip在lo上，lo不是接收设备的进口)
#使用最好的ip来回应，什么是最好的ip？同一个网段内子网掩码最长的

RS网卡信息：

[root@el8 lvs]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet 192.168.137.60/32 brd 192.168.137.60 scope global lo:0
       valid_lft forever preferred_lft forever
    inet 192.168.137.70/32 brd 192.168.137.70 scope global lo:1
       valid_lft forever preferred_lft forever
    inet 39.134.209.254/32 brd 39.134.209.254 scope global lo:254
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:3f:10:fe brd ff:ff:ff:ff:ff:ff
    inet 192.168.137.3/24 brd 192.168.137.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe3f:10fe/64 scope link
       valid_lft forever preferred_lft forever

查看lvs状态：

ipvsadm -Ln --persistent-conn
echo
ipvsadm -Ln --stats

如：

[root@el7 lvs]# sh watch-ipvsadm.sh
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port            Weight    PersistConn ActiveConn InActConn
  -> RemoteAddress:Port
TCP  192.168.137.60:80 wlc persistent 600
  -> 192.168.137.3:80             1         0           0          0         
  -> 192.168.137.4:80             1         0           0          0         
TCP  192.168.137.70:80 wlc persistent 600
  -> 192.168.137.3:80             1         0           0          0         
  -> 192.168.137.4:80             1         0           0          0         

IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port               Conns   InPkts  OutPkts  InBytes OutBytes
  -> RemoteAddress:Port
TCP  192.168.137.60:80                   2      216        0     9006        0
  -> 192.168.137.3:80                    2      216        0     9006        0
  -> 192.168.137.4:80                    0        0        0        0        0
TCP  192.168.137.70:80                   9       61        0     3097        0
  -> 192.168.137.3:80                    9       61        0     3097        0
  -> 192.168.137.4:80                    0        0        0        0        0

ipvsadm管理命令：

#手工添加vip和dr规则
ip addr add $vip dev bond0
ipvsadm -A -t $vip:80 -s wrr
ipvsadm -a -t $vip:80 -r $rs1:80 -g
ipvsadm -a -t $vip:80 -r $rs2:80 -g#将某个RS的权重调为0：
ipvsadm -e -t 192.168.137.70:80 -r 192.168.137.3 -g -w 0#删除某个vip
ipvsadm -D -t 192.168.137.70:80

客户端测试：

curl -vo /dev/null "http://mytest.com/test.mp3" -H "User-Agent:mytest" -m 600 -x 192.168.137.60:80

vrrp抓包：

tcpdump -i ens33 vrrp -n

lvs tcpdump抓包：

tcpdump -i any -e -n -nn -s 0 '(host $lvs_vip or host $client_ip) and tcp port $service_port'

如：

 tcpdump -i any -e -n -nn -s 0 '(host 192.168.137.70 or host 192.168.137.1) and tcp port 80'

错误处理：

问题1：IPVS (cmd 1159,errno 2): No such file or directory

解决：ipvs cmd 1160 is IP_VS_SO_SET_DELDEST and cmd 1159 is IP_VS_SO_SET_ADDDEST so it appears to be attempting to delete the failed real server and then add the real server.

--end--