30.2 lvs-dr

LVS-DR模式实验

清空防火墙策略iptables -F
关闭selinux
如图
这里写图片描述

[root@Client ~ ]#for i in {1..100};do curl 10.0.0.100;sleep 0.5;done
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
^C

1、客户端配置
配置ip 172.18.0.222/16
路由

[root@Client ~ ]#route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
172.18.0.0      0.0.0.0         255.255.0.0     U     0      0        0 eth1
10.0.0.0        172.18.0.6      255.0.0.0       UG    0      0        0 eth1

2、路由器配置
路由器的192.168.31.6/24(接受RS提供的httpd服务的包,包里封装的是172.18.0.222,10.0.0.100的包)和10.0.0.1/8(客户端访问10.0.0.100时的路由转发功能)需在同一个网卡上,一共3个ip

[root@cos37 / ]#ip a
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:65:83:74 brd ff:ff:ff:ff:ff:ff
    inet `192.168.31.6/24` brd 192.168.31.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet `10.0.0.1/8` scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe65:8374/64 scope link 
       valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:65:83:7e brd ff:ff:ff:ff:ff:ff
    inet `172.18.0.6/16` brd 172.18.255.255 scope global noprefixroute eth1
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe65:837e/64 scope link 
       valid_lft forever preferred_lft forever

[root@cos37 / ]#route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
10.0.0.0        0.0.0.0         255.0.0.0       U     0      0        0 eth0
172.18.0.0      0.0.0.0         255.255.0.0     U     101    0        0 eth1
192.168.31.0    0.0.0.0         255.255.255.0   U     100    0        0 eth0

3、lvs配置
内网ip192.168.31.7/24(把客户端的包172.18.0.222,10.0.0.100的包调度到RS服务器),公网ip10.0.0.100/32对互联网提供服务

[root@lvs ~ ]#ip a
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:3b:cf:92 brd ff:ff:ff:ff:ff:ff
    inet `192.168.31.7/24` brd 192.168.31.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet `10.0.0.100/32` brd 10.255.255.255 scope global eth0:1
       valid_lft forever preferred_lft forever
    inet6 fe80::a441:7018:d3a4:d5d6/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

\#路由192.168.31.66,理论上可以不添加,因为响应报文不经过lvs,但实际行不通,没有此路由,当请求报文到达lvs调度器时,检查没有响应报文需要的路由,则会直接拒绝
[root@lvs ~ ]#route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.31.66   0.0.0.0         UG    0      0        0 eth0
10.0.0.100      0.0.0.0         255.255.255.255 UH    100    0        0 eth0
192.168.31.0    0.0.0.0         255.255.255.0   U     100    0        0 eth0

脚本1

4、rs配置

[root@R1 ~ ]#ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet `10.0.0.100/32` scope global lo:1
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:50:56:22:38:c9 brd ff:ff:ff:ff:ff:ff
    inet `192.168.31.17/24` brd 192.168.31.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::4619:664b:f1e4:d5b3/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

#路由指向路由器的网关,把封装的报文(源地址10.0.0.100,目的地址172.18.0.222)由192.168.31.17送往192.168.31.6;10.0.0.100为调度器接受了目的到地址是10.0.0.100的包调度到RS提供服务的主机上,由rs的10.0.0.100提供服务,再由rs的另一个内网地址192.168.31.17送往路由
[root@R1 ~ ]#route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.31.6    0.0.0.0         UG    0      0        0 eth0
192.168.31.0    0.0.0.0         255.255.255.0   U     100    0        0 eth0

#另一个RS服务器配置和上面的相同,只是内网ip为192.168.31.27
[root@R2 ~ ]#

脚本2


实现https的安全加密
[root@R1 ~ ]#cd /etc/pki/tls/certs
[root@R1 certs ]#ls
ca-bundle.crt  ca-bundle.trust.crt  localhost.crt  make-dummy-cert  Makefile  renew-dummy-cert

#去掉私钥加密选项-aes128
[root@R1 certs ]#vim Makefile 
%.key:
        umask 77 ; \
        /usr/bin/openssl genrsa -aes128  $(KEYLEN) > $@ 

[root@R1 certs ]#make dhy.crt
umask 77 ; \
/usr/bin/openssl genrsa  2048 > dhy.key
Generating RSA private key, 2048 bit long modulus
..............................+++
.................................................................................................................+++
e is 65537 (0x10001)
umask 77 ; \
/usr/bin/openssl req -utf8 -new -key dhy.key -x509 -days 365 -out dhy.crt 
You are about to be asked to enter information that will be incorporated
into your certificate request.
What you are about to enter is what is called a Distinguished Name or a DN.
There are quite a few fields but you can leave some blank
For some fields there will be a default value,
If you enter '.', the field will be left blank.

-----
Country Name (2 letter code) [XX]:CN
State or Province Name (full name) []:beijing
Locality Name (eg, city) [Default City]:beijing
Organization Name (eg, company) [Default Company Ltd]:dhy.com
Organizational Unit Name (eg, section) []:opt
Common Name (eg, your name or your server's hostname) []:denghaiyang
Email Address []:
[root@R1 certs ]#mv dhy.* /etc/httpd/conf.d/
[root@R1 conf.d ]#yum install mod_ssl
[root@R1 conf.d ]#vim ssl.conf
[root@R1 conf.d ]#scp ssl.conf dhy.* 192.168.31.27:/etc/httpd/conf.d/
[email protected]'s password: 
ssl.conf                                   100% 9427     8.4MB/s   00:00    
dhy.crt                                    100% 1318    11.1KB/s   00:00    
dhy.key                                    100% 1675   130.3KB/s   00:00   
[root@R1 conf.d ]#systemctl restart httpd
#端口改为443
[root@lvs ~ ]#vim lvs_dr_vs.sh 
mask='255.255.255.255'
port='443'

[root@lvs ~ ]#bash lvs_dr_vs.sh stop
The VS Server is Canceled!
[root@lvs ~ ]#bash lvs_dr_vs.sh start
The VS Server is Ready!
[root@lvs ~ ]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.0.0.100:443 wrr
  -> 192.168.31.17:443            Route   1      0          0         
  -> 192.168.31.27:443            Route   1      0          0  
[root@Client ~ ]#for i in {1..10000};do curl -k https://10.0.0.100;sleep 0.5;done
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
^C

window访问
C:\Users\29256>route –help
C:\WINDOWS\system32>route add 10.0.0.0/8 172.18.0.6
操作完成!

如图
这里写图片描述
C:\WINDOWS\system32>route delete 10.0.0.0
操作完成!

FireWall Mark

1、FWM:FireWall Mark
2、MARK target 可用于给特定的报文打标记 –set-mark value 其中:value 可为0xffff格式,表示十六进制数字
3、借助于防火墙标记来分类报文,而后基于标记定义集群服务;可将多个不同的 应用使用同一个集群服务进行调度
4、实现方法:
在Director主机打标记:

iptables -t mangle -A PREROUTING -d 10.0.0.100 -p tcp -m multiport –dports 80,443 -j MARK –set-mark 6

在Director主机基于标记定义集群服务:
ipvsadm -A -f 6 -s rr
ipvsadm -a -f 6 -r 192.168.31.17 -g
ipvsadm -a -f 6 -r 192.168.31.27 -g
ipvsadm -Ln
如下:

[root@lvs ~ ]#ipvsadm -C    #清空
[root@lvs ~ ]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
[root@lvs ~ ]#iptables -t mangle -A PREROUTING -d 10.0.0.100 -p tcp -m multiport --dports 80,443 -j MARK --set-mark 6 
[root@lvs ~ ]#ipvsadm -A -f 6 -s rr
[root@lvs ~ ]#ipvsadm -a -f 6 -r 192.168.31.17 -g
[root@lvs ~ ]#ipvsadm -a -f 6 -r 192.168.31.27 -g
[root@lvs ~ ]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
FWM  6 rr
  -> 192.168.31.17:0              Route   1      0          0         
  -> 192.168.31.27:0              Route   1      0          0         

[root@Client ~ ]#for i in {1..10000};do curl -k https://10.0.0.100;curl http://10.0.0.100;sleep 0.5;done
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
^C
持久连接

1、 session 绑定:对共享同一组RS的多个集群服务,需要统一进行绑定,lvs sh算法无法实现
2、 持久连接( lvs persistence )模板:实现无论使用任何调度算法,在一段时间内(默认360s ),能够实现将来自同一个地址的请求始终发往同一个RS
ipvsadm -A|E -t|u|f service-address [-s scheduler] [-p [timeout]]
3、 持久连接实现方式:
4、 每端口持久(PPC):每个端口对应定义为一个集群服务,每集群服务单独调度
5、 每防火墙标记持久(PFWMC):基于防火墙标记定义集群服务;可实现将多个端口上的应用统一调度,即所谓的port Affinity
6、 每客户端持久(PCC):基于0端口(表示所有服务)定义集群服务,即将客户端对所有应用的请求都调度至后端主机,必须定义为持久模式

每防火墙持久标记
[root@lvs ~ ]#ipvsadm -E -f 6 -s rr -p
[root@lvs ~ ]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
FWM  6 rr persistent 360   #360秒
  -> 192.168.31.17:0              Route   1      0          5         
  -> 192.168.31.27:0              Route   1      0          5 

[root@Client ~ ]#for i in {1..10000};do curl -k https://10.0.0.100;curl http://10.0.0.100;sleep 0.5;done
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
^C
每客户端持久

ipvsadm -A -t 10.0.0.100:0 -s rr -p #0表示客户端访问10.0.0.100:端口是一一对应调度到相应的RS服务器,这样不要,端口都暴露了

#修改为不是持久连接

ipvsadm -E -f 10 -s rr

LVS高可用性

1 Director不可用,整个系统将不可用;SPoF Single Point of Failure
解决方案:高可用
keepalived heartbeat/corosync
2某RS不可用时,Director依然会调度请求至此RS
解决方案: 由Director对各RS健康状态进行检查,失败时禁用,成功时启用
keepalived heartbeat/corosync ldirectord
检测方式:
(a) 网络层检测,icmp
(b) 传输层检测,端口探测
(c) 应用层检测,请求某关键资源
RS全不用时:backup server, sorry server

lvs不具备健康性检查,由director实现,至于lvs宕机,需要keepalived解决,keepalived更加强大

ldirectord
ldirectord:监控和控制LVS守护进程,可管理LVS规则  
包名:ldirectord-3.9.6-0rc1.1.1.x86_64.rpm  
文件:
/etc/ha.d/ldirectord.cf  主配置文件 
/usr/share/doc/ldirectord-3.9.6/ldirectord.cf 配置模版 
/usr/lib/systemd/system/ldirectord.service 服务
/usr/sbin/ldirectord     主程序 
/var/log/ldirectord.log  日志 
/var/run/ldirectord.ldirectord.pid   pid文件 
Ldirectord               #配置文件示例
checktimeout=3           #检查超时为3秒未响应,则处理
checkinterval=1          #每1秒检查一次
autoreload=yes
logfile=“/var/log/ldirectord.log#日志文件 
quiescent=no            #down时yes权重为0,no为删除 
virtual=5               #指定VS的FWM或IP:port  
real=172.16.0.7:80 gate 2
real=172.16.0.8:80 gate 1
fallback=127.0.0.1:80 gate     #sorry server service=http scheduler=wrr
checktype=negotiate 
checkport=80 
request="index.html" 
receive="rs" 
实验ldirectord

http://download.opensuse.org/repositories/network:/ha-clustering:/Stable/CentOS_CentOS-7/x86_64/

[root@lvs ~ ]#rpm -qpl ldirectord-3.9.6-0rc1.1.2.x86_64.rpm 
warning: ldirectord-3.9.6-0rc1.1.2.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID 17280ddf: NOKEY
/etc/ha.d
/etc/ha.d/resource.d
/etc/ha.d/resource.d/ldirectord
/etc/logrotate.d/ldirectord
/usr/lib/ocf/resource.d/heartbeat/ldirectord
/usr/lib/systemd/system/ldirectord.service
/usr/sbin/ldirectord                  #主程序,perl语言脚本
/usr/share/doc/ldirectord-3.9.6
/usr/share/doc/ldirectord-3.9.6/COPYING
/usr/share/doc/ldirectord-3.9.6/ldirectord.cf  #配置文件参考文档
/usr/share/man/man8/ldirectord.8.gz
[root@lvs ~ ]#yum install ldirectord-3.9.6-0rc1.1.2.x86_64.rpm -y


cat /usr/lib/systemd/system/ldirectord.service
cp /usr/share/doc/ldirectord-3.9.6/ldirectord.cf /etc/ha.d

#ldirectord把lvs的策略都写在配置文件里面了
[root@lvs ha.d ]#vim /etc/ha.d/ldirectord.cf  
virtual=10.0.0.100:80
        real=192.168.31.17:80 gate
        real=192.168.31.27:80 gate 2
        service=http
        scheduler=wrr
        #persistent=600
        #netmask=255.255.255.255
        protocol=tcp
        checktype=negotiate
        checkport=80
        request="index.html"
        receive="cos"

[root@lvs ~ ]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn

[root@lvs ha.d ]#systemctl start ldirectord
[root@lvs ha.d ]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.0.0.100:80 wrr
  -> 192.168.31.17:80             Route   1      0          0         
  -> 192.168.31.27:80             Route   2      0          0 

[root@Client ~ ]#for i in {1..10000};do curl http://10.0.0.100;sleep 0.5;done
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
^C


[root@R2 ~ ]#systemctl stop httpd
[root@Client ~ ]#for i in {1..10000};do curl http://10.0.0.100;sleep 0.5;done
<h1>R1.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R1.localdomain</h1>
[root@lvs ha.d ]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.0.0.100:80 wrr
  -> 192.168.31.17:80             Route   1      0          24 
ldirectord提供宕机提示页面
[root@lvs ha.d ]#yum install httpd
[root@lvs ha.d ]#echo Sorry down > /var/www/html/index.html
\#lvs把网关指向路由器的内网ip地址,lvs对外提供抱歉页面
[root@lvs ~ ]#route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.31.6   0.0.0.0         UG    0      0        0 eth0
10.0.0.100      0.0.0.0         255.255.255.255 UH    100    0        0 eth0
192.168.31.0    0.0.0.0         255.255.255.0   U     100    0        0 eth0
ldirectord通过firewall标签来提供服务
[root@lvs ~ ]#iptables -t mangle -A PREROUTING -d 10.0.0.100 -p tcp -m multiport --dports 80,443 -j MARK --set-mark 6 
[root@lvs ha.d ]#iptables -vnL -t mangle
Chain PREROUTING (policy ACCEPT 48 packets, 4205 bytes)
 pkts bytes target     prot opt in     out     source       destination         
   24  1964 MARK       tcp  --  *      *       0.0.0.0/0    10.0.0.100     multiport dports 80,443 MARK set 0x6

[root@lvs ha.d ]#vim ldirectord.cf 
virtual=6
        real=192.168.31.17 gate
        real=192.168.31.27 gate 2
        service=http
        scheduler=wrr
        #persistent=600
        #netmask=255.255.255.255
        #protocol=tcp
        checktype=negotiate
        checkport=80
        request="index.html"
        receive="cos"
[root@lvs ha.d ]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
FWM  6 wrr
  -> 192.168.31.17:0              Route   1      0          126       
  -> 192.168.31.27:0              Route   2      0          0  
[root@Client ~ ]#for i in {1..10000};do curl -k https://10.0.0.100;curl http://10.0.0.100;sleep 0.5;done
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
^C

[root@lvs ~ ]#cat lvs_dr_vs.sh 
#!/bin/bash
vip='10.0.0.100'
iface='eth0:1'
mask='255.255.255.255'
port='80'
rs1='192.168.31.17'
rs2='192.168.31.27'
scheduler='wrr'
type='-g'
rpm -q ipvsadm &> /dev/null || yum -y install ipvsadm &> /dev/null

case $1 in
start)
    ifconfig $iface $vip netmask $mask #broadcast $vip up
    iptables -F

    ipvsadm -A -t ${vip}:${port} -s $scheduler
    ipvsadm -a -t ${vip}:${port} -r ${rs1} $type -w 1
    ipvsadm -a -t ${vip}:${port} -r ${rs2} $type -w 1
    echo "The VS Server is Ready!"
    ;;
stop)
    ipvsadm -C
    ifconfig $iface down
    echo "The VS Server is Canceled!"
    ;;
*)
    echo "Usage: $(basename $0) start|stop"
    exit 1
    ;;
esac
[root@R1 ~ ]#cat lvs_dr_rs.sh 
#!/bin/bash
vip=10.0.0.100
mask='255.255.255.255'
dev=lo:1
rpm -q httpd &> /dev/null || yum -y install httpd &>/dev/null
service httpd start &> /dev/null && echo "The httpd Server is Ready!"
echo "<h1>`hostname`</h1>" > /var/www/html/index.html

case $1 in
start)
    echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
    echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
    ifconfig $dev $vip netmask $mask #broadcast $vip up
    #route add -host $vip dev $dev
    echo "The RS Server is Ready!"
    ;;
stop)
    ifconfig $dev down
    echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
    echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
    echo "The RS Server is Canceled!"
    ;;
*) 
    echo "Usage: $(basename $0) start|stop"
    exit 1
    ;;
esac

  1. lvs_dr_vs.sh
  2. lvs_dr_rs.sh

猜你喜欢

转载自blog.csdn.net/csdn_immortal/article/details/82082533