LVS-DR模式实验
清空防火墙策略iptables -F
关闭selinux
如图
[root@Client ~ ]#for i in {1..100};do curl 10.0.0.100;sleep 0.5;done
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
^C
1、客户端配置
配置ip 172.18.0.222/16
路由
[root@Client ~ ]#route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
172.18.0.0 0.0.0.0 255.255.0.0 U 0 0 0 eth1
10.0.0.0 172.18.0.6 255.0.0.0 UG 0 0 0 eth1
2、路由器配置
路由器的192.168.31.6/24(接受RS提供的httpd服务的包,包里封装的是172.18.0.222,10.0.0.100的包)和10.0.0.1/8(客户端访问10.0.0.100时的路由转发功能)需在同一个网卡上,一共3个ip
[root@cos37 / ]#ip a
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:65:83:74 brd ff:ff:ff:ff:ff:ff
inet `192.168.31.6/24` brd 192.168.31.255 scope global noprefixroute eth0
valid_lft forever preferred_lft forever
inet `10.0.0.1/8` scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe65:8374/64 scope link
valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:65:83:7e brd ff:ff:ff:ff:ff:ff
inet `172.18.0.6/16` brd 172.18.255.255 scope global noprefixroute eth1
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe65:837e/64 scope link
valid_lft forever preferred_lft forever
[root@cos37 / ]#route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
10.0.0.0 0.0.0.0 255.0.0.0 U 0 0 0 eth0
172.18.0.0 0.0.0.0 255.255.0.0 U 101 0 0 eth1
192.168.31.0 0.0.0.0 255.255.255.0 U 100 0 0 eth0
3、lvs配置
内网ip192.168.31.7/24(把客户端的包172.18.0.222,10.0.0.100的包调度到RS服务器),公网ip10.0.0.100/32对互联网提供服务
[root@lvs ~ ]#ip a
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:3b:cf:92 brd ff:ff:ff:ff:ff:ff
inet `192.168.31.7/24` brd 192.168.31.255 scope global noprefixroute eth0
valid_lft forever preferred_lft forever
inet `10.0.0.100/32` brd 10.255.255.255 scope global eth0:1
valid_lft forever preferred_lft forever
inet6 fe80::a441:7018:d3a4:d5d6/64 scope link noprefixroute
valid_lft forever preferred_lft forever
\#路由192.168.31.66,理论上可以不添加,因为响应报文不经过lvs,但实际行不通,没有此路由,当请求报文到达lvs调度器时,检查没有响应报文需要的路由,则会直接拒绝
[root@lvs ~ ]#route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.31.66 0.0.0.0 UG 0 0 0 eth0
10.0.0.100 0.0.0.0 255.255.255.255 UH 100 0 0 eth0
192.168.31.0 0.0.0.0 255.255.255.0 U 100 0 0 eth0
脚本
1
4、rs配置
[root@R1 ~ ]#ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet `10.0.0.100/32` scope global lo:1
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:50:56:22:38:c9 brd ff:ff:ff:ff:ff:ff
inet `192.168.31.17/24` brd 192.168.31.255 scope global noprefixroute eth0
valid_lft forever preferred_lft forever
inet6 fe80::4619:664b:f1e4:d5b3/64 scope link noprefixroute
valid_lft forever preferred_lft forever
#路由指向路由器的网关,把封装的报文(源地址10.0.0.100,目的地址172.18.0.222)由192.168.31.17送往192.168.31.6;10.0.0.100为调度器接受了目的到地址是10.0.0.100的包调度到RS提供服务的主机上,由rs的10.0.0.100提供服务,再由rs的另一个内网地址192.168.31.17送往路由
[root@R1 ~ ]#route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.31.6 0.0.0.0 UG 0 0 0 eth0
192.168.31.0 0.0.0.0 255.255.255.0 U 100 0 0 eth0
#另一个RS服务器配置和上面的相同,只是内网ip为192.168.31.27
[root@R2 ~ ]#
脚本
2
实现https的安全加密
[root@R1 ~ ]#cd /etc/pki/tls/certs
[root@R1 certs ]#ls
ca-bundle.crt ca-bundle.trust.crt localhost.crt make-dummy-cert Makefile renew-dummy-cert
#去掉私钥加密选项-aes128
[root@R1 certs ]#vim Makefile
%.key:
umask 77 ; \
/usr/bin/openssl genrsa -aes128 $(KEYLEN) > $@
[root@R1 certs ]#make dhy.crt
umask 77 ; \
/usr/bin/openssl genrsa 2048 > dhy.key
Generating RSA private key, 2048 bit long modulus
..............................+++
.................................................................................................................+++
e is 65537 (0x10001)
umask 77 ; \
/usr/bin/openssl req -utf8 -new -key dhy.key -x509 -days 365 -out dhy.crt
You are about to be asked to enter information that will be incorporated
into your certificate request.
What you are about to enter is what is called a Distinguished Name or a DN.
There are quite a few fields but you can leave some blank
For some fields there will be a default value,
If you enter '.', the field will be left blank.
-----
Country Name (2 letter code) [XX]:CN
State or Province Name (full name) []:beijing
Locality Name (eg, city) [Default City]:beijing
Organization Name (eg, company) [Default Company Ltd]:dhy.com
Organizational Unit Name (eg, section) []:opt
Common Name (eg, your name or your server's hostname) []:denghaiyang
Email Address []:
[root@R1 certs ]#mv dhy.* /etc/httpd/conf.d/
[root@R1 conf.d ]#yum install mod_ssl
[root@R1 conf.d ]#vim ssl.conf
[root@R1 conf.d ]#scp ssl.conf dhy.* 192.168.31.27:/etc/httpd/conf.d/
[email protected]'s password:
ssl.conf 100% 9427 8.4MB/s 00:00
dhy.crt 100% 1318 11.1KB/s 00:00
dhy.key 100% 1675 130.3KB/s 00:00
[root@R1 conf.d ]#systemctl restart httpd
#端口改为443
[root@lvs ~ ]#vim lvs_dr_vs.sh
mask='255.255.255.255'
port='443'
[root@lvs ~ ]#bash lvs_dr_vs.sh stop
The VS Server is Canceled!
[root@lvs ~ ]#bash lvs_dr_vs.sh start
The VS Server is Ready!
[root@lvs ~ ]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.0.0.100:443 wrr
-> 192.168.31.17:443 Route 1 0 0
-> 192.168.31.27:443 Route 1 0 0
[root@Client ~ ]#for i in {1..10000};do curl -k https://10.0.0.100;sleep 0.5;done
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
^C
window访问
C:\Users\29256>route –help
C:\WINDOWS\system32>route add 10.0.0.0/8 172.18.0.6
操作完成!
如图
C:\WINDOWS\system32>route delete 10.0.0.0
操作完成!
FireWall Mark
1、FWM:FireWall Mark
2、MARK target 可用于给特定的报文打标记 –set-mark value 其中:value 可为0xffff格式,表示十六进制数字
3、借助于防火墙标记来分类报文,而后基于标记定义集群服务;可将多个不同的 应用使用同一个集群服务进行调度
4、实现方法:
在Director主机打标记:
iptables -t mangle -A PREROUTING -d 10.0.0.100 -p tcp -m multiport –dports 80,443 -j MARK –set-mark 6
在Director主机基于标记定义集群服务:
ipvsadm -A -f 6 -s rr
ipvsadm -a -f 6 -r 192.168.31.17 -g
ipvsadm -a -f 6 -r 192.168.31.27 -g
ipvsadm -Ln
如下:
[root@lvs ~ ]#ipvsadm -C #清空
[root@lvs ~ ]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
[root@lvs ~ ]#iptables -t mangle -A PREROUTING -d 10.0.0.100 -p tcp -m multiport --dports 80,443 -j MARK --set-mark 6
[root@lvs ~ ]#ipvsadm -A -f 6 -s rr
[root@lvs ~ ]#ipvsadm -a -f 6 -r 192.168.31.17 -g
[root@lvs ~ ]#ipvsadm -a -f 6 -r 192.168.31.27 -g
[root@lvs ~ ]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
FWM 6 rr
-> 192.168.31.17:0 Route 1 0 0
-> 192.168.31.27:0 Route 1 0 0
[root@Client ~ ]#for i in {1..10000};do curl -k https://10.0.0.100;curl http://10.0.0.100;sleep 0.5;done
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
^C
持久连接
1、 session 绑定:对共享同一组RS的多个集群服务,需要统一进行绑定,lvs sh算法无法实现
2、 持久连接( lvs persistence )模板:实现无论使用任何调度算法,在一段时间内(默认360s ),能够实现将来自同一个地址的请求始终发往同一个RS
ipvsadm -A|E -t|u|f service-address [-s scheduler] [-p [timeout]]
3、 持久连接实现方式:
4、 每端口持久(PPC):每个端口对应定义为一个集群服务,每集群服务单独调度
5、 每防火墙标记持久(PFWMC):基于防火墙标记定义集群服务;可实现将多个端口上的应用统一调度,即所谓的port Affinity
6、 每客户端持久(PCC):基于0端口(表示所有服务)定义集群服务,即将客户端对所有应用的请求都调度至后端主机,必须定义为持久模式
每防火墙持久标记
[root@lvs ~ ]#ipvsadm -E -f 6 -s rr -p
[root@lvs ~ ]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
FWM 6 rr persistent 360 #360秒
-> 192.168.31.17:0 Route 1 0 5
-> 192.168.31.27:0 Route 1 0 5
[root@Client ~ ]#for i in {1..10000};do curl -k https://10.0.0.100;curl http://10.0.0.100;sleep 0.5;done
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
^C
每客户端持久
ipvsadm -A -t 10.0.0.100:0 -s rr -p #0表示客户端访问10.0.0.100:端口是一一对应调度到相应的RS服务器,这样不要,端口都暴露了
#修改为不是持久连接
ipvsadm -E -f 10 -s rr
LVS高可用性
1 Director不可用,整个系统将不可用;SPoF Single Point of Failure
解决方案:高可用
keepalived heartbeat/corosync
2某RS不可用时,Director依然会调度请求至此RS
解决方案: 由Director对各RS健康状态进行检查,失败时禁用,成功时启用
keepalived heartbeat/corosync ldirectord
检测方式:
(a) 网络层检测,icmp
(b) 传输层检测,端口探测
(c) 应用层检测,请求某关键资源
RS全不用时:backup server, sorry server
lvs不具备健康性检查,由director实现,至于lvs宕机,需要keepalived解决,keepalived更加强大
ldirectord
ldirectord:监控和控制LVS守护进程,可管理LVS规则
包名:ldirectord-3.9.6-0rc1.1.1.x86_64.rpm
文件:
/etc/ha.d/ldirectord.cf 主配置文件
/usr/share/doc/ldirectord-3.9.6/ldirectord.cf 配置模版
/usr/lib/systemd/system/ldirectord.service 服务
/usr/sbin/ldirectord 主程序
/var/log/ldirectord.log 日志
/var/run/ldirectord.ldirectord.pid pid文件
Ldirectord #配置文件示例
checktimeout=3 #检查超时为3秒未响应,则处理
checkinterval=1 #每1秒检查一次
autoreload=yes
logfile=“/var/log/ldirectord.log“ #日志文件
quiescent=no #down时yes权重为0,no为删除
virtual=5 #指定VS的FWM或IP:port
real=172.16.0.7:80 gate 2
real=172.16.0.8:80 gate 1
fallback=127.0.0.1:80 gate #sorry server service=http scheduler=wrr
checktype=negotiate
checkport=80
request="index.html"
receive="rs"
实验ldirectord
http://download.opensuse.org/repositories/network:/ha-clustering:/Stable/CentOS_CentOS-7/x86_64/
[root@lvs ~ ]#rpm -qpl ldirectord-3.9.6-0rc1.1.2.x86_64.rpm
warning: ldirectord-3.9.6-0rc1.1.2.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID 17280ddf: NOKEY
/etc/ha.d
/etc/ha.d/resource.d
/etc/ha.d/resource.d/ldirectord
/etc/logrotate.d/ldirectord
/usr/lib/ocf/resource.d/heartbeat/ldirectord
/usr/lib/systemd/system/ldirectord.service
/usr/sbin/ldirectord #主程序,perl语言脚本
/usr/share/doc/ldirectord-3.9.6
/usr/share/doc/ldirectord-3.9.6/COPYING
/usr/share/doc/ldirectord-3.9.6/ldirectord.cf #配置文件参考文档
/usr/share/man/man8/ldirectord.8.gz
[root@lvs ~ ]#yum install ldirectord-3.9.6-0rc1.1.2.x86_64.rpm -y
cat /usr/lib/systemd/system/ldirectord.service
cp /usr/share/doc/ldirectord-3.9.6/ldirectord.cf /etc/ha.d
#ldirectord把lvs的策略都写在配置文件里面了
[root@lvs ha.d ]#vim /etc/ha.d/ldirectord.cf
virtual=10.0.0.100:80
real=192.168.31.17:80 gate
real=192.168.31.27:80 gate 2
service=http
scheduler=wrr
#persistent=600
#netmask=255.255.255.255
protocol=tcp
checktype=negotiate
checkport=80
request="index.html"
receive="cos"
[root@lvs ~ ]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
[root@lvs ha.d ]#systemctl start ldirectord
[root@lvs ha.d ]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.0.0.100:80 wrr
-> 192.168.31.17:80 Route 1 0 0
-> 192.168.31.27:80 Route 2 0 0
[root@Client ~ ]#for i in {1..10000};do curl http://10.0.0.100;sleep 0.5;done
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
^C
[root@R2 ~ ]#systemctl stop httpd
[root@Client ~ ]#for i in {1..10000};do curl http://10.0.0.100;sleep 0.5;done
<h1>R1.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R1.localdomain</h1>
[root@lvs ha.d ]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.0.0.100:80 wrr
-> 192.168.31.17:80 Route 1 0 24
ldirectord提供宕机提示页面
[root@lvs ha.d ]#yum install httpd
[root@lvs ha.d ]#echo Sorry down > /var/www/html/index.html
\#lvs把网关指向路由器的内网ip地址,lvs对外提供抱歉页面
[root@lvs ~ ]#route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.31.6 0.0.0.0 UG 0 0 0 eth0
10.0.0.100 0.0.0.0 255.255.255.255 UH 100 0 0 eth0
192.168.31.0 0.0.0.0 255.255.255.0 U 100 0 0 eth0
ldirectord通过firewall标签来提供服务
[root@lvs ~ ]#iptables -t mangle -A PREROUTING -d 10.0.0.100 -p tcp -m multiport --dports 80,443 -j MARK --set-mark 6
[root@lvs ha.d ]#iptables -vnL -t mangle
Chain PREROUTING (policy ACCEPT 48 packets, 4205 bytes)
pkts bytes target prot opt in out source destination
24 1964 MARK tcp -- * * 0.0.0.0/0 10.0.0.100 multiport dports 80,443 MARK set 0x6
[root@lvs ha.d ]#vim ldirectord.cf
virtual=6
real=192.168.31.17 gate
real=192.168.31.27 gate 2
service=http
scheduler=wrr
#persistent=600
#netmask=255.255.255.255
#protocol=tcp
checktype=negotiate
checkport=80
request="index.html"
receive="cos"
[root@lvs ha.d ]#ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
FWM 6 wrr
-> 192.168.31.17:0 Route 1 0 126
-> 192.168.31.27:0 Route 2 0 0
[root@Client ~ ]#for i in {1..10000};do curl -k https://10.0.0.100;curl http://10.0.0.100;sleep 0.5;done
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R1.localdomain</h1>
<h1>R2.localdomain</h1>
<h1>R2.localdomain</h1>
^C
[root@lvs ~ ]#cat lvs_dr_vs.sh
#!/bin/bash
vip='10.0.0.100'
iface='eth0:1'
mask='255.255.255.255'
port='80'
rs1='192.168.31.17'
rs2='192.168.31.27'
scheduler='wrr'
type='-g'
rpm -q ipvsadm &> /dev/null || yum -y install ipvsadm &> /dev/null
case $1 in
start)
ifconfig $iface $vip netmask $mask #broadcast $vip up
iptables -F
ipvsadm -A -t ${vip}:${port} -s $scheduler
ipvsadm -a -t ${vip}:${port} -r ${rs1} $type -w 1
ipvsadm -a -t ${vip}:${port} -r ${rs2} $type -w 1
echo "The VS Server is Ready!"
;;
stop)
ipvsadm -C
ifconfig $iface down
echo "The VS Server is Canceled!"
;;
*)
echo "Usage: $(basename $0) start|stop"
exit 1
;;
esac
[root@R1 ~ ]#cat lvs_dr_rs.sh
#!/bin/bash
vip=10.0.0.100
mask='255.255.255.255'
dev=lo:1
rpm -q httpd &> /dev/null || yum -y install httpd &>/dev/null
service httpd start &> /dev/null && echo "The httpd Server is Ready!"
echo "<h1>`hostname`</h1>" > /var/www/html/index.html
case $1 in
start)
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
ifconfig $dev $vip netmask $mask #broadcast $vip up
#route add -host $vip dev $dev
echo "The RS Server is Ready!"
;;
stop)
ifconfig $dev down
echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo "The RS Server is Canceled!"
;;
*)
echo "Usage: $(basename $0) start|stop"
exit 1
;;
esac