nginx负载均衡调度器配置高可用服务
一、反向代理与负载均衡
nginx通常被用作后端服务器的反向代理，这样就可以很方便的实现动静分离以及负载均衡，从而大大提高服务器的处理能力。
nginx实现动静分离，其实就是在反向代理的时候，如果是静态资源，就直接从nginx发布的路径去读取，而不需要从后台服务器获取了。
但是要注意，这种情况下需要保证后端跟前端的程序保持一致，可以使用Rsync做服务端自动同步或者使用NFS、MFS分布式共享存储。
Http Proxy`模块，功能很多，最常用的是`proxy_pass`和`proxy_cache
如果要使用proxy_cache，需要集成第三方的ngx_cache_purge模块，用来清除指定的URL缓存。这个集成需要在安装nginx的时候去做，如：
./configure --add-module=../ngx_cache_purge-1.0 ......
nginx通过upstream模块来实现简单的负载均衡，upstream需要定义在http段内
在upstream段内，定义一个服务器列表，默认的方式是轮询，如果要确定同一个访问者发出的请求总是由同一个后端服务器来处理，可以设置ip_hash，如：
upstream idfsoft.com {
ip_hash;
server 127.0.0.1:9080 weight=5;
server 127.0.0.1:8080 weight=5;
server 127.0.0.1:1111;
}
注意：这个方法本质还是轮询，而且由于客户端的ip可能是不断变化的，比如动态ip，代理，翻墙等，因此ip_hash并不能完全保证同一个客户端总是由同一个服务器来处理。
定义好upstream后，需要在server段内添加如下内容：
server {
location / {
proxy_pass http://idfsoft.com;
}
}
二、nginx负载均衡调度器高可用配置
环境说明
| 主机名 | IP地址 | 安装应用 | 系统 |
|---|---|---|---|
| 135 | 192.168.183.135 | nginx、keepalived | centos8 |
| 136 | 192.168.183.136 | nginx、keepalived | centos8 |
| 137 | 192.168.183.137 | httpd | centos8 |
| 138 | 192.168.183.138 | nginx | centos8 |
注意：在配置之前记得关闭防火墙和selinux，selinux模式为disabled模式！
配置137主机
[root@137 ~]# dnf -y install httpd
[root@137 ~]# echo 'apache!' > /var/www/html/index.html
[root@137 ~]# systemctl enable --now httpd
[root@137 ~]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
LISTEN 0 128 *:80 *:*
[root@137 ~]# curl 192.168.183.137
apache!
配置138主机
[root@138 ~]# dnf -y install nginx
[root@138 ~]# echo 'nginx!' > /usr/share/nginx/html/index.html
[root@138 ~]# systemctl enable --now nginx
Created symlink /etc/systemd/system/multi-user.target.wants/nginx.service → /usr/lib/systemd/system/nginx.service.
[root@138 ~]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 0.0.0.0:80 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
LISTEN 0 128 [::]:80 [::]:*
[root@138 ~]# curl 192.168.183.138
nginx!
配置136主机
//安装依赖
[root@136 ~]# dnf -y install pcre-devel openssl openssl-devel gd-devel gcc gcc-c++ make wget vim
//创建用户
[root@136 ~]# useradd -rMs /sbin/nologin nginx
//下载软件包并解压编译
[root@136 ~]# wget http://nginx.org/download/nginx-1.20.2.tar.gz
[root@136 ~]# tar -xf nginx-1.20.2.tar.gz
[root@136 ~]# cd nginx-1.20.2
[root@136 nginx-1.20.2]# ./configure \
> --prefix=/usr/local/nginx \
> --user=nginx \
> --group=nginx \
> --with-debug \
> --with-http_ssl_module \
> --with-http_realip_module \
> --with-http_image_filter_module \
> --with-http_gunzip_module \
> --with-http_gzip_static_module \
> --with-http_stub_status_module \
> --http-log-path=/var/log/nginx/access.log \
> --error-log-path=/var/log/nginx/error.log
[root@136 ~]# make -j $(grep 'processor' /proc/cpuinfo | wc -l) && make install
//配置环境变量
[root@136 ~]# echo "export PATH=$PATH:/usr/local/nginx/sbin" > /etc/profile.d/nginx.sh
[root@136 ~]# source /etc/profile.d/nginx.sh
//启动
[root@136 ~]# cat > /usr/lib/systemd/system/nginx.service << EOF
> [Unit]
> Description=nginx server daemon
> After=network.target
>
> [Service]
> Type=forking
> ExecStart=/usr/local/nginx/sbin/nginx
> ExecStop=/usr/local/nginx/sbin/nginx -s stop
> ExecReload=/bin/kill -HUP \$MAINPID
>
> [Install]
> WantedBy=multi-user.target
> EOF
[root@136 ~]# systemctl daemon-reload
[root@136 ~]# systemctl enable --now nginx
Created symlink /etc/systemd/system/multi-user.target.wants/nginx.service → /usr/lib/systemd/system/nginx.service.
[root@136 ~]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 0.0.0.0:80 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
//配置负载均衡反向代理
[root@136 ~]# vim /usr/local/nginx/conf/nginx.conf
upstream webservers {
server 192.168.183.137;
server 192.168.183.138;
}
······
location / {
proxy_pass http://webservers;
}
//访问测试
[root@136 ~]# curl 192.168.183.136
apache!
[root@136 ~]# curl 192.168.183.136
nginx!
[root@136 ~]# curl 192.168.183.136
apache!
[root@136 ~]# curl 192.168.183.136
nginx!
[root@136 ~]# curl 192.168.183.136
apache!
[root@136 ~]# curl 192.168.183.136
nginx!
配置多次访问apache
[root@136 ~]# vim /usr/local/nginx/conf/nginx.conf
upstream webservers {
server 192.168.183.137 weight=3;
server 192.168.183.138;
}
[root@136 ~]# !system
systemctl restart nginx
//测试
[root@136 ~]# curl 192.168.183.136
apache!
[root@136 ~]# curl 192.168.183.136
apache!
[root@136 ~]# curl 192.168.183.136
apache!
[root@136 ~]# curl 192.168.183.136
nginx!
[root@136 ~]# curl 192.168.183.136
apache!
[root@136 ~]# curl 192.168.183.136
apache!
[root@136 ~]# curl 192.168.183.136
apache!
[root@136 ~]# curl 192.168.183.136
nginx!
配置ip_hash，第一个访问到apache，那么就一直是apache
upstream webservers {
ip_hash;
server 192.168.183.137 weight=3;
server 192.168.183.138;
}
//访问测试
[root@136 ~]# curl 192.168.183.136
apache!
[root@136 ~]# curl 192.168.183.136
apache!
[root@136 ~]# curl 192.168.183.136
apache!
[root@136 ~]# curl 192.168.183.136
apache!
[root@136 ~]# curl 192.168.183.136
apache!
[root@136 ~]# curl 192.168.183.136
apache!
[root@136 ~]# curl 192.168.183.136
apache!
配置高可用
配置135主机
//安装nginx
[root@135 ~]# dnf -y install pcre-devel openssl openssl-devel gd-devel gcc gcc-c++ make wget vim
[root@135 ~]# useradd -rMs /sbin/nologin nginx
[root@135 ~]# wget http://nginx.org/download/nginx-1.20.2.tar.gz
[root@135 ~]# tar -xf nginx-1.20.2.tar.gz
[root@135 ~]# cd nginx-1.20.2
[root@135 nginx-1.20.2]# ./configure \
> --prefix=/usr/local/nginx \
> --user=nginx \
> --group=nginx \
> --with-debug \
> --with-http_ssl_module \
> --with-http_realip_module \
> --with-http_image_filter_module \
> --with-http_gunzip_module \
> --with-http_gzip_static_module \
> --with-http_stub_status_module \
> --http-log-path=/var/log/nginx/access.log \
> --error-log-path=/var/log/nginx/error.log
[root@135 nginx-1.20.2]# make -j $(grep 'processor' /proc/cpuinfo | wc -l) && make install
[root@135 nginx-1.20.2]# echo "export PATH=$PATH:/usr/local/nginx/sbin" > /etc/profile.d/nginx.sh
[root@135 nginx-1.20.2]# source /etc/profile.d/nginx.sh
[root@135 nginx-1.20.2]# cat > /usr/lib/systemd/system/nginx.service << EOF
> [Unit]
> Description=nginx server daemon
> After=network.target
>
> [Service]
> Type=forking
> ExecStart=/usr/local/nginx/sbin/nginx
> ExecStop=/usr/local/nginx/sbin/nginx -s stop
> ExecReload=/bin/kill -HUP \$MAINPID
>
> [Install]
> WantedBy=multi-user.target
> EOF
[root@135 nginx-1.20.2]# systemctl daemon-reload
[root@135 nginx-1.20.2]# systemctl enable --now nginx
Created symlink /etc/systemd/system/multi-user.target.wants/nginx.service → /usr/lib/systemd/system/nginx.service.
[root@135 ~]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port Process
LISTEN 0 128 0.0.0.0:80 0.0.0.0:*
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
关闭ip_hash功能
修改135主机配置文件
[root@135 ~]# vim /usr/local/nginx/conf/nginx.conf
upstream webservers {
server 192.168.183.137;
server 192.168.183.138;
}
······
location / {
proxy_pass http://webservers;
}
修改136主机配置文件
[root@135 ~]# vim /usr/local/nginx/conf/nginx.conf
upstream webservers {
server 192.168.183.137;
server 192.168.183.138;
}
······
location / {
proxy_pass http://webservers;
}
安装keepalived
配置135主机
[root@135 ~]# dnf -y install keepalived
[root@135 ~]# cd /etc/keepalived/
[root@135 keepalived]# mv keepalived.conf{,-bak}
[root@135 keepalived]# cat keepalived.conf
global_defs {
router_id LVS_Server
}
vrrp_instance VI_1 {
state MASTER
interface ens33
virtual_router_id 51
priority 100
nopreempt
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
192.168.183.250 dev ens33
}
}
virtual_server 192.168.183.250 80 {
delay_loop 3
lvs_sched rr
lvs_method DR
protocol TCP
real_server 192.168.183.135 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.183.136 8080 {
weight 1
TCP_CHECK {
connect_port 8080
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
[root@135 keepalived]# systemctl enable --now keepalived
Created symlink /etc/systemd/system/multi-user.target.wants/keepalived.service → /usr/lib/systemd/system/keepalived.service.
[root@135 keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:7f:37:b0 brd ff:ff:ff:ff:ff:ff
inet 192.168.183.135/24 brd 192.168.183.255 scope global dynamic noprefixroute ens33
valid_lft 1122sec preferred_lft 1122sec
inet 192.168.183.250/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe7f:37b0/64 scope link noprefixroute
valid_lft forever preferred_lft forever
配置136主机
[root@136 ~]# dnf -y install keepalived
[root@136 ~]# cd /etc/keepalived/
[root@136 keepalived]# mv keepalived.conf{,-bak}
[root@136 keepalived]# cat keepalived.conf
bal_defs {
router_id LVS_Server
}
vrrp_instance VI_1 {
state BACKUP
interface ens33
virtual_router_id 51
priority 90
nopreempt
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
192.168.183.250 dev ens33
}
}
virtual_server 192.168.183.250 80 {
delay_loop 3
lvs_sched rr
lvs_method DR
protocol TCP
real_server 192.168.183.135 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.183.136 8080 {
weight 1
TCP_CHECK {
connect_port 8080
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
[root@136 keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:07:de:9b brd ff:ff:ff:ff:ff:ff
inet 192.168.183.136/24 brd 192.168.183.255 scope global dynamic noprefixroute ens33
valid_lft 1502sec preferred_lft 1502sec
inet6 fe80::20c:29ff:fe07:de9b/64 scope link noprefixroute
valid_lft forever preferred_lft forever
模拟135主服务器故障
[root@135 ~]# systemctl stop keepalived
[root@135 ~]# systemctl stop nginx
[root@135 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:7f:37:b0 brd ff:ff:ff:ff:ff:ff
inet 192.168.183.135/24 brd 192.168.183.255 scope global dynamic noprefixroute ens33
valid_lft 1395sec preferred_lft 1395sec
inet6 fe80::20c:29ff:fe7f:37b0/64 scope link noprefixroute
valid_lft forever preferred_lft forever
虚拟IP会到136备服务器
[root@136 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:07:de:9b brd ff:ff:ff:ff:ff:ff
inet 192.168.183.136/24 brd 192.168.183.255 scope global dynamic noprefixroute ens33
valid_lft 1387sec preferred_lft 1387sec
inet 192.168.183.250/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe07:de9b/64 scope link noprefixroute
valid_lft forever preferred_lft forever
测试
[root@136 ~]# curl 192.168.183.250
apache!