Administrator
发布于 2025-03-12 / 7 阅读
0

keepalive+nginx高可用

环境

vip(虚拟IP) 192.168.210.199

master 192.168.210.200 keepalived-master机器

backup 192.168.210.201 keepalive-backup机器

web-1 192.168.210.202 web服务器

web-2 192.168.210.203 web备服务器

1、安装keepalive服务(两台机器,master,backup)

[root@master ~]# yum -y install keepalived

[root@backup ~]# yum -y install keepalived

2、创建keepalived配置文件(两台机器内容有区别)

参数介绍:

global_defs {
    router_id lb-5 # 路由器ID,每个机器不一样
}

vrrp_instance VIP_1 {       # 设置VRRP路由器组组名,属于同一组的名字一样
    state MASTER            # 角色,master、backup两种            
    interface eth0          # VIP绑定的网卡         
    virtual_router_id 50    # 虚拟路由IP,同一组的一样    
    priority 150            # 优先级,优先级越高,权重越高
    advert_int 1            # 发送组播间隔1s
    authentication {        # 认证形式
        auth_type PASS      # 密码认证,明文密码 1111
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.200.199            # 指定虚拟VIP地址,必须没人使用
    }
}

master机器keepalive配置文件内容如下:

global_defs {
    router_id master
}

vrrp_instance VIP_1 {    
    state MASTER                                      
    interface eth0                         
    virtual_router_id 50                              
    priority 150         
    advert_int 1
    authentication {  
        auth_type PASS 
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.210.199 
    }
}

backup机器keepalive配置文件内容

global_defs {
    router_id backup
}

vrrp_instance VIP_1 {
    state BACKUP
    interface eth0
    virtual_router_id 50 
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.210.199
    }
}

启动两台机器keepalive服务

1、查看两台机器网卡情况

[root@master ~]# ip addr show eth0
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 52:54:00:99:a5:e0 brd ff:ff:ff:ff:ff:ff
    inet 192.168.200.200/24 brd 192.168.200.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet6 240e:86c:8:a1f4:80cf:be2c:7656:252/64 scope global noprefixroute dynamic 
       valid_lft 1851062359sec preferred_lft 1851062359sec
    inet6 fe80::bb27:eb6d:8b6:d38c/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
[root@backup ~]# ip addr show eth0
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 52:54:00:a0:6e:b2 brd ff:ff:ff:ff:ff:ff
    inet 192.168.200.201/24 brd 192.168.200.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet6 240e:86c:8:a1f4:ff4f:a151:7e74:1fd9/64 scope global noprefixroute dynamic 
       valid_lft 1851062310sec preferred_lft 1851062310sec
    inet6 fe80::9ace:152d:969:9611/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

启动服务

[root@master ~]# systemctl start keepalived && systemctl enable keepalived

再次检查网卡情况

[root@master ~]# ip addr show eth0
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 52:54:00:99:a5:e0 brd ff:ff:ff:ff:ff:ff
    inet 192.168.200.200/24 brd 192.168.200.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet 192.168.200.199/32 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 240e:86c:8:a1f4:80cf:be2c:7656:252/64 scope global noprefixroute dynamic 
       valid_lft 1851062345sec preferred_lft 1851062345sec
    inet6 fe80::bb27:eb6d:8b6:d38c/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
[root@backup ~]# ip addr show eth0
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 52:54:00:a0:6e:b2 brd ff:ff:ff:ff:ff:ff
    inet 192.168.200.201/24 brd 192.168.200.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet 192.168.200.199/32 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 240e:86c:8:a1f4:ff4f:a151:7e74:1fd9/64 scope global noprefixroute dynamic 
       valid_lft 1851062405sec preferred_lft 1851062405sec
    inet6 fe80::9ace:152d:969:9611/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

可以看到在master机器上有一个192.168.200.199的ip地址

本机ping下虚拟ip看下是否能ping通

安装下tcpdump服务看下vrrp协议

[root@master ~]# yum -y install tcpdump

web1,web2安装nginx服务,模拟本地web页面,设置各自不同访问内容

[root@web1 ~]# yum -y install nginx

[root@web1 ~]# echo "WEB-1"> /usr/share/nginx/html/index.html

[root@web1 ~]# systemctl start nginx && systemctl enable nginx

[root@web2 ~]# yum -y install nginx

[root@web2 ~]# echo "WEB-2"> /usr/share/nginx/html/index.html

[root@web2 ~]# systemctl start nginx && systemctl enable nginx

master、backup机器安装nginx做反向代理web1、web2

[root@master ~]# yum -y install nginx

[root@master ~]# vi /etc/nginx/nginx.conf

[root@master ~]# systemctl enable nginx && systemctl start nginx

测试master机器nginx服务访问后端代理是否正常

[root@backup ~]# yum -y install nginx

[root@backup ~]# vi /etc/nginx/nginx.conf

[root@backup ~]# systemctl enable nginx && systemctl start nginx

浏览器访问下backup机器的nginx服务看下反向代理是否生效

master机器,backup机器代理服务访问都正常,测试使用虚拟vip地址访问看是否正常访问

模拟master故障

手动停止master上的keepalive服务,浏览器访问虚拟ip查看是否能访问web服务,查看虚拟ip连接是否切换到backup机器上

停止master上的keepalive服务

再次启动master上的keepalive服务

手动停止master的keepalive服务发现虚拟vip还可以访问web页面

测试nginx服务的反向代理proxy_set_header能否获取到用户真实的ip还是虚拟ip地址

master、backup nginx服务添加反向代理参数

[root@master ~]# vi /etc/nginx/nginx.conf

[root@master ~]# nginx -t

nginx: the configuration file /etc/nginx/nginx.conf syntax is ok

nginx: configuration file /etc/nginx/nginx.conf test is successful

[root@master ~]# nginx -s reload

[root@backup ~]# vi /etc/nginx/nginx.conf

[root@backup ~]# nginx -t

nginx: the configuration file /etc/nginx/nginx.conf syntax is ok

nginx: configuration file /etc/nginx/nginx.conf test is successful

[root@backup ~]# nginx -s reload

浏览器访问虚拟vip再次测试

没修改参数之前记录的是master,backup主机地址

修改之后

手动开启防火墙拦截master和backup之间心跳传输,实验脑裂

开启backup主机的防火墙服务

[root@backup ~]# systemctl start firewalld

master和backup主机都绑定了vip地址

我们浏览器访问下vip地址看是否能正常访问(虽然发生了脑裂,但是还能正常访问,速度会慢很多)

发生脑裂会导致集群无法切换,停止master的keepalive服务

[root@master ~]# systemctl stop keepalived

组播心跳变成backup备机了

再次访问网页(发现备机也无法访问)

停止backup防火墙服务

[root@backup ~]# systemctl stop firewalld

再次测试web访问恢复正常

解决脑裂方案

防火墙规则

只要启动防火墙服务就会抢夺vip

systemctl restart firewalld

vip应该会消失,vrrp正常

iptables -I INPUT -i eth0 -d 224.0.0.0/8 -p vrrp -j ACCEPT

防止脑裂脚本开发

确保backup机器定时去判断master的vip状态,如果发生脑裂,backup自杀

思路(针对backup备机):

1、backup定期检查master的nginx服务是否运行

2、backup定期检查自己是否有vip

3、如果有如下情况,backup也有vip就是脑裂

  • master的nginx正常

  • master有vip

  • backup有vip

4、如果backup有脑裂,就杀掉自己

5、告知管理员

backup脚本开发

前置条件,backup可以免密登录master

[root@backup keepalived]# vim /etc/keepalived/backup_check_vip.sh

#!/bin/bash
CHECK_MASTER_VIP=$(ssh 192.168.200.200 ip a| grep 192.168.200.199 |wc -l)
CHECK_BACKUP_VIP=$(ip a |grep 192.168.200.199 |wc -l)

if [ ${CHECK_MASTER_VIP} == 1 -a ${CHECK_BACKUP_VIP} == 1 ]
then
        systemctl stop keepalived
fi

给脚本设置执行权限

[root@backup keepalived]# chmod a+x backup_check_vip.sh

keepalive配置文件添加调用脚本

[root@backup keepalived]# vi /etc/keepalived/keepalived.conf

global_defs {
    router_id lb-6
}
#定义脚本
vrrp_script check_vip {
    script "/etc/keepalived/backup_check_vip.sh"
    interval 5 # 脚本执行时间
}

vrrp_instance VIP_1 {
    state BACKUP
    interface eth0
    virtual_router_id 50
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.200.199
    }
    track_script {
    check_vip
    }
}

重启keepalived服务

[root@backup keepalived]# systemctl restart keepalived

测试backup脚本能够恢复脑裂

1、确保master、backup正常

手动开启backup机器防火墙

发现有在抢夺vip,查看keepalived服务是否停止

master机器nginx服务出现问题

监控master故障的脚本

思路:

master机器脚本

1、如果自己的nginx已经不存在的,keepalived服务还活着,就尝试重启nginx

2、如果nginx重启失败,就干掉自己的keepalived服务,放弃master资源,自动切换到backup

脚本如下:

[root@master ~]# vim /etc/keepalived/check_web.sh

#!/bin/bash
NGINX_STATUS=$(ps -ef |grep nginx[x] |wc -l )

if [ ${NGINX_STATUS} == 0 ]
then
    systemctl restart nginx
    if [ $? == 1 ]
    then
        #keepalived kill
        systemctl stop keepalived
    fi
fi

[root@master ~]# chmod a+x /etc/keepalived/check_web.sh

keepalived配置文件调用sh脚本

[root@master ~]# vi /etc/keepalived/keepalived.conf

global_defs {
    router_id lb-5
}
vrrp_script check_web {
    script "/etc/keepalived/check_web.sh"
    interval 5
}
vrrp_instance VIP_1 {
    state MASTER
    interface eth0
    virtual_router_id 50
    priority 150
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.200.199
    }
    track_script {
    check_web
    }
}

重启keepalive使配置更新生效

[root@master ~]# systemctl restart keepalived

模拟master 的nginx服务故障

1、手动停止nginx服务看是否会启动起来

[root@master ~]# systemctl stop nginx

2、模拟nginx配置文件错误,导致nginx无法启动

[root@master keepalived]# echo "." >> /etc/nginx/nginx.conf

[root@master keepalived]# nginx -t

nginx: [emerg] unexpected end of file, expecting ";" or "}" in /etc/nginx/nginx.conf:98

nginx: configuration file /etc/nginx/nginx.conf test failed

停止nginx服务

查看master的keepalive服务是否停止

实验到此完成