14 Openstack-Ussuri集群-改装linuxbridge为openvswitch + vxlan -centos8

这里写目录标题

1 前情提要

1.1 现应用分布情况

2 前置工作

2.1 所有安装过linuxbridge的节点都删除 - compute163/164
2.2 安装openvswitch - compute163/164

3 相关配置修改

3.1 控制节点配置修改
3.2 计算节点配置修改

X.过程中遇到的问题

1 前情提要

#根据之前的集群部署教程，我们已经部署完了openstack最小化安装版本的应用，之前使用的是官网提供的linuxbridge+vxlan模式；
#本教程，我们要将linuxbridge+vxlan模式改装成openvswitch+vxlan模式！

1.1 现应用分布情况

在这里插入图片描述
#由于这里我将网络节点直接部署在计算节点上，因此，我们只需要将compute节点上的 neutron-linuxbridge-agent改成neutron-openvswitch-agent即可；

2 前置工作

2.1 所有安装过linuxbridge的节点都删除 - compute163/164

#删除已经配置的linuxbridge网络，可直接在dashboard上面操作
#删除顺序如下：释放虚拟ip端口–>删除路由–>删除网络
#验证是否还有未删除网络，输出为空

[root@controller160 ~]# openstack network list

#查看安装节点

[root@controller160 ~]# openstack network agent list |grep linuxbridge
| b7cf0b1d-4ff0-4314-9427-8cfda3419e15 | Linux bridge agent | compute164 | None              | :-)   | UP    | neutron-linuxbridge-agent |
| c510acb9-7123-4ae3-850c-5e5184b6542a | Linux bridge agent | compute163 | None              | :-)   | UP    | neutron-linuxbridge-agent |

#从安装节点上关闭并卸载neutron-linuxbridge-agent

systemctl disable neutron-linuxbridge-agent
systemctl stop neutron-linuxbridge-agent
yum remove -y openstack-neutron-linuxbridge

2.2 安装openvswitch - compute163/164

#安装openstack-neutron-openvswitch

yum install -y openstack-neutron-openvswitch

#内核配置

echo '
net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.default.rp_filter=0
'>>/etc/sysctl.conf
sysctl -p
[root@controller160 ~]# sysctl -p
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0

3 相关配置修改

3.1 控制节点配置修改

#设置开启route，前面已经设置过的，可不执行

openstack-config --set /etc/neutron/neutron.conf DEFAULT service_plugins router

#备份ml2配置及修改

cp /etc/neutron/plugins/ml2/ml2_conf.ini{,bak}
echo '#
[DEFAULT]
[ml2]
# ...
type_drivers = flat,vlan,vxlan
tenant_network_types = vxlan
mechanism_drivers = openvswitch,l2population
extension_drivers = port_security

[ml2_type_flat]
# ...
flat_networks = provider

[ml2_type_vxlan]
# ...
vni_ranges = 1:1000

[securitygroup]
# ...
enable_ipset = true
#'>/etc/neutron/plugins/ml2/ml2_conf.ini

#重启服务

systemctl restart neutron-server

3.2 计算节点配置修改

#配置备份

cp /etc/neutron/plugins/ml2/ml2_conf.ini{,bak1}
cp /etc/neutron/l3_agent.ini{,bak1}
cp /etc/neutron/dhcp_agent.ini{,bak1}
cp /etc/neutron/plugins/ml2/openvswitch_agent.ini{,.bak}

#修改ml2_conf.ini

echo '#
[DEFAULT]
[ml2]
# ...
type_drivers = flat,vlan,vxlan
tenant_network_types = vxlan
mechanism_drivers = openvswitch,l2population
extension_drivers = port_security

[ml2_type_flat]
# ...
flat_networks = provider

[ml2_type_vxlan]
# ...
vni_ranges = 1:1000

[securitygroup]
# ...
enable_ipset = true
#'>/etc/neutron/plugins/ml2/ml2_conf.ini

#修改l3_agent.ini

echo '#
[DEFAULT]
interface_driver=neutron.agent.linux.interface.OVSInterfaceDriver
external_network_bridge = br-ex
#'>/etc/neutron/l3_agent.ini

#修改dhcp_agent.ini

echo '#
[DEFAULT]
interface_driver=neutron.agent.linux.interface.OVSInterfaceDriver
dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq
enable_isolated_metadata = true
#'>/etc/neutron/dhcp_agent.ini

#修改openvswitch_agent.ini

echo '#
[ovs]
integration_bridge = br-int
tunnel_bridge = br-tun
local_ip = 172.16.1.163
#bridge_mappings = br-ex
#bridge_mappings = provider:br-ex
bridge_mappings = 
[agent]
tunnel_types = vxlan,gre
l2_population = true
arp_responder = true
[securitygroup]
enable_security_group = true
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
#'>/etc/neutron/plugins/ml2/openvswitch_agent.ini

#启动openvswitch服务

systemctl enable openvswitch
systemctl restart openvswitch

#创建网桥，将ip转移到网桥，添加到开机启动

echo '#
ovs-vsctl add-br br-ex
ovs-vsctl add-port br-ex eth1
ovs-vsctl show
ifconfig eth1 0.0.0.0 
ifconfig br-ex 172.16.2.164/24
#route add default gw 172.16.2.164 #可选，添加默认路由
#'>>/etc/rc.local
chmod +x /etc/rc.d/rc.local ;tail -n 8 /etc/rc.local |bash

#验证

[root@compute163 ~]# chmod +x /etc/rc.d/rc.local ;tail -n 8 /etc/rc.local |bash
41c15882-e730-4486-be0a-096f778954a2
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth1"
            Interface "eth1"
    ovs_version: "2.12.0"
[root@compute163 ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether a2:ac:97:0d:c4:af brd ff:ff:ff:ff:ff:ff
    inet 172.16.1.163/24 brd 172.16.1.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::a0ac:97ff:fe0d:c4af/64 scope link
       valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel master ovs-system state UP group default qlen 1000
    link/ether 62:b6:31:97:e0:e6 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::60b6:31ff:fe97:e0e6/64 scope link
       valid_lft forever preferred_lft forever
4: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 42:1e:bc:4c:8c:c9 brd ff:ff:ff:ff:ff:ff
    inet 172.16.3.163/24 brd 172.16.3.255 scope global noprefixroute eth2
       valid_lft forever preferred_lft forever
    inet6 fe80::401e:bcff:fe4c:8cc9/64 scope link
       valid_lft forever preferred_lft forever
5: ovs-system: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether 9e:35:f2:a3:b8:cc brd ff:ff:ff:ff:ff:ff
6: br-ex: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether 8e:43:3d:49:86:4a brd ff:ff:ff:ff:ff:ff
    inet 172.16.2.163/24 brd 172.16.2.255 scope global br-ex
       valid_lft forever preferred_lft forever
    inet6 fe80::8c43:3dff:fe49:864a/64 scope link
       valid_lft forever preferred_lft forever

#关闭网卡的开机自启，修改以下项
#vim /etc/sysconfig/network-scripts/ifcfg-eth1

ONBOOT=no

#重启服务

systemctl restart neutron-dhcp-agent\
   neutron-metadata-agent\
   neutron-l3-agent.service\
   neutron-l3-agent\
   neutron-openvswitch-agent

#控制节点验证

[root@controller160 ~]# openstack network agent list
+--------------------------------------+--------------------+------------+-------------------+-------+-------+---------------------------+
| ID                                   | Agent Type         | Host       | Availability Zone | Alive | State | Binary                    |
+--------------------------------------+--------------------+------------+-------------------+-------+-------+---------------------------+
| 12f5026f-2a62-49a2-8d87-ee0ca71688ea | Metadata agent     | compute163 | None              | :-)   | UP    | neutron-metadata-agent    |
| 7014261e-6719-447e-882a-19f45033a6c9 | Metadata agent     | compute164 | None              | :-)   | UP    | neutron-metadata-agent    |
| 767a9db2-7b3a-4edc-833a-39f4c3812b70 | DHCP agent         | compute163 | nova              | :-)   | UP    | neutron-dhcp-agent        |
| 8df5c9ee-8086-4351-8c66-866b2d2577fa | L3 agent           | compute164 | nova              | :-)   | UP    | neutron-l3-agent          |
| aa234650-cf75-4fe4-af48-0ea14495b1a6 | DHCP agent         | compute164 | nova              | :-)   | UP    | neutron-dhcp-agent        |
| b7cf0b1d-4ff0-4314-9427-8cfda3419e15 | Linux bridge agent | compute164 | None              | XXX   | UP    | neutron-linuxbridge-agent |
| c510acb9-7123-4ae3-850c-5e5184b6542a | Linux bridge agent | compute163 | None              | XXX   | UP    | neutron-linuxbridge-agent |
| c6660e76-c613-4980-90ba-23c5585b79d5 | Open vSwitch agent | compute163 | None              | :-)   | UP    | neutron-openvswitch-agent |
| dadea42c-e242-494a-a876-098cd71bcfc4 | L3 agent           | compute163 | nova              | :-)   | UP    | neutron-l3-agent          |
| e611f91f-f162-47be-aebc-d6eaf824e0d0 | Open vSwitch agent | compute164 | None              | :-)   | UP    | neutron-openvswitch-agent |
+--------------------------------------+--------------------+------------+-------------------+-------+-------+---------------------------+

X.过程中遇到的问题

eg1.[root@compute163 ~]# sysctl -p
sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-iptables: No such file or directory
sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-ip6tables: No such file or directory
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0
原因是：模块没有载入
解决方案：modprobe br_netfilter #载入模块即可解决

eg2. Interface "eth1"
                error: "could not add network device eth1 to ofproto (Device or resource busy)"
[root@compute164 ~]# ip add
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel master brq53b98327-3a state UP group default qlen 1000
    link/ether 1a:21:74:6f:df:7f brd ff:ff:ff:ff:ff:ff
    inet6 fe80::1821:74ff:fe6f:df7f/64 scope link
       valid_lft forever preferred_lft forever
8: brq53b98327-3a: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 1a:21:74:6f:df:7f brd ff:ff:ff:ff:ff:ff
    inet 172.16.2.164/24 brd 172.16.2.255 scope global brq53b98327-3a
       valid_lft forever preferred_lft forever
    inet6 fe80::dc45:46ff:fe42:553f/64 scope link
       valid_lft forever preferred_lft forever
12: br-ex: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
    link/ether ba:69:2a:ec:86:42 brd ff:ff:ff:ff:ff:ff
    inet 172.16.2.164/24 brd 172.16.2.255 scope global br-ex
       valid_lft forever preferred_lft forever
    inet6 fe80::b869:2aff:feec:8642/64 scope link
       valid_lft forever preferred_lft forever
[root@compute164 ~]# nmcli connection
NAME            UUID                                  TYPE      DEVICE
System eth0     5fb06bd0-0bb0-7ffb-45f1-d6edd65f3e03  ethernet  eth0
System eth2     3a73717e-65ab-93e8-b518-24f5af32dc0d  ethernet  eth2
brq53b98327-3a  69b74146-17da-4da5-ad79-a5894b97cfb7  bridge    brq53b98327-3a
eth1            dc0c8e24-9f74-4e35-a0b8-518ac89154c2  ethernet  eth1
ens3            421b5593-cd40-421f-8cfd-3b23942345f6  ethernet  --
System eth1     9c92fad9-6ecb-3e6c-eb4d-8a47c6f50c04  ethernet  --
[root@compute164 ~]# nmcli c del brq53b98327-3a
Connection 'brq53b98327-3a' (7c70103d-586e-4db6-8d4c-e9a2e21de51f) successfully deleted.

原因是：之前配置linuxbridge的时候，有关联eth1，导致部署openvswitch的时候，没办法切换eth1的关联，所以报这个错
解决方案：
nmcli c down brq53b98327-3a
nmcli c down eth1
nmcli c del brq53b98327-3a
nmcli c 
然后再重新执行一遍
chmod +x /etc/rc.d/rc.local ;tail -n 8 /etc/rc.local |bash