一、环境准备及安装
服务器环境准备
安装环境依赖
yum -y install epel-release yum-utils device-mapper-persistent-data lvm2 wget #yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
准备所需 yum 源，在 master 上配置，然后执行下面命令复制到其他4个节点即可。
[root@localhost yum.repos.d]# for i in {103..106};do scp CentOS-Base-7.repo kubernetes.repo docker.repo [email protected].$i:/etc/yum.repos.d/;done
CentOS yum源
cat > /etc/yum.repos.d/CentOS-Base-7.repo << EOF [base] name=CentOS-$releasever - Base - mirrors.aliyun.com failovermethod=priority baseurl=http://mirrors.aliyun.com/centos/$releasever/os/$basearch/ gpgcheck=1 gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7 #released updates [updates] name=CentOS-$releasever - Updates - mirrors.aliyun.com failovermethod=priority baseurl=http://mirrors.aliyun.com/centos/$releasever/updates/$basearch/ gpgcheck=1 gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7 #additional packages that may be useful [extras] name=CentOS-$releasever - Extras - mirrors.aliyun.com baseurl=http://mirrors.aliyun.com/centos/$releasever/extras/$basearch/ gpgcheck=1 gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7 #additional packages that extend functionality of existing packages [centosplus] name=CentOS-$releasever - Plus - mirrors.aliyun.com failovermethod=priority baseurl=http://mirrors.aliyun.com/centos/$releasever/centosplus/$basearch/ gpgcheck=1 enabled=0 gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7 #contrib - packages by Centos Users [contrib] name=CentOS-$releasever - Contrib - mirrors.aliyun.com failovermethod=priority baseurl=http://mirrors.aliyun.com/centos/$releasever/contrib/$basearch/ gpgcheck=1 enabled=0 gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7 EOF
k8s aliyun源
cat > /etc/yum.repos.d/kubernetes.repo <<EOF [kubernetes] name=Kubernetes baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64 enabled=1 gpgcheck=0 repo_gpgcheck=0 gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg EOF k8s google源 cat > /etc/yum.repos.d/kubernetes.repo <<EOF [kubernetes] name=Kubernetes baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 enabled=1 gpgcheck=0 repo_gpgcheck=0 gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg EOF
docker yum源
cat > /etc/yum.repos.d/docker-ce.repo << EOF [docker-ce-stable] name=Docker CE Stable - $basearch baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/$basearch/stable enabled=1 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-stable-debuginfo] name=Docker CE Stable - Debuginfo $basearch baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/debug-$basearch/stable enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-stable-source] name=Docker CE Stable - Sources baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/source/stable enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-edge] name=Docker CE Edge - $basearch baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/$basearch/edge enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-edge-debuginfo] name=Docker CE Edge - Debuginfo $basearch baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/debug-$basearch/edge enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-edge-source] name=Docker CE Edge - Sources baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/source/edge enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-test] name=Docker CE Test - $basearch baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/$basearch/test enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-test-debuginfo] name=Docker CE Test - Debuginfo $basearch baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/debug-$basearch/test enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-test-source] name=Docker CE Test - Sources baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/source/test enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-nightly] name=Docker CE Nightly - $basearch baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/$basearch/nightly enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-nightly-debuginfo] name=Docker CE Nightly - Debuginfo $basearch baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/debug-$basearch/nightly enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg [docker-ce-nightly-source] name=Docker CE Nightly - Sources baseurl=https://mirrors.aliyun.com/docker-ce/linux/centos/7/source/nightly enabled=0 gpgcheck=1 gpgkey=https://mirrors.aliyun.com/docker-ce/linux/centos/gpg EOF
gcloud源
tee -a /etc/yum.repos.d/google-cloud-sdk.repo << EOM [google-cloud-sdk] name=Google Cloud SDK baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el7-x86_64 enabled=1 gpgcheck=1 repo_gpgcheck=1 gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg EOM
创建 yum 元数据
yum clean all yum makecache fast
yum 安装 etcd、kubernetes
yum -y install etcd kubernetes google-cloud-sdk
最新版：
curl -fsSL https://get.docker.com/ | sh
注意：docker必须跟着kubernetes版本走
创建 kubernetes、etcd 密钥目录
mkdir -p /wdata/kubernetes/ssl
在 master 上建立免密登录私钥（执行 ssh-keygen ，一路回车）
[root@localhost ~]# ssh-keygen
[root@localhost ~]# for i in {103..106};do ssh-copy-id -i /root/.ssh/id_rsa.pub [email protected].$i;done
二、启动
在master节点上执行
systemctl start etcd systemctl start docker systemctl start kube-apiserver systemctl start kube-controller-manager systemctl start kube-scheduler systemctl start kubelet systemctl start kube-proxy
在所有slave节点上执行
systemctl start etcd systemctl start docker systemctl start kubelet systemctl start kube-proxy
三、开机启动
在master节点上执行
systemctl enable etcd systemctl enable docker systemctl enable kube-apiserver systemctl enable kube-controller-manager systemctl enable kube-scheduler systemctl enable kubelet systemctl enable kube-proxy
在所有slave节点上执行
systemctl enable etcd systemctl enable docker systemctl enable kubelet systemctl enable kube-proxy
四、查询状态
在master节点上执行
systemctl status etcd systemctl status docker systemctl status kube-apiserver systemctl status kube-controller-manager systemctl status kube-scheduler systemctl status kubelet systemctl status kube-proxy
在所有slave节点上执行
systemctl status etcd systemctl status docker systemctl status kubelet systemctl status kube-proxy
五、在所有节点编辑 /lib/systemd/system/docker.service ，在ExecStart=..上面加入
[root@localhost ~]# vim /lib/systemd/system/docker.service ExecStartPost=/sbin/iptables -I FORWARD -s 0.0.0.0/0 -j ACCEPT
重启docker
[root@localhost ~]# systemctl daemon-reload && systemctl restart docker && systemctl status docker
或者在master节点修改完之后分发到其他节点
for i in {103..106};do scp /lib/systemd/system/docker.service [email protected].$i:/lib/systemd/system/;done
六、所有节点设置k8s启动参数
cat > /etc/sysctl.d/k8s.conf <<EOF net.ipv4.ip_forward = 1 net.bridge.bridge-nf-call-ip6tables = 1 net.bridge.bridge-nf-call-iptables = 1 EOF [root@localhost ~]# sysctl -p /etc/sysctl.d/k8s.conf
七、在 master 需要安装CFSSL工具，这将会用来建立 TLS certificates。
[root@localhost ~]# export CFSSL_URL="https://pkg.cfssl.org/R1.2"
[root@localhost ~]# wget "${CFSSL_URL}/cfssl_linux-amd64" -O /usr/local/bin/cfssl
[root@localhost ~]# wget "${CFSSL_URL}/cfssljson_linux-amd64" -O /usr/local/bin/cfssljson
[root@localhost ~]# wget "${CFSSL_URL}/cfssl-certinfo_linux-amd64" -O /usr/local/bin/cfssl-certinfo
[root@localhost ~]# chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo
[root@localhost ~]# export PATH=/usr/local/bin:$PATH
八、关闭selinux
sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config
九、修改系统 hosts 文件，加入下面 5 行
[root@localhost ~]# vim /etc/hosts 192.168.10.102 master 192.168.10.103 node1 192.168.10.104 node2 192.168.10.105 node3 192.168.10.106 node4
或者
cat >> /etc/hosts <<EOF 192.168.10.102 master 192.168.10.103 node1 192.168.10.104 node2 192.168.10.105 node3 192.168.10.106 node4 EOF
十、创建集群 CA 与 Certificates，在主节点master上进行操作
配置CA，进入/wdata/kubernetes/ssl目录，然后进入该目录下进行操作
[root@master ~]# cd /wdata/kubernetes/ssl
[root@master ssl]# cfssl print-defaults config > config.json
[root@master ssl]# cfssl print-defaults csr > csr.json# 根据config.json文件的格式创建如下的ca-config.json文件
# 过期时间设置成了 87600hcat > ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "87600h"
}
}
}
}
EOF
创建CA证书签名请求，即创建ca-csr.json，格式如下
cat >ca-csr.json<<EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
],
"ca": {
"expiry": "87600h"
}
}
EOF
生成CA证书和私钥
cfssl gencert -initca ca-csr.json | cfssljson -bare ca ls ca* ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem
创建Kubernetes证书签名请求文件 kubernetes-csr.json，注意记得替换相应ip
配置CA，进入 /wdata/kubernetes/ssl 目录进行操作
[root@master ssl]# cd /wdata/kubernetes/ssl [root@master ssl]# cfssl print-defaults config > config.json [root@master ssl]# cfssl print-defaults csr > csr.json
# 根据config.json文件的格式创建如下的ca-config.json文件 # 过期时间设置成了 87600h
cat > ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "87600h"
}
}
}
}
EOF
创建CA证书签名请求，即创建ca-csr.json，格式如下
cat >ca-csr.json<<EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
],
"ca": {
"expiry": "87600h"
}
}
EOF
创建 kubernetes-csr.json 文件
cat > kubernetes-csr.json <<EOF
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"192.168.10.102",
"192.168.10.103",
"192.168.10.104",
"192.168.10.105",
"192.168.10.106",
"master",
"node1",
"node2",
"node3",
"node4"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
生成CA证书和私钥
cfssl gencert -initca ca-csr.json | cfssljson -bare ca ls ca* ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem
生成kubernetes证书和私钥
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes ls kubernetes* kubernetes.csr kubernetes-csr.json kubernetes-key.pem kubernetes.pem
创建admin签名文件 admin-csr-json
cat >admin-csr.json <<EOF
{
"CN": "admin",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "system:masters",
"OU": "System"
}
]
}
EOF
生成admin证书和密钥
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin ls admin* admin.csr admin-csr.json admin-key.pem admin.pem
创建kube-proxy 证书签名请求文件 kube-proxy-csr.json
cat > kube-proxy-csr.json<<EOF
{
"CN": "system:kube-proxy",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
生成 kube-proxy 客户端证书和密钥
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy ls kube-proxy* kube-proxy.csr kube-proxy-csr.json kube-proxy-key.pem kube-proxy.pem
分发证书，将生成的证书和密钥文件（后缀名为.pem）拷贝到所有机器的/etc/kubernetes/ssl目录下备用
mkdir -p /etc/kubernetes/ssl
cp /wdata/kubernetes/ssl/*.pem /etc/kubernetes/ssl
cd /etc/kubernetes/ssl
for i in {103..106};do scp *.pem 192.168.10.$i:/etc/kubernetes/ssl/;done
创建 kubectl kuberconfig 文件
export KUBE_APISERVER="https://192.168.10.102:6443"
# 设置集群参数
kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/ssl/ca.pem --embed-certs=false --server=${KUBE_APISERVER}
# 设置客户端认证参数
kubectl config set-credentials admin --client-certificate=/etc/kubernetes/ssl/admin.pem --embed-certs=false --client-key=/etc/kubernetes/ssl/admin-key.pem
# 设置上下文参数
kubectl config set-context kubernetes --cluster=kubernetes --user=admin
# 设置默认上下文
kubectl config use-context kubernetes
创建 TLS Bootstrapping Token
export BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')
cat > token.csv << EOF
${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF
cp token.csv /etc/kubernetes/
创建 kubelet bootstrapping kubeconfig 文件
cd /etc/kubernetes export KUBE_APISERVER="https://192.168.10.102:6443"
# 设置集群参数
kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/ssl/ca.pem --embed-certs=true --server=${KUBE_APISERVER} --kubeconfig=bootstrap.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap --token=${BOOTSTRAP_TOKEN} --kubeconfig=bootstrap.kubeconfig
# 设置上下文参数
kubectl config set-context default --cluster=kubernetes --user=kubelet-bootstrap --kubeconfig=bootstrap.kubeconfig
# 设置默认上下文
kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
创建 kube-proxy kubeconfig 文件
export KUBE_APISERVER="https://192.168.10.102:6443"
# 设置集群参数
kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/ssl/ca.pem --embed-certs=true --server=${KUBE_APISERVER} --kubeconfig=kube-proxy.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials kube-proxy --client-certificate=/etc/kubernetes/ssl/kube-proxy.pem --client-key=/etc/kubernetes/ssl/kube-proxy-key.pem --embed-certs=true --kubeconfig=kube-proxy.kubeconfig
# 设置上下文参数
kubectl config set-context default --cluster=kubernetes --user=kube-proxy --kubeconfig=kube-proxy.kubeconfig
# 设置默认上下文
kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
分发 kubeconfig 文件，将两个 kubeconfig 文件分发到所有 node 节点服务器的 /etc/kubernetes/ 目录
for i in {103..106};do scp bootstrap.kubeconfig kube-proxy.kubeconfig [email protected].$i:/etc/kubernetes/;done
到这里，创建证书以及密钥就告一段落了，相信有很多人都有所迷惑，因为刚才创建了好多密钥和证书，下面总结下证书的作用：
生成的 CA 证书和秘钥文件如下：
ca-key.pem
ca.pem
kubernetes-key.pem
kubernetes.pem
kube-proxy.pem
kube-proxy-key.pem
admin.pem
admin-key.pem
使用证书的组件如下：
etcd：使用 ca.pem、kubernetes-key.pem、kubernetes.pem；
kube-apiserver：使用 ca.pem、kubernetes-key.pem、kubernetes.pem；
kubelet：使用 ca.pem；
kube-proxy：使用 ca.pem、kube-proxy-key.pem、kube-proxy.pem；
kubectl：使用 ca.pem、admin-key.pem、admin.pem；
kube-controller-manager：使用 ca-key.pem、ca.pem
相信看完上面的总结就一目了然了，OK下面我们来进行etcd集群的安装。
十一、所有节点部署 etcd
Kuberntes 使用 etcd 来存储所有数据，下面我们来创建三节点etcd集群，也就是master、node1、node2前面我们已经创建了很多TLS证书，咱们这里就复用下kubernetes的证书，以下操作在所有节点执行。
由于上面已经 yum 安装了 etcd ，因此，以下1、2两部可忽略。
1、下载 etcd 源码文件
wget https://github.com/coreos/etcd/releases/download/v3.1.5/etcd-v3.1.5-linux-amd64.tar.gz tar -xvf etcd-v3.1.5-linux-amd64.tar.gz mv etcd-v3.1.5-linux-amd64/etcd* /usr/local/bin
2、更新 etcd.server 文件
[root@master ~]# vim /lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
EnvironmentFile=-/etc/etcd/etcd.conf
User=etcd
# set GOMAXPROCS to number of processors
#ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/bin/etcd --name=\"${ETCD_NAME}\" --data-dir=\"${ETCD_DATA_DIR}\" --listen-client-urls=\"${ETCD_LISTEN_CLIENT_URLS}\""
ExecStart=/usr/bin/etcd \
--name=${ETCD_NAME} \
--data-dir=${ETCD_DATA_DIR} \
--listen-client-urls=${ETCD_LISTEN_CLIENT_URLS} \
--listen-peer-urls=${ETCD_LISTEN_PEER_URLS} \
--advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster-state=${ETCD_INITIAL_CLUSTER_STATE} \
--initial-cluster=${ETCD_INITIAL_CLUSTER}
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
参数说明（下同）：
name 节点名称
data-dir 指定节点的数据存储目录
listen-peer-urls 监听URL，用于与其他节点通讯
listen-client-urls 对外提供服务的地址：比如 http://ip:2379,http://127.0.0.1:2379 ，客户端会连接到这里和 etcd 交互
initial-advertise-peer-urls 该节点同伴监听地址，这个值会告诉集群中其他节点
initial-cluster-state 集群中所有节点的信息，格式为 node1=http://ip1:2380,node2=http://ip2:2380,… 。注意：这里的 node1 是节点的 --name 指定的名字；后面的 ip1:2380 是 --initial-advertise-peer-urls 指定的值
initial-cluster-state 新建集群的时候，这个值为 new ；假如已经存在的集群，这个值为 existing
initial-cluster-token 创建集群的 token，这个值每个集群保持唯一。这样的话，如果你要重新创建集群，即使配置和之前一样，也会再次生成新的集群和节点 uuid；否则会导致多个集群之间的冲突，造成未知的错误
advertise-client-urls 对外公告的该节点客户端监听地址，这个值会告诉集群中其他节点
注意：etcd 的数据目录为 /var/lib/etcd，需在启动服务前创建这个目录，否则启动服务的时候会报错“Failed at step CHDIR spawning /usr/bin/etcd: No such file or directory”；
3、环境变量配置文件/etc/etcd/etcd.conf，每个节点单独配置，共5个节点
#[Member] ETCD_DATA_DIR="/var/lib/etcd/default.etcd" ETCD_LISTEN_PEER_URLS="https://192.168.10.102:2380" ETCD_LISTEN_CLIENT_URLS="https://192.168.10.102:2379" ETCD_NAME="master" #[Clustering] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.10.102:2380" ETCD_ADVERTISE_CLIENT_URLS="https://192.168.10.102:2379" ETCD_INITIAL_CLUSTER="master=https://192.168.10.102:2380,node1=https://192.168.10.103:2380,node2=https://192.168.10.104:2380,node3=https://192.168.10.105:2380,node4=https://192.168.10.106:2380" ETCD_INITIAL_CLUSTER_TOKEN="kubernetes" ETCD_INITIAL_CLUSTER_STATE="new" ########################################################################################################################### #[Member] ETCD_DATA_DIR="/var/lib/etcd/default.etcd" ETCD_LISTEN_PEER_URLS="https://192.168.10.103:2380" ETCD_LISTEN_CLIENT_URLS="https://192.168.10.103:2379" ETCD_NAME="node1" #[Clustering] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.10.103:2380" ETCD_ADVERTISE_CLIENT_URLS="https://192.168.10.103:2379" ETCD_INITIAL_CLUSTER="master=https://192.168.10.102:2380,node1=https://192.168.10.103:2380,node2=https://192.168.10.104:2380,node3=https://192.168.10.105:2380,node4=https://192.168.10.106:2380" ETCD_INITIAL_CLUSTER_TOKEN="kubernetes" ETCD_INITIAL_CLUSTER_STATE="new" ########################################################################################################################### #[Member] ETCD_DATA_DIR="/var/lib/etcd/default.etcd" ETCD_LISTEN_PEER_URLS="https://192.168.10.104:2380" ETCD_LISTEN_CLIENT_URLS="https://192.168.10.104:2379" ETCD_NAME="node2" #[Clustering] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.10.104:2380" ETCD_ADVERTISE_CLIENT_URLS="https://192.168.10.104:2379" ETCD_INITIAL_CLUSTER="master=https://192.168.10.102:2380,node1=https://192.168.10.103:2380,node2=https://192.168.10.104:2380,node3=https://192.168.10.105:2380,node4=https://192.168.10.106:2380" ETCD_INITIAL_CLUSTER_TOKEN="kubernetes" ETCD_INITIAL_CLUSTER_STATE="new" ########################################################################################################################### #[Member] ETCD_DATA_DIR="/var/lib/etcd/default.etcd" ETCD_LISTEN_PEER_URLS="https://192.168.10.105:2380" ETCD_LISTEN_CLIENT_URLS="https://192.168.10.105:2379" ETCD_NAME="node3" #[Clustering] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.10.105:2380" ETCD_ADVERTISE_CLIENT_URLS="https://192.168.10.105:2379" ETCD_INITIAL_CLUSTER="master=https://192.168.10.102:2380,node1=https://192.168.10.103:2380,node2=https://192.168.10.104:2380,node3=https://192.168.10.105:2380,node4=https://192.168.10.106:2380" ETCD_INITIAL_CLUSTER_TOKEN="kubernetes" ETCD_INITIAL_CLUSTER_STATE="new" ########################################################################################################################### #[Member] ETCD_DATA_DIR="/var/lib/etcd/default.etcd" ETCD_LISTEN_PEER_URLS="https://192.168.10.106:2380" ETCD_LISTEN_CLIENT_URLS="https://192.168.10.106:2379" ETCD_NAME="node4" #[Clustering] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.10.106:2380" ETCD_ADVERTISE_CLIENT_URLS="https://192.168.10.106:2379" ETCD_INITIAL_CLUSTER="master=https://192.168.10.102:2380,node1=https://192.168.10.103:2380,node2=https://192.168.10.104:2380,node3=https://192.168.10.105:2380,node4=https://192.168.10.106:2380" ETCD_INITIAL_CLUSTER_TOKEN="kubernetes" ETCD_INITIAL_CLUSTER_STATE="new"
4、重启 etcd 服务
systemctl daemon-reload systemctl restart etcd systemctl status etcd
# 在所有的 kubernetes节点重复上面的步骤，直到所有机器的 etcd 服务都已启动。
5、验证服务
[root@master ~]# etcdctl member list 33e828919ccd6e00: name=etcd3 peerURLs=http://192.168.10.104:2380 clientURLs=http://127.0.0.1:2379,http://192.168.10.104:2379 isLeader=false 63f491cf242db4b3: name=etcd1 peerURLs=http://192.168.10.102:2380 clientURLs=http://127.0.0.1:2379,http://192.168.10.102:2379 isLeader=true d8703b918bb9d544: name=etcd2 peerURLs=http://192.168.10.103:2380 clientURLs=http://127.0.0.1:2379,http://192.168.10.103:2379 isLeader=false
[root@master ~]# etcdctl cluster-health member 33e828919ccd6e00 is healthy: got healthy result from http://127.0.0.1:2379 member 63f491cf242db4b3 is healthy: got healthy result from http://127.0.0.1:2379 member d8703b918bb9d544 is healthy: got healthy result from http://127.0.0.1:2379 cluster is healthy
6、问题总结
①、如果验证输出以下内容
[root@master ~]# etcdctl member list 8e9e05c52164694d: name=etcd1 peerURLs=http://localhost:2380 clientURLs=http://127.0.0.1:2379,http://192.168.10.102:2379 isLeader=true
此问题是在搭建集群前，先进行了单实例的etcd部署测试， 即 etcd2、etcd3 节点分别进行了单实例部署测试，也就是历史数据的影响。删除历史数据即可。
rm -rf /var/lib/etcd/*
②、如果有其他问题，则需要将 etcd.service 中的 User=etcd 修改为 User=root ，且 ExecStart 中不能有空格
③、一个集群中只有一个Leader，如果不是一个就是报错。
以下内容为搭建带认证的 etcd 集群，可忽略
7、创建 etcd 密钥文件
cat > ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "87600h"
}
}
}
}
EOF
生成 etcd-ca-csr.json 文件
cat >etcd-ca-csr.json<<EOF
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "etcd",
"OU": "System"
}
],
"ca": {
"expiry": "87600h"
}
}
EOF
生成 etcd 的证书签名请求文件 etcd-csr.json
cat > etcd-csr.json << EOF
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"192.168.10.102",
"192.168.10.103",
"192.168.10.104",
"etcd1",
"etcd1",
"etcd2",
"etcd3"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "BeiJing",
"L": "BeiJing",
"O": "etcd",
"OU": "System"
}
]
}
EOF
生成 etcd 客户端证书和密钥
cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare etcd-ca cfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
8、修改 etcd 服务文件 etcd.service ,在所有节点加入以下标红的内容
[root@master ~]# vim /lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
EnvironmentFile=-/etc/etcd/etcd.conf
User=etcd
# set GOMAXPROCS to number of processors
# ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /usr/bin/etcd --name=\"${ETCD_NAME}\" --data-dir=\"${ETCD_DATA_DIR}\" --listen-client-urls=\"${ETCD_LISTEN_CLIENT_URLS}\""
ExecStart=/usr/bin/etcd \
--name=${ETCD_NAME} \
--data-dir=${ETCD_DATA_DIR} \
--listen-client-urls=${ETCD_LISTEN_CLIENT_URLS} \
--listen-peer-urls=${ETCD_LISTEN_PEER_URLS} \
--advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster-state=${ETCD_INITIAL_CLUSTER_STATE} \
--initial-cluster=${ETCD_INITIAL_CLUSTER} \
--cert-file=${ETCD_CERT_FILE} \
--key-file=${ETCD_KEY_FILE} \
--peer-cert-file=${ETCD_CERT_FILE} \
--peer-key-file=${ETCD_KEY_FILE} \
--trusted-ca-file=${ETCD_TRUSTED_CA_FILE} \
--peer-trusted-ca-file=${ETCD_PEER_TRUSTED_CA_FILE}
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
9、修改 etcd 配置文件 etcd.conf ,加入以下内容
#[Security] ETCD_CERT_FILE="/etc/kubernetes/ssl/kubernetes.pem" ETCD_KEY_FILE="/etc/kubernetes/ssl/kubernetes-key.pem" ETCD_TRUSTED_CA_FILE="/etc/kubernetes/ssl/kubernetes.pem" ETCD_PEER_CERT_FILE="/etc/kubernetes/ssl/kubernetes.pem" ETCD_PEER_KEY_FILE="/etc/kubernetes/ssl/kubernetes-key.pem" ETCD_PEER_TRUSTED_CA_FILE="/etc/kubernetes/ssl/kubernetes.pem"
10、验证集群
[root@master ~]# etcdctl --ca-file=/etc/kubernetes/ssl/ca.pem --cert-file=/etc/kubernetes/ssl/kubernetes.pem --key-file=/etc/kubernetes/ssl/kubernetes-key.pem --endpoints=https://192.168.10.102:2379 member list 5590395074cc9636: name=node2 peerURLs=https://192.168.10.104:2380 clientURLs=https://0.0.0.0:4001,https://127.0.0.1:2379,https://192.168.10.104:2379 isLeader=false 5c724b8641f4fa1c: name=master peerURLs=https://192.168.10.102:2380 clientURLs=https://0.0.0.0:4001,https://127.0.0.1:2379,https://192.168.10.102:2379 isLeader=false 748537d982cf5c99: name=node3 peerURLs=https://192.168.10.105:2380 clientURLs=https://0.0.0.0:4001,https://127.0.0.1:2379,https://192.168.10.105:2379 isLeader=false c05394aa51d11fda: name=node4 peerURLs=https://192.168.10.106:2380 clientURLs=https://0.0.0.0:4001,https://127.0.0.1:2379,https://192.168.10.106:2379 isLeader=false d3993eda148039ba: name=node1 peerURLs=https://192.168.10.103:2380 clientURLs=https://0.0.0.0:4001,https://127.0.0.1:2379,https://192.168.10.103:2379 isLeader=true
[root@master ~]# etcdctl --ca-file=/etc/kubernetes/ssl/ca.pem --cert-file=/etc/kubernetes/ssl/kubernetes.pem --key-file=/etc/kubernetes/ssl/kubernetes-key.pem --endpoints=https://192.168.10.102:2379 cluster-health member 5590395074cc9636 is healthy: got healthy result from https://192.168.10.104:2379 member 5c724b8641f4fa1c is healthy: got healthy result from https://192.168.10.102:2379 member 748537d982cf5c99 is healthy: got healthy result from https://192.168.10.105:2379 member c05394aa51d11fda is healthy: got healthy result from https://192.168.10.106:2379 member d3993eda148039ba is healthy: got healthy result from https://192.168.10.103:2379 cluster is healthy
十二、Kubernetes 集群部署
经过上面的步骤，前期准备工作完成，下面开始正式部署 Kubernetes 。以下操作均在 master 节点进行
1、修改 kube-apiserver.service 服务文件
[root@master ~]# vim /usr/lib/systemd/system/kube-apiserver.service [Unit] Description=Kubernetes API Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes After=network.target After=etcd.service [Service] EnvironmentFile=-/etc/kubernetes/config EnvironmentFile=-/etc/kubernetes/apiserver User=kube ExecStart=/usr/bin/kube-apiserver \ $KUBE_LOGTOSTDERR \ $KUBE_LOG_LEVEL \ $KUBE_ETCD_SERVERS \ $KUBE_API_ADDRESS \ $KUBE_API_PORT \ $KUBELET_PORT \ $KUBE_ALLOW_PRIV \ $KUBE_SERVICE_ADDRESSES \ $KUBE_ADMISSION_CONTROL \ $KUBE_API_ARGS Restart=on-failure Type=notify LimitNOFILE=65536 [Install] WantedBy=multi-user.target
2、修改kube-apiserver配置文件
[root@master ~]# vim /etc/kubernetes/config KUBE_LOGTOSTDERR="--logtostderr=true" KUBE_LOG_LEVEL="--v=0" KUBE_ALLOW_PRIV="--allow-privileged=false" KUBE_MASTER="--master=http://127.0.0.1:8080" ############################# 将以上默认配置文件修改为以下内容 ############################# KUBE_LOGTOSTDERR="--logtostderr=false" KUBE_LOG_LEVEL="--v=0" KUBE_ALLOW_PRIV="--allow-privileged=false" KUBE_MASTER="--master=https://192.168.10.102:6443"
说明：该配置文件同时被kube-apiserver、kube-controller-manager、kube-scheduler、kubelet、kube-proxy使用。
3、修改 apiserver 文件
[root@master ~]# vim /etc/kubernetes/apiserver ### # kubernetes system config # # The following values are used to configure the kube-apiserver # # The address on the local server to listen to. KUBE_API_ADDRESS="--insecure-bind-address=192.168.10.102" # The port on the local server to listen on. # KUBE_API_PORT="--port=8080" # Port minions listen on # KUBELET_PORT="--kubelet-port=10250" # Comma separated list of nodes in the etcd cluster KUBE_ETCD_SERVERS="--etcd-servers=https://192.168.10.102:2379,https://192.168.10.103:2379,https://192.168.10.104:2379,https://192.168.10.105:2379,https://192.168.10.106:2379" # Address range to use for services KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16" # default admission control policies KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota" # Add your own! KUBE_API_ARGS="--authorization-mode=AlwaysAllow --advertise-address=192.168.10.102 --bind-address=192.168.10.102 --secure-port=6443 --insecure-port=8080 --kubelet-https=true --service-node-port-range=1-65535 --storage-backend=etcd3 --etcd-servers=https://192.168.10.102:2379,https://192.168.10.103:2379,https://192.168.10.104:2379,https://192.168.10.105:2379,https://192.168.10.106:2379 --token-auth-file=/etc/kubernetes/token.csv --tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem --tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem --client-ca-file=/etc/kubernetes/ssl/ca.pem --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem --etcd-cafile=/etc/kubernetes/ssl/ca.pem --etcd-certfile=/etc/kubernetes/ssl/kubernetes.pem --etcd-keyfile=/etc/kubernetes/ssl/kubernetes-key.pem --enable-swagger-ui=true --allow-privileged=true --apiserver-count=1 --audit-log-maxage=30 --audit-log-maxbackup=3 --audit-log-maxsize=100 --audit-log-path=/var/log/kubernetes/apiserver/audit.log --log-dir=/var/log/kubernetes --v=0 --event-ttl=1h"
参数说明：
# --admission-control：kuberneres集群的准入控制机制，各控制模块以插件的形式依次生效，集群时必须包含ServiceAccount；
AlwaysAdmit, AlwaysDeny, AlwaysPullImages, DefaultStorageClass, DefaultTolerationSeconds, DenyEscalatingExec, DenyExecOnPrivileged, EventRateLimit, ExtendedResourceToleration, ImagePolicyWebhook, Initializers, LimitPodHardAntiAffinityTopology, LimitRanger, MutatingAdmissionWebhook, NamespaceAutoProvision, NamespaceExists, NamespaceLifecycle, NodeRestriction, OwnerReferencesPermissionEnforcement, PersistentVolumeClaimResize, PersistentVolumeLabel, PodNodeSelector, PodPreset, PodSecurityPolicy, PodTolerationRestriction, Priority, ResourceQuota, SecurityContextDeny, ServiceAccount, StorageObjectInUseProtection, TaintNodesByCondition, ValidatingAdmissionWebhook # --bind-address：不能为 127.0.0.1；在本地址的6443端口开启https服务，默认值0.0.0.0；
# --insecure-port=0：禁用不安全的http服务，默认开启，端口8080，设置为0禁用；
# --secure-port=6443：https安全端口，默认即6443，0表示禁用；
# --authorization-mode：在安全端口使用 RBAC 授权模式，未通过授权的请求拒绝；
# --service-cluster-ip-range：指定 Service Cluster IP 地址段，该地址段外部路由不可达；
# --service-node-port-range：指定 NodePort 的端口范围；
# --storage-backend：持久化存储类型，v1.6版本后默认即etcd3；
# --enable-swagger-ui：设置为true时，启用swagger-ui网页，可通过apiserver的usl/swagger-ui访问，默认为false；
# --allow-privileged：设置为true时，kubernetes允许在Pod中运行拥有系统特权的容器应用；
# --audit-log-*：审计日志相关；
# --event-ttl：apiserver中各时间保留时间，默认即1h，通常用于审计与追踪；
# --logtostderr：默认为true，输出到stderr，不输出到日志；
# --log-dir：日志目录；
# --v：日志级别
创建日志目录及相关日志文件并授权
[root@master ~]# mkdir -p /var/log/kubernetes/apiserver/ [root@master ~]# touch /var/log/kubernetes/apiserver/audit.log [root@master ~]# chown -R kube.kube /var/log/kubernetes
重启 apiserver 服务
systemctl daemon-reload systemctl enable kube-apiserver systemctl restart kube-apiserver systemctl status kube-apiserver
4、修改 kube-control-manager.service 服务文件
[root@master ~]# vim /usr/lib/systemd/system/kube-controller-manager.service [Unit] Description=Kubernetes Controller Manager Documentation=https://github.com/GoogleCloudPlatform/kubernetes [Service] EnvironmentFile=-/etc/kubernetes/config EnvironmentFile=-/etc/kubernetes/controller-manager User=kube ExecStart=/usr/bin/kube-controller-manager \ $KUBE_LOGTOSTDERR \ $KUBE_LOG_LEVEL \ $KUBE_MASTER \ $KUBE_CONTROLLER_MANAGER_ARGS Restart=on-failure LimitNOFILE=65536 [Install] WantedBy=multi-user.target
5、修改 controller-manager 配置文件
[root@master ~]# vim /etc/kubernetes/controller-manager ### # The following values are used to configure the kubernetes controller-manager # defaults from config and apiserver should be adequate # Add your own! KUBE_CONTROLLER_MANAGER_ARGS="--address=127.0.0.1 --service-cluster-ip-range=10.254.0.0/16 --cluster-name=kubernetes --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem --service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem --root-ca-file=/etc/kubernetes/ssl/ca.pem --leader-elect=true"
重启 kube-control-manager 服务
systemctl daemon-reload systemctl enable kube-controller-manager systemctl restart kube-controller-manager systemctl status kube-controller-manager
6、修改 kube-scheduler.service 服务文件
[root@master ~]# vim /usr/lib/systemd/system/kube-scheduler.service [Unit] Description=Kubernetes Scheduler Plugin Documentation=https://github.com/GoogleCloudPlatform/kubernetes [Service] EnvironmentFile=-/etc/kubernetes/config EnvironmentFile=-/etc/kubernetes/scheduler User=kube ExecStart=/usr/bin/kube-scheduler \ $KUBE_LOGTOSTDERR \ $KUBE_LOG_LEVEL \ $KUBE_MASTER \ $KUBE_SCHEDULER_ARGS Restart=on-failure LimitNOFILE=65536 [Install] WantedBy=multi-user.target
7、修改 scheduler 文件
[root@master ~]# vim /etc/kubernetes/scheduler ### # kubernetes scheduler config # default config should be adequate # Add your own! KUBE_SCHEDULER_ARGS="--leader-elect=true --address=127.0.0.1"
重启 scheduler 服务
systemctl daemon-reload systemctl enable kube-scheduler systemctl restart kube-scheduler systemctl status kube-scheduler
8、修改修改 kubelet.service 服务文件
[root@master ~]# vim /usr/lib/systemd/system/kubelet.service [Unit] Description=Kubernetes Kubelet Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes After=docker.service Requires=docker.service [Service] WorkingDirectory=/var/lib/kubelet EnvironmentFile=-/etc/kubernetes/config EnvironmentFile=-/etc/kubernetes/kubelet ExecStart=/usr/bin/kubelet \ $KUBE_LOGTOSTDERR \ $KUBE_LOG_LEVEL \ $KUBELET_API_SERVER \ $KUBELET_ADDRESS \ $KUBELET_PORT \ $KUBELET_HOSTNAME \ $KUBE_ALLOW_PRIV \ $KUBELET_POD_INFRA_CONTAINER \ $KUBELET_ARGS Restart=on-failure [Install] WantedBy=multi-user.target
9、修改 kubelet 配置文件
[root@master ~]# vim /etc/kubernetes/kubelet ### # kubernetes kubelet (minion) config # The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces) KUBELET_ADDRESS="--address=192.168.10.102" # The port for the info server to serve on KUBELET_PORT="--port=10250" # You may leave this blank to use the actual hostname KUBELET_HOSTNAME="--hostname-override=192.168.10.102" # location of the api-server KUBELET_API_SERVER="--api-servers=https://192.168.10.102:6443" # pod infrastructure container KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest" # Add your own! ### # kubernetes kubelet (minion) config # The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces) KUBELET_ADDRESS="--address=192.168.10.102" # The port for the info server to serve on # KUBELET_PORT="--port=10250" # You may leave this blank to use the actual hostname KUBELET_HOSTNAME="--hostname-override=192.168.10.102" # location of the api-server KUBELET_API_SERVER="--api-servers=https://192.168.10.102:6443" # pod infrastructure container KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest" # Add your own! KUBELET_ARGS="--pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0 --kubeconfig=/etc/kubernetes/kubelet --cert-dir=/etc/kubernetes/ssl --cluster-dns=10.254.0.1 --cluster-domain=cluster.k8s. --allow-privileged=true --serialize-image-pulls=false --logtostderr=false --log-dir=/var/log/kubernetes/kubelet --v=0" # KUBELET_ARGS="--pod-infra-container-image=gcr.io/google_containers/pause-amd64:3.0 --kubeconfig=/etc/kubernetes/kubelet --cluster-dns=10.254.0.1 --cluster-domain=cluster.k8s. --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem --service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem --root-ca-file=/etc/kubernetes/ssl/ca.pem --allow-privileged=true --serialize-image-pulls=false --logtostderr=false --log-dir=/var/log/kubernetes/kubelet --v=0"
重启 kubelet 服务
systemctl daemon-reload systemctl enable kubelet systemctl restart kubelet systemctl status kubelet
10、验证 Master 节点功能
[root@master kubernetes]# kubectl get cs
NAME STATUS MESSAGE ERROR
controller-manager Healthy ok
scheduler Healthy ok
etcd-0 Healthy {"health":"true"}
etcd-2 Healthy {"health":"true"}
etcd-4 Healthy {"health":"true"}
etcd-3 Healthy {"health":"true"}
etcd-1 Healthy {"health":"true"}
如果出现以下错误：
[root@master kubernetes]# kubectl get cs
NAME STATUS MESSAGE ERROR
scheduler Unhealthy Get http://127.0.0.1:10251/healthz: dial tcp 127.0.0.1:10251: getsockopt: connection refused
controller-manager Unhealthy Get http://127.0.0.1:10252/healthz: dial tcp 127.0.0.1:10252: getsockopt: connection refused
etcd-2 Healthy {"health":"true"}
etcd-1 Healthy {"health":"true"}
etcd-3 Healthy {"health":"true"}
etcd-0 Healthy {"health":"true"}
etcd-4 Healthy {"health":"true"}
则需要将scheduler文件和controller-manager文件中的address修改为127.0.0.1，因为当前kube-apiserver期望scheduler和controller-manager在同一台服务器。
此处也可使用 kubectl get cs ，如果报错 Unable to
connect to the server: x509: certificate signed by unknown authority ，则使用上面的方式进行验证，或者做如下操作
从〜/.kube/config中删除嵌入的根证书并运行此config命令：
kubectl config set-cluster ${KUBE_CONTEXT} --insecure-skip-tls-verify=true --server=${KUBE_CONTEXT}
说明：${KUBE_CONTEXT} 此参数可自定义，此处为 kubernetes
十三、所有节点部署 Flannel
所有的node节点都需要安装Flannel网络插件，才能让所有的Pod加入到同一个局域网中，所以下面的操作在所有节点都需要执行一遍。建议直接使用yum安装flanneld，除非对版本有特殊需求，默认安装的是0.7.1版本的flannel。
1、yum 安装 flannel 插件
[root@master ~]# yum -y install flannel
2、修改 flanneld.service 服务启动文件，加入下面标红的两
[root@master ~]# vim /usr/lib/systemd/system/flanneld.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service
[Service]
Type=notify
EnvironmentFile=/etc/sysconfig/flanneld
EnvironmentFile=-/etc/sysconfig/docker-network
ExecStart=/usr/bin/flanneld-start -etcd-endpoints=${FLANNEL_ETCD_ENDPOINTS} -etcd-prefix=${FLANNEL_ETCD_PREFIX} $FLANNEL_OPTIONS
ExecStartPost=/usr/libexec/flannel/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker
Restart=on-failure
[Install]
WantedBy=multi-user.target
WantedBy=docker.service
3、修改 /etc/sysconfig/flanneld 配置文件
[root@master ~]# vim /etc/sysconfig/flanneld # Flanneld configuration options # etcd url location. Point this to the server where etcd runs FLANNEL_ETCD_ENDPOINTS="https://192.168.10.102:2379,https://192.168.10.103:2379,https://192.168.10.104:2379,https://192.168.10.105:2379,https://192.168.10.106:2379" # etcd config key. This is the configuration key that flannel queries # For address range assignment FLANNEL_ETCD_PREFIX="/atomic.io/network" # FLANNEL_ETCD_PREFIX="/kube-centos/network" # Any additional options that you want to pass FLANNEL_OPTIONS="-etcd-cafile=/etc/kubernetes/ssl/ca.pem -etcd-certfile=/etc/kubernetes/ssl/kubernetes.pem -etcd-keyfile=/etc/kubernetes/ssl/kubernetes-key.pem"
注意：如果是多网卡（例如vagran环境），则需要在 FLANNEL_OPENTION 中添加指定的外网出口的网卡，例如iface=eth0
4、在 etcd 中创建网络配置（这个配置只在 master 节点操作即可）
[root@master ~]# etcdctl --endpoints https://192.168.10.102:2379,https://192.168.10.103:2379,https://192.168.10.104:2379,https://192.168.10.105:2379,https://192.168.10.106:2379 \
--ca-file=/etc/kubernetes/ssl/ca.pem \
--cert-file=/etc/kubernetes/ssl/kubernetes.pem \
--key-file=/etc/kubernetes/ssl/kubernetes-key.pem \
mkdir /kube-centos/network
[root@master ~]# etcdctl --endpoints https://192.168.10.102:2379,https://192.168.10.103:2379,https://192.168.10.104:2379,https://192.168.10.105:2379,https://192.168.10.106:2379 \
--ca-file=/etc/kubernetes/ssl/ca.pem \
--cert-file=/etc/kubernetes/ssl/kubernetes.pem \
--key-file=/etc/kubernetes/ssl/kubernetes-key.pem \
mk /atomic.io/network/config '{"Network":"10.254.0.0/16"}'
说明：如果此处想使用 vxlan ，在可以直接将 host-gw 修改为 vxlan 即可。
5、启动 flannel
systemctl daemon-reload systemctl enable flanneld systemctl start flanneld systemctl status flanneld
6、查看flannel状态
[root@master ~]# systemctl status flanneld
如果有如上图提示，证明配置成功
十四、部署 node 节点
到此为止我们已经完成了Master节点服务、etcd集群、flannel集群都已经搭建完成，下面我们来部署 node 节点的服务。首先需要确认下node节点的flannel、docker、etcd是否启动，其次检查下/etc/kubernetes/下的证书和配置文件是否在，具体操作这里就不再赘述了。
1、修改 docker 配置，使其可以使用 flannel 网络
使用systemctl命令启动flanneld后，会自动执行./mk-docker-opts.sh -i在/run/flannel/目录下生成如下两个文件环境变量文件：
[root@master ~]# ll /run/flannel/ 总用量 8 -rw-r--r-- 1 root root 174 7月 22 20:44 docker -rw-r--r-- 1 root root 98 7月 22 20:44 subnet.env
Docker将会读取这两个环境变量文件作为容器启动参数，修改 docker 的配置文件 /usr/lib/systemd/system/docker.service，增加一条环境变量配置：
EnvironmentFile=-/run/flannel/docker
3、安装 conntrack
[root@master ~]# yum -y install conntrack-tools
4、修改 kube-proxy 的 service 配置文件 /usr/lib/systemd/system/kube-proxy.service
[root@master ~]# vim /usr/lib/systemd/system/kube-proxy.service [Unit] Description=Kubernetes Kube-Proxy Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes After=network.target [Service] EnvironmentFile=-/etc/kubernetes/config EnvironmentFile=-/etc/kubernetes/proxy ExecStart=/usr/bin/kube-proxy \ $KUBE_LOGTOSTDERR \ $KUBE_LOG_LEVEL \ $KUBE_MASTER \ $KUBE_PROXY_ARGS Restart=on-failure LimitNOFILE=65536 [Install] WantedBy=multi-user.target
5、修改 kube-proxy 配置文件 /etc/kubernetes/proxy ，在每个节点都进行修改，这里只记录一条，其余的修改IP即可
[root@master ~]# vim /etc/kubernetes/proxy KUBE_PROXY_ARGS="--bind-address=192.168.10.102 --hostname-override=192.168.10.102 --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig --cluster-cidr=10.254.0.0/16"
重启 kube-proxy 服务
systemctl daemon-reload systemctl enable kube-proxy systemctl restart kube-proxy systemctl status kube-proxy
重启 docker 服务
systemctl daemon-reload systemctl enable docker systemctl restart docker systemctl status docker
6、集群测试
[root@master ~]# kubectl --insecure-skip-tls-verify=true run nginx --replicas=2 --labels="run=load-balancer-example" --image=nginx --port=80 deployment "nginx" created [root@master ~]# kubectl expose deployment nginx --type=NodePort --name=example-service service "example-service" exposed [root@master kubernetes]# kubectl describe svc example-service Name:example-service Namespace:default Labels:run=load-balancer-example Selector:run=load-balancer-example Type:NodePort IP:10.254.253.193 Port:<unset>80/TCP NodePort:<unset>57593/TCP Endpoints:<none> Session Affinity:None No events.
