一、StatefulSet操作
–
1、初体验
可以发现等待前一个启动好了，在启动下一个，因此容器的健康检查失败也会影响启动
域名：web-2.nginx.default.svc.cluster.local,一般写到web-2.nginx.default就可以了
[root@master01 ~]# cat ssweb.yaml
apiVersion: v1
kind: Service
metadata:
name: nginx
labels:
app: nginx
spec:
ports:
- port: 80
name: web
clusterIP: None
selector:
app: nginx
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: web
spec:
serviceName: "nginx"
replicas: 3
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:latest
ports:
- containerPort: 80
[root@master01 ~]# kubectl apply -f ssweb.yaml
[root@master01 ~]# kubectl get pod -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
dns-test 1/1 Running 0 2m2s 10.0.4.103 master01 <none> <none>
web-0 1/1 Running 0 9m45s 10.0.4.157 master01 <none> <none>
web-1 1/1 Running 0 9m42s 10.0.0.9 master03 <none> <none>
web-2 1/1 Running 0 9m39s 10.0.2.74 node01 <none> <none>
[root@master01 ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 2d2h
nginx ClusterIP None <none> 80/TCP 41s
[root@master01 ~]# kubectl get pod -w
测试：
[root@master01 ~]# kubectl run -i --tty --image busybox:1.28 dns-test --restart Never --rm
主机名、DNS不会改变
可以发现IP地址改变，其他都没有改变
[root@master01 ~]# kubectl delete pod -l app=nginx
[root@master01 ~]# kubectl get pod -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
dns-test 1/1 Running 0 5m3s 10.0.4.103 master01 <none> <none>
web-0 1/1 Running 0 47s 10.0.4.152 master01 <none> <none>
web-1 1/1 Running 0 44s 10.0.0.211 master03 <none> <none>
web-2 1/1 Running 0 41s 10.0.2.48 node01 <none> <none>
2、扩容和缩容
（1）扩容
跟着之前顺序扩容
[root@master01 ~]# kubectl scale statefulset web --replicas 8
[root@master01 ~]# kubectl get pod -w
（2）缩容
scale和path都可以，可以发现先删除web-7，然后在删除web-6
[root@master01 ~]# kubectl scale statefulset web --replicas 3
[root@master01 ~]# kubectl patch sts web -p '{"spec":{"replicas":3}}'
[root@master01 ~]# kubectl get pod -w
（3）OnDelete更新
默认是RollingUpdate
[root@master01 ~]# cat ssweb.yaml
apiVersion: v1
kind: Service
metadata:
name: nginx
labels:
app: nginx
spec:
ports:
- port: 80
name: web
clusterIP: None
selector:
app: nginx
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: web
spec:
updateStrategy:
type: OnDelete
serviceName: "nginx"
replicas: 3
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:latest
ports:
- containerPort: 80
[root@master01 ~]# kubectl apply -f ssweb.yaml
[root@master01 ~]# kubectl describe sts web
测试：
发现镜像并不会更新
[root@master01 ~]# kubectl patch statefulsets.apps web --type='json' -p '[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"nginx:1.19"}]'
[root@master01 ~]# kubectl get sts -owide
NAME READY AGE CONTAINERS IMAGES
web 3/3 9m55s nginx nginx:1.19
[root@master01 ~]# kubectl describe pod web-0|grep Image:
Image: nginx:latest
[root@master01 ~]# kubectl delete pod web-1
[root@master01 ~]# kubectl get pod -w
[root@master01 ~]# for i in web-0 web-1 web-2;do echo -n $i:;kubectl describe pod $i|grep Image:;done
web-0: Image: nginx:latest
web-1: Image: nginx:1.19
web-2: Image: nginx:latest
（4）RollingUpdate更新
[root@master01 ~]# cat ssweb.yaml
apiVersion: v1
kind: Service
metadata:
name: nginx
labels:
app: nginx
spec:
ports:
- port: 80
name: web
clusterIP: None
selector:
app: nginx
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: web
spec:
updateStrategy:
type: OnDelete
serviceName: "nginx"
replicas: 3
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:latest
ports:
- containerPort: 80
[root@master01 ~]# kubectl apply -f ssweb.yaml
[root@master01 ~]# kubectl describe sts web|grep Update
Update Strategy: RollingUpdate
[root@master01 ~]# kubectl set image statefulset/web nginx=nginx:1.19
[root@master01 ~]# for i in web-0 web-1 web-2;do echo -n $i:;kubectl describe pod $i|grep Image:;done
web-0: Image: nginx:1.19
web-1: Image: nginx:1.19
web-2: Image: nginx:1.19
3、灰度发布
（1）partition
设置partition为3，保护3个不更新
[root@master01 ~]# cat ssweb.yaml
apiVersion: v1
kind: Service
metadata:
name: nginx
labels:
app: nginx
spec:
ports:
- port: 80
name: web
clusterIP: None
selector:
app: nginx
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: web
spec:
updateStrategy:
type: OnDelete
serviceName: "nginx"
replicas: 3
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:latest
ports:
- containerPort: 80
[root@master01 ~]# kubectl apply -f ssweb.yaml
[root@master01 ~]# kubectl patch statefulsets.apps web -p '{"spec":{"updateStrategy":{"type":"RollingUpdate","rollingUpdate":{"partition":3}}}}'
[root@master01 ~]# kubectl set image statefulset/web nginx=nginx:1.21
[root@master01 ~]# for i in web-0 web-1 web-2;do echo -n $i:;kubectl describe pod $i|grep Image:;done
web-0: Image: nginx:1.19
web-1: Image: nginx:1.19
web-2: Image: nginx:1.19
[root@master01 ~]# cat ssweb.yaml
apiVersion: v1
kind: Service
metadata:
name: nginx
labels:
app: nginx
spec:
ports:
- port: 80
name: web
clusterIP: None
selector:
app: nginx
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: web
spec:
serviceName: "nginx"
replicas: 10
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:latest
ports:
- containerPort: 80
[root@master01 ~]# kubectl apply -f ssweb.yaml
[root@master01 ~]# kubectl set image statefulset/web nginx=nginx:1.19
[root@master01 ~]# for i in web-{0..9};do echo -n $i:;kubectl describe pod $i|grep Image:;done
web-0: Image: nginx:latest
web-1: Image: nginx:latest
web-2: Image: nginx:latest
web-3: Image: nginx:1.19
web-4: Image: nginx:1.19
web-5: Image: nginx:1.19
web-6: Image: nginx:1.19
web-7: Image: nginx:1.19
web-8: Image: nginx:1.19
web-9: Image: nginx:1.19
[root@master01 ~]# kubectl patch statefulsets.apps web -p '{"spec":{"updateStrategy":{"type":"RollingUpdate","rollingUpdate":{"partition":2}}}}'
[root@master01 ~]# for i in web-{0..9};do echo -n $i:;kubectl describe pod $i|grep Image:;done
web-0: Image: nginx:latest
web-1: Image: nginx:latest
web-2: Image: nginx:1.19
web-3: Image: nginx:1.19
web-4: Image: nginx:1.19
web-5: Image: nginx:1.19
web-6: Image: nginx:1.19
web-7: Image: nginx:1.19
web-8: Image: nginx:1.19
web-9: Image: nginx:1.19
（2）回滚
[root@master01 ~]# kubectl rollout history statefulset web
statefulset.apps/web
REVISION CHANGE-CAUSE
1 <none>
2 <none>
[root@master01 ~]# kubectl rollout history statefulset web --revision 2
statefulset.apps/web with revision #2
Pod Template:
Labels: app=nginx
Containers:
nginx:
Image: nginx:1.19
Port: 80/TCP
Host Port: 0/TCP
Environment: <none>
Mounts: <none>
Volumes: <none>
[root@master01 ~]# kubectl rollout undo statefulset web --to-revision 1
[root@master01 ~]# for i in web-{0..9};do echo -n $i:;kubectl describe pod $i|grep Image:;done
web-0: Image: nginx:latest
web-1: Image: nginx:latest
web-2: Image: nginx:latest
web-3: Image: nginx:latest
web-4: Image: nginx:latest
web-5: Image: nginx:latest
web-6: Image: nginx:latest
web-7: Image: nginx:latest
web-8: Image: nginx:latest
web-9: Image: nginx:latest
（3）通过yaml文件设置
[root@master01 ~]# cat ssweb.yaml
apiVersion: v1
kind: Service
metadata:
name: nginx
labels:
app: nginx
spec:
ports:
- port: 80
name: web
clusterIP: None
selector:
app: nginx
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: web
spec:
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 10%
partition: 3
serviceName: "nginx"
replicas: 10
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:latest
ports:
- containerPort: 80
[root@master01 ~]# kubectl apply -f ssweb.yaml
[root@master01 ~]# kubectl describe sts web
（4）通过yaml文件设置（并行）
[root@master01 ~]# cat ssweb.yaml
apiVersion: v1
kind: Service
metadata:
name: nginx
labels:
app: nginx
spec:
ports:
- port: 80
name: web
clusterIP: None
selector:
app: nginx
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: web
spec:
podManagementPolicy: Parallel
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 10%
partition: 3
serviceName: "nginx"
replicas: 10
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:latest
ports:
- containerPort: 80
[root@master01 ~]# kubectl apply -f ssweb.yaml
[root@master01 ~]# kubectl get pod -w
级联删除（方式一）
[root@master01 ~]# kubectl delete sts web
statefulset.apps "web" deleted
[root@master01 ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 2d4h
nginx ClusterIP None <none> 80/TCP 3m50s
[root@master01 ~]# kubectl delete svc nginx
级联删除（方式二）
[root@master01 ~]# kubectl delete sts web --cascade=orphan
[root@master01 ~]# kubectl delete pod -l app=nginx
强制删除
不推荐使用
[root@master01 ~]# kubectl apply -f ssweb.yaml
[root@master01 ~]# kubectl set image statefulset/web nginx=nginx:haha
[root@master01 ~]# kubectl delete pods web-9 --grace-period=0 --force
正常删除
Terminating或者Unknown得时候，推荐
[root@master01 ~]# kubectl patch pod web-9 -p '{"metadata":{"finalizers":null}}'
二、Service操作
1、初体验
[root@master01 ~]# cat nginx.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: frontend
spec:
replicas: 3
selector:
matchLabels:
app: frontend
template:
metadata:
labels:
app: frontend
spec:
containers:
- name: nginx
image: nginx:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
[root@master01 ~]# cat nginx-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: nginx-svc
spec:
ports:
- protocol: TCP
port: 8080
targetPort: 80
selector:
app: frontend
[root@master01 ~]# kubectl apply -f nginx.yaml
[root@master01 ~]# kubectl apply -f nginx-svc.yaml
[root@master01 ~]# kubectl get pod -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
frontend-64dccd9ff7-bdmk8 1/1 Running 0 6m48s 10.0.0.230 master03 <none> <none>
frontend-64dccd9ff7-f9xqq 1/1 Running 0 6m48s 10.0.2.157 node01 <none> <none>
frontend-64dccd9ff7-shrn9 1/1 Running 0 6m48s 10.0.4.32 master01 <none> <none>
[root@master01 ~]# kubectl get svc -owide
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 2d5h <none>
nginx-svc ClusterIP 10.103.140.145 <none> 8080/TCP 37s app=frontend
[root@master01 ~]# kubectl describe svc nginx-svc
Name: nginx-svc
Namespace: default
Labels: <none>
Annotations: <none>
Selector: app=frontend
Type: ClusterIP
IP Family Policy: SingleStack
IP Families: IPv4
IP: 10.103.140.145
IPs: 10.103.140.145
Port: <unset> 8080/TCP
TargetPort: 80/TCP
Endpoints: 10.0.0.230:80,10.0.2.157:80,10.0.4.32:80
Session Affinity: None
Events: <none>
2、扩容测试
[root@master01 ~]# kubectl scale deployment frontend --replicas 6
[root@master01 ~]# kubectl describe svc nginx-svc
Name: nginx-svc
Namespace: default
Labels: <none>
Annotations: <none>
Selector: app=frontend
Type: ClusterIP
IP Family Policy: SingleStack
IP Families: IPv4
IP: 10.103.140.145
IPs: 10.103.140.145
Port: <unset> 8080/TCP
TargetPort: 80/TCP
Endpoints: 10.0.0.230:80,10.0.1.163:80,10.0.2.157:80 + 3 more...
Session Affinity: None
Events: <none>
[root@master01 ~]# kubectl get ep
NAME ENDPOINTS AGE
kubernetes 10.10.10.21:6443,10.10.10.22:6443,10.10.10.23:6443 2d5h
nginx-svc 10.0.0.230:80,10.0.1.163:80,10.0.2.157:80 + 3 more... 5m59s
[root@master01 ~]# kubectl describe ep nginx-svc
Name: nginx-svc
Namespace: default
Labels: <none>
Annotations: endpoints.kubernetes.io/last-change-trigger-time: 2023-06-01T09:30:53Z
Subsets:
Addresses: 10.0.0.230,10.0.1.163,10.0.2.157,10.0.3.9,10.0.4.153,10.0.4.32
NotReadyAddresses: <none>
Ports:
Name Port Protocol
---- ---- --------
<unset> 80 TCP
Events: <none>
3、测试集群IP
[root@master01 ~]# kubectl get service
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 2d5h
nginx-svc ClusterIP 10.103.140.145 <none> 8080/TCP 7m44s
[root@master01 ~]# ping 10.103.140.145
[root@master01 ~]# kubectl get pod -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
frontend-64dccd9ff7-69675 1/1 Running 0 8m45s 10.0.3.9 node02 <none> <none>
frontend-64dccd9ff7-bdmk8 1/1 Running 0 17m 10.0.0.230 master03 <none> <none>
frontend-64dccd9ff7-f9xqq 1/1 Running 0 17m 10.0.2.157 node01 <none> <none>
frontend-64dccd9ff7-h5bnk 1/1 Running 0 8m45s 10.0.1.163 master02 <none> <none>
frontend-64dccd9ff7-jvx6z 1/1 Running 0 8m45s 10.0.4.153 master01 <none> <none>
frontend-64dccd9ff7-shrn9 1/1 Running 0 17m 10.0.4.32 master01 <none> <none>
[root@master01 ~]# ipvsadm -Ln|grep -A 6 10.103.140.145
TCP 10.103.140.145:8080 rr
-> 10.0.0.230:80 Masq 1 0 0
-> 10.0.1.163:80 Masq 1 0 0
-> 10.0.2.157:80 Masq 1 0 0
-> 10.0.3.9:80 Masq 1 0 0
-> 10.0.4.32:80 Masq 1 0 0
-> 10.0.4.153:80 Masq 1 0 0
4、测试ns-nginx
[root@master01 ~]# kubectl create ns test
[root@master01 ~]# cat ns-test.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: ns-nginx
namespace: test
spec:
replicas: 3
selector:
matchLabels:
app: nstest
template:
metadata:
labels:
app: nstest
spec:
containers:
- name: nginx
image: nginx:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
name: ns-svc
namespace: test
spec:
ports:
- protocol: TCP
port: 8080
targetPort: 80
selector:
app: nstest
[root@master01 ~]# kubectl apply -f ns-test.yaml
[root@master01 ~]# kubectl get pod -n test
NAME READY STATUS RESTARTS AGE
ns-nginx-867c4f9bcb-g7db5 1/1 Running 0 17s
ns-nginx-867c4f9bcb-vc86d 1/1 Running 0 17s
ns-nginx-867c4f9bcb-x2vf5 1/1 Running 0 17s
[root@master01 ~]# kubectl get svc -n test
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
ns-svc ClusterIP 10.104.254.251 <none> 8080/TCP 24s
[root@master01 ~]# kubectl run box -it --rm --image busybox:1.28 /bin/sh
/ # cat /etc/resolv.conf
search default.svc.cluster.local svc.cluster.local cluster.local
nameserver 10.96.0.10
options ndots:5
/ # nslookup ns-svc.test
Server: 10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local
Name: ns-svc.test
Address 1: 10.104.254.251 ns-svc.test.svc.cluster.local
/ # wget ns-svc.test:8080
Connecting to ns-svc.test:8080 (10.104.254.251:8080)
index.html 100% |****************************************************************************************************************| 615 0:00:00 ETA
/ # cat index.html
<!DOCTYPE html>
...
</html>
5、设置hostname
[root@master01 ~]# cat domainname.yaml
apiVersion: v1
kind: Service
metadata:
name: default-subdomain
spec:
selector:
name: busybox
clusterIP: None
ports:
- name: foo
port: 1234
targetPort: 1234
---
apiVersion: v1
kind: Pod
metadata:
name: busybox1
labels:
name: busybox
spec:
hostname: busybox-1
subdomain: default-subdomain
containers:
- name: busybox
image: busybox:1.28
command:
- sleep
- "100000000"
---
apiVersion: v1
kind: Pod
metadata:
name: busybox2
labels:
name: busybox
spec:
hostname: busybox-2
subdomain: default-subdomain
containers:
- name: busybox
image: busybox:1.28
command:
- sleep
- "100000000"
[root@master01 ~]# kubectl apply -f domainname.yaml
[root@master01 ~]# kubectl get pod -owide|grep busy
busybox1 1/1 Running 0 4s 10.0.0.188 master03 <none> <none>
busybox2 1/1 Running 0 4s 10.0.4.197 master01 <none> <none>
[root@master01 ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
default-subdomain ClusterIP None <none> 1234/TCP 82s
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 2d6h
nginx-svc ClusterIP 10.103.140.145 <none> 8080/TCP 95m
[root@master01 ~]# kubectl describe svc default-subdomain
Name: default-subdomain
Namespace: default
Labels: <none>
Annotations: <none>
Selector: name=busybox
Type: ClusterIP
IP Family Policy: SingleStack
IP Families: IPv4
IP: None
IPs: None
Port: foo 1234/TCP
TargetPort: 1234/TCP
Endpoints: 10.0.4.159:1234,10.0.4.46:1234
Session Affinity: None
Events: <none>
测试访问：
[root@master01 ~]# kubectl exec -it busybox1 -- sh
/ # nslookup default-subdomain
Server: 10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local
Name: default-subdomain
Address 1: 10.0.4.197 busybox-2.default-subdomain.default.svc.cluster.local
Address 2: 10.0.0.188 busybox-1.default-subdomain.default.svc.cluster.local
/ # nslookup busybox-1
Server: 10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local
Name: busybox-1
Address 1: 10.0.0.188 busybox-1.default-subdomain.default.svc.cluster.local
/ # nslookup busybox-2
Server: 10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local
nslookup: can't resolve 'busybox-2'
/ # nslookup busybox-2.default-subdomain
Server: 10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local
Name: busybox-2.default-subdomain
Address 1: 10.0.4.197 busybox-2.default-subdomain.default.svc.cluster.local
6、setHostnameAsFQDN
FQDN：最多64得字符，否则会导致pending
[root@master01 ~]# cat domainname.yaml
apiVersion: v1
kind: Service
metadata:
name: default-subdomain
spec:
selector:
name: busybox
clusterIP: None
ports:
- name: foo
port: 1234
targetPort: 1234
---
apiVersion: v1
kind: Pod
metadata:
name: busybox1
labels:
name: busybox
spec:
setHostnameAsFQDN: true
hostname: busybox-1
subdomain: default-subdomain
containers:
- name: busybox
image: busybox:1.28
command:
- sleep
- "100000000"
---
apiVersion: v1
kind: Pod
metadata:
name: busybox2
labels:
name: busybox
spec:
hostname: busybox-2
subdomain: default-subdomain
containers:
- name: busybox
image: busybox:1.28
command:
- sleep
- "100000000"
[root@master01 ~]# kubectl apply -f domainname.yaml
service/default-subdomain created
[root@master01 ~]# kubectl exec -it busybox1 -- sh
/ # hostname
busybox-1.default-subdomain.default.svc.cluster.local
[root@master01 ~]# kubectl exec -it busybox2 -- sh
/ # hostname
busybox-2
/ # hostname -f
busybox-2.default-subdomain.default.svc.cluster.local
7、DNS策略
（1）默认ClusterFirst
[root@master01 ~]# kubectl edit pod busybox1
（2）ClusterFirstWithHostNet
[root@master01 ~]# cat domainname.yaml
apiVersion: v1
kind: Service
metadata:
name: default-subdomain
spec:
selector:
name: busybox
clusterIP: None
ports:
- name: foo
port: 1234
targetPort: 1234
---
apiVersion: v1
kind: Pod
metadata:
name: busybox1
labels:
name: busybox
spec:
hostname: busybox-1
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: busybox
image: busybox:1.28
command:
- sleep
- "100000000"
---
apiVersion: v1
kind: Pod
metadata:
name: busybox2
labels:
name: busybox
spec:
hostname: busybox-2
subdomain: default-subdomain
containers:
- name: busybox
image: busybox:1.28
command:
- sleep
- "100000000"
[root@master01 ~]# kubectl apply -f domainname.yaml
[root@master01 ~]# kubectl get pod|grep busy
busybox1 1/1 Running 0 26s
busybox2 1/1 Running 0 26s
[root@master01 ~]# kubectl exec -it busybox1 -- sh #部署到了master03节点上了
/ # hostname
master03
/ # exit
[root@master01 ~]# kubectl exec -it busybox2 -- sh
/ # hostname
busybox-2
/ # exit
（3）None
自己设置dns，searches：搜索域名
[root@master01 ~]# cat domainname.yaml
apiVersion: v1
kind: Service
metadata:
name: default-subdomain
spec:
selector:
name: busybox
clusterIP: None
ports:
- name: foo
port: 1234
targetPort: 1234
---
apiVersion: v1
kind: Pod
metadata:
name: busybox1
labels:
name: busybox
spec:
dnsPolicy: None
dnsConfig:
nameservers:
- 114.114.114.114
searches:
- ns1.my.dns.search
containers:
- name: busybox
image: busybox:1.28
command:
- sleep
- "100000000"
---
apiVersion: v1
kind: Pod
metadata:
name: busybox2
labels:
name: busybox
spec:
hostname: busybox-2
subdomain: default-subdomain
containers:
- name: busybox
image: busybox:1.28
command:
- sleep
- "100000000"
[root@master01 ~]# kubectl delete -f domainname.yaml
[root@master01 ~]# kubectl apply -f domainname.yaml
测试：
[root@master01 ~]# kubectl exec -it busybox1 -- sh
/ # ping www.baidu.com
PING www.baidu.com (14.119.104.254): 56 data bytes
64 bytes from 14.119.104.254: seq=0 ttl=50 time=42.844 ms
64 bytes from 14.119.104.254: seq=1 ttl=50 time=42.574 ms
^C
--- www.baidu.com ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max = 42.574/42.709/42.844 ms
/ # nslookup www.baidu.com
Server: 114.114.114.114
Address 1: 114.114.114.114 public1.114dns.com
Name: www.baidu.com
Address 1: 14.119.104.189
Address 2: 14.119.104.254
/ # cat /etc/resolv.conf
search ns1.my.dns.search
nameserver 114.114.114.114
/ # hostname -f
busybox1
8、sessionAffinity亲和性
ClientIP和None模式可以做亲和
[root@master01 ~]# cat nginx-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: nginx-svc
spec:
sessionAffinity: ClientIP
ports:
- protocol: TCP
port: 8080
targetPort: 80
selector:
app: frontend
[root@master01 ~]# cat nginx.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: frontend
spec:
replicas: 3
selector:
matchLabels:
app: frontend
template:
metadata:
labels:
app: frontend
spec:
containers:
- name: nginx
image: nginx:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
[root@master01 ~]# kubectl apply -f nginx.yaml
[root@master01 ~]# kubectl apply -f nginx-svc.yaml
[root@master01 ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 2d23h
nginx-svc ClusterIP 10.103.140.145 <none> 8080/TCP 18h
测试：
可以发现每次都访问到一台机器上
[root@master01 ~]# curl 10.103.140.145:8080
[root@master01 ~]# kubectl logs frontend-64dccd9ff7-bn4wq
9、自定义http-web-svc
[root@master01 ~]# cat nginx.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: frontend
spec:
replicas: 3
selector:
matchLabels:
app: frontend
template:
metadata:
labels:
app: frontend
spec:
containers:
- name: nginx
image: nginx:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
name: http-web-svc
[root@master01 ~]# cat nginx-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: nginx-svc
spec:
sessionAffinity: ClientIP
ports:
- protocol: TCP
port: 8080
targetPort: http-web-svc
selector:
app: frontend
[root@master01 ~]# kubectl delete -f nginx.yaml
[root@master01 ~]# kubectl delete -f nginx-svc.yaml
[root@master01 ~]# kubectl apply -f nginx.yaml
[root@master01 ~]# kubectl apply -f nginx-svc.yaml
测试：
[root@master01 ~]# kubectl get service
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 2d23h
nginx-svc ClusterIP 10.99.182.168 <none> 8080/TCP 3s
[root@master01 ~]# kubectl describe svc nginx-svc
Name: nginx-svc
Namespace: default
Labels: <none>
Annotations: <none>
Selector: app=frontend
Type: ClusterIP
IP Family Policy: SingleStack
IP Families: IPv4
IP: 10.99.182.168
IPs: 10.99.182.168
Port: <unset> 8080/TCP
TargetPort: http-web-svc/TCP
Endpoints: 10.0.0.87:80,10.0.1.247:80,10.0.2.1:80
Session Affinity: ClientIP
Events: <none>
10、无标签访问外部数据
（1）创建一个httpd
[root@node01 ~]# yum install -y httpd
[root@node01 ~]# systemctl start httpd
[root@node01 ~]# echo "<h1>node01</h1>" > /var/www/html/index.html
（2）创建Endpoints
[root@master01 ~]# cat ex-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: my-service
spec:
ports:
- protocol: TCP
port: 80
targetPort: 80
---
apiVersion: v1
kind: Endpoints
metadata:
name: my-service
subsets:
- addresses:
- ip: 10.10.10.24
ports:
- port: 80
[root@master01 ~]# kubectl apply -f ex-svc.yaml
（3）测试
[root@master01 ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 162m
my-service ClusterIP 10.109.96.132 <none> 80/TCP 41s
[root@master01 ~]# kubectl describe svc my-service
Name: my-service
Namespace: default
Labels: <none>
Annotations: <none>
Selector: <none>
Type: ClusterIP
IP Family Policy: SingleStack
IP Families: IPv4
IP: 10.109.96.132
IPs: 10.109.96.132
Port: <unset> 80/TCP
TargetPort: 80/TCP
Endpoints: 10.10.10.24:80
Session Affinity: None
Events: <none>
[root@master01 ~]# curl 10.109.96.132
<h1>node01</h1>
11、NodePort
云厂商得话，我们可以使用LoadBalancer
可以在apiserver中设置端口范围
[root@master01 ~]# cat nginx.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: frontend
spec:
replicas: 3
selector:
matchLabels:
app: frontend
template:
metadata:
labels:
app: frontend
spec:
containers:
- name: nginx
image: nginx:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
[root@master01 ~]# cat nginx-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: nginx-svc
spec:
type: NodePort
ports:
- protocol: TCP
port: 8080
targetPort: 80
selector:
app: frontend
[root@master01 ~]# kubectl apply -f nginx.yaml
[root@master01 ~]# kubectl apply -f nginx-svc.yaml
[root@master01 ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 3h3m
my-service ClusterIP 10.109.96.132 <none> 80/TCP 21m
nginx-svc NodePort 10.110.184.239 <none> 8080:30649/TCP 10m
测试：
所有节点得30649都能访问到
三、Ingress操作
1、Ingress安装
https://github.com/kubernetes/ingress-nginx
https://github.com/kubernetes/ingress-nginx/blob/main/deploy/static/provider/cloud/deploy.yaml
更新得话，只能replace或者edit，不能apply
[root@master01 ~]# cat deploy.yaml
apiVersion: v1
kind: Namespace
metadata:
labels:
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
name: ingress-nginx
---
apiVersion: v1
automountServiceAccountToken: true
kind: ServiceAccount
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx
namespace: ingress-nginx
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx-admission
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx
namespace: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- namespaces
verbs:
- get
- apiGroups:
- ""
resources:
- configmaps
- pods
- secrets
- endpoints
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io
resources:
- ingressclasses
verbs:
- get
- list
- watch
- apiGroups:
- coordination.k8s.io
resourceNames:
- ingress-nginx-leader
resources:
- leases
verbs:
- get
- update
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- create
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- discovery.k8s.io
resources:
- endpointslices
verbs:
- list
- watch
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx-admission
namespace: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- endpoints
- nodes
- pods
- secrets
- namespaces
verbs:
- list
- watch
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- networking.k8s.io
resources:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io
resources:
- ingressclasses
verbs:
- get
- list
- watch
- apiGroups:
- discovery.k8s.io
resources:
- endpointslices
verbs:
- list
- watch
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx-admission
rules:
- apiGroups:
- admissionregistration.k8s.io
resources:
- validatingwebhookconfigurations
verbs:
- get
- update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx
namespace: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ingress-nginx
subjects:
- kind: ServiceAccount
name: ingress-nginx
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx-admission
namespace: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ingress-nginx-admission
subjects:
- kind: ServiceAccount
name: ingress-nginx-admission
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: ingress-nginx
subjects:
- kind: ServiceAccount
name: ingress-nginx
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx-admission
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: ingress-nginx-admission
subjects:
- kind: ServiceAccount
name: ingress-nginx-admission
namespace: ingress-nginx
---
apiVersion: v1
data:
allow-snippet-annotations: "true"
kind: ConfigMap
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx-controller
namespace: ingress-nginx
---
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx-controller
namespace: ingress-nginx
spec:
externalTrafficPolicy: Local
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
ports:
- appProtocol: http
name: http
port: 80
protocol: TCP
targetPort: http
nodePort: 30080
- appProtocol: https
name: https
port: 443
protocol: TCP
targetPort: https
nodePort: 30443
selector:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
#type: LoadBalancer
type: NodePort
---
apiVersion: v1
kind: Service
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx-controller-admission
namespace: ingress-nginx
spec:
ports:
- appProtocol: https
name: https-webhook
port: 443
targetPort: webhook
selector:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
type: ClusterIP
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx-controller
namespace: ingress-nginx
spec:
minReadySeconds: 0
revisionHistoryLimit: 10
selector:
matchLabels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
template:
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
spec:
# hostNetwork: tue
containers:
- args:
- /nginx-ingress-controller
- --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller
- --election-id=ingress-nginx-leader
- --controller-class=k8s.io/ingress-nginx
- --ingress-class=nginx
- --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
- --validating-webhook=:8443
- --validating-webhook-certificate=/usr/local/certificates/cert
- --validating-webhook-key=/usr/local/certificates/key
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: LD_PRELOAD
value: /usr/local/lib/libmimalloc.so
image: bitnami/nginx-ingress-controller:1.8.0
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /wait-shutdown
livenessProbe:
failureThreshold: 5
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
name: controller
ports:
- containerPort: 80
name: http
protocol: TCP
- containerPort: 443
name: https
protocol: TCP
- containerPort: 8443
name: webhook
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
resources:
requests:
cpu: 100m
memory: 90Mi
securityContext:
allowPrivilegeEscalation: true
capabilities:
add:
- NET_BIND_SERVICE
drop:
- ALL
runAsUser: 101
volumeMounts:
- mountPath: /usr/local/certificates/
name: webhook-cert
readOnly: true
dnsPolicy: ClusterFirst
nodeSelector:
kubernetes.io/os: linux
serviceAccountName: ingress-nginx
terminationGracePeriodSeconds: 300
volumes:
- name: webhook-cert
secret:
secretName: ingress-nginx-admission
---
apiVersion: batch/v1
kind: Job
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx-admission-create
namespace: ingress-nginx
spec:
template:
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx-admission-create
spec:
containers:
- args:
- create
- --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc
- --namespace=$(POD_NAMESPACE)
- --secret-name=ingress-nginx-admission
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: registry.cn-hangzhou.aliyuncs.com/google_containers/kube-webhook-certgen:v1.1.1
imagePullPolicy: IfNotPresent
name: create
securityContext:
allowPrivilegeEscalation: false
nodeSelector:
kubernetes.io/os: linux
restartPolicy: OnFailure
securityContext:
fsGroup: 2000
runAsNonRoot: true
runAsUser: 2000
serviceAccountName: ingress-nginx-admission
---
apiVersion: batch/v1
kind: Job
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx-admission-patch
namespace: ingress-nginx
spec:
template:
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx-admission-patch
spec:
containers:
- args:
- patch
- --webhook-name=ingress-nginx-admission
- --namespace=$(POD_NAMESPACE)
- --patch-mutating=false
- --secret-name=ingress-nginx-admission
- --patch-failure-policy=Fail
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
image: registry.cn-hangzhou.aliyuncs.com/google_containers/kube-webhook-certgen:v1.1.1
imagePullPolicy: IfNotPresent
name: patch
securityContext:
allowPrivilegeEscalation: false
nodeSelector:
kubernetes.io/os: linux
restartPolicy: OnFailure
securityContext:
fsGroup: 2000
runAsNonRoot: true
runAsUser: 2000
serviceAccountName: ingress-nginx-admission
---
apiVersion: networking.k8s.io/v1
kind: IngressClass
metadata:
labels:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: nginx
spec:
controller: k8s.io/ingress-nginx
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
labels:
app.kubernetes.io/component: admission-webhook
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
app.kubernetes.io/version: 1.8.0
name: ingress-nginx-admission
webhooks:
- admissionReviewVersions:
- v1
clientConfig:
service:
name: ingress-nginx-controller-admission
namespace: ingress-nginx
path: /networking/v1/ingresses
failurePolicy: Fail
matchPolicy: Equivalent
name: validate.nginx.ingress.kubernetes.io
rules:
- apiGroups:
- networking.k8s.io
apiVersions:
- v1
operations:
- CREATE
- UPDATE
resources:
- ingresses
sideEffects: None
[root@master01 ~]# kubectl apply -f deploy.yaml
2、案例测试
（1）nginx环境
# 每个节点都执行
[root@master01 ~]# mkdir -p /www/nginx && echo "master01" >/www/nginx/index.html
[root@master01 ~]# cat nginx.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-cluster
spec:
replicas: 3
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
volumeMounts:
- name: www
mountPath: /usr/share/nginx/html/
volumes:
- name: www
hostPath:
path: /www/nginx
---
apiVersion: v1
kind: Service
metadata:
name: nginxsvc
spec:
selector:
app: nginx
ports:
- name: http
port: 80
targetPort: 80
[root@master01 ~]# kubectl apply -f nginx.yaml
[root@master01 ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 5h17m
my-service ClusterIP 10.109.96.132 <none> 80/TCP 155m
nginxsvc ClusterIP 10.102.155.12 <none> 80/TCP 2m6s
[root@master01 ~]# curl 10.102.155.12
master01
（2）生成ingress-nginx
[root@master01 ~]# cat ingress-single.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: test-ingress
spec:
ingressClassName: nginx
defaultBackend:
service:
name: nginxsvc
port:
number: 80
[root@master01 ~]# kubectl apply -f ingress-single.yaml
[root@master01 ~]# kubectl get ingress
NAME CLASS HOSTS ADDRESS PORTS AGE
test-ingress nginx * 10.104.238.113 80 107s
[root@master01 ~]# kubectl describe ingress test-ingress
Name: test-ingress
Labels: <none>
Namespace: default
Address: 10.104.238.113
Ingress Class: nginx
Default backend: nginxsvc:80 (10.0.1.120:80,10.0.2.66:80,10.0.3.217:80)
Rules:
Host Path Backends
---- ---- --------
* * nginxsvc:80 (10.0.1.120:80,10.0.2.66:80,10.0.3.217:80)
Annotations: <none>
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Sync 110s (x2 over 2m21s) nginx-ingress-controller Scheduled for sync
[root@master01 ~]# kubectl get pod -owide -n ingress-nginx
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
ingress-nginx-admission-create-rdnrn 0/1 Completed 0 29m 10.0.3.128 master02 <none> <none>
ingress-nginx-admission-patch-vtgr4 0/1 Completed 0 29m 10.0.2.60 node02 <none> <none>
ingress-nginx-controller-59b655fb7b-b4j9q 1/1 Running 0 29m 10.0.0.50 node01 <none> <none>
[root@master01 ~]# curl 10.0.0.50
master02
[root@master01 ~]# curl 10.0.0.50
node02
[root@master01 ~]# curl 10.0.0.50
master01
（3）创建httpd
# 每个节点都执行
[root@master01 ~]# mkdir -p /www/httpd && echo "master01 httpd" >/www/httpd/index.html
[root@master01 ~]# cat httpd.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: httpd-cluster
spec:
replicas: 3
selector:
matchLabels:
app: httpd
template:
metadata:
labels:
app: httpd
spec:
containers:
- name: httpd
image: httpd:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
volumeMounts:
- name: www
mountPath: /usr/local/apache2/htdocs
volumes:
- name: www
hostPath:
path: /www/httpd
---
apiVersion: v1
kind: Service
metadata:
name: httpdsvc
spec:
selector:
app: httpd
ports:
- name: httpd
port: 80
targetPort: 80
[root@master01 ~]# kubectl apply -f httpd.yaml
[root@master01 ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
httpdsvc ClusterIP 10.105.174.147 <none> 80/TCP 32s
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 5h38m
my-service ClusterIP 10.109.96.132 <none> 80/TCP 177m
nginxsvc ClusterIP 10.102.155.12 <none> 80/TCP 23m
[root@master01 ~]# cat ingress-httpd.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: http-ingress
spec:
ingressClassName: nginx
rules:
- host: "nginx.wielun.com"
http:
paths:
- pathType: Prefix
path: /
backend:
service:
name: nginxsvc
port:
number: 80
- host: "httpd.wielun.com"
http:
paths:
- pathType: Prefix
path: /
backend:
service:
name: httpdsvc
port:
number: 80
[root@master01 ~]# kubectl apply -f ingress-httpd.yaml
（4）测试结果
[root@master01 ~]# vim /etc/hosts
10.104.238.113 nginx.wielun.com httpd.wielun.com
[root@master01 ~]# curl -H "Host: httpd.wielun.com" 10.104.238.113
master01 httpd
[root@master01 ~]# curl -H "Host: nginx.wielun.com" 10.104.238.113
master01
[root@master01 ~]# kubectl describe ingressclasses.networking.k8s.io nginx
Name: nginx
Labels: app.kubernetes.io/component=controller
app.kubernetes.io/instance=ingress-nginx
app.kubernetes.io/name=ingress-nginx
app.kubernetes.io/part-of=ingress-nginx
app.kubernetes.io/version=1.8.0
Annotations: <none>
Controller: k8s.io/ingress-nginx
Events: <none>
[root@master01 ~]# kubectl get ingressclasses.networking.k8s.io
NAME CONTROLLER PARAMETERS AGE
nginx k8s.io/ingress-nginx <none> 73m
[root@master01 ~]# kubectl describe ingress http-ingress
Name: http-ingress
Labels: <none>
Namespace: default
Address: 10.104.238.113
Ingress Class: nginx
Default backend: <default>
Rules:
Host Path Backends
---- ---- --------
nginx.wielun.com
/ nginxsvc:80 (10.0.1.120:80,10.0.2.66:80,10.0.3.217:80)
httpd.wielun.com
/ httpdsvc:80 (10.0.0.132:80,10.0.1.114:80,10.0.3.164:80)
Annotations: <none>
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Sync 28m (x2 over 29m) nginx-ingress-controller Scheduled for sync
3、通过路径匹配
（1）生成路径
[root@master01 ~]# mkdir -p /www/httpd/test && cp /www/httpd/index.html /www/httpd/test
[root@master01 ~]#mkdir -p /www/nginx/prod && cp /www/nginx/index.html /www/nginx/prod
（2）创建service和ingress
[root@master01 ~]# cat nginx.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-cluster
spec:
replicas: 3
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
volumeMounts:
- name: www
mountPath: /usr/share/nginx/html/
volumes:
- name: www
hostPath:
path: /www/nginx
---
apiVersion: v1
kind: Service
metadata:
name: nginxsvc
spec:
selector:
app: nginx
ports:
- name: http
port: 8080
targetPort: 80
[root@master01 ~]# cat httpd.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: httpd-cluster
spec:
replicas: 3
selector:
matchLabels:
app: httpd
template:
metadata:
labels:
app: httpd
spec:
containers:
- name: httpd
image: httpd:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
volumeMounts:
- name: www
mountPath: /usr/local/apache2/htdocs
volumes:
- name: www
hostPath:
path: /www/httpd
---
apiVersion: v1
kind: Service
metadata:
name: httpdsvc
spec:
selector:
app: httpd
ports:
- name: httpd
port: 8081
targetPort: 80
[root@master01 ~]# cat ingress-httpd.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: http-ingress
spec:
ingressClassName: nginx
rules:
- host: "nginx.wielun.com"
http:
paths:
- pathType: Prefix
path: /prod
backend:
service:
name: nginxsvc
port:
number: 8080
- pathType: Prefix
path: /test
backend:
service:
name: httpdsvc
port:
number: 8081
[root@master01 ~]# kubectl apply -f httpd.yaml
[root@master01 ~]# kubectl apply -f nginx.yaml
[root@master01 ~]# kubectl apply -f ingress-httpd.yaml
[root@master01 ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
httpdsvc ClusterIP 10.110.116.208 <none> 8081/TCP 31s
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 25h
my-service ClusterIP 10.109.96.132 <none> 80/TCP 22h
nginxsvc ClusterIP 10.110.201.244 <none> 8080/TCP 28s
（3）测试
[root@master01 ~]# curl nginx.wielun.com/prod/
<html>
<head><title>403 Forbidden</title></head>
<body>
<center><h1>403 Forbidden</h1></center>
<hr><center>nginx/1.25.0</center>
</body>
</html>
[root@master01 ~]# curl nginx.wielun.com/test
<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<html><head>
<title>301 Moved Permanently</title>
</head><body>
<h1>Moved Permanently</h1>
<p>The document has moved <a href="http://nginx.wielun.com/test/">here</a>.</p>
</body></html>
[root@master01 ~]# curl nginx.wielun.com/test/
master01 httpd
[root@master01 ~]# kubectl describe ingress http-ingress
Name: http-ingress
Labels: <none>
Namespace: default
Address: 10.104.238.113
Ingress Class: nginx
Default backend: <default>
Rules:
Host Path Backends
---- ---- --------
nginx.wielun.com
/prod nginxsvc:8080 (10.0.0.108:80,10.0.1.159:80,10.0.3.47:80)
/test httpdsvc:8081 (10.0.1.204:80,10.0.2.47:80,10.0.3.242:80)
Annotations: <none>
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Sync 19h (x2 over 19h) nginx-ingress-controller Scheduled for sync
Normal Sync 5m41s (x2 over 10m) nginx-ingress-controller Scheduled for sync
4、rewrite实现从定项
[root@master01 ~]# cat ingress-rewrite.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: http-ingress
annotations:
nginx.ingress.kubernetes.io/rewite-target: https://www.baidu.com
spec:
ingressClassName: nginx
rules:
- host: "nginx.wielun.com"
http:
paths:
- pathType: Prefix
path: /
backend:
service:
name: nginxsvc
port:
number: 80
[root@master01 ~]# kubectl delete -f ingress-httpd.yaml
[root@master01 ~]# kubectl apply -f ingress-rewrite.yaml
四、灰度发布
1、灰度发布方式
- canary：蓝绿和灰度发布，社区版本
- service：蓝绿和灰度发布，云端版本
流量分类：
canary-by-header - > canary-by-cookie - > canary-weight
- 基于Request Header得流量切分，适用于灰度发布及AB测试
- 基于Cookie得流量切分，适用于灰度发布及AB测试
- 基于权重得流量切分，适用于蓝绿发布
2、环境准备
[root@master01 ~]# mkdir -p /www/{old,new}
[root@master01 ~]# echo "old version" >> /www/old/index.html && echo "new version" >> /www/new/index.html
3、创建老环境
（1）配置canary-nginx.yaml
sessionAffinity：生产要打开
[root@master01 ~]# cat canary-nginx.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: old-nginx
spec:
replicas: 3
selector:
matchLabels:
app: old-nginx
template:
metadata:
labels:
app: old-nginx
spec:
containers:
- name: old-nginx
image: nginx:1.19
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
protocol: TCP
volumeMounts:
- name: old
mountPath: /usr/share/nginx/html/
volumes:
- name: old
hostPath:
path: /www/old
---
apiVersion: v1
kind: Service
metadata:
name: old-nginx
spec:
ports:
- port: 80
protocol: TCP
targetPort: 80
selector:
app: old-nginx
sessionAffinity: None
[root@master01 ~]# kubectl apply -f canary-nginx.yaml
[root@master01 ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 27h
my-service ClusterIP 10.109.96.132 <none> 80/TCP 24h
old-nginx ClusterIP 10.101.60.205 <none> 80/TCP 4s
[root@master01 ~]# kubectl get deployment -owide
NAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR
old-nginx 3/3 3 3 94s old-nginx nginx:1.19 app=old-nginx
（2）配置canary-ingress.yaml
[root@master01 ~]# cat canary-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: old-release
spec:
ingressClassName: nginx
rules:
- host: "www.wielun.com"
http:
paths:
- pathType: ImplementationSpecific
path: /
backend:
service:
name: old-nginx
port:
number: 80
[root@master01 ~]# kubectl apply -f canary-ingress.yaml
[root@master01 ~]# kubectl get ingress
NAME CLASS HOSTS ADDRESS PORTS AGE
old-release nginx www.wielun.com 10.104.238.113 80 1m
[root@master01 ~]# vim /etc/hosts
10.104.238.113 www.wielun.com
[root@master01 ~]# curl www.wielun.com
old version
4、请求头方式
（1）配置canary-nginx-new.yaml
[root@master01 ~]# cat canary-nginx-new.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: new-nginx
spec:
replicas: 3
selector:
matchLabels:
app: new-nginx
template:
metadata:
labels:
app: new-nginx
spec:
containers:
- name: new-nginx
image: nginx:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
protocol: TCP
volumeMounts:
- name: new
mountPath: /usr/share/nginx/html/
volumes:
- name: new
hostPath:
path: /www/new
---
apiVersion: v1
kind: Service
metadata:
name: new-nginx
spec:
ports:
- port: 80
protocol: TCP
targetPort: 80
selector:
app: new-nginx
sessionAffinity: None
[root@master01 ~]# kubectl apply -f canary-nginx-new.yaml
（2）配置canary-ingress-new.yaml
[root@master01 ~]# cat canary-ingress-new.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: new-release
annotations:
nginx.ingress.kubernetes.io/canary: "true"
nginx.ingress.kubernetes.io/canary-by-header: "foo"
nginx.ingress.kubernetes.io/canary-by-header-value: "bar"
spec:
ingressClassName: nginx
rules:
- host: "www.wielun.com"
http:
paths:
- pathType: ImplementationSpecific
path: /
backend:
service:
name: new-nginx
port:
number: 80
[root@master01 ~]# kubectl apply -f canary-ingress-new.yaml
[root@master01 ~]# kubectl get ingress
NAME CLASS HOSTS ADDRESS PORTS AGE
new-release nginx www.wielun.com 10.104.238.113 80 5m
old-release nginx www.wielun.com 10.104.238.113 80 11m
（3）测试
[root@master01 ~]# kubectl describe ingress old-release
Name: old-release
Labels: <none>
Namespace: default
Address: 10.104.238.113
Ingress Class: nginx
Default backend: <default>
Rules:
Host Path Backends
---- ---- --------
www.wielun.com
/ old-nginx:80 (10.0.1.50:80,10.0.2.213:80,10.0.3.160:80)
Annotations: <none>
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Sync 13m (x2 over 14m) nginx-ingress-controller Scheduled for sync
[root@master01 ~]# kubectl describe ingress new-release
Name: new-release
Labels: <none>
Namespace: default
Address: 10.104.238.113
Ingress Class: nginx
Default backend: <default>
Rules:
Host Path Backends
---- ---- --------
www.wielun.com
/ new-nginx:80 (10.0.0.143:80,10.0.1.245:80,10.0.3.8:80)
Annotations: nginx.ingress.kubernetes.io/canary: true
nginx.ingress.kubernetes.io/canary-by-header: foo
nginx.ingress.kubernetes.io/canary-by-header-value: bar
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Sync 2m59s (x2 over 3m36s) nginx-ingress-controller Scheduled for sync
[root@master01 ~]# curl www.wielun.com
old version
[root@master01 ~]# curl -H "foo:bar" www.wielun.com
new version
5、权重
[root@master01 ~]# cat canary-ingress-new.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: new-release
annotations:
nginx.ingress.kubernetes.io/canary: "true"
nginx.ingress.kubernetes.io/canary-weight: "50"
spec:
ingressClassName: nginx
rules:
- host: "www.wielun.com"
http:
paths:
- pathType: ImplementationSpecific
path: /
backend:
service:
name: new-nginx
port:
number: 80
[root@master01 ~]# kubectl apply -f canary-ingress-new.yaml
[root@master01 ~]# for i in {0..10};do curl www.wielun.com;done
new version
new version
new version
old version
new version
old version
old version
old version
new version
old version
new version
6、全部切换到新版本
[root@master01 ~]# cat canary-nginx.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: old-nginx
spec:
replicas: 3
selector:
matchLabels:
app: old-nginx
template:
metadata:
labels:
app: old-nginx
spec:
containers:
- name: old-nginx
image: nginx:1.19
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
protocol: TCP
volumeMounts:
- name: old
mountPath: /usr/share/nginx/html/
volumes:
- name: old
hostPath:
path: /www/old
---
apiVersion: v1
kind: Service
metadata:
name: old-nginx
spec:
ports:
- port: 80
protocol: TCP
targetPort: 80
selector:
app: new-nginx
sessionAffinity: None
[root@master01 ~]# kubectl apply -f canary-nginx.yaml
[root@master01 ~]# for i in {0..10};do curl www.wielun.com;done
new version
new version
new version
new version
new version
new version
new version
new version
new version
new version
new version