nginx-ingress的灰度可以使用注解来完成 nginx.ingress.kubernetes.io/canary: "true"。目前常见的灰度方式有:

  • 基于权重灰度:例如20%流量切到灰度版本
  • 基于header灰度
    • 基于 header 键: 例如只要存在header 键为“canary-header”
    • 基于 header 值:例如存在header键为“canary-header”,其值为:“canary-v1”
  • 基于cookie灰度:例如cookie值为 “h5-v1”

准备测试资源

旧版资源

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
# 旧版deployment
[root@node1 demo]# cat nginx-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-canary
labels:
app: nginx-canary
spec:
replicas: 10
selector:
matchLabels:
app: nginx-canary
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 10%
maxSurge: 10%
template:
metadata:
labels:
app: nginx-canary
spec:
initContainers:
- name: init-container
image: busybox:latest
imagePullPolicy: IfNotPresent
command: ["sh"]
env:
# - name: MY_POD_NAME
# valueFrom:
# fieldRef:
# fieldPath: metadata.name
- name: MY_POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
args:
[
"-c",
"echo ${HOSTNAME} ${MY_POD_IP} canary-v1 > /wwwroot/index.html",
]
volumeMounts:
- name: wwwroot
mountPath: "/wwwroot"
containers:
- name: nginx
image: nginx:latest
ports:
- containerPort: 80
protocol: TCP
volumeMounts:
- name: wwwroot
mountPath: /usr/share/nginx/html/index.html
subPath: index.html
volumes:
- name: wwwroot
emptyDir: {}

# 旧版service
[root@node1 demo]# cat nginx-canary-service.yaml
apiVersion: v1
kind: Service
metadata:
name: nginx-canary
labels:
app: nginx-canary
spec:
ports:
- port: 80
targetPort: 80
name: nginx-canary
selector:
app: nginx-canary

# 旧版ingress
[root@node1 demo]# cat canary-weight-ingress-old.yaml
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: gray-release
namespace: default
spec:
rules:
- host: 'canary.xadocker.cn'
http:
paths:
- path: /
backend:
serviceName: nginx-canary
servicePort: 80


[root@node1 demo]# kubectl apply -f nginx-deployment.yaml
[root@node1 demo]# kubectl apply -f nginx-canary-service.yaml
[root@node1 demo]# kubectl apply -f canary-weight-ingress-old.yaml

新版资源

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
# 新版deployment
[root@node1 demo]# cat nginx-deployment-new.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-canary-new
labels:
app: nginx-canary-new
spec:
replicas: 10
selector:
matchLabels:
app: nginx-canary-new
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 10%
maxSurge: 10%
template:
metadata:
labels:
app: nginx-canary-new
spec:
initContainers:
- name: init-container
image: busybox:latest
imagePullPolicy: IfNotPresent
command: ["sh"]
env:
# - name: MY_POD_NAME
# valueFrom:
# fieldRef:
# fieldPath: metadata.name
- name: MY_POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
args:
[
"-c",
"echo ${HOSTNAME} ${MY_POD_IP} canary-v2 > /wwwroot/index.html",
]
volumeMounts:
- name: wwwroot
mountPath: "/wwwroot"
containers:
- name: nginx
image: nginx:latest
ports:
- containerPort: 80
protocol: TCP
volumeMounts:
- name: wwwroot
mountPath: /usr/share/nginx/html/index.html
subPath: index.html
volumes:
- name: wwwroot
emptyDir: {}

# 新版service
[root@node1 demo]# cat nginx-canary-service-new.yaml
apiVersion: v1
kind: Service
metadata:
name: nginx-canary-new
labels:
app: nginx-canary-new
spec:
ports:
- port: 80
targetPort: 80
name: nginx-canary-new
selector:
app: nginx-canary-new


[root@node1 demo]# kubectl apply -f nginx-deployment-new.yaml
[root@node1 demo]# kubectl apply -f nginx-canary-service-new.yaml

基于权重的灰度

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# 创建权重灰度
[root@node1 demo]# cat canary-weight-ingress-new.yaml
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: canary-new
namespace: default
annotations:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/canary: "true"
nginx.ingress.kubernetes.io/canary-weight: "20"
spec:
rules:
- host: canary.xadocker.cn
http:
paths:
- path: /
backend:
serviceName: nginx-canary-new
servicePort: 80

[root@node1 demo]# kubectl apply -f canary-weight-ingress-new.yaml

测试验证

1
2
3
4
5
6
7
8
9
[root@node1 demo]# kubectl get ingress
NAME CLASS HOSTS ADDRESS PORTS AGE
canary-new <none> canary.xadocker.cn 80 18m
gray-release <none> canary.xadocker.cn 80 17m

[root@node1 demo]# for i in `seq 1000`;do curl -H "HOST:canary.xadocker.cn" 127.0.0.1 -s >> test.txt;done
[root@node1 demo]# cat test.txt awk '{print $3}' sort -r uniq -c
191 canary-v2
809 canary-v1

基于header的灰度

基于header键的灰度

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
# 
[root@node1 demo]# kubectl delete -f canary-weight-ingress-new.yaml
[root@node1 demo]# cat >canary-weight-ingress-new.yaml<<'EOF'
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: canary-new
namespace: default
annotations:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/canary: "true"
nginx.ingress.kubernetes.io/canary-by-header: "canary-v2"
spec:
rules:
- host: canary.xadocker.cn
http:
paths:
- path: /
backend:
serviceName: nginx-canary-new
servicePort: 80
EOF
[root@node1 demo]# kubectl apply -f canary-weight-ingress-new.yaml

# 测试的时候其键值必须为always,否则都会访问旧版
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" 127.0.0.1
nginx-canary-7fc7ccb7bd-tspvm 10.100.166.160 canary-v1
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-v2: always" 127.0.0.1
nginx-canary-new-87b757b7d-2csns 10.100.166.169 canary-v2
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-v2: as" 127.0.0.1
nginx-canary-7fc7ccb7bd-rwscr 10.100.166.157 canary-v1

基于header键值的灰度

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
# 配置灰度header为canary-release,其值为v2
[root@node1 demo]# kubectl delete -f canary-weight-ingress-new.yaml
[root@node1 demo]# cat >canary-weight-ingress-new.yaml<<'EOF'
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: canary-new
namespace: default
annotations:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/canary: "true"
nginx.ingress.kubernetes.io/canary-by-header: "canary-release"
nginx.ingress.kubernetes.io/canary-by-header-value: "v2"
spec:
rules:
- host: canary.xadocker.cn
http:
paths:
- path: /
backend:
serviceName: nginx-canary-new
servicePort: 80
EOF
[root@node1 demo]# kubectl apply -f canary-weight-ingress-new.yaml

# 测试
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-: always" 127.0.0.1
nginx-canary-7fc7ccb7bd-tspvm 10.100.166.160 canary-v1
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-: always" 127.0.0.1
nginx-canary-7fc7ccb7bd-rz2f4 10.100.166.159 canary-v1
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-release: always" 127.0.0.1
nginx-canary-7fc7ccb7bd-tspvm 10.100.166.160 canary-v1
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-release: v1" 127.0.0.1
nginx-canary-7fc7ccb7bd-8vn7c 10.100.166.162 canary-v1
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-release: v2" 127.0.0.1
nginx-canary-new-87b757b7d-wqcpw 10.100.166.165 canary-v2
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-release: v2" 127.0.0.1

# 基于header的键值正则匹配
[root@node1 demo]# kubectl delete -f canary-weight-ingress-new.yaml
[root@node1 demo]# cat >canary-weight-ingress-new.yaml<<'EOF'
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: canary-new
namespace: default
annotations:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/canary: "true"
nginx.ingress.kubernetes.io/canary-by-header: "canary-release"
nginx.ingress.kubernetes.io/canary-by-header-pattern: "v2v3"
spec:
rules:
- host: canary.xadocker.cn
http:
paths:
- path: /
backend:
serviceName: nginx-canary-new
servicePort: 80
EOF

[root@node1 demo]# kubectl apply -f canary-weight-ingress-new.yaml
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-release: v2" 127.0.0.1
nginx-canary-new-87b757b7d-fxrq4 10.100.166.164 canary-v2
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-release: v3" 127.0.0.1
nginx-canary-new-87b757b7d-2csns 10.100.166.169 canary-v2
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" -H "canary-release: v1" 127.0.0.1
nginx-canary-7fc7ccb7bd-rvfcz 10.100.166.161 canary-v1

基于cookie的灰度

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
# 其cookie值必须为always才能切流到新版
[root@node1 demo]# kubectl delete -f canary-weight-ingress-new.yaml
[root@node1 demo]# cat canary-weight-ingress-new.yaml
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: canary-new
namespace: default
annotations:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/canary: "true"
nginx.ingress.kubernetes.io/canary-by-cookie: "userid12345678"
spec:
rules:
- host: canary.xadocker.cn
http:
paths:
- path: /
backend:
serviceName: nginx-canary-new
servicePort: 80

[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" --cookie "userid12345678" 127.0.0.1
nginx-canary-7fc7ccb7bd-rvfcz 10.100.166.161 canary-v1
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" --cookie "userid12345678=always" 127.0.0.1
nginx-canary-new-87b757b7d-z2z87 10.100.166.167 canary-v2
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" --cookie "userid1234567sdf8=always" 127.0.0.1
nginx-canary-7fc7ccb7bd-rz2f4 10.100.166.159 canary-v1
[root@node1 demo]# curl -H "HOST:canary.xadocker.cn" 127.0.0.1
nginx-canary-7fc7ccb7bd-6qdhb 10.100.166.163 canary-v1