一、开启会话保持

和 Nginx 一样，Ingress Nginx 也支持基于 cookie 的会话保持。

环境准备

1.创建tls类型的secret

[root@k8s-master01 ~]# kubectl create secret tls ca-secret --cert=test.zhang-qing.com.crt --key=test.zhang-qing.com.key -n study-ingress

[root@k8s-master01 day012]# kubectl get secret -n study-ingress
NAME        TYPE                DATA   AGE
ca-secret   kubernetes.io/tls   2      7s

2.配置 Ingress 添加 TLS 配置

[root@k8s-master01 ~]# vim ingress-ssl.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  creationTimestamp: null
  name: nginx-ingress
  namespace: study-ingress
  # annotations:
   # kubernetes.io/ingress.class: nginx
spec:
  ingressClassName: nginx # for k8s >= 1.22+
  rules:
  - host: test.zhang-qing.com
    http:
      paths:
      - backend:
          service:
            name: nginx
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific
  tls:
  - hosts:
    - test.zhang-qing.com
    secretName: ca-secret

上面参数说明：

  • hosts：证书所授权的域名列表
  • secretName：证书的Secret名字

说明：1.22+版本的k8s需要配置ingressClassName

3.更新该Ingress

[root@k8s-master01 ~]# kaf ingress-ssl.yaml

开启会话保持

1、扩容 nginx 服务至2个副本

[root@k8s-master01 day012]# k scale deploy nginx --replicas=2 -n study-ingress

# 扩容后进行验证
[root@k8s-master01 day012]# kg deploy -n study-ingress
NAME    READY   UP-TO-DATE   AVAILABLE   AGE
nginx   2/2     2            2           43m

2、没有配置会话保持之前一直访问test.zhang-qing.com，观察到流量会轮询分发到这两个副本中

[root@k8s-master01 day012]# kg po -n study-ingress
NAME                     READY   STATUS    RESTARTS   AGE
nginx-84568d57dd-5cc8p   1/1     Running   0          44s
nginx-84568d57dd-dhk7h   1/1     Running   0          44m

[root@k8s-master01 ~]# k logs -f nginx-84568d57dd-5cc8p -n study-ingress
192.168.58.192 - - [19/Mar/2025:00:43:16 +0000] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0" "10.0.0.1"
...
...

[root@k8s-master01 ~]# k logs -f nginx-84568d57dd-dhk7h  -n study-ingress
10.0.0.22 - - [19/Mar/2025:00:43:12 +0000] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0" "10.0.0.1"
...
...

3、重新定义yaml文件，增加会话保持配置

[root@k8s-master01 ~]# k delete -f ingress-ssl.yaml

[root@k8s-master01 ~]# vim ingress-ssl-keep.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: nginx-ingress
  namespace: study-ingress
  annotations:
    nginx.ingress.kubernetes.io/proxy-body-size: 16m
    nginx.ingress.kubernetes.io/affinity: "cookie"
    nginx.ingress.kubernetes.io/session-cookie-name: "route"
    nginx.ingress.kubernetes.io/session-cookie-expires: "172800"
    nginx.ingress.kubernetes.io/session-cookie-max-age: "172800"
    nginx.ingress.kubernetes.io/affinity-mode: persistent
spec:
  ingressClassName: nginx # for k8s >= 1.22+
  rules:
  - host: test.zhang-qing.com
    http:
      paths:
      - backend:
          service:
            name: nginx
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific
  tls:
  - hosts:
    - test.zhang-qing.com
    secretName: ca-secret

会话保持相关配置说明：

  • nginx.ingress.kubernetes.io/proxy-body-size: 设置客户端请求体（如文件上传、表单数据等）的最大允许大小，这里设置为16M
  • nginx.ingress.kubernetes.io/affinity：启用 会话亲和性（Session Affinity），即通过 Cookie 实现“粘性会话”（Sticky Session）
  • nginx.ingress.kubernetes.io/session-cookie-name：定义用于会话亲和性的 Cookie 名称
  • nginx.ingress.kubernetes.io/session-cookie-expires：设置会话 Cookie 的绝对过期时间
  • nginx.ingress.kubernetes.io/session-cookie-max-age：设置会话 Cookie 的相对过期时间
  • nginx.ingress.kubernetes.io/affinity-mode：定义会话亲和性的持久化模式
  • sticky(默认模式)：确保来自同一客户端的请求始终路由到同一个后端 Pod，只要该 Pod 可用。
  • persistent模式：即使 Pod 重启或更新，只要 Pod 名称和命名空间不变，会话会持久化到原 Pod。
  • balanced模式：在保持会话亲和性的同时，动态平衡后端 Pod 的负载。

4、重新创建ingress

[root@k8s-master01 ~]# kaf ingress-ssl-keep.yaml

# 验证
[root@k8s-master01 day012]# kg ingress -n study-ingress
NAME            CLASS   HOSTS                 ADDRESS   PORTS     AGE
nginx-ingress   nginx   test.zhang-qing.com             80, 443   31s

5、多次访问test.zhang-qing.com，查看两个pod的访问日志，观察到只有一个Pod上面产生访问日志信息

[root@k8s-master01 ~]# k logs -f nginx-84568d57dd-5cc8p -n study-ingress
192.168.58.192 - - [19/Mar/2025:00:43:16 +0000] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0" "10.0.0.1"
...
...

[root@k8s-master01 ~]# k logs -f nginx-84568d57dd-dhk7h  -n study-ingress

6、浏览器访问test.zhang-qing.com的同时，查看请求标头产生Cookie信息，观察到Cookie信息已配置成功

image-20250319090931824

同时也可以进入pod查看配置已经生效

[root@k8s-master01 ~]# k exec -it ingress-nginx-controller-qwmkm -n ingress-nginx -- bash
k8s-node02:/etc/nginx$ cat nginx.conf
...
...
client_max_body_size                    16m;
...
...

7、重新定义会话亲和性的持久化模式为balanced

[root@k8s-master01 ~]# vim ingress-ssl-keep.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: nginx-ingress
  namespace: study-ingress
  annotations:
    nginx.ingress.kubernetes.io/proxy-body-size: 16m
    nginx.ingress.kubernetes.io/affinity: "cookie"
    nginx.ingress.kubernetes.io/session-cookie-name: "route"
    nginx.ingress.kubernetes.io/session-cookie-expires: "172800"
    nginx.ingress.kubernetes.io/session-cookie-max-age: "172800"
    nginx.ingress.kubernetes.io/affinity-mode: balanced
spec:
  ingressClassName: nginx # for k8s >= 1.22+
  rules:
  - host: test.zhang-qing.com
    http:
      paths:
      - backend:
          service:
            name: nginx
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific
  tls:
  - hosts:
    - test.zhang-qing.com
    secretName: ca-secret

[root@k8s-master01 ~]# kaf ingress-ssl-keep.yaml

8、扩容副本数为3

[root@k8s-master01 day012]# k scale deploy nginx --replicas=3 -n study-ingress

# 验证
[root@k8s-master01 day012]# kgp -n study-ingress
NAME                     READY   STATUS    RESTARTS   AGE
nginx-84568d57dd-5cc8p   1/1     Running   0          48m
nginx-84568d57dd-dhk7h   1/1     Running   0          91m
nginx-84568d57dd-ptsdk   1/1     Running   0          13s

9、修改会话亲和性的持久化模式为balanced后多次访问test.zhang-qing.com再次进行验证

[root@k8s-master01 day012]# k logs -f -n study-ingress nginx-84568d57dd-5cc8p
0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0" "10.0.0.1"
192.168.58.192 - - [19/Mar/2025:01:34:13 +0000] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0" "10.0.0.1"
...
...

[root@k8s-master01 day012]# k logs -f -n study-ingress nginx-84568d57dd-dhk7h
10.0.0.22 - - [19/Mar/2025:01:34:02 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.76.1" "10.0.0.20"
10.0.0.22 - - [19/Mar/2025:01:34:07 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.76.1" "10.0.0.20"
...
...

[root@k8s-master01 day012]# k logs -f -n study-ingress nginx-84568d57dd-ptsdk
10.0.0.22 - - [19/Mar/2025:01:34:02 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.76.1" "10.0.0.20"
10.0.0.22 - - [19/Mar/2025:01:34:07 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.76.1" "10.0.0.20"
...
...

问题汇总

问题描述：

当修改会话亲和性的持久化模式为balanced后，扩容副本后在客户端多次访问test.zhang-qing.com还是只往一个pod中打访问日志信息

解决方法：

1、清除浏览器缓存，重新多次访问

2、尝试在不同客户端进行访问，比如在别的linux机器上执行如下命令进行访问

# 客户端1（携带 Cookie）
curl -k -H "Cookie: route=abcd1234" https://test.zhang-qing.com

# 客户端2（不携带 Cookie）
curl -k https://test.zhang-qing.com

环境复原

删除之前创建的ingress

k delete -f ingress-ssl-keep.yaml

二、配置流式返回 SSE

配置流式返回 SSE

如果后端服务需要持续的输出数据，或者需要长连接，此时需要更改请求头升级链接为长连接

1、修改ingress-nginx的configmap

#新增下面信息
data:
  allow-snippet-annotations: "true"
  annotations-risk-level: Critical

#新增metadata.annotations.kubectl.kubernetes.io/last-applied-configuration下面的data信息"data":{"allow-snippet-annotations":"true","annotations-risk-level":"Critical"}
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","data":{"allow-snippet-annotations":"true","annotations-risk-level":"Critical"},"kind":"ConfigMap","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"controller","app.kubernetes.io/instance":"ingress-nginx","app.kubernetes.io/name":"ingress-nginx","app.kubernetes.io/part-of":"ingress-nginx","app.kubernetes.io/version":"1.12.0"},"name":"ingress-nginx-controller","namespace":"ingress-nginx"}}

#修改后的内容如下
[root@k8s-master01 day012]# k edit cm ingress-nginx-controller  -n ingress-nginx
apiVersion: v1
data:
  allow-snippet-annotations: "true"
  annotations-risk-level: Critical
kind: ConfigMap
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","data":{"allow-snippet-annotations":"true","annotations-risk-level":"Critical"},"kind":"ConfigMap","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"controller","app.kubernetes.io/instance":"ingress-nginx","app.kubernetes.io/name":"ingress-nginx","app.kubernetes.io/part-of":"ingress-nginx","app.kubernetes.io/version":"1.12.0"},"name":"ingress-nginx-controller","namespace":"ingress-nginx"}}
  creationTimestamp: "2025-03-19T02:57:51Z"
  labels:
    app.kubernetes.io/component: controller
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.0
  name: ingress-nginx-controller
  namespace: ingress-nginx
  resourceVersion: "31199"
  uid: 24c226f2-68bd-485f-9747-a0a619e7d6e3

2、重启ingress-nginx-controller

[root@k8s-master01 day012]# k delete po ingress-nginx-controller-5ctqn  -n ingress-nginx

3、重新定义ingress

[root@k8s-master01 ~]# vim ingress-see.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: timeout
  namespace: study-ingress
  annotations:
    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
    nginx.ingress.kubernetes.io/proxy-buffering: "off"
    nginx.ingress.kubernetes.io/configuration-snippet: |
      proxy_set_header Updrade $http_updrade;
      proxy_set_header Connection 'upgrade';
spec:
  ingressClassName: nginx # for k8s >= 1.22+
  rules:
  - host: test.zhang-qing.com
    http:
      paths:
      - backend:
          service:
            name: nginx
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific

[root@k8s-master01 ~]# kaf ingress-see.yaml

相关参数：

  • nginx.ingress.kubernetes.io/proxy-http-version:指定http版本为1.1
  • nginx.ingress.kubernetes.io/proxy-buffering:禁用缓冲

4、登录到ingress-nginx的pod查看配置是否生效

[root@k8s-master01 ~]# kgp -n ingress-nginx | grep ingress-nginx-controller
ingress-nginx-controller-xqhks         1/1     Running     0          13m

[root@k8s-master01 ~]# k exec -it ingress-nginx-controller-xqhks -n ingress-nginx -- bash
k8s-node02:/etc/nginx$ vi nginx.conf
...
...
                        proxy_set_header                        Upgrade           $http_upgrade;

                        proxy_set_header                        Connection        $connection_upgrade;
...
...

问题汇总

问题描述：

当应用ingress-see.yaml文件时，会提示ingress-nginx注解没有打开

参考链接：https://github.com/kubernetes/ingress-nginx/issues/12648

[root@k8s-master01 day012]# kaf ingress-see.yaml
Error from server (BadRequest): error when creating "ingress-see.yaml": admission webhook "validate.nginx.ingress.kubernetes.io" denied the request: nginx.ingress.kubernetes.io/configuration-snippet annotation cannot be used. Snippet directives are disabled by the Ingress administrator

解决方法：

1、修改ingress-nginx的configmap

#新增下面信息
data:
  allow-snippet-annotations: "true"
  annotations-risk-level: Critical

#新增metadata.annotations.kubectl.kubernetes.io/last-applied-configuration下面的data信息"data":{"allow-snippet-annotations":"true","annotations-risk-level":"Critical"}
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","data":{"allow-snippet-annotations":"true","annotations-risk-level":"Critical"},"kind":"ConfigMap","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"controller","app.kubernetes.io/instance":"ingress-nginx","app.kubernetes.io/name":"ingress-nginx","app.kubernetes.io/part-of":"ingress-nginx","app.kubernetes.io/version":"1.12.0"},"name":"ingress-nginx-controller","namespace":"ingress-nginx"}}

#修改后的内容如下
[root@k8s-master01 day012]# k edit cm ingress-nginx-controller  -n ingress-nginx
apiVersion: v1
data:
  allow-snippet-annotations: "true"
  annotations-risk-level: Critical
kind: ConfigMap
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","data":{"allow-snippet-annotations":"true","annotations-risk-level":"Critical"},"kind":"ConfigMap","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"controller","app.kubernetes.io/instance":"ingress-nginx","app.kubernetes.io/name":"ingress-nginx","app.kubernetes.io/part-of":"ingress-nginx","app.kubernetes.io/version":"1.12.0"},"name":"ingress-nginx-controller","namespace":"ingress-nginx"}}
  creationTimestamp: "2025-03-19T02:57:51Z"
  labels:
    app.kubernetes.io/component: controller
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.0
  name: ingress-nginx-controller
  namespace: ingress-nginx
  resourceVersion: "31199"
  uid: 24c226f2-68bd-485f-9747-a0a619e7d6e3

2、重启ingress-nginx-controller

[root@k8s-master01 day012]# k delete po ingress-nginx-controller-5ctqn  -n ingress-nginx

3、重新应用ingress

[root@k8s-master01 day012]# kaf ingress-see.yaml

环境复原

删除之前创建的ingress

[root@k8s-master01 day012]# k delete -f  ingress-see.yaml