Multiple Ingress controllers(多ingress部署)

背景：

1、业务有个性化需求，例如需要在nginx 上面部署agent 分析日志并做告警，但该业务不关注其他业务的日志

2、每次业务变更，nginx worker进程都得执行reload。随着业务体量增加，reload会越来越频繁，拆分ingress 可以有效避免业务互相影响

# 为机器加上不同的标签，如azone/bzone 用来区分A专区跟B专区

kubectl label node test-node-1.1.1.1 ingress-role="azone" 

kubectl label node test-node-2.2.2.2 ingress-role="bzone"

# 创建ingress

root@ubuntu:/home/test# kubectl apply -f nginx-ingress-controller-ds-azone.yml 

root@ubuntu:/home/test# kubectl apply -f nginx-ingress-controller-ds-bzone.yml

# 查看部署ingress实例

root@ubuntu:/home/test# kubectl get pod -n kube-system -o wide |grep nginx 

azone-nginx-ingress-controller-d92zq      1/1       Running   0          2m        10.26.129.21     test-node-1.1.1.1

bzone-nginx-ingress-controller-dswv9   1/1       Running   0          2m        10.26.129.22    test-node-2.2.2.2

# nginx-controller 配置如下

apiVersion: extensions/v1beta1

kind: DaemonSet

metadata:

  name: azone-nginx-ingress-controller

  labels:

    app: ingress-nginx

  namespace: kube-system

spec:

  template:

    metadata:

      labels:

        app: ingress-nginx

      annotations:

        prometheus.io/scrape: "true"

        prometheus.io/port: "10254"

        prometheus.io/type: "ingress-nginx"

    spec:

      hostNetwork: true

      tolerations:

      - key: "node-role.kubernetes.io/ingress"

        operator: "Equal"

        value: "true"

        effect: "NoSchedule"

      nodeSelector:

        node-role.kubernetes.io/ingress: "true"

        ingress-role: "azone"                        # 添加指定标签，绑定固定部署机器

      serviceAccountName: admin

      containers:

        - name: azone-nginx-ingress-controller

          image: registry.cn-hangzhou.aliyuncs.com/test/ingress-controller:0.15.0-10

          args:

            - /nginx-ingress-controller

            - --default-backend-service=$(POD_NAMESPACE)/default-http-backend

            - --configmap=$(POD_NAMESPACE)/nginx-configuration

            - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services

            - --udp-services-configmap=$(POD_NAMESPACE)/udp-services

            - --publish-service=$(POD_NAMESPACE)/ingress-nginx

            - --annotations-prefix=nginx.ingress.kubernetes.io

            - --v=2

            - --enable-dynamic-configuration=true

            - --ingress-class=azone                # 指定ingress-class 属性

          env:

            - name: POD_NAME

              valueFrom:

                fieldRef:

                  fieldPath: metadata.name

            - name: COLLECT_LOG_DOCKER_DATA_WEBLOG

              value: "true"

            - name: POD_NAMESPACE

              valueFrom:

                fieldRef:

                  fieldPath: metadata.namespace

          ports:

          - name: http

            containerPort: 80

          - name: https

            containerPort: 443

          volumeMounts:

          - name: localtime-config

            mountPath: /etc/localtime

          livenessProbe:

            failureThreshold: 3

            httpGet:

              path: /healthz

              port: 10254

              scheme: HTTP

            initialDelaySeconds: 10

            periodSeconds: 10

            successThreshold: 1

            timeoutSeconds: 1

          readinessProbe:

            failureThreshold: 3

            httpGet:

              path: /healthz

              port: 10254

              scheme: HTTP

            periodSeconds: 10

            successThreshold: 1

            timeoutSeconds: 1

      volumes:

        - name: localtime-config

          hostPath:

            path: /etc/localtime

# 创建 ingress，配置里面绑定class

root@ubuntu:/home/test# cat azone-test.aaa.com-ingress.yml

apiVersion: extensions/v1beta1

kind: Ingress

metadata:

  name: azone-test-ingress-https

  annotations:

    kubernetes.io/ingress.class: "azone"                    # 绑定ingress-class

    nginx.ingress.kubernetes.io/ssl-redirect: "false"

spec:

  rules:

  - host: azone-test.aaa.com

    http:

      paths:

      - path: /

        backend:

          serviceName: azone-test-svc

          servicePort: 80

# 查看绑定情况

root@ubuntu:/home/wuguihong1# kubectl -n kube-system get pod -o wide|grep nginx

azone-ingress-controller-d92zq    1/1       Running   0          16h       10.26.129.21    test-node-1.1.1.1

bzone-ingress-controller-62458   1/1       Running   0          15h       10.26.129.22    test-node-2.2.2.2

root@ubuntu:/home/test# kubectl -n kube-system exec  azone-ingress-controller-d92zq cat /etc/nginx/nginx.conf |grep azone-test.aaa.com

server_name azone-test.aaa.com ;

root@ubuntu:/home/test# kubectl -n kube-system exec bzone-ingress-controller-62458  cat /etc/nginx/nginx.conf|grep azone-test.aaa.com

可以看到2台node节点上各运行一个ingress-controller ， 并且azone 上面绑定了azone-test.aaa.com 的域名，而bzone 上面没绑定

参考资料：

Multiple Ingress controllers

https://kubernetes.github.io/ingress-nginx/user-guide/multiple-ingress/