Update ingress-nginx addon

deploy/addons/ingress/ingress-configmap.yaml.tmpl

@@ -15,7 +15,6 @@
 apiVersion: v1
 data:
   # see https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/configmap.md for all possible options and their description
-  map-hash-bucket-size: "128"
   hsts: "false"
 kind: ConfigMap
 metadata:

deploy/addons/ingress/ingress-dp.yaml.tmpl

@@ -16,11 +16,13 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: nginx-ingress-controller
+  name: ingress-nginx-controller
   namespace: kube-system
   labels:
-    app.kubernetes.io/name: nginx-ingress-controller
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
     app.kubernetes.io/part-of: kube-system
+    app.kubernetes.io/component: controller
     addonmanager.kubernetes.io/mode: Reconcile
 spec:
   replicas: 1
@@ -32,67 +34,251 @@ spec:
       maxSurge: 1
   selector:
     matchLabels:
-      app.kubernetes.io/name: nginx-ingress-controller
+      app.kubernetes.io/name: ingress-nginx
-      app.kubernetes.io/part-of: kube-system
+      app.kubernetes.io/instance: ingress-nginx
-      addonmanager.kubernetes.io/mode: Reconcile
+      app.kubernetes.io/component: controller
   template:
     metadata:
       labels:
-        app.kubernetes.io/name: nginx-ingress-controller
+        app.kubernetes.io/name: ingress-nginx
-        app.kubernetes.io/part-of: kube-system
+        app.kubernetes.io/instance: ingress-nginx
+        app.kubernetes.io/component: controller
         addonmanager.kubernetes.io/mode: Reconcile
-      annotations:
-        prometheus.io/port: '10254'
-        prometheus.io/scrape: 'true'
     spec:
-      serviceAccountName: nginx-ingress
+      serviceAccountName: ingress-nginx
-      terminationGracePeriodSeconds: 60
       containers:
-      - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller{{.ExoticArch}}:0.26.1
+        - name: controller
-        name: nginx-ingress-controller
+          image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.32.0
-        imagePullPolicy: IfNotPresent
+          imagePullPolicy: IfNotPresent
-        readinessProbe:
+          lifecycle:
-          httpGet:
+            preStop:
-            path: /healthz
+              exec:
-            port: 10254
+                command:
-            scheme: HTTP
+                  - /wait-shutdown
-        livenessProbe:
+          args:
-          httpGet:
+            - /nginx-ingress-controller
-            path: /healthz
+            - --configmap=$(POD_NAMESPACE)/nginx-load-balancer-conf
-            port: 10254
+            - --report-node-internal-ip-address
-            scheme: HTTP
+            - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
-          initialDelaySeconds: 10
+            - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
-          timeoutSeconds: 1
+            - --validating-webhook=:8443
-        env:
+            - --validating-webhook-certificate=/usr/local/certificates/cert
-          - name: POD_NAME
+            - --validating-webhook-key=/usr/local/certificates/key
-            valueFrom:
+          securityContext:
-              fieldRef:
+            capabilities:
-                fieldPath: metadata.name
-          - name: POD_NAMESPACE
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.namespace
-        ports:
-        - containerPort: 80
-          hostPort: 80
-        - containerPort: 443
-          hostPort: 443
-        # (Optional) we expose 18080 to access nginx stats in url /nginx-status
-        - containerPort: 18080
-          hostPort: 18080
-        args:
-        - /nginx-ingress-controller
-        - --configmap=$(POD_NAMESPACE)/nginx-load-balancer-conf
-        - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
-        - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
-        - --annotations-prefix=nginx.ingress.kubernetes.io
-        # use minikube IP address in ingress status field
-        - --report-node-internal-ip-address
-        securityContext:
-          capabilities:
               drop:
-              - ALL
+                - ALL
               add:
-              - NET_BIND_SERVICE
+                - NET_BIND_SERVICE
-          # www-data -> 33
+            runAsUser: 101
-          runAsUser: 33
+            allowPrivilegeEscalation: true
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 10254
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 1
+            successThreshold: 1
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: 10254
+              scheme: HTTP
+            initialDelaySeconds: 10
+            timeoutSeconds: 1
+            successThreshold: 1
+            failureThreshold: 3
+          ports:
+            - name: http
+              containerPort: 80
+              protocol: TCP
+              hostPort: 80
+            - name: https
+              containerPort: 443
+              protocol: TCP
+              hostPort: 443
+            - name: webhook
+              containerPort: 8443
+              protocol: TCP
+          volumeMounts:
+            - name: webhook-cert
+              mountPath: /usr/local/certificates/
+              readOnly: true
+          resources:
+            requests:
+              cpu: 100m
+              memory: 90Mi
+      volumes:
+        - name: webhook-cert
+          secret:
+            secretName: ingress-nginx-admission
+
+---
+
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/component: admission-webhook
+  name: ingress-nginx-admission
+  namespace: kube-system
+webhooks:
+  - name: validate.nginx.ingress.kubernetes.io
+    rules:
+      - apiGroups:
+          - extensions
+          - networking.k8s.io
+        apiVersions:
+          - v1beta1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - ingresses
+    failurePolicy: Fail
+    clientConfig:
+      service:
+        namespace: kube-system
+        name: ingress-nginx-controller-admission
+        path: /extensions/v1beta1/ingresses
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: ingress-nginx-admission
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/component: admission-webhook
+  namespace: kube-system
+rules:
+  - apiGroups:
+      - admissionregistration.k8s.io
+    resources:
+      - validatingwebhookconfigurations
+    verbs:
+      - get
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: ingress-nginx-admission
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/component: admission-webhook
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: ingress-nginx-admission
+subjects:
+  - kind: ServiceAccount
+    name: ingress-nginx-admission
+    namespace: kube-system
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ingress-nginx-admission-create
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/component: admission-webhook
+  namespace: kube-system
+spec:
+  template:
+    metadata:
+      name: ingress-nginx-admission-create
+      labels:
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/instance: ingress-nginx
+        app.kubernetes.io/component: admission-webhook
+    spec:
+      containers:
+        - name: create
+          image: jettech/kube-webhook-certgen:v1.2.0
+          imagePullPolicy: IfNotPresent
+          args:
+            - create
+            - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.kube-system.svc
+            - --namespace=kube-system
+            - --secret-name=ingress-nginx-admission
+      restartPolicy: OnFailure
+      serviceAccountName: ingress-nginx-admission
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 2000
+
+---
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ingress-nginx-admission-patch
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/component: admission-webhook
+  namespace: kube-system
+spec:
+  template:
+    metadata:
+      name: ingress-nginx-admission-patch
+      labels:
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/instance: ingress-nginx
+        app.kubernetes.io/component: admission-webhook
+    spec:
+      containers:
+        - name: patch
+          image: jettech/kube-webhook-certgen:v1.2.0
+          imagePullPolicy:
+          args:
+            - patch
+            - --webhook-name=ingress-nginx-admission
+            - --namespace=kube-system
+            - --patch-mutating=false
+            - --secret-name=ingress-nginx-admission
+            - --patch-failure-policy=Fail
+      restartPolicy: OnFailure
+      serviceAccountName: ingress-nginx-admission
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 2000
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/component: controller
+    addonmanager.kubernetes.io/mode: Reconcile
+  name: ingress-nginx-controller-admission
+  namespace: kube-system
+spec:
+  ports:
+    - name: https-webhook
+      port: 443
+      targetPort: webhook
+  selector:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/component: controller

deploy/addons/ingress/ingress-rbac.yaml.tmpl

@@ -3,122 +3,186 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: nginx-ingress
+  name: ingress-nginx
   namespace: kube-system
   labels:
     addonmanager.kubernetes.io/mode: Reconcile
 
 ---
 
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ingress-nginx-admission
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/component: admission-webhook
+    addonmanager.kubernetes.io/mode: Reconcile
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: system:nginx-ingress
+  name: system::ingress-nginx
   labels:
     kubernetes.io/bootstrapping: rbac-defaults
-    addonmanager.kubernetes.io/mode: Reconcile
 rules:
-- apiGroups:
+  - apiGroups:
-  - ""
+      - ''
-  resources:
+    resources:
-  - configmaps
+      - configmaps
-  - endpoints
+      - endpoints
-  - nodes
+      - nodes
-  - pods
+      - pods
-  - secrets
+      - secrets
-  verbs:
+    verbs:
-  - list
+      - list
-  - watch
+      - watch
-- apiGroups:
+  - apiGroups:
-  - ""
+      - ''
-  resources:
+    resources:
-  - nodes
+      - nodes
-  verbs:
+    verbs:
-  - get
+      - get
-- apiGroups:
+  - apiGroups:
-  - ""
+      - ''
-  resources:
+    resources:
-  - services
+      - services
-  verbs:
+    verbs:
-  - get
+      - get
-  - list
+      - list
-  - watch
+      - watch
-- apiGroups:
+  - apiGroups:
-  - "extensions"
+      - extensions
-  - "networking.k8s.io"
+      - networking.k8s.io   # k8s 1.14+
-  resources:
+    resources:
-  - ingresses
+      - ingresses
-  verbs:
+    verbs:
-  - get
+      - get
-  - list
+      - list
-  - watch
+      - watch
-- apiGroups:
+  - apiGroups:
-  - ""
+      - ''
-  resources:
+    resources:
-  - events
+      - events
-  verbs:
+    verbs:
-  - create
+      - create
-  - patch
+      - patch
-- apiGroups:
+  - apiGroups:
-  - "extensions"
+      - extensions
-  - "networking.k8s.io"
+      - networking.k8s.io   # k8s 1.14+
-  resources:
+    resources:
-  - ingresses/status
+      - ingresses/status
-  verbs:
+    verbs:
-  - update
+      - update
+  - apiGroups:
+      - networking.k8s.io   # k8s 1.18+
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
 
 ---
 
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: system::nginx-ingress-role
+  name: system::ingress-nginx
   namespace: kube-system
   labels:
     kubernetes.io/bootstrapping: rbac-defaults
     addonmanager.kubernetes.io/mode: Reconcile
 rules:
-- apiGroups:
+  - apiGroups:
-  - ""
+      - ''
-  resources:
+    resources:
-  - configmaps
+      - namespaces
-  - pods
+    verbs:
-  - secrets
+      - get
-  - namespaces
+  - apiGroups:
-  verbs:
+      - ''
-  - get
+    resources:
-- apiGroups:
+      - configmaps
-  - ""
+      - pods
-  resources:
+      - secrets
-  - configmaps
+      - endpoints
-  resourceNames:
+    verbs:
-  # Defaults to "<election-id>-<ingress-class>"
+      - get
-  # Here: "<ingress-controller-leader>-<nginx>"
+      - list
-  # This has to be adapted if you change either parameter
+      - watch
-  # when launching the nginx-ingress-controller.
+  - apiGroups:
-  - ingress-controller-leader-nginx
+      - ''
-  verbs:
+    resources:
-  - get
+      - services
-  - update
+    verbs:
-- apiGroups:
+      - get
-  - ""
+      - list
-  resources:
+      - update
-  - configmaps
+      - watch
-  verbs:
+  - apiGroups:
-  - create
+      - extensions
-- apiGroups:
+      - networking.k8s.io   # k8s 1.14+
-  - ""
+    resources:
-  resources:
+      - ingresses
-  - endpoints
+    verbs:
-  verbs:
+      - get
-  - get
+      - list
+      - watch
+  - apiGroups:
+      - extensions
+      - networking.k8s.io   # k8s 1.14+
+    resources:
+      - ingresses/status
+    verbs:
+      - update
+  - apiGroups:
+      - networking.k8s.io   # k8s 1.18+
+    resources:
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ''
+    resources:
+      - configmaps
+    resourceNames:
+      - ingress-controller-leader-nginx
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ''
+    resources:
+      - configmaps
+    verbs:
+      - create
+  - apiGroups:
+      - ''
+    resources:
+      - endpoints
+    verbs:
+      - get
+  - apiGroups:
+      - ''
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
 
 ---
 
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: RoleBinding
 metadata:
-  name: system::nginx-ingress-role-binding
+  name: system::ingress-nginx
   namespace: kube-system
   labels:
     kubernetes.io/bootstrapping: rbac-defaults
@@ -126,10 +190,10 @@ metadata:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
-  name: system::nginx-ingress-role
+  name: system::ingress-nginx
 subjects:
 - kind: ServiceAccount
-  name: nginx-ingress
+  name: ingress-nginx
   namespace: kube-system
 
 ---
@@ -137,15 +201,59 @@ subjects:
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRoleBinding
 metadata:
-  name: system:nginx-ingress
+  name: system::ingress-nginx
   labels:
     kubernetes.io/bootstrapping: rbac-defaults
     addonmanager.kubernetes.io/mode: EnsureExists
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: system:nginx-ingress
+  name: system::ingress-nginx
 subjects:
 - kind: ServiceAccount
-  name: nginx-ingress
+  name: ingress-nginx
   namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: ingress-nginx-admission
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/component: admission-webhook
+    kubernetes.io/bootstrapping: rbac-defaults
+    addonmanager.kubernetes.io/mode: EnsureExists
+  namespace: kube-system
+rules:
+  - apiGroups:
+      - ''
+    resources:
+      - secrets
+    verbs:
+      - get
+      - create
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: system::ingress-nginx-admission
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/instance: ingress-nginx
+    app.kubernetes.io/component: admission-webhook
+    kubernetes.io/bootstrapping: rbac-defaults
+    addonmanager.kubernetes.io/mode: EnsureExists
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: ingress-nginx-admission
+subjects:
+  - kind: ServiceAccount
+    name: ingress-nginx-admission
+    namespace: kube-system

test/integration/addons_test.go

@@ -91,18 +91,29 @@ func validateIngressAddon(ctx context.Context, t *testing.T, profile string) {
 		t.Fatalf("failed to get kubernetes client: %v", client)
 	}
 
-	if err := kapi.WaitForDeploymentToStabilize(client, "kube-system", "nginx-ingress-controller", Minutes(6)); err != nil {
+	if err := kapi.WaitForDeploymentToStabilize(client, "kube-system", "ingress-nginx-controller", Minutes(6)); err != nil {
 		t.Errorf("failed waiting for ingress-controller deployment to stabilize: %v", err)
 	}
-	if _, err := PodWait(ctx, t, profile, "kube-system", "app.kubernetes.io/name=nginx-ingress-controller", Minutes(12)); err != nil {
+	if _, err := PodWait(ctx, t, profile, "kube-system", "app.kubernetes.io/name=ingress-nginx", Minutes(12)); err != nil {
 		t.Fatalf("failed waititing for nginx-ingress-controller : %v", err)
 	}
 
-	rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-ing.yaml")))
+	createIngress := func() error {
-	if err != nil {
+		rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-ing.yaml")))
-		t.Errorf("failed to kubectl replace nginx-ing. args %q. %v", rr.Command(), err)
+		if err != nil {
+			return err
+		}
+		if rr.Stderr.String() != "" {
+			t.Logf("%v: unexpected stderr: %s (may be temproary)", rr.Command(), rr.Stderr)
+		}
+		return nil
 	}
-	rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-pod-svc.yaml")))
+
+	if err := retry.Expo(createIngress, 1*time.Second, Seconds(90)); err != nil {
+		t.Errorf("failed to create ingress: %v", err)
+	}
+
+	rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-pod-svc.yaml")))
 	if err != nil {
 		t.Errorf("failed to kubectl replace nginx-pod-svc. args %q. %v", rr.Command(), err)
 	}