From 422b43ece4fc1bd6e3fd86ad57c8b4f0e43b18d6 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Sat, 22 Jan 2022 23:23:58 +0000 Subject: [PATCH 1/4] fix ingress for multinode --- .../addons/ingress/ingress-deploy.yaml.tmpl | 1114 ++++++++--------- pkg/minikube/assets/addons.go | 8 +- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 10 +- .../testdata/ingress-dns-example-v1.yaml | 48 +- .../testdata/ingress-dns-example-v1beta1.yaml | 40 +- .../testdata/nginx-ingress-v1.yaml | 14 + .../testdata/nginx-ingress-v1beta1.yaml | 14 + 7 files changed, 637 insertions(+), 611 deletions(-) diff --git a/deploy/addons/ingress/ingress-deploy.yaml.tmpl b/deploy/addons/ingress/ingress-deploy.yaml.tmpl index 46ceac8bff..e8e0e1ea0a 100644 --- a/deploy/addons/ingress/ingress-deploy.yaml.tmpl +++ b/deploy/addons/ingress/ingress-deploy.yaml.tmpl @@ -12,42 +12,311 @@ # See the License for the specific language governing permissions and # limitations under the License. +//ref: https://github.com/kubernetes/ingress-nginx/blob/main/deploy/static/provider/kind/deploy.yaml + apiVersion: v1 kind: Namespace metadata: - name: ingress-nginx labels: - app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx +--- +apiVersion: v1 +automountServiceAccountToken: true +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx + namespace: ingress-nginx --- -# Source: ingress-nginx/templates/controller-serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: labels: - app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/component: admission-webhook app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/component: controller - name: ingress-nginx + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx-admission namespace: ingress-nginx -automountServiceAccountToken: true --- -# Source: ingress-nginx/templates/controller-configmap.yaml -apiVersion: v1 -kind: ConfigMap +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role metadata: labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/component: controller - name: ingress-nginx-controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx namespace: ingress-nginx +rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +- apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + {{- if eq .IngressAPIVersion "v1beta1"}} + - extensions + {{- end}} + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + {{- if eq .IngressAPIVersion "v1beta1"}} + - extensions + {{- end}} + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resourceNames: + - ingress-controller-leader + resources: + - configmaps + verbs: + - get + - update +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx-admission + namespace: ingress-nginx +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx +rules: +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + - namespaces + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + {{- if eq .IngressAPIVersion "v1beta1"}} + - extensions + {{- end}} + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + {{- if eq .IngressAPIVersion "v1beta1"}} + - extensions + {{- end}} + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx-admission +rules: +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx +subjects: +- kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx-admission + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx-admission +subjects: +- kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx +subjects: +- kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx-admission +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-admission +subjects: +- kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +apiVersion: v1 data: # see https://github.com/kubernetes/ingress-nginx/blob/master/docs/user-guide/nginx-configuration/configmap.md for all possible options and their description hsts: "false" -# see https://github.com/kubernetes/minikube/pull/12702#discussion_r727519180: 'allow-snippet-annotations' should be used only if strictly required by another part of the deployment -# allow-snippet-annotations: 'true' + # see https://github.com/kubernetes/minikube/pull/12702#discussion_r727519180: 'allow-snippet-annotations' should be used only if strictly required by another part of the deployment +# allow-snippet-annotations: "true" +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx-controller + namespace: ingress-nginx --- apiVersion: v1 kind: ConfigMap @@ -69,607 +338,328 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/component: controller --- -# Source: ingress-nginx/templates/clusterrole.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx - name: ingress-nginx -rules: - - apiGroups: - - '' - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - verbs: - - list - - watch - - apiGroups: - - '' - resources: - - nodes - verbs: - - get - - apiGroups: - - '' - resources: - - services - verbs: - - get - - list - - watch - - apiGroups: - {{- if eq .IngressAPIVersion "v1beta1"}} - - extensions - {{- end}} - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch - - apiGroups: - - '' - resources: - - events - verbs: - - create - - patch - - apiGroups: - {{- if eq .IngressAPIVersion "v1beta1"}} - - extensions - {{- end}} - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update - - apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch ---- -# Source: ingress-nginx/templates/clusterrolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx - name: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx -subjects: - - kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -# Source: ingress-nginx/templates/controller-role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/component: controller - name: ingress-nginx - namespace: ingress-nginx -rules: - - apiGroups: - - '' - resources: - - namespaces - verbs: - - get - - apiGroups: - - '' - resources: - - configmaps - - pods - - secrets - - endpoints - verbs: - - get - - list - - watch - - apiGroups: - - '' - resources: - - services - verbs: - - get - - list - - watch - - apiGroups: - {{- if eq .IngressAPIVersion "v1beta1"}} - - extensions - {{- end}} - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch - - apiGroups: - {{- if eq .IngressAPIVersion "v1beta1"}} - - extensions - {{- end}} - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update - - apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch - - apiGroups: - - '' - resources: - - configmaps - resourceNames: - - ingress-controller-leader - verbs: - - get - - update - - apiGroups: - - '' - resources: - - configmaps - verbs: - - create - - apiGroups: - - '' - resources: - - events - verbs: - - create - - patch ---- -# Source: ingress-nginx/templates/controller-rolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/component: controller - name: ingress-nginx - namespace: ingress-nginx -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx -subjects: - - kind: ServiceAccount - name: ingress-nginx - namespace: ingress-nginx ---- -# Source: ingress-nginx/templates/controller-service-webhook.yaml apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/component: controller - name: ingress-nginx-controller-admission - namespace: ingress-nginx -spec: - type: ClusterIP - ports: - - name: https-webhook - port: 443 - targetPort: webhook - {{- if eq .IngressAPIVersion "v1"}} - appProtocol: https - {{- end}} - selector: - app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/component: controller ---- -# Source: ingress-nginx/templates/controller-service.yaml -apiVersion: v1 -kind: Service -metadata: - annotations: - labels: app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/component: controller name: ingress-nginx-controller namespace: ingress-nginx spec: - type: NodePort {{- if and (eq .IngressAPIVersion "v1") (not .PreOneTwentyKubernetes)}} - ipFamilyPolicy: SingleStack ipFamilies: - - IPv4 + - IPv4 + ipFamilyPolicy: SingleStack {{- end}} ports: - - name: http - port: 80 - protocol: TCP - targetPort: http - {{- if eq .IngressAPIVersion "v1"}} - appProtocol: http - {{- end}} - - name: https - port: 443 - protocol: TCP - targetPort: https - {{- if eq .IngressAPIVersion "v1"}} - appProtocol: https - {{- end}} + - name: http + port: 80 + protocol: TCP + targetPort: http + {{- if eq .IngressAPIVersion "v1"}} + appProtocol: http + {{- end}} + - name: https + port: 443 + protocol: TCP + targetPort: https + {{- if eq .IngressAPIVersion "v1"}} + appProtocol: https + {{- end}} selector: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + type: NodePort +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx-controller-admission + namespace: ingress-nginx +spec: + ports: + - name: https-webhook + port: 443 + targetPort: webhook + {{- if eq .IngressAPIVersion "v1"}} + appProtocol: https + {{- end}} + selector: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + type: ClusterIP --- -# Source: ingress-nginx/templates/controller-deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx name: ingress-nginx-controller namespace: ingress-nginx spec: + minReadySeconds: 0 + revisionHistoryLimit: 10 selector: matchLabels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/component: controller - revisionHistoryLimit: 10 - minReadySeconds: 0 + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx gcp-auth-skip-secret: "true" spec: - dnsPolicy: ClusterFirst containers: - - name: controller - image: {{.CustomRegistries.IngressController | default .ImageRepository | default .Registries.IngressController}}{{.Images.IngressController}} - imagePullPolicy: IfNotPresent - lifecycle: - preStop: - exec: - command: - - /wait-shutdown - args: - - /nginx-ingress-controller - - --election-id=ingress-controller-leader - {{- if eq .IngressAPIVersion "v1"}} - - --controller-class=k8s.io/ingress-nginx - - --watch-ingress-without-class=true - {{- end}} - {{- if eq .IngressAPIVersion "v1beta1"}} - - --ingress-class=nginx - {{- end}} - - --publish-status-address=localhost - - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller - - --report-node-internal-ip-address - - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services - - --udp-services-configmap=$(POD_NAMESPACE)/udp-services - - --validating-webhook=:8443 - - --validating-webhook-certificate=/usr/local/certificates/cert - - --validating-webhook-key=/usr/local/certificates/key - {{- if .CustomIngressCert}} - - --default-ssl-certificate={{ .CustomIngressCert }} - {{- end}} - securityContext: - capabilities: - drop: - - ALL - add: - - NET_BIND_SERVICE - runAsUser: 101 - allowPrivilegeEscalation: true - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: LD_PRELOAD - value: /usr/local/lib/libmimalloc.so - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - readinessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - ports: - - name: http - containerPort: 80 - protocol: TCP - hostPort: 80 - - name: https - containerPort: 443 - protocol: TCP - hostPort: 443 - - name: webhook - containerPort: 8443 - protocol: TCP - volumeMounts: - - name: webhook-cert - mountPath: /usr/local/certificates/ - readOnly: true - resources: - requests: - cpu: 100m - memory: 90Mi + - args: + - /nginx-ingress-controller + - --election-id=ingress-controller-leader + {{- if eq .IngressAPIVersion "v1"}} + - --controller-class=k8s.io/ingress-nginx + - --watch-ingress-without-class=true + {{- end}} + {{- if eq .IngressAPIVersion "v1beta1"}} + - --ingress-class=nginx + {{- end}} + - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller + - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services + - --udp-services-configmap=$(POD_NAMESPACE)/udp-services + - --validating-webhook=:8443 + - --validating-webhook-certificate=/usr/local/certificates/cert + - --validating-webhook-key=/usr/local/certificates/key + - --publish-status-address=localhost + {{- if .CustomIngressCert}} + - --default-ssl-certificate={{ .CustomIngressCert }} + {{- end}} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + image: {{.CustomRegistries.IngressController | default .ImageRepository | default .Registries.IngressController}}{{.Images.IngressController}} + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: controller + ports: + - containerPort: 80 + hostPort: 80 + name: http + protocol: TCP + - containerPort: 443 + hostPort: 443 + name: https + protocol: TCP + - containerPort: 8443 + name: webhook + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 90Mi + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + runAsUser: 101 + volumeMounts: + - mountPath: /usr/local/certificates/ + name: webhook-cert + readOnly: true + dnsPolicy: ClusterFirst + nodeSelector: + minikube.k8s.io/primary: "true" + kubernetes.io/os: linux serviceAccountName: ingress-nginx + terminationGracePeriodSeconds: 0 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal volumes: - - name: webhook-cert - secret: - secretName: ingress-nginx-admission + - name: webhook-cert + secret: + secretName: ingress-nginx-admission +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx-admission-create + namespace: ingress-nginx +spec: + template: + metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx-admission-create + spec: + containers: + - args: + - create + - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=ingress-nginx-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: {{.CustomRegistries.KubeWebhookCertgenCreate | default .ImageRepository | default .Registries.KubeWebhookCertgenCreate }}{{.Images.KubeWebhookCertgenCreate}} + imagePullPolicy: IfNotPresent + name: create + securityContext: + allowPrivilegeEscalation: false + nodeSelector: + minikube.k8s.io/primary: "true" + kubernetes.io/os: linux + restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + runAsUser: 2000 + serviceAccountName: ingress-nginx-admission +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx-admission-patch + namespace: ingress-nginx +spec: + template: + metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx-admission-patch + spec: + containers: + - args: + - patch + - --webhook-name=ingress-nginx-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=ingress-nginx-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: {{.CustomRegistries.KubeWebhookCertgenPatch | default .ImageRepository | default .Registries.KubeWebhookCertgenPatch }}{{.Images.KubeWebhookCertgenPatch}} + imagePullPolicy: IfNotPresent + name: patch + securityContext: + allowPrivilegeEscalation: false + nodeSelector: + minikube.k8s.io/primary: "true" + kubernetes.io/os: linux + restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + runAsUser: 2000 + serviceAccountName: ingress-nginx-admission --- {{- if eq .IngressAPIVersion "v1"}} -# Source: ingress-nginx/templates/controller-ingressclass.yaml -# We don't support namespaced ingressClass yet -# So a ClusterRole and a ClusterRoleBinding is required apiVersion: networking.k8s.io/v1 kind: IngressClass metadata: labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx name: nginx - namespace: ingress-nginx - annotations: - ingressclass.kubernetes.io/is-default-class: "true" spec: controller: k8s.io/ingress-nginx --- {{- end}} -# Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml -# before changing this value, check the required kubernetes version -# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx name: ingress-nginx-admission webhooks: - - name: validate.nginx.ingress.kubernetes.io - matchPolicy: Equivalent - rules: - - apiGroups: - - networking.k8s.io - apiVersions: - - {{.IngressAPIVersion}} - operations: - - CREATE - - UPDATE - resources: - - ingresses - failurePolicy: Fail - sideEffects: None - admissionReviewVersions: - - v1 - {{- if eq .IngressAPIVersion "v1beta1"}} - - v1beta1 - {{- end}} - clientConfig: - service: - namespace: ingress-nginx - name: ingress-nginx-controller-admission - path: /networking/{{.IngressAPIVersion}}/ingresses ---- -# Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ingress-nginx-admission - namespace: ingress-nginx - annotations: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/component: admission-webhook ---- -# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: ingress-nginx-admission - annotations: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/component: admission-webhook -rules: +- admissionReviewVersions: + - v1 + {{- if eq .IngressAPIVersion "v1beta1"}} + - v1beta1 + {{- end}} + clientConfig: + service: + name: ingress-nginx-controller-admission + namespace: ingress-nginx + path: /networking/{{.IngressAPIVersion}}/ingresses + failurePolicy: Fail + matchPolicy: Equivalent + name: validate.nginx.ingress.kubernetes.io + rules: - apiGroups: - - admissionregistration.k8s.io + - networking.k8s.io + apiVersions: + - {{.IngressAPIVersion}} + operations: + - CREATE + - UPDATE resources: - - validatingwebhookconfigurations - verbs: - - get - - update ---- -# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: ingress-nginx-admission - annotations: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/component: admission-webhook -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ingress-nginx-admission -subjects: - - kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx ---- -# Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: ingress-nginx-admission - namespace: ingress-nginx - annotations: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/component: admission-webhook -rules: - - apiGroups: - - '' - resources: - - secrets - verbs: - - get - - create ---- -# Source: ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: ingress-nginx-admission - namespace: ingress-nginx - annotations: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/component: admission-webhook -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ingress-nginx-admission -subjects: - - kind: ServiceAccount - name: ingress-nginx-admission - namespace: ingress-nginx ---- -# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml -apiVersion: batch/v1 -kind: Job -metadata: - name: ingress-nginx-admission-create - namespace: ingress-nginx - annotations: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/component: admission-webhook -spec: - template: - metadata: - name: ingress-nginx-admission-create - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/component: admission-webhook - spec: - containers: - - name: create - image: {{.CustomRegistries.KubeWebhookCertgenCreate | default .ImageRepository | default .Registries.KubeWebhookCertgenCreate }}{{.Images.KubeWebhookCertgenCreate}} - imagePullPolicy: IfNotPresent - args: - - create - - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc - - --namespace=$(POD_NAMESPACE) - - --secret-name=ingress-nginx-admission - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission - securityContext: - runAsNonRoot: true - runAsUser: 2000 ---- -# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml -apiVersion: batch/v1 -kind: Job -metadata: - name: ingress-nginx-admission-patch - namespace: ingress-nginx - annotations: - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/component: admission-webhook -spec: - template: - metadata: - name: ingress-nginx-admission-patch - labels: - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/component: admission-webhook - spec: - containers: - - name: patch - image: {{.CustomRegistries.KubeWebhookCertgenPatch | default .ImageRepository | default .Registries.KubeWebhookCertgenPatch }}{{.Images.KubeWebhookCertgenPatch}} - imagePullPolicy: IfNotPresent - args: - - patch - - --webhook-name=ingress-nginx-admission - - --namespace=$(POD_NAMESPACE) - - --patch-mutating=false - - --secret-name=ingress-nginx-admission - - --patch-failure-policy=Fail - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - restartPolicy: OnFailure - serviceAccountName: ingress-nginx-admission - securityContext: - runAsNonRoot: true - runAsUser: 2000 + - ingresses + sideEffects: None diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index d1195234bc..368e13d6f1 100755 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -238,11 +238,11 @@ var Addons = map[string]*Addon{ "ingress-deploy.yaml", "0640"), }, false, "ingress", "", map[string]string{ - // https://github.com/kubernetes/ingress-nginx/blob/14f6b32032b709d3e0f614ca85954c3583c5fe3d/deploy/static/provider/kind/deploy.yaml#L330 - "IngressController": "ingress-nginx/controller:v1.1.0@sha256:f766669fdcf3dc26347ed273a55e754b427eb4411ee075a53f30718b4499076a", - // https://github.com/kubernetes/ingress-nginx/blob/14f6b32032b709d3e0f614ca85954c3583c5fe3d/deploy/static/provider/kind/deploy.yaml#L620 + // https://github.com/kubernetes/ingress-nginx/blob/fc38b9f2aa2d68ee00c417cf97e727b77a00c175/deploy/static/provider/kind/deploy.yaml#L331 + "IngressController": "ingress-nginx/controller:v1.1.1@sha256:0bc88eb15f9e7f84e8e56c14fa5735aaa488b840983f87bd79b1054190e660de", + // https://github.com/kubernetes/ingress-nginx/blob/fc38b9f2aa2d68ee00c417cf97e727b77a00c175/deploy/static/provider/kind/deploy.yaml#L621 "KubeWebhookCertgenCreate": "k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660", - // https://github.com/kubernetes/ingress-nginx/blob/14f6b32032b709d3e0f614ca85954c3583c5fe3d/deploy/static/provider/kind/deploy.yaml#L670 + // https://github.com/kubernetes/ingress-nginx/blob/fc38b9f2aa2d68ee00c417cf97e727b77a00c175/deploy/static/provider/kind/deploy.yaml#L673 "KubeWebhookCertgenPatch": "k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660", }, map[string]string{ "IngressController": "k8s.gcr.io", diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index c7f57b9ea5..b0f67206a5 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -835,6 +835,7 @@ func kubectlPath(cfg config.ClusterConfig) string { } // applyNodeLabels applies minikube labels to all the nodes +// but it's currently called only from kubeadm.StartCluster (via kubeadm.init) where there's only one - first node func (k *Bootstrapper) applyNodeLabels(cfg config.ClusterConfig) error { // time cluster was created. time format is based on ISO 8601 (RFC 3339) // converting - and : to _ because of Kubernetes label restriction @@ -843,12 +844,19 @@ func (k *Bootstrapper) applyNodeLabels(cfg config.ClusterConfig) error { commitLbl := "minikube.k8s.io/commit=" + version.GetGitCommitID() nameLbl := "minikube.k8s.io/name=" + cfg.Name + // ensure that "primary" label is applied only to the 1st node in the cluster (used eg for placing ingress there) + // this is used to uniquely distinguish that from other nodes in multi-master/multi-control-plane cluster config + primaryLbl := "minikube.k8s.io/primary=false" + if len(cfg.Nodes) <= 1 { + primaryLbl = "minikube.k8s.io/primary=true" + } + ctx, cancel := context.WithTimeout(context.Background(), applyTimeoutSeconds*time.Second) defer cancel() // example: // sudo /var/lib/minikube/binaries//kubectl label nodes minikube.k8s.io/version= minikube.k8s.io/commit=aa91f39ffbcf27dcbb93c4ff3f457c54e585cf4a-dirty minikube.k8s.io/name=p1 minikube.k8s.io/updated_at=2020_02_20T12_05_35_0700 --all --overwrite --kubeconfig=/var/lib/minikube/kubeconfig cmd := exec.CommandContext(ctx, "sudo", kubectlPath(cfg), - "label", "nodes", verLbl, commitLbl, nameLbl, createdAtLbl, "--all", "--overwrite", + "label", "nodes", verLbl, commitLbl, nameLbl, createdAtLbl, primaryLbl, "--all", "--overwrite", fmt.Sprintf("--kubeconfig=%s", path.Join(vmpath.GuestPersistentDir, "kubeconfig"))) if _, err := k.c.RunCmd(cmd); err != nil { diff --git a/test/integration/testdata/ingress-dns-example-v1.yaml b/test/integration/testdata/ingress-dns-example-v1.yaml index a4f7d196ec..e67d8cec1b 100644 --- a/test/integration/testdata/ingress-dns-example-v1.yaml +++ b/test/integration/testdata/ingress-dns-example-v1.yaml @@ -28,10 +28,10 @@ spec: app: hello-world-app spec: containers: - - name: hello-world-app - image: gcr.io/google-samples/hello-app:1.0 - ports: - - containerPort: 8080 + - name: hello-world-app + image: gcr.io/google-samples/hello-app:1.0 + ports: + - containerPort: 8080 --- apiVersion: networking.k8s.io/v1 kind: Ingress @@ -41,26 +41,26 @@ metadata: spec: ingressClassName: nginx rules: - - host: hello-john.test - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: hello-world-app - port: - number: 80 - - host: hello-jane.test - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: hello-world-app - port: - number: 80 + - host: hello-john.test + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: hello-world-app + port: + number: 80 + - host: hello-jane.test + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: hello-world-app + port: + number: 80 --- apiVersion: v1 kind: Service diff --git a/test/integration/testdata/ingress-dns-example-v1beta1.yaml b/test/integration/testdata/ingress-dns-example-v1beta1.yaml index 09f2817df4..d2fa35d916 100644 --- a/test/integration/testdata/ingress-dns-example-v1beta1.yaml +++ b/test/integration/testdata/ingress-dns-example-v1beta1.yaml @@ -28,10 +28,10 @@ spec: app: hello-world-app spec: containers: - - name: hello-world-app - image: gcr.io/google-samples/hello-app:1.0 - ports: - - containerPort: 8080 + - name: hello-world-app + image: gcr.io/google-samples/hello-app:1.0 + ports: + - containerPort: 8080 --- apiVersion: networking.k8s.io/v1beta1 kind: Ingress @@ -43,22 +43,22 @@ metadata: kubernetes.io/ingress.class: nginx spec: rules: - - host: hello-john.test - http: - paths: - - path: / - pathType: Prefix - backend: - serviceName: hello-world-app - servicePort: 80 - - host: hello-jane.test - http: - paths: - - path: / - pathType: Prefix - backend: - serviceName: hello-world-app - servicePort: 80 + - host: hello-john.test + http: + paths: + - path: / + pathType: Prefix + backend: + serviceName: hello-world-app + servicePort: 80 + - host: hello-jane.test + http: + paths: + - path: / + pathType: Prefix + backend: + serviceName: hello-world-app + servicePort: 80 --- apiVersion: v1 kind: Service diff --git a/test/integration/testdata/nginx-ingress-v1.yaml b/test/integration/testdata/nginx-ingress-v1.yaml index e5c6766b05..0db2ad2286 100644 --- a/test/integration/testdata/nginx-ingress-v1.yaml +++ b/test/integration/testdata/nginx-ingress-v1.yaml @@ -1,3 +1,17 @@ +# Copyright 2022 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + apiVersion: networking.k8s.io/v1 kind: Ingress metadata: diff --git a/test/integration/testdata/nginx-ingress-v1beta1.yaml b/test/integration/testdata/nginx-ingress-v1beta1.yaml index 5e3fcc2350..12ebaa764d 100644 --- a/test/integration/testdata/nginx-ingress-v1beta1.yaml +++ b/test/integration/testdata/nginx-ingress-v1beta1.yaml @@ -1,3 +1,17 @@ +# Copyright 2022 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: From 2be6b99d79651f6353cdd1543684bf868bea904b Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Sun, 23 Jan 2022 01:00:41 +0000 Subject: [PATCH 2/4] remove 'publish-status-address=localhost' and fix comment --- deploy/addons/ingress/ingress-deploy.yaml.tmpl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy/addons/ingress/ingress-deploy.yaml.tmpl b/deploy/addons/ingress/ingress-deploy.yaml.tmpl index e8e0e1ea0a..85be5784d6 100644 --- a/deploy/addons/ingress/ingress-deploy.yaml.tmpl +++ b/deploy/addons/ingress/ingress-deploy.yaml.tmpl @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -//ref: https://github.com/kubernetes/ingress-nginx/blob/main/deploy/static/provider/kind/deploy.yaml +# ref: https://github.com/kubernetes/ingress-nginx/blob/main/deploy/static/provider/kind/deploy.yaml apiVersion: v1 kind: Namespace @@ -443,7 +443,6 @@ spec: - --validating-webhook=:8443 - --validating-webhook-certificate=/usr/local/certificates/cert - --validating-webhook-key=/usr/local/certificates/key - - --publish-status-address=localhost {{- if .CustomIngressCert}} - --default-ssl-certificate={{ .CustomIngressCert }} {{- end}} From 9048b0b91a5ae8608de789b8be08a410a2d41511 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Sun, 23 Jan 2022 01:16:32 +0000 Subject: [PATCH 3/4] add 'is-default-class' --- deploy/addons/ingress/ingress-deploy.yaml.tmpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/addons/ingress/ingress-deploy.yaml.tmpl b/deploy/addons/ingress/ingress-deploy.yaml.tmpl index 85be5784d6..b90b1a85ea 100644 --- a/deploy/addons/ingress/ingress-deploy.yaml.tmpl +++ b/deploy/addons/ingress/ingress-deploy.yaml.tmpl @@ -625,6 +625,8 @@ metadata: app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx name: nginx + annotations: + ingressclass.kubernetes.io/is-default-class: "true" spec: controller: k8s.io/ingress-nginx --- From 75a63be3c8ed71f3c9522a3bb940f2ceca2e7fcb Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Sun, 23 Jan 2022 01:24:57 +0000 Subject: [PATCH 4/4] add 'is-default-class' --- deploy/addons/ingress/ingress-deploy.yaml.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/addons/ingress/ingress-deploy.yaml.tmpl b/deploy/addons/ingress/ingress-deploy.yaml.tmpl index b90b1a85ea..170b4bc691 100644 --- a/deploy/addons/ingress/ingress-deploy.yaml.tmpl +++ b/deploy/addons/ingress/ingress-deploy.yaml.tmpl @@ -620,13 +620,13 @@ spec: apiVersion: networking.k8s.io/v1 kind: IngressClass metadata: + annotations: + ingressclass.kubernetes.io/is-default-class: "true" labels: app.kubernetes.io/component: controller app.kubernetes.io/instance: ingress-nginx app.kubernetes.io/name: ingress-nginx name: nginx - annotations: - ingressclass.kubernetes.io/is-default-class: "true" spec: controller: k8s.io/ingress-nginx ---