From 3de8a8d5aa5910ae2df154a261a463e33b4c16d0 Mon Sep 17 00:00:00 2001 From: Ian Stewart Date: Thu, 24 Nov 2022 20:55:44 -0500 Subject: [PATCH 01/12] Moved image pull secret creation code to the webhook --- .../gcp-auth/gcp-auth-webhook.yaml.tmpl | 18 +++ pkg/addons/addons_gcpauth.go | 127 +++++------------- 2 files changed, 50 insertions(+), 95 deletions(-) diff --git a/deploy/addons/gcp-auth/gcp-auth-webhook.yaml.tmpl b/deploy/addons/gcp-auth/gcp-auth-webhook.yaml.tmpl index dbac1650b7..d1afa12549 100644 --- a/deploy/addons/gcp-auth/gcp-auth-webhook.yaml.tmpl +++ b/deploy/addons/gcp-auth/gcp-auth-webhook.yaml.tmpl @@ -38,6 +38,13 @@ rules: verbs: - get - update + - apiGroups: + - '' + resources: + - namespaces + verbs: + - list + - watch --- apiVersion: rbac.authorization.k8s.io/v1 @@ -92,10 +99,14 @@ spec: app: gcp-auth kubernetes.io/minikube-addons: gcp-auth spec: + serviceAccountName: minikube-gcp-auth-certs containers: - name: gcp-auth image: {{.CustomRegistries.GCPAuthWebhook | default .ImageRepository | default .Registries.GCPAuthWebhook}}{{.Images.GCPAuthWebhook}} imagePullPolicy: IfNotPresent + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /google-app-creds.json ports: - containerPort: 8443 volumeMounts: @@ -105,6 +116,9 @@ spec: - name: gcp-project mountPath: /var/lib/minikube/google_cloud_project readOnly: true + - name: gcp-creds + mountPath: /google-app-creds.json + readOnly: true volumes: - name: webhook-certs secret: @@ -113,6 +127,10 @@ spec: hostPath: path: /var/lib/minikube/google_cloud_project type: File + - name: gcp-creds + hostPath: + path: /var/lib/minikube/google_application_credentials.json + type: File --- apiVersion: batch/v1 kind: Job diff --git a/pkg/addons/addons_gcpauth.go b/pkg/addons/addons_gcpauth.go index 697c104d01..da9e67b4bd 100644 --- a/pkg/addons/addons_gcpauth.go +++ b/pkg/addons/addons_gcpauth.go @@ -19,15 +19,12 @@ package addons import ( "bytes" "context" - "fmt" "os" "os/exec" "path" "strconv" - "strings" "time" - gcr_config "github.com/GoogleCloudPlatform/docker-credential-gcr/config" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -88,9 +85,9 @@ func enableAddonGCPAuth(cfg *config.ClusterConfig) error { } } - // Create a registry secret in every namespace we can find - // Always create the pull secret, no matter where we are - if err := createPullSecret(cfg, creds); err != nil { + // Patch service accounts for all namespaces to include the image pull secret. + // The image registry pull secret is added to the namespaces in the webhook. + if err := patchServiceAccounts(cfg); err != nil { return errors.Wrap(err, "pull secret") } @@ -101,7 +98,7 @@ func enableAddonGCPAuth(cfg *config.ClusterConfig) error { } if creds.JSON == nil { - out.WarningT("You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue. The image pull secret has been imported.") + out.WarningT("You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue.") return nil } @@ -139,110 +136,50 @@ or set the GOOGLE_CLOUD_PROJECT environment variable.`) } -func createPullSecret(cc *config.ClusterConfig, creds *google.Credentials) error { - if creds == nil { - return errors.New("no credentials, skipping creating pull secret") +func patchServiceAccounts(cc *config.ClusterConfig) error { + client, err := service.K8s.GetCoreClient(cc.Name) + if err != nil { + return err } - token, err := creds.TokenSource.Token() - // Only try to add secret if Token was found - if err == nil { - client, err := service.K8s.GetCoreClient(cc.Name) + namespaces, err := client.Namespaces().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return err + } + + for _, n := range namespaces.Items { + // Now patch the secret into all the service accounts we can find + serviceaccounts := client.ServiceAccounts(n.Name) + salist, err := serviceaccounts.List(context.TODO(), metav1.ListOptions{}) if err != nil { return err } - namespaces, err := client.Namespaces().List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return err - } - - var dockercfg string - registries := append(gcr_config.DefaultGCRRegistries[:], gcr_config.DefaultARRegistries[:]...) - for _, reg := range registries { - dockercfg += fmt.Sprintf(`"https://%s":{"username":"oauth2accesstoken","password":"%s","email":"none"},`, reg, token.AccessToken) - } - - dockercfg = strings.TrimSuffix(dockercfg, ",") - - data := map[string][]byte{ - ".dockercfg": []byte(fmt.Sprintf(`{%s}`, dockercfg)), - } - - for _, n := range namespaces.Items { - if skipNamespace(n.Name) { - continue - } - secrets := client.Secrets(n.Name) - - exists := false - secList, err := secrets.List(context.TODO(), metav1.ListOptions{}) + // Let's make sure we at least find the default service account + for len(salist.Items) == 0 { + salist, err = serviceaccounts.List(context.TODO(), metav1.ListOptions{}) if err != nil { return err } - for _, s := range secList.Items { - if s.Name == secretName { - exists = true + time.Sleep(1 * time.Second) + } + + ips := corev1.LocalObjectReference{Name: secretName} + for _, sa := range salist.Items { + add := true + for _, ps := range sa.ImagePullSecrets { + if ps.Name == secretName { + add = false break } } - - if !exists || Refresh { - secretObj := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - }, - Data: data, - Type: "kubernetes.io/dockercfg", - } - - if exists && Refresh { - _, err := secrets.Update(context.TODO(), secretObj, metav1.UpdateOptions{}) - if err != nil { - return err - } - } else { - _, err = secrets.Create(context.TODO(), secretObj, metav1.CreateOptions{}) - if err != nil { - return err - } - } - } - - // Now patch the secret into all the service accounts we can find - serviceaccounts := client.ServiceAccounts(n.Name) - salist, err := serviceaccounts.List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return err - } - - // Let's make sure we at least find the default service account - for len(salist.Items) == 0 { - salist, err = serviceaccounts.List(context.TODO(), metav1.ListOptions{}) + if add { + sa.ImagePullSecrets = append(sa.ImagePullSecrets, ips) + _, err := serviceaccounts.Update(context.TODO(), &sa, metav1.UpdateOptions{}) if err != nil { return err } - time.Sleep(1 * time.Second) } - - ips := corev1.LocalObjectReference{Name: secretName} - for _, sa := range salist.Items { - add := true - for _, ps := range sa.ImagePullSecrets { - if ps.Name == secretName { - add = false - break - } - } - if add { - sa.ImagePullSecrets = append(sa.ImagePullSecrets, ips) - _, err := serviceaccounts.Update(context.TODO(), &sa, metav1.UpdateOptions{}) - if err != nil { - return err - } - } - } - } } return nil From 64d0c4eec016bf1438485f07cf133cea89978adc Mon Sep 17 00:00:00 2001 From: Ian Stewart Date: Tue, 29 Nov 2022 15:46:54 -0500 Subject: [PATCH 02/12] updated GCPAuthWebhook image to v0.0.12 --- pkg/minikube/assets/addons.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index 9e26f58cfe..a614cfa41a 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -577,7 +577,7 @@ var Addons = map[string]*Addon{ "0640"), }, false, "gcp-auth", "Google", "", "https://minikube.sigs.k8s.io/docs/handbook/addons/gcp-auth/", map[string]string{ "KubeWebhookCertgen": "ingress-nginx/kube-webhook-certgen:v1.0@sha256:f3b6b39a6062328c095337b4cadcefd1612348fdd5190b1dcbcb9b9e90bd8068", - "GCPAuthWebhook": "k8s-minikube/gcp-auth-webhook:v0.0.11@sha256:82efb346863dc47701586bebadd4cef998d4c6692d802ec3de68d451c87fb613", + "GCPAuthWebhook": "k8s-minikube/gcp-auth-webhook:v0.0.12@sha256:0f2df44dd380376ff32144390b94283b52b0be00c1c1f117803578dd2c5dcadd", }, map[string]string{ "GCPAuthWebhook": "gcr.io", "KubeWebhookCertgen": "k8s.gcr.io", From 6ee591c1f3bb5377b49b7d9b51d9625829946fc2 Mon Sep 17 00:00:00 2001 From: Ian Stewart Date: Tue, 29 Nov 2022 15:54:49 -0500 Subject: [PATCH 03/12] addressed PR comments --- pkg/addons/addons_gcpauth.go | 2 +- translations/de.json | 2 +- translations/es.json | 2 +- translations/fr.json | 1 + translations/ja.json | 1 + translations/ko.json | 2 +- translations/pl.json | 2 +- translations/ru.json | 2 +- translations/strings.txt | 2 +- translations/zh-CN.json | 2 +- 10 files changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/addons/addons_gcpauth.go b/pkg/addons/addons_gcpauth.go index da9e67b4bd..3af84d62e4 100644 --- a/pkg/addons/addons_gcpauth.go +++ b/pkg/addons/addons_gcpauth.go @@ -88,7 +88,7 @@ func enableAddonGCPAuth(cfg *config.ClusterConfig) error { // Patch service accounts for all namespaces to include the image pull secret. // The image registry pull secret is added to the namespaces in the webhook. if err := patchServiceAccounts(cfg); err != nil { - return errors.Wrap(err, "pull secret") + return errors.Wrap(err, "patching service accounts") } // If the env var is explicitly set, even in GCE, then defer to the user and continue diff --git a/translations/de.json b/translations/de.json index d56b610c8b..a8beebb309 100644 --- a/translations/de.json +++ b/translations/de.json @@ -899,7 +899,7 @@ "You cannot change the CPUs for an existing minikube cluster. Please first delete the cluster.": "Die Anzahl der CPUs eines existierenden Minikube Clusters kann nicht geändert werden. Bitte löschen Sie den Cluster zuerst.", "You cannot change the disk size for an existing minikube cluster. Please first delete the cluster.": "Die Plattengröße eines existierenden Minikube Clusters kann nicht geändert werden. Bitte löschen Sie den Cluster zuerst.", "You cannot change the memory size for an existing minikube cluster. Please first delete the cluster.": "Die Speichergröße eines existierenden Minikube Clusters kann nicht geändert werden. Bitte löschen Sie den Cluster zuerst.", - "You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue. The image pull secret has been imported.": "", + "You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue.": "", "You have authenticated with a service account that does not have an associated JSON. The GCP Auth requires credentials with a JSON file to in order to continue. The image pull secret has been imported.": "Sie haben sich mit einem Service Account authentifiziert, welcher kein zugehöriges JSON besitzt. GCP Auth benötigt Zugangsdaten in einer JSON-Datei um weitermachen zu können. Das Image Pull Secret wurde importiert.", "You have chosen to disable the CNI but the \"{{.name}}\" container runtime requires CNI": "Sie haben den CNI Treiber deaktiviert, aber die \"{{.name}}\" Container Laufzeitumgebung benötigt ein CNI", "You have selected \"virtualbox\" driver, but there are better options !\nFor better performance and support consider using a different driver: {{.drivers}}\n\nTo turn off this warning run:\n\n\t$ minikube config set WantVirtualBoxDriverWarning false\n\n\nTo learn more about on minikube drivers checkout https://minikube.sigs.k8s.io/docs/drivers/\nTo see benchmarks checkout https://minikube.sigs.k8s.io/docs/benchmarks/cpuusage/\n\n": "Sie haben den \"virtualbox\" Treiber ausgewählt, aber es existieren bessere Möglichkeiten !\nFür eine bessere Performanz und besseren Support erwägen Sie die Verwendung eines anderen Treibers: {{.drivers}}\n\nUm diese Warnung zu deaktivieren, führen Sie folgendes aus:\n\n\t$ minikube config set WantVirtualBoxDriverWarning false\n\n\nUm mehr über die Minikube-Treiber zu erfahren, lesen Sie https://minikube.sigs.k8s.io/docs/drivers/\nZu Benchmarks lesen Sie https://minikube.sigs.k8s.io/docs/benchmarks/cpuusage/\n\n", diff --git a/translations/es.json b/translations/es.json index cc13400ba1..f5db97ef40 100644 --- a/translations/es.json +++ b/translations/es.json @@ -899,7 +899,7 @@ "You cannot change the CPUs for an existing minikube cluster. Please first delete the cluster.": "", "You cannot change the disk size for an existing minikube cluster. Please first delete the cluster.": "", "You cannot change the memory size for an existing minikube cluster. Please first delete the cluster.": "", - "You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue. The image pull secret has been imported.": "", + "You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue.": "", "You have chosen to disable the CNI but the \"{{.name}}\" container runtime requires CNI": "", "You have selected \"virtualbox\" driver, but there are better options !\nFor better performance and support consider using a different driver: {{.drivers}}\n\nTo turn off this warning run:\n\n\t$ minikube config set WantVirtualBoxDriverWarning false\n\n\nTo learn more about on minikube drivers checkout https://minikube.sigs.k8s.io/docs/drivers/\nTo see benchmarks checkout https://minikube.sigs.k8s.io/docs/benchmarks/cpuusage/\n\n": "", "You may need to manually remove the \"{{.name}}\" VM from your hypervisor": "Puede que tengas que retirar manualmente la VM \"{{.name}}\" de tu hipervisor", diff --git a/translations/fr.json b/translations/fr.json index 6158c40dc4..166866ea77 100644 --- a/translations/fr.json +++ b/translations/fr.json @@ -872,6 +872,7 @@ "You cannot change the CPUs for an existing minikube cluster. Please first delete the cluster.": "Vous ne pouvez pas modifier les processeurs d'un cluster minikube existant. Veuillez d'abord supprimer le cluster.", "You cannot change the disk size for an existing minikube cluster. Please first delete the cluster.": "Vous ne pouvez pas modifier la taille du disque pour un cluster minikube existant. Veuillez d'abord supprimer le cluster.", "You cannot change the memory size for an existing minikube cluster. Please first delete the cluster.": "Vous ne pouvez pas modifier la taille de la mémoire d'un cluster minikube existant. Veuillez d'abord supprimer le cluster.", + "You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue.": "", "You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue. The image pull secret has been imported.": "Vous vous êtes authentifié avec un compte de service qui n'a pas de fichier JSON associé. Le module complémentaire GCP Auth nécessite des informations d'identification avec un fichier JSON pour continuer. Le secret d'extraction d'image a été importé.", "You have authenticated with a service account that does not have an associated JSON. The GCP Auth requires credentials with a JSON file in order to continue. The image pull secret has been imported.": "Vous vous êtes authentifié avec un compte de service qui n'a pas de fichier JSON associé. L'authentification GCP nécessite des informations d'identification avec un fichier JSON pour continuer. Le secret d'extraction d'image a été importé.", "You have authenticated with a service account that does not have an associated JSON. The GCP Auth requires credentials with a JSON file to in order to continue. The image pull secret has been imported.": "Vous vous êtes authentifié avec un compte de service qui n'a pas de fichier JSON associé. L'authentification GCP nécessite des informations d'identification avec un fichier JSON pour continuer. Le secret d'extraction d'image a été importé.", diff --git a/translations/ja.json b/translations/ja.json index d4ffdf4a4e..644d1c018e 100644 --- a/translations/ja.json +++ b/translations/ja.json @@ -835,6 +835,7 @@ "You cannot change the CPUs for an existing minikube cluster. Please first delete the cluster.": "既存の minikube クラスターに対して、CPU を変更できません。最初にクラスターを削除してください。", "You cannot change the disk size for an existing minikube cluster. Please first delete the cluster.": "既存の minikube クラスターに対して、ディスクサイズを変更できません。最初にクラスターを削除してください。", "You cannot change the memory size for an existing minikube cluster. Please first delete the cluster.": "既存の minikube クラスターに対して、メモリサイズを変更できません。最初にクラスターを削除してください。", + "You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue.": "", "You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue. The image pull secret has been imported.": "関連する JSON ファイルがないサービスアカウントで認証しています。GCP Auth アドオンは、作業を続行するために JSON ファイル付きクレデンシャルを要求します。イメージ取得シークレットがインポートされました。", "You have chosen to disable the CNI but the \"{{.name}}\" container runtime requires CNI": "CNI 無効が選択されましたが、「{{.name}}」コンテナランタイムは CNI が必要です", "You have selected \"virtualbox\" driver, but there are better options !\nFor better performance and support consider using a different driver: {{.drivers}}\n\nTo turn off this warning run:\n\n\t$ minikube config set WantVirtualBoxDriverWarning false\n\n\nTo learn more about on minikube drivers checkout https://minikube.sigs.k8s.io/docs/drivers/\nTo see benchmarks checkout https://minikube.sigs.k8s.io/docs/benchmarks/cpuusage/\n\n": "「virtualbox」ドライバーが選択されましたが、より良い選択肢があります！\n性能と機能の向上のため、別のドライバー使用を検討してください: {{.drivers}}\n\nこの警告を表示させないためには、以下を実行してください:\n\n\t$ minikube config set WantVirtualBoxDriverWarning false\n\n\nminikube ドライバーについてもっと知るためには、https://minikube.sigs.k8s.io/docs/drivers/ を確認してください。\nベンチマークについては https://minikube.sigs.k8s.io/docs/benchmarks/cpuusage/ を確認してください\n\n", diff --git a/translations/ko.json b/translations/ko.json index 749d176eb1..3b537d72ab 100644 --- a/translations/ko.json +++ b/translations/ko.json @@ -898,7 +898,7 @@ "You cannot change the CPUs for an existing minikube cluster. Please first delete the cluster.": "", "You cannot change the disk size for an existing minikube cluster. Please first delete the cluster.": "", "You cannot change the memory size for an existing minikube cluster. Please first delete the cluster.": "", - "You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue. The image pull secret has been imported.": "", + "You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue.": "", "You have chosen to disable the CNI but the \"{{.name}}\" container runtime requires CNI": "", "You have selected \"virtualbox\" driver, but there are better options !\nFor better performance and support consider using a different driver: {{.drivers}}\n\nTo turn off this warning run:\n\n\t$ minikube config set WantVirtualBoxDriverWarning false\n\n\nTo learn more about on minikube drivers checkout https://minikube.sigs.k8s.io/docs/drivers/\nTo see benchmarks checkout https://minikube.sigs.k8s.io/docs/benchmarks/cpuusage/\n\n": "", "You may need to manually remove the \"{{.name}}\" VM from your hypervisor": "", diff --git a/translations/pl.json b/translations/pl.json index fb7792115b..5de73100b0 100644 --- a/translations/pl.json +++ b/translations/pl.json @@ -910,7 +910,7 @@ "You cannot change the CPUs for an existing minikube cluster. Please first delete the cluster.": "", "You cannot change the disk size for an existing minikube cluster. Please first delete the cluster.": "", "You cannot change the memory size for an existing minikube cluster. Please first delete the cluster.": "", - "You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue. The image pull secret has been imported.": "", + "You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue.": "", "You have chosen to disable the CNI but the \"{{.name}}\" container runtime requires CNI": "", "You have selected \"virtualbox\" driver, but there are better options !\nFor better performance and support consider using a different driver: {{.drivers}}\n\nTo turn off this warning run:\n\n\t$ minikube config set WantVirtualBoxDriverWarning false\n\n\nTo learn more about on minikube drivers checkout https://minikube.sigs.k8s.io/docs/drivers/\nTo see benchmarks checkout https://minikube.sigs.k8s.io/docs/benchmarks/cpuusage/\n\n": "", "You may need to manually remove the \"{{.name}}\" VM from your hypervisor": "", diff --git a/translations/ru.json b/translations/ru.json index ba55142afc..a8aa5afa00 100644 --- a/translations/ru.json +++ b/translations/ru.json @@ -831,7 +831,7 @@ "You cannot change the CPUs for an existing minikube cluster. Please first delete the cluster.": "", "You cannot change the disk size for an existing minikube cluster. Please first delete the cluster.": "", "You cannot change the memory size for an existing minikube cluster. Please first delete the cluster.": "", - "You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue. The image pull secret has been imported.": "", + "You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue.": "", "You have chosen to disable the CNI but the \"{{.name}}\" container runtime requires CNI": "", "You have selected \"virtualbox\" driver, but there are better options !\nFor better performance and support consider using a different driver: {{.drivers}}\n\nTo turn off this warning run:\n\n\t$ minikube config set WantVirtualBoxDriverWarning false\n\n\nTo learn more about on minikube drivers checkout https://minikube.sigs.k8s.io/docs/drivers/\nTo see benchmarks checkout https://minikube.sigs.k8s.io/docs/benchmarks/cpuusage/\n\n": "", "You may need to manually remove the \"{{.name}}\" VM from your hypervisor": "", diff --git a/translations/strings.txt b/translations/strings.txt index 8a15801cd5..0f91e7de62 100644 --- a/translations/strings.txt +++ b/translations/strings.txt @@ -831,7 +831,7 @@ "You cannot change the CPUs for an existing minikube cluster. Please first delete the cluster.": "", "You cannot change the disk size for an existing minikube cluster. Please first delete the cluster.": "", "You cannot change the memory size for an existing minikube cluster. Please first delete the cluster.": "", - "You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue. The image pull secret has been imported.": "", + "You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue.": "", "You have chosen to disable the CNI but the \"{{.name}}\" container runtime requires CNI": "", "You have selected \"virtualbox\" driver, but there are better options !\nFor better performance and support consider using a different driver: {{.drivers}}\n\nTo turn off this warning run:\n\n\t$ minikube config set WantVirtualBoxDriverWarning false\n\n\nTo learn more about on minikube drivers checkout https://minikube.sigs.k8s.io/docs/drivers/\nTo see benchmarks checkout https://minikube.sigs.k8s.io/docs/benchmarks/cpuusage/\n\n": "", "You may need to manually remove the \"{{.name}}\" VM from your hypervisor": "", diff --git a/translations/zh-CN.json b/translations/zh-CN.json index 263f7408e5..9ef08d2374 100644 --- a/translations/zh-CN.json +++ b/translations/zh-CN.json @@ -1018,7 +1018,7 @@ "You cannot change the CPUs for an existing minikube cluster. Please first delete the cluster.": "", "You cannot change the disk size for an existing minikube cluster. Please first delete the cluster.": "", "You cannot change the memory size for an existing minikube cluster. Please first delete the cluster.": "", - "You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue. The image pull secret has been imported.": "", + "You have authenticated with a service account that does not have an associated JSON file. The GCP Auth addon requires credentials with a JSON file in order to continue.": "", "You have chosen to disable the CNI but the \"{{.name}}\" container runtime requires CNI": "", "You have selected \"virtualbox\" driver, but there are better options !\nFor better performance and support consider using a different driver: {{.drivers}}\n\nTo turn off this warning run:\n\n\t$ minikube config set WantVirtualBoxDriverWarning false\n\n\nTo learn more about on minikube drivers checkout https://minikube.sigs.k8s.io/docs/drivers/\nTo see benchmarks checkout https://minikube.sigs.k8s.io/docs/benchmarks/cpuusage/\n\n": "", "You may need to manually remove the \"{{.name}}\" VM from your hypervisor": "您可能需要从管理程序中手动移除“{{.name}}”虚拟机", From bec22f0e129c619c365f68e912f5000e59f1e832 Mon Sep 17 00:00:00 2001 From: Ian Stewart Date: Wed, 30 Nov 2022 18:50:32 -0500 Subject: [PATCH 04/12] added integration test to ensure new namespaces include the image pull secret --- test/integration/addons_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index c879047634..c919abe371 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -598,10 +598,25 @@ func validateCSIDriverAndSnapshots(ctx context.Context, t *testing.T, profile st } } +// validateGCPAuthNamespaces validates that newly created namespaces contain the gcp-auth secret. +func validateGCPAuthNamespaces(ctx context.Context, t *testing.T, profile string) { + rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "ns", "new-namespace")) + if err != nil { + t.Fatalf("%s failed: %v", rr.Command(), err) + } + + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "secret", "gcp-auth", "-n", "new-namespace")) + if err != nil { + t.Errorf("%s failed: %v", rr.Command(), err) + } +} + // validateGCPAuthAddon tests the GCP Auth addon with either phony or real credentials and makes sure the files are mounted into pods correctly func validateGCPAuthAddon(ctx context.Context, t *testing.T, profile string) { defer PostMortemLogs(t, profile) + validateGCPAuthNamespaces(ctx, t, profile) + // schedule a pod to check environment variables rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "busybox.yaml"))) if err != nil { From 2b93c08c5a1365289855ac64a3ca88afa8135883 Mon Sep 17 00:00:00 2001 From: Ian Stewart Date: Thu, 1 Dec 2022 17:24:41 -0500 Subject: [PATCH 05/12] wrapped gcp-auth namespaces test as a subtest --- test/integration/addons_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index c919abe371..31e3371032 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -615,7 +615,9 @@ func validateGCPAuthNamespaces(ctx context.Context, t *testing.T, profile string func validateGCPAuthAddon(ctx context.Context, t *testing.T, profile string) { defer PostMortemLogs(t, profile) - validateGCPAuthNamespaces(ctx, t, profile) + t.Run("Namespaces", func(t *testing.T) { + validateGCPAuthNamespaces(ctx, t, profile) + }) // schedule a pod to check environment variables rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "busybox.yaml"))) From d2a94988eedf4121e0e513c607cc8fe8c74c548f Mon Sep 17 00:00:00 2001 From: Ian Stewart Date: Thu, 1 Dec 2022 17:42:38 -0500 Subject: [PATCH 06/12] retry getting the gcp-auth secret for 10 seconds --- test/integration/addons_test.go | 13 ++++++++++--- test/integration/main_test.go | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 31e3371032..8b5dabdf55 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -605,9 +605,16 @@ func validateGCPAuthNamespaces(ctx context.Context, t *testing.T, profile string t.Fatalf("%s failed: %v", rr.Command(), err) } - rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "secret", "gcp-auth", "-n", "new-namespace")) - if err != nil { - t.Errorf("%s failed: %v", rr.Command(), err) + getSecret := func() error { + _, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "secret", "gcp-auth", "-n", "new-namespace")) + if err != nil { + return err + } + return nil + } + + if err := retry.Expo(getSecret, Seconds(1), Seconds(10)); err != nil { + t.Errorf("failed to get secret: %v", err) } } diff --git a/test/integration/main_test.go b/test/integration/main_test.go index 8863c7f105..2fe79de386 100644 --- a/test/integration/main_test.go +++ b/test/integration/main_test.go @@ -186,7 +186,7 @@ func Minutes(n int) time.Duration { return time.Duration(*timeOutMultiplier) * time.Duration(n) * time.Minute } -// Seconds will return timeout in minutes based on how slow the machine is +// Seconds will return timeout in seconds based on how slow the machine is func Seconds(n int) time.Duration { return time.Duration(*timeOutMultiplier) * time.Duration(n) * time.Second } From 2c3ab40370c9ba0b1eaba46bc7bfb0c83713f352 Mon Sep 17 00:00:00 2001 From: Ian Stewart Date: Fri, 2 Dec 2022 12:09:23 -0500 Subject: [PATCH 07/12] increased wait time for getting namespace secret --- test/integration/addons_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 8b5dabdf55..4f8dec0414 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -613,7 +613,7 @@ func validateGCPAuthNamespaces(ctx context.Context, t *testing.T, profile string return nil } - if err := retry.Expo(getSecret, Seconds(1), Seconds(10)); err != nil { + if err := retry.Expo(getSecret, Minutes(2), Minutes(10)); err != nil { t.Errorf("failed to get secret: %v", err) } } From a2ba24f1d649541bc515d6582a47adc3bad3535c Mon Sep 17 00:00:00 2001 From: Ian Stewart Date: Mon, 5 Dec 2022 11:34:40 -0500 Subject: [PATCH 08/12] returning gcp-auth container logs if namespace test fails to help debugging --- test/integration/addons_test.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 4f8dec0414..da3383ca82 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -22,6 +22,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "net/http" "net/url" @@ -605,15 +606,23 @@ func validateGCPAuthNamespaces(ctx context.Context, t *testing.T, profile string t.Fatalf("%s failed: %v", rr.Command(), err) } - getSecret := func() error { - _, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "secret", "gcp-auth", "-n", "new-namespace")) + getLogs := func() error { + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "logs", "-l", "app=gcp-auth", "-n", "gcp-auth")) if err != nil { return err } - return nil + return errors.New(rr.Output()) } - if err := retry.Expo(getSecret, Minutes(2), Minutes(10)); err != nil { + getSecret := func() error { + _, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "secret", "gcp-auth", "-n", "new-namespace")) + if err != nil { + err = getLogs() + } + return err + } + + if err := retry.Expo(getSecret, Seconds(2), Minutes(1)); err != nil { t.Errorf("failed to get secret: %v", err) } } From b76d247f12de7b2909b5e250107dbdacf5055461 Mon Sep 17 00:00:00 2001 From: Ian Stewart Date: Wed, 7 Dec 2022 18:05:31 -0500 Subject: [PATCH 09/12] added MOCK_GOOGLE_TOKEN env var --- pkg/minikube/assets/addons.go | 2 +- test/integration/addons_test.go | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index a614cfa41a..e8e9187e70 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -577,7 +577,7 @@ var Addons = map[string]*Addon{ "0640"), }, false, "gcp-auth", "Google", "", "https://minikube.sigs.k8s.io/docs/handbook/addons/gcp-auth/", map[string]string{ "KubeWebhookCertgen": "ingress-nginx/kube-webhook-certgen:v1.0@sha256:f3b6b39a6062328c095337b4cadcefd1612348fdd5190b1dcbcb9b9e90bd8068", - "GCPAuthWebhook": "k8s-minikube/gcp-auth-webhook:v0.0.12@sha256:0f2df44dd380376ff32144390b94283b52b0be00c1c1f117803578dd2c5dcadd", + "GCPAuthWebhook": "k8s-minikube/gcp-auth-webhook:v0.0.13@sha256:08a49cb7a588d81723b7e02c16082c75418b6e0a54cf2e44668bd77f79a41a40", }, map[string]string{ "GCPAuthWebhook": "gcr.io", "KubeWebhookCertgen": "k8s.gcr.io", diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index da3383ca82..ad004c704d 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -67,6 +67,15 @@ func TestAddons(t *testing.T) { t.Fatalf("Failed setting GOOGLE_CLOUD_PROJECT env var: %v", err) } + // MOCK_GOOGLE_TOKEN forces the gcp-auth webhook to use a mock token instead of trying to get a valid one from the credentials. + err = os.Setenv("MOCK_GOOGLE_TOKEN", "true") + t.Cleanup(func() { + os.Unsetenv("MOCK_GOOGLE_TOKEN") + }) + if err != nil { + t.Fatalf("Failed setting MOCK_GOOGLE_TOKEN env var: %v", err) + } + args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth", "--addons=cloud-spanner"}, StartArgs()...) if !NoneDriver() { // none driver does not support ingress args = append(args, "--addons=ingress", "--addons=ingress-dns") @@ -606,7 +615,7 @@ func validateGCPAuthNamespaces(ctx context.Context, t *testing.T, profile string t.Fatalf("%s failed: %v", rr.Command(), err) } - getLogs := func() error { + logsAsError := func() error { rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "logs", "-l", "app=gcp-auth", "-n", "gcp-auth")) if err != nil { return err @@ -617,7 +626,7 @@ func validateGCPAuthNamespaces(ctx context.Context, t *testing.T, profile string getSecret := func() error { _, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "secret", "gcp-auth", "-n", "new-namespace")) if err != nil { - err = getLogs() + err = fmt.Errorf("%w: gcp-auth container logs: %v", err, logsAsError()) } return err } From 7b60a5376d9eb94bdecf9733f5312f7882c3beb1 Mon Sep 17 00:00:00 2001 From: Ian Stewart Date: Thu, 8 Dec 2022 13:46:38 -0500 Subject: [PATCH 10/12] Added MOCK_GOOGLE_TOKEN env var to gcp-auth container. Reading env var from images map. This should be moved. --- deploy/addons/gcp-auth/gcp-auth-webhook.yaml.tmpl | 2 ++ pkg/minikube/assets/addons.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/deploy/addons/gcp-auth/gcp-auth-webhook.yaml.tmpl b/deploy/addons/gcp-auth/gcp-auth-webhook.yaml.tmpl index d1afa12549..3f7a6ab5da 100644 --- a/deploy/addons/gcp-auth/gcp-auth-webhook.yaml.tmpl +++ b/deploy/addons/gcp-auth/gcp-auth-webhook.yaml.tmpl @@ -107,6 +107,8 @@ spec: env: - name: GOOGLE_APPLICATION_CREDENTIALS value: /google-app-creds.json + - name: MOCK_GOOGLE_TOKEN + value: "{{.Images.MockGoogleToken}}" ports: - containerPort: 8443 volumeMounts: diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index e8e9187e70..794b48c184 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -18,6 +18,7 @@ package assets import ( "fmt" + "os" "runtime" "strings" @@ -578,6 +579,7 @@ var Addons = map[string]*Addon{ }, false, "gcp-auth", "Google", "", "https://minikube.sigs.k8s.io/docs/handbook/addons/gcp-auth/", map[string]string{ "KubeWebhookCertgen": "ingress-nginx/kube-webhook-certgen:v1.0@sha256:f3b6b39a6062328c095337b4cadcefd1612348fdd5190b1dcbcb9b9e90bd8068", "GCPAuthWebhook": "k8s-minikube/gcp-auth-webhook:v0.0.13@sha256:08a49cb7a588d81723b7e02c16082c75418b6e0a54cf2e44668bd77f79a41a40", + "MockGoogleToken": os.Getenv("MOCK_GOOGLE_TOKEN"), // TODO: move this. Maybe we should have a misc map? }, map[string]string{ "GCPAuthWebhook": "gcr.io", "KubeWebhookCertgen": "k8s.gcr.io", From 44f8364c5ffb5258e917bbb93cb803e78826241e Mon Sep 17 00:00:00 2001 From: Ian Stewart Date: Thu, 8 Dec 2022 17:45:51 -0500 Subject: [PATCH 11/12] Added Environment to the template data --- .../gcp-auth/gcp-auth-webhook.yaml.tmpl | 2 +- pkg/minikube/assets/addons.go | 35 ++++++++++--------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/deploy/addons/gcp-auth/gcp-auth-webhook.yaml.tmpl b/deploy/addons/gcp-auth/gcp-auth-webhook.yaml.tmpl index 3f7a6ab5da..d767f269a6 100644 --- a/deploy/addons/gcp-auth/gcp-auth-webhook.yaml.tmpl +++ b/deploy/addons/gcp-auth/gcp-auth-webhook.yaml.tmpl @@ -108,7 +108,7 @@ spec: - name: GOOGLE_APPLICATION_CREDENTIALS value: /google-app-creds.json - name: MOCK_GOOGLE_TOKEN - value: "{{.Images.MockGoogleToken}}" + value: "{{.Environment.MockGoogleToken}}" ports: - containerPort: 8443 volumeMounts: diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index 4d8ac35e62..0c002f9879 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -579,7 +579,6 @@ var Addons = map[string]*Addon{ }, false, "gcp-auth", "Google", "", "https://minikube.sigs.k8s.io/docs/handbook/addons/gcp-auth/", map[string]string{ "KubeWebhookCertgen": "ingress-nginx/kube-webhook-certgen:v1.0@sha256:f3b6b39a6062328c095337b4cadcefd1612348fdd5190b1dcbcb9b9e90bd8068", "GCPAuthWebhook": "k8s-minikube/gcp-auth-webhook:v0.0.13@sha256:08a49cb7a588d81723b7e02c16082c75418b6e0a54cf2e44668bd77f79a41a40", - "MockGoogleToken": os.Getenv("MOCK_GOOGLE_TOKEN"), // TODO: move this. Maybe we should have a misc map? }, map[string]string{ "GCPAuthWebhook": "gcr.io", "KubeWebhookCertgen": "k8s.gcr.io", @@ -886,24 +885,28 @@ func GenerateTemplateData(addon *Addon, cc *config.ClusterConfig, netInfo Networ Registries map[string]string CustomRegistries map[string]string NetworkInfo map[string]string + Environment map[string]string LegacyPodSecurityPolicy bool LegacyRuntimeClass bool }{ - KubernetesVersion: make(map[string]uint64), - PreOneTwentyKubernetes: false, - Arch: a, - ExoticArch: ea, - ImageRepository: cfg.ImageRepository, - LoadBalancerStartIP: cfg.LoadBalancerStartIP, - LoadBalancerEndIP: cfg.LoadBalancerEndIP, - CustomIngressCert: cfg.CustomIngressCert, - RegistryAliases: cfg.RegistryAliases, - IngressAPIVersion: "v1", // api version for ingress (eg, "v1beta1"; defaults to "v1" for k8s 1.19+) - ContainerRuntime: cfg.ContainerRuntime, - Images: images, - Registries: addon.Registries, - CustomRegistries: customRegistries, - NetworkInfo: make(map[string]string), + KubernetesVersion: make(map[string]uint64), + PreOneTwentyKubernetes: false, + Arch: a, + ExoticArch: ea, + ImageRepository: cfg.ImageRepository, + LoadBalancerStartIP: cfg.LoadBalancerStartIP, + LoadBalancerEndIP: cfg.LoadBalancerEndIP, + CustomIngressCert: cfg.CustomIngressCert, + RegistryAliases: cfg.RegistryAliases, + IngressAPIVersion: "v1", // api version for ingress (eg, "v1beta1"; defaults to "v1" for k8s 1.19+) + ContainerRuntime: cfg.ContainerRuntime, + Images: images, + Registries: addon.Registries, + CustomRegistries: customRegistries, + NetworkInfo: make(map[string]string), + Environment: map[string]string{ + "MockGoogleToken": os.Getenv("MOCK_GOOGLE_TOKEN"), + }, LegacyPodSecurityPolicy: v.LT(semver.Version{Major: 1, Minor: 25}), LegacyRuntimeClass: v.LT(semver.Version{Major: 1, Minor: 25}), } From a6bfa55152dc767a532df9cd7bde41b314756164 Mon Sep 17 00:00:00 2001 From: Ian Stewart Date: Fri, 9 Dec 2022 14:12:13 -0500 Subject: [PATCH 12/12] Using t.Setenv instead of os.Setenv --- test/integration/addons_test.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index ad004c704d..88e1434a6d 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -68,13 +68,7 @@ func TestAddons(t *testing.T) { } // MOCK_GOOGLE_TOKEN forces the gcp-auth webhook to use a mock token instead of trying to get a valid one from the credentials. - err = os.Setenv("MOCK_GOOGLE_TOKEN", "true") - t.Cleanup(func() { - os.Unsetenv("MOCK_GOOGLE_TOKEN") - }) - if err != nil { - t.Fatalf("Failed setting MOCK_GOOGLE_TOKEN env var: %v", err) - } + t.Setenv("MOCK_GOOGLE_TOKEN", "true") args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth", "--addons=cloud-spanner"}, StartArgs()...) if !NoneDriver() { // none driver does not support ingress