Merge pull request #11486 from sharifelgamal/gcp-auth-restart

gcp-auth addon: do not reapply gcp-auth yamls on minikube restart

cmd/minikube/cmd/config/enable.go

@@ -77,5 +77,6 @@ func init() {
 	addonsEnableCmd.Flags().StringVar(&images, "images", "", "Images used by this addon. Separated by commas.")
 	addonsEnableCmd.Flags().StringVar(&registries, "registries", "", "Registries used by this addon. Separated by commas.")
 	addonsEnableCmd.Flags().BoolVar(&addons.Force, "force", false, "If true, will perform potentially dangerous operations. Use with discretion.")
+	addonsEnableCmd.Flags().BoolVar(&addons.Refresh, "refresh", false, "If true, pods might get deleted and restarted on addon enable")
 	AddonsCmd.AddCommand(addonsEnableCmd)
 }

deploy/addons/gcp-auth/gcp-auth-webhook.yaml.tmpl.tmpl

@@ -164,7 +164,7 @@ webhooks:
       namespace: gcp-auth
       path: "/mutate"
   rules:
-  - operations: ["CREATE", "UPDATE"]
+  - operations: ["CREATE"]
     apiGroups: ["*"]
     apiVersions: ["*"]
     resources: ["pods"]

pkg/addons/addons.go

@@ -50,6 +50,10 @@ import (
 // Force is used to override checks for addons
 var Force bool = false
 
+// Refresh is used to refresh pods in specific cases when an addon is enabled
+// Currently only used for gcp-auth
+var Refresh bool = false
+
 // RunCallbacks runs all actions associated to an addon, but does not set it (thread-safe)
 func RunCallbacks(cc *config.ClusterConfig, name string, value string) error {
 	klog.Infof("Setting %s=%s in profile %q", name, value, cc.Name)
@@ -137,6 +141,9 @@ func EnableOrDisableAddon(cc *config.ClusterConfig, name string, val string) err
 
 	// check addon status before enabling/disabling it
 	if isAddonAlreadySet(cc, addon, enable) {
+		if addon.Name() == "gcp-auth" {
+			return nil
+		}
 		klog.Warningf("addon %s should already be in state %v", name, val)
 		if !enable {
 			return nil

pkg/addons/addons_gcpauth.go

@@ -45,6 +45,7 @@ const (
 	credentialsPath = "/var/lib/minikube/google_application_credentials.json"
 	projectPath     = "/var/lib/minikube/google_cloud_project"
 	secretName      = "gcp-auth"
+	namespaceName   = "gcp-auth"
 )
 
 // enableOrDisableGCPAuth enables or disables the gcp-auth addon depending on the val parameter
@@ -84,7 +85,45 @@ func enableAddonGCPAuth(cfg *config.ClusterConfig) error {
 	}
 
 	// Create a registry secret in every namespace we can find
-	client, err := service.K8s.GetCoreClient(cfg.Name)
+	err = createPullSecret(cfg, creds)
+	if err != nil {
+		return errors.Wrap(err, "pull secret")
+	}
+
+	// First check if the project env var is explicitly set
+	projectEnv := os.Getenv("GOOGLE_CLOUD_PROJECT")
+	if projectEnv != "" {
+		f := assets.NewMemoryAssetTarget([]byte(projectEnv), projectPath, "0444")
+		return r.Copy(f)
+	}
+
+	// We're currently assuming gcloud is installed and in the user's path
+	proj, err := exec.Command("gcloud", "config", "get-value", "project").Output()
+	if err == nil && len(proj) > 0 {
+		f := assets.NewMemoryAssetTarget(bytes.TrimSpace(proj), projectPath, "0444")
+		return r.Copy(f)
+	}
+
+	out.WarningT("Could not determine a Google Cloud project, which might be ok.")
+	out.Styled(style.Tip, `To set your Google Cloud project,  run:
+
+		gcloud config set project <project name>
+
+or set the GOOGLE_CLOUD_PROJECT environment variable.`)
+
+	// Copy an empty file in to avoid errors about missing files
+	emptyFile := assets.NewMemoryAssetTarget([]byte{}, projectPath, "0444")
+	return r.Copy(emptyFile)
+
+}
+
+func createPullSecret(cc *config.ClusterConfig, creds *google.Credentials) error {
+	client, err := service.K8s.GetCoreClient(cc.Name)
+	if err != nil {
+		return err
+	}
+
+	namespaces, err := client.Namespaces().List(context.TODO(), metav1.ListOptions{})
 	if err != nil {
 		return err
 	}
@@ -96,11 +135,6 @@ func enableAddonGCPAuth(cfg *config.ClusterConfig) error {
 			".dockercfg": []byte(fmt.Sprintf(`{"https://gcr.io":{"username":"oauth2accesstoken","password":"%s","email":"none"}, "https://us-docker.pkg.dev":{"username":"oauth2accesstoken","password":"%s","email":"none"}}`, token.AccessToken, token.AccessToken)),
 		}
 
-		namespaces, err := client.Namespaces().List(context.TODO(), metav1.ListOptions{})
-		if err != nil {
-			return err
-		}
-
 		for _, n := range namespaces.Items {
 			secrets := client.Secrets(n.Name)
 
@@ -147,7 +181,7 @@ func enableAddonGCPAuth(cfg *config.ClusterConfig) error {
 				time.Sleep(1 * time.Second)
 			}
 
-			ips := corev1.LocalObjectReference{Name: "gcp-auth"}
+			ips := corev1.LocalObjectReference{Name: secretName}
 			for _, sa := range salist.Items {
 				sa.ImagePullSecrets = append(sa.ImagePullSecrets, ips)
 				_, err := serviceaccounts.Update(context.TODO(), &sa, metav1.UpdateOptions{})
@@ -158,31 +192,59 @@ func enableAddonGCPAuth(cfg *config.ClusterConfig) error {
 
 		}
 	}
-
-	// First check if the project env var is explicitly set
-	projectEnv := os.Getenv("GOOGLE_CLOUD_PROJECT")
-	if projectEnv != "" {
-		f := assets.NewMemoryAssetTarget([]byte(projectEnv), projectPath, "0444")
-		return r.Copy(f)
+	return nil
 }
 
-	// We're currently assuming gcloud is installed and in the user's path
-	project, err := exec.Command("gcloud", "config", "get-value", "project").Output()
-	if err == nil && len(project) > 0 {
-		f := assets.NewMemoryAssetTarget(bytes.TrimSpace(project), projectPath, "0444")
-		return r.Copy(f)
+func refreshExistingPods(cc *config.ClusterConfig) error {
+	client, err := service.K8s.GetCoreClient(cc.Name)
+	if err != nil {
+		return err
 	}
 
-	out.WarningT("Could not determine a Google Cloud project, which might be ok.")
-	out.Styled(style.Tip, `To set your Google Cloud project,  run: 
+	namespaces, err := client.Namespaces().List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, n := range namespaces.Items {
+		// Ignore kube-system and gcp-auth namespaces
+		if n.Name == metav1.NamespaceSystem || n.Name == namespaceName {
+			continue
+		}
 
-		gcloud config set project <project name>
+		pods := client.Pods(n.Name)
+		podList, err := pods.List(context.TODO(), metav1.ListOptions{})
+		if err != nil {
+			return err
+		}
 
-or set the GOOGLE_CLOUD_PROJECT environment variable.`)
+		for _, p := range podList.Items {
+			// Skip pods we're explicitly told to skip
+			if _, ok := p.Labels["gcp-auth-skip-secret"]; ok {
+				continue
+			}
 
-	// Copy an empty file in to avoid errors about missing files
-	emptyFile := assets.NewMemoryAssetTarget([]byte{}, projectPath, "0444")
-	return r.Copy(emptyFile)
+			// Recreating the pod should pickup the necessary changes
+			err := pods.Delete(context.TODO(), p.Name, metav1.DeleteOptions{})
+			if err != nil {
+				return err
+			}
+
+			p.ResourceVersion = ""
+
+			_, err = pods.Get(context.TODO(), p.Name, metav1.GetOptions{})
+
+			for err == nil {
+				time.Sleep(time.Second)
+				_, err = pods.Get(context.TODO(), p.Name, metav1.GetOptions{})
+			}
+
+			_, err = pods.Create(context.TODO(), &p, metav1.CreateOptions{})
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
 }
 
 func disableAddonGCPAuth(cfg *config.ClusterConfig) error {
@@ -231,10 +293,23 @@ func verifyGCPAuthAddon(cc *config.ClusterConfig, name string, val string) error
 		return errors.Wrapf(err, "parsing bool: %s", name)
 	}
 	err = verifyAddonStatusInternal(cc, name, val, "gcp-auth")
+	if err != nil {
+		return err
+	}
+
+	if Refresh {
+		err = refreshExistingPods(cc)
+		if err != nil {
+			return err
+		}
+	}
 
 	if enable && err == nil {
 		out.Styled(style.Notice, "Your GCP credentials will now be mounted into every pod created in the {{.name}} cluster.", out.V{"name": cc.Name})
 		out.Styled(style.Notice, "If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.")
+		if !Refresh {
+			out.Styled(style.Notice, "If you want existing pods to be mounted with credentials, either recreate them or rerun addons enable with --refresh.")
+		}
 	}
 
 	return err

site/content/en/docs/commands/addons.md

@@ -130,6 +130,7 @@ minikube addons enable dashboard
 ```
       --force               If true, will perform potentially dangerous operations. Use with discretion.
       --images string       Images used by this addon. Separated by commas.
+      --refresh             If true, pods might get deleted and restarted on addon enable
       --registries string   Registries used by this addon. Separated by commas.
 ```
 

site/content/en/docs/handbook/addons/gcp-auth.md

@@ -5,6 +5,8 @@ weight: 1
 date: 2020-07-15
 ---
 
+## Tutorial
+
 If you have a containerized GCP app with a Kubernetes yaml, you can automatically add your credentials to all your deployed pods dynamically with this minikube addon. You just need to have a credentials file, which can be generated with `gcloud auth application-default login`. If you already have a json credentials file you want specify, use the GOOGLE_APPLICATION_CREDENTIALS environment variable.
 
 - Start a cluster:
@@ -80,3 +82,12 @@ spec:
         ports:
           - containerPort: 80
 </pre>
+
+
+## Refreshing existing pods
+
+If you had already deployed pods to your minikube cluster before enabling the gcp-auth addon, then these pods will not have any GCP credentials. There are two ways to solve this issue.  
+
+1. If you use a Deployment to deploy your pods, just delete the existing pods with `kubectl delete pod <pod_name>`. The deployment will then automatically recreate the pod and it will have the correct credentials.
+
+2. minikube can delete and recreate your pods for you, by running `minikube addons enable gcp-auth --refresh`. It does not matter if you have already enabled the addon or not. 

translations/de.json

@@ -312,6 +312,7 @@
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none.": "",
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --vm-driver=none.": "Wenn true, speichern Sie Docker-Images für den aktuellen Bootstrapper zwischen und laden Sie sie auf den Computer. Immer falsch mit --vm-driver = none.",
 	"If true, only download and cache files for later use - don't install or start anything.": "Wenn true, laden Sie nur Dateien für die spätere Verwendung herunter und speichern Sie sie – installieren oder starten Sie nichts.",
+	"If true, pods might get deleted and restarted on addon enable": "",
 	"If true, returns list of profiles faster by skipping validating the status of the cluster.": "",
 	"If true, the added node will be marked for work. Defaults to true.": "",
 	"If true, the node added will also be a control plane in addition to a worker.": "",
@@ -319,6 +320,7 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
+	"If you want existing pods to be mounted with credentials, either recreate them or rerun addons enable with --refresh.": "",
 	"Ignoring empty custom image {{.name}}": "",
 	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",

translations/es.json

@@ -317,6 +317,7 @@
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none.": "",
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --vm-driver=none.": "Si el valor es \"true\", las imágenes de Docker del programa previo actual se almacenan en caché y se cargan en la máquina. Siempre es \"false\" si se especifica --vm-driver=none.",
 	"If true, only download and cache files for later use - don't install or start anything.": "Si el valor es \"true\", los archivos solo se descargan y almacenan en caché (no se instala ni inicia nada).",
+	"If true, pods might get deleted and restarted on addon enable": "",
 	"If true, returns list of profiles faster by skipping validating the status of the cluster.": "",
 	"If true, the added node will be marked for work. Defaults to true.": "",
 	"If true, the node added will also be a control plane in addition to a worker.": "",
@@ -324,6 +325,7 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
+	"If you want existing pods to be mounted with credentials, either recreate them or rerun addons enable with --refresh.": "",
 	"Ignoring empty custom image {{.name}}": "",
 	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",

translations/fr.json

@@ -314,6 +314,7 @@
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none.": "",
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --vm-driver=none.": "Si la valeur est \"true\", mettez les images Docker en cache pour l'amorceur actuel et chargez-les dans la machine. La valeur est toujours \"false\" avec --vm-driver=none.",
 	"If true, only download and cache files for later use - don't install or start anything.": "Si la valeur est \"true\", téléchargez les fichiers et mettez-les en cache uniquement pour une utilisation future. Ne lancez pas d'installation et ne commencez aucun processus.",
+	"If true, pods might get deleted and restarted on addon enable": "",
 	"If true, returns list of profiles faster by skipping validating the status of the cluster.": "",
 	"If true, the added node will be marked for work. Defaults to true.": "",
 	"If true, the node added will also be a control plane in addition to a worker.": "",
@@ -321,6 +322,7 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
+	"If you want existing pods to be mounted with credentials, either recreate them or rerun addons enable with --refresh.": "",
 	"Ignoring empty custom image {{.name}}": "",
 	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",

translations/ja.json

@@ -302,6 +302,7 @@
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none.": "",
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --vm-driver=none.": "true の場合、現在のブートストラッパの Docker イメージをキャッシュに保存して、マシンに読み込みます。--vm-driver=none の場合は常に false です",
 	"If true, only download and cache files for later use - don't install or start anything.": "true の場合、後で使用できるようにファイルのダウンロードとキャッシュ保存だけが行われます。インストールも起動も行われません",
+	"If true, pods might get deleted and restarted on addon enable": "",
 	"If true, returns list of profiles faster by skipping validating the status of the cluster.": "",
 	"If true, the added node will be marked for work. Defaults to true.": "",
 	"If true, the node added will also be a control plane in addition to a worker.": "",
@@ -309,6 +310,7 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
+	"If you want existing pods to be mounted with credentials, either recreate them or rerun addons enable with --refresh.": "",
 	"Ignoring empty custom image {{.name}}": "",
 	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",

translations/ko.json

@@ -336,6 +336,7 @@
 	"If the above advice does not help, please let us know:": "",
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none.": "",
 	"If true, only download and cache files for later use - don't install or start anything.": "",
+	"If true, pods might get deleted and restarted on addon enable": "",
 	"If true, returns list of profiles faster by skipping validating the status of the cluster.": "",
 	"If true, the added node will be marked for work. Defaults to true.": "",
 	"If true, the node added will also be a control plane in addition to a worker.": "",
@@ -343,6 +344,7 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
+	"If you want existing pods to be mounted with credentials, either recreate them or rerun addons enable with --refresh.": "",
 	"Ignoring empty custom image {{.name}}": "",
 	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",

translations/pl.json

@@ -323,6 +323,7 @@
 	"If the above advice does not help, please let us know:": "",
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none.": "",
 	"If true, only download and cache files for later use - don't install or start anything.": "",
+	"If true, pods might get deleted and restarted on addon enable": "",
 	"If true, returns list of profiles faster by skipping validating the status of the cluster.": "",
 	"If true, the added node will be marked for work. Defaults to true.": "",
 	"If true, the node added will also be a control plane in addition to a worker.": "",
@@ -331,6 +332,7 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
+	"If you want existing pods to be mounted with credentials, either recreate them or rerun addons enable with --refresh.": "",
 	"Ignoring empty custom image {{.name}}": "",
 	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",

translations/strings.txt

@@ -292,6 +292,7 @@
 	"If the above advice does not help, please let us know:": "",
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none.": "",
 	"If true, only download and cache files for later use - don't install or start anything.": "",
+	"If true, pods might get deleted and restarted on addon enable": "",
 	"If true, returns list of profiles faster by skipping validating the status of the cluster.": "",
 	"If true, the added node will be marked for work. Defaults to true.": "",
 	"If true, the node added will also be a control plane in addition to a worker.": "",
@@ -299,6 +300,7 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
+	"If you want existing pods to be mounted with credentials, either recreate them or rerun addons enable with --refresh.": "",
 	"Ignoring empty custom image {{.name}}": "",
 	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",

translations/zh-CN.json

@@ -393,6 +393,7 @@
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none.": "",
 	"If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --vm-driver=none.": "如果为 true，请缓存当前引导程序的 docker 镜像并将其加载到机器中。在 --vm-driver=none 情况下始终为 false。",
 	"If true, only download and cache files for later use - don't install or start anything.": "如果为 true，仅会下载和缓存文件以备后用 - 不会安装或启动任何项。",
+	"If true, pods might get deleted and restarted on addon enable": "",
 	"If true, returns list of profiles faster by skipping validating the status of the cluster.": "",
 	"If true, the added node will be marked for work. Defaults to true.": "",
 	"If true, the node added will also be a control plane in addition to a worker.": "",
@@ -400,6 +401,7 @@
 	"If you are running minikube within a VM, consider using --driver=none:": "",
 	"If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:": "",
 	"If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.": "",
+	"If you want existing pods to be mounted with credentials, either recreate them or rerun addons enable with --refresh.": "",
 	"Ignoring empty custom image {{.name}}": "",
 	"Ignoring invalid pair entry {{.pair}}": "",
 	"Ignoring unknown custom image {{.name}}": "",