minikube/pkg/addons/addons_gcpauth.go

/*
Copyright 2021 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package addons

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"time"

	gcr_config "github.com/GoogleCloudPlatform/docker-credential-gcr/config"
	"github.com/pkg/errors"
	"golang.org/x/oauth2/google"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/klog/v2"
	"k8s.io/minikube/pkg/minikube/assets"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/detect"
	"k8s.io/minikube/pkg/minikube/exit"
	"k8s.io/minikube/pkg/minikube/mustload"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/minikube/reason"
	"k8s.io/minikube/pkg/minikube/service"
	"k8s.io/minikube/pkg/minikube/style"
)

const (
	credentialsPath = "/var/lib/minikube/google_application_credentials.json"
	projectPath     = "/var/lib/minikube/google_cloud_project"
	secretName      = "gcp-auth"
	namespaceName   = "gcp-auth"
)

// enableOrDisableGCPAuth enables or disables the gcp-auth addon depending on the val parameter
func enableOrDisableGCPAuth(cfg *config.ClusterConfig, name string, val string) error {
	enable, err := strconv.ParseBool(val)
	if err != nil {
		return errors.Wrapf(err, "parsing bool: %s", name)
	}
	if enable {
		return enableAddonGCPAuth(cfg)
	}
	return disableAddonGCPAuth(cfg)
}

func enableAddonGCPAuth(cfg *config.ClusterConfig) error {
	// Grab command runner from running cluster
	cc := mustload.Running(cfg.Name)
	r := cc.CP.Runner

	// Grab credentials from where GCP would normally look
	ctx := context.Background()
	creds, err := google.FindDefaultCredentials(ctx)
	if err != nil {
		if detect.IsCloudShell() {
			if c := os.Getenv("CLOUDSDK_CONFIG"); c != "" {
				f, err := ioutil.ReadFile(path.Join(c, "application_default_credentials.json"))
				if err == nil {
					creds, _ = google.CredentialsFromJSON(ctx, f)
				}
			}
		} else {
			exit.Message(reason.InternalCredsNotFound, "Could not find any GCP credentials. Either run `gcloud auth application-default login` or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to the path of your credentials file.")
		}
	}

	// Create a registry secret in every namespace we can find
	// Always create the pull secret, no matter where we are
	err = createPullSecret(cfg, creds)
	if err != nil {
		return errors.Wrap(err, "pull secret")
	}

	// If the env var is explicitly set, even in GCE, then defer to the user and continue
	if !Force && detect.IsOnGCE() && os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" {
		out.WarningT("It seems that you are running in GCE, which means authentication should work without the GCP Auth addon. If you would still like to authenticate using a credentials file, use the --force flag.")
		return nil
	}

	if creds.JSON == nil {
		out.WarningT("You have authenticated with a service account that does not have an associated JSON. The GCP Auth requires credentials with a JSON file to in order to continue. The image pull secret has been imported.")
		return nil
	}

	// Actually copy the creds over
	f := assets.NewMemoryAssetTarget(creds.JSON, credentialsPath, "0444")

	err = r.Copy(f)
	if err != nil {
		return err
	}

	// First check if the project env var is explicitly set
	projectEnv := os.Getenv("GOOGLE_CLOUD_PROJECT")
	if projectEnv != "" {
		f := assets.NewMemoryAssetTarget([]byte(projectEnv), projectPath, "0444")
		return r.Copy(f)
	}

	// We're currently assuming gcloud is installed and in the user's path
	proj, err := exec.Command("gcloud", "config", "get-value", "project").Output()
	if err == nil && len(proj) > 0 {
		f := assets.NewMemoryAssetTarget(bytes.TrimSpace(proj), projectPath, "0444")
		return r.Copy(f)
	}

	out.WarningT("Could not determine a Google Cloud project, which might be ok.")
	out.Styled(style.Tip, `To set your Google Cloud project,  run:

		gcloud config set project <project name>

or set the GOOGLE_CLOUD_PROJECT environment variable.`)

	// Copy an empty file in to avoid errors about missing files
	emptyFile := assets.NewMemoryAssetTarget([]byte{}, projectPath, "0444")
	return r.Copy(emptyFile)

}

func createPullSecret(cc *config.ClusterConfig, creds *google.Credentials) error {
	if creds == nil {
		return errors.New("no credentials, skipping creating pull secret")
	}

	token, err := creds.TokenSource.Token()
	// Only try to add secret if Token was found
	if err == nil {
		client, err := service.K8s.GetCoreClient(cc.Name)
		if err != nil {
			return err
		}

		namespaces, err := client.Namespaces().List(context.TODO(), metav1.ListOptions{})
		if err != nil {
			return err
		}

		dockercfg := ""
		registries := append(gcr_config.DefaultGCRRegistries[:], gcr_config.DefaultARRegistries[:]...)
		for _, reg := range registries {
			dockercfg += fmt.Sprintf(`"https://%s":{"username":"oauth2accesstoken","password":"%s","email":"none"},`, reg, token.AccessToken)
		}

		dockercfg = strings.TrimSuffix(dockercfg, ",")

		data := map[string][]byte{
			".dockercfg": []byte(fmt.Sprintf(`{%s}`, dockercfg)),
		}

		for _, n := range namespaces.Items {
			if skipNamespace(n.Name) {
				continue
			}
			secrets := client.Secrets(n.Name)

			exists := false
			secList, err := secrets.List(context.TODO(), metav1.ListOptions{})
			if err != nil {
				return err
			}
			for _, s := range secList.Items {
				if s.Name == secretName {
					exists = true
					break
				}
			}

			if !exists || Refresh {
				secretObj := &corev1.Secret{
					ObjectMeta: metav1.ObjectMeta{
						Name: secretName,
					},
					Data: data,
					Type: "kubernetes.io/dockercfg",
				}

				if exists && Refresh {
					_, err := secrets.Update(context.TODO(), secretObj, metav1.UpdateOptions{})
					if err != nil {
						return err
					}
				} else {
					_, err = secrets.Create(context.TODO(), secretObj, metav1.CreateOptions{})
					if err != nil {
						return err
					}
				}
			}

			// Now patch the secret into all the service accounts we can find
			serviceaccounts := client.ServiceAccounts(n.Name)
			salist, err := serviceaccounts.List(context.TODO(), metav1.ListOptions{})
			if err != nil {
				return err
			}

			// Let's make sure we at least find the default service account
			for len(salist.Items) == 0 {
				salist, err = serviceaccounts.List(context.TODO(), metav1.ListOptions{})
				if err != nil {
					return err
				}
				time.Sleep(1 * time.Second)
			}

			ips := corev1.LocalObjectReference{Name: secretName}
			for _, sa := range salist.Items {
				add := true
				for _, ps := range sa.ImagePullSecrets {
					if ps.Name == secretName {
						add = false
						break
					}
				}
				if add {
					sa.ImagePullSecrets = append(sa.ImagePullSecrets, ips)
					_, err := serviceaccounts.Update(context.TODO(), &sa, metav1.UpdateOptions{})
					if err != nil {
						return err
					}
				}
			}

		}
	}
	return nil
}

func refreshExistingPods(cc *config.ClusterConfig) error {
	client, err := service.K8s.GetCoreClient(cc.Name)
	if err != nil {
		return err
	}

	namespaces, err := client.Namespaces().List(context.TODO(), metav1.ListOptions{})
	if err != nil {
		return err
	}
	for _, n := range namespaces.Items {
		// Ignore kube-system and gcp-auth namespaces
		if skipNamespace(n.Name) {
			continue
		}

		pods := client.Pods(n.Name)
		podList, err := pods.List(context.TODO(), metav1.ListOptions{})
		if err != nil {
			return err
		}

		for _, p := range podList.Items {
			// Skip pods we're explicitly told to skip
			if _, ok := p.Labels["gcp-auth-skip-secret"]; ok {
				continue
			}

			// Recreating the pod should pickup the necessary changes
			err := pods.Delete(context.TODO(), p.Name, metav1.DeleteOptions{})
			if err != nil {
				return err
			}

			p.ResourceVersion = ""

			_, err = pods.Get(context.TODO(), p.Name, metav1.GetOptions{})

			for err == nil {
				time.Sleep(time.Second)
				_, err = pods.Get(context.TODO(), p.Name, metav1.GetOptions{})
			}

			_, err = pods.Create(context.TODO(), &p, metav1.CreateOptions{})
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func disableAddonGCPAuth(cfg *config.ClusterConfig) error {
	// Grab command runner from running cluster
	cc := mustload.Running(cfg.Name)
	r := cc.CP.Runner

	// Clean up the files generated when enabling the addon
	creds := assets.NewMemoryAssetTarget([]byte{}, credentialsPath, "0444")
	err := r.Remove(creds)
	if err != nil {
		return err
	}

	project := assets.NewMemoryAssetTarget([]byte{}, projectPath, "0444")
	err = r.Remove(project)
	if err != nil {
		return err
	}

	client, err := service.K8s.GetCoreClient(cfg.Name)
	if err != nil {
		return err
	}

	namespaces, err := client.Namespaces().List(context.TODO(), metav1.ListOptions{})
	if err != nil {
		return err
	}

	// No need to check for an error here, if the secret doesn't exist, no harm done.
	for _, n := range namespaces.Items {
		if skipNamespace(n.Name) {
			continue
		}
		secrets := client.Secrets(n.Name)
		err := secrets.Delete(context.TODO(), secretName, metav1.DeleteOptions{})
		if err != nil {
			klog.Infof("error deleting secret: %v", err)
		}

		serviceaccounts := client.ServiceAccounts(n.Name)
		salist, err := serviceaccounts.List(context.TODO(), metav1.ListOptions{})
		if err != nil {
			klog.Infof("error getting service accounts: %v", err)
			return err
		}
		for _, sa := range salist.Items {
			for i, ps := range sa.ImagePullSecrets {
				if ps.Name == secretName {
					sa.ImagePullSecrets = append(sa.ImagePullSecrets[:i], sa.ImagePullSecrets[i+1:]...)
					_, err := serviceaccounts.Update(context.TODO(), &sa, metav1.UpdateOptions{})
					if err != nil {
						return err
					}
					break
				}
			}
		}
	}

	return nil
}

func verifyGCPAuthAddon(cc *config.ClusterConfig, name string, val string) error {
	enable, err := strconv.ParseBool(val)
	if err != nil {
		return errors.Wrapf(err, "parsing bool: %s", name)
	}

	// If we're in GCE and didn't actually start the gcp-auth pods, don't check for them.
	// We also don't want to actually set the addon as enabled, so just exit completely.
	if enable && !Force && detect.IsOnGCE() && os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" {
		return ErrSkipThisAddon
	}

	err = verifyAddonStatusInternal(cc, name, val, "gcp-auth")
	if err != nil {
		return err
	}

	if Refresh {
		err = refreshExistingPods(cc)
		if err != nil {
			return err
		}
	}

	if enable && err == nil {
		out.Styled(style.Notice, "Your GCP credentials will now be mounted into every pod created in the {{.name}} cluster.", out.V{"name": cc.Name})
		out.Styled(style.Notice, "If you don't want your credentials mounted into a specific pod, add a label with the `gcp-auth-skip-secret` key to your pod configuration.")
		if !Refresh {
			out.Styled(style.Notice, "If you want existing pods to be mounted with credentials, either recreate them or rerun addons enable with --refresh.")
		}
	}

	return err
}

func skipNamespace(name string) bool {
	return name == metav1.NamespaceSystem || name == namespaceName
}