From b70115d00e1b61ae2449d4c5fe2592732f73b148 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 25 Mar 2020 21:47:33 -0700 Subject: [PATCH 001/105] Add hosts aliases for control-plane and VM host --- pkg/minikube/bootstrapper/bootstrapper.go | 2 +- pkg/minikube/bootstrapper/bsutil/kubeadm.go | 2 +- pkg/minikube/bootstrapper/certs.go | 3 ++- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 16 ++++++++++++++-- pkg/minikube/constants/constants.go | 4 ++++ pkg/minikube/machine/start.go | 16 ++++++++++++++++ pkg/minikube/node/start.go | 16 +++++++++++++++- 7 files changed, 53 insertions(+), 6 deletions(-) diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index 598397b171..1b940390c4 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -41,7 +41,7 @@ type Bootstrapper interface { DeleteCluster(config.KubernetesConfig) error WaitForNode(config.ClusterConfig, config.Node, time.Duration) error JoinCluster(config.ClusterConfig, config.Node, string) error - UpdateNode(config.ClusterConfig, config.Node, cruntime.Manager) error + UpdateNode(config.ClusterConfig, config.Node, cruntime.Manager, config.Node) error GenerateToken(config.ClusterConfig) (string, error) // LogCommands returns a map of log type to a command which will display that log. LogCommands(config.ClusterConfig, LogOptions) map[string]string diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm.go b/pkg/minikube/bootstrapper/bsutil/kubeadm.go index 46104c1d44..5dd5908f2a 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubeadm.go +++ b/pkg/minikube/bootstrapper/bsutil/kubeadm.go @@ -101,7 +101,7 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana NoTaintMaster: false, // That does not work with k8s 1.12+ DNSDomain: k8s.DNSDomain, NodeIP: n.IP, - ControlPlaneAddress: cp.IP, + ControlPlaneAddress: constants.ControlPlaneAlias, } if k8s.ServiceCIDR != "" { diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index de938082cc..77fad2434e 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -39,6 +39,7 @@ import ( "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/kubeconfig" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/vmpath" @@ -189,7 +190,7 @@ func generateProfileCerts(k8s config.KubernetesConfig, n config.Node, ccs CACert apiServerIPs := append( k8s.APIServerIPs, []net.IP{net.ParseIP(n.IP), serviceIP, net.ParseIP(oci.DefaultBindIPV4), net.ParseIP("10.0.0.1")}...) - apiServerNames := append(k8s.APIServerNames, k8s.APIServerName) + apiServerNames := append(k8s.APIServerNames, k8s.APIServerName, constants.ControlPlaneAlias) apiServerAlternateNames := append( apiServerNames, util.GetAlternateDNS(k8s.DNSDomain)...) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 4ccc402698..1447cb78d9 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -586,8 +586,14 @@ func (k *Bootstrapper) UpdateCluster(cfg config.ClusterConfig) error { } } + // In case of no port assigned, use default + cp, err := config.PrimaryControlPlane(&cfg) + if err != nil { + return errors.Wrap(err, "getting control plane") + } + for _, n := range cfg.Nodes { - err := k.UpdateNode(cfg, n, r) + err := k.UpdateNode(cfg, n, r, cp) if err != nil { return errors.Wrap(err, "updating node") } @@ -597,7 +603,9 @@ func (k *Bootstrapper) UpdateCluster(cfg config.ClusterConfig) error { } // UpdateNode updates a node. -func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cruntime.Manager) error { +func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cruntime.Manager, cp config.Node) error { + glog.Infof("UPDATING NODE: %v", n.Name) + kubeadmCfg, err := bsutil.GenerateKubeadmYAML(cfg, n, r) if err != nil { return errors.Wrap(err, "generating kubeadm cfg") @@ -630,6 +638,10 @@ func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cru return err } + if err := machine.AddHostAlias(k.c, constants.ControlPlaneAlias, net.ParseIP(cp.IP)); err != nil { + return errors.Wrap(err, "host alias") + } + if err := reloadKubelet(k.c); err != nil { return err } diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index af053eee32..e2dda5671d 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -47,6 +47,10 @@ const ( ClusterDNSDomain = "cluster.local" // DefaultServiceCIDR is The CIDR to be used for service cluster IPs DefaultServiceCIDR = "10.96.0.0/12" + // HostAlias is a DNS alias to the the container/VM host IP + HostAlias = "host.minikube.internal" + // ControlPaneAlias is a DNS alias pointing to the apiserver frontend + ControlPlaneAlias = "apiserver.minikube.internal" // DockerHostEnv is used for docker daemon settings DockerHostEnv = "DOCKER_HOST" diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index c41b285a2f..d848f0724e 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -19,6 +19,7 @@ package machine import ( "encoding/json" "fmt" + "net" "os/exec" "path" "path/filepath" @@ -288,3 +289,18 @@ func showHostInfo(cfg config.ClusterConfig) { } out.T(out.StartingVM, "Creating {{.driver_name}} {{.machine_type}} (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"driver_name": cfg.Driver, "number_of_cpus": cfg.CPUs, "memory_size": cfg.Memory, "disk_size": cfg.DiskSize, "machine_type": machineType}) } + +// AddHostAlias makes fine adjustments to pod resources that aren't possible via kubeadm config. +func AddHostAlias(c command.Runner, name string, ip net.IP) error { + glog.Infof("checking") + record := fmt.Sprintf("%s\t%s", ip, name) + if _, err := c.RunCmd(exec.Command("grep", record+"$", "/etc/hosts")); err == nil { + return nil + } + + script := fmt.Sprintf(`{ grep -v '\t%s$' /etc/hosts; echo "%s"; } > /tmp/h.$$; sudo mv /tmp/h.$$ /etc/hosts`, name, record) + if _, err := c.RunCmd(exec.Command("/bin/bash", "-c", script)); err != nil { + return errors.Wrap(err, "hosts update") + } + return nil +} diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 5f7b131a6e..9ee2317e86 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -100,6 +100,14 @@ func Start(cc config.ClusterConfig, n config.Node, existingAddons map[string]boo cr := configureRuntimes(mRunner, cc.Driver, cc.KubernetesConfig, sv) showVersionInfo(n.KubernetesVersion, cr) + hostIP, err := cluster.GetVMHostIP(host) + if err != nil { + exit.WithError("Unable to get VM IP", err) + } + if err := machine.AddHostAlias(mRunner, constants.HostAlias, hostIP); err != nil { + exit.WithError("Unable to add host alias", err) + } + var bs bootstrapper.Bootstrapper var kubeconfig *kubeconfig.Settings if apiServer { @@ -138,6 +146,12 @@ func Start(cc config.ClusterConfig, n config.Node, existingAddons map[string]boo addons.Start(viper.GetString(config.ProfileName), existingAddons, config.AddonList) } + // In case of no port assigned, use default + cp, err := config.PrimaryControlPlane(&cc) + if err != nil { + exit.WithError("Unable to get control plane", err) + } + if apiServer { // special ops for none , like change minikube directory. // multinode super doesn't work on the none driver @@ -152,7 +166,7 @@ func Start(cc config.ClusterConfig, n config.Node, existingAddons map[string]boo } } } else { - if err := bs.UpdateNode(cc, n, cr); err != nil { + if err := bs.UpdateNode(cc, n, cr, cp); err != nil { exit.WithError("Updating node", err) } From f75616304457fa4cb87a161e5bb8d832a2b4aa06 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 25 Mar 2020 21:53:17 -0700 Subject: [PATCH 002/105] apiserver -> control-plane --- pkg/minikube/constants/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index e2dda5671d..e777cdd3da 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -50,7 +50,7 @@ const ( // HostAlias is a DNS alias to the the container/VM host IP HostAlias = "host.minikube.internal" // ControlPaneAlias is a DNS alias pointing to the apiserver frontend - ControlPlaneAlias = "apiserver.minikube.internal" + ControlPlaneAlias = "control-plane.minikube.internal" // DockerHostEnv is used for docker daemon settings DockerHostEnv = "DOCKER_HOST" From d9fe5afa866381ba15c86930bce2bc36e3804e54 Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Wed, 4 Mar 2020 23:36:37 +0800 Subject: [PATCH 003/105] Adds MetalLB addon --- .../addons/metallb/metallb-config.yaml.tmpl | 12 + deploy/addons/metallb/metallb.yaml | 293 ++++++++++++++++++ pkg/addons/config.go | 5 + pkg/minikube/assets/addons.go | 30 +- pkg/minikube/config/types.go | 28 +- 5 files changed, 349 insertions(+), 19 deletions(-) create mode 100644 deploy/addons/metallb/metallb-config.yaml.tmpl create mode 100644 deploy/addons/metallb/metallb.yaml diff --git a/deploy/addons/metallb/metallb-config.yaml.tmpl b/deploy/addons/metallb/metallb-config.yaml.tmpl new file mode 100644 index 0000000000..d89de8f582 --- /dev/null +++ b/deploy/addons/metallb/metallb-config.yaml.tmpl @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: metallb-system + name: config +data: + config: | + address-pools: + - name: default + protocol: layer2 + addresses: + - {{ .LoadBalancerStartIP }}-{{ .LoadBalancerEndIP }} diff --git a/deploy/addons/metallb/metallb.yaml b/deploy/addons/metallb/metallb.yaml new file mode 100644 index 0000000000..0384232eab --- /dev/null +++ b/deploy/addons/metallb/metallb.yaml @@ -0,0 +1,293 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + app: metallb + name: metallb-system +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + labels: + app: metallb + name: speaker + namespace: metallb-system +spec: + allowPrivilegeEscalation: false + allowedCapabilities: + - NET_ADMIN + - NET_RAW + - SYS_ADMIN + fsGroup: + rule: RunAsAny + hostNetwork: true + hostPorts: + - max: 7472 + min: 7472 + privileged: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - '*' +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: metallb + name: controller + namespace: metallb-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: metallb + name: speaker + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: metallb + name: metallb-system:controller +rules: +- apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - update +- apiGroups: + - '' + resources: + - services/status + verbs: + - update +- apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: metallb + name: metallb-system:speaker +rules: +- apiGroups: + - '' + resources: + - services + - endpoints + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +- apiGroups: + - extensions + resourceNames: + - speaker + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app: metallb + name: config-watcher + namespace: metallb-system +rules: +- apiGroups: + - '' + resources: + - configmaps + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: metallb + name: metallb-system:controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb-system:controller +subjects: +- kind: ServiceAccount + name: controller + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: metallb + name: metallb-system:speaker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb-system:speaker +subjects: +- kind: ServiceAccount + name: speaker + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: metallb + name: config-watcher + namespace: metallb-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: config-watcher +subjects: +- kind: ServiceAccount + name: controller +- kind: ServiceAccount + name: speaker +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: metallb + component: speaker + name: speaker + namespace: metallb-system +spec: + selector: + matchLabels: + app: metallb + component: speaker + template: + metadata: + annotations: + prometheus.io/port: '7472' + prometheus.io/scrape: 'true' + labels: + app: metallb + component: speaker + spec: + containers: + - args: + - --port=7472 + - --config=config + env: + - name: METALLB_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: METALLB_HOST + valueFrom: + fieldRef: + fieldPath: status.hostIP + image: metallb/speaker:v0.8.2 + imagePullPolicy: IfNotPresent + name: speaker + ports: + - containerPort: 7472 + name: monitoring + resources: + limits: + cpu: 100m + memory: 100Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_ADMIN + - NET_RAW + - SYS_ADMIN + drop: + - ALL + readOnlyRootFilesystem: true + hostNetwork: true + nodeSelector: + beta.kubernetes.io/os: linux + serviceAccountName: speaker + terminationGracePeriodSeconds: 0 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: metallb + component: controller + name: controller + namespace: metallb-system +spec: + revisionHistoryLimit: 3 + selector: + matchLabels: + app: metallb + component: controller + template: + metadata: + annotations: + prometheus.io/port: '7472' + prometheus.io/scrape: 'true' + labels: + app: metallb + component: controller + spec: + containers: + - args: + - --port=7472 + - --config=config + image: metallb/controller:v0.8.2 + imagePullPolicy: IfNotPresent + name: controller + ports: + - containerPort: 7472 + name: monitoring + resources: + limits: + cpu: 100m + memory: 100Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + readOnlyRootFilesystem: true + nodeSelector: + beta.kubernetes.io/os: linux + securityContext: + runAsNonRoot: true + runAsUser: 65534 + serviceAccountName: controller + terminationGracePeriodSeconds: 0 diff --git a/pkg/addons/config.go b/pkg/addons/config.go index 059ddf7929..40863db42b 100644 --- a/pkg/addons/config.go +++ b/pkg/addons/config.go @@ -122,4 +122,9 @@ var Addons = []*Addon{ set: SetBool, callbacks: []setFn{enableOrDisableStorageClasses}, }, + { + name: "metallb", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + }, } diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index b25446e8f6..0056883ba0 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -332,6 +332,20 @@ var Addons = map[string]*Addon{ "0640", false), }, false, "ingress-dns"), + "metallb": NewAddon([]*BinAsset{ + MustBinAsset( + "deploy/addons/metallb/metallb.yaml", + vmpath.GuestAddonsDir, + "metallb.yaml", + "0640", + false), + MustBinAsset( + "deploy/addons/metallb/metallb-config.yaml.tmpl", + vmpath.GuestAddonsDir, + "metallb-config.yaml", + "0640", + true), + }, false, "metallb"), } // GenerateTemplateData generates template data for template assets @@ -345,13 +359,17 @@ func GenerateTemplateData(cfg config.KubernetesConfig) interface{} { ea = "-" + runtime.GOARCH } opts := struct { - Arch string - ExoticArch string - ImageRepository string + Arch string + ExoticArch string + ImageRepository string + LoadBalancerStartIP string + LoadBalancerEndIP string }{ - Arch: a, - ExoticArch: ea, - ImageRepository: cfg.ImageRepository, + Arch: a, + ExoticArch: ea, + ImageRepository: cfg.ImageRepository, + LoadBalancerStartIP: cfg.LoadBalancerStartIP, + LoadBalancerEndIP: cfg.LoadBalancerEndIP, } return opts diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 6513efd2d9..e9a9581689 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -71,19 +71,21 @@ type MachineConfig struct { // KubernetesConfig contains the parameters used to configure the VM Kubernetes. type KubernetesConfig struct { - KubernetesVersion string - ClusterName string - APIServerName string - APIServerNames []string - APIServerIPs []net.IP - DNSDomain string - ContainerRuntime string - CRISocket string - NetworkPlugin string - FeatureGates string // https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ - ServiceCIDR string // the subnet which kubernetes services will be deployed to - ImageRepository string - ExtraOptions ExtraOptionSlice + KubernetesVersion string + ClusterName string + APIServerName string + APIServerNames []string + APIServerIPs []net.IP + DNSDomain string + ContainerRuntime string + CRISocket string + NetworkPlugin string + FeatureGates string // https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ + ServiceCIDR string // the subnet which kubernetes services will be deployed to + ImageRepository string + LoadBalancerStartIP string + LoadBalancerEndIP string + ExtraOptions ExtraOptionSlice ShouldLoadCachedImages bool EnableDefaultCNI bool From 377a4746f9d6dcb187a392f5e32422064ac7fe48 Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Thu, 5 Mar 2020 01:41:45 +0800 Subject: [PATCH 004/105] Allows to configure load balancer start/end IP --- cmd/minikube/cmd/config/configure.go | 28 ++++++++++++++++++++++++++++ cmd/minikube/cmd/config/prompt.go | 20 ++++++++++++++++++++ cmd/minikube/cmd/start.go | 6 ++++++ 3 files changed, 54 insertions(+) diff --git a/cmd/minikube/cmd/config/configure.go b/cmd/minikube/cmd/config/configure.go index 412fa1204d..5bcecc893a 100644 --- a/cmd/minikube/cmd/config/configure.go +++ b/cmd/minikube/cmd/config/configure.go @@ -18,8 +18,12 @@ package config import ( "io/ioutil" + "net" "github.com/spf13/cobra" + "github.com/spf13/viper" + + "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/service" @@ -178,6 +182,30 @@ var addonsConfigureCmd = &cobra.Command{ out.WarningT("ERROR creating `registry-creds-acr` secret") } + case "metallb": + profile := viper.GetString(config.MachineProfile) + cfg, err := config.Load(profile) + if err != nil { + out.ErrT(out.FatalType, "Failed to load config {{.profile}}", out.V{"profile": profile}) + } + + validator := func(s string) bool { + return net.ParseIP(s) != nil + } + + if cfg.KubernetesConfig.LoadBalancerStartIP == "" { + cfg.KubernetesConfig.LoadBalancerStartIP = AskForStaticValidatedValue("-- Enter Load Balancer Start IP: ", validator) + } + + if cfg.KubernetesConfig.LoadBalancerEndIP == "" { + cfg.KubernetesConfig.LoadBalancerEndIP = AskForStaticValidatedValue("-- Enter Load Balancer End IP: ", validator) + } + + err = config.SaveProfile(profile, cfg) + if err != nil { + out.ErrT(out.FatalType, "Failed to save config {{.profile}}", out.V{"profile": profile}) + } + default: out.FailureT("{{.name}} has no available configuration options", out.V{"name": addon}) return diff --git a/cmd/minikube/cmd/config/prompt.go b/cmd/minikube/cmd/config/prompt.go index 10c00a7f0f..8f85d6061c 100644 --- a/cmd/minikube/cmd/config/prompt.go +++ b/cmd/minikube/cmd/config/prompt.go @@ -153,3 +153,23 @@ func posString(slice []string, element string) int { func containsString(slice []string, element string) bool { return posString(slice, element) != -1 } + +// AskForStaticValidatedValue asks for a single value to enter and check for valid input +func AskForStaticValidatedValue(s string, validator func(s string) bool) string { + reader := bufio.NewReader(os.Stdin) + + for { + response := getStaticValue(reader, s) + + // Can't have zero length + if len(response) == 0 { + out.Err("--Error, please enter a value:") + continue + } + if !validator(response) { + out.Err("--Invalid input, please enter a value:") + continue + } + return response + } +} diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index de5002a4c9..8990756d4b 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -118,6 +118,8 @@ const ( autoUpdate = "auto-update-drivers" hostOnlyNicType = "host-only-nic-type" natNicType = "nat-nic-type" + loadBalancerStartIP = "load-balancer-start-ip" + loadBalancerEndIP = "load-balancer-end-ip" ) var ( @@ -185,6 +187,8 @@ func initKubernetesFlags() { startCmd.Flags().String(apiServerName, constants.APIServerName, "The apiserver name which is used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine") startCmd.Flags().StringArrayVar(&apiServerNames, "apiserver-names", nil, "A set of apiserver names which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine") startCmd.Flags().IPSliceVar(&apiServerIPs, "apiserver-ips", nil, "A set of apiserver IP Addresses which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine") + startCmd.Flags().String(loadBalancerStartIP, "", "The first Load Balancer IP within a range. This can be used if you want to set up Load Balancer (MetalLB)") + startCmd.Flags().String(loadBalancerEndIP, "", "The last Load Balancer IP within a range. This can be used if you want to set up Load Balancer (MetalLB)") } // initDriverFlags inits the commandline flags for vm drivers @@ -824,6 +828,8 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string) ExtraOptions: node.ExtraOptions, ShouldLoadCachedImages: viper.GetBool(cacheImages), EnableDefaultCNI: selectedEnableDefaultCNI, + LoadBalancerStartIP: viper.GetString(loadBalancerStartIP), + LoadBalancerEndIP: viper.GetString(loadBalancerEndIP), }, Nodes: []config.Node{cp}, } From 30f08824180d71dd75b3e1dacac49760d53d6c65 Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Sun, 29 Mar 2020 11:45:53 +0800 Subject: [PATCH 005/105] Fixes MachineProfile no longer exist --- cmd/minikube/cmd/config/configure.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/minikube/cmd/config/configure.go b/cmd/minikube/cmd/config/configure.go index 5bcecc893a..3f361f3bbe 100644 --- a/cmd/minikube/cmd/config/configure.go +++ b/cmd/minikube/cmd/config/configure.go @@ -21,8 +21,6 @@ import ( "net" "github.com/spf13/cobra" - "github.com/spf13/viper" - "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/out" @@ -183,7 +181,7 @@ var addonsConfigureCmd = &cobra.Command{ } case "metallb": - profile := viper.GetString(config.MachineProfile) + profile := ClusterFlagValue() cfg, err := config.Load(profile) if err != nil { out.ErrT(out.FatalType, "Failed to load config {{.profile}}", out.V{"profile": profile}) From c57375c48626962d66ffd2494a45a2b8e65b0864 Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Sun, 29 Mar 2020 12:53:25 +0800 Subject: [PATCH 006/105] Remove load balancer start/end IP from start flags --- cmd/minikube/cmd/start.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index fcb32a9052..01f27f68c4 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -122,8 +122,6 @@ const ( autoUpdate = "auto-update-drivers" hostOnlyNicType = "host-only-nic-type" natNicType = "nat-nic-type" - loadBalancerStartIP = "load-balancer-start-ip" - loadBalancerEndIP = "load-balancer-end-ip" nodes = "nodes" ) @@ -193,8 +191,6 @@ func initKubernetesFlags() { startCmd.Flags().String(apiServerName, constants.APIServerName, "The apiserver name which is used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine") startCmd.Flags().StringArrayVar(&apiServerNames, "apiserver-names", nil, "A set of apiserver names which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine") startCmd.Flags().IPSliceVar(&apiServerIPs, "apiserver-ips", nil, "A set of apiserver IP Addresses which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine") - startCmd.Flags().String(loadBalancerStartIP, "", "The first Load Balancer IP within a range. This can be used if you want to set up Load Balancer (MetalLB)") - startCmd.Flags().String(loadBalancerEndIP, "", "The last Load Balancer IP within a range. This can be used if you want to set up Load Balancer (MetalLB)") } // initDriverFlags inits the commandline flags for vm drivers @@ -1009,8 +1005,6 @@ func createNode(cmd *cobra.Command, k8sVersion, kubeNodeName, drvName, repositor ExtraOptions: config.ExtraOptions, ShouldLoadCachedImages: viper.GetBool(cacheImages), EnableDefaultCNI: selectedEnableDefaultCNI, - LoadBalancerStartIP: viper.GetString(loadBalancerStartIP), - LoadBalancerEndIP: viper.GetString(loadBalancerEndIP), }, Nodes: []config.Node{cp}, } From f078cce376e1aa0be291f140a2232274af263576 Mon Sep 17 00:00:00 2001 From: Zhongcheng Lao Date: Thu, 9 Apr 2020 23:39:06 +0800 Subject: [PATCH 007/105] Add comment on LoadBalancerStartIP/LoadBalancerEndIP --- pkg/minikube/config/types.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index a7850a6ef5..3008f8b2dd 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -81,8 +81,8 @@ type KubernetesConfig struct { FeatureGates string // https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ ServiceCIDR string // the subnet which kubernetes services will be deployed to ImageRepository string - LoadBalancerStartIP string - LoadBalancerEndIP string + LoadBalancerStartIP string // currently only used by MetalLB addon + LoadBalancerEndIP string // currently only used by MetalLB addon ExtraOptions ExtraOptionSlice ShouldLoadCachedImages bool From d7dc7bf7b2135cedf835b68285c9b0377720498d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 12 Apr 2020 16:12:30 +0200 Subject: [PATCH 008/105] The podman driver should not be run as root Use sudo for the podman commands instead Wrap the docker commands with env prefix --- cmd/minikube/cmd/delete.go | 12 +-- .../jenkins/linux_integration_tests_podman.sh | 1 - hack/preload-images/generate.go | 1 + pkg/drivers/kic/kic.go | 35 +++++---- pkg/drivers/kic/oci/info.go | 2 +- pkg/drivers/kic/oci/network.go | 8 +- pkg/drivers/kic/oci/oci.go | 74 +++++++++---------- pkg/drivers/kic/oci/types.go | 1 + pkg/drivers/kic/oci/volumes.go | 14 ++-- pkg/drivers/kic/types.go | 1 + pkg/minikube/command/kic_runner.go | 11 ++- pkg/minikube/config/profile.go | 2 +- pkg/minikube/driver/driver.go | 2 +- pkg/minikube/machine/delete.go | 12 +-- pkg/minikube/machine/stop.go | 2 +- pkg/minikube/registry/drvs/docker/docker.go | 3 +- pkg/minikube/registry/drvs/podman/podman.go | 7 +- 17 files changed, 99 insertions(+), 89 deletions(-) diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go index b25696ae34..e37949957c 100644 --- a/cmd/minikube/cmd/delete.go +++ b/cmd/minikube/cmd/delete.go @@ -90,17 +90,17 @@ func init() { func deleteContainersAndVolumes() { delLabel := fmt.Sprintf("%s=%s", oci.CreatedByLabelKey, "true") - errs := oci.DeleteContainersByLabel(oci.Docker, delLabel) + errs := oci.DeleteContainersByLabel("env", oci.Docker, delLabel) if len(errs) > 0 { // it will error if there is no container to delete glog.Infof("error delete containers by label %q (might be okay): %+v", delLabel, errs) } - errs = oci.DeleteAllVolumesByLabel(oci.Docker, delLabel) + errs = oci.DeleteAllVolumesByLabel("env", oci.Docker, delLabel) if len(errs) > 0 { // it will not error if there is nothing to delete glog.Warningf("error delete volumes by label %q (might be okay): %+v", delLabel, errs) } - errs = oci.PruneAllVolumesByLabel(oci.Docker, delLabel) + errs = oci.PruneAllVolumesByLabel("env", oci.Docker, delLabel) if len(errs) > 0 { // it will not error if there is nothing to delete glog.Warningf("error pruning volumes by label %q (might be okay): %+v", delLabel, errs) } @@ -191,16 +191,16 @@ func DeleteProfiles(profiles []*config.Profile) []error { func deleteProfileContainersAndVolumes(name string) { delLabel := fmt.Sprintf("%s=%s", oci.ProfileLabelKey, name) - errs := oci.DeleteContainersByLabel(oci.Docker, delLabel) + errs := oci.DeleteContainersByLabel("env", oci.Docker, delLabel) if errs != nil { // it will error if there is no container to delete glog.Infof("error deleting containers for %s (might be okay):\n%v", name, errs) } - errs = oci.DeleteAllVolumesByLabel(oci.Docker, delLabel) + errs = oci.DeleteAllVolumesByLabel("env", oci.Docker, delLabel) if errs != nil { // it will not error if there is nothing to delete glog.Warningf("error deleting volumes (might be okay).\nTo see the list of volumes run: 'docker volume ls'\n:%v", errs) } - errs = oci.PruneAllVolumesByLabel(oci.Docker, delLabel) + errs = oci.PruneAllVolumesByLabel("env", oci.Docker, delLabel) if len(errs) > 0 { // it will not error if there is nothing to delete glog.Warningf("error pruning volume (might be okay):\n%v", errs) } diff --git a/hack/jenkins/linux_integration_tests_podman.sh b/hack/jenkins/linux_integration_tests_podman.sh index 1dfbdb4456..455af2955b 100755 --- a/hack/jenkins/linux_integration_tests_podman.sh +++ b/hack/jenkins/linux_integration_tests_podman.sh @@ -31,7 +31,6 @@ JOB_NAME="Podman_Linux" mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES" sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP" -SUDO_PREFIX="sudo -E " EXTRA_ARGS="--container-runtime=containerd" diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go index 1a22e7404d..7475901fae 100644 --- a/hack/preload-images/generate.go +++ b/hack/preload-images/generate.go @@ -44,6 +44,7 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string driver := kic.NewDriver(kic.Config{ KubernetesVersion: kubernetesVersion, ContainerRuntime: driver.Docker, + OCIPrefix: "env", OCIBinary: oci.Docker, MachineName: profile, ImageDigest: kic.BaseImage, diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 05b1e325ca..cd7dae2803 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -48,6 +48,7 @@ type Driver struct { URL string exec command.Runner NodeConfig Config + OCIPrefix string // env, sudo OCIBinary string // docker,podman } @@ -58,8 +59,9 @@ func NewDriver(c Config) *Driver { MachineName: c.MachineName, StorePath: c.StorePath, }, - exec: command.NewKICRunner(c.MachineName, c.OCIBinary), + exec: command.NewKICRunner(c.MachineName, c.OCIPrefix, c.OCIBinary), NodeConfig: c, + OCIPrefix: c.OCIPrefix, OCIBinary: c.OCIBinary, } return d @@ -76,6 +78,7 @@ func (d *Driver) Create() error { Memory: strconv.Itoa(d.NodeConfig.Memory) + "mb", Envs: d.NodeConfig.Envs, ExtraArgs: []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.APIServerPort)}, + OCIPrefix: d.NodeConfig.OCIPrefix, OCIBinary: d.NodeConfig.OCIBinary, APIServerPort: d.NodeConfig.APIServerPort, } @@ -99,15 +102,15 @@ func (d *Driver) Create() error { }, ) - exists, err := oci.ContainerExists(d.OCIBinary, params.Name) + exists, err := oci.ContainerExists(d.OCIPrefix, d.OCIBinary, params.Name) if err != nil { glog.Warningf("failed to check if container already exists: %v", err) } if exists { // if container was created by minikube it is safe to delete and recreate it. - if oci.IsCreatedByMinikube(d.OCIBinary, params.Name) { + if oci.IsCreatedByMinikube(d.OCIPrefix, d.OCIBinary, params.Name) { glog.Info("Found already existing abandoned minikube container, will try to delete.") - if err := oci.DeleteContainer(d.OCIBinary, params.Name); err != nil { + if err := oci.DeleteContainer(d.OCIPrefix, d.OCIBinary, params.Name); err != nil { glog.Errorf("Failed to delete a conflicting minikube container %s. You might need to restart your %s daemon and delete it manually and try again: %v", params.Name, params.OCIBinary, err) } } else { @@ -159,7 +162,7 @@ func (d *Driver) prepareSSH() error { return errors.Wrap(err, "generate ssh key") } - cmder := command.NewKICRunner(d.NodeConfig.MachineName, d.NodeConfig.OCIBinary) + cmder := command.NewKICRunner(d.NodeConfig.MachineName, d.NodeConfig.OCIPrefix, d.NodeConfig.OCIBinary) f, err := assets.NewFileAsset(d.GetSSHKeyPath()+".pub", "/home/docker/.ssh/", "authorized_keys", "0644") if err != nil { return errors.Wrap(err, "create pubkey assetfile ") @@ -234,7 +237,7 @@ func (d *Driver) GetURL() (string, error) { // GetState returns the state that the host is in (running, stopped, etc) func (d *Driver) GetState() (state.State, error) { - out, err := oci.WarnIfSlow(d.NodeConfig.OCIBinary, "inspect", "-f", "{{.State.Status}}", d.MachineName) + out, err := oci.WarnIfSlow(d.NodeConfig.OCIPrefix, d.NodeConfig.OCIBinary, "inspect", "-f", "{{.State.Status}}", d.MachineName) if err != nil { return state.Error, err } @@ -259,17 +262,17 @@ func (d *Driver) GetState() (state.State, error) { // Kill stops a host forcefully, including any containers that we are managing. func (d *Driver) Kill() error { // on init this doesn't get filled when called from cmd - d.exec = command.NewKICRunner(d.MachineName, d.OCIBinary) + d.exec = command.NewKICRunner(d.MachineName, d.OCIPrefix, d.OCIBinary) if err := sysinit.New(d.exec).ForceStop("kubelet"); err != nil { glog.Warningf("couldn't force stop kubelet. will continue with kill anyways: %v", err) } - if err := oci.ShutDown(d.OCIBinary, d.MachineName); err != nil { + if err := oci.ShutDown(d.OCIPrefix, d.OCIBinary, d.MachineName); err != nil { glog.Warningf("couldn't shutdown the container, will continue with kill anyways: %v", err) } cr := command.NewExecRunner() // using exec runner for interacting with dameon. - if _, err := cr.RunCmd(exec.Command(d.NodeConfig.OCIBinary, "kill", d.MachineName)); err != nil { + if _, err := cr.RunCmd(exec.Command(d.NodeConfig.OCIPrefix, d.NodeConfig.OCIBinary, "kill", d.MachineName)); err != nil { return errors.Wrapf(err, "killing %q", d.MachineName) } return nil @@ -277,11 +280,11 @@ func (d *Driver) Kill() error { // Remove will delete the Kic Node Container func (d *Driver) Remove() error { - if _, err := oci.ContainerID(d.OCIBinary, d.MachineName); err != nil { + if _, err := oci.ContainerID(d.OCIPrefix, d.OCIBinary, d.MachineName); err != nil { glog.Infof("could not find the container %s to remove it. will try anyways", d.MachineName) } - if err := oci.DeleteContainer(d.NodeConfig.OCIBinary, d.MachineName); err != nil { + if err := oci.DeleteContainer(d.NodeConfig.OCIPrefix, d.NodeConfig.OCIBinary, d.MachineName); err != nil { if strings.Contains(err.Error(), "is already in progress") { return errors.Wrap(err, "stuck delete") } @@ -292,7 +295,7 @@ func (d *Driver) Remove() error { } // check there be no container left after delete - if id, err := oci.ContainerID(d.OCIBinary, d.MachineName); err == nil && id != "" { + if id, err := oci.ContainerID(d.OCIPrefix, d.OCIBinary, d.MachineName); err == nil && id != "" { return fmt.Errorf("expected no container ID be found for %q after delete. but got %q", d.MachineName, id) } return nil @@ -320,11 +323,11 @@ func (d *Driver) Restart() error { // Start an already created kic container func (d *Driver) Start() error { cr := command.NewExecRunner() // using exec runner for interacting with docker/podman daemon - if _, err := cr.RunCmd(exec.Command(d.NodeConfig.OCIBinary, "start", d.MachineName)); err != nil { + if _, err := cr.RunCmd(exec.Command(d.NodeConfig.OCIPrefix, d.NodeConfig.OCIBinary, "start", d.MachineName)); err != nil { return errors.Wrap(err, "start") } checkRunning := func() error { - s, err := oci.ContainerStatus(d.NodeConfig.OCIBinary, d.MachineName) + s, err := oci.ContainerStatus(d.NodeConfig.OCIPrefix, d.NodeConfig.OCIBinary, d.MachineName) if err != nil { return err } @@ -344,7 +347,7 @@ func (d *Driver) Start() error { // Stop a host gracefully, including any containers that we are managing. func (d *Driver) Stop() error { // on init this doesn't get filled when called from cmd - d.exec = command.NewKICRunner(d.MachineName, d.OCIBinary) + d.exec = command.NewKICRunner(d.MachineName, d.OCIPrefix, d.OCIBinary) // docker does not send right SIG for systemd to know to stop the systemd. // to avoid bind address be taken on an upgrade. more info https://github.com/kubernetes/minikube/issues/7171 if err := sysinit.New(d.exec).Stop("kubelet"); err != nil { @@ -379,7 +382,7 @@ func (d *Driver) Stop() error { glog.Warningf("couldn't stop kube-apiserver proc: %v", err) } - cmd := exec.Command(d.NodeConfig.OCIBinary, "stop", d.MachineName) + cmd := exec.Command(d.NodeConfig.OCIPrefix, d.NodeConfig.OCIBinary, "stop", d.MachineName) if err := cmd.Run(); err != nil { return errors.Wrapf(err, "stopping %s", d.MachineName) } diff --git a/pkg/drivers/kic/oci/info.go b/pkg/drivers/kic/oci/info.go index 548c071f16..73a944d7bb 100644 --- a/pkg/drivers/kic/oci/info.go +++ b/pkg/drivers/kic/oci/info.go @@ -234,7 +234,7 @@ func dockerSystemInfo() (dockerSysInfo, error) { // podmanSysInfo returns podman system info --format '{{json .}}' func podmanSystemInfo() (podmanSysInfo, error) { var ps podmanSysInfo - cmd := exec.Command(Podman, "system", "info", "--format", "'{{json .}}'") + cmd := exec.Command("sudo", Podman, "system", "info", "--format", "'{{json .}}'") out, err := cmd.CombinedOutput() if err != nil { return ps, errors.Wrap(err, "get podman system info") diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index 1dbb8232a5..600e77ab95 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -66,7 +66,7 @@ func dockerGatewayIP() (net.IP, error) { } bridgeID := strings.TrimSpace(string(out)) - cmd = exec.Command(Docker, "inspect", + cmd = exec.Command("env", Docker, "inspect", "--format", "{{(index .IPAM.Config 0).Gateway}}", bridgeID) out, err = cmd.CombinedOutput() @@ -90,13 +90,13 @@ func ForwardedPort(ociBinary string, ociID string, contPort int) (int, error) { if ociBinary == Podman { //podman inspect -f "{{range .NetworkSettings.Ports}}{{if eq .ContainerPort "80"}}{{.HostPort}}{{end}}{{end}}" - cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("{{range .NetworkSettings.Ports}}{{if eq .ContainerPort %s}}{{.HostPort}}{{end}}{{end}}", fmt.Sprint(contPort)), ociID) + cmd := exec.Command("sudo", ociBinary, "inspect", "-f", fmt.Sprintf("{{range .NetworkSettings.Ports}}{{if eq .ContainerPort %s}}{{.HostPort}}{{end}}{{end}}", fmt.Sprint(contPort)), ociID) out, err = cmd.CombinedOutput() if err != nil { return 0, errors.Wrapf(err, "get host-bind port %d for %q, output %s", contPort, ociID, out) } } else { - cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID) + cmd := exec.Command("env", ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID) out, err = cmd.CombinedOutput() if err != nil { return 0, errors.Wrapf(err, "get host-bind port %d for %q, output %s", contPort, ociID, out) @@ -141,7 +141,7 @@ func podmanConttainerIP(name string) (string, string, error) { // dockerContainerIP returns ipv4, ipv6 of container or error func dockerContainerIP(name string) (string, string, error) { // retrieve the IP address of the node using docker inspect - lines, err := inspect(Docker, name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}") + lines, err := inspect("env", Docker, name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}") if err != nil { return "", "", errors.Wrap(err, "inspecting NetworkSettings.Networks") } diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index d508784874..cf3ebe15c2 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -40,10 +40,10 @@ import ( // DeleteContainersByLabel deletes all containers that have a specific label // if there no containers found with the given label, it will return nil -func DeleteContainersByLabel(ociBin string, label string) []error { +func DeleteContainersByLabel(prefix string, ociBin string, label string) []error { var deleteErrs []error - cs, err := listContainersByLabel(ociBin, label) + cs, err := listContainersByLabel(prefix, ociBin, label) if err != nil { return []error{fmt.Errorf("listing containers by label %q", label)} } @@ -53,7 +53,7 @@ func DeleteContainersByLabel(ociBin string, label string) []error { } for _, c := range cs { - _, err := ContainerStatus(ociBin, c) + _, err := ContainerStatus(prefix, ociBin, c) // only try to delete if docker/podman inspect returns // if it doesn't it means docker daemon is stuck and needs restart if err != nil { @@ -61,10 +61,10 @@ func DeleteContainersByLabel(ociBin string, label string) []error { glog.Errorf("%s daemon seems to be stuck. Please try restarting your %s. :%v", ociBin, ociBin, err) continue } - if err := ShutDown(ociBin, c); err != nil { + if err := ShutDown(prefix, ociBin, c); err != nil { glog.Infof("couldn't shut down %s (might be okay): %v ", c, err) } - cmd := exec.Command(ociBin, "rm", "-f", "-v", c) + cmd := exec.Command(prefix, ociBin, "rm", "-f", "-v", c) if out, err := cmd.CombinedOutput(); err != nil { deleteErrs = append(deleteErrs, errors.Wrapf(err, "delete container %s: output %s", c, out)) } @@ -74,17 +74,17 @@ func DeleteContainersByLabel(ociBin string, label string) []error { } // DeleteContainer deletes a container by ID or Name -func DeleteContainer(ociBin string, name string) error { +func DeleteContainer(prefix string, ociBin string, name string) error { - _, err := ContainerStatus(ociBin, name) + _, err := ContainerStatus(prefix, ociBin, name) if err != nil { glog.Errorf("%s daemon seems to be stuck. Please try restarting your %s. Will try to delete anyways: %v", ociBin, ociBin, err) } // try to delete anyways - if err := ShutDown(ociBin, name); err != nil { + if err := ShutDown(prefix, ociBin, name); err != nil { glog.Infof("couldn't shut down %s (might be okay): %v ", name, err) } - cmd := exec.Command(ociBin, "rm", "-f", "-v", name) + cmd := exec.Command(prefix, ociBin, "rm", "-f", "-v", name) if out, err := cmd.CombinedOutput(); err != nil { return errors.Wrapf(err, "delete container %s: output %s", name, out) } @@ -157,18 +157,18 @@ func CreateContainerNode(p CreateParams) error { // adds node specific args runArgs = append(runArgs, p.ExtraArgs...) - if enabled := isUsernsRemapEnabled(p.OCIBinary); enabled { + if enabled := isUsernsRemapEnabled(p.OCIPrefix, p.OCIBinary); enabled { // We need this argument in order to make this command work // in systems that have userns-remap enabled on the docker daemon runArgs = append(runArgs, "--userns=host") } - if err := createContainer(p.OCIBinary, p.Image, withRunArgs(runArgs...), withMounts(p.Mounts), withPortMappings(p.PortMappings)); err != nil { + if err := createContainer(p.OCIPrefix, p.OCIBinary, p.Image, withRunArgs(runArgs...), withMounts(p.Mounts), withPortMappings(p.PortMappings)); err != nil { return errors.Wrap(err, "create container") } checkRunning := func() error { - s, err := ContainerStatus(p.OCIBinary, p.Name) + s, err := ContainerStatus(p.OCIPrefix, p.OCIBinary, p.Name) if err != nil { return fmt.Errorf("temporary error checking status for %q : %v", p.Name, err) } @@ -188,7 +188,7 @@ func CreateContainerNode(p CreateParams) error { } // CreateContainer creates a container with "docker/podman run" -func createContainer(ociBinary string, image string, opts ...createOpt) error { +func createContainer(prefix string, ociBin string, image string, opts ...createOpt) error { o := &createOpts{} for _, opt := range opts { o = opt(o) @@ -202,10 +202,10 @@ func createContainer(ociBinary string, image string, opts ...createOpt) error { runArgs = append(runArgs, generatePortMappings(portMapping)...) } // construct the actual docker run argv - args := []string{"run"} + args := []string{ociBin, "run"} // to run nested container from privileged container in podman https://bugzilla.redhat.com/show_bug.cgi?id=1687713 - if ociBinary == Podman { + if ociBin == Podman { args = append(args, "--cgroup-manager", "cgroupfs") } @@ -213,7 +213,7 @@ func createContainer(ociBinary string, image string, opts ...createOpt) error { args = append(args, image) args = append(args, o.ContainerArgs...) - out, err := exec.Command(ociBinary, args...).CombinedOutput() + out, err := exec.Command(prefix, args...).CombinedOutput() if err != nil { return errors.Wrapf(err, "failed args: %v output: %s", args, out) } @@ -222,13 +222,13 @@ func createContainer(ociBinary string, image string, opts ...createOpt) error { } // Copy copies a local asset into the container -func Copy(ociBinary string, ociID string, targetDir string, fName string) error { +func Copy(prefix string, ociBin string, ociID string, targetDir string, fName string) error { if _, err := os.Stat(fName); os.IsNotExist(err) { return errors.Wrapf(err, "error source %s does not exist", fName) } destination := fmt.Sprintf("%s:%s", ociID, targetDir) - cmd := exec.Command(ociBinary, "cp", fName, destination) + cmd := exec.Command(prefix, ociBin, "cp", fName, destination) if err := cmd.Run(); err != nil { return errors.Wrapf(err, "error copying %s into node", fName) } @@ -237,8 +237,8 @@ func Copy(ociBinary string, ociID string, targetDir string, fName string) error } // ContainerID returns id of a container name -func ContainerID(ociBinary string, nameOrID string) (string, error) { - cmd := exec.Command(ociBinary, "inspect", "-f", "{{.Id}}", nameOrID) +func ContainerID(prefix string, ociBin string, nameOrID string) (string, error) { + cmd := exec.Command(prefix, ociBin, "inspect", "-f", "{{.Id}}", nameOrID) out, err := cmd.CombinedOutput() if err != nil { // don't return error if not found, only return empty string @@ -287,8 +287,8 @@ func WarnIfSlow(args ...string) ([]byte, error) { } // ContainerExists checks if container name exists (either running or exited) -func ContainerExists(ociBin string, name string) (bool, error) { - out, err := WarnIfSlow(ociBin, "ps", "-a", "--format", "{{.Names}}") +func ContainerExists(prefix string, ociBin string, name string) (bool, error) { + out, err := WarnIfSlow(prefix, ociBin, "ps", "-a", "--format", "{{.Names}}") if err != nil { return false, errors.Wrapf(err, string(out)) } @@ -305,8 +305,8 @@ func ContainerExists(ociBin string, name string) (bool, error) { // IsCreatedByMinikube returns true if the container was created by minikube // with default assumption that it is not created by minikube when we don't know for sure -func IsCreatedByMinikube(ociBinary string, nameOrID string) bool { - cmd := exec.Command(ociBinary, "inspect", nameOrID, "--format", "{{.Config.Labels}}") +func IsCreatedByMinikube(prefix string, ociBin string, nameOrID string) bool { + cmd := exec.Command(prefix, ociBin, "inspect", nameOrID, "--format", "{{.Config.Labels}}") out, err := cmd.CombinedOutput() if err != nil { @@ -321,14 +321,14 @@ func IsCreatedByMinikube(ociBinary string, nameOrID string) bool { } // ListOwnedContainers lists all the containres that kic driver created on user's machine using a label -func ListOwnedContainers(ociBinary string) ([]string, error) { - return listContainersByLabel(ociBinary, ProfileLabelKey) +func ListOwnedContainers(prefix string, ociBin string) ([]string, error) { + return listContainersByLabel(prefix, ociBin, ProfileLabelKey) } // inspect return low-level information on containers -func inspect(ociBinary string, containerNameOrID, format string) ([]string, error) { +func inspect(prefix string, ociBin string, containerNameOrID, format string) ([]string, error) { - cmd := exec.Command(ociBinary, "inspect", + cmd := exec.Command(prefix, ociBin, "inspect", "-f", format, containerNameOrID) // ... against the "node" container var buff bytes.Buffer @@ -390,8 +390,8 @@ func generateMountBindings(mounts ...Mount) []string { } // isUsernsRemapEnabled checks if userns-remap is enabled in docker -func isUsernsRemapEnabled(ociBinary string) bool { - cmd := exec.Command(ociBinary, "info", "--format", "'{{json .SecurityOptions}}'") +func isUsernsRemapEnabled(prefix string, ociBin string) bool { + cmd := exec.Command(prefix, ociBin, "info", "--format", "'{{json .SecurityOptions}}'") var buff bytes.Buffer cmd.Stdout = &buff cmd.Stderr = &buff @@ -453,8 +453,8 @@ func withPortMappings(portMappings []PortMapping) createOpt { } // listContainersByLabel returns all the container names with a specified label -func listContainersByLabel(ociBinary string, label string) ([]string, error) { - stdout, err := WarnIfSlow(ociBinary, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}") +func listContainersByLabel(prefix string, ociBin string, label string) ([]string, error) { + stdout, err := WarnIfSlow(prefix, ociBin, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}") if err != nil { return nil, err } @@ -489,8 +489,8 @@ func PointToHostDockerDaemon() error { } // ContainerStatus returns status of a container running,exited,... -func ContainerStatus(ociBin string, name string) (state.State, error) { - out, err := WarnIfSlow(ociBin, "inspect", name, "--format={{.State.Status}}") +func ContainerStatus(prefix string, ociBin string, name string) (state.State, error) { + out, err := WarnIfSlow(prefix, ociBin, "inspect", name, "--format={{.State.Status}}") o := strings.TrimSpace(string(out)) switch o { case "running": @@ -511,8 +511,8 @@ func ContainerStatus(ociBin string, name string) (state.State, error) { // Shutdown will run command to shut down the container // to ensure the containers process and networking bindings are all closed // to avoid containers getting stuck before delete https://github.com/kubernetes/minikube/issues/7657 -func ShutDown(ociBin string, name string) error { - cmd := exec.Command(ociBin, "exec", "--privileged", "-t", name, "/bin/bash", "-c", "sudo init 0") +func ShutDown(prefix string, ociBin string, name string) error { + cmd := exec.Command(prefix, ociBin, "exec", "--privileged", "-t", name, "/bin/bash", "-c", "sudo init 0") if out, err := cmd.CombinedOutput(); err != nil { glog.Infof("error shutdown %s output %q : %v", name, out, err) } @@ -520,7 +520,7 @@ func ShutDown(ociBin string, name string) error { time.Sleep(time.Second * 1) // wait till it is stoped stopped := func() error { - st, err := ContainerStatus(ociBin, name) + st, err := ContainerStatus(prefix, ociBin, name) if st == state.Stopped { glog.Infof("container %s status is %s", name, st) return nil diff --git a/pkg/drivers/kic/oci/types.go b/pkg/drivers/kic/oci/types.go index 8c43ce7d8b..1a13e1708e 100644 --- a/pkg/drivers/kic/oci/types.go +++ b/pkg/drivers/kic/oci/types.go @@ -47,6 +47,7 @@ type CreateParams struct { Memory string // memory (mbs) to assign to the container Envs map[string]string // environment variables to pass to the container ExtraArgs []string // a list of any extra option to pass to oci binary during creation time, for example --expose 8080... + OCIPrefix string // env or sudo OCIBinary string // docker or podman } diff --git a/pkg/drivers/kic/oci/volumes.go b/pkg/drivers/kic/oci/volumes.go index 2fbd9f32b0..9babe0196b 100644 --- a/pkg/drivers/kic/oci/volumes.go +++ b/pkg/drivers/kic/oci/volumes.go @@ -29,18 +29,18 @@ import ( // DeleteAllVolumesByLabel deletes all volumes that have a specific label // if there is no volume to delete it will return nil -func DeleteAllVolumesByLabel(ociBin string, label string) []error { +func DeleteAllVolumesByLabel(prefix string, ociBin string, label string) []error { var deleteErrs []error glog.Infof("trying to delete all %s volumes with label %s", ociBin, label) - vs, err := allVolumesByLabel(ociBin, label) + vs, err := allVolumesByLabel(prefix, ociBin, label) if err != nil { return []error{fmt.Errorf("listing volumes by label %q: %v", label, err)} } for _, v := range vs { - if _, err := WarnIfSlow(ociBin, "volume", "rm", "--force", v); err != nil { + if _, err := WarnIfSlow(prefix, ociBin, "volume", "rm", "--force", v); err != nil { deleteErrs = append(deleteErrs, fmt.Errorf("deleting %q", v)) } } @@ -51,11 +51,11 @@ func DeleteAllVolumesByLabel(ociBin string, label string) []error { // PruneAllVolumesByLabel deletes all volumes that have a specific label // if there is no volume to delete it will return nil // example: docker volume prune -f --filter label=name.minikube.sigs.k8s.io=minikube -func PruneAllVolumesByLabel(ociBin string, label string) []error { +func PruneAllVolumesByLabel(prefix string, ociBin string, label string) []error { var deleteErrs []error glog.Infof("trying to prune all %s volumes with label %s", ociBin, label) - if _, err := WarnIfSlow(ociBin, "volume", "prune", "-f", "--filter", "label="+label); err != nil { + if _, err := WarnIfSlow(prefix, ociBin, "volume", "prune", "-f", "--filter", "label="+label); err != nil { deleteErrs = append(deleteErrs, errors.Wrapf(err, "prune volume by label %s", label)) } @@ -64,8 +64,8 @@ func PruneAllVolumesByLabel(ociBin string, label string) []error { // allVolumesByLabel returns name of all docker volumes by a specific label // will not return error if there is no volume found. -func allVolumesByLabel(ociBin string, label string) ([]string, error) { - cmd := exec.Command(ociBin, "volume", "ls", "--filter", "label="+label, "--format", "{{.Name}}") +func allVolumesByLabel(prefix string, ociBin string, label string) ([]string, error) { + cmd := exec.Command(prefix, ociBin, "volume", "ls", "--filter", "label="+label, "--format", "{{.Name}}") stdout, err := cmd.Output() s := bufio.NewScanner(bytes.NewReader(stdout)) var vols []string diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go index 2b01019b2e..3d68269378 100644 --- a/pkg/drivers/kic/types.go +++ b/pkg/drivers/kic/types.go @@ -49,6 +49,7 @@ type Config struct { CPU int // Number of CPU cores assigned to the container Memory int // max memory in MB StorePath string // libmachine store path + OCIPrefix string // prefix to use (env, sudo, ...) OCIBinary string // oci tool to use (docker, podman,...) ImageDigest string // image name with sha to use for the node Mounts []oci.Mount // mounts diff --git a/pkg/minikube/command/kic_runner.go b/pkg/minikube/command/kic_runner.go index 018fe3bfae..4b5a223371 100644 --- a/pkg/minikube/command/kic_runner.go +++ b/pkg/minikube/command/kic_runner.go @@ -38,19 +38,22 @@ import ( // It implements the CommandRunner interface. type kicRunner struct { nameOrID string + prefix string ociBin string } // NewKICRunner returns a kicRunner implementor of runner which runs cmds inside a container -func NewKICRunner(containerNameOrID string, oci string) Runner { +func NewKICRunner(containerNameOrID string, prefix string, oci string) Runner { return &kicRunner{ nameOrID: containerNameOrID, - ociBin: oci, // docker or podman + prefix: prefix, // env or sudo + ociBin: oci, // docker or podman } } func (k *kicRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) { args := []string{ + k.ociBin, "exec", // run with privileges so we can remount etc.. "--privileged", @@ -81,7 +84,7 @@ func (k *kicRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) { args, cmd.Args..., ) - oc := exec.Command(k.ociBin, args...) + oc := exec.Command(k.prefix, args...) oc.Stdin = cmd.Stdin oc.Stdout = cmd.Stdout oc.Stderr = cmd.Stderr @@ -199,7 +202,7 @@ func (k *kicRunner) chmod(dst string, perm string) error { // Podman cp command doesn't match docker and doesn't have -a func copyToPodman(src string, dest string) error { - if out, err := exec.Command(oci.Podman, "cp", src, dest).CombinedOutput(); err != nil { + if out, err := exec.Command("sudo", oci.Podman, "cp", src, dest).CombinedOutput(); err != nil { return errors.Wrapf(err, "podman copy %s into %s, output: %s", src, dest, string(out)) } return nil diff --git a/pkg/minikube/config/profile.go b/pkg/minikube/config/profile.go index 8345341c51..404800b670 100644 --- a/pkg/minikube/config/profile.go +++ b/pkg/minikube/config/profile.go @@ -203,7 +203,7 @@ func ListProfiles(miniHome ...string) (validPs []*Profile, inValidPs []*Profile, return nil, nil, err } // try to get profiles list based on all contrainers created by docker driver - cs, err := oci.ListOwnedContainers(oci.Docker) + cs, err := oci.ListOwnedContainers("env", oci.Docker) if err == nil { pDirs = append(pDirs, cs...) } diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index a55f7de441..1990f39733 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -126,7 +126,7 @@ func BareMetal(name string) bool { // NeedsRoot returns true if driver needs to run with root privileges func NeedsRoot(name string) bool { - return name == None || name == Podman + return name == None } // NeedsPortForward returns true if driver is unable provide direct IP connectivity diff --git a/pkg/minikube/machine/delete.go b/pkg/minikube/machine/delete.go index 8fc568e43a..2e4fdb473c 100644 --- a/pkg/minikube/machine/delete.go +++ b/pkg/minikube/machine/delete.go @@ -35,12 +35,12 @@ import ( // deleteOrphanedKIC attempts to delete an orphaned docker instance for machines without a config file // used as last effort clean up not returning errors, wont warn user. -func deleteOrphanedKIC(ociBin string, name string) { +func deleteOrphanedKIC(prefix string, ociBin string, name string) { if !(ociBin == oci.Podman || ociBin == oci.Docker) { return } - _, err := oci.ContainerStatus(ociBin, name) + _, err := oci.ContainerStatus(prefix, ociBin, name) if err != nil { glog.Infof("couldn't inspect container %q before deleting, %s-daemon might needs a restart!: %v", name, ociBin, err) return @@ -49,10 +49,10 @@ func deleteOrphanedKIC(ociBin string, name string) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - if err := oci.ShutDown(ociBin, name); err != nil { + if err := oci.ShutDown(prefix, ociBin, name); err != nil { glog.Infof("couldn't shut down %s (might be okay): %v ", name, err) } - cmd := exec.CommandContext(ctx, ociBin, "rm", "-f", "-v", name) + cmd := exec.CommandContext(ctx, prefix, ociBin, "rm", "-f", "-v", name) err = cmd.Run() if err == nil { glog.Infof("Found stale kic container and successfully cleaned it up!") @@ -63,8 +63,8 @@ func deleteOrphanedKIC(ociBin string, name string) { func DeleteHost(api libmachine.API, machineName string) error { host, err := api.Load(machineName) if err != nil && host == nil { - deleteOrphanedKIC(oci.Docker, machineName) - deleteOrphanedKIC(oci.Podman, machineName) + deleteOrphanedKIC("env", oci.Docker, machineName) + deleteOrphanedKIC("sudo", oci.Podman, machineName) // Keep going even if minikube does not know about the host } diff --git a/pkg/minikube/machine/stop.go b/pkg/minikube/machine/stop.go index d5cac2b2d3..f8d5fa3cf3 100644 --- a/pkg/minikube/machine/stop.go +++ b/pkg/minikube/machine/stop.go @@ -80,7 +80,7 @@ func trySSHPowerOff(h *host.Host) error { out.T(out.Shutdown, `Powering off "{{.profile_name}}" via SSH ...`, out.V{"profile_name": h.Name}) // differnet for kic because RunSSHCommand is not implemented by kic if driver.IsKIC(h.DriverName) { - err := oci.ShutDown(h.DriverName, h.Name) + err := oci.ShutDown("sudo", h.DriverName, h.Name) glog.Infof("shutdown container: err=%v", err) } else { out, err := h.RunSSHCommand("sudo poweroff") diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index 88a190897a..ee651b487d 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -47,7 +47,7 @@ func init() { if err := registry.Register(registry.DriverDef{ Name: driver.Docker, Config: configure, - Init: func() drivers.Driver { return kic.NewDriver(kic.Config{OCIBinary: oci.Docker}) }, + Init: func() drivers.Driver { return kic.NewDriver(kic.Config{OCIPrefix: "env", OCIBinary: oci.Docker}) }, Status: status, Priority: priority, }); err != nil { @@ -62,6 +62,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { ImageDigest: kic.BaseImage, CPU: cc.CPUs, Memory: cc.Memory, + OCIPrefix: "env", OCIBinary: oci.Docker, APIServerPort: cc.Nodes[0].Port, KubernetesVersion: cc.KubernetesConfig.KubernetesVersion, diff --git a/pkg/minikube/registry/drvs/podman/podman.go b/pkg/minikube/registry/drvs/podman/podman.go index 468b1814b8..2d199c7523 100644 --- a/pkg/minikube/registry/drvs/podman/podman.go +++ b/pkg/minikube/registry/drvs/podman/podman.go @@ -41,7 +41,7 @@ func init() { if err := registry.Register(registry.DriverDef{ Name: driver.Podman, Config: configure, - Init: func() drivers.Driver { return kic.NewDriver(kic.Config{OCIBinary: oci.Podman}) }, + Init: func() drivers.Driver { return kic.NewDriver(kic.Config{OCIPrefix: "sudo", OCIBinary: oci.Podman}) }, Status: status, Priority: registry.Experimental, }); err != nil { @@ -56,6 +56,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { ImageDigest: strings.Split(kic.BaseImage, "@")[0], // for podman does not support docker images references with both a tag and digest. CPU: cc.CPUs, Memory: cc.Memory, + OCIPrefix: "sudo", OCIBinary: oci.Podman, APIServerPort: cc.Nodes[0].Port, }), nil @@ -71,7 +72,7 @@ func status() registry.State { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - cmd := exec.CommandContext(ctx, oci.Podman, "version", "-f", "{{.Version}}") + cmd := exec.CommandContext(ctx, "sudo", oci.Podman, "version", "-f", "{{.Version}}") o, err := cmd.CombinedOutput() output := string(o) if err != nil { @@ -89,7 +90,7 @@ func status() registry.State { // Allow no more than 3 seconds for querying state ctx, cancel = context.WithTimeout(context.Background(), 3*time.Second) defer cancel() - err = exec.CommandContext(ctx, oci.Podman, "info").Run() + err = exec.CommandContext(ctx, "sudo", oci.Podman, "info").Run() if err != nil { return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Podman is not running or taking too long to respond. Try: restarting podman."} } From f264ac171e7b12334c550067202839e9e739f907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 13 Apr 2020 10:35:34 +0200 Subject: [PATCH 009/105] Remove extra single-quotes from format parameter podman Error: template parsing error: template: image:1: function "json" not defined --- pkg/drivers/kic/oci/info.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/drivers/kic/oci/info.go b/pkg/drivers/kic/oci/info.go index 73a944d7bb..b0fcbc981e 100644 --- a/pkg/drivers/kic/oci/info.go +++ b/pkg/drivers/kic/oci/info.go @@ -234,7 +234,7 @@ func dockerSystemInfo() (dockerSysInfo, error) { // podmanSysInfo returns podman system info --format '{{json .}}' func podmanSystemInfo() (podmanSysInfo, error) { var ps podmanSysInfo - cmd := exec.Command("sudo", Podman, "system", "info", "--format", "'{{json .}}'") + cmd := exec.Command("sudo", Podman, "system", "info", "--format", "{{json .}}") out, err := cmd.CombinedOutput() if err != nil { return ps, errors.Wrap(err, "get podman system info") From 401e94c854bee74330d0b479c7bee9072f368c3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 13 Apr 2020 10:39:25 +0200 Subject: [PATCH 010/105] Don't download podman images to docker daemon --- pkg/minikube/node/cache.go | 20 +++++++++++++------- pkg/minikube/node/start.go | 2 +- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go index dd01433ca8..96176b6abd 100644 --- a/pkg/minikube/node/cache.go +++ b/pkg/minikube/node/cache.go @@ -99,14 +99,20 @@ func doCacheBinaries(k8sVersion string) error { } // BeginDownloadKicArtifacts downloads the kic image + preload tarball, returns true if preload is available -func beginDownloadKicArtifacts(g *errgroup.Group) { +func beginDownloadKicArtifacts(g *errgroup.Group, driver string, cRuntime string) { glog.Info("Beginning downloading kic artifacts") - if !image.ExistsImageInDaemon(kic.BaseImage) { - out.T(out.Pulling, "Pulling base image ...") - g.Go(func() error { - glog.Infof("Downloading %s to local daemon", kic.BaseImage) - return image.WriteImageToDaemon(kic.BaseImage) - }) + glog.Infof("Beginning downloading kic artifacts for %s with %s", driver, cRuntime) + if driver == "docker" { + if !image.ExistsImageInDaemon(kic.BaseImage) { + out.T(out.Pulling, "Pulling base image ...") + g.Go(func() error { + glog.Infof("Downloading %s to local daemon", kic.BaseImage) + return image.WriteImageToDaemon(kic.BaseImage) + }) + } + } else { + // TODO: driver == "podman" + glog.Info("Container runtime isn't docker, skipping download") } } diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 5f2aecb0d9..6bc46d9c1f 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -201,7 +201,7 @@ func Provision(cc *config.ClusterConfig, n *config.Node, apiServer bool) (comman } if driver.IsKIC(cc.Driver) { - beginDownloadKicArtifacts(&kicGroup) + beginDownloadKicArtifacts(&kicGroup, cc.Driver, cc.KubernetesConfig.ContainerRuntime) } if !driver.BareMetal(cc.Driver) { From 22aa1aff228a11036110ed1893c487ca4b8c43ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 13 Apr 2020 13:43:45 +0200 Subject: [PATCH 011/105] Handle more state and status for the podman driver --- pkg/drivers/kic/kic.go | 2 ++ pkg/drivers/kic/oci/oci.go | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index cd7dae2803..89ff735e86 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -244,6 +244,8 @@ func (d *Driver) GetState() (state.State, error) { o := strings.TrimSpace(string(out)) switch o { + case "configured": + return state.Stopped, nil case "running": return state.Running, nil case "exited": diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index cf3ebe15c2..0edd983af8 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -36,6 +36,7 @@ import ( "fmt" "os/exec" "strings" + "strconv" ) // DeleteContainersByLabel deletes all containers that have a specific label @@ -168,6 +169,13 @@ func CreateContainerNode(p CreateParams) error { } checkRunning := func() error { + r, err := ContainerRunning(p.OCIPrefix, p.OCIBinary, p.Name) + if err != nil { + return fmt.Errorf("temporary error checking running for %q : %v", p.Name, err) + } + if !r { + return fmt.Errorf("temporary error created container %q is not running yet", p.Name) + } s, err := ContainerStatus(p.OCIPrefix, p.OCIBinary, p.Name) if err != nil { return fmt.Errorf("temporary error checking status for %q : %v", p.Name, err) @@ -488,6 +496,15 @@ func PointToHostDockerDaemon() error { return nil } +// ContainerRunning returns running state of a container +func ContainerRunning(prefix string, ociBin string, name string) (bool, error) { + out, err := WarnIfSlow(prefix, ociBin, "inspect", name, "--format={{.State.Running}}") + if err != nil { + return false, err + } + return strconv.ParseBool(strings.TrimSpace(string(out))) +} + // ContainerStatus returns status of a container running,exited,... func ContainerStatus(prefix string, ociBin string, name string) (state.State, error) { out, err := WarnIfSlow(prefix, ociBin, "inspect", name, "--format={{.State.Status}}") From f57faf8036c8ecbcb93a6621324e2b547b794d40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 13 Apr 2020 16:04:43 +0200 Subject: [PATCH 012/105] Only remount the /var/lib/minikube directory --- pkg/drivers/kic/oci/oci.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 0edd983af8..eeb491e873 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -137,12 +137,12 @@ func CreateContainerNode(p CreateParams) error { if p.OCIBinary == Podman { // enable execing in /var // volume path in minikube home folder to mount to /var - hostVarVolPath := filepath.Join(localpath.MiniPath(), "machines", p.Name, "var") + hostVarVolPath := filepath.Join(localpath.MiniPath(), "machines", p.Name, "var", "lib", "minikube") if err := os.MkdirAll(hostVarVolPath, 0711); err != nil { return errors.Wrapf(err, "create var dir %s", hostVarVolPath) } // podman mounts var/lib with no-exec by default https://github.com/containers/libpod/issues/5103 - runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var:exec", hostVarVolPath)) + runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var/lib/minikube:exec", hostVarVolPath)) } if p.OCIBinary == Docker { runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name)) From df3aec6b0a494e72351f2e9807b9594a1e45606a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 13 Apr 2020 16:06:12 +0200 Subject: [PATCH 013/105] Missed to wrap one podman inspect in sudo proper --- pkg/drivers/kic/oci/network.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index 600e77ab95..cdb9ce091b 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -124,7 +124,7 @@ func ContainerIPs(ociBinary string, name string) (string, string, error) { // podmanConttainerIP returns ipv4, ipv6 of container or error func podmanConttainerIP(name string) (string, string, error) { - cmd := exec.Command(Podman, "inspect", + cmd := exec.Command("sudo", Podman, "inspect", "-f", "{{.NetworkSettings.IPAddress}}", name) out, err := cmd.CombinedOutput() From 223424652cbce7d638b722e09a32520c9873d721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 13 Apr 2020 16:49:07 +0200 Subject: [PATCH 014/105] Also create and mount the container runtime dirs --- pkg/drivers/kic/oci/oci.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index eeb491e873..20397cbe57 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -35,8 +35,8 @@ import ( "fmt" "os/exec" - "strings" "strconv" + "strings" ) // DeleteContainersByLabel deletes all containers that have a specific label @@ -137,12 +137,24 @@ func CreateContainerNode(p CreateParams) error { if p.OCIBinary == Podman { // enable execing in /var // volume path in minikube home folder to mount to /var - hostVarVolPath := filepath.Join(localpath.MiniPath(), "machines", p.Name, "var", "lib", "minikube") - if err := os.MkdirAll(hostVarVolPath, 0711); err != nil { + hostVarVolPath := filepath.Join(localpath.MiniPath(), "machines", p.Name, "var", "lib") + if err := os.MkdirAll(hostVarVolPath, 0755); err != nil { return errors.Wrapf(err, "create var dir %s", hostVarVolPath) } + if err := os.Mkdir(fmt.Sprintf("%s/%s", hostVarVolPath, "minikube"), 0711); err != nil { + return errors.Wrapf(err, "create var dir %s/%s", hostVarVolPath, "minikube") + } + hostVarLibSubdirs := []string{"docker", "containerd", "containers"} + for _, subdir := range hostVarLibSubdirs { + if err := os.Mkdir(fmt.Sprintf("%s/%s", hostVarVolPath, subdir), 0711); err != nil { + return errors.Wrapf(err, "create var dir %s/%s", hostVarVolPath, subdir) + } + } // podman mounts var/lib with no-exec by default https://github.com/containers/libpod/issues/5103 - runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var/lib/minikube:exec", hostVarVolPath)) + runArgs = append(runArgs, "--volume", fmt.Sprintf("%s/minikube:/var/lib/minikube:exec", hostVarVolPath)) + for _, subdir := range hostVarLibSubdirs { + runArgs = append(runArgs, "--volume", fmt.Sprintf("%s/%s:/var/lib/%s", hostVarVolPath, subdir, subdir)) + } } if p.OCIBinary == Docker { runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name)) From 8987b1067acf2a6b5c8915de149723e67b8540bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 13 Apr 2020 17:33:48 +0200 Subject: [PATCH 015/105] Also persist the kubelet and the cni state dirs --- pkg/drivers/kic/oci/oci.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 20397cbe57..55a4a24f31 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -144,7 +144,7 @@ func CreateContainerNode(p CreateParams) error { if err := os.Mkdir(fmt.Sprintf("%s/%s", hostVarVolPath, "minikube"), 0711); err != nil { return errors.Wrapf(err, "create var dir %s/%s", hostVarVolPath, "minikube") } - hostVarLibSubdirs := []string{"docker", "containerd", "containers"} + hostVarLibSubdirs := []string{"docker", "containerd", "containers", "kubelet", "cni"} for _, subdir := range hostVarLibSubdirs { if err := os.Mkdir(fmt.Sprintf("%s/%s", hostVarVolPath, subdir), 0711); err != nil { return errors.Wrapf(err, "create var dir %s/%s", hostVarVolPath, subdir) From 45ec38d2e63ee7b576e32e5addc5fb962a6c46c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 13 Apr 2020 18:07:45 +0200 Subject: [PATCH 016/105] Fix hack binary, as pointed out by golangci-lint --- hack/preload-images/generate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go index 7475901fae..b915a2a9b3 100644 --- a/hack/preload-images/generate.go +++ b/hack/preload-images/generate.go @@ -87,7 +87,7 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string kcfg := config.KubernetesConfig{ KubernetesVersion: kubernetesVersion, } - runner := command.NewKICRunner(profile, driver.OCIBinary) + runner := command.NewKICRunner(profile, driver.OCIPrefix, driver.OCIBinary) sm := sysinit.New(runner) if err := bsutil.TransferBinaries(kcfg, runner, sm); err != nil { From 024cd6b108ed35aa17cee599bb0c1cc353a718bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Tue, 14 Apr 2020 21:39:00 +0200 Subject: [PATCH 017/105] Use the same kind of named /var mount for podman The internal anonymous mounts do not seem to have the same issues as the external path mounts, so we can do all of /var --- pkg/drivers/kic/oci/oci.go | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 55a4a24f31..78714ba0ea 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -19,7 +19,6 @@ package oci import ( "context" "os" - "path/filepath" "time" "bufio" @@ -29,7 +28,6 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/constants" - "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/util/retry" @@ -136,25 +134,8 @@ func CreateContainerNode(p CreateParams) error { } if p.OCIBinary == Podman { // enable execing in /var - // volume path in minikube home folder to mount to /var - hostVarVolPath := filepath.Join(localpath.MiniPath(), "machines", p.Name, "var", "lib") - if err := os.MkdirAll(hostVarVolPath, 0755); err != nil { - return errors.Wrapf(err, "create var dir %s", hostVarVolPath) - } - if err := os.Mkdir(fmt.Sprintf("%s/%s", hostVarVolPath, "minikube"), 0711); err != nil { - return errors.Wrapf(err, "create var dir %s/%s", hostVarVolPath, "minikube") - } - hostVarLibSubdirs := []string{"docker", "containerd", "containers", "kubelet", "cni"} - for _, subdir := range hostVarLibSubdirs { - if err := os.Mkdir(fmt.Sprintf("%s/%s", hostVarVolPath, subdir), 0711); err != nil { - return errors.Wrapf(err, "create var dir %s/%s", hostVarVolPath, subdir) - } - } // podman mounts var/lib with no-exec by default https://github.com/containers/libpod/issues/5103 - runArgs = append(runArgs, "--volume", fmt.Sprintf("%s/minikube:/var/lib/minikube:exec", hostVarVolPath)) - for _, subdir := range hostVarLibSubdirs { - runArgs = append(runArgs, "--volume", fmt.Sprintf("%s/%s:/var/lib/%s", hostVarVolPath, subdir, subdir)) - } + runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var:exec", p.Name)) } if p.OCIBinary == Docker { runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name)) From 19be5619d6b5a0fe02d02c76a0a4f76958244088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Tue, 14 Apr 2020 21:40:20 +0200 Subject: [PATCH 018/105] Resource limits seem to work ok when not rootless --- pkg/drivers/kic/oci/oci.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 78714ba0ea..60c7997498 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -139,11 +139,10 @@ func CreateContainerNode(p CreateParams) error { } if p.OCIBinary == Docker { runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name)) - // setting resource limit in privileged mode is only supported by docker - // podman error: "Error: invalid configuration, cannot set resources with rootless containers not using cgroups v2 unified mode" - runArgs = append(runArgs, fmt.Sprintf("--cpus=%s", p.CPUs), fmt.Sprintf("--memory=%s", p.Memory)) } + runArgs = append(runArgs, fmt.Sprintf("--cpus=%s", p.CPUs), fmt.Sprintf("--memory=%s", p.Memory)) + for key, val := range p.Envs { runArgs = append(runArgs, "-e", fmt.Sprintf("%s=%s", key, val)) } From e99340b4ac8daa800ae9955a377afd4c44cba16a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sat, 18 Apr 2020 18:19:37 +0200 Subject: [PATCH 019/105] Improve podman status when checking for sudo Only use sudo for info and running on linux --- pkg/minikube/registry/drvs/podman/podman.go | 46 +++++++++++++++++---- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/pkg/minikube/registry/drvs/podman/podman.go b/pkg/minikube/registry/drvs/podman/podman.go index 2d199c7523..d7034c9df0 100644 --- a/pkg/minikube/registry/drvs/podman/podman.go +++ b/pkg/minikube/registry/drvs/podman/podman.go @@ -19,7 +19,10 @@ package podman import ( "context" "fmt" + "os" "os/exec" + "os/user" + "runtime" "strings" "time" @@ -63,16 +66,17 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { } func status() registry.State { - _, err := exec.LookPath(oci.Podman) + docURL := "https://minikube.sigs.k8s.io/docs/drivers/podman/" + podman, err := exec.LookPath(oci.Podman) if err != nil { - return registry.State{Error: err, Installed: false, Healthy: false, Fix: "Podman is required.", Doc: "https://minikube.sigs.k8s.io/docs/reference/drivers/podman/"} + return registry.State{Error: err, Installed: false, Healthy: false, Fix: "Install Podman", Doc: docURL} } // Allow no more than 2 seconds for version command ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - cmd := exec.CommandContext(ctx, "sudo", oci.Podman, "version", "-f", "{{.Version}}") + cmd := exec.CommandContext(ctx, oci.Podman, "version", "-f", "{{.Version}}") o, err := cmd.CombinedOutput() output := string(o) if err != nil { @@ -87,13 +91,41 @@ func status() registry.State { if v.LT(minReqPodmanVer) { glog.Warningf("Warning ! mininim required version for podman is %s. your version is %q. minikube might not work. use at your own risk. To install latest version please see https://podman.io/getting-started/installation.html ", minReqPodmanVer.String(), v.String()) } + // Allow no more than 3 seconds for querying state ctx, cancel = context.WithTimeout(context.Background(), 3*time.Second) defer cancel() - err = exec.CommandContext(ctx, "sudo", oci.Podman, "info").Run() - if err != nil { - return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Podman is not running or taking too long to respond. Try: restarting podman."} + + // Run with sudo on linux (local), otherwise podman-remote (as podman) + if runtime.GOOS == "linux" { + cmd = exec.CommandContext(ctx, "sudo", "-n", oci.Podman, "info") + cmd.Env = append(os.Environ(), "LANG=C", "LC_ALL=C") // sudo is localized + } else { + cmd = exec.CommandContext(ctx, oci.Podman, "info") + } + _, err = cmd.Output() + if err == nil { + return registry.State{Installed: true, Healthy: true} } - return registry.State{Installed: true, Healthy: true} + glog.Warningf("podman returned error: %v", err) + + if exitErr, ok := err.(*exec.ExitError); ok { + stderr := strings.TrimSpace(string(exitErr.Stderr)) + newErr := fmt.Errorf(`%q %v: %s`, strings.Join(cmd.Args, " "), exitErr, stderr) + + username := "$USER" + if u, err := user.Current(); err == nil { + username = u.Username + } + + if strings.Contains(stderr, "a password is required") && runtime.GOOS == "linux" { + return registry.State{Error: newErr, Installed: true, Healthy: false, Fix: fmt.Sprintf("Add your user to the 'sudoers' file: '%s ALL=(ALL) NOPASSWD: %s'", username, podman), Doc: "https://podman.io"} + } + + // We don't have good advice, but at least we can provide a good error message + return registry.State{Error: newErr, Installed: true, Healthy: false, Doc: docURL} + } + + return registry.State{Error: err, Installed: true, Healthy: false, Doc: docURL} } From a2d180ec075146c2a900fcbcc505434e2f4c01e9 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 20 Apr 2020 17:43:03 -0700 Subject: [PATCH 020/105] Docker driver: force docker to use systemd as cgroup manager Since minikube is running systemd, kubeadm expects kubeadm to be the cgroup manager. If docker is using a different cgroup manager like cgroupfs, this can cause unstable resource allocation. We were seeing this in Cloud Shell, and forcing docker to use systemd resolved the issue. --- pkg/minikube/cruntime/containerd.go | 5 +++++ pkg/minikube/cruntime/crio.go | 5 +++++ pkg/minikube/cruntime/cruntime.go | 2 ++ pkg/minikube/cruntime/docker.go | 19 +++++++++++++++++++ pkg/minikube/node/start.go | 7 +++++++ 5 files changed, 38 insertions(+) diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 73261c9819..1636a14257 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -429,3 +429,8 @@ func addRepoTagToImageName(imgName string) string { } // else it already has repo name dont add anything return imgName } + +// TODO: Enable for containerd +func (r *Containerd) ForceSystemdCgroupManager() error { + return nil +} diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 250d765df4..a335f45134 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -229,3 +229,8 @@ func (r *CRIO) Preload(cfg config.KubernetesConfig) error { } return fmt.Errorf("not yet implemented for %s", r.Name()) } + +// ForceSystemdCgroupManager does nothing since CRIO already uses systemd +func (r *CRIO) ForceSystemdCgroupManager() error { + return nil +} diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index f1866ff1aa..8b0e231376 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -104,6 +104,8 @@ type Manager interface { SystemLogCmd(int) string // Preload preloads the container runtime with k8s images Preload(config.KubernetesConfig) error + // ForceSystemdCgroupManager forces the runtime to use systemd as cgroup manager + ForceSystemdCgroupManager() error } // Config is runtime configuration diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index eb11b58e83..ce1785e50b 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -365,6 +365,25 @@ func dockerImagesPreloaded(runner command.Runner, images []string) bool { return true } +// ForceSystemdCgroupManager forces docker to use systemd as cgroup manager, as recommended in k8s docs: +// https://kubernetes.io/docs/setup/production-environment/container-runtimes/#docker +func (r *Docker) ForceSystemdCgroupManager() error { + daemonConfig := `{ +"exec-opts": ["native.cgroupdriver=systemd"], +"log-driver": "json-file", +"log-opts": { + "max-size": "100m" +}, +"storage-driver": "overlay2" +} +` + ma := assets.NewMemoryAsset([]byte(daemonConfig), "/etc/docker", "daemon.json", "0644") + if err := r.Runner.Copy(ma); err != nil { + return errors.Wrap(err, "copying daemon config") + } + return r.Restart() +} + func dockerBoundToContainerd(runner command.Runner) bool { // NOTE: assumes systemd rr, err := runner.RunCmd(exec.Command("sudo", "systemctl", "cat", "docker.service")) diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 5f2aecb0d9..2bc18f4736 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -99,6 +99,7 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { // setup kubeadm (must come after setupKubeconfig) bs = setupKubeAdm(starter.MachineAPI, *starter.Cfg, *starter.Node, starter.Runner) + err = bs.StartCluster(*starter.Cfg) if err != nil { @@ -261,6 +262,12 @@ func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, k exit.WithError("Failed to enable container runtime", err) } + if driver.IsKIC(cc.Driver) { + if err := cr.ForceSystemdCgroupManager(); err != nil { + glog.Warningf("Failed to force %s to use systemd as cgroup manager (this might be ok): %v", cr.Name(), err) + } + } + return cr } From b50e816d1078c86ef6ff5ea17e307bb15c73e0a5 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Tue, 21 Apr 2020 11:14:50 -0700 Subject: [PATCH 021/105] Make inability to add host IP non-fatal --- pkg/minikube/node/start.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index fb4324026d..39233950dd 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -88,12 +88,14 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { cr := configureRuntimes(starter.Runner, *starter.Cfg, sv) showVersionInfo(starter.Node.KubernetesVersion, cr) - hostIP, err := cluster.GetVMHostIP(starter.Host) + // Add "host.minikube.internal" DNS alias (intentionally non-fatal) + hostIP, err := cluster.HostIP(starter.Host) if err != nil { - exit.WithError("Unable to get VM IP", err) - } - if err := machine.AddHostAlias(starter.Runner, constants.HostAlias, hostIP); err != nil { - exit.WithError("Unable to add host alias", err) + glog.Errorf("Unable to get VM IP", err) + } else { + if err := machine.AddHostAlias(starter.Runner, constants.HostAlias, hostIP); err != nil { + glog.Errorf("Unable to add host alias", err) + } } var bs bootstrapper.Bootstrapper From 23fa28f739d7e89043759784fa0f97b92b0788c4 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Tue, 21 Apr 2020 11:15:04 -0700 Subject: [PATCH 022/105] Add HostIP support for none, reduce tech debt --- cmd/minikube/cmd/mount.go | 2 +- cmd/minikube/cmd/status.go | 2 +- pkg/minikube/cluster/ip.go | 12 +++++++----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cmd/minikube/cmd/mount.go b/cmd/minikube/cmd/mount.go index fa0db9f75d..ec64cd6638 100644 --- a/cmd/minikube/cmd/mount.go +++ b/cmd/minikube/cmd/mount.go @@ -106,7 +106,7 @@ var mountCmd = &cobra.Command{ var ip net.IP var err error if mountIP == "" { - ip, err = cluster.GetVMHostIP(co.CP.Host) + ip, err = cluster.HostIP(co.CP.Host) if err != nil { exit.WithError("Error getting the host IP address to use from within the VM", err) } diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index 5f4ca77767..da21d748c2 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -188,7 +188,7 @@ func status(api libmachine.API, cc config.ClusterConfig, n config.Node) (*Status } // We have a fully operational host, now we can check for details - if _, err := cluster.GetHostDriverIP(api, name); err != nil { + if _, err := cluster.DriverIP(api, name); err != nil { glog.Errorf("failed to get driver ip: %v", err) st.Host = state.Error.String() return st, err diff --git a/pkg/minikube/cluster/ip.go b/pkg/minikube/cluster/ip.go index 7ec11064c2..2addca2bf6 100644 --- a/pkg/minikube/cluster/ip.go +++ b/pkg/minikube/cluster/ip.go @@ -30,8 +30,8 @@ import ( "k8s.io/minikube/pkg/minikube/machine" ) -// GetVMHostIP gets the ip address to be used for mapping host -> VM and VM -> host -func GetVMHostIP(host *host.Host) (net.IP, error) { +// HostIP gets the ip address to be used for mapping host -> VM and VM -> host +func HostIP(host *host.Host) (net.IP, error) { switch host.DriverName { case driver.Docker: return oci.RoutableHostIPFromInside(oci.Docker, host.Name) @@ -72,13 +72,15 @@ func GetVMHostIP(host *host.Host) (net.IP, error) { return []byte{}, errors.Wrap(err, "Error converting VM IP address to IPv4 address") } return net.IPv4(vmIP[0], vmIP[1], vmIP[2], byte(1)), nil + case driver.None: + return net.ParseIP("127.0.0.1"), nil default: - return []byte{}, errors.New("Error, attempted to get host ip address for unsupported driver") + return []byte{}, fmt.Errorf("HostIP not yet implemented for %q driver", host.DriverName) } } -// GetHostDriverIP gets the ip address of the current minikube cluster -func GetHostDriverIP(api libmachine.API, machineName string) (net.IP, error) { +// DriverIP gets the ip address of the current minikube cluster +func DriverIP(api libmachine.API, machineName string) (net.IP, error) { host, err := machine.LoadHost(api, machineName) if err != nil { return nil, err From 6e2d0867212ec39452928d2593aebf6700dd89c3 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Tue, 21 Apr 2020 11:18:44 -0700 Subject: [PATCH 023/105] Address lint errors --- pkg/minikube/node/start.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 39233950dd..cb3931dbff 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -91,11 +91,9 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { // Add "host.minikube.internal" DNS alias (intentionally non-fatal) hostIP, err := cluster.HostIP(starter.Host) if err != nil { - glog.Errorf("Unable to get VM IP", err) - } else { - if err := machine.AddHostAlias(starter.Runner, constants.HostAlias, hostIP); err != nil { - glog.Errorf("Unable to add host alias", err) - } + glog.Errorf("Unable to get host IP: %v", err) + } else if err := machine.AddHostAlias(starter.Runner, constants.HostAlias, hostIP); err != nil { + glog.Errorf("Unable to add host alias: %v", err) } var bs bootstrapper.Bootstrapper From bb6200516c8d98756126af55f3ceaea0858800cc Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Tue, 21 Apr 2020 11:21:19 -0700 Subject: [PATCH 024/105] Update examples --- .../bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml | 2 +- .../bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.11/image-repository.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml | 2 +- .../bsutil/testdata/v1.12/containerd-pod-network-cidr.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.12/default.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.12/dns.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.12/image-repository.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.12/options.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.13/containerd-api-port.yaml | 2 +- .../bsutil/testdata/v1.13/containerd-pod-network-cidr.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.13/crio-options-gates.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.13/default.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.13/dns.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.13/image-repository.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.13/options.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml | 2 +- .../bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.14/image-repository.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml | 2 +- .../bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.15/image-repository.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml | 2 +- .../bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.16/image-repository.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml | 2 +- .../bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.17/image-repository.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.18/containerd-api-port.yaml | 2 +- .../bsutil/testdata/v1.18/containerd-pod-network-cidr.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.18/containerd.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.18/crio-options-gates.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.18/crio.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.18/default.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.18/dns.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.18/image-repository.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.18/options.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.19/containerd-api-port.yaml | 2 +- .../bsutil/testdata/v1.19/containerd-pod-network-cidr.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.19/containerd.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.19/crio-options-gates.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.19/crio.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.19/default.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.19/dns.yaml | 2 +- .../bootstrapper/bsutil/testdata/v1.19/image-repository.yaml | 2 +- pkg/minikube/bootstrapper/bsutil/testdata/v1.19/options.yaml | 2 +- 80 files changed, 80 insertions(+), 80 deletions(-) diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml index ae79c8aa7a..be58f504e9 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml @@ -4,7 +4,7 @@ noTaintMaster: true api: advertiseAddress: 1.1.1.1 bindPort: 12345 - controlPlaneEndpoint: 1.1.1.1 + controlPlaneEndpoint: control-plane.minikube.internal kubernetesVersion: v1.11.0 certificatesDir: /var/lib/minikube/certs networking: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml index a8ce3c8dc7..043abb8632 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml @@ -4,7 +4,7 @@ noTaintMaster: true api: advertiseAddress: 1.1.1.1 bindPort: 8443 - controlPlaneEndpoint: 1.1.1.1 + controlPlaneEndpoint: control-plane.minikube.internal kubernetesVersion: v1.11.0 certificatesDir: /var/lib/minikube/certs networking: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml index a8ce3c8dc7..043abb8632 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml @@ -4,7 +4,7 @@ noTaintMaster: true api: advertiseAddress: 1.1.1.1 bindPort: 8443 - controlPlaneEndpoint: 1.1.1.1 + controlPlaneEndpoint: control-plane.minikube.internal kubernetesVersion: v1.11.0 certificatesDir: /var/lib/minikube/certs networking: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml index 1a4d370e84..1256ec38b5 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml @@ -4,7 +4,7 @@ noTaintMaster: true api: advertiseAddress: 1.1.1.1 bindPort: 8443 - controlPlaneEndpoint: 1.1.1.1 + controlPlaneEndpoint: control-plane.minikube.internal kubernetesVersion: v1.11.0 certificatesDir: /var/lib/minikube/certs networking: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml index e179fbf4e3..6481d10b90 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml @@ -4,7 +4,7 @@ noTaintMaster: true api: advertiseAddress: 1.1.1.1 bindPort: 8443 - controlPlaneEndpoint: 1.1.1.1 + controlPlaneEndpoint: control-plane.minikube.internal kubernetesVersion: v1.11.0 certificatesDir: /var/lib/minikube/certs networking: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml index 68429da7bc..ee015643cf 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml @@ -4,7 +4,7 @@ noTaintMaster: true api: advertiseAddress: 1.1.1.1 bindPort: 8443 - controlPlaneEndpoint: 1.1.1.1 + controlPlaneEndpoint: control-plane.minikube.internal kubernetesVersion: v1.11.0 certificatesDir: /var/lib/minikube/certs networking: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/image-repository.yaml index 651706493c..9ce35121be 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/image-repository.yaml @@ -4,7 +4,7 @@ noTaintMaster: true api: advertiseAddress: 1.1.1.1 bindPort: 8443 - controlPlaneEndpoint: 1.1.1.1 + controlPlaneEndpoint: control-plane.minikube.internal kubernetesVersion: v1.11.0 certificatesDir: /var/lib/minikube/certs networking: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml index 5b192e1cfd..2512d833de 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml @@ -4,7 +4,7 @@ noTaintMaster: true api: advertiseAddress: 1.1.1.1 bindPort: 8443 - controlPlaneEndpoint: 1.1.1.1 + controlPlaneEndpoint: control-plane.minikube.internal kubernetesVersion: v1.11.0 certificatesDir: /var/lib/minikube/certs networking: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml index e7ae37e44a..b46a0f3d4c 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml @@ -24,7 +24,7 @@ apiServerExtraArgs: certificatesDir: /var/lib/minikube/certs clusterName: mk apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] -controlPlaneEndpoint: 1.1.1.1:12345 +controlPlaneEndpoint: control-plane.minikube.internal:12345 etcd: local: dataDir: /var/lib/minikube/etcd diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-pod-network-cidr.yaml index d49b9ec306..f9fee2ad6c 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-pod-network-cidr.yaml @@ -24,7 +24,7 @@ apiServerExtraArgs: certificatesDir: /var/lib/minikube/certs clusterName: mk apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 etcd: local: dataDir: /var/lib/minikube/etcd diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml index e8bc0ce7da..ca48887959 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml @@ -24,7 +24,7 @@ apiServerExtraArgs: certificatesDir: /var/lib/minikube/certs clusterName: mk apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 etcd: local: dataDir: /var/lib/minikube/etcd diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml index 71c60951ac..cb5090b8e0 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml @@ -32,7 +32,7 @@ schedulerExtraArgs: certificatesDir: /var/lib/minikube/certs clusterName: mk apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 etcd: local: dataDir: /var/lib/minikube/etcd diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio.yaml index 1f79aeb4d6..f3fa3252cc 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio.yaml @@ -24,7 +24,7 @@ apiServerExtraArgs: certificatesDir: /var/lib/minikube/certs clusterName: mk apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 etcd: local: dataDir: /var/lib/minikube/etcd diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/default.yaml index 5588fe6c75..ce8452e0d9 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/default.yaml @@ -24,7 +24,7 @@ apiServerExtraArgs: certificatesDir: /var/lib/minikube/certs clusterName: mk apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 etcd: local: dataDir: /var/lib/minikube/etcd diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/dns.yaml index 7092fb6277..4608706132 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/dns.yaml @@ -24,7 +24,7 @@ apiServerExtraArgs: certificatesDir: /var/lib/minikube/certs clusterName: mk apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 etcd: local: dataDir: /var/lib/minikube/etcd diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/image-repository.yaml index a22a4d75cc..62ab4b080f 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/image-repository.yaml @@ -25,7 +25,7 @@ apiServerExtraArgs: certificatesDir: /var/lib/minikube/certs clusterName: mk apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 etcd: local: dataDir: /var/lib/minikube/etcd diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/options.yaml index 1303e6b6ac..9e2e66869d 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.12/options.yaml @@ -29,7 +29,7 @@ schedulerExtraArgs: certificatesDir: /var/lib/minikube/certs clusterName: mk apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 etcd: local: dataDir: /var/lib/minikube/etcd diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-api-port.yaml index 4389e10b5a..12512a305a 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-api-port.yaml @@ -24,7 +24,7 @@ apiServerExtraArgs: certificatesDir: /var/lib/minikube/certs clusterName: mk apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] -controlPlaneEndpoint: 1.1.1.1:12345 +controlPlaneEndpoint: control-plane.minikube.internal:12345 etcd: local: dataDir: /var/lib/minikube/etcd diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-pod-network-cidr.yaml index 71d7a676be..8eeed433f9 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-pod-network-cidr.yaml @@ -24,7 +24,7 @@ apiServerExtraArgs: certificatesDir: /var/lib/minikube/certs clusterName: mk apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 etcd: local: dataDir: /var/lib/minikube/etcd diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd.yaml index 485394334f..bf3700c474 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd.yaml @@ -24,7 +24,7 @@ apiServerExtraArgs: certificatesDir: /var/lib/minikube/certs clusterName: mk apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 etcd: local: dataDir: /var/lib/minikube/etcd diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio-options-gates.yaml index 4fa17efa5e..929e4d1230 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio-options-gates.yaml @@ -32,7 +32,7 @@ schedulerExtraArgs: certificatesDir: /var/lib/minikube/certs clusterName: mk apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 etcd: local: dataDir: /var/lib/minikube/etcd diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio.yaml index f5daeef55d..659215754a 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio.yaml @@ -24,7 +24,7 @@ apiServerExtraArgs: certificatesDir: /var/lib/minikube/certs clusterName: mk apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 etcd: local: dataDir: /var/lib/minikube/etcd diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/default.yaml index d99a3fbfaf..b9bf43eb41 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/default.yaml @@ -24,7 +24,7 @@ apiServerExtraArgs: certificatesDir: /var/lib/minikube/certs clusterName: mk apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 etcd: local: dataDir: /var/lib/minikube/etcd diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/dns.yaml index 23bac82149..a5caf8deff 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/dns.yaml @@ -24,7 +24,7 @@ apiServerExtraArgs: certificatesDir: /var/lib/minikube/certs clusterName: mk apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 etcd: local: dataDir: /var/lib/minikube/etcd diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/image-repository.yaml index 924f4fc5a9..eea514431c 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/image-repository.yaml @@ -25,7 +25,7 @@ apiServerExtraArgs: certificatesDir: /var/lib/minikube/certs clusterName: mk apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 etcd: local: dataDir: /var/lib/minikube/etcd diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/options.yaml index 3d9770bed6..c6e5c52122 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.13/options.yaml @@ -29,7 +29,7 @@ schedulerExtraArgs: certificatesDir: /var/lib/minikube/certs clusterName: mk apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"] -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 etcd: local: dataDir: /var/lib/minikube/etcd diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml index fc26257b55..c64e862d15 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:12345 +controlPlaneEndpoint: control-plane.minikube.internal:12345 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml index 9608b92860..5025da9446 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml index c3a54825dd..ef62f65b5a 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml index 7b43973e7d..677b4220df 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml @@ -35,7 +35,7 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml index fa5b62cb02..966784ad94 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml index 84f68f98fc..068fbc1553 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml index 8791cf879f..87fcf83e2c 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml index b1c12c218e..d714f61e43 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml @@ -26,7 +26,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml index 72302c5654..abab48e16d 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml @@ -32,7 +32,7 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml index 307f517ace..03d394574b 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:12345 +controlPlaneEndpoint: control-plane.minikube.internal:12345 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml index 9a0077f387..93d158074f 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml index 3fee964be4..998960e6a0 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml index b07e909cc2..8fc07ef58f 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml @@ -35,7 +35,7 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml index c30eed071b..23483d115a 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml index 99c6275ebd..0555cfec6a 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml index 7a41b4cd55..045e9baa24 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/image-repository.yaml index 053a5977f9..749ed067a2 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/image-repository.yaml @@ -26,7 +26,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml index bac9bdd0c1..d71ad09635 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml @@ -32,7 +32,7 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml index 1ac14c682a..94576d3a8d 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:12345 +controlPlaneEndpoint: control-plane.minikube.internal:12345 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml index 0a43dd453b..b5ef52316a 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml index 13ebbb05a2..2f4932f1d7 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml index 56388066f7..58520fcc10 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml @@ -35,7 +35,7 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml index 70d0f5219b..6867c38197 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml index fd99dfc616..5ea28708d4 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml index 8c880bada8..a82d51453d 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/image-repository.yaml index 919063dda8..e86a63b95d 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/image-repository.yaml @@ -26,7 +26,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml index bf5753a224..90053250f3 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml @@ -32,7 +32,7 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml index 43b450f5c0..598edbdc78 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:12345 +controlPlaneEndpoint: control-plane.minikube.internal:12345 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml index 9cc48c0109..7b720b1feb 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml index fd4f8c324d..9974058e1c 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml index 658763f28f..35728b48d2 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml @@ -35,7 +35,7 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml index f4c255c0c1..19344925c9 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml index 03b1632e50..2ed68ad9fe 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml index bf54c68796..b037f034e7 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/image-repository.yaml index 607c43532c..837c3115f5 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/image-repository.yaml @@ -26,7 +26,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml index 2dfa4b76fd..6c41f706bf 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml @@ -32,7 +32,7 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/containerd-api-port.yaml index 376b6dd2fa..b58afd1527 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/containerd-api-port.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:12345 +controlPlaneEndpoint: control-plane.minikube.internal:12345 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/containerd-pod-network-cidr.yaml index 910e9e9596..14fae0f92b 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/containerd-pod-network-cidr.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/containerd.yaml index 30b062ee58..699175017f 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/containerd.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/crio-options-gates.yaml index 2f787eb95f..a5e764ab34 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/crio-options-gates.yaml @@ -35,7 +35,7 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/crio.yaml index ab798b8bf7..6c27fb0677 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/crio.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/default.yaml index 12ae87fcd3..fbe19f0bb9 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/default.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/dns.yaml index abb27ec9fb..c513fd7fa4 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/dns.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/image-repository.yaml index c22581b268..2aa15c494b 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/image-repository.yaml @@ -26,7 +26,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/options.yaml index 65e991dbce..97167238f9 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.18/options.yaml @@ -32,7 +32,7 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/containerd-api-port.yaml index 5c7c731951..5515f204a5 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/containerd-api-port.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/containerd-api-port.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:12345 +controlPlaneEndpoint: control-plane.minikube.internal:12345 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/containerd-pod-network-cidr.yaml index 833a7ce604..fa98955de8 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/containerd-pod-network-cidr.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/containerd-pod-network-cidr.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/containerd.yaml index 6e2bca9c65..32cd70beb6 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/containerd.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/containerd.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/crio-options-gates.yaml index 9183325cff..c2327fc534 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/crio-options-gates.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/crio-options-gates.yaml @@ -35,7 +35,7 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/crio.yaml index 7c8a91362f..6108566892 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/crio.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/crio.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/default.yaml index efe78f9ea4..3522b8ae65 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/default.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/default.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/dns.yaml index c771a624ee..24e7e299e8 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/dns.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/dns.yaml @@ -25,7 +25,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/image-repository.yaml index 0d9e4b7b4a..facd3abe08 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/image-repository.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/image-repository.yaml @@ -26,7 +26,7 @@ apiServer: enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/options.yaml index c288ba36c4..1b8da1d8c9 100644 --- a/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/options.yaml +++ b/pkg/minikube/bootstrapper/bsutil/testdata/v1.19/options.yaml @@ -32,7 +32,7 @@ scheduler: scheduler-name: "mini-scheduler" certificatesDir: /var/lib/minikube/certs clusterName: mk -controlPlaneEndpoint: 1.1.1.1:8443 +controlPlaneEndpoint: control-plane.minikube.internal:8443 dns: type: CoreDNS etcd: From 80b331c6cdf05c845b3142d77d3206f18e3a7175 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 21 Apr 2020 13:46:17 -0700 Subject: [PATCH 025/105] Set systemd in kicbase Dockerfile --- pkg/minikube/cruntime/containerd.go | 5 ----- pkg/minikube/cruntime/crio.go | 5 ----- pkg/minikube/cruntime/cruntime.go | 2 -- pkg/minikube/cruntime/docker.go | 19 ------------------- pkg/minikube/node/start.go | 7 ------- 5 files changed, 38 deletions(-) diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 1636a14257..73261c9819 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -429,8 +429,3 @@ func addRepoTagToImageName(imgName string) string { } // else it already has repo name dont add anything return imgName } - -// TODO: Enable for containerd -func (r *Containerd) ForceSystemdCgroupManager() error { - return nil -} diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index a335f45134..250d765df4 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -229,8 +229,3 @@ func (r *CRIO) Preload(cfg config.KubernetesConfig) error { } return fmt.Errorf("not yet implemented for %s", r.Name()) } - -// ForceSystemdCgroupManager does nothing since CRIO already uses systemd -func (r *CRIO) ForceSystemdCgroupManager() error { - return nil -} diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 8b0e231376..f1866ff1aa 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -104,8 +104,6 @@ type Manager interface { SystemLogCmd(int) string // Preload preloads the container runtime with k8s images Preload(config.KubernetesConfig) error - // ForceSystemdCgroupManager forces the runtime to use systemd as cgroup manager - ForceSystemdCgroupManager() error } // Config is runtime configuration diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index ce1785e50b..eb11b58e83 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -365,25 +365,6 @@ func dockerImagesPreloaded(runner command.Runner, images []string) bool { return true } -// ForceSystemdCgroupManager forces docker to use systemd as cgroup manager, as recommended in k8s docs: -// https://kubernetes.io/docs/setup/production-environment/container-runtimes/#docker -func (r *Docker) ForceSystemdCgroupManager() error { - daemonConfig := `{ -"exec-opts": ["native.cgroupdriver=systemd"], -"log-driver": "json-file", -"log-opts": { - "max-size": "100m" -}, -"storage-driver": "overlay2" -} -` - ma := assets.NewMemoryAsset([]byte(daemonConfig), "/etc/docker", "daemon.json", "0644") - if err := r.Runner.Copy(ma); err != nil { - return errors.Wrap(err, "copying daemon config") - } - return r.Restart() -} - func dockerBoundToContainerd(runner command.Runner) bool { // NOTE: assumes systemd rr, err := runner.RunCmd(exec.Command("sudo", "systemctl", "cat", "docker.service")) diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 2bc18f4736..5f2aecb0d9 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -99,7 +99,6 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { // setup kubeadm (must come after setupKubeconfig) bs = setupKubeAdm(starter.MachineAPI, *starter.Cfg, *starter.Node, starter.Runner) - err = bs.StartCluster(*starter.Cfg) if err != nil { @@ -262,12 +261,6 @@ func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, k exit.WithError("Failed to enable container runtime", err) } - if driver.IsKIC(cc.Driver) { - if err := cr.ForceSystemdCgroupManager(); err != nil { - glog.Warningf("Failed to force %s to use systemd as cgroup manager (this might be ok): %v", cr.Name(), err) - } - } - return cr } From 3ae85a1f627ddfa72128f2930d5805dd9a70bc48 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 21 Apr 2020 14:17:48 -0700 Subject: [PATCH 026/105] Update kic to v0.0.10, upgrade crio and podman so that the kic base image builds --- hack/images/daemon.json | 8 ++++++++ hack/images/kicbase.Dockerfile | 13 +++++++++++-- pkg/drivers/kic/types.go | 4 ++-- 3 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 hack/images/daemon.json diff --git a/hack/images/daemon.json b/hack/images/daemon.json new file mode 100644 index 0000000000..4785467515 --- /dev/null +++ b/hack/images/daemon.json @@ -0,0 +1,8 @@ +{ + "exec-opts": ["native.cgroupdriver=systemd"], + "log-driver": "json-file", + "log-opts": { + "max-size": "100m" + }, + "storage-driver": "overlay2" +} diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile index 97a29b7371..ccb8e38ff6 100644 --- a/hack/images/kicbase.Dockerfile +++ b/hack/images/kicbase.Dockerfile @@ -15,17 +15,26 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # libglib2.0-0 is required for conmon, which is required for podman libglib2.0-0=2.62.1-1 \ && rm /etc/crictl.yaml + # install cri-o based on https://github.com/cri-o/cri-o/commit/96b0c34b31a9fc181e46d7d8e34fb8ee6c4dc4e1#diff-04c6e90faac2675aa89e2176d2eec7d8R128 +ENV CRIO_VERSION="1.17=1.17.3~2" RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \ curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \ apt-key add - < Release.key && apt-get update && \ - apt-get install -y --no-install-recommends cri-o-1.17=1.17.2~1 + apt-get install -y --no-install-recommends cri-o-${CRIO_VERSION} + # install podman -RUN apt-get install -y --no-install-recommends podman=1.8.2~144 +ENV PODMAN_VERSION=1.9.0~2 +RUN apt-get install -y --no-install-recommends podman=${PODMAN_VERSION} + # disable non-docker runtimes by default RUN systemctl disable containerd && systemctl disable crio && rm /etc/crictl.yaml + +# copy over docker daemon config which sets cgroup manager to systemd +COPY hack/images/daemon.json /etc/docker/daemon.json # enable docker which is default RUN systemctl enable docker + # making SSH work for docker container # based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile RUN mkdir /var/run/sshd diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go index 2b01019b2e..b3840b45d3 100644 --- a/pkg/drivers/kic/types.go +++ b/pkg/drivers/kic/types.go @@ -30,9 +30,9 @@ const ( DefaultPodCIDR = "10.244.0.0/16" // Version is the current version of kic - Version = "v0.0.9" + Version = "v0.0.10" // SHA of the kic base image - baseImageSHA = "82a826cc03c3e59ead5969b8020ca138de98f366c1907293df91fc57205dbb53" + baseImageSHA = "d591f4ef60f4c48f2d6fddfefa88213e81d8be505d3b663cc764326a59cd0c31" // OverlayImage is the cni plugin used for overlay image, created by kind. // CNI plugin image used for kic drivers created by kind. OverlayImage = "kindest/kindnetd:0.5.4" From 1b2707d086dd4cb68a82bad90de5e2181e517627 Mon Sep 17 00:00:00 2001 From: ZouYu Date: Wed, 22 Apr 2020 10:06:57 +0800 Subject: [PATCH 027/105] Add unit test for function ChownR&MaybeChownDirRecursiveToMinikubeUser Signed-off-by: ZouYu --- pkg/util/utils_test.go | 120 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/pkg/util/utils_test.go b/pkg/util/utils_test.go index 55392d7ebc..81952d73c3 100644 --- a/pkg/util/utils_test.go +++ b/pkg/util/utils_test.go @@ -17,6 +17,10 @@ limitations under the License. package util import ( + "io/ioutil" + "os" + "os/user" + "syscall" "testing" "github.com/blang/semver" @@ -74,3 +78,119 @@ func TestParseKubernetesVersion(t *testing.T) { t.Errorf("Expected: %s, Actual:%s", "1.8.0-alpha.5", version) } } + +func TestChownR(t *testing.T) { + testDir, err := ioutil.TempDir(os.TempDir(), "") + if nil != err { + return + } + _, err = os.Create(testDir + "/TestChownR") + if nil != err { + return + } + defer func() { //clean up tempdir + err := os.RemoveAll(testDir) + if err != nil { + t.Errorf("failed to clean up temp folder %q", testDir) + } + }() + + cases := []struct { + name string + uid int + gid int + expectedError bool + }{ + { + name: "normal", + uid: os.Getuid(), + gid: os.Getgid(), + expectedError: false, + }, + { + name: "invalid uid", + uid: 2147483647, + gid: os.Getgid(), + expectedError: true, + }, + { + name: "invalid gid", + uid: os.Getuid(), + gid: 2147483647, + expectedError: true, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + err = ChownR(testDir+"/TestChownR", c.uid, c.gid) + fileInfo, _ := os.Stat(testDir + "/TestChownR") + fileSys := fileInfo.Sys() + if (nil != err) != c.expectedError || ((false == c.expectedError) && (fileSys.(*syscall.Stat_t).Gid != uint32(c.gid) || fileSys.(*syscall.Stat_t).Uid != uint32(c.uid))) { + t.Errorf("expectedError: %v, got: %v", c.expectedError, err) + } + }) + } +} + +func TestMaybeChownDirRecursiveToMinikubeUser(t *testing.T) { + testDir, err := ioutil.TempDir(os.TempDir(), "") + if nil != err { + return + } + _, err = os.Create(testDir + "/TestChownR") + if nil != err { + return + } + + defer func() { //clean up tempdir + err := os.RemoveAll(testDir) + if err != nil { + t.Errorf("failed to clean up temp folder %q", testDir) + } + }() + + if os.Getenv("CHANGE_MINIKUBE_NONE_USER") == "" { + err = os.Setenv("CHANGE_MINIKUBE_NONE_USER", "1") + if nil != err { + t.Error("failed to set env: CHANGE_MINIKUBE_NONE_USER") + } + } + + if os.Getenv("SUDO_USER") == "" { + user, err := user.Current() + if nil != err { + t.Error("fail to get user") + } + os.Setenv("SUDO_USER", user.Username) + err = os.Setenv("SUDO_USER", user.Username) + if nil != err { + t.Error("failed to set env: SUDO_USER") + } + } + + cases := []struct { + name string + dir string + expectedError bool + }{ + { + name: "normal", + dir: testDir, + expectedError: false, + }, + { + name: "invaild dir", + dir: "./utils_test", + expectedError: true, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + err = MaybeChownDirRecursiveToMinikubeUser(c.dir) + if (nil != err) != c.expectedError { + t.Errorf("expectedError: %v, got: %v", c.expectedError, err) + } + }) + } +} From bf618d663283e9ab96437555002f9b0fdeedc1c6 Mon Sep 17 00:00:00 2001 From: Radoslaw Smigielski Date: Wed, 22 Apr 2020 17:08:44 +0100 Subject: [PATCH 028/105] Add list option for "minikube node" command Add option to list the nodes. So far there was no "minikube node list" command. --- cmd/minikube/cmd/node.go | 2 +- cmd/minikube/cmd/node_list.go | 58 +++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 cmd/minikube/cmd/node_list.go diff --git a/cmd/minikube/cmd/node.go b/cmd/minikube/cmd/node.go index 39dbac6c7c..6dac2ca409 100644 --- a/cmd/minikube/cmd/node.go +++ b/cmd/minikube/cmd/node.go @@ -27,6 +27,6 @@ var nodeCmd = &cobra.Command{ Short: "Node operations", Long: "Operations on nodes", Run: func(cmd *cobra.Command, args []string) { - exit.UsageT("Usage: minikube node [add|start|stop|delete]") + exit.UsageT("Usage: minikube node [add|start|stop|delete|list]") }, } diff --git a/cmd/minikube/cmd/node_list.go b/cmd/minikube/cmd/node_list.go new file mode 100644 index 0000000000..d2cfd4f7e9 --- /dev/null +++ b/cmd/minikube/cmd/node_list.go @@ -0,0 +1,58 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "os" + "fmt" + + "github.com/golang/glog" + "github.com/spf13/cobra" + "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/mustload" +) + +var nodeListCmd = &cobra.Command{ + Use: "list", + Short: "List nodes.", + Long: "List existing Minikube nodes.", + Run: func(cmd *cobra.Command, args []string) { + if len(args) != 0 { + exit.UsageT("Usage: minikube node list") + } + + cname := ClusterFlagValue() + _, cc := mustload.Partial(cname) + + if len(cc.Nodes) < 1 { + glog.Warningf("Did not found any Minikube node.") + } else { + glog.Infof("%v", cc.Nodes) + } + + for _, n := range cc.Nodes { + machineName := driver.MachineName(*cc, n) + fmt.Printf("%s\n", machineName) + } + os.Exit(0) + }, +} + +func init() { + nodeCmd.AddCommand(nodeListCmd) +} From face3a43b9009ae0866ff2ca7d785c3bbf1ec759 Mon Sep 17 00:00:00 2001 From: Radoslaw Smigielski Date: Wed, 22 Apr 2020 17:20:26 +0100 Subject: [PATCH 029/105] Fix lint problem --- cmd/minikube/cmd/node_list.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/node_list.go b/cmd/minikube/cmd/node_list.go index d2cfd4f7e9..e5c8975652 100644 --- a/cmd/minikube/cmd/node_list.go +++ b/cmd/minikube/cmd/node_list.go @@ -17,8 +17,8 @@ limitations under the License. package cmd import ( - "os" "fmt" + "os" "github.com/golang/glog" "github.com/spf13/cobra" From 62c93c465742e33ccbc062d68c0ade5338abe198 Mon Sep 17 00:00:00 2001 From: Radoslaw Smigielski Date: Wed, 22 Apr 2020 17:23:05 +0100 Subject: [PATCH 030/105] Add docs for node list command --- site/content/en/docs/commands/node.md | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/site/content/en/docs/commands/node.md b/site/content/en/docs/commands/node.md index 9aec995f71..d73f31a195 100644 --- a/site/content/en/docs/commands/node.md +++ b/site/content/en/docs/commands/node.md @@ -138,6 +138,38 @@ minikube node help [command] [flags] --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging ``` +## minikube node list + +List nodes. + +### Synopsis + +List existing Minikube nodes. + +``` +minikube node list [flags] +``` + +### Options + +``` + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --alsologtostderr log to standard error as well as files + -b, --bootstrapper string The name of the cluster bootstrapper that will set up the kubernetes cluster. (default "kubeadm") + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --logtostderr log to standard error instead of files + -p, --profile string The name of the minikube VM being used. This can be set to allow having multiple instances of minikube independently. (default "minikube") + --stderrthreshold severity logs at or above this threshold go to stderr (default 2) + -v, --v Level log level for V logs + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging +``` + ## minikube node start Starts a node. From ae388d0edb43a49a1fb86fc92adbd4449f3b6cd1 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 22 Apr 2020 14:10:16 -0700 Subject: [PATCH 031/105] Add --force-systemd flag to force container runtime to use systemd --- cmd/minikube/cmd/start_flags.go | 2 ++ pkg/minikube/cruntime/containerd.go | 5 +++++ pkg/minikube/cruntime/crio.go | 5 +++++ pkg/minikube/cruntime/cruntime.go | 2 ++ pkg/minikube/cruntime/docker.go | 23 +++++++++++++++++++++++ 5 files changed, 37 insertions(+) diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go index c735e8025d..af42ee7c59 100644 --- a/cmd/minikube/cmd/start_flags.go +++ b/cmd/minikube/cmd/start_flags.go @@ -98,6 +98,7 @@ const ( nodes = "nodes" preload = "preload" deleteOnFailure = "delete-on-failure" + forceSystemd = "force-systemd" ) // initMinikubeFlags includes commandline flags for minikube. @@ -135,6 +136,7 @@ func initMinikubeFlags() { startCmd.Flags().IntP(nodes, "n", 1, "The number of nodes to spin up. Defaults to 1.") startCmd.Flags().Bool(preload, true, "If set, download tarball of preloaded images if available to improve start time. Defaults to true.") startCmd.Flags().Bool(deleteOnFailure, false, "If set, delete the current cluster if start fails and try again. Defaults to false.") + startCmd.Flags().Bool(forceSystemd, false, "If set, force the container runtime to use sytemd as cgroup manager. Currently avaiable for docker and crio. Defaults to false.") } // initKubernetesFlags inits the commandline flags for kubernetes related options diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 73261c9819..570407b61a 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -429,3 +429,8 @@ func addRepoTagToImageName(imgName string) string { } // else it already has repo name dont add anything return imgName } + +// TODO: Implement for containerd +func (r *Containerd) ForceSystemd() error { + return nil +} diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 250d765df4..4ff52cc37e 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -229,3 +229,8 @@ func (r *CRIO) Preload(cfg config.KubernetesConfig) error { } return fmt.Errorf("not yet implemented for %s", r.Name()) } + +// ForceSystemd does nothing for crio since it already uses systemd as cgroup manager +func (r *CRIO) ForceSystemd() error { + return nil +} diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index f1866ff1aa..3a5ad43d89 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -104,6 +104,8 @@ type Manager interface { SystemLogCmd(int) string // Preload preloads the container runtime with k8s images Preload(config.KubernetesConfig) error + // ForceSystemd forces the container runtime to use systemd as cgroup manager + ForceSystemd() error } // Config is runtime configuration diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index eb11b58e83..eb4c8030df 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -25,6 +25,7 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" + "github.com/spf13/viper" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/command" @@ -110,6 +111,10 @@ func (r *Docker) Enable(disOthers bool) error { } } + if err := r.ForceSystemd(); err != nil { + return errors.Wrap(err, "forcing systemd") + } + return r.Init.Start("docker") } @@ -274,6 +279,24 @@ func (r *Docker) SystemLogCmd(len int) string { return fmt.Sprintf("sudo journalctl -u docker -n %d", len) } +// ForceSystemd forces the docker daemon to use systemd as cgroup manager +func (r *Docker) ForceSystemd() error { + if !viper.GetBool("force-systemd") { + return nil + } + daemonConfig := `{ +"exec-opts": ["native.cgroupdriver=systemd"], +"log-driver": "json-file", +"log-opts": { + "max-size": "100m" +}, +"storage-driver": "overlay2" +} +` + ma := assets.NewMemoryAsset([]byte(daemonConfig), "/etc/docker", "daemon.json", "0644") + return r.Runner.Copy(ma) +} + // Preload preloads docker with k8s images: // 1. Copy over the preloaded tarball into the VM // 2. Extract the preloaded tarball to the correct directory From 1f2b32c4edbe9d18a567b01dd204276b3402827e Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 22 Apr 2020 14:11:23 -0700 Subject: [PATCH 032/105] Update docs --- site/content/en/docs/commands/start.md | 1 + 1 file changed, 1 insertion(+) diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md index 7f2a0b837e..a1b0c098fe 100644 --- a/site/content/en/docs/commands/start.md +++ b/site/content/en/docs/commands/start.md @@ -49,6 +49,7 @@ minikube start [flags] Valid kubeadm parameters: ignore-preflight-errors, dry-run, kubeconfig, kubeconfig-dir, node-name, cri-socket, experimental-upload-certs, certificate-key, rootfs, skip-phases, pod-network-cidr --feature-gates string A set of key=value pairs that describe feature gates for alpha/experimental features. --force Force minikube to perform possibly dangerous operations + --force-systemd If set, force the container runtime to use sytemd as cgroup manager. Currently avaiable for docker and crio. Defaults to false. -h, --help help for start --host-dns-resolver Enable host resolver for NAT DNS requests (virtualbox driver only) (default true) --host-only-cidr string The CIDR to be used for the minikube VM (virtualbox driver only) (default "192.168.99.1/24") From 56fa90fd8763dc5f14cd51d1579e54488c9de679 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 22 Apr 2020 14:19:26 -0700 Subject: [PATCH 033/105] Add integration test --- hack/images/kicbase.Dockerfile | 13 ++----------- test/integration/docker_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile index ccb8e38ff6..97a29b7371 100644 --- a/hack/images/kicbase.Dockerfile +++ b/hack/images/kicbase.Dockerfile @@ -15,26 +15,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # libglib2.0-0 is required for conmon, which is required for podman libglib2.0-0=2.62.1-1 \ && rm /etc/crictl.yaml - # install cri-o based on https://github.com/cri-o/cri-o/commit/96b0c34b31a9fc181e46d7d8e34fb8ee6c4dc4e1#diff-04c6e90faac2675aa89e2176d2eec7d8R128 -ENV CRIO_VERSION="1.17=1.17.3~2" RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \ curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \ apt-key add - < Release.key && apt-get update && \ - apt-get install -y --no-install-recommends cri-o-${CRIO_VERSION} - + apt-get install -y --no-install-recommends cri-o-1.17=1.17.2~1 # install podman -ENV PODMAN_VERSION=1.9.0~2 -RUN apt-get install -y --no-install-recommends podman=${PODMAN_VERSION} - +RUN apt-get install -y --no-install-recommends podman=1.8.2~144 # disable non-docker runtimes by default RUN systemctl disable containerd && systemctl disable crio && rm /etc/crictl.yaml - -# copy over docker daemon config which sets cgroup manager to systemd -COPY hack/images/daemon.json /etc/docker/daemon.json # enable docker which is default RUN systemctl enable docker - # making SSH work for docker container # based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile RUN mkdir /var/run/sshd diff --git a/test/integration/docker_test.go b/test/integration/docker_test.go index dd0a27de06..25d0da8eb4 100644 --- a/test/integration/docker_test.go +++ b/test/integration/docker_test.go @@ -63,3 +63,30 @@ func TestDockerFlags(t *testing.T) { } } } + +func TestForceSystemd(t *testing.T) { + if NoneDriver() { + t.Skip("skipping: none driver does not support ssh or bundle docker") + } + MaybeParallel(t) + + profile := UniqueProfileName("force-systemd") + ctx, cancel := context.WithTimeout(context.Background(), Minutes(30)) + defer CleanupWithLogs(t, profile, cancel) + + // Use the most verbose logging for the simplest test. If it fails, something is very wrong. + args := append([]string{"start", "-p", profile, "--force-systemd", "--alsologtostderr", "-v=5"}, StartArgs()...) + rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) + if err != nil { + t.Errorf("failed to start minikube with args: %q : %v", rr.Command(), err) + } + + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "docker info --format {{.CgroupDriver}}")) + if err != nil { + t.Errorf("failed to get docker cgroup driver. args %q: %v", rr.Command(), err) + } + + if !strings.Contains(rr.Output(), "systemd") { + t.Fatalf("expected systemd cgroup driver, got: %v", rr.Output()) + } +} From 9bba297cf153261633dab20a0dee4f0de2ff5d1b Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 22 Apr 2020 14:19:33 -0700 Subject: [PATCH 034/105] revert unneeded changes --- hack/images/daemon.json | 8 -------- pkg/drivers/kic/types.go | 4 ++-- 2 files changed, 2 insertions(+), 10 deletions(-) delete mode 100644 hack/images/daemon.json diff --git a/hack/images/daemon.json b/hack/images/daemon.json deleted file mode 100644 index 4785467515..0000000000 --- a/hack/images/daemon.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "exec-opts": ["native.cgroupdriver=systemd"], - "log-driver": "json-file", - "log-opts": { - "max-size": "100m" - }, - "storage-driver": "overlay2" -} diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go index b3840b45d3..2b01019b2e 100644 --- a/pkg/drivers/kic/types.go +++ b/pkg/drivers/kic/types.go @@ -30,9 +30,9 @@ const ( DefaultPodCIDR = "10.244.0.0/16" // Version is the current version of kic - Version = "v0.0.10" + Version = "v0.0.9" // SHA of the kic base image - baseImageSHA = "d591f4ef60f4c48f2d6fddfefa88213e81d8be505d3b663cc764326a59cd0c31" + baseImageSHA = "82a826cc03c3e59ead5969b8020ca138de98f366c1907293df91fc57205dbb53" // OverlayImage is the cni plugin used for overlay image, created by kind. // CNI plugin image used for kic drivers created by kind. OverlayImage = "kindest/kindnetd:0.5.4" From 7a3db5078eec5957137e2d5d22898669ca7a4f52 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 22 Apr 2020 14:41:49 -0700 Subject: [PATCH 035/105] Update crio and podman in docker driver base image The dockerfile currently doesn't build because it can't find the specified versions of crio and podman to install. --- hack/images/kicbase.Dockerfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile index 97a29b7371..be6ebb2bce 100644 --- a/hack/images/kicbase.Dockerfile +++ b/hack/images/kicbase.Dockerfile @@ -16,12 +16,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libglib2.0-0=2.62.1-1 \ && rm /etc/crictl.yaml # install cri-o based on https://github.com/cri-o/cri-o/commit/96b0c34b31a9fc181e46d7d8e34fb8ee6c4dc4e1#diff-04c6e90faac2675aa89e2176d2eec7d8R128 +ENV CRIO_VERSION="1.17=1.17.3~2" RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \ curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \ apt-key add - < Release.key && apt-get update && \ - apt-get install -y --no-install-recommends cri-o-1.17=1.17.2~1 + apt-get install -y --no-install-recommends cri-o-${CRIO_VERSION} + # install podman -RUN apt-get install -y --no-install-recommends podman=1.8.2~144 +ENV PODMAN_VERSION=1.9.0~2 +RUN apt-get install -y --no-install-recommends podman=${PODMAN_VERSION} + # disable non-docker runtimes by default RUN systemctl disable containerd && systemctl disable crio && rm /etc/crictl.yaml # enable docker which is default From d39b822a3c8d685ca59b08f4b8f0c1479eec9249 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 22 Apr 2020 14:44:24 -0700 Subject: [PATCH 036/105] upgrade to v0.0.10 --- pkg/drivers/kic/types.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go index 2b01019b2e..f441f0f8b6 100644 --- a/pkg/drivers/kic/types.go +++ b/pkg/drivers/kic/types.go @@ -30,9 +30,9 @@ const ( DefaultPodCIDR = "10.244.0.0/16" // Version is the current version of kic - Version = "v0.0.9" + Version = "v0.0.10" // SHA of the kic base image - baseImageSHA = "82a826cc03c3e59ead5969b8020ca138de98f366c1907293df91fc57205dbb53" + baseImageSHA = "0c3666c7dacb3c7170a86b237d0d15d37afb5a6d2f0c45046546b5c09f40378c" // OverlayImage is the cni plugin used for overlay image, created by kind. // CNI plugin image used for kic drivers created by kind. OverlayImage = "kindest/kindnetd:0.5.4" From e5d633df9f9c94527fba94e730f240ac8c7a8f11 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 22 Apr 2020 14:51:36 -0700 Subject: [PATCH 037/105] fix lint --- cmd/minikube/cmd/start_flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go index af42ee7c59..144f8caf95 100644 --- a/cmd/minikube/cmd/start_flags.go +++ b/cmd/minikube/cmd/start_flags.go @@ -136,7 +136,7 @@ func initMinikubeFlags() { startCmd.Flags().IntP(nodes, "n", 1, "The number of nodes to spin up. Defaults to 1.") startCmd.Flags().Bool(preload, true, "If set, download tarball of preloaded images if available to improve start time. Defaults to true.") startCmd.Flags().Bool(deleteOnFailure, false, "If set, delete the current cluster if start fails and try again. Defaults to false.") - startCmd.Flags().Bool(forceSystemd, false, "If set, force the container runtime to use sytemd as cgroup manager. Currently avaiable for docker and crio. Defaults to false.") + startCmd.Flags().Bool(forceSystemd, false, "If set, force the container runtime to use sytemd as cgroup manager. Currently available for docker and crio. Defaults to false.") } // initKubernetesFlags inits the commandline flags for kubernetes related options From fad745f6074fee720c027e98ac4e9e44b4cac58c Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 22 Apr 2020 22:16:18 -0700 Subject: [PATCH 038/105] adding runner for oci package --- pkg/drivers/kic/oci/runner.go | 112 ++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 pkg/drivers/kic/oci/runner.go diff --git a/pkg/drivers/kic/oci/runner.go b/pkg/drivers/kic/oci/runner.go new file mode 100644 index 0000000000..1cf7733a7c --- /dev/null +++ b/pkg/drivers/kic/oci/runner.go @@ -0,0 +1,112 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oci + +import ( + "bytes" + "fmt" + "io" + "os/exec" + "strings" + "time" + + "github.com/golang/glog" +) + +//runner runs commands using the os/exec package. +type runner struct { +} + +// RunResult holds the results of a Runner +type RunResult struct { + Stdout bytes.Buffer + Stderr bytes.Buffer + ExitCode int + Args []string // the args that was passed to Runner +} + +// Command returns a human readable command string that does not induce eye fatigue +func (rr RunResult) Command() string { + var sb strings.Builder + sb.WriteString(rr.Args[0]) + for _, a := range rr.Args[1:] { + if strings.Contains(a, " ") { + sb.WriteString(fmt.Sprintf(` "%s"`, a)) + continue + } + sb.WriteString(fmt.Sprintf(" %s", a)) + } + return sb.String() +} + +// Output returns human-readable output for an execution result +func (rr RunResult) Output() string { + var sb strings.Builder + if rr.Stdout.Len() > 0 { + sb.WriteString(fmt.Sprintf("-- stdout --\n%s\n-- /stdout --", rr.Stdout.Bytes())) + } + if rr.Stderr.Len() > 0 { + sb.WriteString(fmt.Sprintf("\n** stderr ** \n%s\n** /stderr **", rr.Stderr.Bytes())) + } + return sb.String() +} + +// newRunner returns an oci runner +func newRunner() *runner { + return &runner{} +} + +// RunCmd implements the Command Runner interface to run a exec.Cmd object +func (*runner) RunCmd(cmd *exec.Cmd) (*RunResult, error) { + rr := &RunResult{Args: cmd.Args} + glog.Infof("Run: %v", rr.Command()) + + var outb, errb io.Writer + if cmd.Stdout == nil { + var so bytes.Buffer + outb = io.MultiWriter(&so, &rr.Stdout) + } else { + outb = io.MultiWriter(cmd.Stdout, &rr.Stdout) + } + + if cmd.Stderr == nil { + var se bytes.Buffer + errb = io.MultiWriter(&se, &rr.Stderr) + } else { + errb = io.MultiWriter(cmd.Stderr, &rr.Stderr) + } + + cmd.Stdout = outb + cmd.Stderr = errb + + start := time.Now() + err := cmd.Run() + elapsed := time.Since(start) + + if exitError, ok := err.(*exec.ExitError); ok { + rr.ExitCode = exitError.ExitCode() + } + // Decrease log spam + if elapsed > (1 * time.Second) { + glog.Infof("Completed: %s: (%s)", rr.Command(), elapsed) + } + if err == nil { + return rr, nil + } + + return rr, fmt.Errorf("%s: %v\nstdout:\n%s\nstderr:\n%s", rr.Command(), err, rr.Stdout.String(), rr.Stderr.String()) +} From c6895c20587700c614e2179eab741a06afb4fbcd Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 22 Apr 2020 22:34:11 -0700 Subject: [PATCH 039/105] add cli runner to info --- pkg/drivers/kic/oci/info.go | 12 ++++-------- pkg/drivers/kic/oci/runner.go | 12 +++++++----- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/pkg/drivers/kic/oci/info.go b/pkg/drivers/kic/oci/info.go index 548c071f16..0d0f324f59 100644 --- a/pkg/drivers/kic/oci/info.go +++ b/pkg/drivers/kic/oci/info.go @@ -216,15 +216,12 @@ type podmanSysInfo struct { // dockerSystemInfo returns docker system info --format '{{json .}}' func dockerSystemInfo() (dockerSysInfo, error) { var ds dockerSysInfo - - cmd := exec.Command(Docker, "system", "info", "--format", "{{json .}}") - out, err := cmd.CombinedOutput() - + rr, err := cli.RunCmd(exec.Command(Docker, "system", "info", "--format", "{{json .}}")) if err != nil { return ds, errors.Wrap(err, "get docker system info") } - if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &ds); err != nil { + if err := json.Unmarshal([]byte(strings.TrimSpace(string(rr.Stdout.Bytes()))), &ds); err != nil { return ds, errors.Wrapf(err, "unmarshal docker system info") } @@ -234,12 +231,11 @@ func dockerSystemInfo() (dockerSysInfo, error) { // podmanSysInfo returns podman system info --format '{{json .}}' func podmanSystemInfo() (podmanSysInfo, error) { var ps podmanSysInfo - cmd := exec.Command(Podman, "system", "info", "--format", "'{{json .}}'") - out, err := cmd.CombinedOutput() + rr, err := cli.RunCmd(exec.Command(Podman, "system", "info", "--format", "'{{json .}}'")) if err != nil { return ps, errors.Wrap(err, "get podman system info") } - if err := json.Unmarshal([]byte(strings.TrimSpace(string(out))), &ps); err != nil { + if err := json.Unmarshal([]byte(strings.TrimSpace(string(rr.Stdout.Bytes()))), &ps); err != nil { return ps, errors.Wrapf(err, "unmarshal podman system info") } return ps, nil diff --git a/pkg/drivers/kic/oci/runner.go b/pkg/drivers/kic/oci/runner.go index 1cf7733a7c..fec1cb48cf 100644 --- a/pkg/drivers/kic/oci/runner.go +++ b/pkg/drivers/kic/oci/runner.go @@ -27,8 +27,10 @@ import ( "github.com/golang/glog" ) -//runner runs commands using the os/exec package. -type runner struct { +var cli = newRunner() + +//cliRunner runs commands using the os/exec package. +type cliRunner struct { } // RunResult holds the results of a Runner @@ -66,12 +68,12 @@ func (rr RunResult) Output() string { } // newRunner returns an oci runner -func newRunner() *runner { - return &runner{} +func newRunner() *cliRunner { + return &cliRunner{} } // RunCmd implements the Command Runner interface to run a exec.Cmd object -func (*runner) RunCmd(cmd *exec.Cmd) (*RunResult, error) { +func (*cliRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) { rr := &RunResult{Args: cmd.Args} glog.Infof("Run: %v", rr.Command()) From 72baa1dad2ad00ff34d4477e7fd974cc15605e54 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 22 Apr 2020 22:49:48 -0700 Subject: [PATCH 040/105] convert execs to cliRunner --- pkg/drivers/kic/oci/network.go | 46 ++++++++++++++-------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index 1dbb8232a5..0a16132adc 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -43,12 +43,11 @@ func RoutableHostIPFromInside(ociBin string, containerName string) (net.IP, erro // digDNS will get the IP record for a dns func digDNS(ociBin, containerName, dns string) (net.IP, error) { - cmd := exec.Command(ociBin, "exec", "-t", containerName, "dig", "+short", dns) - out, err := cmd.CombinedOutput() - ip := net.ParseIP(strings.TrimSpace(string(out))) + rr, err := cli.RunCmd(exec.Command(ociBin, "exec", "-t", containerName, "dig", "+short", dns)) + ip := net.ParseIP(strings.TrimSpace(rr.Stdout.String())) if err != nil { - return ip, errors.Wrapf(err, "resolve dns to ip: %s", string(out)) + return ip, errors.Wrapf(err, "resolve dns to ip: %s", rr.Output()) } glog.Infof("got host ip for mount in container by digging dns: %s", ip.String()) @@ -58,23 +57,19 @@ func digDNS(ociBin, containerName, dns string) (net.IP, error) { // dockerGatewayIP gets the default gateway ip for the docker bridge on the user's host machine // gets the ip from user's host docker func dockerGatewayIP() (net.IP, error) { - cmd := exec.Command(Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}") - out, err := cmd.CombinedOutput() - + rr, err := cli.RunCmd(exec.Command(Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}")) if err != nil { - return nil, errors.Wrapf(err, "get network bridge. output: %s", string(out)) + return nil, errors.Wrapf(err, "get network bridge. output: %s", rr.Output()) } - bridgeID := strings.TrimSpace(string(out)) - cmd = exec.Command(Docker, "inspect", - "--format", "{{(index .IPAM.Config 0).Gateway}}", bridgeID) - out, err = cmd.CombinedOutput() - + bridgeID := strings.TrimSpace(rr.Stdout.String()) + rr, err = cli.RunCmd(exec.Command(Docker, "inspect", + "--format", "{{(index .IPAM.Config 0).Gateway}}", bridgeID)) if err != nil { - return nil, errors.Wrapf(err, "inspect IP gatway for bridge network: %q. output: %s", string(out), bridgeID) + return nil, errors.Wrapf(err, "inspect IP gatway for bridge network: %q. output: %s", rr.Output(), bridgeID) } - ip := net.ParseIP(strings.TrimSpace(string(out))) + ip := net.ParseIP(strings.TrimSpace(rr.Stdout.String())) glog.Infof("got host ip for mount in container by inspect docker network: %s", ip.String()) return ip, nil } @@ -85,25 +80,23 @@ func dockerGatewayIP() (net.IP, error) { // 32769, nil // only supports TCP ports func ForwardedPort(ociBinary string, ociID string, contPort int) (int, error) { - var out []byte + var rr *RunResult var err error if ociBinary == Podman { //podman inspect -f "{{range .NetworkSettings.Ports}}{{if eq .ContainerPort "80"}}{{.HostPort}}{{end}}{{end}}" - cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("{{range .NetworkSettings.Ports}}{{if eq .ContainerPort %s}}{{.HostPort}}{{end}}{{end}}", fmt.Sprint(contPort)), ociID) - out, err = cmd.CombinedOutput() + rr, err = cli.RunCmd(exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("{{range .NetworkSettings.Ports}}{{if eq .ContainerPort %s}}{{.HostPort}}{{end}}{{end}}", fmt.Sprint(contPort)), ociID)) if err != nil { - return 0, errors.Wrapf(err, "get host-bind port %d for %q, output %s", contPort, ociID, out) + return 0, errors.Wrapf(err, "get host-bind port %d for %q, output %s", contPort, ociID, rr.Output()) } } else { - cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID) - out, err = cmd.CombinedOutput() + rr, err = cli.RunCmd(exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)) if err != nil { - return 0, errors.Wrapf(err, "get host-bind port %d for %q, output %s", contPort, ociID, out) + return 0, errors.Wrapf(err, "get host-bind port %d for %q, output %s", contPort, ociID, rr.Output()) } } - o := strings.TrimSpace(string(out)) + o := strings.TrimSpace(rr.Stdout.String()) o = strings.Trim(o, "'") p, err := strconv.Atoi(o) @@ -124,14 +117,13 @@ func ContainerIPs(ociBinary string, name string) (string, string, error) { // podmanConttainerIP returns ipv4, ipv6 of container or error func podmanConttainerIP(name string) (string, string, error) { - cmd := exec.Command(Podman, "inspect", + rr, err := cli.RunCmd(exec.Command(Podman, "inspect", "-f", "{{.NetworkSettings.IPAddress}}", - name) - out, err := cmd.CombinedOutput() + name)) if err != nil { return "", "", errors.Wrapf(err, "podman inspect ip %s", name) } - output := strings.TrimSpace(string(out)) + output := strings.TrimSpace(rr.Stdout.String()) if err == nil && output == "" { // podman returns empty for 127.0.0.1 return DefaultBindIPV4, "", nil } From 915058ef8e038e0328621a1008435be4a3748b44 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 22 Apr 2020 23:01:04 -0700 Subject: [PATCH 041/105] lint --- pkg/drivers/kic/oci/info.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/drivers/kic/oci/info.go b/pkg/drivers/kic/oci/info.go index 0d0f324f59..ab678ab688 100644 --- a/pkg/drivers/kic/oci/info.go +++ b/pkg/drivers/kic/oci/info.go @@ -221,7 +221,7 @@ func dockerSystemInfo() (dockerSysInfo, error) { return ds, errors.Wrap(err, "get docker system info") } - if err := json.Unmarshal([]byte(strings.TrimSpace(string(rr.Stdout.Bytes()))), &ds); err != nil { + if err := json.Unmarshal([]byte(strings.TrimSpace(rr.Stdout.String())), &ds); err != nil { return ds, errors.Wrapf(err, "unmarshal docker system info") } @@ -235,7 +235,8 @@ func podmanSystemInfo() (podmanSysInfo, error) { if err != nil { return ps, errors.Wrap(err, "get podman system info") } - if err := json.Unmarshal([]byte(strings.TrimSpace(string(rr.Stdout.Bytes()))), &ps); err != nil { + + if err := json.Unmarshal([]byte(strings.TrimSpace(rr.Stdout.String())), &ps); err != nil { return ps, errors.Wrapf(err, "unmarshal podman system info") } return ps, nil From 29df4260efc7c05a2b26e96689078a5d477f7c8a Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 22 Apr 2020 23:27:54 -0700 Subject: [PATCH 042/105] make all oci volume calls go through runner --- .../kic/oci/{runner.go => cli_runner.go} | 0 pkg/drivers/kic/oci/volumes.go | 21 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) rename pkg/drivers/kic/oci/{runner.go => cli_runner.go} (100%) diff --git a/pkg/drivers/kic/oci/runner.go b/pkg/drivers/kic/oci/cli_runner.go similarity index 100% rename from pkg/drivers/kic/oci/runner.go rename to pkg/drivers/kic/oci/cli_runner.go diff --git a/pkg/drivers/kic/oci/volumes.go b/pkg/drivers/kic/oci/volumes.go index 2fbd9f32b0..3afd7ec6ec 100644 --- a/pkg/drivers/kic/oci/volumes.go +++ b/pkg/drivers/kic/oci/volumes.go @@ -65,9 +65,8 @@ func PruneAllVolumesByLabel(ociBin string, label string) []error { // allVolumesByLabel returns name of all docker volumes by a specific label // will not return error if there is no volume found. func allVolumesByLabel(ociBin string, label string) ([]string, error) { - cmd := exec.Command(ociBin, "volume", "ls", "--filter", "label="+label, "--format", "{{.Name}}") - stdout, err := cmd.Output() - s := bufio.NewScanner(bytes.NewReader(stdout)) + rr, err := cli.RunCmd(exec.Command(ociBin, "volume", "ls", "--filter", "label="+label, "--format", "{{.Name}}")) + s := bufio.NewScanner(bytes.NewReader(rr.Stdout.Bytes())) var vols []string for s.Scan() { v := strings.TrimSpace(s.Text()) @@ -81,10 +80,10 @@ func allVolumesByLabel(ociBin string, label string) ([]string, error) { // ExtractTarballToVolume runs a docker image imageName which extracts the tarball at tarballPath // to the volume named volumeName func ExtractTarballToVolume(tarballPath, volumeName, imageName string) error { - cmd := exec.Command(Docker, "run", "--rm", "--entrypoint", "/usr/bin/tar", "-v", fmt.Sprintf("%s:/preloaded.tar:ro", tarballPath), "-v", fmt.Sprintf("%s:/extractDir", volumeName), imageName, "-I", "lz4", "-xvf", "/preloaded.tar", "-C", "/extractDir") - glog.Infof("executing: %s", cmd.Args) - if out, err := cmd.CombinedOutput(); err != nil { - return errors.Wrapf(err, "output %s", string(out)) + rr, err := cli.RunCmd(exec.Command(Docker, "run", "--rm", "--entrypoint", "/usr/bin/tar", "-v", fmt.Sprintf("%s:/preloaded.tar:ro", tarballPath), "-v", fmt.Sprintf("%s:/extractDir", volumeName), imageName, "-I", "lz4", "-xvf", "/preloaded.tar", "-C", "/extractDir")) + glog.Infof("executed: %s", rr.Command()) + if err != nil { + return errors.Wrapf(err, "output: %s", rr.Output()) } return nil } @@ -93,10 +92,10 @@ func ExtractTarballToVolume(tarballPath, volumeName, imageName string) error { // Caution ! if volume already exists does NOT return an error and will not apply the minikube labels on it. // TODO: this should be fixed as a part of https://github.com/kubernetes/minikube/issues/6530 func createDockerVolume(profile string, nodeName string) error { - cmd := exec.Command(Docker, "volume", "create", nodeName, "--label", fmt.Sprintf("%s=%s", ProfileLabelKey, profile), "--label", fmt.Sprintf("%s=%s", CreatedByLabelKey, "true")) - glog.Infof("executing: %s", cmd.Args) - if out, err := cmd.CombinedOutput(); err != nil { - return errors.Wrapf(err, "output %s", string(out)) + rr, err := cli.RunCmd(exec.Command(Docker, "volume", "create", nodeName, "--label", fmt.Sprintf("%s=%s", ProfileLabelKey, profile), "--label", fmt.Sprintf("%s=%s", CreatedByLabelKey, "true"))) + glog.Infof("executed: %s", rr.Command()) + if err != nil { + return errors.Wrapf(err, "output: %s", rr.Output()) } return nil } From c6e1445ccb3de3ea8cd461eb43584213025e2e3e Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Thu, 23 Apr 2020 00:12:41 -0700 Subject: [PATCH 043/105] convert more to cli runner --- pkg/drivers/kic/oci/network.go | 11 +++---- pkg/drivers/kic/oci/oci.go | 56 +++++++++++----------------------- pkg/drivers/kic/oci/volumes.go | 13 +++----- 3 files changed, 28 insertions(+), 52 deletions(-) diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index 0a16132adc..576607d378 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -45,9 +45,8 @@ func RoutableHostIPFromInside(ociBin string, containerName string) (net.IP, erro func digDNS(ociBin, containerName, dns string) (net.IP, error) { rr, err := cli.RunCmd(exec.Command(ociBin, "exec", "-t", containerName, "dig", "+short", dns)) ip := net.ParseIP(strings.TrimSpace(rr.Stdout.String())) - if err != nil { - return ip, errors.Wrapf(err, "resolve dns to ip: %s", rr.Output()) + return ip, errors.Wrapf(err, "resolve dns to ip") } glog.Infof("got host ip for mount in container by digging dns: %s", ip.String()) @@ -59,14 +58,14 @@ func digDNS(ociBin, containerName, dns string) (net.IP, error) { func dockerGatewayIP() (net.IP, error) { rr, err := cli.RunCmd(exec.Command(Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}")) if err != nil { - return nil, errors.Wrapf(err, "get network bridge. output: %s", rr.Output()) + return nil, errors.Wrapf(err, "get network bridge") } bridgeID := strings.TrimSpace(rr.Stdout.String()) rr, err = cli.RunCmd(exec.Command(Docker, "inspect", "--format", "{{(index .IPAM.Config 0).Gateway}}", bridgeID)) if err != nil { - return nil, errors.Wrapf(err, "inspect IP gatway for bridge network: %q. output: %s", rr.Output(), bridgeID) + return nil, errors.Wrapf(err, "inspect IP bridge network %q.", bridgeID) } ip := net.ParseIP(strings.TrimSpace(rr.Stdout.String())) @@ -87,12 +86,12 @@ func ForwardedPort(ociBinary string, ociID string, contPort int) (int, error) { //podman inspect -f "{{range .NetworkSettings.Ports}}{{if eq .ContainerPort "80"}}{{.HostPort}}{{end}}{{end}}" rr, err = cli.RunCmd(exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("{{range .NetworkSettings.Ports}}{{if eq .ContainerPort %s}}{{.HostPort}}{{end}}{{end}}", fmt.Sprint(contPort)), ociID)) if err != nil { - return 0, errors.Wrapf(err, "get host-bind port %d for %q, output %s", contPort, ociID, rr.Output()) + return 0, errors.Wrapf(err, "get port %d for %q", contPort, ociID) } } else { rr, err = cli.RunCmd(exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)) if err != nil { - return 0, errors.Wrapf(err, "get host-bind port %d for %q, output %s", contPort, ociID, rr.Output()) + return 0, errors.Wrapf(err, "get port %d for %q", contPort, ociID) } } diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 89bc22df71..c0eddc6154 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -64,9 +64,9 @@ func DeleteContainersByLabel(ociBin string, label string) []error { if err := ShutDown(ociBin, c); err != nil { glog.Infof("couldn't shut down %s (might be okay): %v ", c, err) } - cmd := exec.Command(ociBin, "rm", "-f", "-v", c) - if out, err := cmd.CombinedOutput(); err != nil { - deleteErrs = append(deleteErrs, errors.Wrapf(err, "delete container %s: output %s", c, out)) + + if _, err := cli.RunCmd(exec.Command(ociBin, "rm", "-f", "-v", c)); err != nil { + deleteErrs = append(deleteErrs, errors.Wrapf(err, "delete container %s: output %s", c, err)) } } @@ -84,9 +84,9 @@ func DeleteContainer(ociBin string, name string) error { if err := ShutDown(ociBin, name); err != nil { glog.Infof("couldn't shut down %s (might be okay): %v ", name, err) } - cmd := exec.Command(ociBin, "rm", "-f", "-v", name) - if out, err := cmd.CombinedOutput(); err != nil { - return errors.Wrapf(err, "delete container %s: output %s", name, out) + + if _, err := cli.RunCmd(exec.Command(ociBin, "rm", "-f", "-v", name)); err != nil { + return errors.Wrapf(err, "delete %s", name) } return nil } @@ -213,24 +213,8 @@ func createContainer(ociBinary string, image string, opts ...createOpt) error { args = append(args, image) args = append(args, o.ContainerArgs...) - out, err := exec.Command(ociBinary, args...).CombinedOutput() - if err != nil { - return errors.Wrapf(err, "failed args: %v output: %s", args, out) - } - - return nil -} - -// Copy copies a local asset into the container -func Copy(ociBinary string, ociID string, targetDir string, fName string) error { - if _, err := os.Stat(fName); os.IsNotExist(err) { - return errors.Wrapf(err, "error source %s does not exist", fName) - } - - destination := fmt.Sprintf("%s:%s", ociID, targetDir) - cmd := exec.Command(ociBinary, "cp", fName, destination) - if err := cmd.Run(); err != nil { - return errors.Wrapf(err, "error copying %s into node", fName) + if _, err := cli.RunCmd(exec.Command(ociBinary, args...)); err != nil { + return err } return nil @@ -238,17 +222,14 @@ func Copy(ociBinary string, ociID string, targetDir string, fName string) error // ContainerID returns id of a container name func ContainerID(ociBinary string, nameOrID string) (string, error) { - cmd := exec.Command(ociBinary, "inspect", "-f", "{{.Id}}", nameOrID) - out, err := cmd.CombinedOutput() - + rr, err := cli.RunCmd(exec.Command(ociBinary, "inspect", "-f", "{{.Id}}", nameOrID)) if err != nil { // don't return error if not found, only return empty string - if strings.Contains(string(out), "Error: No such object:") || strings.Contains(string(out), "unable to find") { + if strings.Contains(rr.Stdout.String(), "Error: No such object:") || strings.Contains(rr.Stdout.String(), "unable to find") { err = nil } - out = []byte{} + return "", err } - - return string(out), err + return rr.Stdout.String(), nil } // WarnIfSlow runs an oci command, warning about performance issues @@ -266,24 +247,23 @@ func WarnIfSlow(args ...string) ([]byte, error) { start := time.Now() glog.Infof("executing with %s timeout: %v", args, killTime) - cmd := exec.CommandContext(ctx, args[0], args[1:]...) - stdout, err := cmd.Output() + rr, err := cli.RunCmd(exec.CommandContext(ctx, args[0], args[1:]...)) d := time.Since(start) if d > warnTime { - out.WarningT(`Executing "{{.command}}" took an unusually long time: {{.duration}}`, out.V{"command": strings.Join(cmd.Args, " "), "duration": d}) + out.WarningT(`Executing "{{.command}}" took an unusually long time: {{.duration}}`, out.V{"command": rr.Command(), "duration": d}) out.ErrT(out.Tip, `Restarting the {{.name}} service may improve performance.`, out.V{"name": args[0]}) } if ctx.Err() == context.DeadlineExceeded { - return stdout, fmt.Errorf("%q timed out after %s", strings.Join(cmd.Args, " "), killTime) + return []byte(rr.Output()), fmt.Errorf("%q timed out after %s", rr.Command(), killTime) } if err != nil { if exitErr, ok := err.(*exec.ExitError); ok { - return stdout, fmt.Errorf("%q failed: %v: %s", strings.Join(cmd.Args, " "), exitErr, exitErr.Stderr) + return []byte(rr.Output()), fmt.Errorf("%q failed: %v: %s", rr.Command(), exitErr, exitErr.Stderr) } - return stdout, fmt.Errorf("%q failed: %v", strings.Join(cmd.Args, " "), err) + return []byte(rr.Output()), err } - return stdout, nil + return []byte(rr.Output()), nil } // ContainerExists checks if container name exists (either running or exited) diff --git a/pkg/drivers/kic/oci/volumes.go b/pkg/drivers/kic/oci/volumes.go index 3afd7ec6ec..917c871b17 100644 --- a/pkg/drivers/kic/oci/volumes.go +++ b/pkg/drivers/kic/oci/volumes.go @@ -80,10 +80,9 @@ func allVolumesByLabel(ociBin string, label string) ([]string, error) { // ExtractTarballToVolume runs a docker image imageName which extracts the tarball at tarballPath // to the volume named volumeName func ExtractTarballToVolume(tarballPath, volumeName, imageName string) error { - rr, err := cli.RunCmd(exec.Command(Docker, "run", "--rm", "--entrypoint", "/usr/bin/tar", "-v", fmt.Sprintf("%s:/preloaded.tar:ro", tarballPath), "-v", fmt.Sprintf("%s:/extractDir", volumeName), imageName, "-I", "lz4", "-xvf", "/preloaded.tar", "-C", "/extractDir")) - glog.Infof("executed: %s", rr.Command()) - if err != nil { - return errors.Wrapf(err, "output: %s", rr.Output()) + cmd := exec.Command(Docker, "run", "--rm", "--entrypoint", "/usr/bin/tar", "-v", fmt.Sprintf("%s:/preloaded.tar:ro", tarballPath), "-v", fmt.Sprintf("%s:/extractDir", volumeName), imageName, "-I", "lz4", "-xvf", "/preloaded.tar", "-C", "/extractDir") + if _, err := cli.RunCmd(cmd); err != nil { + return err } return nil } @@ -92,10 +91,8 @@ func ExtractTarballToVolume(tarballPath, volumeName, imageName string) error { // Caution ! if volume already exists does NOT return an error and will not apply the minikube labels on it. // TODO: this should be fixed as a part of https://github.com/kubernetes/minikube/issues/6530 func createDockerVolume(profile string, nodeName string) error { - rr, err := cli.RunCmd(exec.Command(Docker, "volume", "create", nodeName, "--label", fmt.Sprintf("%s=%s", ProfileLabelKey, profile), "--label", fmt.Sprintf("%s=%s", CreatedByLabelKey, "true"))) - glog.Infof("executed: %s", rr.Command()) - if err != nil { - return errors.Wrapf(err, "output: %s", rr.Output()) + if _, err := cli.RunCmd(exec.Command(Docker, "volume", "create", nodeName, "--label", fmt.Sprintf("%s=%s", ProfileLabelKey, profile), "--label", fmt.Sprintf("%s=%s", CreatedByLabelKey, "true"))); err != nil { + return err } return nil } From 8a0a210763012222c3e1fd03369f463e072f27df Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Thu, 23 Apr 2020 00:29:37 -0700 Subject: [PATCH 044/105] convert more funcs to use cli runner --- pkg/drivers/kic/oci/oci.go | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index c0eddc6154..96f12611f9 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -286,14 +286,12 @@ func ContainerExists(ociBin string, name string) (bool, error) { // IsCreatedByMinikube returns true if the container was created by minikube // with default assumption that it is not created by minikube when we don't know for sure func IsCreatedByMinikube(ociBinary string, nameOrID string) bool { - cmd := exec.Command(ociBinary, "inspect", nameOrID, "--format", "{{.Config.Labels}}") - out, err := cmd.CombinedOutput() - + rr, err := cli.RunCmd(exec.Command(ociBinary, "inspect", nameOrID, "--format", "{{.Config.Labels}}")) if err != nil { return false } - if strings.Contains(string(out), fmt.Sprintf("%s:true", CreatedByLabelKey)) { + if strings.Contains(rr.Stdout.String(), fmt.Sprintf("%s:true", CreatedByLabelKey)) { return true } @@ -307,14 +305,13 @@ func ListOwnedContainers(ociBinary string) ([]string, error) { // inspect return low-level information on containers func inspect(ociBinary string, containerNameOrID, format string) ([]string, error) { - cmd := exec.Command(ociBinary, "inspect", "-f", format, containerNameOrID) // ... against the "node" container var buff bytes.Buffer cmd.Stdout = &buff cmd.Stderr = &buff - err := cmd.Run() + _, err := cli.RunCmd(cmd) scanner := bufio.NewScanner(&buff) var lines []string for scanner.Scan() { @@ -375,8 +372,8 @@ func isUsernsRemapEnabled(ociBinary string) bool { var buff bytes.Buffer cmd.Stdout = &buff cmd.Stderr = &buff - err := cmd.Run() - if err != nil { + + if _, err := cli.RunCmd(cmd); err != nil { return false } @@ -488,13 +485,12 @@ func ContainerStatus(ociBin string, name string) (state.State, error) { } } -// Shutdown will run command to shut down the container +// ShutDown will run command to shut down the container // to ensure the containers process and networking bindings are all closed // to avoid containers getting stuck before delete https://github.com/kubernetes/minikube/issues/7657 func ShutDown(ociBin string, name string) error { - cmd := exec.Command(ociBin, "exec", "--privileged", "-t", name, "/bin/bash", "-c", "sudo init 0") - if out, err := cmd.CombinedOutput(); err != nil { - glog.Infof("error shutdown %s output %q : %v", name, out, err) + if _, err := cli.RunCmd(exec.Command(ociBin, "exec", "--privileged", "-t", name, "/bin/bash", "-c", "sudo init 0")); err != nil { + glog.Infof("error shutdown %s: %v", name, err) } // helps with allowing docker realize the container is exited and report its status correctly. time.Sleep(time.Second * 1) From 038aef6b48012c940d26312bca0403922ff65843 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Thu, 23 Apr 2020 00:33:40 -0700 Subject: [PATCH 045/105] unexport warnIfSlow func --- pkg/drivers/kic/kic.go | 21 +-------------------- pkg/drivers/kic/oci/oci.go | 12 ++++++------ 2 files changed, 7 insertions(+), 26 deletions(-) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 0c1a4e4f48..c3605780e3 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -234,26 +234,7 @@ func (d *Driver) GetURL() (string, error) { // GetState returns the state that the host is in (running, stopped, etc) func (d *Driver) GetState() (state.State, error) { - out, err := oci.WarnIfSlow(d.NodeConfig.OCIBinary, "inspect", "-f", "{{.State.Status}}", d.MachineName) - if err != nil { - return state.Error, err - } - - o := strings.TrimSpace(string(out)) - switch o { - case "running": - return state.Running, nil - case "exited": - return state.Stopped, nil - case "paused": - return state.Paused, nil - case "restarting": - return state.Starting, nil - case "dead": - return state.Error, nil - default: - return state.None, fmt.Errorf("unknown state") - } + return oci.ContainerStatus(d.OCIBinary, d.MachineName) } // Kill stops a host forcefully, including any containers that we are managing. diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 96f12611f9..eacd091e43 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -232,8 +232,8 @@ func ContainerID(ociBinary string, nameOrID string) (string, error) { return rr.Stdout.String(), nil } -// WarnIfSlow runs an oci command, warning about performance issues -func WarnIfSlow(args ...string) ([]byte, error) { +// warnIfSlow runs an oci command, warning about performance issues +func warnIfSlow(args ...string) ([]byte, error) { killTime := 19 * time.Second warnTime := 2 * time.Second @@ -268,7 +268,7 @@ func WarnIfSlow(args ...string) ([]byte, error) { // ContainerExists checks if container name exists (either running or exited) func ContainerExists(ociBin string, name string) (bool, error) { - out, err := WarnIfSlow(ociBin, "ps", "-a", "--format", "{{.Names}}") + out, err := warnIfSlow(ociBin, "ps", "-a", "--format", "{{.Names}}") if err != nil { return false, errors.Wrapf(err, string(out)) } @@ -431,7 +431,7 @@ func withPortMappings(portMappings []PortMapping) createOpt { // ListContainersByLabel returns all the container names with a specified label func ListContainersByLabel(ociBinary string, label string) ([]string, error) { - stdout, err := WarnIfSlow(ociBinary, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}") + stdout, err := warnIfSlow(ociBinary, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}") if err != nil { return nil, err } @@ -466,8 +466,8 @@ func PointToHostDockerDaemon() error { } // ContainerStatus returns status of a container running,exited,... -func ContainerStatus(ociBin string, name string) (state.State, error) { - out, err := WarnIfSlow(ociBin, "inspect", name, "--format={{.State.Status}}") +func ContainerStatus(ociBin string, name string, warnSlow ...bool) (state.State, error) { + out, err := warnIfSlow(ociBin, "inspect", name, "--format={{.State.Status}}") o := strings.TrimSpace(string(out)) switch o { case "running": From d9f1b532d0b34d3c314108d44c82e8a69d89da40 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Thu, 23 Apr 2020 00:33:59 -0700 Subject: [PATCH 046/105] unexport warnIfSlow func --- pkg/drivers/kic/oci/volumes.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/drivers/kic/oci/volumes.go b/pkg/drivers/kic/oci/volumes.go index 917c871b17..b583ca07d9 100644 --- a/pkg/drivers/kic/oci/volumes.go +++ b/pkg/drivers/kic/oci/volumes.go @@ -40,7 +40,7 @@ func DeleteAllVolumesByLabel(ociBin string, label string) []error { } for _, v := range vs { - if _, err := WarnIfSlow(ociBin, "volume", "rm", "--force", v); err != nil { + if _, err := warnIfSlow(ociBin, "volume", "rm", "--force", v); err != nil { deleteErrs = append(deleteErrs, fmt.Errorf("deleting %q", v)) } } @@ -55,7 +55,7 @@ func PruneAllVolumesByLabel(ociBin string, label string) []error { var deleteErrs []error glog.Infof("trying to prune all %s volumes with label %s", ociBin, label) - if _, err := WarnIfSlow(ociBin, "volume", "prune", "-f", "--filter", "label="+label); err != nil { + if _, err := warnIfSlow(ociBin, "volume", "prune", "-f", "--filter", "label="+label); err != nil { deleteErrs = append(deleteErrs, errors.Wrapf(err, "prune volume by label %s", label)) } From 17f420cc712025cf4396e368bb91be1bdd2af5e6 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Thu, 23 Apr 2020 01:23:49 -0700 Subject: [PATCH 047/105] warn slow part of cli runner --- pkg/drivers/kic/oci/cli_runner.go | 36 +++++++++++++- pkg/drivers/kic/oci/network.go | 12 ++--- pkg/drivers/kic/oci/oci.go | 81 +++++++++---------------------- pkg/drivers/kic/oci/volumes.go | 10 ++-- 4 files changed, 69 insertions(+), 70 deletions(-) diff --git a/pkg/drivers/kic/oci/cli_runner.go b/pkg/drivers/kic/oci/cli_runner.go index fec1cb48cf..fb82e91ec5 100644 --- a/pkg/drivers/kic/oci/cli_runner.go +++ b/pkg/drivers/kic/oci/cli_runner.go @@ -18,6 +18,7 @@ package oci import ( "bytes" + "context" "fmt" "io" "os/exec" @@ -25,6 +26,7 @@ import ( "time" "github.com/golang/glog" + "k8s.io/minikube/pkg/minikube/out" ) var cli = newRunner() @@ -73,7 +75,29 @@ func newRunner() *cliRunner { } // RunCmd implements the Command Runner interface to run a exec.Cmd object -func (*cliRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) { +func (*cliRunner) RunCmd(cmd *exec.Cmd, warnSlow ...bool) (*RunResult, error) { + warn := false + if len(warnSlow) > 0 { + warn = warnSlow[0] + } + + killTime := 19 * time.Second // this will be applied only if warnSlow is true + warnTime := 2 * time.Second + ctx, cancel := context.WithTimeout(context.Background(), killTime) + defer cancel() + + if cmd.Args[1] == "volume" || cmd.Args[1] == "ps" { // volume and ps requires more time than inspect + killTime = 30 * time.Second + warnTime = 3 * time.Second + } + + if warn { // convert exec.Command to with context + cmdWithCtx := exec.CommandContext(ctx, cmd.Args[0], cmd.Args[1:]...) + cmdWithCtx.Stdout = cmd.Stdout //copying the original command + cmdWithCtx.Stderr = cmd.Stderr + cmd = cmdWithCtx + } + rr := &RunResult{Args: cmd.Args} glog.Infof("Run: %v", rr.Command()) @@ -98,6 +122,16 @@ func (*cliRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) { start := time.Now() err := cmd.Run() elapsed := time.Since(start) + if warn { + if elapsed > warnTime { + out.WarningT(`Executing "{{.command}}" took an unusually long time: {{.duration}}`, out.V{"command": rr.Command(), "duration": elapsed}) + out.ErrT(out.Tip, `Restarting the {{.name}} service may improve performance.`, out.V{"name": cmd.Args[0]}) + } + + if ctx.Err() == context.DeadlineExceeded { + return rr, fmt.Errorf("%q timed out after %s", rr.Command(), killTime) + } + } if exitError, ok := err.(*exec.ExitError); ok { rr.ExitCode = exitError.ExitCode() diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index 576607d378..12ebe82128 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -78,18 +78,18 @@ func dockerGatewayIP() (net.IP, error) { // will return the docker assigned port: // 32769, nil // only supports TCP ports -func ForwardedPort(ociBinary string, ociID string, contPort int) (int, error) { +func ForwardedPort(ociBin string, ociID string, contPort int) (int, error) { var rr *RunResult var err error - if ociBinary == Podman { + if ociBin == Podman { //podman inspect -f "{{range .NetworkSettings.Ports}}{{if eq .ContainerPort "80"}}{{.HostPort}}{{end}}{{end}}" - rr, err = cli.RunCmd(exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("{{range .NetworkSettings.Ports}}{{if eq .ContainerPort %s}}{{.HostPort}}{{end}}{{end}}", fmt.Sprint(contPort)), ociID)) + rr, err = cli.RunCmd(exec.Command(ociBin, "inspect", "-f", fmt.Sprintf("{{range .NetworkSettings.Ports}}{{if eq .ContainerPort %s}}{{.HostPort}}{{end}}{{end}}", fmt.Sprint(contPort)), ociID)) if err != nil { return 0, errors.Wrapf(err, "get port %d for %q", contPort, ociID) } } else { - rr, err = cli.RunCmd(exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)) + rr, err = cli.RunCmd(exec.Command(ociBin, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)) if err != nil { return 0, errors.Wrapf(err, "get port %d for %q", contPort, ociID) } @@ -107,8 +107,8 @@ func ForwardedPort(ociBinary string, ociID string, contPort int) (int, error) { } // ContainerIPs returns ipv4,ipv6, error of a container by their name -func ContainerIPs(ociBinary string, name string) (string, string, error) { - if ociBinary == Podman { +func ContainerIPs(ociBin string, name string) (string, string, error) { + if ociBin == Podman { return podmanConttainerIP(name) } return dockerContainerIP(name) diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index eacd091e43..ba8907ca48 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -17,7 +17,6 @@ limitations under the License. package oci import ( - "context" "os" "path/filepath" "time" @@ -30,7 +29,6 @@ import ( "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/localpath" - "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/util/retry" "fmt" @@ -188,7 +186,7 @@ func CreateContainerNode(p CreateParams) error { } // CreateContainer creates a container with "docker/podman run" -func createContainer(ociBinary string, image string, opts ...createOpt) error { +func createContainer(ociBin string, image string, opts ...createOpt) error { o := &createOpts{} for _, opt := range opts { o = opt(o) @@ -205,7 +203,7 @@ func createContainer(ociBinary string, image string, opts ...createOpt) error { args := []string{"run"} // to run nested container from privileged container in podman https://bugzilla.redhat.com/show_bug.cgi?id=1687713 - if ociBinary == Podman { + if ociBin == Podman { args = append(args, "--cgroup-manager", "cgroupfs") } @@ -213,7 +211,7 @@ func createContainer(ociBinary string, image string, opts ...createOpt) error { args = append(args, image) args = append(args, o.ContainerArgs...) - if _, err := cli.RunCmd(exec.Command(ociBinary, args...)); err != nil { + if _, err := cli.RunCmd(exec.Command(ociBin, args...)); err != nil { return err } @@ -221,8 +219,8 @@ func createContainer(ociBinary string, image string, opts ...createOpt) error { } // ContainerID returns id of a container name -func ContainerID(ociBinary string, nameOrID string) (string, error) { - rr, err := cli.RunCmd(exec.Command(ociBinary, "inspect", "-f", "{{.Id}}", nameOrID)) +func ContainerID(ociBin string, nameOrID string) (string, error) { + rr, err := cli.RunCmd(exec.Command(ociBin, "inspect", "-f", "{{.Id}}", nameOrID)) if err != nil { // don't return error if not found, only return empty string if strings.Contains(rr.Stdout.String(), "Error: No such object:") || strings.Contains(rr.Stdout.String(), "unable to find") { err = nil @@ -232,48 +230,14 @@ func ContainerID(ociBinary string, nameOrID string) (string, error) { return rr.Stdout.String(), nil } -// warnIfSlow runs an oci command, warning about performance issues -func warnIfSlow(args ...string) ([]byte, error) { - killTime := 19 * time.Second - warnTime := 2 * time.Second - - if args[1] == "volume" || args[1] == "ps" { // volume and ps requires more time than inspect - killTime = 30 * time.Second - warnTime = 3 * time.Second - } - - ctx, cancel := context.WithTimeout(context.Background(), killTime) - defer cancel() - - start := time.Now() - glog.Infof("executing with %s timeout: %v", args, killTime) - rr, err := cli.RunCmd(exec.CommandContext(ctx, args[0], args[1:]...)) - d := time.Since(start) - if d > warnTime { - out.WarningT(`Executing "{{.command}}" took an unusually long time: {{.duration}}`, out.V{"command": rr.Command(), "duration": d}) - out.ErrT(out.Tip, `Restarting the {{.name}} service may improve performance.`, out.V{"name": args[0]}) - } - - if ctx.Err() == context.DeadlineExceeded { - return []byte(rr.Output()), fmt.Errorf("%q timed out after %s", rr.Command(), killTime) - } - if err != nil { - if exitErr, ok := err.(*exec.ExitError); ok { - return []byte(rr.Output()), fmt.Errorf("%q failed: %v: %s", rr.Command(), exitErr, exitErr.Stderr) - } - return []byte(rr.Output()), err - } - return []byte(rr.Output()), nil -} - // ContainerExists checks if container name exists (either running or exited) -func ContainerExists(ociBin string, name string) (bool, error) { - out, err := warnIfSlow(ociBin, "ps", "-a", "--format", "{{.Names}}") +func ContainerExists(ociBin string, name string, warnSlow ...bool) (bool, error) { + rr, err := cli.RunCmd(exec.Command(ociBin, "ps", "-a", "--format", "{{.Names}}"), true) if err != nil { - return false, errors.Wrapf(err, string(out)) + return false, err } - containers := strings.Split(string(out), "\n") + containers := strings.Split(rr.Stdout.String(), "\n") for _, c := range containers { if strings.TrimSpace(c) == name { return true, nil @@ -285,8 +249,8 @@ func ContainerExists(ociBin string, name string) (bool, error) { // IsCreatedByMinikube returns true if the container was created by minikube // with default assumption that it is not created by minikube when we don't know for sure -func IsCreatedByMinikube(ociBinary string, nameOrID string) bool { - rr, err := cli.RunCmd(exec.Command(ociBinary, "inspect", nameOrID, "--format", "{{.Config.Labels}}")) +func IsCreatedByMinikube(ociBin string, nameOrID string) bool { + rr, err := cli.RunCmd(exec.Command(ociBin, "inspect", nameOrID, "--format", "{{.Config.Labels}}")) if err != nil { return false } @@ -299,13 +263,13 @@ func IsCreatedByMinikube(ociBinary string, nameOrID string) bool { } // ListOwnedContainers lists all the containres that kic driver created on user's machine using a label -func ListOwnedContainers(ociBinary string) ([]string, error) { - return ListContainersByLabel(ociBinary, ProfileLabelKey) +func ListOwnedContainers(ociBin string) ([]string, error) { + return ListContainersByLabel(ociBin, ProfileLabelKey) } // inspect return low-level information on containers -func inspect(ociBinary string, containerNameOrID, format string) ([]string, error) { - cmd := exec.Command(ociBinary, "inspect", +func inspect(ociBin string, containerNameOrID, format string) ([]string, error) { + cmd := exec.Command(ociBin, "inspect", "-f", format, containerNameOrID) // ... against the "node" container var buff bytes.Buffer @@ -367,8 +331,8 @@ func generateMountBindings(mounts ...Mount) []string { } // isUsernsRemapEnabled checks if userns-remap is enabled in docker -func isUsernsRemapEnabled(ociBinary string) bool { - cmd := exec.Command(ociBinary, "info", "--format", "'{{json .SecurityOptions}}'") +func isUsernsRemapEnabled(ociBin string) bool { + cmd := exec.Command(ociBin, "info", "--format", "'{{json .SecurityOptions}}'") var buff bytes.Buffer cmd.Stdout = &buff cmd.Stderr = &buff @@ -430,12 +394,12 @@ func withPortMappings(portMappings []PortMapping) createOpt { } // ListContainersByLabel returns all the container names with a specified label -func ListContainersByLabel(ociBinary string, label string) ([]string, error) { - stdout, err := warnIfSlow(ociBinary, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}") +func ListContainersByLabel(ociBin string, label string, warnSlow ...bool) ([]string, error) { + rr, err := cli.RunCmd(exec.Command(ociBin, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}"), true) if err != nil { return nil, err } - s := bufio.NewScanner(bytes.NewReader(stdout)) + s := bufio.NewScanner(bytes.NewReader(rr.Stdout.Bytes())) var names []string for s.Scan() { n := strings.TrimSpace(s.Text()) @@ -467,8 +431,9 @@ func PointToHostDockerDaemon() error { // ContainerStatus returns status of a container running,exited,... func ContainerStatus(ociBin string, name string, warnSlow ...bool) (state.State, error) { - out, err := warnIfSlow(ociBin, "inspect", name, "--format={{.State.Status}}") - o := strings.TrimSpace(string(out)) + cmd := exec.Command(ociBin, "inspect", name, "--format={{.State.Status}}") + rr, err := cli.RunCmd(cmd, true) + o := strings.TrimSpace(rr.Stdout.String()) switch o { case "running": return state.Running, nil diff --git a/pkg/drivers/kic/oci/volumes.go b/pkg/drivers/kic/oci/volumes.go index b583ca07d9..ba105826c7 100644 --- a/pkg/drivers/kic/oci/volumes.go +++ b/pkg/drivers/kic/oci/volumes.go @@ -29,7 +29,7 @@ import ( // DeleteAllVolumesByLabel deletes all volumes that have a specific label // if there is no volume to delete it will return nil -func DeleteAllVolumesByLabel(ociBin string, label string) []error { +func DeleteAllVolumesByLabel(ociBin string, label string, warnSlow ...bool) []error { var deleteErrs []error glog.Infof("trying to delete all %s volumes with label %s", ociBin, label) @@ -40,7 +40,7 @@ func DeleteAllVolumesByLabel(ociBin string, label string) []error { } for _, v := range vs { - if _, err := warnIfSlow(ociBin, "volume", "rm", "--force", v); err != nil { + if _, err := cli.RunCmd(exec.Command(ociBin, "volume", "rm", "--force", v), true); err != nil { deleteErrs = append(deleteErrs, fmt.Errorf("deleting %q", v)) } } @@ -51,11 +51,11 @@ func DeleteAllVolumesByLabel(ociBin string, label string) []error { // PruneAllVolumesByLabel deletes all volumes that have a specific label // if there is no volume to delete it will return nil // example: docker volume prune -f --filter label=name.minikube.sigs.k8s.io=minikube -func PruneAllVolumesByLabel(ociBin string, label string) []error { +func PruneAllVolumesByLabel(ociBin string, label string, warnSlow ...bool) []error { var deleteErrs []error glog.Infof("trying to prune all %s volumes with label %s", ociBin, label) - - if _, err := warnIfSlow(ociBin, "volume", "prune", "-f", "--filter", "label="+label); err != nil { + cmd := exec.Command(ociBin, "volume", "prune", "-f", "--filter", "label="+label) + if _, err := cli.RunCmd(cmd, true); err != nil { deleteErrs = append(deleteErrs, errors.Wrapf(err, "prune volume by label %s", label)) } From d81547ed45a0b36f0fccd9ad37a6c59ec59bee06 Mon Sep 17 00:00:00 2001 From: Marcin Niemira Date: Thu, 23 Apr 2020 23:42:59 +1000 Subject: [PATCH 048/105] add test file --- pkg/minikube/perf/binary_test.go | 51 ++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 pkg/minikube/perf/binary_test.go diff --git a/pkg/minikube/perf/binary_test.go b/pkg/minikube/perf/binary_test.go new file mode 100644 index 0000000000..81885fc0e6 --- /dev/null +++ b/pkg/minikube/perf/binary_test.go @@ -0,0 +1,51 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package perf + +import "testing" + +func TestNewBinary(t *testing.T) { + tests := []struct { + input string + errExpected bool + }{ + { + input: "/bin/sh", + errExpected: true, + }, + { + input: "foo", + errExpected: true, + }, { + input: prPrefix + "foo", + }, + } + + for _, test := range tests { + t.Run(test.input, func(t *testing.T) { + bin, err := NewBinary(test.input) + if err != nil && test.errExpected { + t.Fatalf("Input %v returned unexpected error", test.input) + } + if test.errExpected { + return + } + if bin != nil { + t.Fatalf("Input %v returned unexpected empty binary", test.input) + } + }) + } +} From 78486a55d341bca026f2f5bfb5a1093f04410cf0 Mon Sep 17 00:00:00 2001 From: Travis Mehlinger Date: Thu, 23 Apr 2020 09:49:25 -0500 Subject: [PATCH 049/105] change hosts update container image The most recent Fedora image doesn't include a `diff` binary. This changes the container image to use Ubuntu Bionic instead, which _does_ provide a diff binary. --- deploy/addons/registry-aliases/node-etc-hosts-update.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/addons/registry-aliases/node-etc-hosts-update.tmpl b/deploy/addons/registry-aliases/node-etc-hosts-update.tmpl index 0ef938876b..4b58c26060 100644 --- a/deploy/addons/registry-aliases/node-etc-hosts-update.tmpl +++ b/deploy/addons/registry-aliases/node-etc-hosts-update.tmpl @@ -17,7 +17,7 @@ spec: spec: initContainers: - name: update - image: registry.fedoraproject.org/fedora + image: ubuntu:bionic volumeMounts: - name: etchosts mountPath: /host-etc/hosts From 8f0f1a7d76133791faff6f23c5193b62664193cd Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Thu, 23 Apr 2020 10:06:21 -0700 Subject: [PATCH 050/105] Check for Get-Wmiobject err before parsing output --- pkg/minikube/registry/drvs/hyperv/hyperv.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/minikube/registry/drvs/hyperv/hyperv.go b/pkg/minikube/registry/drvs/hyperv/hyperv.go index 5dbe874ecf..8c2b534309 100644 --- a/pkg/minikube/registry/drvs/hyperv/hyperv.go +++ b/pkg/minikube/registry/drvs/hyperv/hyperv.go @@ -91,17 +91,19 @@ func status() registry.State { cmd := exec.CommandContext(ctx, path, "@(Get-Wmiobject Win32_ComputerSystem).HypervisorPresent") out, err := cmd.CombinedOutput() - if string(out) != "True\r\n" { - errorMessage := fmt.Errorf("%s failed:\n%s", strings.Join(cmd.Args, " "), out) - fixMessage := "Start PowerShell as Administrator, and run: 'Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -All'" - // If timed out, prompt different error and suggestion messages - // See https://github.com/kubernetes/minikube/issues/6579 - if ctx.Err() != nil { - errorMessage = fmt.Errorf("%s exited unexpectedly:\n%s", strings.Join(cmd.Args, " "), ctx.Err()) - fixMessage = "If you have Hyper-V configured correctly, please try start again with `--force` specified" - } + if err != nil { + errorMessage := fmt.Errorf("%s failed:\n%s", strings.Join(cmd.Args, " "), out) + fixMessage := "Start PowerShell as an Administrator" return registry.State{Installed: false, Error: errorMessage, Fix: fixMessage, Doc: docURL} } + + // Get-Wmiobject does not return an error code for false + if strings.TrimSpace(string(out)) != "True" { + errorMessage := fmt.Errorf("%s returned %q", strings.Join(cmd.Args, " "), out) + fixMessage := "Enable Hyper-V: Start PowerShell as Administrator, and run: 'Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -All'" + return registry.State{Installed: false, Error: errorMessage, Fix: fixMessage, Doc: docURL} + } + return registry.State{Installed: true, Healthy: true} } From b90a28cc52c754dbf8e1e8b9425347a492c21cd0 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 23 Apr 2020 11:05:47 -0700 Subject: [PATCH 051/105] Download crio from GCS bucket --- hack/images/kicbase.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile index be6ebb2bce..4676f87437 100644 --- a/hack/images/kicbase.Dockerfile +++ b/hack/images/kicbase.Dockerfile @@ -15,6 +15,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # libglib2.0-0 is required for conmon, which is required for podman libglib2.0-0=2.62.1-1 \ && rm /etc/crictl.yaml + # install cri-o based on https://github.com/cri-o/cri-o/commit/96b0c34b31a9fc181e46d7d8e34fb8ee6c4dc4e1#diff-04c6e90faac2675aa89e2176d2eec7d8R128 ENV CRIO_VERSION="1.17=1.17.3~2" RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \ From e4731f1793bc869ef7cff9767b5e2d8d7116b1e0 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 23 Apr 2020 12:08:40 -0700 Subject: [PATCH 052/105] Add --cache-from to kic base image so that we reuse layers from previous images --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 061d62039d..bc525005a8 100755 --- a/Makefile +++ b/Makefile @@ -529,7 +529,7 @@ storage-provisioner-image: out/storage-provisioner-$(GOARCH) ## Build storage-pr .PHONY: kic-base-image kic-base-image: ## builds the base image used for kic. docker rmi -f $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot || true - docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --target base . + docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --cache-from $(REGISTRY)/kicbase:$(KIC_VERSION) --target base . .PHONY: upload-preloaded-images-tar upload-preloaded-images-tar: out/minikube # Upload the preloaded images for oldest supported, newest supported, and default kubernetes versions to GCS. From 82886dff440c38e380b7e5354b842b41340b97cd Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 23 Apr 2020 12:09:58 -0700 Subject: [PATCH 053/105] change env to arg --- hack/images/kicbase.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile index 4676f87437..86224d676e 100644 --- a/hack/images/kicbase.Dockerfile +++ b/hack/images/kicbase.Dockerfile @@ -17,14 +17,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm /etc/crictl.yaml # install cri-o based on https://github.com/cri-o/cri-o/commit/96b0c34b31a9fc181e46d7d8e34fb8ee6c4dc4e1#diff-04c6e90faac2675aa89e2176d2eec7d8R128 -ENV CRIO_VERSION="1.17=1.17.3~2" +ARG CRIO_VERSION="1.17=1.17.3~2" RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \ curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \ apt-key add - < Release.key && apt-get update && \ apt-get install -y --no-install-recommends cri-o-${CRIO_VERSION} # install podman -ENV PODMAN_VERSION=1.9.0~2 +ARG PODMAN_VERSION=1.9.0~2 RUN apt-get install -y --no-install-recommends podman=${PODMAN_VERSION} # disable non-docker runtimes by default From da8253d879706770f6ee0109ec8d58a3f8603c43 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 23 Apr 2020 12:46:37 -0700 Subject: [PATCH 054/105] update docs --- site/content/en/docs/commands/start.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md index ce3df92ee9..eb5bb649d2 100644 --- a/site/content/en/docs/commands/start.md +++ b/site/content/en/docs/commands/start.md @@ -27,7 +27,7 @@ minikube start [flags] --apiserver-names stringArray A set of apiserver names which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine --apiserver-port int The apiserver listening port (default 8443) --auto-update-drivers If set, automatically updates drivers to the latest version. Defaults to true. (default true) - --base-image string The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.9@sha256:82a826cc03c3e59ead5969b8020ca138de98f366c1907293df91fc57205dbb53") + --base-image string The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.10@sha256:0c3666c7dacb3c7170a86b237d0d15d37afb5a6d2f0c45046546b5c09f40378c") --cache-images If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true) --container-runtime string The container runtime to be used (docker, crio, containerd). (default "docker") --cpus int Number of CPUs allocated to Kubernetes. (default 2) From ef01ed481ceb1ec827bcaab472163d075a3d831f Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Thu, 23 Apr 2020 14:37:03 -0700 Subject: [PATCH 055/105] get rid of global CliRunner --- pkg/drivers/kic/oci/cli_runner.go | 15 ++------------- pkg/drivers/kic/oci/info.go | 4 ++-- pkg/drivers/kic/oci/network.go | 12 ++++++------ pkg/drivers/kic/oci/oci.go | 22 +++++++++++----------- pkg/drivers/kic/oci/volumes.go | 10 +++++----- 5 files changed, 26 insertions(+), 37 deletions(-) diff --git a/pkg/drivers/kic/oci/cli_runner.go b/pkg/drivers/kic/oci/cli_runner.go index fb82e91ec5..f740580423 100644 --- a/pkg/drivers/kic/oci/cli_runner.go +++ b/pkg/drivers/kic/oci/cli_runner.go @@ -29,12 +29,6 @@ import ( "k8s.io/minikube/pkg/minikube/out" ) -var cli = newRunner() - -//cliRunner runs commands using the os/exec package. -type cliRunner struct { -} - // RunResult holds the results of a Runner type RunResult struct { Stdout bytes.Buffer @@ -69,13 +63,8 @@ func (rr RunResult) Output() string { return sb.String() } -// newRunner returns an oci runner -func newRunner() *cliRunner { - return &cliRunner{} -} - -// RunCmd implements the Command Runner interface to run a exec.Cmd object -func (*cliRunner) RunCmd(cmd *exec.Cmd, warnSlow ...bool) (*RunResult, error) { +// runCmd runs a command exec.Command against docker daemon or podman +func runCmd(cmd *exec.Cmd, warnSlow ...bool) (*RunResult, error) { warn := false if len(warnSlow) > 0 { warn = warnSlow[0] diff --git a/pkg/drivers/kic/oci/info.go b/pkg/drivers/kic/oci/info.go index ab678ab688..17c7e16379 100644 --- a/pkg/drivers/kic/oci/info.go +++ b/pkg/drivers/kic/oci/info.go @@ -216,7 +216,7 @@ type podmanSysInfo struct { // dockerSystemInfo returns docker system info --format '{{json .}}' func dockerSystemInfo() (dockerSysInfo, error) { var ds dockerSysInfo - rr, err := cli.RunCmd(exec.Command(Docker, "system", "info", "--format", "{{json .}}")) + rr, err := runCmd(exec.Command(Docker, "system", "info", "--format", "{{json .}}")) if err != nil { return ds, errors.Wrap(err, "get docker system info") } @@ -231,7 +231,7 @@ func dockerSystemInfo() (dockerSysInfo, error) { // podmanSysInfo returns podman system info --format '{{json .}}' func podmanSystemInfo() (podmanSysInfo, error) { var ps podmanSysInfo - rr, err := cli.RunCmd(exec.Command(Podman, "system", "info", "--format", "'{{json .}}'")) + rr, err := runCmd(exec.Command(Podman, "system", "info", "--format", "'{{json .}}'")) if err != nil { return ps, errors.Wrap(err, "get podman system info") } diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index 12ebe82128..2ba57b06be 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -43,7 +43,7 @@ func RoutableHostIPFromInside(ociBin string, containerName string) (net.IP, erro // digDNS will get the IP record for a dns func digDNS(ociBin, containerName, dns string) (net.IP, error) { - rr, err := cli.RunCmd(exec.Command(ociBin, "exec", "-t", containerName, "dig", "+short", dns)) + rr, err := runCmd(exec.Command(ociBin, "exec", "-t", containerName, "dig", "+short", dns)) ip := net.ParseIP(strings.TrimSpace(rr.Stdout.String())) if err != nil { return ip, errors.Wrapf(err, "resolve dns to ip") @@ -56,13 +56,13 @@ func digDNS(ociBin, containerName, dns string) (net.IP, error) { // dockerGatewayIP gets the default gateway ip for the docker bridge on the user's host machine // gets the ip from user's host docker func dockerGatewayIP() (net.IP, error) { - rr, err := cli.RunCmd(exec.Command(Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}")) + rr, err := runCmd(exec.Command(Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}")) if err != nil { return nil, errors.Wrapf(err, "get network bridge") } bridgeID := strings.TrimSpace(rr.Stdout.String()) - rr, err = cli.RunCmd(exec.Command(Docker, "inspect", + rr, err = runCmd(exec.Command(Docker, "inspect", "--format", "{{(index .IPAM.Config 0).Gateway}}", bridgeID)) if err != nil { return nil, errors.Wrapf(err, "inspect IP bridge network %q.", bridgeID) @@ -84,12 +84,12 @@ func ForwardedPort(ociBin string, ociID string, contPort int) (int, error) { if ociBin == Podman { //podman inspect -f "{{range .NetworkSettings.Ports}}{{if eq .ContainerPort "80"}}{{.HostPort}}{{end}}{{end}}" - rr, err = cli.RunCmd(exec.Command(ociBin, "inspect", "-f", fmt.Sprintf("{{range .NetworkSettings.Ports}}{{if eq .ContainerPort %s}}{{.HostPort}}{{end}}{{end}}", fmt.Sprint(contPort)), ociID)) + rr, err = runCmd(exec.Command(ociBin, "inspect", "-f", fmt.Sprintf("{{range .NetworkSettings.Ports}}{{if eq .ContainerPort %s}}{{.HostPort}}{{end}}{{end}}", fmt.Sprint(contPort)), ociID)) if err != nil { return 0, errors.Wrapf(err, "get port %d for %q", contPort, ociID) } } else { - rr, err = cli.RunCmd(exec.Command(ociBin, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)) + rr, err = runCmd(exec.Command(ociBin, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)) if err != nil { return 0, errors.Wrapf(err, "get port %d for %q", contPort, ociID) } @@ -116,7 +116,7 @@ func ContainerIPs(ociBin string, name string) (string, string, error) { // podmanConttainerIP returns ipv4, ipv6 of container or error func podmanConttainerIP(name string) (string, string, error) { - rr, err := cli.RunCmd(exec.Command(Podman, "inspect", + rr, err := runCmd(exec.Command(Podman, "inspect", "-f", "{{.NetworkSettings.IPAddress}}", name)) if err != nil { diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index ba8907ca48..99fcb6c20f 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -63,7 +63,7 @@ func DeleteContainersByLabel(ociBin string, label string) []error { glog.Infof("couldn't shut down %s (might be okay): %v ", c, err) } - if _, err := cli.RunCmd(exec.Command(ociBin, "rm", "-f", "-v", c)); err != nil { + if _, err := runCmd(exec.Command(ociBin, "rm", "-f", "-v", c)); err != nil { deleteErrs = append(deleteErrs, errors.Wrapf(err, "delete container %s: output %s", c, err)) } @@ -83,7 +83,7 @@ func DeleteContainer(ociBin string, name string) error { glog.Infof("couldn't shut down %s (might be okay): %v ", name, err) } - if _, err := cli.RunCmd(exec.Command(ociBin, "rm", "-f", "-v", name)); err != nil { + if _, err := runCmd(exec.Command(ociBin, "rm", "-f", "-v", name)); err != nil { return errors.Wrapf(err, "delete %s", name) } return nil @@ -211,7 +211,7 @@ func createContainer(ociBin string, image string, opts ...createOpt) error { args = append(args, image) args = append(args, o.ContainerArgs...) - if _, err := cli.RunCmd(exec.Command(ociBin, args...)); err != nil { + if _, err := runCmd(exec.Command(ociBin, args...)); err != nil { return err } @@ -220,7 +220,7 @@ func createContainer(ociBin string, image string, opts ...createOpt) error { // ContainerID returns id of a container name func ContainerID(ociBin string, nameOrID string) (string, error) { - rr, err := cli.RunCmd(exec.Command(ociBin, "inspect", "-f", "{{.Id}}", nameOrID)) + rr, err := runCmd(exec.Command(ociBin, "inspect", "-f", "{{.Id}}", nameOrID)) if err != nil { // don't return error if not found, only return empty string if strings.Contains(rr.Stdout.String(), "Error: No such object:") || strings.Contains(rr.Stdout.String(), "unable to find") { err = nil @@ -232,7 +232,7 @@ func ContainerID(ociBin string, nameOrID string) (string, error) { // ContainerExists checks if container name exists (either running or exited) func ContainerExists(ociBin string, name string, warnSlow ...bool) (bool, error) { - rr, err := cli.RunCmd(exec.Command(ociBin, "ps", "-a", "--format", "{{.Names}}"), true) + rr, err := runCmd(exec.Command(ociBin, "ps", "-a", "--format", "{{.Names}}"), true) if err != nil { return false, err } @@ -250,7 +250,7 @@ func ContainerExists(ociBin string, name string, warnSlow ...bool) (bool, error) // IsCreatedByMinikube returns true if the container was created by minikube // with default assumption that it is not created by minikube when we don't know for sure func IsCreatedByMinikube(ociBin string, nameOrID string) bool { - rr, err := cli.RunCmd(exec.Command(ociBin, "inspect", nameOrID, "--format", "{{.Config.Labels}}")) + rr, err := runCmd(exec.Command(ociBin, "inspect", nameOrID, "--format", "{{.Config.Labels}}")) if err != nil { return false } @@ -275,7 +275,7 @@ func inspect(ociBin string, containerNameOrID, format string) ([]string, error) var buff bytes.Buffer cmd.Stdout = &buff cmd.Stderr = &buff - _, err := cli.RunCmd(cmd) + _, err := runCmd(cmd) scanner := bufio.NewScanner(&buff) var lines []string for scanner.Scan() { @@ -337,7 +337,7 @@ func isUsernsRemapEnabled(ociBin string) bool { cmd.Stdout = &buff cmd.Stderr = &buff - if _, err := cli.RunCmd(cmd); err != nil { + if _, err := runCmd(cmd); err != nil { return false } @@ -395,7 +395,7 @@ func withPortMappings(portMappings []PortMapping) createOpt { // ListContainersByLabel returns all the container names with a specified label func ListContainersByLabel(ociBin string, label string, warnSlow ...bool) ([]string, error) { - rr, err := cli.RunCmd(exec.Command(ociBin, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}"), true) + rr, err := runCmd(exec.Command(ociBin, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}"), true) if err != nil { return nil, err } @@ -432,7 +432,7 @@ func PointToHostDockerDaemon() error { // ContainerStatus returns status of a container running,exited,... func ContainerStatus(ociBin string, name string, warnSlow ...bool) (state.State, error) { cmd := exec.Command(ociBin, "inspect", name, "--format={{.State.Status}}") - rr, err := cli.RunCmd(cmd, true) + rr, err := runCmd(cmd, true) o := strings.TrimSpace(rr.Stdout.String()) switch o { case "running": @@ -454,7 +454,7 @@ func ContainerStatus(ociBin string, name string, warnSlow ...bool) (state.State, // to ensure the containers process and networking bindings are all closed // to avoid containers getting stuck before delete https://github.com/kubernetes/minikube/issues/7657 func ShutDown(ociBin string, name string) error { - if _, err := cli.RunCmd(exec.Command(ociBin, "exec", "--privileged", "-t", name, "/bin/bash", "-c", "sudo init 0")); err != nil { + if _, err := runCmd(exec.Command(ociBin, "exec", "--privileged", "-t", name, "/bin/bash", "-c", "sudo init 0")); err != nil { glog.Infof("error shutdown %s: %v", name, err) } // helps with allowing docker realize the container is exited and report its status correctly. diff --git a/pkg/drivers/kic/oci/volumes.go b/pkg/drivers/kic/oci/volumes.go index ba105826c7..a881dc6670 100644 --- a/pkg/drivers/kic/oci/volumes.go +++ b/pkg/drivers/kic/oci/volumes.go @@ -40,7 +40,7 @@ func DeleteAllVolumesByLabel(ociBin string, label string, warnSlow ...bool) []er } for _, v := range vs { - if _, err := cli.RunCmd(exec.Command(ociBin, "volume", "rm", "--force", v), true); err != nil { + if _, err := runCmd(exec.Command(ociBin, "volume", "rm", "--force", v), true); err != nil { deleteErrs = append(deleteErrs, fmt.Errorf("deleting %q", v)) } } @@ -55,7 +55,7 @@ func PruneAllVolumesByLabel(ociBin string, label string, warnSlow ...bool) []err var deleteErrs []error glog.Infof("trying to prune all %s volumes with label %s", ociBin, label) cmd := exec.Command(ociBin, "volume", "prune", "-f", "--filter", "label="+label) - if _, err := cli.RunCmd(cmd, true); err != nil { + if _, err := runCmd(cmd, true); err != nil { deleteErrs = append(deleteErrs, errors.Wrapf(err, "prune volume by label %s", label)) } @@ -65,7 +65,7 @@ func PruneAllVolumesByLabel(ociBin string, label string, warnSlow ...bool) []err // allVolumesByLabel returns name of all docker volumes by a specific label // will not return error if there is no volume found. func allVolumesByLabel(ociBin string, label string) ([]string, error) { - rr, err := cli.RunCmd(exec.Command(ociBin, "volume", "ls", "--filter", "label="+label, "--format", "{{.Name}}")) + rr, err := runCmd(exec.Command(ociBin, "volume", "ls", "--filter", "label="+label, "--format", "{{.Name}}")) s := bufio.NewScanner(bytes.NewReader(rr.Stdout.Bytes())) var vols []string for s.Scan() { @@ -81,7 +81,7 @@ func allVolumesByLabel(ociBin string, label string) ([]string, error) { // to the volume named volumeName func ExtractTarballToVolume(tarballPath, volumeName, imageName string) error { cmd := exec.Command(Docker, "run", "--rm", "--entrypoint", "/usr/bin/tar", "-v", fmt.Sprintf("%s:/preloaded.tar:ro", tarballPath), "-v", fmt.Sprintf("%s:/extractDir", volumeName), imageName, "-I", "lz4", "-xvf", "/preloaded.tar", "-C", "/extractDir") - if _, err := cli.RunCmd(cmd); err != nil { + if _, err := runCmd(cmd); err != nil { return err } return nil @@ -91,7 +91,7 @@ func ExtractTarballToVolume(tarballPath, volumeName, imageName string) error { // Caution ! if volume already exists does NOT return an error and will not apply the minikube labels on it. // TODO: this should be fixed as a part of https://github.com/kubernetes/minikube/issues/6530 func createDockerVolume(profile string, nodeName string) error { - if _, err := cli.RunCmd(exec.Command(Docker, "volume", "create", nodeName, "--label", fmt.Sprintf("%s=%s", ProfileLabelKey, profile), "--label", fmt.Sprintf("%s=%s", CreatedByLabelKey, "true"))); err != nil { + if _, err := runCmd(exec.Command(Docker, "volume", "create", nodeName, "--label", fmt.Sprintf("%s=%s", ProfileLabelKey, profile), "--label", fmt.Sprintf("%s=%s", CreatedByLabelKey, "true"))); err != nil { return err } return nil From 045b1e9a3ac9e9d213037d1d2e46d0867185abf1 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 23 Apr 2020 15:02:14 -0700 Subject: [PATCH 056/105] update docs --- site/content/en/docs/commands/start.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md index 5f1e19b23e..ede8a83f0b 100644 --- a/site/content/en/docs/commands/start.md +++ b/site/content/en/docs/commands/start.md @@ -50,7 +50,7 @@ minikube start [flags] Valid kubeadm parameters: ignore-preflight-errors, dry-run, kubeconfig, kubeconfig-dir, node-name, cri-socket, experimental-upload-certs, certificate-key, rootfs, skip-phases, pod-network-cidr --feature-gates string A set of key=value pairs that describe feature gates for alpha/experimental features. --force Force minikube to perform possibly dangerous operations - --force-systemd If set, force the container runtime to use sytemd as cgroup manager. Currently avaiable for docker and crio. Defaults to false. + --force-systemd If set, force the container runtime to use sytemd as cgroup manager. Currently available for docker and crio. Defaults to false. -h, --help help for start --host-dns-resolver Enable host resolver for NAT DNS requests (virtualbox driver only) (default true) --host-only-cidr string The CIDR to be used for the minikube VM (virtualbox driver only) (default "192.168.99.1/24") From 1087f56b7a674e17d1a284938d7e3eb357d1eed9 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 23 Apr 2020 15:11:40 -0700 Subject: [PATCH 057/105] Pass forceSystemd into cRuntime lib to avoid errors with cobra --- pkg/minikube/cruntime/containerd.go | 7 ++++++- pkg/minikube/cruntime/crio.go | 7 ++++++- pkg/minikube/cruntime/cruntime.go | 2 +- pkg/minikube/cruntime/docker.go | 12 +++++------- pkg/minikube/node/start.go | 2 +- 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 570407b61a..864f70b067 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -197,12 +197,17 @@ func generateContainerdConfig(cr CommandRunner, imageRepository string, kv semve } // Enable idempotently enables containerd on a host -func (r *Containerd) Enable(disOthers bool) error { +func (r *Containerd) Enable(disOthers, forceSystemd bool) error { if disOthers { if err := disableOthers(r, r.Runner); err != nil { glog.Warningf("disableOthers: %v", err) } } + if forceSystemd { + if err := r.ForceSystemd(); err != nil { + return err + } + } if err := populateCRIConfig(r.Runner, r.SocketPath()); err != nil { return err } diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 4ff52cc37e..b9ab117459 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -110,12 +110,17 @@ func (r *CRIO) Active() bool { } // Enable idempotently enables CRIO on a host -func (r *CRIO) Enable(disOthers bool) error { +func (r *CRIO) Enable(disOthers, forceSystemd bool) error { if disOthers { if err := disableOthers(r, r.Runner); err != nil { glog.Warningf("disableOthers: %v", err) } } + if forceSystemd { + if err := r.ForceSystemd(); err != nil { + return err + } + } if err := populateCRIConfig(r.Runner, r.SocketPath()); err != nil { return err } diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 3a5ad43d89..212709ee5b 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -63,7 +63,7 @@ type Manager interface { // Version retrieves the current version of this runtime Version() (string, error) // Enable idempotently enables this runtime on a host - Enable(bool) error + Enable(bool, bool) error // Disable idempotently disables this runtime on a host Disable() error // Active returns whether or not a runtime is active on a host diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index eb4c8030df..7f51fb3a3d 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -25,7 +25,6 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" - "github.com/spf13/viper" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/command" @@ -104,15 +103,17 @@ func (r *Docker) Active() bool { } // Enable idempotently enables Docker on a host -func (r *Docker) Enable(disOthers bool) error { +func (r *Docker) Enable(disOthers, forceSystemd bool) error { if disOthers { if err := disableOthers(r, r.Runner); err != nil { glog.Warningf("disableOthers: %v", err) } } - if err := r.ForceSystemd(); err != nil { - return errors.Wrap(err, "forcing systemd") + if forceSystemd { + if err := r.ForceSystemd(); err != nil { + return err + } } return r.Init.Start("docker") @@ -281,9 +282,6 @@ func (r *Docker) SystemLogCmd(len int) string { // ForceSystemd forces the docker daemon to use systemd as cgroup manager func (r *Docker) ForceSystemd() error { - if !viper.GetBool("force-systemd") { - return nil - } daemonConfig := `{ "exec-opts": ["native.cgroupdriver=systemd"], "log-driver": "json-file", diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 5f2aecb0d9..aa6a908e0b 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -255,7 +255,7 @@ func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, k } } - err = cr.Enable(disableOthers) + err = cr.Enable(disableOthers, viper.GetBool("force-systemd")) if err != nil { debug.PrintStack() exit.WithError("Failed to enable container runtime", err) From 746d225bfbd3d6f51165b47430e8c23d7dc135ba Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 23 Apr 2020 15:15:57 -0700 Subject: [PATCH 058/105] remove unused code --- pkg/minikube/cruntime/containerd.go | 7 +------ pkg/minikube/cruntime/crio.go | 7 +------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 864f70b067..4694a58edb 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -197,17 +197,12 @@ func generateContainerdConfig(cr CommandRunner, imageRepository string, kv semve } // Enable idempotently enables containerd on a host -func (r *Containerd) Enable(disOthers, forceSystemd bool) error { +func (r *Containerd) Enable(disOthers, _ bool) error { if disOthers { if err := disableOthers(r, r.Runner); err != nil { glog.Warningf("disableOthers: %v", err) } } - if forceSystemd { - if err := r.ForceSystemd(); err != nil { - return err - } - } if err := populateCRIConfig(r.Runner, r.SocketPath()); err != nil { return err } diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index b9ab117459..015b723ae0 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -110,17 +110,12 @@ func (r *CRIO) Active() bool { } // Enable idempotently enables CRIO on a host -func (r *CRIO) Enable(disOthers, forceSystemd bool) error { +func (r *CRIO) Enable(disOthers, _ bool) error { if disOthers { if err := disableOthers(r, r.Runner); err != nil { glog.Warningf("disableOthers: %v", err) } } - if forceSystemd { - if err := r.ForceSystemd(); err != nil { - return err - } - } if err := populateCRIConfig(r.Runner, r.SocketPath()); err != nil { return err } From cfb1d2198202b2685aa38a70b7d7dd0f4180de24 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 23 Apr 2020 16:10:20 -0700 Subject: [PATCH 059/105] fix lint --- hack/preload-images/generate.go | 2 +- pkg/minikube/cruntime/cruntime_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go index daa9e77f09..54c680bc48 100644 --- a/hack/preload-images/generate.go +++ b/hack/preload-images/generate.go @@ -87,7 +87,7 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string if err != nil { return errors.Wrap(err, "failed create new runtime") } - if err := cr.Enable(true); err != nil { + if err := cr.Enable(true, false); err != nil { return errors.Wrap(err, "enable container runtime") } diff --git a/pkg/minikube/cruntime/cruntime_test.go b/pkg/minikube/cruntime/cruntime_test.go index aef420cf84..c2a5b4324d 100644 --- a/pkg/minikube/cruntime/cruntime_test.go +++ b/pkg/minikube/cruntime/cruntime_test.go @@ -581,7 +581,7 @@ func TestEnable(t *testing.T) { if err != nil { t.Fatalf("New(%s): %v", tc.runtime, err) } - err = cr.Enable(true) + err = cr.Enable(true, false) if err != nil { t.Errorf("%s disable unexpected error: %v", tc.runtime, err) } From 52b57374fcc02d9b17ae62bc4f499a63d617fd44 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 23 Apr 2020 16:56:30 -0700 Subject: [PATCH 060/105] make forceSystemd private --- pkg/minikube/cruntime/containerd.go | 5 ----- pkg/minikube/cruntime/crio.go | 5 ----- pkg/minikube/cruntime/cruntime.go | 2 -- pkg/minikube/cruntime/docker.go | 4 ++-- 4 files changed, 2 insertions(+), 14 deletions(-) diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 4694a58edb..9dabf8a466 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -429,8 +429,3 @@ func addRepoTagToImageName(imgName string) string { } // else it already has repo name dont add anything return imgName } - -// TODO: Implement for containerd -func (r *Containerd) ForceSystemd() error { - return nil -} diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 015b723ae0..8caa5cda97 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -229,8 +229,3 @@ func (r *CRIO) Preload(cfg config.KubernetesConfig) error { } return fmt.Errorf("not yet implemented for %s", r.Name()) } - -// ForceSystemd does nothing for crio since it already uses systemd as cgroup manager -func (r *CRIO) ForceSystemd() error { - return nil -} diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 212709ee5b..3ba208a47a 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -104,8 +104,6 @@ type Manager interface { SystemLogCmd(int) string // Preload preloads the container runtime with k8s images Preload(config.KubernetesConfig) error - // ForceSystemd forces the container runtime to use systemd as cgroup manager - ForceSystemd() error } // Config is runtime configuration diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 7f51fb3a3d..882a4a18d8 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -111,7 +111,7 @@ func (r *Docker) Enable(disOthers, forceSystemd bool) error { } if forceSystemd { - if err := r.ForceSystemd(); err != nil { + if err := r.forceSystemd(); err != nil { return err } } @@ -281,7 +281,7 @@ func (r *Docker) SystemLogCmd(len int) string { } // ForceSystemd forces the docker daemon to use systemd as cgroup manager -func (r *Docker) ForceSystemd() error { +func (r *Docker) forceSystemd() error { daemonConfig := `{ "exec-opts": ["native.cgroupdriver=systemd"], "log-driver": "json-file", From c247e6f2ee30374c12ad2eef28ed148d0831b088 Mon Sep 17 00:00:00 2001 From: Marcin Niemira Date: Fri, 24 Apr 2020 10:24:31 +1000 Subject: [PATCH 061/105] add tests for binary(perf package --- pkg/minikube/perf/binary_test.go | 52 ++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/pkg/minikube/perf/binary_test.go b/pkg/minikube/perf/binary_test.go index 81885fc0e6..021833e6cf 100644 --- a/pkg/minikube/perf/binary_test.go +++ b/pkg/minikube/perf/binary_test.go @@ -16,35 +16,67 @@ limitations under the License. package perf import "testing" +import "strings" + +func TestBinaryName(t *testing.T) { + tests := []struct { + expected string + binary Binary + }{ + { + expected: "foo", + binary: Binary{path: "foo", pr: 0}, + }, + { + expected: "Minikube (PR 1)", + binary: Binary{path: "bar", pr: 1}, + }, + } + + for _, test := range tests { + t.Run(test.expected, func(t *testing.T) { + name := test.binary.Name() + if name != test.expected { + t.Fatalf("Binary name(%v) doesn't match expected name(%v)", name, test.expected) + } + }) + + } +} func TestNewBinary(t *testing.T) { tests := []struct { - input string - errExpected bool + input, prNum string + errExpected bool }{ { - input: "/bin/sh", - errExpected: true, + input: prPrefix + "42", + prNum: "42", }, { - input: "foo", + input: "42", + prNum: "42", + }, + { + input: prPrefix + "XYZ", errExpected: true, - }, { - input: prPrefix + "foo", }, } for _, test := range tests { t.Run(test.input, func(t *testing.T) { bin, err := NewBinary(test.input) - if err != nil && test.errExpected { + if err == nil && test.errExpected { t.Fatalf("Input %v returned unexpected error", test.input) } if test.errExpected { return } - if bin != nil { - t.Fatalf("Input %v returned unexpected empty binary", test.input) + if bin == nil { + t.Fatalf("Input string(%v) returned unexpected empty binary", test.input) + } + if !strings.Contains(bin.path, test.prNum) { + t.Fatalf("Binary path(%v) doesn't contain expected(%v)", bin.path, test.prNum) } }) } From a687a721ec052ae5ec9b236b6c9090bc5e60a3e8 Mon Sep 17 00:00:00 2001 From: Marcin Niemira Date: Fri, 24 Apr 2020 10:27:16 +1000 Subject: [PATCH 062/105] keep imports syntax coherent --- pkg/minikube/perf/binary_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/minikube/perf/binary_test.go b/pkg/minikube/perf/binary_test.go index 021833e6cf..935ba933c9 100644 --- a/pkg/minikube/perf/binary_test.go +++ b/pkg/minikube/perf/binary_test.go @@ -15,8 +15,10 @@ limitations under the License. */ package perf -import "testing" -import "strings" +import ( + "strings" + "testing" +) func TestBinaryName(t *testing.T) { tests := []struct { From 9994f287e0a401f7793c92c8f76e81ec56c75b66 Mon Sep 17 00:00:00 2001 From: Marcin Niemira Date: Fri, 24 Apr 2020 10:33:24 +1000 Subject: [PATCH 063/105] boilerplate fix --- pkg/minikube/perf/binary_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/minikube/perf/binary_test.go b/pkg/minikube/perf/binary_test.go index 935ba933c9..e068a1d7e1 100644 --- a/pkg/minikube/perf/binary_test.go +++ b/pkg/minikube/perf/binary_test.go @@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package perf import ( From 644b41989ddc5a43e9960aaad37484fea77703c4 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Thu, 23 Apr 2020 20:11:31 -0700 Subject: [PATCH 064/105] Fix sysctl fs.protected_regular=1 typo --- pkg/minikube/problem/err_map.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/problem/err_map.go b/pkg/minikube/problem/err_map.go index 02ba0dd04e..5e56e4f25c 100644 --- a/pkg/minikube/problem/err_map.go +++ b/pkg/minikube/problem/err_map.go @@ -543,7 +543,7 @@ var osProblems = map[string]match{ }, "JUJU_LOCK_DENIED": { Regexp: re(`unable to open /tmp/juju.*: permission denied`), - Advice: "Run 'sudo sysctl fs.protected_regular=1', or try a driver which does not require root, such as '--driver=docker'", + Advice: "Run 'sudo sysctl fs.protected_regular=0', or try a driver which does not require root, such as '--driver=docker'", GOOS: []string{"linux"}, Issues: []int{6391}, }, From f361ae5811a04717a9056a89d92357b013d9df8c Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Fri, 24 Apr 2020 09:42:37 -0700 Subject: [PATCH 065/105] Add proposal for scheduled shutdown/pause --- .../20200424-scheduled-shutdown.md | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 enhancements/proposed/20200424-scheduled-shutdown/20200424-scheduled-shutdown.md diff --git a/enhancements/proposed/20200424-scheduled-shutdown/20200424-scheduled-shutdown.md b/enhancements/proposed/20200424-scheduled-shutdown/20200424-scheduled-shutdown.md new file mode 100644 index 0000000000..11d9cb1081 --- /dev/null +++ b/enhancements/proposed/20200424-scheduled-shutdown/20200424-scheduled-shutdown.md @@ -0,0 +1,86 @@ +# Scheduled shutdown and pause + +* First proposed: 2020-04-20 +* Authors: Thomas Stromberg (@tstromberg) + +## Reviewer Priorities + +Please review this proposal with the following priorities: + +* Does this fit with minikube's [principles](https://minikube.sigs.k8s.io/docs/concepts/principles/)? +* Are there other approaches to consider? +* Could the implementation be made simpler? +* Are there usability, reliability, or technical debt concerns? + +Please leave the above text in your proposal as instructions to the reader. + +## Summary + +Add the ability to schedule a future shutdown or pause event. + +This is useful for two sets of users: + +* command-lines which interact with a minikube on per-invocation basis. These command-line tools may not be aware of when the final invocation is, but would like low-latency between-commands +* IDE's which start minikube on an as-needed basis. Not all IDE's have the ability to trigger exit hooks when closed. + +## Goals + +* The ability to schedule a pause or shutdown event +* The ability to defer or update the scheduled event +* "minikube start" transparently clears pending scheduled events + +## Non-Goals + +* Automatically idle detection. This is a related, but complimentary idea. + +## Design Details + +### Proposed interface: + +* `minikube pause --schedule 5m` +* `minikube stop --schedule 5m` + +* Each scheduled pause would overwrite the previous scheduled event. +* Each scheduled stop would overwrite the previous scheduled event. +* Each call to `minikube start` would clear scheduled events + +As a `keep-alive` implementation, tools will repeat the command to reset the clock, and move the event 5 minutes into the future. + +### Implementation idea #1: host-based + +* If `--schedule` is used, the minikube command will daemonize, storing a pid in a well-known location, such as `$HOME/.minikube/profiles//scheduled_pause.pid`. +* If the pid already exists, the previous process will be killed, cancelling the scheduled event. + +Advantages: + +* Able to re-use all of the existing `pause` and `stop` implementation within minikube. +* Built-in handling for multiple architectures +* Does not consume memory reserved for the VM + +Disadvantages: + +* Runs a background task on the host +* Daemonization may require different handling on Windows + +### Implementation idea #2: guest-based + +minikube would connect to the control-plane via SSH, and run the equivalent of: + +```shell +killall minikube-schedule +sleep 300 + +for node in $other-nodes; do + ssh $node halt +done +halt +``` + +Advantages: + +* Consistent execution environment + +Disadvantages: + +* Requires creation of a helper binary that runs within the VM +* Untested: some drivers may not fully release resources if shutdown from inside the VM From 1e8c4bb8da7249d84d0029301a5d6fc28fd462c3 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Fri, 24 Apr 2020 09:52:03 -0700 Subject: [PATCH 066/105] Use --after for duration rather than --schedule --- .../20200424-scheduled-shutdown.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/enhancements/proposed/20200424-scheduled-shutdown/20200424-scheduled-shutdown.md b/enhancements/proposed/20200424-scheduled-shutdown/20200424-scheduled-shutdown.md index 11d9cb1081..8bf6e9458b 100644 --- a/enhancements/proposed/20200424-scheduled-shutdown/20200424-scheduled-shutdown.md +++ b/enhancements/proposed/20200424-scheduled-shutdown/20200424-scheduled-shutdown.md @@ -37,8 +37,8 @@ This is useful for two sets of users: ### Proposed interface: -* `minikube pause --schedule 5m` -* `minikube stop --schedule 5m` +* `minikube pause --after 5m` +* `minikube stop --after 5m` * Each scheduled pause would overwrite the previous scheduled event. * Each scheduled stop would overwrite the previous scheduled event. From f3d4db330af75082a9a64c3fc75776146bb918cf Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Fri, 24 Apr 2020 12:07:49 -0700 Subject: [PATCH 067/105] adding ability to detect multinode clusters --- pkg/minikube/cluster/cluster.go | 21 ++++++++++++++++++++- pkg/minikube/config/config.go | 15 +++++++++++++++ pkg/minikube/driver/driver.go | 2 +- pkg/minikube/node/start.go | 15 +-------------- 4 files changed, 37 insertions(+), 16 deletions(-) diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go index 9e385686f2..7d63f050f0 100644 --- a/pkg/minikube/cluster/cluster.go +++ b/pkg/minikube/cluster/cluster.go @@ -28,7 +28,9 @@ import ( "k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/machine" ) // This init function is used to set the logtostderr variable to false so that INFO level log info does not clutter the CLI @@ -44,7 +46,6 @@ func init() { } // Bootstrapper returns a new bootstrapper for the cluster -// TODO(#6891): Remove node as an argument func Bootstrapper(api libmachine.API, bootstrapperName string, cc config.ClusterConfig, r command.Runner) (bootstrapper.Bootstrapper, error) { var b bootstrapper.Bootstrapper var err error @@ -59,3 +60,21 @@ func Bootstrapper(api libmachine.API, bootstrapperName string, cc config.Cluster } return b, nil } + +// ControlPlaneBootstrapper returns the bootstrapper for the cluster's control plane +func ControlPlaneBootstrapper(mAPI libmachine.API, cc *config.ClusterConfig, bootstrapperName string) (bootstrapper.Bootstrapper, error) { + cp, err := config.PrimaryControlPlane(cc) + if err != nil { + return nil, errors.Wrap(err, "getting primary control plane") + } + h, err := machine.LoadHost(mAPI, driver.MachineName(*cc, cp)) + if err != nil { + return nil, errors.Wrap(err, "getting control plane host") + } + cpr, err := machine.CommandRunner(h) + if err != nil { + return nil, errors.Wrap(err, "getting control plane command runner") + } + + return Bootstrapper(mAPI, bootstrapperName, *cc, cpr) +} diff --git a/pkg/minikube/config/config.go b/pkg/minikube/config/config.go index 4db4af8dc5..4fe3f79f08 100644 --- a/pkg/minikube/config/config.go +++ b/pkg/minikube/config/config.go @@ -25,6 +25,7 @@ import ( "github.com/pkg/errors" + "k8s.io/minikube/pkg/drivers/kic" "k8s.io/minikube/pkg/minikube/localpath" ) @@ -200,3 +201,17 @@ func (c *simpleConfigLoader) WriteConfigToFile(profileName string, cc *ClusterCo } return ioutil.WriteFile(path, contents, 0644) } + +// MultiNodeCNIConfig add default CNI config needed for multinode clusters and saves off the config +func MultiNodeCNIConfig(cc *ClusterConfig) { + cc.KubernetesConfig.NetworkPlugin = "cni" + cc.KubernetesConfig.ExtraOptions.Set(fmt.Sprintf("kubeadm.pod-network-cidr=%s", kic.DefaultPodCIDR)) + SaveProfile(cc.Name, cc) +} + +// MultiNode returns true if the cluster +func MultiNode(cc ClusterConfig) bool { + if len(cc.Nodes) > 1 { + return true + } +} diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index a55f7de441..fe9efd580e 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -163,7 +163,7 @@ func FlagDefaults(name string) FlagHints { fh := FlagHints{} if name != None { fh.CacheImages = true - // only for kic, till other run-times are available we auto-set containerd. + // only for kic, until other runtimes are available we auto-set containerd. if name == Docker { fh.ExtraOptions = append(fh.ExtraOptions, fmt.Sprintf("kubeadm.pod-network-cidr=%s", kic.DefaultPodCIDR)) } diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 5f2aecb0d9..19ddf2ef9b 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -156,20 +156,7 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { } // Make sure to use the command runner for the control plane to generate the join token - cp, err := config.PrimaryControlPlane(starter.Cfg) - if err != nil { - return nil, errors.Wrap(err, "getting primary control plane") - } - h, err := machine.LoadHost(starter.MachineAPI, driver.MachineName(*starter.Cfg, cp)) - if err != nil { - return nil, errors.Wrap(err, "getting control plane host") - } - cpr, err := machine.CommandRunner(h) - if err != nil { - return nil, errors.Wrap(err, "getting control plane command runner") - } - - cpBs, err := cluster.Bootstrapper(starter.MachineAPI, viper.GetString(cmdcfg.Bootstrapper), *starter.Cfg, cpr) + cpBs, err := cluster.ControlPlaneBootstrapper(starter.MachineAPI, starter.Cfg, viper.GetString(cmdcfg.Bootstrapper)) if err != nil { return nil, errors.Wrap(err, "getting control plane bootstrapper") } From 475bbc4ff227d9a8b6967f834dcb3b427bed43a1 Mon Sep 17 00:00:00 2001 From: Travis Mehlinger Date: Fri, 24 Apr 2020 14:10:24 -0500 Subject: [PATCH 068/105] use alpine instead --- deploy/addons/registry-aliases/node-etc-hosts-update.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/addons/registry-aliases/node-etc-hosts-update.tmpl b/deploy/addons/registry-aliases/node-etc-hosts-update.tmpl index 4b58c26060..dce9320c60 100644 --- a/deploy/addons/registry-aliases/node-etc-hosts-update.tmpl +++ b/deploy/addons/registry-aliases/node-etc-hosts-update.tmpl @@ -17,7 +17,7 @@ spec: spec: initContainers: - name: update - image: ubuntu:bionic + image: alpine:3.11 volumeMounts: - name: etchosts mountPath: /host-etc/hosts @@ -39,7 +39,7 @@ spec: for H in $REGISTRY_ALIASES; do echo "$HOSTS" | grep "$H" || HOSTS="$HOSTS$NL$REGISTRY_SERVICE_HOST$TAB$H"; done; - echo "$HOSTS" | diff -u /host-etc/hosts - || echo "$HOSTS" > /host-etc/hosts + echo "$HOSTS" | diff -U 3 /host-etc/hosts - || echo "$HOSTS" > /host-etc/hosts echo "Done." containers: - name: pause-for-update From 18255467574c7da109aec80d20253b77775b78b5 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 24 Apr 2020 14:05:35 -0700 Subject: [PATCH 069/105] Restart docker if we've changed the daemon config --- pkg/minikube/cruntime/docker.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 882a4a18d8..bbc5b7ffcf 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -114,6 +114,7 @@ func (r *Docker) Enable(disOthers, forceSystemd bool) error { if err := r.forceSystemd(); err != nil { return err } + return r.Init.Restart("docker") } return r.Init.Start("docker") @@ -282,6 +283,7 @@ func (r *Docker) SystemLogCmd(len int) string { // ForceSystemd forces the docker daemon to use systemd as cgroup manager func (r *Docker) forceSystemd() error { + glog.Infof("Forcing docker to use systemd as cgroup manager...") daemonConfig := `{ "exec-opts": ["native.cgroupdriver=systemd"], "log-driver": "json-file", From 63f9bfbb61946103977c88ee3064de42633e9dce Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 24 Apr 2020 14:07:54 -0700 Subject: [PATCH 070/105] addres review comments --- pkg/drivers/kic/kic.go | 4 ++-- pkg/drivers/kic/oci/oci.go | 6 +++--- pkg/drivers/kic/oci/volumes.go | 4 ++-- pkg/minikube/machine/delete.go | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index c3605780e3..300945752f 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -99,7 +99,7 @@ func (d *Driver) Create() error { }, ) - exists, err := oci.ContainerExists(d.OCIBinary, params.Name) + exists, err := oci.ContainerExists(d.OCIBinary, params.Name, true) if err != nil { glog.Warningf("failed to check if container already exists: %v", err) } @@ -234,7 +234,7 @@ func (d *Driver) GetURL() (string, error) { // GetState returns the state that the host is in (running, stopped, etc) func (d *Driver) GetState() (state.State, error) { - return oci.ContainerStatus(d.OCIBinary, d.MachineName) + return oci.ContainerStatus(d.OCIBinary, d.MachineName, true) } // Kill stops a host forcefully, including any containers that we are managing. diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 99fcb6c20f..130095bacc 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -232,7 +232,7 @@ func ContainerID(ociBin string, nameOrID string) (string, error) { // ContainerExists checks if container name exists (either running or exited) func ContainerExists(ociBin string, name string, warnSlow ...bool) (bool, error) { - rr, err := runCmd(exec.Command(ociBin, "ps", "-a", "--format", "{{.Names}}"), true) + rr, err := runCmd(exec.Command(ociBin, "ps", "-a", "--format", "{{.Names}}"), warnSlow...) if err != nil { return false, err } @@ -395,7 +395,7 @@ func withPortMappings(portMappings []PortMapping) createOpt { // ListContainersByLabel returns all the container names with a specified label func ListContainersByLabel(ociBin string, label string, warnSlow ...bool) ([]string, error) { - rr, err := runCmd(exec.Command(ociBin, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}"), true) + rr, err := runCmd(exec.Command(ociBin, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}"), warnSlow...) if err != nil { return nil, err } @@ -432,7 +432,7 @@ func PointToHostDockerDaemon() error { // ContainerStatus returns status of a container running,exited,... func ContainerStatus(ociBin string, name string, warnSlow ...bool) (state.State, error) { cmd := exec.Command(ociBin, "inspect", name, "--format={{.State.Status}}") - rr, err := runCmd(cmd, true) + rr, err := runCmd(cmd, warnSlow...) o := strings.TrimSpace(rr.Stdout.String()) switch o { case "running": diff --git a/pkg/drivers/kic/oci/volumes.go b/pkg/drivers/kic/oci/volumes.go index a881dc6670..319d07c342 100644 --- a/pkg/drivers/kic/oci/volumes.go +++ b/pkg/drivers/kic/oci/volumes.go @@ -40,7 +40,7 @@ func DeleteAllVolumesByLabel(ociBin string, label string, warnSlow ...bool) []er } for _, v := range vs { - if _, err := runCmd(exec.Command(ociBin, "volume", "rm", "--force", v), true); err != nil { + if _, err := runCmd(exec.Command(ociBin, "volume", "rm", "--force", v), warnSlow...); err != nil { deleteErrs = append(deleteErrs, fmt.Errorf("deleting %q", v)) } } @@ -55,7 +55,7 @@ func PruneAllVolumesByLabel(ociBin string, label string, warnSlow ...bool) []err var deleteErrs []error glog.Infof("trying to prune all %s volumes with label %s", ociBin, label) cmd := exec.Command(ociBin, "volume", "prune", "-f", "--filter", "label="+label) - if _, err := runCmd(cmd, true); err != nil { + if _, err := runCmd(cmd, warnSlow...); err != nil { deleteErrs = append(deleteErrs, errors.Wrapf(err, "prune volume by label %s", label)) } diff --git a/pkg/minikube/machine/delete.go b/pkg/minikube/machine/delete.go index 8fc568e43a..4cd55d37d0 100644 --- a/pkg/minikube/machine/delete.go +++ b/pkg/minikube/machine/delete.go @@ -42,7 +42,7 @@ func deleteOrphanedKIC(ociBin string, name string) { _, err := oci.ContainerStatus(ociBin, name) if err != nil { - glog.Infof("couldn't inspect container %q before deleting, %s-daemon might needs a restart!: %v", name, ociBin, err) + glog.Infof("couldn't inspect container %q before deleting: %v", name, err) return } // allow no more than 5 seconds for delting the container From 450637941fa263eac6b0cca4289474639568cec2 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Fri, 24 Apr 2020 15:18:04 -0700 Subject: [PATCH 071/105] Clarify non-goals --- .../20200424-scheduled-shutdown.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/enhancements/proposed/20200424-scheduled-shutdown/20200424-scheduled-shutdown.md b/enhancements/proposed/20200424-scheduled-shutdown/20200424-scheduled-shutdown.md index 8bf6e9458b..a5890ee127 100644 --- a/enhancements/proposed/20200424-scheduled-shutdown/20200424-scheduled-shutdown.md +++ b/enhancements/proposed/20200424-scheduled-shutdown/20200424-scheduled-shutdown.md @@ -26,12 +26,14 @@ This is useful for two sets of users: ## Goals * The ability to schedule a pause or shutdown event -* The ability to defer or update the scheduled event +* The ability to defer the scheduled pause or shutdown event * "minikube start" transparently clears pending scheduled events ## Non-Goals -* Automatically idle detection. This is a related, but complimentary idea. +* Automatic idle detection: This is a more advanced, but complimentary idea, that would keep users from having to invoke a keep-alive command. It is possible to create using the same mechanisms, but is beyond the scope of this proposal. + +* Automatic re-activation: This is significantly more advanced, particularly for the scheduled shutdown case. It would be possible to create this, but it to is out of scope for this proposal. ## Design Details From 556e8871b7cd9d0f36ad03146ba5bc2683688188 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 24 Apr 2020 19:13:39 -0700 Subject: [PATCH 072/105] wait for service --- cmd/minikube/cmd/service.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/minikube/cmd/service.go b/cmd/minikube/cmd/service.go index fccc771f3a..7e3407b853 100644 --- a/cmd/minikube/cmd/service.go +++ b/cmd/minikube/cmd/service.go @@ -80,11 +80,6 @@ var serviceCmd = &cobra.Command{ cname := ClusterFlagValue() co := mustload.Healthy(cname) - if driver.NeedsPortForward(co.Config.Driver) { - startKicServiceTunnel(svc, cname) - return - } - urls, err := service.WaitForService(co.API, co.Config.Name, namespace, svc, serviceURLTemplate, serviceURLMode, https, wait, interval) if err != nil { var s *service.SVCNotFoundError @@ -95,6 +90,11 @@ You may select another namespace by using 'minikube service {{.service}} -n Date: Sat, 25 Apr 2020 06:54:02 +0100 Subject: [PATCH 073/105] Sort available style constants We have a number of style constant names defined in completly random order, let's make a little bit of cleanup here. --- pkg/minikube/out/style_enum.go | 130 ++++++++++++++++----------------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/pkg/minikube/out/style_enum.go b/pkg/minikube/out/style_enum.go index b8cca45448..a495952f16 100644 --- a/pkg/minikube/out/style_enum.go +++ b/pkg/minikube/out/style_enum.go @@ -21,76 +21,76 @@ type StyleEnum int // All the Style constants available const ( - Happy StyleEnum = iota - SuccessType - FailureType - Celebration - Conflict - FatalType - Notice - Ready - Running - Provisioning - Restarting - Stopping - Stopped - Warning - Waiting - WaitingPods - Usage - Launch - Sad - ThumbsUp - ThumbsDown - Option - Command - LogEntry - Deleted - URL - Documentation - Issues - Issue - Check - ISODownload - FileDownload + AddonDisable = iota + AddonEnable Caching - StartingVM - StartingNone - Provisioner - Resetting - DeletingHost - Copying - Connectivity - Confused - Internet - Mounting Celebrate - ContainerRuntime - Docker - CRIO + Celebration + Check + Command + Conflict + Confused + Connectivity Containerd - Permissions - Enabling - Shutdown - Pulling - HealthCheck - Verifying - VerifyingNoLine - Kubectl - Meh + ContainerRuntime + Copying + CRIO + Deleted + DeletingHost + Docker + Documentation + DryRun Embarrassed + Empty + Enabling + FailureType + FatalType + FileDownload + Fileserver + Happy StyleEnum + HealthCheck + Internet + ISODownload + Issue + Issues + Kubectl + Launch + LogEntry + Meh + Mounting + MountOptions + New + Notice + Option + Pause + Permissions + Provisioner + Provisioning + Pulling + Ready + Resetting + Restarting + Running + Sad + Shrug + Shutdown + Sparkle + StartingNone + StartingVM + Stopped + Stopping + SuccessType + ThumbsDown + ThumbsUp Tip Unmount - MountOptions - Fileserver - Empty - Workaround - Sparkle - Pause Unpause - DryRun - AddonEnable - AddonDisable - Shrug - New + URL + Usage + Verifying + VerifyingNoLine + Waiting + WaitingPods + Warning + Workaround ) From fe59e387481ba53d993f847f7a81d55540e9df69 Mon Sep 17 00:00:00 2001 From: Radoslaw Smigielski Date: Sat, 25 Apr 2020 07:15:29 +0100 Subject: [PATCH 074/105] Sort map of styles' names to style struct --- pkg/minikube/out/style.go | 118 ++++++++++++++++----------------- pkg/minikube/out/style_enum.go | 4 +- 2 files changed, 61 insertions(+), 61 deletions(-) diff --git a/pkg/minikube/out/style.go b/pkg/minikube/out/style.go index 12365b435a..c6ab21d8c0 100644 --- a/pkg/minikube/out/style.go +++ b/pkg/minikube/out/style.go @@ -49,79 +49,79 @@ type style struct { // styles is a map of style name to style struct // For consistency, ensure that emojis added render with the same width across platforms. var styles = map[StyleEnum]style{ - Empty: {Prefix: "", LowPrefix: ""}, - Happy: {Prefix: "😄 "}, - SuccessType: {Prefix: "✅ "}, - FailureType: {Prefix: "❌ "}, - Conflict: {Prefix: "💥 ", LowPrefix: lowWarning}, - FatalType: {Prefix: "💣 ", LowPrefix: lowError}, - Notice: {Prefix: "📌 "}, - Ready: {Prefix: "🏄 "}, - Running: {Prefix: "🏃 "}, - Provisioning: {Prefix: "🌱 "}, - Restarting: {Prefix: "🔄 "}, - Stopping: {Prefix: "✋ "}, - Stopped: {Prefix: "🛑 "}, - Warning: {Prefix: "❗ ", LowPrefix: lowWarning}, - Waiting: {Prefix: "⌛ "}, - Usage: {Prefix: "💡 "}, - Launch: {Prefix: "🚀 "}, - Sad: {Prefix: "😿 "}, - ThumbsUp: {Prefix: "👍 "}, - ThumbsDown: {Prefix: "👎 "}, - Option: {Prefix: " ▪ ", LowPrefix: lowIndent}, // Indented bullet - Command: {Prefix: " ▪ ", LowPrefix: lowIndent}, // Indented bullet - LogEntry: {Prefix: " "}, // Indent - Deleted: {Prefix: "💀 "}, - URL: {Prefix: "👉 ", LowPrefix: lowIndent}, - Documentation: {Prefix: "📘 "}, - Issues: {Prefix: "⁉️ "}, - Issue: {Prefix: " ▪ ", LowPrefix: lowIndent}, // Indented bullet - Check: {Prefix: "✅ "}, Celebration: {Prefix: "🎉 "}, - Workaround: {Prefix: "👉 ", LowPrefix: lowIndent}, - Sparkle: {Prefix: "✨ "}, - Pause: {Prefix: "⏸️ "}, - Unpause: {Prefix: "⏯️ "}, + Check: {Prefix: "✅ "}, + Command: {Prefix: " ▪ ", LowPrefix: lowIndent}, // Indented bullet + Conflict: {Prefix: "💥 ", LowPrefix: lowWarning}, Confused: {Prefix: "😕 "}, - Shrug: {Prefix: "🤷 "}, + Deleted: {Prefix: "💀 "}, + Documentation: {Prefix: "📘 "}, + Empty: {Prefix: "", LowPrefix: ""}, + FailureType: {Prefix: "❌ "}, + FatalType: {Prefix: "💣 ", LowPrefix: lowError}, + Happy: {Prefix: "😄 "}, + Issue: {Prefix: " ▪ ", LowPrefix: lowIndent}, // Indented bullet + Issues: {Prefix: "⁉️ "}, + Launch: {Prefix: "🚀 "}, + LogEntry: {Prefix: " "}, // Indent New: {Prefix: "🆕 "}, + Notice: {Prefix: "📌 "}, + Option: {Prefix: " ▪ ", LowPrefix: lowIndent}, // Indented bullet + Pause: {Prefix: "⏸️ "}, + Provisioning: {Prefix: "🌱 "}, + Ready: {Prefix: "🏄 "}, + Restarting: {Prefix: "🔄 "}, + Running: {Prefix: "🏃 "}, + Sad: {Prefix: "😿 "}, + Shrug: {Prefix: "🤷 "}, + Sparkle: {Prefix: "✨ "}, + Stopped: {Prefix: "🛑 "}, + Stopping: {Prefix: "✋ "}, + SuccessType: {Prefix: "✅ "}, + ThumbsDown: {Prefix: "👎 "}, + ThumbsUp: {Prefix: "👍 "}, + Unpause: {Prefix: "⏯️ "}, + URL: {Prefix: "👉 ", LowPrefix: lowIndent}, + Usage: {Prefix: "💡 "}, + Waiting: {Prefix: "⌛ "}, + Warning: {Prefix: "❗ ", LowPrefix: lowWarning}, + Workaround: {Prefix: "👉 ", LowPrefix: lowIndent}, // Specialized purpose styles - ISODownload: {Prefix: "💿 "}, - FileDownload: {Prefix: "💾 "}, + AddonDisable: {Prefix: "🌑 "}, + AddonEnable: {Prefix: "🌟 "}, Caching: {Prefix: "🤹 "}, - StartingVM: {Prefix: "🔥 "}, - StartingNone: {Prefix: "🤹 "}, - Provisioner: {Prefix: "ℹ️ "}, - Resetting: {Prefix: "🔄 "}, - DeletingHost: {Prefix: "🔥 "}, - Copying: {Prefix: "✨ "}, - Connectivity: {Prefix: "📶 "}, - Internet: {Prefix: "🌐 "}, - Mounting: {Prefix: "📁 "}, Celebrate: {Prefix: "🎉 "}, - ContainerRuntime: {Prefix: "🎁 "}, - Docker: {Prefix: "🐳 "}, - CRIO: {Prefix: "🎁 "}, // This should be a snow-flake, but the emoji has a strange width on macOS + Connectivity: {Prefix: "📶 "}, Containerd: {Prefix: "📦 "}, - Permissions: {Prefix: "🔑 "}, + ContainerRuntime: {Prefix: "🎁 "}, + Copying: {Prefix: "✨ "}, + CRIO: {Prefix: "🎁 "}, // This should be a snow-flake, but the emoji has a strange width on macOS + DeletingHost: {Prefix: "🔥 "}, + Docker: {Prefix: "🐳 "}, + DryRun: {Prefix: "🌵 "}, + Embarrassed: {Prefix: "🤦 ", LowPrefix: lowWarning}, Enabling: {Prefix: "🔌 "}, - Shutdown: {Prefix: "🛑 "}, - Pulling: {Prefix: "🚜 "}, + FileDownload: {Prefix: "💾 "}, + Fileserver: {Prefix: "🚀 ", OmitNewline: true}, HealthCheck: {Prefix: "🔎 "}, - Verifying: {Prefix: "🤔 "}, - VerifyingNoLine: {Prefix: "🤔 ", OmitNewline: true}, + Internet: {Prefix: "🌐 "}, + ISODownload: {Prefix: "💿 "}, Kubectl: {Prefix: "💗 "}, Meh: {Prefix: "🙄 ", LowPrefix: lowWarning}, - Embarrassed: {Prefix: "🤦 ", LowPrefix: lowWarning}, + Mounting: {Prefix: "📁 "}, + MountOptions: {Prefix: "💾 "}, + Permissions: {Prefix: "🔑 "}, + Provisioner: {Prefix: "ℹ️ "}, + Pulling: {Prefix: "🚜 "}, + Resetting: {Prefix: "🔄 "}, + Shutdown: {Prefix: "🛑 "}, + StartingNone: {Prefix: "🤹 "}, + StartingVM: {Prefix: "🔥 "}, Tip: {Prefix: "💡 "}, Unmount: {Prefix: "🔥 "}, - MountOptions: {Prefix: "💾 "}, - Fileserver: {Prefix: "🚀 ", OmitNewline: true}, - DryRun: {Prefix: "🌵 "}, - AddonEnable: {Prefix: "🌟 "}, - AddonDisable: {Prefix: "🌑 "}, + VerifyingNoLine: {Prefix: "🤔 ", OmitNewline: true}, + Verifying: {Prefix: "🤔 "}, } // Add a prefix to a string diff --git a/pkg/minikube/out/style_enum.go b/pkg/minikube/out/style_enum.go index a495952f16..9fe6362e2d 100644 --- a/pkg/minikube/out/style_enum.go +++ b/pkg/minikube/out/style_enum.go @@ -21,7 +21,7 @@ type StyleEnum int // All the Style constants available const ( - AddonDisable = iota + AddonDisable StyleEnum = iota AddonEnable Caching Celebrate @@ -47,7 +47,7 @@ const ( FatalType FileDownload Fileserver - Happy StyleEnum + Happy HealthCheck Internet ISODownload From d42a0a145cc0661a89b40dc095b66a2ecd224ee9 Mon Sep 17 00:00:00 2001 From: Radoslaw Smigielski Date: Sat, 25 Apr 2020 07:32:40 +0100 Subject: [PATCH 075/105] Fix lint error --- pkg/minikube/out/style.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/out/style.go b/pkg/minikube/out/style.go index c6ab21d8c0..2d2a64762b 100644 --- a/pkg/minikube/out/style.go +++ b/pkg/minikube/out/style.go @@ -63,7 +63,7 @@ var styles = map[StyleEnum]style{ Issue: {Prefix: " ▪ ", LowPrefix: lowIndent}, // Indented bullet Issues: {Prefix: "⁉️ "}, Launch: {Prefix: "🚀 "}, - LogEntry: {Prefix: " "}, // Indent + LogEntry: {Prefix: " "}, // Indent New: {Prefix: "🆕 "}, Notice: {Prefix: "📌 "}, Option: {Prefix: " ▪ ", LowPrefix: lowIndent}, // Indented bullet From bb37c7098ca9d3f047e4f967d4259e6bcb581b54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sat, 25 Apr 2020 21:56:40 +0200 Subject: [PATCH 076/105] Fix log message to say driver and not cruntime --- pkg/minikube/node/cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go index 282ceeb1d6..1b032f320d 100644 --- a/pkg/minikube/node/cache.go +++ b/pkg/minikube/node/cache.go @@ -112,7 +112,7 @@ func beginDownloadKicArtifacts(g *errgroup.Group, driver string, cRuntime string } } else { // TODO: driver == "podman" - glog.Info("Container runtime isn't docker, skipping download") + glog.Info("Driver isn't docker, skipping base-image download") } } From 7647b1f998b36401a396acceddd7c147fb736f19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sat, 25 Apr 2020 22:01:51 +0200 Subject: [PATCH 077/105] Don't try to limit podman memory without cgroup Podman requires both memcg and memcg_swap for it (--memory). Docker will still limit memory, without swap limit support. --- pkg/drivers/kic/oci/oci.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 64da623425..79bb2e9976 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -31,6 +31,7 @@ import ( "fmt" "os/exec" + "runtime" "strconv" "strings" ) @@ -139,7 +140,23 @@ func CreateContainerNode(p CreateParams) error { runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name)) } - runArgs = append(runArgs, fmt.Sprintf("--cpus=%s", p.CPUs), fmt.Sprintf("--memory=%s", p.Memory)) + runArgs = append(runArgs, fmt.Sprintf("--cpus=%s", p.CPUs)) + + memcgSwap := true + if runtime.GOOS == "linux" { + if _, err := os.Stat("/sys/fs/cgroup/memory/memsw.limit_in_bytes"); os.IsNotExist(err) { + // requires CONFIG_MEMCG_SWAP_ENABLED or cgroup_enable=memory in grub + glog.Warning("Your kernel does not support swap limit capabilities or the cgroup is not mounted.") + memcgSwap = false + } + } + + if p.OCIBinary == Podman && memcgSwap { // swap is required for memory + runArgs = append(runArgs, fmt.Sprintf("--memory=%s", p.Memory)) + } + if p.OCIBinary == Docker { // swap is only required for --memory-swap + runArgs = append(runArgs, fmt.Sprintf("--memory=%s", p.Memory)) + } for key, val := range p.Envs { runArgs = append(runArgs, "-e", fmt.Sprintf("%s=%s", key, val)) From 7185140c57c4212a9d61747d5bf4e2f9f377092d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sat, 25 Apr 2020 22:09:52 +0200 Subject: [PATCH 078/105] Remove extra quotes added in conflict resolution Reverted f264ac171e7b12334c550067202839e9e739f907 by accident --- pkg/drivers/kic/oci/info.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/drivers/kic/oci/info.go b/pkg/drivers/kic/oci/info.go index 633c154804..41dd09ef56 100644 --- a/pkg/drivers/kic/oci/info.go +++ b/pkg/drivers/kic/oci/info.go @@ -231,7 +231,7 @@ func dockerSystemInfo() (dockerSysInfo, error) { // podmanSysInfo returns podman system info --format '{{json .}}' func podmanSystemInfo() (podmanSysInfo, error) { var ps podmanSysInfo - rr, err := runCmd(exec.Command("sudo", Podman, "system", "info", "--format", "'{{json .}}'")) + rr, err := runCmd(exec.Command("sudo", Podman, "system", "info", "--format", "{{json .}}")) if err != nil { return ps, errors.Wrap(err, "get podman system info") } From 88c8a24d55501670613e9c472714122246c59ce8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sat, 25 Apr 2020 22:43:40 +0200 Subject: [PATCH 079/105] Pass the container implementation on to systemd --- pkg/drivers/kic/oci/oci.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 79bb2e9976..7ff83d4737 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -158,6 +158,16 @@ func CreateContainerNode(p CreateParams) error { runArgs = append(runArgs, fmt.Sprintf("--memory=%s", p.Memory)) } + // https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/ + var virtualization string + if p.OCIBinary == Podman { + virtualization = "podman" // VIRTUALIZATION_PODMAN + } + if p.OCIBinary == Docker { + virtualization = "docker" // VIRTUALIZATION_DOCKER + } + runArgs = append(runArgs, "-e", fmt.Sprintf("%s=%s", "container", virtualization)) + for key, val := range p.Envs { runArgs = append(runArgs, "-e", fmt.Sprintf("%s=%s", key, val)) } From d96d9d34ed4df2e375a2389003a733cb66803029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 26 Apr 2020 15:12:04 +0200 Subject: [PATCH 080/105] Include any output from podman version in the log --- pkg/minikube/registry/drvs/podman/podman.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/minikube/registry/drvs/podman/podman.go b/pkg/minikube/registry/drvs/podman/podman.go index 8f72a4deb0..6191c8b46e 100644 --- a/pkg/minikube/registry/drvs/podman/podman.go +++ b/pkg/minikube/registry/drvs/podman/podman.go @@ -82,6 +82,7 @@ func status() registry.State { o, err := cmd.CombinedOutput() output := string(o) if err != nil { + glog.Warningf("podman version returned %s", output) return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Cant verify mininim required version for podman . See podman website for installation guide.", Doc: "https://podman.io/getting-started/installation.html"} } From 28106fa2d71449a85506e7d8b28fa857bfaa078a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 26 Apr 2020 15:22:37 +0200 Subject: [PATCH 081/105] Use constants instead of the KIC prefix strings --- cmd/minikube/cmd/delete.go | 8 ++++---- hack/preload-images/generate.go | 2 +- pkg/drivers/kic/oci/info.go | 2 +- pkg/drivers/kic/oci/network.go | 12 ++++++------ pkg/drivers/kic/oci/types.go | 4 ++++ pkg/minikube/command/kic_runner.go | 2 +- pkg/minikube/config/profile.go | 2 +- pkg/minikube/machine/delete.go | 4 ++-- pkg/minikube/registry/drvs/docker/docker.go | 4 ++-- pkg/minikube/registry/drvs/podman/podman.go | 6 +++--- 10 files changed, 25 insertions(+), 21 deletions(-) diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go index 052d393f57..df9aa79301 100644 --- a/cmd/minikube/cmd/delete.go +++ b/cmd/minikube/cmd/delete.go @@ -90,17 +90,17 @@ func init() { func deleteContainersAndVolumes() { delLabel := fmt.Sprintf("%s=%s", oci.CreatedByLabelKey, "true") - errs := oci.DeleteContainersByLabel("env", oci.Docker, delLabel) + errs := oci.DeleteContainersByLabel(oci.Env, oci.Docker, delLabel) if len(errs) > 0 { // it will error if there is no container to delete glog.Infof("error delete containers by label %q (might be okay): %+v", delLabel, errs) } - errs = oci.DeleteAllVolumesByLabel("env", oci.Docker, delLabel) + errs = oci.DeleteAllVolumesByLabel(oci.Env, oci.Docker, delLabel) if len(errs) > 0 { // it will not error if there is nothing to delete glog.Warningf("error delete volumes by label %q (might be okay): %+v", delLabel, errs) } - errs = oci.PruneAllVolumesByLabel("env", oci.Docker, delLabel) + errs = oci.PruneAllVolumesByLabel(oci.Env, oci.Docker, delLabel) if len(errs) > 0 { // it will not error if there is nothing to delete glog.Warningf("error pruning volumes by label %q (might be okay): %+v", delLabel, errs) } @@ -193,7 +193,7 @@ func DeleteProfiles(profiles []*config.Profile) []error { func deletePossibleKicLeftOver(name string) { delLabel := fmt.Sprintf("%s=%s", oci.ProfileLabelKey, name) - prefixes := []string{"env", "sudo"} + prefixes := []string{oci.Env, oci.Sudo} for i, bin := range []string{oci.Docker, oci.Podman} { prefix := prefixes[i] cs, err := oci.ListContainersByLabel(prefix, bin, delLabel) diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go index 52b9c90af6..054f5c5c60 100644 --- a/hack/preload-images/generate.go +++ b/hack/preload-images/generate.go @@ -41,7 +41,7 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string driver := kic.NewDriver(kic.Config{ KubernetesVersion: kubernetesVersion, ContainerRuntime: containerRuntime, - OCIPrefix: "env", + OCIPrefix: oci.Env, OCIBinary: oci.Docker, MachineName: profile, ImageDigest: kic.BaseImage, diff --git a/pkg/drivers/kic/oci/info.go b/pkg/drivers/kic/oci/info.go index 41dd09ef56..e5835eea6c 100644 --- a/pkg/drivers/kic/oci/info.go +++ b/pkg/drivers/kic/oci/info.go @@ -231,7 +231,7 @@ func dockerSystemInfo() (dockerSysInfo, error) { // podmanSysInfo returns podman system info --format '{{json .}}' func podmanSystemInfo() (podmanSysInfo, error) { var ps podmanSysInfo - rr, err := runCmd(exec.Command("sudo", Podman, "system", "info", "--format", "{{json .}}")) + rr, err := runCmd(exec.Command(Sudo, Podman, "system", "info", "--format", "{{json .}}")) if err != nil { return ps, errors.Wrap(err, "get podman system info") } diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index 5dc39b2c08..1bb5cdeaaf 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -56,13 +56,13 @@ func digDNS(ociBin, containerName, dns string) (net.IP, error) { // dockerGatewayIP gets the default gateway ip for the docker bridge on the user's host machine // gets the ip from user's host docker func dockerGatewayIP() (net.IP, error) { - rr, err := runCmd(exec.Command(Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}")) + rr, err := runCmd(exec.Command(Env, Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}")) if err != nil { return nil, errors.Wrapf(err, "get network bridge") } bridgeID := strings.TrimSpace(rr.Stdout.String()) - rr, err = runCmd(exec.Command("env", Docker, "inspect", + rr, err = runCmd(exec.Command(Env, Docker, "inspect", "--format", "{{(index .IPAM.Config 0).Gateway}}", bridgeID)) if err != nil { return nil, errors.Wrapf(err, "inspect IP bridge network %q.", bridgeID) @@ -84,12 +84,12 @@ func ForwardedPort(ociBin string, ociID string, contPort int) (int, error) { if ociBin == Podman { //podman inspect -f "{{range .NetworkSettings.Ports}}{{if eq .ContainerPort "80"}}{{.HostPort}}{{end}}{{end}}" - rr, err = runCmd(exec.Command("sudo", ociBin, "inspect", "-f", fmt.Sprintf("{{range .NetworkSettings.Ports}}{{if eq .ContainerPort %s}}{{.HostPort}}{{end}}{{end}}", fmt.Sprint(contPort)), ociID)) + rr, err = runCmd(exec.Command(Sudo, ociBin, "inspect", "-f", fmt.Sprintf("{{range .NetworkSettings.Ports}}{{if eq .ContainerPort %s}}{{.HostPort}}{{end}}{{end}}", fmt.Sprint(contPort)), ociID)) if err != nil { return 0, errors.Wrapf(err, "get port %d for %q", contPort, ociID) } } else { - rr, err = runCmd(exec.Command("env", ociBin, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)) + rr, err = runCmd(exec.Command(Env, ociBin, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)) if err != nil { return 0, errors.Wrapf(err, "get port %d for %q", contPort, ociID) } @@ -116,7 +116,7 @@ func ContainerIPs(ociBin string, name string) (string, string, error) { // podmanConttainerIP returns ipv4, ipv6 of container or error func podmanConttainerIP(name string) (string, string, error) { - rr, err := runCmd(exec.Command("sudo", Podman, "inspect", + rr, err := runCmd(exec.Command(Sudo, Podman, "inspect", "-f", "{{.NetworkSettings.IPAddress}}", name)) if err != nil { @@ -132,7 +132,7 @@ func podmanConttainerIP(name string) (string, string, error) { // dockerContainerIP returns ipv4, ipv6 of container or error func dockerContainerIP(name string) (string, string, error) { // retrieve the IP address of the node using docker inspect - lines, err := inspect("env", Docker, name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}") + lines, err := inspect(Env, Docker, name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}") if err != nil { return "", "", errors.Wrap(err, "inspecting NetworkSettings.Networks") } diff --git a/pkg/drivers/kic/oci/types.go b/pkg/drivers/kic/oci/types.go index 1a13e1708e..45c38522b9 100644 --- a/pkg/drivers/kic/oci/types.go +++ b/pkg/drivers/kic/oci/types.go @@ -19,6 +19,10 @@ package oci const ( // DefaultBindIPV4 is The default IP the container will listen on. DefaultBindIPV4 = "127.0.0.1" + // Env is env + Env = "env" + // Sudo is sudo + Sudo = "sudo" // Docker is docker Docker = "docker" // Podman is podman diff --git a/pkg/minikube/command/kic_runner.go b/pkg/minikube/command/kic_runner.go index 4b5a223371..98c66e5426 100644 --- a/pkg/minikube/command/kic_runner.go +++ b/pkg/minikube/command/kic_runner.go @@ -202,7 +202,7 @@ func (k *kicRunner) chmod(dst string, perm string) error { // Podman cp command doesn't match docker and doesn't have -a func copyToPodman(src string, dest string) error { - if out, err := exec.Command("sudo", oci.Podman, "cp", src, dest).CombinedOutput(); err != nil { + if out, err := exec.Command(oci.Sudo, oci.Podman, "cp", src, dest).CombinedOutput(); err != nil { return errors.Wrapf(err, "podman copy %s into %s, output: %s", src, dest, string(out)) } return nil diff --git a/pkg/minikube/config/profile.go b/pkg/minikube/config/profile.go index 404800b670..788d98afea 100644 --- a/pkg/minikube/config/profile.go +++ b/pkg/minikube/config/profile.go @@ -203,7 +203,7 @@ func ListProfiles(miniHome ...string) (validPs []*Profile, inValidPs []*Profile, return nil, nil, err } // try to get profiles list based on all contrainers created by docker driver - cs, err := oci.ListOwnedContainers("env", oci.Docker) + cs, err := oci.ListOwnedContainers(oci.Env, oci.Docker) if err == nil { pDirs = append(pDirs, cs...) } diff --git a/pkg/minikube/machine/delete.go b/pkg/minikube/machine/delete.go index f8c4b44ec4..134c6a74eb 100644 --- a/pkg/minikube/machine/delete.go +++ b/pkg/minikube/machine/delete.go @@ -63,8 +63,8 @@ func deleteOrphanedKIC(prefix string, ociBin string, name string) { func DeleteHost(api libmachine.API, machineName string) error { host, err := api.Load(machineName) if err != nil && host == nil { - deleteOrphanedKIC("env", oci.Docker, machineName) - deleteOrphanedKIC("sudo", oci.Podman, machineName) + deleteOrphanedKIC(oci.Env, oci.Docker, machineName) + deleteOrphanedKIC(oci.Sudo, oci.Podman, machineName) // Keep going even if minikube does not know about the host } diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index ab5439a044..13a4102e69 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -48,7 +48,7 @@ func init() { if err := registry.Register(registry.DriverDef{ Name: driver.Docker, Config: configure, - Init: func() drivers.Driver { return kic.NewDriver(kic.Config{OCIPrefix: "env", OCIBinary: oci.Docker}) }, + Init: func() drivers.Driver { return kic.NewDriver(kic.Config{OCIPrefix: oci.Env, OCIBinary: oci.Docker}) }, Status: status, Priority: priority, }); err != nil { @@ -63,7 +63,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { ImageDigest: viper.GetString("base-image"), CPU: cc.CPUs, Memory: cc.Memory, - OCIPrefix: "env", + OCIPrefix: oci.Env, OCIBinary: oci.Docker, APIServerPort: cc.Nodes[0].Port, KubernetesVersion: cc.KubernetesConfig.KubernetesVersion, diff --git a/pkg/minikube/registry/drvs/podman/podman.go b/pkg/minikube/registry/drvs/podman/podman.go index 6191c8b46e..79815c0d2e 100644 --- a/pkg/minikube/registry/drvs/podman/podman.go +++ b/pkg/minikube/registry/drvs/podman/podman.go @@ -45,7 +45,7 @@ func init() { if err := registry.Register(registry.DriverDef{ Name: driver.Podman, Config: configure, - Init: func() drivers.Driver { return kic.NewDriver(kic.Config{OCIPrefix: "sudo", OCIBinary: oci.Podman}) }, + Init: func() drivers.Driver { return kic.NewDriver(kic.Config{OCIPrefix: oci.Sudo, OCIBinary: oci.Podman}) }, Status: status, Priority: registry.Experimental, }); err != nil { @@ -61,7 +61,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { ImageDigest: strings.Split(baseImage, "@")[0], // for podman does not support docker images references with both a tag and digest. CPU: cc.CPUs, Memory: cc.Memory, - OCIPrefix: "sudo", + OCIPrefix: oci.Sudo, OCIBinary: oci.Podman, APIServerPort: cc.Nodes[0].Port, }), nil @@ -101,7 +101,7 @@ func status() registry.State { // Run with sudo on linux (local), otherwise podman-remote (as podman) if runtime.GOOS == "linux" { - cmd = exec.CommandContext(ctx, "sudo", "-n", oci.Podman, "info") + cmd = exec.CommandContext(ctx, oci.Sudo, "-n", oci.Podman, "info") cmd.Env = append(os.Environ(), "LANG=C", "LC_ALL=C") // sudo is localized } else { cmd = exec.CommandContext(ctx, oci.Podman, "info") From e95ae3280c2130d17993a65cbefe46b4ddf4c156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 26 Apr 2020 22:45:38 +0200 Subject: [PATCH 082/105] Fix copy/paste error on the podman usage page --- site/content/en/docs/drivers/includes/podman_usage.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/content/en/docs/drivers/includes/podman_usage.inc b/site/content/en/docs/drivers/includes/podman_usage.inc index 186c5bb604..5472e8faa4 100644 --- a/site/content/en/docs/drivers/includes/podman_usage.inc +++ b/site/content/en/docs/drivers/includes/podman_usage.inc @@ -9,7 +9,7 @@ for a better kubernetes in container experience, use docker [driver](https://min ## Usage -Start a cluster using the docker driver: +Start a cluster using the podman driver: ```shell minikube start --driver=podman From 795128131f17e3115d148879ad7f235f97ac680e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 26 Apr 2020 22:52:10 +0200 Subject: [PATCH 083/105] Remove old Copy function added in merge conflict --- pkg/drivers/kic/oci/oci.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 7ff83d4737..633d8912d3 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -245,20 +245,6 @@ func createContainer(prefix string, ociBin string, image string, opts ...createO return nil } -// Copy copies a local asset into the container -func Copy(prefix string, ociBin string, ociID string, targetDir string, fName string) error { - if _, err := os.Stat(fName); os.IsNotExist(err) { - return errors.Wrapf(err, "error source %s does not exist", fName) - } - - destination := fmt.Sprintf("%s:%s", ociID, targetDir) - if _, err := runCmd(exec.Command(prefix, ociBin, "cp", fName, destination)); err != nil { - return err - } - - return nil -} - // ContainerID returns id of a container name func ContainerID(prefix, ociBin string, nameOrID string) (string, error) { rr, err := runCmd(exec.Command(prefix, ociBin, "inspect", "-f", "{{.Id}}", nameOrID)) From 998ab840fbb139b0f9bdda71aa509b19554fab15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 26 Apr 2020 22:54:45 +0200 Subject: [PATCH 084/105] Use simple version of podman info format parameter So that it works with the older versions of podman --- pkg/drivers/kic/oci/info.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/drivers/kic/oci/info.go b/pkg/drivers/kic/oci/info.go index e5835eea6c..45752f1959 100644 --- a/pkg/drivers/kic/oci/info.go +++ b/pkg/drivers/kic/oci/info.go @@ -231,7 +231,7 @@ func dockerSystemInfo() (dockerSysInfo, error) { // podmanSysInfo returns podman system info --format '{{json .}}' func podmanSystemInfo() (podmanSysInfo, error) { var ps podmanSysInfo - rr, err := runCmd(exec.Command(Sudo, Podman, "system", "info", "--format", "{{json .}}")) + rr, err := runCmd(exec.Command(Sudo, Podman, "system", "info", "--format", "json")) if err != nil { return ps, errors.Wrap(err, "get podman system info") } From 95c95599c2b266d23ab6d3487e53195b73f4b8fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 27 Apr 2020 12:41:16 +0200 Subject: [PATCH 085/105] Align podman driver with docker and improve check Make sure to use "sudo podman version" on Linux (need user namespace support for "podman version") And improve error output handling for podman-remote (when the remote service is not running properly) --- pkg/minikube/registry/drvs/podman/podman.go | 91 +++++++++++---------- 1 file changed, 50 insertions(+), 41 deletions(-) diff --git a/pkg/minikube/registry/drvs/podman/podman.go b/pkg/minikube/registry/drvs/podman/podman.go index 79815c0d2e..dd54e998d8 100644 --- a/pkg/minikube/registry/drvs/podman/podman.go +++ b/pkg/minikube/registry/drvs/podman/podman.go @@ -42,12 +42,18 @@ import ( var minReqPodmanVer = semver.Version{Major: 1, Minor: 7, Patch: 0} func init() { + priority := registry.Experimental + // Staged rollout for default: + // - Linux + // - macOS (podman-remote) + // - Windows (podman-remote) + if err := registry.Register(registry.DriverDef{ Name: driver.Podman, Config: configure, Init: func() drivers.Driver { return kic.NewDriver(kic.Config{OCIPrefix: oci.Sudo, OCIBinary: oci.Podman}) }, Status: status, - Priority: registry.Experimental, + Priority: priority, }); err != nil { panic(fmt.Sprintf("register failed: %v", err)) } @@ -56,14 +62,16 @@ func init() { func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { baseImage := viper.GetString("base-image") return kic.NewDriver(kic.Config{ - MachineName: driver.MachineName(cc, n), - StorePath: localpath.MiniPath(), - ImageDigest: strings.Split(baseImage, "@")[0], // for podman does not support docker images references with both a tag and digest. - CPU: cc.CPUs, - Memory: cc.Memory, - OCIPrefix: oci.Sudo, - OCIBinary: oci.Podman, - APIServerPort: cc.Nodes[0].Port, + MachineName: driver.MachineName(cc, n), + StorePath: localpath.MiniPath(), + ImageDigest: strings.Split(baseImage, "@")[0], // for podman does not support docker images references with both a tag and digest. + CPU: cc.CPUs, + Memory: cc.Memory, + OCIPrefix: oci.Sudo, + OCIBinary: oci.Podman, + APIServerPort: cc.Nodes[0].Port, + KubernetesVersion: cc.KubernetesConfig.KubernetesVersion, + ContainerRuntime: cc.KubernetesConfig.ContainerRuntime, }), nil } @@ -74,58 +82,59 @@ func status() registry.State { return registry.State{Error: err, Installed: false, Healthy: false, Fix: "Install Podman", Doc: docURL} } - // Allow no more than 2 seconds for version command - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - - cmd := exec.CommandContext(ctx, oci.Podman, "version", "-f", "{{.Version}}") - o, err := cmd.CombinedOutput() - output := string(o) - if err != nil { - glog.Warningf("podman version returned %s", output) - return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Cant verify mininim required version for podman . See podman website for installation guide.", Doc: "https://podman.io/getting-started/installation.html"} - } - - v, err := semver.Make(output) - if err != nil { - return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Cant verify mininim required version for podman . See podman website for installation guide.", Doc: "https://podman.io/getting-started/installation.html"} - } - - if v.LT(minReqPodmanVer) { - glog.Warningf("Warning ! mininim required version for podman is %s. your version is %q. minikube might not work. use at your own risk. To install latest version please see https://podman.io/getting-started/installation.html ", minReqPodmanVer.String(), v.String()) - } - - // Allow no more than 3 seconds for querying state - ctx, cancel = context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) defer cancel() + // Quickly returns an error code if service is not running + cmd := exec.CommandContext(ctx, oci.Podman, "version", "--format", "{{.Server.Version}}") // Run with sudo on linux (local), otherwise podman-remote (as podman) if runtime.GOOS == "linux" { - cmd = exec.CommandContext(ctx, oci.Sudo, "-n", oci.Podman, "info") + cmd = exec.CommandContext(ctx, oci.Sudo, "-n", oci.Podman, "version", "--format", "{{.Version}}") cmd.Env = append(os.Environ(), "LANG=C", "LC_ALL=C") // sudo is localized - } else { - cmd = exec.CommandContext(ctx, oci.Podman, "info") } - _, err = cmd.Output() + o, err := cmd.Output() + output := string(o) if err == nil { + v, err := semver.Make(output) + if err != nil { + return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Cant verify minimum required version for podman . See podman website for installation guide.", Doc: "https://podman.io/getting-started/installation.html"} + } + + if v.LT(minReqPodmanVer) { + glog.Warningf("Warning ! minimum required version for podman is %s. your version is %q. minikube might not work. use at your own risk. To install latest version please see https://podman.io/getting-started/installation.html ", minReqPodmanVer.String(), v.String()) + } + return registry.State{Installed: true, Healthy: true} } glog.Warningf("podman returned error: %v", err) + // Basic timeout + if ctx.Err() == context.DeadlineExceeded { + return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Restart the Podman service", Doc: docURL} + } + + username := "$USER" + if u, err := user.Current(); err == nil { + username = u.Username + } + if exitErr, ok := err.(*exec.ExitError); ok { stderr := strings.TrimSpace(string(exitErr.Stderr)) newErr := fmt.Errorf(`%q %v: %s`, strings.Join(cmd.Args, " "), exitErr, stderr) - username := "$USER" - if u, err := user.Current(); err == nil { - username = u.Username - } - if strings.Contains(stderr, "a password is required") && runtime.GOOS == "linux" { return registry.State{Error: newErr, Installed: true, Healthy: false, Fix: fmt.Sprintf("Add your user to the 'sudoers' file: '%s ALL=(ALL) NOPASSWD: %s'", username, podman), Doc: "https://podman.io"} } + // Typical low-level errors from running podman-remote: + // - local: "dial unix /run/podman/io.podman: connect: no such file or directory" + // - remote: "unexpected EOF" (ssh varlink isn't so great at handling rejections) + + if strings.Contains(stderr, "could not get runtime") || strings.Contains(stderr, "Unable to obtain server version information") { + return registry.State{Error: newErr, Installed: true, Healthy: false, Fix: "Start the Podman service", Doc: docURL} + } + // We don't have good advice, but at least we can provide a good error message return registry.State{Error: newErr, Installed: true, Healthy: false, Doc: docURL} } From a041f4012ae96150b2b0a2e940df4689aaf88fd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 27 Apr 2020 17:02:13 +0200 Subject: [PATCH 086/105] Podman allows setting --cpus and --memory now --- pkg/minikube/driver/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index 1990f39733..d4dd5e2762 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -137,7 +137,7 @@ func NeedsPortForward(name string) bool { // HasResourceLimits returns true if driver can set resource limits such as memory size or CPU count. func HasResourceLimits(name string) bool { - return !(name == None || name == Podman) + return name != None } // NeedsShutdown returns true if driver needs manual shutdown command before stopping. From 90b029e359d874802ce700745629f9b7a890b84d Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 27 Apr 2020 10:45:34 -0700 Subject: [PATCH 087/105] Add varlink for podman-env ' --- hack/images/kicbase.Dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile index 86224d676e..35be4fde1d 100644 --- a/hack/images/kicbase.Dockerfile +++ b/hack/images/kicbase.Dockerfile @@ -17,15 +17,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm /etc/crictl.yaml # install cri-o based on https://github.com/cri-o/cri-o/commit/96b0c34b31a9fc181e46d7d8e34fb8ee6c4dc4e1#diff-04c6e90faac2675aa89e2176d2eec7d8R128 -ARG CRIO_VERSION="1.17=1.17.3~2" RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \ curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \ apt-key add - < Release.key && apt-get update && \ - apt-get install -y --no-install-recommends cri-o-${CRIO_VERSION} + apt-get install -y --no-install-recommends cri-o-1.17 # install podman -ARG PODMAN_VERSION=1.9.0~2 -RUN apt-get install -y --no-install-recommends podman=${PODMAN_VERSION} +RUN apt-get install -y --no-install-recommends podman + +# install varlink +RUN apt-get install -y --no-install-recommends varlink + # disable non-docker runtimes by default RUN systemctl disable containerd && systemctl disable crio && rm /etc/crictl.yaml From 2459caea75433ef0abfa13d5a8e2eafef21463d3 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 27 Apr 2020 10:47:10 -0700 Subject: [PATCH 088/105] update SHA --- pkg/drivers/kic/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go index f441f0f8b6..020ae1ed8e 100644 --- a/pkg/drivers/kic/types.go +++ b/pkg/drivers/kic/types.go @@ -32,7 +32,7 @@ const ( // Version is the current version of kic Version = "v0.0.10" // SHA of the kic base image - baseImageSHA = "0c3666c7dacb3c7170a86b237d0d15d37afb5a6d2f0c45046546b5c09f40378c" + baseImageSHA = "f58e0c4662bac8a9b5dda7984b185bad8502ade5d9fa364bf2755d636ab51438" // OverlayImage is the cni plugin used for overlay image, created by kind. // CNI plugin image used for kic drivers created by kind. OverlayImage = "kindest/kindnetd:0.5.4" From 67efd1120e1a36f840b545d987f299d38f3a7535 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 27 Apr 2020 10:47:43 -0700 Subject: [PATCH 089/105] update docs --- site/content/en/docs/commands/start.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md index b5ca983541..2a381ef790 100644 --- a/site/content/en/docs/commands/start.md +++ b/site/content/en/docs/commands/start.md @@ -27,7 +27,7 @@ minikube start [flags] --apiserver-names stringArray A set of apiserver names which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine --apiserver-port int The apiserver listening port (default 8443) --auto-update-drivers If set, automatically updates drivers to the latest version. Defaults to true. (default true) - --base-image string The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.10@sha256:0c3666c7dacb3c7170a86b237d0d15d37afb5a6d2f0c45046546b5c09f40378c") + --base-image string The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase:v0.0.10@sha256:f58e0c4662bac8a9b5dda7984b185bad8502ade5d9fa364bf2755d636ab51438") --cache-images If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true) --container-runtime string The container runtime to be used (docker, crio, containerd). (default "docker") --cpus int Number of CPUs allocated to Kubernetes. (default 2) From bf4aa78ca6367aba419a9a935a9124a14d1a367b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 27 Apr 2020 21:58:21 +0200 Subject: [PATCH 090/105] Log the version used of the two KIC runtimes --- pkg/minikube/registry/drvs/docker/docker.go | 4 +++- pkg/minikube/registry/drvs/podman/podman.go | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index 13a4102e69..ada73686c0 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -83,8 +83,10 @@ func status() registry.State { // Quickly returns an error code if server is not running cmd := exec.CommandContext(ctx, oci.Docker, "version", "--format", "{{.Server.Version}}") - _, err = cmd.Output() + o, err := cmd.Output() + output := string(o) if err == nil { + glog.Infof("docker version: %s", output) return registry.State{Installed: true, Healthy: true} } diff --git a/pkg/minikube/registry/drvs/podman/podman.go b/pkg/minikube/registry/drvs/podman/podman.go index dd54e998d8..ffde7a19b9 100644 --- a/pkg/minikube/registry/drvs/podman/podman.go +++ b/pkg/minikube/registry/drvs/podman/podman.go @@ -95,6 +95,8 @@ func status() registry.State { o, err := cmd.Output() output := string(o) if err == nil { + glog.Infof("podman version: %s", output) + v, err := semver.Make(output) if err != nil { return registry.State{Error: err, Installed: true, Healthy: false, Fix: "Cant verify minimum required version for podman . See podman website for installation guide.", Doc: "https://podman.io/getting-started/installation.html"} From 1744ffe7732954e6e760e91adae6c8d399dfdc36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Mon, 27 Apr 2020 22:27:30 +0200 Subject: [PATCH 091/105] Restore the configured state added for podman Reverted 22aa1aff228a11036110ed1893c487ca4b8c43ca by accident --- pkg/drivers/kic/oci/oci.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 633d8912d3..10120ed97c 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -470,6 +470,8 @@ func ContainerStatus(prefix string, ociBin string, name string, warnSlow ...bool rr, err := runCmd(cmd, warnSlow...) o := strings.TrimSpace(rr.Stdout.String()) switch o { + case "configured": + return state.Stopped, nil case "running": return state.Running, nil case "exited": From a843b3083ce9721007e92bd888ca8505c82c4497 Mon Sep 17 00:00:00 2001 From: Radoslaw Smigielski Date: Mon, 27 Apr 2020 21:51:58 +0100 Subject: [PATCH 092/105] Also show IP address of the node --- cmd/minikube/cmd/node_list.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/node_list.go b/cmd/minikube/cmd/node_list.go index e5c8975652..0295837ceb 100644 --- a/cmd/minikube/cmd/node_list.go +++ b/cmd/minikube/cmd/node_list.go @@ -47,7 +47,7 @@ var nodeListCmd = &cobra.Command{ for _, n := range cc.Nodes { machineName := driver.MachineName(*cc, n) - fmt.Printf("%s\n", machineName) + fmt.Printf("%s\t%s\n", machineName, n.IP) } os.Exit(0) }, From 228c1b092cb6b1975aa2dbb66cce8134a7b5a3fb Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Mon, 27 Apr 2020 14:15:12 -0700 Subject: [PATCH 093/105] automatically apply kic overlay when detecting multinode clusters --- cmd/minikube/cmd/node_add.go | 4 ++++ pkg/drivers/kic/types.go | 6 ------ pkg/minikube/bootstrapper/bootstrapper.go | 1 + pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 7 ++++++- pkg/minikube/config/config.go | 17 +++++++++++++---- pkg/minikube/config/profile.go | 5 +++++ pkg/minikube/config/types.go | 8 ++++++++ pkg/minikube/driver/driver.go | 3 +-- pkg/minikube/node/start.go | 6 +++++- 9 files changed, 43 insertions(+), 14 deletions(-) diff --git a/cmd/minikube/cmd/node_add.go b/cmd/minikube/cmd/node_add.go index 3344065794..5d668f7531 100644 --- a/cmd/minikube/cmd/node_add.go +++ b/cmd/minikube/cmd/node_add.go @@ -61,6 +61,10 @@ var nodeAddCmd = &cobra.Command{ } } + // Add CNI config if it's not already there + // We need to run kubeadm.init here as well + config.MultiNodeCNIConfig(cc) + out.T(out.Ready, "Successfully added {{.name}} to {{.cluster}}!", out.V{"name": name, "cluster": cc.Name}) }, } diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go index 2b01019b2e..c8602c9000 100644 --- a/pkg/drivers/kic/types.go +++ b/pkg/drivers/kic/types.go @@ -23,12 +23,6 @@ import ( ) const ( - // DefaultNetwork is the Docker default bridge network named "bridge" - // (https://docs.docker.com/network/bridge/#use-the-default-bridge-network) - DefaultNetwork = "bridge" - // DefaultPodCIDR is The CIDR to be used for pods inside the node. - DefaultPodCIDR = "10.244.0.0/16" - // Version is the current version of kic Version = "v0.0.9" // SHA of the kic base image diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index 0019b1750d..e7ae9ba385 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -46,6 +46,7 @@ type Bootstrapper interface { LogCommands(config.ClusterConfig, LogOptions) map[string]string SetupCerts(config.KubernetesConfig, config.Node) error GetAPIServerStatus(string, int) (string, error) + ApplyCNI(config.ClusterConfig) error } const ( diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index d41ba6d248..2c1b57aaaf 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -226,7 +226,7 @@ func (k *Bootstrapper) init(cfg config.ClusterConfig) error { glog.Errorf("unable to create cluster role binding, some addons might not work: %v", err) } // the overlay is required for containerd and cri-o runtime: see #7428 - if driver.IsKIC(cfg.Driver) && cfg.KubernetesConfig.ContainerRuntime != "docker" { + if config.MultiNode(cfg) || (driver.IsKIC(cfg.Driver) && cfg.KubernetesConfig.ContainerRuntime != "docker") { if err := k.applyKICOverlay(cfg); err != nil { glog.Errorf("failed to apply kic overlay: %v", err) } @@ -809,8 +809,13 @@ func kubectlPath(cfg config.ClusterConfig) string { return path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl") } +func (k *Bootstrapper) ApplyCNI(cfg config.ClusterConfig) error { + return k.applyKICOverlay(cfg) +} + // applyKICOverlay applies the CNI plugin needed to make kic work func (k *Bootstrapper) applyKICOverlay(cfg config.ClusterConfig) error { + fmt.Println("APPLYING KIC OVERLAY") b := bytes.Buffer{} if err := kicCNIConfig.Execute(&b, struct{ ImageName string }{ImageName: kic.OverlayImage}); err != nil { return err diff --git a/pkg/minikube/config/config.go b/pkg/minikube/config/config.go index 4fe3f79f08..facedad5a6 100644 --- a/pkg/minikube/config/config.go +++ b/pkg/minikube/config/config.go @@ -24,8 +24,8 @@ import ( "os" "github.com/pkg/errors" + "github.com/spf13/viper" - "k8s.io/minikube/pkg/drivers/kic" "k8s.io/minikube/pkg/minikube/localpath" ) @@ -204,9 +204,12 @@ func (c *simpleConfigLoader) WriteConfigToFile(profileName string, cc *ClusterCo // MultiNodeCNIConfig add default CNI config needed for multinode clusters and saves off the config func MultiNodeCNIConfig(cc *ClusterConfig) { - cc.KubernetesConfig.NetworkPlugin = "cni" - cc.KubernetesConfig.ExtraOptions.Set(fmt.Sprintf("kubeadm.pod-network-cidr=%s", kic.DefaultPodCIDR)) - SaveProfile(cc.Name, cc) + if cc.KubernetesConfig.ExtraOptions.Get("pod-network-cidr", "kubeadm") == "" { + fmt.Println("SETTING CNI CONFIG") + cc.KubernetesConfig.NetworkPlugin = "cni" + cc.KubernetesConfig.ExtraOptions.Set(fmt.Sprintf("kubeadm.pod-network-cidr=%s", DefaultPodCIDR)) + SaveProfile(cc.Name, cc) + } } // MultiNode returns true if the cluster @@ -214,4 +217,10 @@ func MultiNode(cc ClusterConfig) bool { if len(cc.Nodes) > 1 { return true } + + if viper.GetInt("nodes") > 1 { + return true + } + + return false } diff --git a/pkg/minikube/config/profile.go b/pkg/minikube/config/profile.go index 8345341c51..6a476eaf00 100644 --- a/pkg/minikube/config/profile.go +++ b/pkg/minikube/config/profile.go @@ -136,6 +136,11 @@ func SaveNode(cfg *ClusterConfig, node *Node) error { if !update { cfg.Nodes = append(cfg.Nodes, *node) } + + if MultiNode(*cfg) { + MultiNodeCNIConfig(cfg) + } + return SaveProfile(viper.GetString(ProfileName), cfg) } diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 95991d5f33..4ce0470953 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -22,6 +22,14 @@ import ( "github.com/blang/semver" ) +const ( + // DefaultNetwork is the Docker default bridge network named "bridge" + // (https://docs.docker.com/network/bridge/#use-the-default-bridge-network) + DefaultNetwork = "bridge" + // DefaultPodCIDR is The CIDR to be used for pods inside the node. + DefaultPodCIDR = "10.244.0.0/16" +) + // Profile represents a minikube profile type Profile struct { Name string diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index fe9efd580e..4fcd44c059 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -24,7 +24,6 @@ import ( "strings" "github.com/golang/glog" - "k8s.io/minikube/pkg/drivers/kic" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/registry" ) @@ -165,7 +164,7 @@ func FlagDefaults(name string) FlagHints { fh.CacheImages = true // only for kic, until other runtimes are available we auto-set containerd. if name == Docker { - fh.ExtraOptions = append(fh.ExtraOptions, fmt.Sprintf("kubeadm.pod-network-cidr=%s", kic.DefaultPodCIDR)) + fh.ExtraOptions = append(fh.ExtraOptions, fmt.Sprintf("kubeadm.pod-network-cidr=%s", config.DefaultPodCIDR)) } return fh } diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 19ddf2ef9b..92458dbc74 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -169,6 +169,10 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { if err = bs.JoinCluster(*starter.Cfg, *starter.Node, joinCmd); err != nil { return nil, errors.Wrap(err, "joining cluster") } + + if err = cpBs.ApplyCNI(*starter.Cfg); err != nil { + return nil, errors.Wrap(err, "applying CNI") + } } wg.Wait() @@ -196,7 +200,7 @@ func Provision(cc *config.ClusterConfig, n *config.Node, apiServer bool) (comman } // Abstraction leakage alert: startHost requires the config to be saved, to satistfy pkg/provision/buildroot. - // Hence, saveConfig must be called before startHost, and again afterwards when we know the IP. + // Hence, SaveProfile must be called before startHost, and again afterwards when we know the IP. if err := config.SaveProfile(viper.GetString(config.ProfileName), cc); err != nil { return nil, false, nil, nil, errors.Wrap(err, "Failed to save config") } From e2467b2081e5aac5d83fab11d2492c706f57b19b Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Mon, 27 Apr 2020 17:53:55 -0700 Subject: [PATCH 094/105] just always apply overlay --- pkg/minikube/bootstrapper/bootstrapper.go | 1 - pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 11 +++-------- pkg/minikube/node/start.go | 4 ---- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index e7ae9ba385..0019b1750d 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -46,7 +46,6 @@ type Bootstrapper interface { LogCommands(config.ClusterConfig, LogOptions) map[string]string SetupCerts(config.KubernetesConfig, config.Node) error GetAPIServerStatus(string, int) (string, error) - ApplyCNI(config.ClusterConfig) error } const ( diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 2c1b57aaaf..7111d0e567 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -226,11 +226,10 @@ func (k *Bootstrapper) init(cfg config.ClusterConfig) error { glog.Errorf("unable to create cluster role binding, some addons might not work: %v", err) } // the overlay is required for containerd and cri-o runtime: see #7428 - if config.MultiNode(cfg) || (driver.IsKIC(cfg.Driver) && cfg.KubernetesConfig.ContainerRuntime != "docker") { - if err := k.applyKICOverlay(cfg); err != nil { - glog.Errorf("failed to apply kic overlay: %v", err) - } + if err := k.applyKICOverlay(cfg); err != nil { + glog.Errorf("failed to apply kic overlay: %v", err) } + wg.Done() }() @@ -809,10 +808,6 @@ func kubectlPath(cfg config.ClusterConfig) string { return path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl") } -func (k *Bootstrapper) ApplyCNI(cfg config.ClusterConfig) error { - return k.applyKICOverlay(cfg) -} - // applyKICOverlay applies the CNI plugin needed to make kic work func (k *Bootstrapper) applyKICOverlay(cfg config.ClusterConfig) error { fmt.Println("APPLYING KIC OVERLAY") diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 92458dbc74..aeb825965c 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -169,10 +169,6 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { if err = bs.JoinCluster(*starter.Cfg, *starter.Node, joinCmd); err != nil { return nil, errors.Wrap(err, "joining cluster") } - - if err = cpBs.ApplyCNI(*starter.Cfg); err != nil { - return nil, errors.Wrap(err, "applying CNI") - } } wg.Wait() From c1cd59821b56b69885fb10fd94a37f435adde130 Mon Sep 17 00:00:00 2001 From: Radoslaw Smigielski Date: Tue, 28 Apr 2020 10:18:57 +0100 Subject: [PATCH 095/105] Example how to enable multiple addons on start command --- site/content/en/docs/handbook/deploying.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/content/en/docs/handbook/deploying.md b/site/content/en/docs/handbook/deploying.md index 0c727afb0a..521cb949f2 100644 --- a/site/content/en/docs/handbook/deploying.md +++ b/site/content/en/docs/handbook/deploying.md @@ -29,10 +29,10 @@ To enable an add-on, see: minikube addons enable ``` -To enable an addon at start-up: +To enable an addon at start-up, where *--addons* option can be specified multiple times: ```shell -minikube start --addons +minikube start --addons --addons ``` For addons that expose a browser endpoint, you can quickly open them with: From 78a22f5056eb171d480d1d932e67c4cf991b390d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Tue, 28 Apr 2020 12:28:00 +0200 Subject: [PATCH 096/105] Remove prefix parameter and add prefix command Move the "sudo" prefix to a central location, instead of having it all over the place. Assume only needed on Linux. --- cmd/minikube/cmd/delete.go | 18 +++-- hack/preload-images/generate.go | 3 +- pkg/drivers/kic/kic.go | 35 +++++----- pkg/drivers/kic/oci/cli_runner.go | 17 +++++ pkg/drivers/kic/oci/info.go | 2 +- pkg/drivers/kic/oci/network.go | 12 ++-- pkg/drivers/kic/oci/oci.go | 73 +++++++++++---------- pkg/drivers/kic/oci/types.go | 5 -- pkg/drivers/kic/oci/volumes.go | 14 ++-- pkg/drivers/kic/types.go | 1 - pkg/minikube/command/kic_runner.go | 16 ++--- pkg/minikube/config/profile.go | 2 +- pkg/minikube/machine/delete.go | 12 ++-- pkg/minikube/machine/stop.go | 2 +- pkg/minikube/registry/drvs/docker/docker.go | 3 +- pkg/minikube/registry/drvs/podman/podman.go | 5 +- 16 files changed, 112 insertions(+), 108 deletions(-) diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go index df9aa79301..c2b05800aa 100644 --- a/cmd/minikube/cmd/delete.go +++ b/cmd/minikube/cmd/delete.go @@ -90,17 +90,17 @@ func init() { func deleteContainersAndVolumes() { delLabel := fmt.Sprintf("%s=%s", oci.CreatedByLabelKey, "true") - errs := oci.DeleteContainersByLabel(oci.Env, oci.Docker, delLabel) + errs := oci.DeleteContainersByLabel(oci.Docker, delLabel) if len(errs) > 0 { // it will error if there is no container to delete glog.Infof("error delete containers by label %q (might be okay): %+v", delLabel, errs) } - errs = oci.DeleteAllVolumesByLabel(oci.Env, oci.Docker, delLabel) + errs = oci.DeleteAllVolumesByLabel(oci.Docker, delLabel) if len(errs) > 0 { // it will not error if there is nothing to delete glog.Warningf("error delete volumes by label %q (might be okay): %+v", delLabel, errs) } - errs = oci.PruneAllVolumesByLabel(oci.Env, oci.Docker, delLabel) + errs = oci.PruneAllVolumesByLabel(oci.Docker, delLabel) if len(errs) > 0 { // it will not error if there is nothing to delete glog.Warningf("error pruning volumes by label %q (might be okay): %+v", delLabel, errs) } @@ -193,14 +193,12 @@ func DeleteProfiles(profiles []*config.Profile) []error { func deletePossibleKicLeftOver(name string) { delLabel := fmt.Sprintf("%s=%s", oci.ProfileLabelKey, name) - prefixes := []string{oci.Env, oci.Sudo} - for i, bin := range []string{oci.Docker, oci.Podman} { - prefix := prefixes[i] - cs, err := oci.ListContainersByLabel(prefix, bin, delLabel) + for _, bin := range []string{oci.Docker, oci.Podman} { + cs, err := oci.ListContainersByLabel(bin, delLabel) if err == nil && len(cs) > 0 { for _, c := range cs { out.T(out.DeletingHost, `Deleting container "{{.name}}" ...`, out.V{"name": name}) - err := oci.DeleteContainer(prefix, bin, c) + err := oci.DeleteContainer(bin, c) if err != nil { // it will error if there is no container to delete glog.Errorf("error deleting container %q. you might want to delete that manually :\n%v", name, err) } @@ -208,12 +206,12 @@ func deletePossibleKicLeftOver(name string) { } } - errs := oci.DeleteAllVolumesByLabel(prefix, bin, delLabel) + errs := oci.DeleteAllVolumesByLabel(bin, delLabel) if errs != nil { // it will not error if there is nothing to delete glog.Warningf("error deleting volumes (might be okay).\nTo see the list of volumes run: 'docker volume ls'\n:%v", errs) } - errs = oci.PruneAllVolumesByLabel(prefix, bin, delLabel) + errs = oci.PruneAllVolumesByLabel(bin, delLabel) if len(errs) > 0 { // it will not error if there is nothing to delete glog.Warningf("error pruning volume (might be okay):\n%v", errs) } diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go index d03b38f1a7..54c680bc48 100644 --- a/hack/preload-images/generate.go +++ b/hack/preload-images/generate.go @@ -41,7 +41,6 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string driver := kic.NewDriver(kic.Config{ KubernetesVersion: kubernetesVersion, ContainerRuntime: containerRuntime, - OCIPrefix: oci.Env, OCIBinary: oci.Docker, MachineName: profile, ImageDigest: kic.BaseImage, @@ -70,7 +69,7 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string imgs = append(imgs, kic.OverlayImage) } - runner := command.NewKICRunner(profile, driver.OCIPrefix, driver.OCIBinary) + runner := command.NewKICRunner(profile, driver.OCIBinary) // will need to do this to enable the container run-time service sv, err := util.ParseKubernetesVersion(kubernetesVersion) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index fb9536113a..1e8e3df9d2 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -48,7 +48,6 @@ type Driver struct { URL string exec command.Runner NodeConfig Config - OCIPrefix string // env, sudo OCIBinary string // docker,podman } @@ -59,9 +58,8 @@ func NewDriver(c Config) *Driver { MachineName: c.MachineName, StorePath: c.StorePath, }, - exec: command.NewKICRunner(c.MachineName, c.OCIPrefix, c.OCIBinary), + exec: command.NewKICRunner(c.MachineName, c.OCIBinary), NodeConfig: c, - OCIPrefix: c.OCIPrefix, OCIBinary: c.OCIBinary, } return d @@ -78,7 +76,6 @@ func (d *Driver) Create() error { Memory: strconv.Itoa(d.NodeConfig.Memory) + "mb", Envs: d.NodeConfig.Envs, ExtraArgs: []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.APIServerPort)}, - OCIPrefix: d.NodeConfig.OCIPrefix, OCIBinary: d.NodeConfig.OCIBinary, APIServerPort: d.NodeConfig.APIServerPort, } @@ -102,15 +99,15 @@ func (d *Driver) Create() error { }, ) - exists, err := oci.ContainerExists(d.OCIPrefix, d.OCIBinary, params.Name, true) + exists, err := oci.ContainerExists(d.OCIBinary, params.Name, true) if err != nil { glog.Warningf("failed to check if container already exists: %v", err) } if exists { // if container was created by minikube it is safe to delete and recreate it. - if oci.IsCreatedByMinikube(d.OCIPrefix, d.OCIBinary, params.Name) { + if oci.IsCreatedByMinikube(d.OCIBinary, params.Name) { glog.Info("Found already existing abandoned minikube container, will try to delete.") - if err := oci.DeleteContainer(d.OCIPrefix, d.OCIBinary, params.Name); err != nil { + if err := oci.DeleteContainer(d.OCIBinary, params.Name); err != nil { glog.Errorf("Failed to delete a conflicting minikube container %s. You might need to restart your %s daemon and delete it manually and try again: %v", params.Name, params.OCIBinary, err) } } else { @@ -162,7 +159,7 @@ func (d *Driver) prepareSSH() error { return errors.Wrap(err, "generate ssh key") } - cmder := command.NewKICRunner(d.NodeConfig.MachineName, d.NodeConfig.OCIPrefix, d.NodeConfig.OCIBinary) + cmder := command.NewKICRunner(d.NodeConfig.MachineName, d.NodeConfig.OCIBinary) f, err := assets.NewFileAsset(d.GetSSHKeyPath()+".pub", "/home/docker/.ssh/", "authorized_keys", "0644") if err != nil { return errors.Wrap(err, "create pubkey assetfile ") @@ -237,23 +234,23 @@ func (d *Driver) GetURL() (string, error) { // GetState returns the state that the host is in (running, stopped, etc) func (d *Driver) GetState() (state.State, error) { - return oci.ContainerStatus(d.OCIPrefix, d.OCIBinary, d.MachineName, true) + return oci.ContainerStatus(d.OCIBinary, d.MachineName, true) } // Kill stops a host forcefully, including any containers that we are managing. func (d *Driver) Kill() error { // on init this doesn't get filled when called from cmd - d.exec = command.NewKICRunner(d.MachineName, d.OCIPrefix, d.OCIBinary) + d.exec = command.NewKICRunner(d.MachineName, d.OCIBinary) if err := sysinit.New(d.exec).ForceStop("kubelet"); err != nil { glog.Warningf("couldn't force stop kubelet. will continue with kill anyways: %v", err) } - if err := oci.ShutDown(d.OCIPrefix, d.OCIBinary, d.MachineName); err != nil { + if err := oci.ShutDown(d.OCIBinary, d.MachineName); err != nil { glog.Warningf("couldn't shutdown the container, will continue with kill anyways: %v", err) } cr := command.NewExecRunner() // using exec runner for interacting with dameon. - if _, err := cr.RunCmd(exec.Command(d.NodeConfig.OCIPrefix, d.NodeConfig.OCIBinary, "kill", d.MachineName)); err != nil { + if _, err := cr.RunCmd(oci.PrefixCmd(exec.Command(d.NodeConfig.OCIBinary, "kill", d.MachineName))); err != nil { return errors.Wrapf(err, "killing %q", d.MachineName) } return nil @@ -261,11 +258,11 @@ func (d *Driver) Kill() error { // Remove will delete the Kic Node Container func (d *Driver) Remove() error { - if _, err := oci.ContainerID(d.OCIPrefix, d.OCIBinary, d.MachineName); err != nil { + if _, err := oci.ContainerID(d.OCIBinary, d.MachineName); err != nil { glog.Infof("could not find the container %s to remove it. will try anyways", d.MachineName) } - if err := oci.DeleteContainer(d.NodeConfig.OCIPrefix, d.NodeConfig.OCIBinary, d.MachineName); err != nil { + if err := oci.DeleteContainer(d.NodeConfig.OCIBinary, d.MachineName); err != nil { if strings.Contains(err.Error(), "is already in progress") { return errors.Wrap(err, "stuck delete") } @@ -276,7 +273,7 @@ func (d *Driver) Remove() error { } // check there be no container left after delete - if id, err := oci.ContainerID(d.OCIPrefix, d.OCIBinary, d.MachineName); err == nil && id != "" { + if id, err := oci.ContainerID(d.OCIBinary, d.MachineName); err == nil && id != "" { return fmt.Errorf("expected no container ID be found for %q after delete. but got %q", d.MachineName, id) } return nil @@ -304,11 +301,11 @@ func (d *Driver) Restart() error { // Start an already created kic container func (d *Driver) Start() error { cr := command.NewExecRunner() // using exec runner for interacting with docker/podman daemon - if _, err := cr.RunCmd(exec.Command(d.NodeConfig.OCIPrefix, d.NodeConfig.OCIBinary, "start", d.MachineName)); err != nil { + if _, err := cr.RunCmd(oci.PrefixCmd(exec.Command(d.NodeConfig.OCIBinary, "start", d.MachineName))); err != nil { return errors.Wrap(err, "start") } checkRunning := func() error { - s, err := oci.ContainerStatus(d.NodeConfig.OCIPrefix, d.NodeConfig.OCIBinary, d.MachineName) + s, err := oci.ContainerStatus(d.NodeConfig.OCIBinary, d.MachineName) if err != nil { return err } @@ -328,7 +325,7 @@ func (d *Driver) Start() error { // Stop a host gracefully, including any containers that we are managing. func (d *Driver) Stop() error { // on init this doesn't get filled when called from cmd - d.exec = command.NewKICRunner(d.MachineName, d.OCIPrefix, d.OCIBinary) + d.exec = command.NewKICRunner(d.MachineName, d.OCIBinary) // docker does not send right SIG for systemd to know to stop the systemd. // to avoid bind address be taken on an upgrade. more info https://github.com/kubernetes/minikube/issues/7171 if err := sysinit.New(d.exec).Stop("kubelet"); err != nil { @@ -363,7 +360,7 @@ func (d *Driver) Stop() error { glog.Warningf("couldn't stop kube-apiserver proc: %v", err) } - cmd := exec.Command(d.NodeConfig.OCIPrefix, d.NodeConfig.OCIBinary, "stop", d.MachineName) + cmd := exec.Command(d.NodeConfig.OCIBinary, "stop", d.MachineName) if err := cmd.Run(); err != nil { return errors.Wrapf(err, "stopping %s", d.MachineName) } diff --git a/pkg/drivers/kic/oci/cli_runner.go b/pkg/drivers/kic/oci/cli_runner.go index f740580423..173b509d27 100644 --- a/pkg/drivers/kic/oci/cli_runner.go +++ b/pkg/drivers/kic/oci/cli_runner.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "os/exec" + "runtime" "strings" "time" @@ -63,8 +64,24 @@ func (rr RunResult) Output() string { return sb.String() } +// PrefixCmd adds any needed prefix (such as sudo) to the command +func PrefixCmd(cmd *exec.Cmd) *exec.Cmd { + if cmd.Args[0] == Podman && runtime.GOOS == "linux" { // want sudo when not running podman-remote + cmdWithSudo := exec.Command("sudo", cmd.Args...) + cmdWithSudo.Env = cmd.Env + cmdWithSudo.Dir = cmd.Dir + cmdWithSudo.Stdin = cmd.Stdin + cmdWithSudo.Stdout = cmd.Stdout + cmdWithSudo.Stderr = cmd.Stderr + cmd = cmdWithSudo + } + return cmd +} + // runCmd runs a command exec.Command against docker daemon or podman func runCmd(cmd *exec.Cmd, warnSlow ...bool) (*RunResult, error) { + cmd = PrefixCmd(cmd) + warn := false if len(warnSlow) > 0 { warn = warnSlow[0] diff --git a/pkg/drivers/kic/oci/info.go b/pkg/drivers/kic/oci/info.go index 45752f1959..4fe381ede5 100644 --- a/pkg/drivers/kic/oci/info.go +++ b/pkg/drivers/kic/oci/info.go @@ -231,7 +231,7 @@ func dockerSystemInfo() (dockerSysInfo, error) { // podmanSysInfo returns podman system info --format '{{json .}}' func podmanSystemInfo() (podmanSysInfo, error) { var ps podmanSysInfo - rr, err := runCmd(exec.Command(Sudo, Podman, "system", "info", "--format", "json")) + rr, err := runCmd(exec.Command(Podman, "system", "info", "--format", "json")) if err != nil { return ps, errors.Wrap(err, "get podman system info") } diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index 1bb5cdeaaf..2ba57b06be 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -56,13 +56,13 @@ func digDNS(ociBin, containerName, dns string) (net.IP, error) { // dockerGatewayIP gets the default gateway ip for the docker bridge on the user's host machine // gets the ip from user's host docker func dockerGatewayIP() (net.IP, error) { - rr, err := runCmd(exec.Command(Env, Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}")) + rr, err := runCmd(exec.Command(Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}")) if err != nil { return nil, errors.Wrapf(err, "get network bridge") } bridgeID := strings.TrimSpace(rr.Stdout.String()) - rr, err = runCmd(exec.Command(Env, Docker, "inspect", + rr, err = runCmd(exec.Command(Docker, "inspect", "--format", "{{(index .IPAM.Config 0).Gateway}}", bridgeID)) if err != nil { return nil, errors.Wrapf(err, "inspect IP bridge network %q.", bridgeID) @@ -84,12 +84,12 @@ func ForwardedPort(ociBin string, ociID string, contPort int) (int, error) { if ociBin == Podman { //podman inspect -f "{{range .NetworkSettings.Ports}}{{if eq .ContainerPort "80"}}{{.HostPort}}{{end}}{{end}}" - rr, err = runCmd(exec.Command(Sudo, ociBin, "inspect", "-f", fmt.Sprintf("{{range .NetworkSettings.Ports}}{{if eq .ContainerPort %s}}{{.HostPort}}{{end}}{{end}}", fmt.Sprint(contPort)), ociID)) + rr, err = runCmd(exec.Command(ociBin, "inspect", "-f", fmt.Sprintf("{{range .NetworkSettings.Ports}}{{if eq .ContainerPort %s}}{{.HostPort}}{{end}}{{end}}", fmt.Sprint(contPort)), ociID)) if err != nil { return 0, errors.Wrapf(err, "get port %d for %q", contPort, ociID) } } else { - rr, err = runCmd(exec.Command(Env, ociBin, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)) + rr, err = runCmd(exec.Command(ociBin, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)) if err != nil { return 0, errors.Wrapf(err, "get port %d for %q", contPort, ociID) } @@ -116,7 +116,7 @@ func ContainerIPs(ociBin string, name string) (string, string, error) { // podmanConttainerIP returns ipv4, ipv6 of container or error func podmanConttainerIP(name string) (string, string, error) { - rr, err := runCmd(exec.Command(Sudo, Podman, "inspect", + rr, err := runCmd(exec.Command(Podman, "inspect", "-f", "{{.NetworkSettings.IPAddress}}", name)) if err != nil { @@ -132,7 +132,7 @@ func podmanConttainerIP(name string) (string, string, error) { // dockerContainerIP returns ipv4, ipv6 of container or error func dockerContainerIP(name string) (string, string, error) { // retrieve the IP address of the node using docker inspect - lines, err := inspect(Env, Docker, name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}") + lines, err := inspect(Docker, name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}") if err != nil { return "", "", errors.Wrap(err, "inspecting NetworkSettings.Networks") } diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 10120ed97c..9e633b9a8e 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -38,10 +38,10 @@ import ( // DeleteContainersByLabel deletes all containers that have a specific label // if there no containers found with the given label, it will return nil -func DeleteContainersByLabel(prefix string, ociBin string, label string) []error { +func DeleteContainersByLabel(ociBin string, label string) []error { var deleteErrs []error - cs, err := ListContainersByLabel(prefix, ociBin, label) + cs, err := ListContainersByLabel(ociBin, label) if err != nil { return []error{fmt.Errorf("listing containers by label %q", label)} } @@ -51,7 +51,7 @@ func DeleteContainersByLabel(prefix string, ociBin string, label string) []error } for _, c := range cs { - _, err := ContainerStatus(prefix, ociBin, c) + _, err := ContainerStatus(ociBin, c) // only try to delete if docker/podman inspect returns // if it doesn't it means docker daemon is stuck and needs restart if err != nil { @@ -59,11 +59,11 @@ func DeleteContainersByLabel(prefix string, ociBin string, label string) []error glog.Errorf("%s daemon seems to be stuck. Please try restarting your %s. :%v", ociBin, ociBin, err) continue } - if err := ShutDown(prefix, ociBin, c); err != nil { + if err := ShutDown(ociBin, c); err != nil { glog.Infof("couldn't shut down %s (might be okay): %v ", c, err) } - if _, err := runCmd(exec.Command(prefix, ociBin, "rm", "-f", "-v", c)); err != nil { + if _, err := runCmd(exec.Command(ociBin, "rm", "-f", "-v", c)); err != nil { deleteErrs = append(deleteErrs, errors.Wrapf(err, "delete container %s: output %s", c, err)) } @@ -72,18 +72,18 @@ func DeleteContainersByLabel(prefix string, ociBin string, label string) []error } // DeleteContainer deletes a container by ID or Name -func DeleteContainer(prefix string, ociBin string, name string) error { +func DeleteContainer(ociBin string, name string) error { - _, err := ContainerStatus(prefix, ociBin, name) + _, err := ContainerStatus(ociBin, name) if err != nil { glog.Errorf("%s daemon seems to be stuck. Please try restarting your %s. Will try to delete anyways: %v", ociBin, ociBin, err) } // try to delete anyways - if err := ShutDown(prefix, ociBin, name); err != nil { + if err := ShutDown(ociBin, name); err != nil { glog.Infof("couldn't shut down %s (might be okay): %v ", name, err) } - if _, err := runCmd(exec.Command(prefix, ociBin, "rm", "-f", "-v", name)); err != nil { + if _, err := runCmd(exec.Command(ociBin, "rm", "-f", "-v", name)); err != nil { return errors.Wrapf(err, "delete %s", name) } return nil @@ -175,25 +175,25 @@ func CreateContainerNode(p CreateParams) error { // adds node specific args runArgs = append(runArgs, p.ExtraArgs...) - if enabled := isUsernsRemapEnabled(p.OCIPrefix, p.OCIBinary); enabled { + if enabled := isUsernsRemapEnabled(p.OCIBinary); enabled { // We need this argument in order to make this command work // in systems that have userns-remap enabled on the docker daemon runArgs = append(runArgs, "--userns=host") } - if err := createContainer(p.OCIPrefix, p.OCIBinary, p.Image, withRunArgs(runArgs...), withMounts(p.Mounts), withPortMappings(p.PortMappings)); err != nil { + if err := createContainer(p.OCIBinary, p.Image, withRunArgs(runArgs...), withMounts(p.Mounts), withPortMappings(p.PortMappings)); err != nil { return errors.Wrap(err, "create container") } checkRunning := func() error { - r, err := ContainerRunning(p.OCIPrefix, p.OCIBinary, p.Name) + r, err := ContainerRunning(p.OCIBinary, p.Name) if err != nil { return fmt.Errorf("temporary error checking running for %q : %v", p.Name, err) } if !r { return fmt.Errorf("temporary error created container %q is not running yet", p.Name) } - s, err := ContainerStatus(p.OCIPrefix, p.OCIBinary, p.Name) + s, err := ContainerStatus(p.OCIBinary, p.Name) if err != nil { return fmt.Errorf("temporary error checking status for %q : %v", p.Name, err) } @@ -213,7 +213,7 @@ func CreateContainerNode(p CreateParams) error { } // CreateContainer creates a container with "docker/podman run" -func createContainer(prefix string, ociBin string, image string, opts ...createOpt) error { +func createContainer(ociBin string, image string, opts ...createOpt) error { o := &createOpts{} for _, opt := range opts { o = opt(o) @@ -227,7 +227,7 @@ func createContainer(prefix string, ociBin string, image string, opts ...createO runArgs = append(runArgs, generatePortMappings(portMapping)...) } // construct the actual docker run argv - args := []string{ociBin, "run"} + args := []string{"run"} // to run nested container from privileged container in podman https://bugzilla.redhat.com/show_bug.cgi?id=1687713 if ociBin == Podman { @@ -238,7 +238,7 @@ func createContainer(prefix string, ociBin string, image string, opts ...createO args = append(args, image) args = append(args, o.ContainerArgs...) - if _, err := runCmd(exec.Command(prefix, args...)); err != nil { + if _, err := runCmd(exec.Command(ociBin, args...)); err != nil { return err } @@ -246,8 +246,8 @@ func createContainer(prefix string, ociBin string, image string, opts ...createO } // ContainerID returns id of a container name -func ContainerID(prefix, ociBin string, nameOrID string) (string, error) { - rr, err := runCmd(exec.Command(prefix, ociBin, "inspect", "-f", "{{.Id}}", nameOrID)) +func ContainerID(ociBin string, nameOrID string) (string, error) { + rr, err := runCmd(exec.Command(ociBin, "inspect", "-f", "{{.Id}}", nameOrID)) if err != nil { // don't return error if not found, only return empty string if strings.Contains(rr.Stdout.String(), "Error: No such object:") || strings.Contains(rr.Stdout.String(), "unable to find") { err = nil @@ -258,7 +258,7 @@ func ContainerID(prefix, ociBin string, nameOrID string) (string, error) { } // ContainerExists checks if container name exists (either running or exited) -func ContainerExists(prefix string, ociBin string, name string, warnSlow ...bool) (bool, error) { +func ContainerExists(ociBin string, name string, warnSlow ...bool) (bool, error) { rr, err := runCmd(exec.Command(ociBin, "ps", "-a", "--format", "{{.Names}}"), warnSlow...) if err != nil { return false, err @@ -276,8 +276,8 @@ func ContainerExists(prefix string, ociBin string, name string, warnSlow ...bool // IsCreatedByMinikube returns true if the container was created by minikube // with default assumption that it is not created by minikube when we don't know for sure -func IsCreatedByMinikube(prefix, ociBin string, nameOrID string) bool { - rr, err := runCmd(exec.Command(prefix, ociBin, "inspect", nameOrID, "--format", "{{.Config.Labels}}")) +func IsCreatedByMinikube(ociBin string, nameOrID string) bool { + rr, err := runCmd(exec.Command(ociBin, "inspect", nameOrID, "--format", "{{.Config.Labels}}")) if err != nil { return false } @@ -289,13 +289,14 @@ func IsCreatedByMinikube(prefix, ociBin string, nameOrID string) bool { return false } -func ListOwnedContainers(prefix string, ociBin string) ([]string, error) { - return ListContainersByLabel(prefix, ociBin, ProfileLabelKey) +// ListOwnedContainers lists all the containres that kic driver created on user's machine using a label +func ListOwnedContainers(ociBin string) ([]string, error) { + return ListContainersByLabel(ociBin, ProfileLabelKey) } // inspect return low-level information on containers -func inspect(prefix string, ociBin string, containerNameOrID, format string) ([]string, error) { - cmd := exec.Command(prefix, ociBin, "inspect", +func inspect(ociBin string, containerNameOrID, format string) ([]string, error) { + cmd := exec.Command(ociBin, "inspect", "-f", format, containerNameOrID) // ... against the "node" container var buff bytes.Buffer @@ -357,8 +358,8 @@ func generateMountBindings(mounts ...Mount) []string { } // isUsernsRemapEnabled checks if userns-remap is enabled in docker -func isUsernsRemapEnabled(prefix string, ociBin string) bool { - cmd := exec.Command(prefix, ociBin, "info", "--format", "'{{json .SecurityOptions}}'") +func isUsernsRemapEnabled(ociBin string) bool { + cmd := exec.Command(ociBin, "info", "--format", "'{{json .SecurityOptions}}'") var buff bytes.Buffer cmd.Stdout = &buff cmd.Stderr = &buff @@ -420,8 +421,8 @@ func withPortMappings(portMappings []PortMapping) createOpt { } // ListContainersByLabel returns all the container names with a specified label -func ListContainersByLabel(prefix string, ociBin string, label string, warnSlow ...bool) ([]string, error) { - rr, err := runCmd(exec.Command(prefix, ociBin, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}"), warnSlow...) +func ListContainersByLabel(ociBin string, label string, warnSlow ...bool) ([]string, error) { + rr, err := runCmd(exec.Command(ociBin, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}"), warnSlow...) if err != nil { return nil, err } @@ -456,8 +457,8 @@ func PointToHostDockerDaemon() error { } // ContainerRunning returns running state of a container -func ContainerRunning(prefix string, ociBin string, name string, warnSlow ...bool) (bool, error) { - rr, err := runCmd(exec.Command(prefix, ociBin, "inspect", name, "--format={{.State.Running}}"), warnSlow...) +func ContainerRunning(ociBin string, name string, warnSlow ...bool) (bool, error) { + rr, err := runCmd(exec.Command(ociBin, "inspect", name, "--format={{.State.Running}}"), warnSlow...) if err != nil { return false, err } @@ -465,8 +466,8 @@ func ContainerRunning(prefix string, ociBin string, name string, warnSlow ...boo } // ContainerStatus returns status of a container running,exited,... -func ContainerStatus(prefix string, ociBin string, name string, warnSlow ...bool) (state.State, error) { - cmd := exec.Command(prefix, ociBin, "inspect", name, "--format={{.State.Status}}") +func ContainerStatus(ociBin string, name string, warnSlow ...bool) (state.State, error) { + cmd := exec.Command(ociBin, "inspect", name, "--format={{.State.Status}}") rr, err := runCmd(cmd, warnSlow...) o := strings.TrimSpace(rr.Stdout.String()) switch o { @@ -490,15 +491,15 @@ func ContainerStatus(prefix string, ociBin string, name string, warnSlow ...bool // ShutDown will run command to shut down the container // to ensure the containers process and networking bindings are all closed // to avoid containers getting stuck before delete https://github.com/kubernetes/minikube/issues/7657 -func ShutDown(prefix string, ociBin string, name string) error { - if _, err := runCmd(exec.Command(prefix, ociBin, "exec", "--privileged", "-t", name, "/bin/bash", "-c", "sudo init 0")); err != nil { +func ShutDown(ociBin string, name string) error { + if _, err := runCmd(exec.Command(ociBin, "exec", "--privileged", "-t", name, "/bin/bash", "-c", "sudo init 0")); err != nil { glog.Infof("error shutdown %s: %v", name, err) } // helps with allowing docker realize the container is exited and report its status correctly. time.Sleep(time.Second * 1) // wait till it is stoped stopped := func() error { - st, err := ContainerStatus(prefix, ociBin, name) + st, err := ContainerStatus(ociBin, name) if st == state.Stopped { glog.Infof("container %s status is %s", name, st) return nil diff --git a/pkg/drivers/kic/oci/types.go b/pkg/drivers/kic/oci/types.go index 45c38522b9..8c43ce7d8b 100644 --- a/pkg/drivers/kic/oci/types.go +++ b/pkg/drivers/kic/oci/types.go @@ -19,10 +19,6 @@ package oci const ( // DefaultBindIPV4 is The default IP the container will listen on. DefaultBindIPV4 = "127.0.0.1" - // Env is env - Env = "env" - // Sudo is sudo - Sudo = "sudo" // Docker is docker Docker = "docker" // Podman is podman @@ -51,7 +47,6 @@ type CreateParams struct { Memory string // memory (mbs) to assign to the container Envs map[string]string // environment variables to pass to the container ExtraArgs []string // a list of any extra option to pass to oci binary during creation time, for example --expose 8080... - OCIPrefix string // env or sudo OCIBinary string // docker or podman } diff --git a/pkg/drivers/kic/oci/volumes.go b/pkg/drivers/kic/oci/volumes.go index 37dea4d35e..319d07c342 100644 --- a/pkg/drivers/kic/oci/volumes.go +++ b/pkg/drivers/kic/oci/volumes.go @@ -29,18 +29,18 @@ import ( // DeleteAllVolumesByLabel deletes all volumes that have a specific label // if there is no volume to delete it will return nil -func DeleteAllVolumesByLabel(prefix string, ociBin string, label string, warnSlow ...bool) []error { +func DeleteAllVolumesByLabel(ociBin string, label string, warnSlow ...bool) []error { var deleteErrs []error glog.Infof("trying to delete all %s volumes with label %s", ociBin, label) - vs, err := allVolumesByLabel(prefix, ociBin, label) + vs, err := allVolumesByLabel(ociBin, label) if err != nil { return []error{fmt.Errorf("listing volumes by label %q: %v", label, err)} } for _, v := range vs { - if _, err := runCmd(exec.Command(prefix, ociBin, "volume", "rm", "--force", v), warnSlow...); err != nil { + if _, err := runCmd(exec.Command(ociBin, "volume", "rm", "--force", v), warnSlow...); err != nil { deleteErrs = append(deleteErrs, fmt.Errorf("deleting %q", v)) } } @@ -51,10 +51,10 @@ func DeleteAllVolumesByLabel(prefix string, ociBin string, label string, warnSlo // PruneAllVolumesByLabel deletes all volumes that have a specific label // if there is no volume to delete it will return nil // example: docker volume prune -f --filter label=name.minikube.sigs.k8s.io=minikube -func PruneAllVolumesByLabel(prefix string, ociBin string, label string, warnSlow ...bool) []error { +func PruneAllVolumesByLabel(ociBin string, label string, warnSlow ...bool) []error { var deleteErrs []error glog.Infof("trying to prune all %s volumes with label %s", ociBin, label) - cmd := exec.Command(prefix, ociBin, "volume", "prune", "-f", "--filter", "label="+label) + cmd := exec.Command(ociBin, "volume", "prune", "-f", "--filter", "label="+label) if _, err := runCmd(cmd, warnSlow...); err != nil { deleteErrs = append(deleteErrs, errors.Wrapf(err, "prune volume by label %s", label)) } @@ -64,8 +64,8 @@ func PruneAllVolumesByLabel(prefix string, ociBin string, label string, warnSlow // allVolumesByLabel returns name of all docker volumes by a specific label // will not return error if there is no volume found. -func allVolumesByLabel(prefix string, ociBin string, label string) ([]string, error) { - rr, err := runCmd(exec.Command(prefix, ociBin, "volume", "ls", "--filter", "label="+label, "--format", "{{.Name}}")) +func allVolumesByLabel(ociBin string, label string) ([]string, error) { + rr, err := runCmd(exec.Command(ociBin, "volume", "ls", "--filter", "label="+label, "--format", "{{.Name}}")) s := bufio.NewScanner(bytes.NewReader(rr.Stdout.Bytes())) var vols []string for s.Scan() { diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go index 3d68269378..2b01019b2e 100644 --- a/pkg/drivers/kic/types.go +++ b/pkg/drivers/kic/types.go @@ -49,7 +49,6 @@ type Config struct { CPU int // Number of CPU cores assigned to the container Memory int // max memory in MB StorePath string // libmachine store path - OCIPrefix string // prefix to use (env, sudo, ...) OCIBinary string // oci tool to use (docker, podman,...) ImageDigest string // image name with sha to use for the node Mounts []oci.Mount // mounts diff --git a/pkg/minikube/command/kic_runner.go b/pkg/minikube/command/kic_runner.go index 98c66e5426..667b3cb1e6 100644 --- a/pkg/minikube/command/kic_runner.go +++ b/pkg/minikube/command/kic_runner.go @@ -38,22 +38,19 @@ import ( // It implements the CommandRunner interface. type kicRunner struct { nameOrID string - prefix string ociBin string } // NewKICRunner returns a kicRunner implementor of runner which runs cmds inside a container -func NewKICRunner(containerNameOrID string, prefix string, oci string) Runner { +func NewKICRunner(containerNameOrID string, oci string) Runner { return &kicRunner{ nameOrID: containerNameOrID, - prefix: prefix, // env or sudo - ociBin: oci, // docker or podman + ociBin: oci, // docker or podman } } func (k *kicRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) { args := []string{ - k.ociBin, "exec", // run with privileges so we can remount etc.. "--privileged", @@ -84,7 +81,7 @@ func (k *kicRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) { args, cmd.Args..., ) - oc := exec.Command(k.prefix, args...) + oc := exec.Command(k.ociBin, args...) oc.Stdin = cmd.Stdin oc.Stdout = cmd.Stdout oc.Stderr = cmd.Stderr @@ -111,6 +108,9 @@ func (k *kicRunner) RunCmd(cmd *exec.Cmd) (*RunResult, error) { oc.Stdout = outb oc.Stderr = errb + oc = oci.PrefixCmd(oc) + glog.Infof("Args: %v", oc.Args) + start := time.Now() err := oc.Run() @@ -202,14 +202,14 @@ func (k *kicRunner) chmod(dst string, perm string) error { // Podman cp command doesn't match docker and doesn't have -a func copyToPodman(src string, dest string) error { - if out, err := exec.Command(oci.Sudo, oci.Podman, "cp", src, dest).CombinedOutput(); err != nil { + if out, err := oci.PrefixCmd(exec.Command(oci.Podman, "cp", src, dest)).CombinedOutput(); err != nil { return errors.Wrapf(err, "podman copy %s into %s, output: %s", src, dest, string(out)) } return nil } func copyToDocker(src string, dest string) error { - if out, err := exec.Command(oci.Docker, "cp", "-a", src, dest).CombinedOutput(); err != nil { + if out, err := oci.PrefixCmd(exec.Command(oci.Docker, "cp", "-a", src, dest)).CombinedOutput(); err != nil { return errors.Wrapf(err, "docker copy %s into %s, output: %s", src, dest, string(out)) } return nil diff --git a/pkg/minikube/config/profile.go b/pkg/minikube/config/profile.go index 788d98afea..8345341c51 100644 --- a/pkg/minikube/config/profile.go +++ b/pkg/minikube/config/profile.go @@ -203,7 +203,7 @@ func ListProfiles(miniHome ...string) (validPs []*Profile, inValidPs []*Profile, return nil, nil, err } // try to get profiles list based on all contrainers created by docker driver - cs, err := oci.ListOwnedContainers(oci.Env, oci.Docker) + cs, err := oci.ListOwnedContainers(oci.Docker) if err == nil { pDirs = append(pDirs, cs...) } diff --git a/pkg/minikube/machine/delete.go b/pkg/minikube/machine/delete.go index 134c6a74eb..4cd55d37d0 100644 --- a/pkg/minikube/machine/delete.go +++ b/pkg/minikube/machine/delete.go @@ -35,12 +35,12 @@ import ( // deleteOrphanedKIC attempts to delete an orphaned docker instance for machines without a config file // used as last effort clean up not returning errors, wont warn user. -func deleteOrphanedKIC(prefix string, ociBin string, name string) { +func deleteOrphanedKIC(ociBin string, name string) { if !(ociBin == oci.Podman || ociBin == oci.Docker) { return } - _, err := oci.ContainerStatus(prefix, ociBin, name) + _, err := oci.ContainerStatus(ociBin, name) if err != nil { glog.Infof("couldn't inspect container %q before deleting: %v", name, err) return @@ -49,10 +49,10 @@ func deleteOrphanedKIC(prefix string, ociBin string, name string) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - if err := oci.ShutDown(prefix, ociBin, name); err != nil { + if err := oci.ShutDown(ociBin, name); err != nil { glog.Infof("couldn't shut down %s (might be okay): %v ", name, err) } - cmd := exec.CommandContext(ctx, prefix, ociBin, "rm", "-f", "-v", name) + cmd := exec.CommandContext(ctx, ociBin, "rm", "-f", "-v", name) err = cmd.Run() if err == nil { glog.Infof("Found stale kic container and successfully cleaned it up!") @@ -63,8 +63,8 @@ func deleteOrphanedKIC(prefix string, ociBin string, name string) { func DeleteHost(api libmachine.API, machineName string) error { host, err := api.Load(machineName) if err != nil && host == nil { - deleteOrphanedKIC(oci.Env, oci.Docker, machineName) - deleteOrphanedKIC(oci.Sudo, oci.Podman, machineName) + deleteOrphanedKIC(oci.Docker, machineName) + deleteOrphanedKIC(oci.Podman, machineName) // Keep going even if minikube does not know about the host } diff --git a/pkg/minikube/machine/stop.go b/pkg/minikube/machine/stop.go index f8d5fa3cf3..d5cac2b2d3 100644 --- a/pkg/minikube/machine/stop.go +++ b/pkg/minikube/machine/stop.go @@ -80,7 +80,7 @@ func trySSHPowerOff(h *host.Host) error { out.T(out.Shutdown, `Powering off "{{.profile_name}}" via SSH ...`, out.V{"profile_name": h.Name}) // differnet for kic because RunSSHCommand is not implemented by kic if driver.IsKIC(h.DriverName) { - err := oci.ShutDown("sudo", h.DriverName, h.Name) + err := oci.ShutDown(h.DriverName, h.Name) glog.Infof("shutdown container: err=%v", err) } else { out, err := h.RunSSHCommand("sudo poweroff") diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index ada73686c0..6f47a34a1d 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -48,7 +48,7 @@ func init() { if err := registry.Register(registry.DriverDef{ Name: driver.Docker, Config: configure, - Init: func() drivers.Driver { return kic.NewDriver(kic.Config{OCIPrefix: oci.Env, OCIBinary: oci.Docker}) }, + Init: func() drivers.Driver { return kic.NewDriver(kic.Config{OCIBinary: oci.Docker}) }, Status: status, Priority: priority, }); err != nil { @@ -63,7 +63,6 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { ImageDigest: viper.GetString("base-image"), CPU: cc.CPUs, Memory: cc.Memory, - OCIPrefix: oci.Env, OCIBinary: oci.Docker, APIServerPort: cc.Nodes[0].Port, KubernetesVersion: cc.KubernetesConfig.KubernetesVersion, diff --git a/pkg/minikube/registry/drvs/podman/podman.go b/pkg/minikube/registry/drvs/podman/podman.go index ffde7a19b9..7f985110b9 100644 --- a/pkg/minikube/registry/drvs/podman/podman.go +++ b/pkg/minikube/registry/drvs/podman/podman.go @@ -51,7 +51,7 @@ func init() { if err := registry.Register(registry.DriverDef{ Name: driver.Podman, Config: configure, - Init: func() drivers.Driver { return kic.NewDriver(kic.Config{OCIPrefix: oci.Sudo, OCIBinary: oci.Podman}) }, + Init: func() drivers.Driver { return kic.NewDriver(kic.Config{OCIBinary: oci.Podman}) }, Status: status, Priority: priority, }); err != nil { @@ -67,7 +67,6 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { ImageDigest: strings.Split(baseImage, "@")[0], // for podman does not support docker images references with both a tag and digest. CPU: cc.CPUs, Memory: cc.Memory, - OCIPrefix: oci.Sudo, OCIBinary: oci.Podman, APIServerPort: cc.Nodes[0].Port, KubernetesVersion: cc.KubernetesConfig.KubernetesVersion, @@ -89,7 +88,7 @@ func status() registry.State { cmd := exec.CommandContext(ctx, oci.Podman, "version", "--format", "{{.Server.Version}}") // Run with sudo on linux (local), otherwise podman-remote (as podman) if runtime.GOOS == "linux" { - cmd = exec.CommandContext(ctx, oci.Sudo, "-n", oci.Podman, "version", "--format", "{{.Version}}") + cmd = exec.CommandContext(ctx, "sudo", "-n", oci.Podman, "version", "--format", "{{.Version}}") cmd.Env = append(os.Environ(), "LANG=C", "LC_ALL=C") // sudo is localized } o, err := cmd.Output() From e61a18cbaaab570fafb5a671e287748bc9346bcf Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 28 Apr 2020 10:10:02 -0700 Subject: [PATCH 097/105] be smarter about allocating memory for multinode clusters --- cmd/minikube/cmd/node_add.go | 6 ++++++ cmd/minikube/cmd/start.go | 4 ++++ pkg/minikube/node/start.go | 3 ++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/node_add.go b/cmd/minikube/cmd/node_add.go index 3344065794..07da8bf080 100644 --- a/cmd/minikube/cmd/node_add.go +++ b/cmd/minikube/cmd/node_add.go @@ -18,6 +18,7 @@ package cmd import ( "github.com/spf13/cobra" + "github.com/spf13/viper" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/exit" @@ -54,6 +55,11 @@ var nodeAddCmd = &cobra.Command{ KubernetesVersion: cc.KubernetesConfig.KubernetesVersion, } + // Make sure to decrease the default amount of memory we use per VM if this is the first worker node + if len(cc.Nodes) == 1 && viper.GetString(memory) == "" { + cc.Memory = 2200 + } + if err := node.Add(cc, n); err != nil { _, err := maybeDeleteAndRetry(*cc, n, nil, err) if err != nil { diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 4acabcc969..c9f1bb83d7 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -737,6 +737,10 @@ func suggestMemoryAllocation(sysLimit int, containerLimit int) int { // Suggest 25% of RAM, rounded to nearest 100MB. Hyper-V requires an even number! suggested := int(float32(sysLimit)/400.0) * 100 + if nodes := viper.GetInt(nodes); nodes > 1 { + suggested = suggested / nodes + } + if suggested > maximum { return maximum } diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index aa6a908e0b..9571557ea1 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -239,7 +239,8 @@ func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, k disableOthers = false } - // Preload is overly invasive for bare metal, and caching is not meaningful. KIC handled elsewhere. + // Preload is overly invasive for bare metal, and caching is not meaningful. + // KIC handles preload elsewhere. if driver.IsVM(cc.Driver) { if err := cr.Preload(cc.KubernetesConfig); err != nil { switch err.(type) { From b116cd15bb7265a0bb2ccf03b758e6da1c16e758 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 28 Apr 2020 10:18:33 -0700 Subject: [PATCH 098/105] lint --- cmd/minikube/cmd/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index c9f1bb83d7..319371396c 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -738,7 +738,7 @@ func suggestMemoryAllocation(sysLimit int, containerLimit int) int { suggested := int(float32(sysLimit)/400.0) * 100 if nodes := viper.GetInt(nodes); nodes > 1 { - suggested = suggested / nodes + suggested /= nodes } if suggested > maximum { From de5e493b1052ecdbf67b01ce8918b5310903ffce Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 28 Apr 2020 11:15:11 -0700 Subject: [PATCH 099/105] add unit tests --- cmd/minikube/cmd/start.go | 4 ++-- cmd/minikube/cmd/start_flags.go | 2 +- cmd/minikube/cmd/start_test.go | 37 ++++++++++++++++++++------------- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 319371396c..7a0faac6f3 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -715,7 +715,7 @@ func memoryLimits(drvName string) (int, int, error) { } // suggestMemoryAllocation calculates the default memory footprint in MB -func suggestMemoryAllocation(sysLimit int, containerLimit int) int { +func suggestMemoryAllocation(sysLimit int, containerLimit int, nodes int) int { if mem := viper.GetInt(memory); mem != 0 { return mem } @@ -737,7 +737,7 @@ func suggestMemoryAllocation(sysLimit int, containerLimit int) int { // Suggest 25% of RAM, rounded to nearest 100MB. Hyper-V requires an even number! suggested := int(float32(sysLimit)/400.0) * 100 - if nodes := viper.GetInt(nodes); nodes > 1 { + if nodes > 1 { suggested /= nodes } diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go index 3294036228..57417471c6 100644 --- a/cmd/minikube/cmd/start_flags.go +++ b/cmd/minikube/cmd/start_flags.go @@ -220,7 +220,7 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k glog.Warningf("Unable to query memory limits: %v", err) } - mem := suggestMemoryAllocation(sysLimit, containerLimit) + mem := suggestMemoryAllocation(sysLimit, containerLimit, viper.GetInt(nodes)) if cmd.Flags().Changed(memory) { mem, err = pkgutil.CalculateSizeInMB(viper.GetString(memory)) if err != nil { diff --git a/cmd/minikube/cmd/start_test.go b/cmd/minikube/cmd/start_test.go index 9c6dfa93a1..3e3e659702 100644 --- a/cmd/minikube/cmd/start_test.go +++ b/cmd/minikube/cmd/start_test.go @@ -185,25 +185,34 @@ func TestSuggestMemoryAllocation(t *testing.T) { description string sysLimit int containerLimit int + nodes int want int }{ - {"128GB sys", 128000, 0, 6000}, - {"64GB sys", 64000, 0, 6000}, - {"16GB sys", 16384, 0, 4000}, - {"odd sys", 14567, 0, 3600}, - {"4GB sys", 4096, 0, 2200}, - {"2GB sys", 2048, 0, 2048}, - {"Unable to poll sys", 0, 0, 2200}, - {"128GB sys, 16GB container", 128000, 16384, 16336}, - {"64GB sys, 16GB container", 64000, 16384, 16000}, - {"16GB sys, 4GB container", 16384, 4096, 4000}, - {"4GB sys, 3.5GB container", 16384, 3500, 3452}, - {"2GB sys, 2GB container", 16384, 2048, 2048}, - {"2GB sys, unable to poll container", 16384, 0, 4000}, + {"128GB sys", 128000, 0, 1, 6000}, + {"64GB sys", 64000, 0, 1, 6000}, + {"32GB sys", 32768, 0, 1, 6000}, + {"16GB sys", 16384, 0, 1, 4000}, + {"odd sys", 14567, 0, 1, 3600}, + {"4GB sys", 4096, 0, 1, 2200}, + {"2GB sys", 2048, 0, 1, 2048}, + {"Unable to poll sys", 0, 0, 1, 2200}, + {"128GB sys, 16GB container", 128000, 16384, 1, 16336}, + {"64GB sys, 16GB container", 64000, 16384, 1, 16000}, + {"16GB sys, 4GB container", 16384, 4096, 1, 4000}, + {"4GB sys, 3.5GB container", 16384, 3500, 1, 3452}, + {"16GB sys, 2GB container", 16384, 2048, 1, 2048}, + {"16GB sys, unable to poll container", 16384, 0, 1, 4000}, + {"128GB sys 2 nodes", 128000, 0, 2, 6000}, + {"8GB sys 3 nodes", 8192, 0, 3, 2200}, + {"16GB sys 2 nodes", 16384, 0, 2, 2200}, + {"32GB sys 2 nodes", 32768, 0, 2, 4050}, + {"odd sys 2 nodes", 14567, 0, 2, 2200}, + {"4GB sys 2 nodes", 4096, 0, 2, 2200}, + {"2GB sys 3 nodes", 2048, 0, 3, 2048}, } for _, test := range tests { t.Run(test.description, func(t *testing.T) { - got := suggestMemoryAllocation(test.sysLimit, test.containerLimit) + got := suggestMemoryAllocation(test.sysLimit, test.containerLimit, test.nodes) if got != test.want { t.Errorf("defaultMemorySize(sys=%d, container=%d) = %d, want: %d", test.sysLimit, test.containerLimit, got, test.want) } From 2d95ec92220a6647065127b7d8c015904e5a75a3 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 28 Apr 2020 11:42:51 -0700 Subject: [PATCH 100/105] check for multinode --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 7111d0e567..e8956b9140 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -226,10 +226,11 @@ func (k *Bootstrapper) init(cfg config.ClusterConfig) error { glog.Errorf("unable to create cluster role binding, some addons might not work: %v", err) } // the overlay is required for containerd and cri-o runtime: see #7428 - if err := k.applyKICOverlay(cfg); err != nil { - glog.Errorf("failed to apply kic overlay: %v", err) + if config.MultiNode(cfg) || (driver.IsKIC(cfg.Driver) && cfg.KubernetesConfig.ContainerRuntime != "docker") { + if err := k.applyKICOverlay(cfg); err != nil { + glog.Errorf("failed to apply kic overlay: %v", err) + } } - wg.Done() }() @@ -810,7 +811,7 @@ func kubectlPath(cfg config.ClusterConfig) string { // applyKICOverlay applies the CNI plugin needed to make kic work func (k *Bootstrapper) applyKICOverlay(cfg config.ClusterConfig) error { - fmt.Println("APPLYING KIC OVERLAY") + fmt.Println("APPLY KIC OVERLAY") b := bytes.Buffer{} if err := kicCNIConfig.Execute(&b, struct{ ImageName string }{ImageName: kic.OverlayImage}); err != nil { return err From 747b18a955307efbe548e949c06ee99b471d1053 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 28 Apr 2020 12:10:27 -0700 Subject: [PATCH 101/105] lint --- cmd/minikube/cmd/node_add.go | 4 +++- pkg/minikube/config/config.go | 9 ++++++--- pkg/minikube/config/profile.go | 4 +++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/cmd/minikube/cmd/node_add.go b/cmd/minikube/cmd/node_add.go index 5d668f7531..a2a243b3e9 100644 --- a/cmd/minikube/cmd/node_add.go +++ b/cmd/minikube/cmd/node_add.go @@ -63,7 +63,9 @@ var nodeAddCmd = &cobra.Command{ // Add CNI config if it's not already there // We need to run kubeadm.init here as well - config.MultiNodeCNIConfig(cc) + if err := config.MultiNodeCNIConfig(cc); err != nil { + exit.WithError("failed to save config", err) + } out.T(out.Ready, "Successfully added {{.name}} to {{.cluster}}!", out.V{"name": name, "cluster": cc.Name}) }, diff --git a/pkg/minikube/config/config.go b/pkg/minikube/config/config.go index facedad5a6..ccd0c7240d 100644 --- a/pkg/minikube/config/config.go +++ b/pkg/minikube/config/config.go @@ -203,13 +203,16 @@ func (c *simpleConfigLoader) WriteConfigToFile(profileName string, cc *ClusterCo } // MultiNodeCNIConfig add default CNI config needed for multinode clusters and saves off the config -func MultiNodeCNIConfig(cc *ClusterConfig) { +func MultiNodeCNIConfig(cc *ClusterConfig) error { if cc.KubernetesConfig.ExtraOptions.Get("pod-network-cidr", "kubeadm") == "" { fmt.Println("SETTING CNI CONFIG") cc.KubernetesConfig.NetworkPlugin = "cni" - cc.KubernetesConfig.ExtraOptions.Set(fmt.Sprintf("kubeadm.pod-network-cidr=%s", DefaultPodCIDR)) - SaveProfile(cc.Name, cc) + if err := cc.KubernetesConfig.ExtraOptions.Set(fmt.Sprintf("kubeadm.pod-network-cidr=%s", DefaultPodCIDR)); err != nil { + return err + } + return SaveProfile(cc.Name, cc) } + return nil } // MultiNode returns true if the cluster diff --git a/pkg/minikube/config/profile.go b/pkg/minikube/config/profile.go index 6a476eaf00..1b6b759f52 100644 --- a/pkg/minikube/config/profile.go +++ b/pkg/minikube/config/profile.go @@ -138,7 +138,9 @@ func SaveNode(cfg *ClusterConfig, node *Node) error { } if MultiNode(*cfg) { - MultiNodeCNIConfig(cfg) + if err := MultiNodeCNIConfig(cfg); err != nil { + return err + } } return SaveProfile(viper.GetString(ProfileName), cfg) From 19705a481e392ab7eca8f27b4b34142438c99703 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 28 Apr 2020 12:40:42 -0700 Subject: [PATCH 102/105] remove debugging --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 1 - pkg/minikube/config/config.go | 1 - pkg/minikube/driver/driver.go | 1 - 3 files changed, 3 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index e8956b9140..d50fbc50fc 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -811,7 +811,6 @@ func kubectlPath(cfg config.ClusterConfig) string { // applyKICOverlay applies the CNI plugin needed to make kic work func (k *Bootstrapper) applyKICOverlay(cfg config.ClusterConfig) error { - fmt.Println("APPLY KIC OVERLAY") b := bytes.Buffer{} if err := kicCNIConfig.Execute(&b, struct{ ImageName string }{ImageName: kic.OverlayImage}); err != nil { return err diff --git a/pkg/minikube/config/config.go b/pkg/minikube/config/config.go index ccd0c7240d..82cf57431d 100644 --- a/pkg/minikube/config/config.go +++ b/pkg/minikube/config/config.go @@ -205,7 +205,6 @@ func (c *simpleConfigLoader) WriteConfigToFile(profileName string, cc *ClusterCo // MultiNodeCNIConfig add default CNI config needed for multinode clusters and saves off the config func MultiNodeCNIConfig(cc *ClusterConfig) error { if cc.KubernetesConfig.ExtraOptions.Get("pod-network-cidr", "kubeadm") == "" { - fmt.Println("SETTING CNI CONFIG") cc.KubernetesConfig.NetworkPlugin = "cni" if err := cc.KubernetesConfig.ExtraOptions.Set(fmt.Sprintf("kubeadm.pod-network-cidr=%s", DefaultPodCIDR)); err != nil { return err diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index 4fcd44c059..074267c48c 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -162,7 +162,6 @@ func FlagDefaults(name string) FlagHints { fh := FlagHints{} if name != None { fh.CacheImages = true - // only for kic, until other runtimes are available we auto-set containerd. if name == Docker { fh.ExtraOptions = append(fh.ExtraOptions, fmt.Sprintf("kubeadm.pod-network-cidr=%s", config.DefaultPodCIDR)) } From 17d7c2a0f8778dd8d4c00d4fea4cf1f39d02e7ca Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 28 Apr 2020 12:45:01 -0700 Subject: [PATCH 103/105] finish comment --- pkg/minikube/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/config/config.go b/pkg/minikube/config/config.go index 82cf57431d..9e2e11a467 100644 --- a/pkg/minikube/config/config.go +++ b/pkg/minikube/config/config.go @@ -214,7 +214,7 @@ func MultiNodeCNIConfig(cc *ClusterConfig) error { return nil } -// MultiNode returns true if the cluster +// MultiNode returns true if the cluster has multiple nodes or if the request is asking for multinode func MultiNode(cc ClusterConfig) bool { if len(cc.Nodes) > 1 { return true From dc0ceeef2ced06cff70e45929ebc47ebf6879271 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 28 Apr 2020 17:06:24 -0700 Subject: [PATCH 104/105] debugging for multinode test --- test/integration/multinode_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go index 5a9cc4f16a..fce2410ea0 100644 --- a/test/integration/multinode_test.go +++ b/test/integration/multinode_test.go @@ -65,7 +65,7 @@ func validateMultiNodeStart(ctx context.Context, t *testing.T, profile string) { } // Make sure minikube status shows 2 nodes - rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "status")) + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "status", "--alsologtostderr")) if err != nil { t.Fatalf("failed to run minikube status. args %q : %v", rr.Command(), err) } @@ -89,7 +89,7 @@ func validateAddNodeToMultiNode(ctx context.Context, t *testing.T, profile strin } // Make sure minikube status shows 3 nodes - rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "status")) + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "status", "--alsologtostderr")) if err != nil { t.Fatalf("failed to run minikube status. args %q : %v", rr.Command(), err) } @@ -121,7 +121,7 @@ func validateStopRunningNode(ctx context.Context, t *testing.T, profile string) } // Make sure minikube status shows 2 running nodes and 1 stopped one - rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "status")) + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "status", "--alsologtostderr")) if err != nil && rr.ExitCode != 7 { t.Fatalf("failed to run minikube status. args %q : %v", rr.Command(), err) } @@ -177,7 +177,7 @@ func validateDeleteNodeFromMultiNode(ctx context.Context, t *testing.T, profile } // Make sure status is back down to 2 hosts - rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "status")) + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "status", "--alsologtostderr")) if err != nil { t.Fatalf("failed to run minikube status. args %q : %v", rr.Command(), err) } From 90cd6c36fbab239a9b1d51669a5859172cc6654d Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Tue, 28 Apr 2020 17:58:12 -0700 Subject: [PATCH 105/105] don't run multinode tests in parallel --- test/integration/multinode_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go index fce2410ea0..c4814cd415 100644 --- a/test/integration/multinode_test.go +++ b/test/integration/multinode_test.go @@ -29,7 +29,6 @@ func TestMultiNode(t *testing.T) { if NoneDriver() { t.Skip("none driver does not support multinode") } - MaybeParallel(t) type validatorFunc func(context.Context, *testing.T, string) profile := UniqueProfileName("multinode")