diff --git a/cmd/minikube/cmd/mount.go b/cmd/minikube/cmd/mount.go index d5cdaf418a..e5f6abf453 100644 --- a/cmd/minikube/cmd/mount.go +++ b/cmd/minikube/cmd/mount.go @@ -107,7 +107,6 @@ var mountCmd = &cobra.Command{ exit.WithError("Error getting config", err) } host, err := api.Load(cc.Name) - if err != nil { exit.WithError("Error loading api", err) } diff --git a/cmd/minikube/cmd/pause.go b/cmd/minikube/cmd/pause.go new file mode 100644 index 0000000000..731c5f87d0 --- /dev/null +++ b/cmd/minikube/cmd/pause.go @@ -0,0 +1,67 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "github.com/spf13/cobra" + + "k8s.io/minikube/pkg/minikube/cluster" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/cruntime" + "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/machine" + "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/pause" +) + +// pauseCmd represents the docker-pause command +var pauseCmd = &cobra.Command{ + Use: "pause", + Short: "pause containers", + Run: func(cmd *cobra.Command, args []string) { + api, err := machine.NewAPIClient() + if err != nil { + exit.WithError("Error getting client", err) + } + defer api.Close() + cc, err := config.Load() + if err != nil { + exit.WithError("Error getting config", err) + } + host, err := cluster.CheckIfHostExistsAndLoad(api, cc.Name) + if err != nil { + exit.WithError("Error getting host", err) + } + + r, err := machine.CommandRunner(host) + if err != nil { + exit.WithError("Failed to get command runner", err) + } + + config := cruntime.Config{Type: cc.ContainerRuntime} + cr, err := cruntime.New(config) + if err != nil { + exit.WithError("Failed runtime", err) + } + + err = pause.Pause(cr, r) + if err != nil { + exit.WithError("Pause", err) + } + out.T(out.Pause, "The '{{.name}}' cluster is now paused", out.V{"name": cc.Name}) + }, +} diff --git a/cmd/minikube/cmd/root.go b/cmd/minikube/cmd/root.go index 38f84246d9..6c09694ebf 100644 --- a/cmd/minikube/cmd/root.go +++ b/cmd/minikube/cmd/root.go @@ -172,6 +172,8 @@ func init() { stopCmd, deleteCmd, dashboardCmd, + pauseCmd, + unpauseCmd, }, }, { diff --git a/cmd/minikube/cmd/unpause.go b/cmd/minikube/cmd/unpause.go new file mode 100644 index 0000000000..3e45eaf711 --- /dev/null +++ b/cmd/minikube/cmd/unpause.go @@ -0,0 +1,64 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "github.com/spf13/cobra" + + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/cruntime" + "k8s.io/minikube/pkg/minikube/exit" + "k8s.io/minikube/pkg/minikube/machine" + "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/pause" +) + +// unpauseCmd represents the docker-pause command +var unpauseCmd = &cobra.Command{ + Use: "unpause", + Short: "unpause Kubernetes", + Run: func(cmd *cobra.Command, args []string) { + api, err := machine.NewAPIClient() + if err != nil { + exit.WithError("Error getting client", err) + } + defer api.Close() + + cc, err := config.Load() + if err != nil { + exit.WithError("Error getting config", err) + } + host, err := api.Load(cc.Name) + + config := cruntime.Config{Type: cc.ContainerRuntime} + cr, err := cruntime.New(config) + if err != nil { + exit.WithError("Failed runtime", err) + } + + r, err := machine.CommandRunner(host) + if err != nil { + exit.WithError("Failed to get command runner", err) + } + + err = pause.Pause(cr, r) + if err != nil { + exit.WithError("Pause", err) + } + out.T(out.Pause, "The '{{.name}}' cluster is now paused", out.V{"name": cc.Name}) + }, +} diff --git a/go.sum b/go.sum index 1971629b40..f8c771f026 100644 --- a/go.sum +++ b/go.sum @@ -720,6 +720,7 @@ k8s.io/kubernetes v1.15.2 h1:RO9EuRw5vlN3oa/lnmPxmywOoJRtg9o40KcklHXNIAQ= k8s.io/kubernetes v1.15.2/go.mod h1:3RE5ikMc73WK+dSxk4pQuQ6ZaJcPXiZX2dj98RcdCuM= k8s.io/kubernetes/staging/src/k8s.io/api v0.0.0-20190623232353-8c3b7d7679cc h1:vZ5+77WP1yImZo23wc75vV5b5zCGq9gv484q8Yw5sBw= k8s.io/kubernetes/staging/src/k8s.io/api v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:pU9hbGZc8Z6+6HlNLEFY1GiNGzcCykU1Glsd4vEea2U= +k8s.io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20190623232353-8c3b7d7679cc h1:8L3YgoEmmOxIGWNv9Hj6WhJuUspT+sw4gJs2nEc0qI0= k8s.io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:Q49J/iUBV6A9nn8loyV72DK2EXhN8sqCR8FyfxIFDA4= k8s.io/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20190623232353-8c3b7d7679cc h1:SHxaBZWgNouwsZCVg2+iffu0Um1ExSLPKgvO1drWShs= k8s.io/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:rRBYbORqofLsn4/tsQWkeXkdKUoGrTfUwbI9s7NhU0Q= @@ -738,10 +739,12 @@ k8s.io/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20190623232353-8c3b7d7679cc/ k8s.io/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:5RWpGgZKzUcW9gCtmSVRq8maZkOetGv87HrohpTrnLI= k8s.io/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:ogOX4l9UCMfFGIF+FZqmsln4NtCGPqf9zTMCIlm2YX4= k8s.io/kubernetes/staging/src/k8s.io/kube-controller-manager v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:o6aAFW1lCnr5CJm1riWnhQskrAHhyt8btyv5UHhgZ6c= +k8s.io/kubernetes/staging/src/k8s.io/kube-proxy v0.0.0-20190623232353-8c3b7d7679cc h1:j30roBbl6b5Mom66efcNOHyjdYXU2RD8UWYnL0Adb8I= k8s.io/kubernetes/staging/src/k8s.io/kube-proxy v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:y0hpsQGN8h3HcNqYbpSZEH4yC1ohi45N35c8ma9yg6M= k8s.io/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:/hbCTKdfutEO2iTQv8NuYcnAxd8Tuu4mMEymYv/EZHk= k8s.io/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20190623232353-8c3b7d7679cc h1:Bsljf/3UDy91qqLkevAiq6y+wl0qJrkLjWfBCQs9rss= k8s.io/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:Vg6Q3IDU3hfYMICKyb43lClOXWtCtOBh2o1FfuQw8mQ= +k8s.io/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20190623232353-8c3b7d7679cc h1:ZUouIndlzPLGsRpeNAswxcs//fyODrNZOYybP6JZ9mM= k8s.io/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:9UInPlSttlDwZBFNMAsqhTtl7zH00dE2M88B9Z0Ennc= k8s.io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:xlTRb77uaXbuT6evILwFescWPMENFKYGYj3a/kOjYQE= k8s.io/kubernetes/staging/src/k8s.io/metrics v0.0.0-20190623232353-8c3b7d7679cc/go.mod h1:6Cs3k9ccbWbJo3CQOrGDu2QEVLwsWbBlu9HitjPhuSk= diff --git a/pkg/drivers/none/none.go b/pkg/drivers/none/none.go index 4d99fd8b8e..9546cee2d6 100644 --- a/pkg/drivers/none/none.go +++ b/pkg/drivers/none/none.go @@ -19,19 +19,17 @@ package none import ( "fmt" "os/exec" - "strings" - "time" "github.com/docker/machine/libmachine/drivers" "github.com/docker/machine/libmachine/state" "github.com/golang/glog" "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/net" + "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet" pkgdrivers "k8s.io/minikube/pkg/drivers" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/vmpath" - "k8s.io/minikube/pkg/util/retry" ) // cleanupPaths are paths to be removed by cleanup, and are used by both kubeadm and minikube. @@ -123,7 +121,7 @@ func (d *Driver) GetURL() (string, error) { // GetState returns the state that the host is in (running, stopped, etc) func (d *Driver) GetState() (state.State, error) { - if err := checkKubelet(d.exec); err != nil { + if err := kubelet.Check(d.exec); err != nil { glog.Infof("kubelet not running: %v", err) return state.Stopped, nil } @@ -132,7 +130,7 @@ func (d *Driver) GetState() (state.State, error) { // Kill stops a host forcefully, including any containers that we are managing. func (d *Driver) Kill() error { - if err := stopKubelet(d.exec); err != nil { + if err := kubelet.TryStopKubelet(d.exec); err != nil { return errors.Wrap(err, "kubelet") } @@ -177,7 +175,7 @@ func (d *Driver) Remove() error { // Restart a host func (d *Driver) Restart() error { - return restartKubelet(d.exec) + return kubelet.Restart(d.exec) } // Start a host @@ -196,7 +194,7 @@ func (d *Driver) Start() error { // Stop a host gracefully, including any containers that we are managing. func (d *Driver) Stop() error { - if err := stopKubelet(d.exec); err != nil { + if err := kubelet.Stop(d.exec); err != nil { return err } containers, err := d.runtime.ListContainers("") @@ -215,49 +213,3 @@ func (d *Driver) Stop() error { func (d *Driver) RunSSHCommandFromDriver() error { return fmt.Errorf("driver does not support ssh commands") } - -// stopKubelet idempotently stops the kubelet -func stopKubelet(cr command.Runner) error { - glog.Infof("stopping kubelet.service ...") - stop := func() error { - cmd := exec.Command("sudo", "systemctl", "stop", "kubelet.service") - if rr, err := cr.RunCmd(cmd); err != nil { - glog.Errorf("temporary error for %q : %v", rr.Command(), err) - } - cmd = exec.Command("sudo", "systemctl", "show", "-p", "SubState", "kubelet") - rr, err := cr.RunCmd(cmd) - if err != nil { - glog.Errorf("temporary error: for %q : %v", rr.Command(), err) - } - if !strings.Contains(rr.Stdout.String(), "dead") && !strings.Contains(rr.Stdout.String(), "failed") { - return fmt.Errorf("unexpected kubelet state: %q", rr.Stdout.String()) - } - return nil - } - - if err := retry.Expo(stop, 2*time.Second, time.Minute*3, 5); err != nil { - return errors.Wrapf(err, "error stopping kubelet") - } - - return nil -} - -// restartKubelet restarts the kubelet -func restartKubelet(cr command.Runner) error { - glog.Infof("restarting kubelet.service ...") - c := exec.Command("sudo", "systemctl", "restart", "kubelet.service") - if _, err := cr.RunCmd(c); err != nil { - return err - } - return nil -} - -// checkKubelet returns an error if the kubelet is not running. -func checkKubelet(cr command.Runner) error { - glog.Infof("checking for running kubelet ...") - c := exec.Command("systemctl", "is-active", "--quiet", "service", "kubelet") - if _, err := cr.RunCmd(c); err != nil { - return errors.Wrap(err, "check kubelet") - } - return nil -} diff --git a/pkg/minikube/config/profile.go.orig b/pkg/minikube/config/profile.go.orig new file mode 100644 index 0000000000..3546ec2e93 --- /dev/null +++ b/pkg/minikube/config/profile.go.orig @@ -0,0 +1,201 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/golang/glog" + "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/util/lock" +) + +var keywords = []string{"start", "stop", "status", "delete", "config", "open", "profile", "addons", "cache", "logs"} + +// IsValid checks if the profile has the essential info needed for a profile +func (p *Profile) IsValid() bool { + if p.Config == nil { + return false + } + if len(p.Config) == 0 { + return false + } + // This will become a loop for multinode + if p.Config[0] == nil { + return false + } + if p.Config[0].VMDriver == "" { + return false + } + if p.Config[0].KubernetesConfig.KubernetesVersion == "" { + return false + } + return true +} + +// ProfileNameInReservedKeywords checks if the profile is an internal keywords +func ProfileNameInReservedKeywords(name string) bool { + for _, v := range keywords { + if strings.EqualFold(v, name) { + return true + } + } + return false +} + +// ProfileExists returns true if there is a profile config (regardless of being valid) +func ProfileExists(name string, miniHome ...string) bool { + miniPath := localpath.MiniPath() + if len(miniHome) > 0 { + miniPath = miniHome[0] + } + + p := profileFilePath(name, miniPath) + _, err := os.Stat(p) + return err == nil +} + +// CreateEmptyProfile creates an empty profile stores in $MINIKUBE_HOME/profiles//config.json +func CreateEmptyProfile(name string, miniHome ...string) error { + cfg := &MachineConfig{} + return CreateProfile(name, cfg, miniHome...) +} + +// CreateProfile creates an profile out of the cfg and stores in $MINIKUBE_HOME/profiles//config.json +func CreateProfile(name string, cfg *MachineConfig, miniHome ...string) error { + data, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return err + } + path := profileFilePath(name, miniHome...) + glog.Infof("Saving config to %s ...", path) + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return err + } + + // If no config file exists, don't worry about swapping paths + if _, err := os.Stat(path); os.IsNotExist(err) { + if err := lock.WriteFile(path, data, 0600); err != nil { + return err + } + return nil + } + + tf, err := ioutil.TempFile(filepath.Dir(path), "config.json.tmp") + if err != nil { + return err + } + defer os.Remove(tf.Name()) + + if err = lock.WriteFile(tf.Name(), data, 0600); err != nil { + return err + } + + if err = tf.Close(); err != nil { + return err + } + + if err = os.Remove(path); err != nil { + return err + } + + if err = os.Rename(tf.Name(), path); err != nil { + return err + } + return nil +} + +// DeleteProfile deletes a profile and removes the profile dir +func DeleteProfile(profile string, miniHome ...string) error { + miniPath := localpath.MiniPath() + if len(miniHome) > 0 { + miniPath = miniHome[0] + } + return os.RemoveAll(ProfileFolderPath(profile, miniPath)) +} + +// ListProfiles returns all valid and invalid (if any) minikube profiles +// invalidPs are the profiles that have a directory or config file but not usable +// invalidPs would be suggested to be deleted +func ListProfiles(miniHome ...string) (validPs []*Profile, inValidPs []*Profile, err error) { + pDirs, err := profileDirs(miniHome...) + if err != nil { + return nil, nil, err + } + for _, n := range pDirs { + p, err := LoadProfile(n, miniHome...) + if err != nil { + inValidPs = append(inValidPs, p) + continue + } + if !p.IsValid() { + inValidPs = append(inValidPs, p) + continue + } + validPs = append(validPs, p) + } + return validPs, inValidPs, nil +} + +// LoadProfile loads type Profile based on its name +func LoadProfile(name string, miniHome ...string) (*Profile, error) { + cfg, err := DefaultLoader.LoadConfigFromFile(name, miniHome...) + p := &Profile{ + Name: name, + Config: []*MachineConfig{cfg}, + } + return p, err +} + +// profileDirs gets all the folders in the user's profiles folder regardless of valid or invalid config +func profileDirs(miniHome ...string) (dirs []string, err error) { + miniPath := localpath.MiniPath() + if len(miniHome) > 0 { + miniPath = miniHome[0] + } + pRootDir := filepath.Join(miniPath, "profiles") + items, err := ioutil.ReadDir(pRootDir) + for _, f := range items { + if f.IsDir() { + dirs = append(dirs, f.Name()) + } + } + return dirs, err +} + +// profileFilePath returns the Minikube profile config file +func profileFilePath(profile string, miniHome ...string) string { + miniPath := localpath.MiniPath() + if len(miniHome) > 0 { + miniPath = miniHome[0] + } + + return filepath.Join(miniPath, "profiles", profile, "config.json") +} + +// ProfileFolderPath returns path of profile folder +func ProfileFolderPath(profile string, miniHome ...string) string { + miniPath := localpath.MiniPath() + if len(miniHome) > 0 { + miniPath = miniHome[0] + } + return filepath.Join(miniPath, "profiles", profile) +} diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 934f65c416..e11d9f8ec5 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -32,6 +32,7 @@ import ( ) const ( + containerdNamespaceRoot = "/run/containerd/runc/k8s.io" // ContainerdConfFile is the path to the containerd configuration containerdConfigFile = "/etc/containerd/config.toml" containerdConfigTemplate = `root = "/var/lib/containerd" @@ -242,8 +243,18 @@ func (r *Containerd) KubeletOptions() map[string]string { } // ListContainers returns a list of managed by this container runtime -func (r *Containerd) ListContainers(filter string) ([]string, error) { - return listCRIContainers(r.Runner, filter) +func (r *Containerd) ListContainers(o ListOptions) ([]string, error) { + return listCRIContainers(r.Runner, containerdNamespaceRoot, o) +} + +// PauseContainers pauses a running container based on ID +func (r *Containerd) PauseContainers(ids []string) error { + return pauseCRIContainers(r.Runner, containerdNamespaceRoot, ids) +} + +// PauseContainers pauses a running container based on ID +func (r *Containerd) UnpauseContainers(ids []string) error { + return unpauseCRIContainers(r.Runner, containerdNamespaceRoot, ids) } // KillContainers removes containers based on ID diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go index 173e652301..ca77a58dab 100644 --- a/pkg/minikube/cruntime/cri.go +++ b/pkg/minikube/cruntime/cri.go @@ -18,7 +18,7 @@ package cruntime import ( "bytes" - "encoding/base64" + "encoding/json" "fmt" "html/template" "os/exec" @@ -27,331 +27,107 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" - "k8s.io/minikube/pkg/minikube/bootstrapper/images" - "k8s.io/minikube/pkg/minikube/command" ) -const ( - // CRIOConfFile is the path to the CRI-O configuration - crioConfigFile = "/etc/crio/crio.conf" - crioConfigTemplate = `# The CRI-O configuration file specifies all of the available configuration -# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime -# daemon, but in a TOML format that can be more easily modified and versioned. -# -# Please refer to crio.conf(5) for details of all configuration options. - -# CRI-O supports partial configuration reload during runtime, which can be -# done by sending SIGHUP to the running process. Currently supported options -# are explicitly mentioned with: 'This option supports live configuration -# reload'. - -# CRI-O reads its storage defaults from the containers-storage.conf(5) file -# located at /etc/containers/storage.conf. Modify this storage configuration if -# you want to change the system's defaults. If you want to modify storage just -# for CRI-O, you can change the storage configuration options here. -[crio] - -# Path to the "root directory". CRI-O stores all of its data, including -# containers images, in this directory. -root = "/var/lib/containers/storage" - -# Path to the "run directory". CRI-O stores all of its state in this directory. -runroot = "/var/run/containers/storage" - -# Storage driver used to manage the storage of images and containers. Please -# refer to containers-storage.conf(5) to see all available storage drivers. -storage_driver = "overlay" - -# List to pass options to the storage driver. Please refer to -# containers-storage.conf(5) to see all available storage options. -#storage_option = [ -#] - -# If set to false, in-memory locking will be used instead of file-based locking. -# **Deprecated** this option will be removed in the future. -file_locking = false - -# Path to the lock file. -# **Deprecated** this option will be removed in the future. -file_locking_path = "/run/crio.lock" - - -# The crio.api table contains settings for the kubelet/gRPC interface. -[crio.api] - -# Path to AF_LOCAL socket on which CRI-O will listen. -listen = "/var/run/crio/crio.sock" - -# IP address on which the stream server will listen. -stream_address = "127.0.0.1" - -# The port on which the stream server will listen. -stream_port = "0" - -# Enable encrypted TLS transport of the stream server. -stream_enable_tls = false - -# Path to the x509 certificate file used to serve the encrypted stream. This -# file can change, and CRI-O will automatically pick up the changes within 5 -# minutes. -stream_tls_cert = "" - -# Path to the key file used to serve the encrypted stream. This file can -# change, and CRI-O will automatically pick up the changes within 5 minutes. -stream_tls_key = "" - -# Path to the x509 CA(s) file used to verify and authenticate client -# communication with the encrypted stream. This file can change, and CRI-O will -# automatically pick up the changes within 5 minutes. -stream_tls_ca = "" - -# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024. -grpc_max_send_msg_size = 16777216 - -# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024. -grpc_max_recv_msg_size = 16777216 - -# The crio.runtime table contains settings pertaining to the OCI runtime used -# and options for how to set up and manage the OCI runtime. -[crio.runtime] - -# A list of ulimits to be set in containers by default, specified as -# "=:", for example: -# "nofile=1024:2048" -# If nothing is set here, settings will be inherited from the CRI-O daemon -#default_ulimits = [ -#] - -# default_runtime is the _name_ of the OCI runtime to be used as the default. -# The name is matched against the runtimes map below. -default_runtime = "runc" - -# If true, the runtime will not use pivot_root, but instead use MS_MOVE. -no_pivot = true - -# Path to the conmon binary, used for monitoring the OCI runtime. -conmon = "/usr/libexec/crio/conmon" - -# Cgroup setting for conmon -conmon_cgroup = "pod" - -# Environment variable list for the conmon process, used for passing necessary -# environment variables to conmon or the runtime. -conmon_env = [ - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", -] - -# If true, SELinux will be used for pod separation on the host. -selinux = false - -# Path to the seccomp.json profile which is used as the default seccomp profile -# for the runtime. If not specified, then the internal default seccomp profile -# will be used. -seccomp_profile = "" - -# Used to change the name of the default AppArmor profile of CRI-O. The default -# profile name is "crio-default-" followed by the version string of CRI-O. -apparmor_profile = "crio-default" - -# Cgroup management implementation used for the runtime. -cgroup_manager = "cgroupfs" - -# List of default capabilities for containers. If it is empty or commented out, -# only the capabilities defined in the containers json file by the user/kube -# will be added. -default_capabilities = [ - "CHOWN", - "DAC_OVERRIDE", - "FSETID", - "FOWNER", - "NET_RAW", - "SETGID", - "SETUID", - "SETPCAP", - "NET_BIND_SERVICE", - "SYS_CHROOT", - "KILL", -] - -# List of default sysctls. If it is empty or commented out, only the sysctls -# defined in the container json file by the user/kube will be added. -default_sysctls = [ -] - -# List of additional devices. specified as -# "::", for example: "--device=/dev/sdc:/dev/xvdc:rwm". -#If it is empty or commented out, only the devices -# defined in the container json file by the user/kube will be added. -additional_devices = [ -] - -# Path to OCI hooks directories for automatically executed hooks. -hooks_dir = [ -] - -# List of default mounts for each container. **Deprecated:** this option will -# be removed in future versions in favor of default_mounts_file. -default_mounts = [ -] - -# Path to the file specifying the defaults mounts for each container. The -# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads -# its default mounts from the following two files: -# -# 1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the -# override file, where users can either add in their own default mounts, or -# override the default mounts shipped with the package. -# -# 2) /usr/share/containers/mounts.conf: This is the default file read for -# mounts. If you want CRI-O to read from a different, specific mounts file, -# you can change the default_mounts_file. Note, if this is done, CRI-O will -# only add mounts it finds in this file. -# -#default_mounts_file = "" - -# Maximum number of processes allowed in a container. -pids_limit = 1024 - -# Maximum sized allowed for the container log file. Negative numbers indicate -# that no size limit is imposed. If it is positive, it must be >= 8192 to -# match/exceed conmon's read buffer. The file is truncated and re-opened so the -# limit is never exceeded. -log_size_max = -1 - -# Whether container output should be logged to journald in addition to the kuberentes log file -log_to_journald = false - -# Path to directory in which container exit files are written to by conmon. -container_exits_dir = "/var/run/crio/exits" - -# Path to directory for container attach sockets. -container_attach_socket_dir = "/var/run/crio" - -# If set to true, all containers will run in read-only mode. -read_only = false - -# Changes the verbosity of the logs based on the level it is set to. Options -# are fatal, panic, error, warn, info, and debug. This option supports live -# configuration reload. -log_level = "error" - -# The default log directory where all logs will go unless directly specified by the kubelet -log_dir = "/var/log/crio/pods" - -# The UID mappings for the user namespace of each container. A range is -# specified in the form containerUID:HostUID:Size. Multiple ranges must be -# separated by comma. -uid_mappings = "" - -# The GID mappings for the user namespace of each container. A range is -# specified in the form containerGID:HostGID:Size. Multiple ranges must be -# separated by comma. -gid_mappings = "" - -# The minimal amount of time in seconds to wait before issuing a timeout -# regarding the proper termination of the container. -ctr_stop_timeout = 0 - -# ManageNetworkNSLifecycle determines whether we pin and remove network namespace -# and manage its lifecycle. -manage_network_ns_lifecycle = false - -# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. -# The runtime to use is picked based on the runtime_handler provided by the CRI. -# If no runtime_handler is provided, the runtime will be picked based on the level -# of trust of the workload. - -[crio.runtime.runtimes.runc] -runtime_path = "/usr/bin/runc" -runtime_type = "oci" -runtime_root = "/run/runc" - - -# The crio.image table contains settings pertaining to the management of OCI images. -# -# CRI-O reads its configured registries defaults from the system wide -# containers-registries.conf(5) located in /etc/containers/registries.conf. If -# you want to modify just CRI-O, you can change the registries configuration in -# this file. Otherwise, leave insecure_registries and registries commented out to -# use the system's defaults from /etc/containers/registries.conf. -[crio.image] - -# Default transport for pulling images from a remote container storage. -default_transport = "docker://" - -# The path to a file containing credentials necessary for pulling images from -# secure registries. The file is similar to that of /var/lib/kubelet/config.json -global_auth_file = "" - -# The image used to instantiate infra containers. -# This option supports live configuration reload. -pause_image = "{{ .PodInfraContainerImage }}" - -# The path to a file containing credentials specific for pulling the pause_image from -# above. The file is similar to that of /var/lib/kubelet/config.json -# This option supports live configuration reload. -pause_image_auth_file = "" - -# The command to run to have a container stay in the paused state. -# This option supports live configuration reload. -pause_command = "/pause" - -# Path to the file which decides what sort of policy we use when deciding -# whether or not to trust an image that we've pulled. It is not recommended that -# this option be used, as the default behavior of using the system-wide default -# policy (i.e., /etc/containers/policy.json) is most often preferred. Please -# refer to containers-policy.json(5) for more details. -signature_policy = "" - -# Controls how image volumes are handled. The valid values are mkdir, bind and -# ignore; the latter will ignore volumes entirely. -image_volumes = "mkdir" - -# List of registries to be used when pulling an unqualified image (e.g., -# "alpine:latest"). By default, registries is set to "docker.io" for -# compatibility reasons. Depending on your workload and usecase you may add more -# registries (e.g., "quay.io", "registry.fedoraproject.org", -# "registry.opensuse.org", etc.). -registries = [ - "docker.io" -] - - -# The crio.network table containers settings pertaining to the management of -# CNI plugins. -[crio.network] - -# Path to the directory where CNI configuration files are located. -network_dir = "/etc/cni/net.d/" - -# Paths to directories where CNI plugin binaries are located. -plugin_dirs = [ - "/opt/cni/bin/", -] -` -) - -// listCRIContainers returns a list of containers using crictl -func listCRIContainers(cr CommandRunner, filter string) ([]string, error) { - var err error - var rr *command.RunResult - state := "Running" - if filter != "" { - c := exec.Command("sudo", "crictl", "ps", "-a", fmt.Sprintf("--name=%s", filter), fmt.Sprintf("--state=%s", state), "--quiet") - rr, err = cr.RunCmd(c) - } else { - rr, err = cr.RunCmd(exec.Command("sudo", "crictl", "ps", "-a", fmt.Sprintf("--state=%s", state), "--quiet")) +// container maps to 'runc list -f json' +type container struct { + pid int + id string + status string + annotations map[string]string +} + +// listCRIContainers returns a list of containers +func listCRIContainers(cr CommandRunner, root string, o ListOptions) ([]string, error) { + // First use crictl, because it reliably matches names + args := []string{"crictl", "ps", "--quiet"} + if o.State == All { + args = append(args, "-a") } + if o.Name != "" { + args = append(args, fmt.Sprintf("--name=%s", o.Name)) + } + rr, err := cr.RunCmd(exec.Command("sudo", args...)) if err != nil { return nil, err } + + // Avoid an id named "" var ids []string - for _, line := range strings.Split(rr.Stderr.String(), "\n") { - if line != "" { - ids = append(ids, line) + seen := map[string]bool{} + for _, id := range strings.Split(rr.Stderr.String(), "\n") { + if id != "" && !seen[id] { + ids = append(ids, id) + seen[id] = true } } - return ids, nil + + if len(ids) == 0 { + return nil, nil + } + if o.State == All { + return ids, nil + } + + // crictl does not understand paused pods + cs := []container{} + args = []string{"runc", "list", "-f", "json"} + if root != "" { + args = append(args, "--root", root) + } + + if _, err := cr.RunCmd(exec.Command("sudo", args...)); err != nil { + return nil, errors.Wrap(err, "runc") + } + + d := json.NewDecoder(bytes.NewReader(rr.Stdout.Bytes())) + if err := d.Decode(&cs); err != nil { + return nil, err + } + + var fids []string + for _, c := range cs { + if !seen[c.id] { + continue + } + if o.State.String() != c.status { + continue + } + fids = append(fids, c.id) + } + return fids, nil +} + +// pauseCRIContainers pauses a list of containers +func pauseCRIContainers(cr CommandRunner, root string, ids []string) error { + args := []string{"runc", "pause"} + if root != "" { + args = append(args, "--root", root) + } + + for _, id := range ids { + cargs := append(args, id) + if _, err := cr.RunCmd(exec.Command("sudo", cargs...)); err != nil { + return errors.Wrap(err, "runc") + } + } + return nil +} + +// unpauseCRIContainers pauses a list of containers +func unpauseCRIContainers(cr CommandRunner, root string, ids []string) error { + args := []string{"runc", "unpause"} + if root != "" { + args = append(args, "--root", root) + } + + for _, id := range ids { + cargs := append(args, id) + if _, err := cr.RunCmd(exec.Command("sudo", cargs...)); err != nil { + return errors.Wrap(err, "runc") + } + } + return nil } // criCRIContainers kills a list of containers using crictl @@ -364,7 +140,7 @@ func killCRIContainers(cr CommandRunner, ids []string) error { args := append([]string{"crictl", "rm"}, ids...) c := exec.Command("sudo", args...) if _, err := cr.RunCmd(c); err != nil { - return errors.Wrap(err, "kill cri containers.") + return errors.Wrap(err, "crictl") } return nil } @@ -378,10 +154,9 @@ func stopCRIContainers(cr CommandRunner, ids []string) error { args := append([]string{"crictl", "rm"}, ids...) c := exec.Command("sudo", args...) if _, err := cr.RunCmd(c); err != nil { - return errors.Wrap(err, "stop cri containers") + return errors.Wrap(err, "crictl") } return nil - } // populateCRIConfig sets up /etc/crictl.yaml @@ -406,27 +181,6 @@ image-endpoint: unix://{{.Socket}} return nil } -// generateCRIOConfig sets up /etc/crio/crio.conf -func generateCRIOConfig(cr CommandRunner, imageRepository string, k8sVersion string) error { - cPath := crioConfigFile - t, err := template.New("crio.conf").Parse(crioConfigTemplate) - if err != nil { - return err - } - pauseImage := images.PauseImage(imageRepository, k8sVersion) - opts := struct{ PodInfraContainerImage string }{PodInfraContainerImage: pauseImage} - var b bytes.Buffer - if err := t.Execute(&b, opts); err != nil { - return err - } - - c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | base64 -d | sudo tee %s", path.Dir(cPath), base64.StdEncoding.EncodeToString(b.Bytes()), cPath)) - if _, err := cr.RunCmd(c); err != nil { - return errors.Wrap(err, "generateCRIOConfig.") - } - return nil -} - // criContainerLogCmd returns the command to retrieve the log for a container based on ID func criContainerLogCmd(id string, len int, follow bool) string { var cmd strings.Builder diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 763e52ac73..e658494aaf 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -17,15 +17,320 @@ limitations under the License. package cruntime import ( + "bytes" + "encoding/base64" "fmt" "os/exec" + "path" "strings" + "text/template" "github.com/golang/glog" "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/out" ) +const ( + // CRIOConfFile is the path to the CRI-O configuration + crioConfigFile = "/etc/crio/crio.conf" + crioConfigTemplate = `# The CRI-O configuration file specifies all of the available configuration +# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime +# daemon, but in a TOML format that can be more easily modified and versioned. +# +# Please refer to crio.conf(5) for details of all configuration options. + +# CRI-O supports partial configuration reload during runtime, which can be +# done by sending SIGHUP to the running process. Currently supported options +# are explicitly mentioned with: 'This option supports live configuration +# reload'. + +# CRI-O reads its storage defaults from the containers-storage.conf(5) file +# located at /etc/containers/storage.conf. Modify this storage configuration if +# you want to change the system's defaults. If you want to modify storage just +# for CRI-O, you can change the storage configuration options here. +[crio] + +# Path to the "root directory". CRI-O stores all of its data, including +# containers images, in this directory. +root = "/var/lib/containers/storage" + +# Path to the "run directory". CRI-O stores all of its state in this directory. +runroot = "/var/run/containers/storage" + +# Storage driver used to manage the storage of images and containers. Please +# refer to containers-storage.conf(5) to see all available storage drivers. +storage_driver = "overlay" + +# List to pass options to the storage driver. Please refer to +# containers-storage.conf(5) to see all available storage options. +#storage_option = [ +#] + +# If set to false, in-memory locking will be used instead of file-based locking. +# **Deprecated** this option will be removed in the future. +file_locking = false + +# Path to the lock file. +# **Deprecated** this option will be removed in the future. +file_locking_path = "/run/crio.lock" + + +# The crio.api table contains settings for the kubelet/gRPC interface. +[crio.api] + +# Path to AF_LOCAL socket on which CRI-O will listen. +listen = "/var/run/crio/crio.sock" + +# IP address on which the stream server will listen. +stream_address = "127.0.0.1" + +# The port on which the stream server will listen. +stream_port = "0" + +# Enable encrypted TLS transport of the stream server. +stream_enable_tls = false + +# Path to the x509 certificate file used to serve the encrypted stream. This +# file can change, and CRI-O will automatically pick up the changes within 5 +# minutes. +stream_tls_cert = "" + +# Path to the key file used to serve the encrypted stream. This file can +# change, and CRI-O will automatically pick up the changes within 5 minutes. +stream_tls_key = "" + +# Path to the x509 CA(s) file used to verify and authenticate client +# communication with the encrypted stream. This file can change, and CRI-O will +# automatically pick up the changes within 5 minutes. +stream_tls_ca = "" + +# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024. +grpc_max_send_msg_size = 16777216 + +# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024. +grpc_max_recv_msg_size = 16777216 + +# The crio.runtime table contains settings pertaining to the OCI runtime used +# and options for how to set up and manage the OCI runtime. +[crio.runtime] + +# A list of ulimits to be set in containers by default, specified as +# "=:", for example: +# "nofile=1024:2048" +# If nothing is set here, settings will be inherited from the CRI-O daemon +#default_ulimits = [ +#] + +# default_runtime is the _name_ of the OCI runtime to be used as the default. +# The name is matched against the runtimes map below. +default_runtime = "runc" + +# If true, the runtime will not use pivot_root, but instead use MS_MOVE. +no_pivot = true + +# Path to the conmon binary, used for monitoring the OCI runtime. +conmon = "/usr/libexec/crio/conmon" + +# Cgroup setting for conmon +conmon_cgroup = "pod" + +# Environment variable list for the conmon process, used for passing necessary +# environment variables to conmon or the runtime. +conmon_env = [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +] + +# If true, SELinux will be used for pod separation on the host. +selinux = false + +# Path to the seccomp.json profile which is used as the default seccomp profile +# for the runtime. If not specified, then the internal default seccomp profile +# will be used. +seccomp_profile = "" + +# Used to change the name of the default AppArmor profile of CRI-O. The default +# profile name is "crio-default-" followed by the version string of CRI-O. +apparmor_profile = "crio-default" + +# Cgroup management implementation used for the runtime. +cgroup_manager = "cgroupfs" + +# List of default capabilities for containers. If it is empty or commented out, +# only the capabilities defined in the containers json file by the user/kube +# will be added. +default_capabilities = [ + "CHOWN", + "DAC_OVERRIDE", + "FSETID", + "FOWNER", + "NET_RAW", + "SETGID", + "SETUID", + "SETPCAP", + "NET_BIND_SERVICE", + "SYS_CHROOT", + "KILL", +] + +# List of default sysctls. If it is empty or commented out, only the sysctls +# defined in the container json file by the user/kube will be added. +default_sysctls = [ +] + +# List of additional devices. specified as +# "::", for example: "--device=/dev/sdc:/dev/xvdc:rwm". +#If it is empty or commented out, only the devices +# defined in the container json file by the user/kube will be added. +additional_devices = [ +] + +# Path to OCI hooks directories for automatically executed hooks. +hooks_dir = [ +] + +# List of default mounts for each container. **Deprecated:** this option will +# be removed in future versions in favor of default_mounts_file. +default_mounts = [ +] + +# Path to the file specifying the defaults mounts for each container. The +# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads +# its default mounts from the following two files: +# +# 1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the +# override file, where users can either add in their own default mounts, or +# override the default mounts shipped with the package. +# +# 2) /usr/share/containers/mounts.conf: This is the default file read for +# mounts. If you want CRI-O to read from a different, specific mounts file, +# you can change the default_mounts_file. Note, if this is done, CRI-O will +# only add mounts it finds in this file. +# +#default_mounts_file = "" + +# Maximum number of processes allowed in a container. +pids_limit = 1024 + +# Maximum sized allowed for the container log file. Negative numbers indicate +# that no size limit is imposed. If it is positive, it must be >= 8192 to +# match/exceed conmon's read buffer. The file is truncated and re-opened so the +# limit is never exceeded. +log_size_max = -1 + +# Whether container output should be logged to journald in addition to the kuberentes log file +log_to_journald = false + +# Path to directory in which container exit files are written to by conmon. +container_exits_dir = "/var/run/crio/exits" + +# Path to directory for container attach sockets. +container_attach_socket_dir = "/var/run/crio" + +# If set to true, all containers will run in read-only mode. +read_only = false + +# Changes the verbosity of the logs based on the level it is set to. Options +# are fatal, panic, error, warn, info, and debug. This option supports live +# configuration reload. +log_level = "error" + +# The default log directory where all logs will go unless directly specified by the kubelet +log_dir = "/var/log/crio/pods" + +# The UID mappings for the user namespace of each container. A range is +# specified in the form containerUID:HostUID:Size. Multiple ranges must be +# separated by comma. +uid_mappings = "" + +# The GID mappings for the user namespace of each container. A range is +# specified in the form containerGID:HostGID:Size. Multiple ranges must be +# separated by comma. +gid_mappings = "" + +# The minimal amount of time in seconds to wait before issuing a timeout +# regarding the proper termination of the container. +ctr_stop_timeout = 0 + +# ManageNetworkNSLifecycle determines whether we pin and remove network namespace +# and manage its lifecycle. +manage_network_ns_lifecycle = false + +# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes. +# The runtime to use is picked based on the runtime_handler provided by the CRI. +# If no runtime_handler is provided, the runtime will be picked based on the level +# of trust of the workload. + +[crio.runtime.runtimes.runc] +runtime_path = "/usr/bin/runc" +runtime_type = "oci" +runtime_root = "/run/runc" + + +# The crio.image table contains settings pertaining to the management of OCI images. +# +# CRI-O reads its configured registries defaults from the system wide +# containers-registries.conf(5) located in /etc/containers/registries.conf. If +# you want to modify just CRI-O, you can change the registries configuration in +# this file. Otherwise, leave insecure_registries and registries commented out to +# use the system's defaults from /etc/containers/registries.conf. +[crio.image] + +# Default transport for pulling images from a remote container storage. +default_transport = "docker://" + +# The path to a file containing credentials necessary for pulling images from +# secure registries. The file is similar to that of /var/lib/kubelet/config.json +global_auth_file = "" + +# The image used to instantiate infra containers. +# This option supports live configuration reload. +pause_image = "{{ .PodInfraContainerImage }}" + +# The path to a file containing credentials specific for pulling the pause_image from +# above. The file is similar to that of /var/lib/kubelet/config.json +# This option supports live configuration reload. +pause_image_auth_file = "" + +# The command to run to have a container stay in the paused state. +# This option supports live configuration reload. +pause_command = "/pause" + +# Path to the file which decides what sort of policy we use when deciding +# whether or not to trust an image that we've pulled. It is not recommended that +# this option be used, as the default behavior of using the system-wide default +# policy (i.e., /etc/containers/policy.json) is most often preferred. Please +# refer to containers-policy.json(5) for more details. +signature_policy = "" + +# Controls how image volumes are handled. The valid values are mkdir, bind and +# ignore; the latter will ignore volumes entirely. +image_volumes = "mkdir" + +# List of registries to be used when pulling an unqualified image (e.g., +# "alpine:latest"). By default, registries is set to "docker.io" for +# compatibility reasons. Depending on your workload and usecase you may add more +# registries (e.g., "quay.io", "registry.fedoraproject.org", +# "registry.opensuse.org", etc.). +registries = [ + "docker.io" +] + + +# The crio.network table containers settings pertaining to the management of +# CNI plugins. +[crio.network] + +# Path to the directory where CNI configuration files are located. +network_dir = "/etc/cni/net.d/" + +# Paths to directories where CNI plugin binaries are located. +plugin_dirs = [ + "/opt/cni/bin/", +] +` +) + // CRIO contains CRIO runtime state type CRIO struct { Socket string @@ -140,8 +445,18 @@ func (r *CRIO) KubeletOptions() map[string]string { } // ListContainers returns a list of managed by this container runtime -func (r *CRIO) ListContainers(filter string) ([]string, error) { - return listCRIContainers(r.Runner, filter) +func (r *CRIO) ListContainers(o ListOptions) ([]string, error) { + return listCRIContainers(r.Runner, "", o) +} + +// PauseContainers pauses a running container based on ID +func (r *CRIO) PauseContainers(ids []string) error { + return pauseCRIContainers(r.Runner, "", ids) +} + +// PauseContainers pauses a running container based on ID +func (r *CRIO) UnpauseContainers(ids []string) error { + return unpauseCRIContainers(r.Runner, "", ids) } // KillContainers removes containers based on ID @@ -163,3 +478,24 @@ func (r *CRIO) ContainerLogCmd(id string, len int, follow bool) string { func (r *CRIO) SystemLogCmd(len int) string { return fmt.Sprintf("sudo journalctl -u crio -n %d", len) } + +// generateCRIOConfig sets up /etc/crio/crio.conf +func generateCRIOConfig(cr CommandRunner, imageRepository string, k8sVersion string) error { + cPath := crioConfigFile + t, err := template.New("crio.conf").Parse(crioConfigTemplate) + if err != nil { + return err + } + pauseImage := images.PauseImage(imageRepository, k8sVersion) + opts := struct{ PodInfraContainerImage string }{PodInfraContainerImage: pauseImage} + var b bytes.Buffer + if err := t.Execute(&b, opts); err != nil { + return err + } + + c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | base64 -d | sudo tee %s", path.Dir(cPath), base64.StdEncoding.EncodeToString(b.Bytes()), cPath)) + if _, err := cr.RunCmd(c); err != nil { + return errors.Wrap(err, "generateCRIOConfig.") + } + return nil +} diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 6566a8d8cc..9ca3e078bf 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -27,6 +27,18 @@ import ( "k8s.io/minikube/pkg/minikube/out" ) +type ContainerState int + +const ( + All ContainerState = iota + Running + Paused +) + +func (cs ContainerState) String() string { + return [...]string{"All", "Running", "Paused"}[cs] +} + // CommandRunner is the subset of command.Runner this package consumes type CommandRunner interface { RunCmd(cmd *exec.Cmd) (*command.RunResult, error) @@ -60,11 +72,15 @@ type Manager interface { LoadImage(string) error // ListContainers returns a list of managed by this container runtime - ListContainers(string) ([]string, error) + ListContainers(ListOptions) ([]string, error) // KillContainers removes containers based on ID KillContainers([]string) error // StopContainers stops containers based on ID StopContainers([]string) error + // PauseContainers pauses containers based on ID + PauseContainers([]string) error + // UnpauseContainers unpauses containers based on ID + UnpauseContainers([]string) error // ContainerLogCmd returns the command to retrieve the log for a container based on ID ContainerLogCmd(string, int, bool) string // SystemLogCmd returns the command to return the system logs @@ -85,6 +101,13 @@ type Config struct { KubernetesVersion string } +type ListOptions struct { + // State is the container state to filter by (All, Running, Paused) + State ContainerState + // Name is a name filter + Name string +} + // New returns an appropriately configured runtime func New(c Config) (Manager, error) { switch c.Type { diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 3d80fca459..1bc2ab6920 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -121,11 +121,18 @@ func (r *Docker) KubeletOptions() map[string]string { } // ListContainers returns a list of containers -func (r *Docker) ListContainers(filter string) ([]string, error) { - filter = KubernetesContainerPrefix + filter - rr, err := r.Runner.RunCmd(exec.Command("docker", "ps", "-a", fmt.Sprintf("--filter=name=%s", filter), "--format=\"{{.ID}}\"")) +func (r *Docker) ListContainers(o ListOptions) ([]string, error) { + args := []string{"ps"} + switch o.State { + case All: + args = append(args, "-a") + case Paused: + args = append(args, "--filter status=paused") + } + args = append(args, fmt.Sprintf("--filter=name=%s", KubernetesContainerPrefix+o.Name), "--format=\"{{.ID}}\"") + rr, err := r.Runner.RunCmd(exec.Command("docker", args...)) if err != nil { - return nil, errors.Wrapf(err, "docker ListContainers. ") + return nil, errors.Wrapf(err, "docker") } var ids []string for _, line := range strings.Split(rr.Stdout.String(), "\n") { @@ -159,7 +166,35 @@ func (r *Docker) StopContainers(ids []string) error { args := append([]string{"stop"}, ids...) c := exec.Command("docker", args...) if _, err := r.Runner.RunCmd(c); err != nil { - return errors.Wrap(err, "stopping containers docker.") + return errors.Wrap(err, "docker") + } + return nil +} + +// PauseContainers pauses a running container based on ID +func (r *Docker) PauseContainers(ids []string) error { + if len(ids) == 0 { + return nil + } + glog.Infof("Pausing containers: %s", ids) + args := append([]string{"pause"}, ids...) + c := exec.Command("docker", args...) + if _, err := r.Runner.RunCmd(c); err != nil { + return errors.Wrap(err, "docker") + } + return nil +} + +// UnpauseContainers unpauses a container based on ID +func (r *Docker) UnpauseContainers(ids []string) error { + if len(ids) == 0 { + return nil + } + glog.Infof("Unpausing containers: %s", ids) + args := append([]string{"unpause"}, ids...) + c := exec.Command("docker", args...) + if _, err := r.Runner.RunCmd(c); err != nil { + return errors.Wrap(err, "docker") } return nil } diff --git a/pkg/minikube/kubelet/kubelet.go b/pkg/minikube/kubelet/kubelet.go new file mode 100644 index 0000000000..c8e0436a2f --- /dev/null +++ b/pkg/minikube/kubelet/kubelet.go @@ -0,0 +1,105 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubelet + +import ( + "fmt" + "os/exec" + "strings" + "time" + + "github.com/golang/glog" + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/command" + "k8s.io/minikube/pkg/util/retry" +) + +// Stop idempotently stops the kubelet +func Stop(cr command.Runner) error { + glog.Infof("stopping kubelet ...") + stop := func() error { + cmd := exec.Command("sudo", "systemctl", "stop", "kubelet.service") + if rr, err := cr.RunCmd(cmd); err != nil { + glog.Errorf("temporary error for %q : %v", rr.Command(), err) + } + cmd = exec.Command("sudo", "systemctl", "show", "-p", "SubState", "kubelet") + rr, err := cr.RunCmd(cmd) + if err != nil { + glog.Errorf("temporary error: for %q : %v", rr.Command(), err) + } + if !strings.Contains(rr.Stdout.String(), "dead") && !strings.Contains(rr.Stdout.String(), "failed") { + return fmt.Errorf("unexpected kubelet state: %q", rr.Stdout.String()) + } + return nil + } + + if err := retry.Expo(stop, 2*time.Second, time.Minute*3, 5); err != nil { + return errors.Wrapf(err, "error stopping kubelet") + } + + return nil +} + +// Start starts the kubelet +func Start(cr command.Runner) error { + glog.Infof("restarting kubelet.service ...") + c := exec.Command("sudo", "systemctl", "start", "kubelet") + if _, err := cr.RunCmd(c); err != nil { + return err + } + return nil +} + +// Restart restarts the kubelet +func Restart(cr command.Runner) error { + glog.Infof("restarting kubelet.service ...") + c := exec.Command("sudo", "systemctl", "restart", "kubelet.service") + if _, err := cr.RunCmd(c); err != nil { + return err + } + return nil +} + +// Check checks on the status of the kubelet +func Check(cr command.Runner) error { + glog.Infof("checking for running kubelet ...") + c := exec.Command("systemctl", "is-active", "--quiet", "service", "kubelet") + if _, err := cr.RunCmd(c); err != nil { + return errors.Wrap(err, "check kubelet") + } + return nil +} + +// Disable disables the Kubelet +func Disable(cr command.Runner) error { + glog.Infof("disabling kubelet ...") + c := exec.Command("systemctl", "disable", "kubelet") + if _, err := cr.RunCmd(c); err != nil { + return errors.Wrap(err, "disable") + } + return nil +} + +// Enable enables the Kubelet +func Enable(cr command.Runner) error { + glog.Infof("enabling kubelet ...") + c := exec.Command("systemctl", "enable", "kubelet") + if _, err := cr.RunCmd(c); err != nil { + return errors.Wrap(err, "enable") + } + return nil +} diff --git a/pkg/minikube/logs/logs.go b/pkg/minikube/logs/logs.go index f562302619..ff5dcdacf0 100644 --- a/pkg/minikube/logs/logs.go +++ b/pkg/minikube/logs/logs.go @@ -170,7 +170,7 @@ func Output(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner command.Run func logCommands(r cruntime.Manager, bs bootstrapper.Bootstrapper, length int, follow bool) map[string]string { cmds := bs.LogCommands(bootstrapper.LogOptions{Lines: length, Follow: follow}) for _, pod := range importantPods { - ids, err := r.ListContainers(pod) + ids, err := r.ListContainers(cruntime.ListOptions{Name: pod}) if err != nil { glog.Errorf("Failed to list containers for %q: %v", pod, err) continue diff --git a/pkg/minikube/out/style.go b/pkg/minikube/out/style.go index b14c9d5883..946ee16f01 100644 --- a/pkg/minikube/out/style.go +++ b/pkg/minikube/out/style.go @@ -81,6 +81,8 @@ var styles = map[StyleEnum]style{ Celebration: {Prefix: "πŸŽ‰ "}, Workaround: {Prefix: "πŸ‘‰ ", LowPrefix: lowIndent}, Sparkle: {Prefix: "✨ "}, + Pause: {Prefix: "⏸️ "}, + Unpause: {Prefix: "⏯️ "}, // Specialized purpose styles ISODownload: {Prefix: "πŸ’Ώ "}, diff --git a/pkg/minikube/out/style_enum.go b/pkg/minikube/out/style_enum.go index 0bc8ac1822..3787dbe904 100644 --- a/pkg/minikube/out/style_enum.go +++ b/pkg/minikube/out/style_enum.go @@ -84,4 +84,6 @@ const ( Empty Workaround Sparkle + Pause + Unpause ) diff --git a/pkg/minikube/pause/pause.go b/pkg/minikube/pause/pause.go new file mode 100644 index 0000000000..8800fabfef --- /dev/null +++ b/pkg/minikube/pause/pause.go @@ -0,0 +1,69 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pause + +import ( + "github.com/golang/glog" + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/command" + "k8s.io/minikube/pkg/minikube/cruntime" + "k8s.io/minikube/pkg/minikube/kubelet" +) + +// Pause pauses a Kubernetes cluster +func Pause(cr cruntime.Manager, r command.Runner) error { + if err := kubelet.Disable(r); err != nil { + return errors.Wrap(err, "kubelet disable") + } + if err := kubelet.Stop(r); err != nil { + return errors.Wrap(err, "kubelet stop") + } + ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Running}) + if err != nil { + return errors.Wrap(err, "list running") + } + if len(ids) == 0 { + glog.Warningf("no running containers to pause") + return nil + } + return cr.PauseContainers(ids) + +} + +// Unpause unpauses a Kubernetes cluster +func Unpause(cr cruntime.Manager, r command.Runner) error { + ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Paused}) + if err != nil { + return errors.Wrap(err, "list paused") + } + + if len(ids) == 0 { + glog.Warningf("no paused containers found") + } else { + if err := cr.UnpauseContainers(ids); err != nil { + return errors.Wrap(err, "unpause") + } + } + + if err := kubelet.Enable(r); err != nil { + return errors.Wrap(err, "kubelet enable") + } + if err := kubelet.Start(r); err != nil { + return errors.Wrap(err, "kubelet start") + } + return nil +}