kubernetes
/
minikube
mirror of https://github.com/kubernetes/minikube.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #10332 from priyawadhwa/delete-timeout 

Add 5 minute timeout to deleting a profile
pull/10435/head
priyawadhwa 2021-02-10 10:40:32 -08:00 committed by GitHub
parent 859ec2a44b 6ed56d842e
commit 09de7ff62f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 47 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
cmd/minikube/cmd/delete.go
View File

@ -17,12 +17,14 @@ limitations under the License.
package cmd
import (
"context"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strconv"
"time"
"github.com/docker/machine/libmachine/mcnerror"
"github.com/mitchellh/go-ps"
@ -94,7 +96,7 @@ func init() {
}
// shotgun cleanup to delete orphaned docker container data
func deleteContainersAndVolumes(ociBin string) {
func deleteContainersAndVolumes(ctx context.Context, ociBin string) {
if _, err := exec.LookPath(ociBin); err != nil {
klog.Infof("skipping deleteContainersAndVolumes for %s: %v", ociBin, err)
return
@ -108,7 +110,7 @@ func deleteContainersAndVolumes(ociBin string) {
klog.Infof("error delete containers by label %q (might be okay): %+v", delLabel, errs)
}
errs = oci.DeleteAllVolumesByLabel(ociBin, delLabel)
errs = oci.DeleteAllVolumesByLabel(ctx, ociBin, delLabel)
if len(errs) > 0 { // it will not error if there is nothing to delete
klog.Warningf("error delete volumes by label %q (might be okay): %+v", delLabel, errs)
}
@ -118,7 +120,7 @@ func deleteContainersAndVolumes(ociBin string) {
return
}
errs = oci.PruneAllVolumesByLabel(ociBin, delLabel)
errs = oci.PruneAllVolumesByLabel(ctx, ociBin, delLabel)
if len(errs) > 0 { // it will not error if there is nothing to delete
klog.Warningf("error pruning volumes by label %q (might be okay): %+v", delLabel, errs)
}
@ -146,10 +148,12 @@ func runDelete(cmd *cobra.Command, args []string) {
}
exit.Message(reason.Usage, "Usage: minikube delete --all --purge")
}
delCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
if deleteAll {
deleteContainersAndVolumes(oci.Docker)
deleteContainersAndVolumes(oci.Podman)
deleteContainersAndVolumes(delCtx, oci.Docker)
deleteContainersAndVolumes(delCtx, oci.Podman)
errs := DeleteProfiles(profilesToDelete)
register.Reg.SetStep(register.Done)
@ -182,8 +186,8 @@ func runDelete(cmd *cobra.Command, args []string) {
if orphan {
// TODO: generalize for non-KIC drivers: #8040
deletePossibleKicLeftOver(cname, driver.Docker)
deletePossibleKicLeftOver(cname, driver.Podman)
deletePossibleKicLeftOver(delCtx, cname, driver.Docker)
deletePossibleKicLeftOver(delCtx, cname, driver.Podman)
}
}
@ -206,7 +210,9 @@ func DeleteProfiles(profiles []*config.Profile) []error {
klog.Infof("DeleteProfiles")
var errs []error
for _, profile := range profiles {
err := deleteProfile(profile)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
err := deleteProfile(ctx, profile)
if err != nil {
mm, loadErr := machine.LoadMachine(profile.Name)
@ -224,7 +230,7 @@ func DeleteProfiles(profiles []*config.Profile) []error {
}
// TODO: remove and/or move to delete package: #8040
func deletePossibleKicLeftOver(cname string, driverName string) {
func deletePossibleKicLeftOver(ctx context.Context, cname string, driverName string) {
bin := ""
switch driverName {
case driver.Docker:
@ -241,13 +247,12 @@ func deletePossibleKicLeftOver(cname string, driverName string) {
}
klog.Infof("deleting possible KIC leftovers for %s (driver=%s) ...", cname, driverName)
delLabel := fmt.Sprintf("%s=%s", oci.ProfileLabelKey, cname)
cs, err := oci.ListContainersByLabel(bin, delLabel)
cs, err := oci.ListContainersByLabel(ctx, bin, delLabel)
if err == nil && len(cs) > 0 {
for _, c := range cs {
out.Step(style.DeletingHost, `Deleting container "{{.name}}" ...`, out.V{"name": cname})
err := oci.DeleteContainer(bin, c)
err := oci.DeleteContainer(ctx, bin, c)
if err != nil { // it will error if there is no container to delete
klog.Errorf("error deleting container %q. You may want to delete it manually :\n%v", cname, err)
}
@ -255,7 +260,7 @@ func deletePossibleKicLeftOver(cname string, driverName string) {
}
}
errs := oci.DeleteAllVolumesByLabel(bin, delLabel)
errs := oci.DeleteAllVolumesByLabel(ctx, bin, delLabel)
if errs != nil { // it will not error if there is nothing to delete
klog.Warningf("error deleting volumes (might be okay).\nTo see the list of volumes run: 'docker volume ls'\n:%v", errs)
}
@ -270,13 +275,13 @@ func deletePossibleKicLeftOver(cname string, driverName string) {
return
}
errs = oci.PruneAllVolumesByLabel(bin, delLabel)
errs = oci.PruneAllVolumesByLabel(ctx, bin, delLabel)
if len(errs) > 0 { // it will not error if there is nothing to delete
klog.Warningf("error pruning volume (might be okay):\n%v", errs)
}
}
func deleteProfile(profile *config.Profile) error {
func deleteProfile(ctx context.Context, profile *config.Profile) error {
klog.Infof("Deleting %s", profile.Name)
register.Reg.SetStep(register.Deleting)
@ -289,7 +294,7 @@ func deleteProfile(profile *config.Profile) error {
out.Step(style.DeletingHost, `Deleting "{{.profile_name}}" in {{.driver_name}} ...`, out.V{"profile_name": profile.Name, "driver_name": profile.Config.Driver})
for _, n := range profile.Config.Nodes {
machineName := config.MachineName(*profile.Config, n)
deletePossibleKicLeftOver(machineName, profile.Config.Driver)
deletePossibleKicLeftOver(ctx, machineName, profile.Config.Driver)
}
}
} else {

7
cmd/minikube/cmd/node_delete.go
View File

@ -17,6 +17,9 @@ limitations under the License.
package cmd
import (
"context"
"time"
"github.com/spf13/cobra"
"k8s.io/minikube/pkg/minikube/config"
"k8s.io/minikube/pkg/minikube/driver"
@ -48,7 +51,9 @@ var nodeDeleteCmd = &cobra.Command{
if driver.IsKIC(co.Config.Driver) {
machineName := config.MachineName(*co.Config, *n)
deletePossibleKicLeftOver(machineName, co.Config.Driver)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
deletePossibleKicLeftOver(ctx, machineName, co.Config.Driver)
}
out.Step(style.Deleted, "Node {{.name}} was successfully deleted.", out.V{"name": name})

9
cmd/minikube/cmd/start.go
View File

@ -17,6 +17,7 @@ limitations under the License.
package cmd
import (
"context"
"encoding/json"
"fmt"
"math"
@ -130,7 +131,7 @@ func platform() string {
// runStart handles the executes the flow of "minikube start"
func runStart(cmd *cobra.Command, args []string) {
register.SetEventLogPath(localpath.EventLog(ClusterFlagValue()))
ctx := context.Background()
out.SetJSON(outputFormat == "json")
if err := pkgtrace.Initialize(viper.GetString(trace)); err != nil {
exit.Message(reason.Usage, "error initializing tracing: {{.Error}}", out.V{"Error": err.Error()})
@ -219,7 +220,7 @@ func runStart(cmd *cobra.Command, args []string) {
klog.Warningf("%s profile does not exist, trying anyways.", ClusterFlagValue())
}
err = deleteProfile(profile)
err = deleteProfile(ctx, profile)
if err != nil {
out.WarningT("Failed to delete cluster {{.name}}, proceeding with retry anyway.", out.V{"name": ClusterFlagValue()})
}
@ -482,7 +483,7 @@ func maybeDeleteAndRetry(cmd *cobra.Command, existing config.ClusterConfig, n co
out.ErrT(style.Meh, `"{{.name}}" profile does not exist, trying anyways.`, out.V{"name": existing.Name})
}
err = deleteProfile(profile)
err = deleteProfile(context.Background(), profile)
if err != nil {
out.WarningT("Failed to delete cluster {{.name}}, proceeding with retry anyway.", out.V{"name": existing.Name})
}
@ -697,7 +698,7 @@ func validateSpecifiedDriver(existing *config.ClusterConfig) {
out.ErrT(style.Meh, `"{{.name}}" profile does not exist, trying anyways.`, out.V{"name": existing.Name})
}
err = deleteProfile(profile)
err = deleteProfile(context.Background(), profile)
if err != nil {
out.WarningT("Failed to delete cluster {{.name}}.", out.V{"name": existing.Name})
}

5
pkg/drivers/kic/kic.go
View File

@ -72,6 +72,7 @@ func NewDriver(c Config) *Driver {
// Create a host using the driver's config
func (d *Driver) Create() error {
ctx := context.Background()
params := oci.CreateParams{
Mounts: d.NodeConfig.Mounts,
Name: d.NodeConfig.MachineName,
@ -136,7 +137,7 @@ func (d *Driver) Create() error {
// if container was created by minikube it is safe to delete and recreate it.
if oci.IsCreatedByMinikube(d.OCIBinary, params.Name) {
klog.Info("Found already existing abandoned minikube container, will try to delete.")
if err := oci.DeleteContainer(d.OCIBinary, params.Name); err != nil {
if err := oci.DeleteContainer(ctx, d.OCIBinary, params.Name); err != nil {
klog.Errorf("Failed to delete a conflicting minikube container %s. You might need to restart your %s daemon and delete it manually and try again: %v", params.Name, params.OCIBinary, err)
}
} else {
@ -338,7 +339,7 @@ func (d *Driver) Remove() error {
klog.Infof("could not find the container %s to remove it. will try anyways", d.MachineName)
}
if err := oci.DeleteContainer(d.NodeConfig.OCIBinary, d.MachineName); err != nil {
if err := oci.DeleteContainer(context.Background(), d.NodeConfig.OCIBinary, d.MachineName); err != nil {
if strings.Contains(err.Error(), "is already in progress") {
return errors.Wrap(err, "stuck delete")
}

14
pkg/drivers/kic/oci/oci.go
View File

@ -43,8 +43,8 @@ import (
// if there no containers found with the given label, it will return nil
func DeleteContainersByLabel(ociBin string, label string) []error {
var deleteErrs []error
cs, err := ListContainersByLabel(ociBin, label)
ctx := context.Background()
cs, err := ListContainersByLabel(ctx, ociBin, label)
if err != nil {
return []error{fmt.Errorf("listing containers by label %q", label)}
}
@ -75,7 +75,7 @@ func DeleteContainersByLabel(ociBin string, label string) []error {
}
// DeleteContainer deletes a container by ID or Name
func DeleteContainer(ociBin string, name string) error {
func DeleteContainer(ctx context.Context, ociBin string, name string) error {
_, err := ContainerStatus(ociBin, name)
if err == context.DeadlineExceeded {
out.WarningT("{{.ocibin}} is taking an unsually long time to respond, consider restarting {{.ocibin}}", out.V{"ociBin": ociBin})
@ -87,7 +87,7 @@ func DeleteContainer(ociBin string, name string) error {
klog.Infof("couldn't shut down %s (might be okay): %v ", name, err)
}
if _, err := runCmd(exec.Command(ociBin, "rm", "-f", "-v", name)); err != nil {
if _, err := runCmd(exec.CommandContext(ctx, ociBin, "rm", "-f", "-v", name)); err != nil {
return errors.Wrapf(err, "delete %s", name)
}
return nil
@ -373,7 +373,7 @@ func IsCreatedByMinikube(ociBin string, nameOrID string) bool {
// ListOwnedContainers lists all the containres that kic driver created on user's machine using a label
func ListOwnedContainers(ociBin string) ([]string, error) {
return ListContainersByLabel(ociBin, ProfileLabelKey)
return ListContainersByLabel(context.Background(), ociBin, ProfileLabelKey)
}
// inspect return low-level information on containers
@ -503,8 +503,8 @@ func withPortMappings(portMappings []PortMapping) createOpt {
}
// ListContainersByLabel returns all the container names with a specified label
func ListContainersByLabel(ociBin string, label string, warnSlow ...bool) ([]string, error) {
rr, err := runCmd(exec.Command(ociBin, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}"), warnSlow...)
func ListContainersByLabel(ctx context.Context, ociBin string, label string, warnSlow ...bool) ([]string, error) {
rr, err := runCmd(exec.CommandContext(ctx, ociBin, "ps", "-a", "--filter", fmt.Sprintf("label=%s", label), "--format", "{{.Names}}"), warnSlow...)
if err != nil {
return nil, err
}

9
pkg/drivers/kic/oci/volumes.go
View File

@ -19,6 +19,7 @@ package oci
import (
"bufio"
"bytes"
"context"
"fmt"
"os/exec"
"runtime"
@ -31,7 +32,7 @@ import (
// DeleteAllVolumesByLabel deletes all volumes that have a specific label
// if there is no volume to delete it will return nil
func DeleteAllVolumesByLabel(ociBin string, label string, warnSlow ...bool) []error {
func DeleteAllVolumesByLabel(ctx context.Context, ociBin string, label string, warnSlow ...bool) []error {
var deleteErrs []error
klog.Infof("trying to delete all %s volumes with label %s", ociBin, label)
@ -42,7 +43,7 @@ func DeleteAllVolumesByLabel(ociBin string, label string, warnSlow ...bool) []er
}
for _, v := range vs {
if _, err := runCmd(exec.Command(ociBin, "volume", "rm", "--force", v), warnSlow...); err != nil {
if _, err := runCmd(exec.CommandContext(ctx, ociBin, "volume", "rm", "--force", v), warnSlow...); err != nil {
deleteErrs = append(deleteErrs, fmt.Errorf("deleting %q", v))
}
}
@ -53,10 +54,10 @@ func DeleteAllVolumesByLabel(ociBin string, label string, warnSlow ...bool) []er
// PruneAllVolumesByLabel deletes all volumes that have a specific label
// if there is no volume to delete it will return nil
// example: docker volume prune -f --filter label=name.minikube.sigs.k8s.io=minikube
func PruneAllVolumesByLabel(ociBin string, label string, warnSlow ...bool) []error {
func PruneAllVolumesByLabel(ctx context.Context, ociBin string, label string, warnSlow ...bool) []error {
var deleteErrs []error
klog.Infof("trying to prune all %s volumes with label %s", ociBin, label)
cmd := exec.Command(ociBin, "volume", "prune", "-f", "--filter", "label="+label)
cmd := exec.CommandContext(ctx, ociBin, "volume", "prune", "-f", "--filter", "label="+label)
if _, err := runCmd(cmd, warnSlow...); err != nil {
deleteErrs = append(deleteErrs, errors.Wrapf(err, "prune volume by label %s", label))
}