Merge pull request #13630 from F1ko/master

Add sanity check to disk space check

pkg/minikube/constants/constants.go

@@ -95,8 +95,10 @@ const (
 	MinikubeActivePodmanEnv = "MINIKUBE_ACTIVE_PODMAN"
 	// MinikubeForceSystemdEnv is used to force systemd as cgroup manager for the container runtime
 	MinikubeForceSystemdEnv = "MINIKUBE_FORCE_SYSTEMD"
-	// TestDiskUsedEnv is used in integration tests for insufficient storage with 'minikube status'
+	// TestDiskUsedEnv is used in integration tests for insufficient storage with 'minikube status' (in %)
 	TestDiskUsedEnv = "MINIKUBE_TEST_STORAGE_CAPACITY"
+	// TestDiskAvailableEnv is used in integration tests for insufficient storage with 'minikube status' (in GiB)
+	TestDiskAvailableEnv = "MINIKUBE_TEST_AVAILABLE_STORAGE"
 
 	// scheduled stop constants
 

pkg/minikube/machine/start.go

@@ -34,6 +34,7 @@ import (
 	"github.com/docker/machine/libmachine/host"
 	"github.com/juju/mutex"
 	"github.com/pkg/errors"
+	"github.com/spf13/viper"
 	"k8s.io/klog/v2"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/minikube/command"
@@ -233,17 +234,36 @@ func postStartValidations(h *host.Host, drvName string) {
 		return
 	}
 
+	if viper.GetBool("force") {
+		return
+	}
+
 	// make sure /var isn't full,  as pod deployments will fail if it is
 	percentageFull, err := DiskUsed(r, "/var")
 	if err != nil {
 		klog.Warningf("error getting percentage of /var that is free: %v", err)
 	}
-	if percentageFull >= 99 {
+
-		exit.Message(kind, `{{.n}} is out of disk space! (/var is at {{.p}}% of capacity)`, out.V{"n": name, "p": percentageFull})
+	availableGiB, err := DiskAvailable(r, "/var")
+	if err != nil {
+		klog.Warningf("error getting GiB of /var that is available: %v", err)
+	}
+	const thresholdGiB = 20
+
+	if percentageFull >= 99 && availableGiB < thresholdGiB {
+		exit.Message(
+			kind,
+			`{{.n}} is out of disk space! (/var is at {{.p}}% of capacity). You can pass '--force' to skip this check.`,
+			out.V{"n": name, "p": percentageFull},
+		)
 	}
 
-	if percentageFull >= 85 {
+	if percentageFull >= 85 && availableGiB < thresholdGiB {
-		out.WarnReason(kind, `{{.n}} is nearly out of disk space, which may cause deployments to fail! ({{.p}}% of capacity)`, out.V{"n": name, "p": percentageFull})
+		out.WarnReason(
+			kind,
+			`{{.n}} is nearly out of disk space, which may cause deployments to fail! ({{.p}}% of capacity). You can pass '--force' to skip this check.`,
+			out.V{"n": name, "p": percentageFull},
+		)
 	}
 }
 
@@ -262,7 +282,22 @@ func DiskUsed(cr command.Runner, dir string) (int, error) {
 	return strconv.Atoi(percentage)
 }
 
-// postStart are functions shared between startHost and fixHost
+// DiskAvailable returns the available capacity of dir in the VM/container in GiB
+func DiskAvailable(cr command.Runner, dir string) (int, error) {
+	if s := os.Getenv(constants.TestDiskAvailableEnv); s != "" {
+		return strconv.Atoi(s)
+	}
+	output, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf("df -BG %s | awk 'NR==2{print $4}'", dir)))
+	if err != nil {
+		klog.Warningf("error running df -BG /var: %v\n%v", err, output.Output())
+		return 0, err
+	}
+	gib := strings.TrimSpace(output.Stdout.String())
+	gib = strings.Trim(gib, "G")
+	return strconv.Atoi(gib)
+}
+
+// postStartSetup are functions shared between startHost and fixHost
 func postStartSetup(h *host.Host, mc config.ClusterConfig) error {
 	klog.Infof("post-start starting for %q (driver=%q)", h.Name, h.DriverName)
 	start := time.Now()

pkg/minikube/machine/start_test.go

@@ -21,6 +21,8 @@ import (
 	"net"
 	"os"
 	"testing"
+
+	"k8s.io/minikube/pkg/minikube/command"
 )
 
 const initialEtcHostsContent string = `127.0.0.1	localhost
@@ -96,3 +98,100 @@ func writeContentToTempFile(content string) (string, error) {
 
 	return path, nil
 }
+
+func TestDiskUsed(t *testing.T) {
+	ex := command.NewFakeCommandRunner()
+	ex.SetCommandToOutput(map[string]string{
+		"sh -c \"df -h /var | awk 'NR==2{print $5}'\"": "20%",
+	})
+	nonex := command.NewFakeCommandRunner()
+	nonex.SetCommandToOutput(map[string]string{
+		"sh -c \"df -h /nonexistent | awk 'NR==2{print $5}'\"": "df: /nonexistent: No such file or directory",
+	})
+
+	type args struct {
+		cr  command.Runner
+		dir string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    int
+		wantErr bool
+	}{
+		{
+			name:    "existent",
+			args:    args{ex, "/var"},
+			want:    20,
+			wantErr: false,
+		},
+		{
+			name:    "existent",
+			args:    args{nonex, "/nonexistent"},
+			want:    0,
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Logf("starting %v", tt.name)
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := DiskUsed(tt.args.cr, tt.args.dir)
+			t.Logf("err: %v\n", err)
+			t.Logf("got: %v\n", got)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("DiskUsed() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if got != tt.want {
+				t.Errorf("DiskUsed() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestDiskAvailable(t *testing.T) {
+	ex := command.NewFakeCommandRunner()
+	ex.SetCommandToOutput(map[string]string{
+		"sh -c \"df -BG /var | awk 'NR==2{print $4}'\"": "20",
+	})
+	nonex := command.NewFakeCommandRunner()
+	nonex.SetCommandToOutput(map[string]string{
+		"sh -c \"df -BG /nonexistent | awk 'NR==2{print $4}'\"": "df: /nonexistent: No such file or directory",
+	})
+
+	type args struct {
+		cr  command.Runner
+		dir string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    int
+		wantErr bool
+	}{
+		{
+			name:    "existent",
+			args:    args{ex, "/var"},
+			want:    20,
+			wantErr: false,
+		},
+		{
+			name:    "existent",
+			args:    args{nonex, "/nonexistent"},
+			want:    0,
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Logf("starting %v", tt.name)
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := DiskAvailable(tt.args.cr, tt.args.dir)
+			t.Logf("err: %v\n", err)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("DiskAvailable() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if got != tt.want {
+				t.Errorf("DiskAvailable() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

test/integration/status_test.go

@@ -46,7 +46,7 @@ func TestInsufficientStorage(t *testing.T) {
 	startArgs = append(startArgs, StartArgs()...)
 	c := exec.CommandContext(ctx, Target(), startArgs...)
 	// artificially set /var to 100% capacity
-	c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestDiskUsedEnv))
+	c.Env = append(os.Environ(), fmt.Sprintf("%s=100", constants.TestDiskUsedEnv), fmt.Sprintf("%s=19", constants.TestDiskAvailableEnv))
 
 	rr, err := Run(t, c)
 	if err == nil {