Merge pull request #6273 from tstromberg/cleaner-int-logs

Only grab post-mortem logs if VM is running

test/integration/helpers.go

@@ -33,6 +33,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/docker/machine/libmachine/state"
 	"github.com/shirou/gopsutil/process"
 	core "k8s.io/api/core/v1"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -177,31 +178,49 @@ func CleanupWithLogs(t *testing.T, profile string, cancel context.CancelFunc) {
 	t.Logf("*** %s FAILED at %s", t.Name(), time.Now())
 
 	if *postMortemLogs {
-		t.Logf(">>> %s FAILED: start of post-mortem logs >>>", t.Name())
-
-		rr, rerr := Run(t, exec.Command("kubectl", "--context", profile, "get", "po", "-A", "--show-labels"))
-		if rerr != nil {
-			t.Logf("%s: %v", rr.Command(), rerr)
-		}
-		t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout)
-
-		rr, err := Run(t, exec.Command("kubectl", "--context", profile, "describe", "node"))
-		if err != nil {
-			t.Logf("%s: %v", rr.Command(), err)
-		} else {
-			t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout)
-		}
-
-		rr, err = Run(t, exec.Command(Target(), "-p", profile, "logs", "--problems"))
-		if err != nil {
-			t.Logf("failed logs error: %v", err)
-		}
-		t.Logf("%s logs: %s", t.Name(), rr.Stdout)
-		t.Logf("<<< %s FAILED: end of post-mortem logs <<<", t.Name())
+		clusterLogs(t, profile)
 	}
 	Cleanup(t, profile, cancel)
 }
 
+// clusterLogs shows logs for debugging a failed cluster
+func clusterLogs(t *testing.T, profile string) {
+	st := Status(context.Background(), t, Target(), profile, "Host")
+	if st != state.Running.String() {
+		t.Logf("%q host is not running, skipping log retrieval (state=%q)", profile, st)
+		return
+	}
+
+	t.Logf("<<< %s FAILED: start of post-mortem logs <<<", t.Name())
+	rr, err := Run(t, exec.Command(Target(), "-p", profile, "logs", "--problems"))
+	if err != nil {
+		t.Logf("failed logs error: %v", err)
+		return
+	}
+	t.Logf("%s logs: %s", t.Name(), rr.Stdout)
+
+	st = Status(context.Background(), t, Target(), profile, "APIServer")
+	if st != state.Running.String() {
+		t.Logf("%q apiserver is not running, skipping kubectl commands (state=%q)", profile, st)
+		return
+	}
+
+	rr, rerr := Run(t, exec.Command("kubectl", "--context", profile, "get", "po", "-A", "--show-labels"))
+	if rerr != nil {
+		t.Logf("%s: %v", rr.Command(), rerr)
+		return
+	}
+	t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout)
+
+	rr, err = Run(t, exec.Command("kubectl", "--context", profile, "describe", "node"))
+	if err != nil {
+		t.Logf("%s: %v", rr.Command(), err)
+	} else {
+		t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout)
+	}
+	t.Logf("<<< %s FAILED: end of post-mortem logs <<<", t.Name())
+}
+
 // podStatusMsg returns a human-readable pod status, for generating debug status
 func podStatusMsg(pod core.Pod) string {
 	var sb strings.Builder
@@ -300,8 +319,26 @@ func PodWait(ctx context.Context, t *testing.T, profile string, ns string, selec
 	return names, fmt.Errorf("%s: %v", fmt.Sprintf("%s within %s", selector, timeout), err)
 }
 
+// Status returns a minikube component status as a string
+func Status(ctx context.Context, t *testing.T, path string, profile string, key string) string {
+	t.Helper()
+	// Reminder of useful keys: "Host", "Kubelet", "APIServer"
+	rr, err := Run(t, exec.CommandContext(ctx, path, "status", fmt.Sprintf("--format={{.%s}}", key), "-p", profile))
+	if err != nil {
+		t.Logf("status error: %v (may be ok)", err)
+	}
+	return strings.TrimSpace(rr.Stdout.String())
+}
+
 // showPodLogs logs debug info for pods
 func showPodLogs(ctx context.Context, t *testing.T, profile string, ns string, names []string) {
+	t.Helper()
+	st := Status(context.Background(), t, Target(), profile, "APIServer")
+	if st != state.Running.String() {
+		t.Logf("%q apiserver is not running, skipping kubectl commands (state=%q)", profile, st)
+		return
+	}
+
 	t.Logf("%s: showing logs for failed pods as of %s", t.Name(), time.Now())
 
 	for _, name := range names {

test/integration/start_stop_delete_test.go

@@ -36,16 +36,6 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
-// status returns a minikube component status as a string
-func status(ctx context.Context, t *testing.T, path string, profile string, key string) string {
-	t.Helper()
-	rr, err := Run(t, exec.CommandContext(ctx, path, "status", fmt.Sprintf("--format={{.%s}}", key), "-p", profile))
-	if err != nil {
-		t.Logf("status error: %v (may be ok)", err)
-	}
-	return strings.TrimSpace(rr.Stdout.String())
-}
-
 func TestStartStop(t *testing.T) {
 	MaybeParallel(t)
 
@@ -115,7 +105,7 @@ func TestStartStop(t *testing.T) {
 					t.Errorf("%s failed: %v", rr.Args, err)
 				}
 
-				got := status(ctx, t, Target(), profile, "Host")
+				got := Status(ctx, t, Target(), profile, "Host")
 				if got != state.Stopped.String() {
 					t.Errorf("status = %q; want = %q", got, state.Stopped)
 				}
@@ -132,7 +122,7 @@ func TestStartStop(t *testing.T) {
 					t.Fatalf("wait: %v", err)
 				}
 
-				got = status(ctx, t, Target(), profile, "Host")
+				got = Status(ctx, t, Target(), profile, "Host")
 				if got != state.Running.String() {
 					t.Errorf("host status = %q; want = %q", got, state.Running)
 				}
@@ -235,12 +225,12 @@ func testPause(ctx context.Context, t *testing.T, profile string) {
 		t.Fatalf("%s failed: %v", rr.Args, err)
 	}
 
-	got := status(ctx, t, Target(), profile, "APIServer")
+	got := Status(ctx, t, Target(), profile, "APIServer")
 	if got != state.Paused.String() {
 		t.Errorf("apiserver status = %q; want = %q", got, state.Paused)
 	}
 
-	got = status(ctx, t, Target(), profile, "Kubelet")
+	got = Status(ctx, t, Target(), profile, "Kubelet")
 	if got != state.Stopped.String() {
 		t.Errorf("kubelet status = %q; want = %q", got, state.Stopped)
 	}
@@ -250,12 +240,12 @@ func testPause(ctx context.Context, t *testing.T, profile string) {
 		t.Fatalf("%s failed: %v", rr.Args, err)
 	}
 
-	got = status(ctx, t, Target(), profile, "APIServer")
+	got = Status(ctx, t, Target(), profile, "APIServer")
 	if got != state.Running.String() {
 		t.Errorf("apiserver status = %q; want = %q", got, state.Running)
 	}
 
-	got = status(ctx, t, Target(), profile, "Kubelet")
+	got = Status(ctx, t, Target(), profile, "Kubelet")
 	if got != state.Running.String() {
 		t.Errorf("kubelet status = %q; want = %q", got, state.Running)
 	}