kubernetes
/
minikube
mirror of https://github.com/kubernetes/minikube.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

adding postmortem logs for worker nodes in integration tests

pull/8238/head
Sharif Elgamal 2020-05-21 14:29:33 -07:00
parent 9fe9513c34
commit 70c228c2c3
3 changed files with 72 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
test/integration/helpers.go
View File

@ -191,7 +191,7 @@ func CleanupWithLogs(t *testing.T, profile string, cancel context.CancelFunc) {
}
// PostMortemLogs shows logs for debugging a failed cluster
func PostMortemLogs(t *testing.T, profile string, node ...string) {
func PostMortemLogs(t *testing.T, profile string, multinode ...bool) {
if !t.Failed() {
return
}
@ -201,57 +201,73 @@ func PostMortemLogs(t *testing.T, profile string, node ...string) {
return
}
m := false
if len(multinode) > 0 {
m = multinode[0]
}
nodes := []string{profile}
if m {
nodes = append(nodes, SecondNodeName, ThirdNodeName)
}
t.Logf("-----------------------post-mortem--------------------------------")
if DockerDriver() {
t.Logf("======> post-mortem[%s]: docker inspect <======", t.Name())
rr, err := Run(t, exec.Command("docker", "inspect", profile))
if err != nil {
t.Logf("failed to get docker inspect: %v", err)
} else {
t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output())
for _, n := range nodes {
machine := profile
if n != profile {
machine = fmt.Sprintf("%s-%s", profile, n)
}
if DockerDriver() {
t.Logf("======> post-mortem[%s]: docker inspect <======", t.Name())
rr, err := Run(t, exec.Command("docker", "inspect", machine))
if err != nil {
t.Logf("failed to get docker inspect: %v", err)
} else {
t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output())
}
}
}
st := Status(context.Background(), t, Target(), profile, "Host")
if st != state.Running.String() {
t.Logf("%q host is not running, skipping log retrieval (state=%q)", profile, st)
return
}
t.Logf("<<< %s FAILED: start of post-mortem logs <<<", t.Name())
t.Logf("======> post-mortem[%s]: minikube logs <======", t.Name())
st := Status(context.Background(), t, Target(), profile, "Host", n)
if st != state.Running.String() {
t.Logf("%q host is not running, skipping log retrieval (state=%q)", profile, st)
return
}
t.Logf("<<< %s FAILED: start of post-mortem logs <<<", t.Name())
t.Logf("======> post-mortem[%s]: minikube logs <======", t.Name())
rr, err := Run(t, exec.Command(Target(), "-p", profile, "logs", "-n", "25"))
if err != nil {
t.Logf("failed logs error: %v", err)
return
}
t.Logf("%s logs: %s", t.Name(), rr.Output())
rr, err := Run(t, exec.Command(Target(), "-p", profile, "logs", "-n", "25"))
if err != nil {
t.Logf("failed logs error: %v", err)
return
}
t.Logf("%s logs: %s", t.Name(), rr.Output())
st = Status(context.Background(), t, Target(), profile, "APIServer")
if st != state.Running.String() {
t.Logf("%q apiserver is not running, skipping kubectl commands (state=%q)", profile, st)
return
}
st = Status(context.Background(), t, Target(), profile, "APIServer", n)
if st != state.Running.String() {
t.Logf("%q apiserver is not running, skipping kubectl commands (state=%q)", profile, st)
return
}
// Get non-running pods. NOTE: This does not yet contain pods which are "running", but not "ready"
rr, rerr := Run(t, exec.Command("kubectl", "--context", profile, "get", "po", "-o=jsonpath={.items[*].metadata.name}", "-A", "--field-selector=status.phase!=Running"))
if rerr != nil {
t.Logf("%s: %v", rr.Command(), rerr)
return
}
notRunning := strings.Split(rr.Stdout.String(), " ")
t.Logf("non-running pods: %s", strings.Join(notRunning, " "))
// Get non-running pods. NOTE: This does not yet contain pods which are "running", but not "ready"
rr, rerr := Run(t, exec.Command("kubectl", "--context", profile, "get", "po", "-o=jsonpath={.items[*].metadata.name}", "-A", "--field-selector=status.phase!=Running"))
if rerr != nil {
t.Logf("%s: %v", rr.Command(), rerr)
return
}
notRunning := strings.Split(rr.Stdout.String(), " ")
t.Logf("non-running pods: %s", strings.Join(notRunning, " "))
t.Logf("======> post-mortem[%s]: describe non-running pods <======", t.Name())
t.Logf("======> post-mortem[%s]: describe non-running pods <======", t.Name())
args := append([]string{"--context", profile, "describe", "pod"}, notRunning...)
rr, rerr = Run(t, exec.Command("kubectl", args...))
if rerr != nil {
t.Logf("%s: %v", rr.Command(), rerr)
return
args := append([]string{"--context", profile, "describe", "pod"}, notRunning...)
rr, rerr = Run(t, exec.Command("kubectl", args...))
if rerr != nil {
t.Logf("%s: %v", rr.Command(), rerr)
return
}
t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output())
}
t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output())
t.Logf("<<< %s FAILED: end of post-mortem logs <<<", t.Name())
t.Logf("---------------------/post-mortem---------------------------------")
@ -356,10 +372,10 @@ func PodWait(ctx context.Context, t *testing.T, profile string, ns string, selec
}
// Status returns a minikube component status as a string
func Status(ctx context.Context, t *testing.T, path string, profile string, key string, node ...string) string {
func Status(ctx context.Context, t *testing.T, path string, profile string, key string, node string) string {
t.Helper()
// Reminder of useful keys: "Host", "Kubelet", "APIServer"
rr, err := Run(t, exec.CommandContext(ctx, path, "status", fmt.Sprintf("--format={{.%s}}", key), "-p", profile))
rr, err := Run(t, exec.CommandContext(ctx, path, "status", fmt.Sprintf("--format={{.%s}}", key), "-p", profile, "-n", node))
if err != nil {
t.Logf("status error: %v (may be ok)", err)
}
@ -369,7 +385,7 @@ func Status(ctx context.Context, t *testing.T, path string, profile string, key
// showPodLogs logs debug info for pods
func showPodLogs(ctx context.Context, t *testing.T, profile string, ns string, names []string) {
t.Helper()
st := Status(context.Background(), t, Target(), profile, "APIServer")
st := Status(context.Background(), t, Target(), profile, "APIServer", profile)
if st != state.Running.String() {
t.Logf("%q apiserver is not running, skipping kubectl commands (state=%q)", profile, st)
return

6
test/integration/main.go
View File

@ -40,6 +40,12 @@ var timeOutMultiplier = flag.Float64("timeout-multiplier", 1, "multiply the time
var binaryPath = flag.String("binary", "../../out/minikube", "path to minikube binary")
var testdataDir = flag.String("testdata-dir", "testdata", "the directory relative to test/integration where the testdata lives")
// Node names are consistent, let's store these for easy access later
const (
SecondNodeName = "m02"
ThirdNodeName = "m03"
)
// TestMain is the test main
func TestMain(m *testing.M) {
flag.Parse()

16
test/integration/multinode_test.go
View File

@ -50,6 +50,7 @@ func TestMultiNode(t *testing.T) {
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
defer PostMortemLogs(t, profile)
tc.validator(ctx, t, profile)
})
}
@ -104,11 +105,8 @@ func validateAddNodeToMultiNode(ctx context.Context, t *testing.T, profile strin
}
func validateStopRunningNode(ctx context.Context, t *testing.T, profile string) {
// Names are autogenerated using the node.Name() function
name := "m03"
// Run minikube node stop on that node
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "stop", name))
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "stop", ThirdNodeName))
if err != nil {
t.Errorf("node stop returned an error. args %q: %v", rr.Command(), err)
}
@ -143,11 +141,8 @@ func validateStartNodeAfterStop(ctx context.Context, t *testing.T, profile strin
// TODO (#7496): remove skip once restarts work
t.Skip("Restarting nodes is broken :(")
// Grab the stopped node
name := "m03"
// Start the node back up
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "start", name))
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "start", ThirdNodeName))
if err != nil {
t.Errorf("node start returned an error. args %q: %v", rr.Command(), err)
}
@ -168,10 +163,9 @@ func validateStartNodeAfterStop(ctx context.Context, t *testing.T, profile strin
}
func validateDeleteNodeFromMultiNode(ctx context.Context, t *testing.T, profile string) {
name := "m03"
// Start the node back up
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "delete", name))
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "delete", ThirdNodeName))
if err != nil {
t.Errorf("node stop returned an error. args %q: %v", rr.Command(), err)
}
@ -195,7 +189,7 @@ func validateDeleteNodeFromMultiNode(ctx context.Context, t *testing.T, profile
if err != nil {
t.Errorf("failed to run %q : %v", rr.Command(), err)
}
if strings.Contains(rr.Stdout.String(), fmt.Sprintf("%s-%s", profile, name)) {
if strings.Contains(rr.Stdout.String(), fmt.Sprintf("%s-%s", profile, ThirdNodeName)) {
t.Errorf("docker volume was not properly deleted: %s", rr.Stdout.String())
}
}