Merge pull request #360 from dlorenc/pfix

Make the PersistenceTest more robust.

test/integration/persistence_test.go

@@ -46,29 +46,18 @@ func TestPersistence(t *testing.T) {
 		t.Fatalf("Error creating test pod: %s", err)
 	}
 
-	p := &api.Pod{}
-	for p.Status.Phase != "Running" {
-		p = kubectlRunner.GetPod(podName, podNamespace)
-		time.Sleep(time.Second)
-	}
-
-	// Now restart minikube and make sure the pod is still there.
-	minikubeRunner.RunCommand("stop", true)
-	minikubeRunner.CheckStatus("Stopped")
-
-	minikubeRunner.RunCommand("start", true)
-	minikubeRunner.CheckStatus("Running")
-
 	checkPod := func() error {
-		_, err := kubectlRunner.RunCommand(
-			[]string{"get", "pod", podName, "--namespace", podNamespace})
-		return err
+		p := kubectlRunner.GetPod(podName, podNamespace)
+		if util.IsPodReady(p) {
+			return nil
+		}
+		return fmt.Errorf("Pod %s is not ready yet.", podName)
 	}
-	if err := commonutil.RetryAfter(5, checkPod, 3*time.Second); err != nil {
+
+	if err := commonutil.RetryAfter(10, checkPod, 6*time.Second); err != nil {
 		t.Fatalf("Error checking the status of pod %s. Err: %s", podName, err)
 	}
 
-	// Now check the dashboard.
 	checkDashboard := func() error {
 		pods := api.PodList{}
 		cmd := []string{"get", "pods", "--namespace=kube-system", "--selector=app=kubernetes-dashboard"}
@@ -79,17 +68,30 @@ func TestPersistence(t *testing.T) {
 			return fmt.Errorf("No pods found matching query: %v", cmd)
 		}
 		db := pods.Items[0]
-		for _, cond := range db.Status.Conditions {
-			if cond.Type == "Ready" {
-				if cond.Status == "True" {
-					return nil
-				}
-				return fmt.Errorf("Dashboard pod not healthy. Healthy: %s. Reason: %s", cond.Status, cond.Reason)
-			}
+		if util.IsPodReady(&db) {
+			return nil
 		}
-		return fmt.Errorf("Unable to find healthy pod condition: %v", db.Status.Conditions)
+		return fmt.Errorf("Dashboard pod is not ready yet.")
 	}
 
+	// Make sure the dashboard is running before we stop the VM.
+	// On slow networks it can take several minutes to pull the addon-manager then the dashboard image.
+	if err := commonutil.RetryAfter(20, checkDashboard, 6*time.Second); err != nil {
+		t.Fatalf("Dashboard pod is not healthy: %s", err)
+	}
+
+	// Now restart minikube and make sure the pod is still there.
+	minikubeRunner.RunCommand("stop", true)
+	minikubeRunner.CheckStatus("Stopped")
+
+	minikubeRunner.RunCommand("start", true)
+	minikubeRunner.CheckStatus("Running")
+
+	if err := commonutil.RetryAfter(5, checkPod, 3*time.Second); err != nil {
+		t.Fatalf("Error checking the status of pod %s. Err: %s", podName, err)
+	}
+
+	// Now make sure it's still running after.
 	if err := commonutil.RetryAfter(5, checkDashboard, 3*time.Second); err != nil {
 		t.Fatalf("Dashboard pod is not healthy: %s", err)
 	}

test/integration/util/util.go

@@ -31,7 +31,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/docker/machine/libmachine/log"
+
 	"k8s.io/kubernetes/pkg/api"
+	commonutil "k8s.io/minikube/pkg/util"
 )
 
 type MinikubeRunner struct {
@@ -40,6 +43,20 @@ type MinikubeRunner struct {
 	Args       string
 }
 
+func IsPodReady(p *api.Pod) bool {
+	for _, cond := range p.Status.Conditions {
+		if cond.Type == "Ready" {
+			if cond.Status == "True" {
+				return true
+			}
+			log.Debugf("Pod %s not ready. Ready: %s. Reason: %s", p.Name, cond.Status, cond.Reason)
+			return false /**/
+		}
+	}
+	log.Debugf("Unable to find ready pod condition: %v", p.Status.Conditions)
+	return false
+}
+
 func (m *MinikubeRunner) RunCommand(command string, checkError bool) string {
 	commandArr := strings.Split(command, " ")
 	path, _ := filepath.Abs(m.BinaryPath)
@@ -120,19 +137,27 @@ func (k *KubectlRunner) RunCommandParseOutput(args []string, outputObj interface
 	return nil
 }
 
-func (k *KubectlRunner) RunCommand(args []string) ([]byte, error) {
-	cmd := exec.Command(k.BinaryPath, args...)
-	stdout, err := cmd.CombinedOutput()
-	if err != nil {
-		return make([]byte, 0), fmt.Errorf("Error running command. Error  %s. Output: %s", err, stdout)
+func (k *KubectlRunner) RunCommand(args []string) (stdout []byte, err error) {
+	inner := func() error {
+		cmd := exec.Command(k.BinaryPath, args...)
+		stdout, err = cmd.CombinedOutput()
+		if err != nil {
+			log.Errorf("Error %s running command %s. Return code: %s", stdout, args, err)
+			return fmt.Errorf("Error running command. Error  %s. Output: %s", err, stdout)
+		}
+		return nil
 	}
-	return stdout, nil
+
+	err = commonutil.RetryAfter(3, inner, 2*time.Second)
+	return stdout, err
 }
 
 func (k *KubectlRunner) CreateRandomNamespace() string {
 	const strLen = 20
 	name := genRandString(strLen)
-	k.RunCommand([]string{"create", "namespace", name})
+	if _, err := k.RunCommand([]string{"create", "namespace", name}); err != nil {
+		k.T.Fatalf("Error creating namespace: %s", err)
+	}
 	return name
 }