minikube/test/integration/util/minikube_runner.go

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/docker/machine/libmachine/state"
	"k8s.io/minikube/pkg/minikube/assets"
	commonutil "k8s.io/minikube/pkg/util"
)

// MinikubeRunner runs a command
type MinikubeRunner struct {
	Profile    string
	T          *testing.T
	BinaryPath string
	GlobalArgs string
	StartArgs  string
	MountArgs  string
	Runtime    string
}

// Copy copies a file
func (m *MinikubeRunner) Copy(f assets.CopyableFile) error {
	path, _ := filepath.Abs(m.BinaryPath)
	cmd := exec.Command("/bin/bash", "-c", path, "ssh", "--", fmt.Sprintf("cat >> %s", filepath.Join(f.GetTargetDir(), f.GetTargetName())))
	Logf("Running: %s", cmd.Args)
	return cmd.Run()
}

// Remove removes a file
func (m *MinikubeRunner) Remove(f assets.CopyableFile) error {
	_, err := m.SSH(fmt.Sprintf("rm -rf %s", filepath.Join(f.GetTargetDir(), f.GetTargetName())))
	return err
}

// teeRun runs a command, streaming stdout, stderr to console
func (m *MinikubeRunner) teeRun(cmd *exec.Cmd, wait ...bool) (string, string, error) {
	w := true
	if wait != nil {
		w = wait[0]
	}

	errPipe, err := cmd.StderrPipe()
	if err != nil {
		return "", "", err
	}
	outPipe, err := cmd.StdoutPipe()
	if err != nil {
		return "", "", err
	}

	if err := cmd.Start(); err != nil {
		return "", "", err
	}
	if w {
		var outB bytes.Buffer
		var errB bytes.Buffer
		var wg sync.WaitGroup
		wg.Add(2)
		go func() {
			if err := commonutil.TeePrefix(commonutil.ErrPrefix, errPipe, &errB, Logf); err != nil {
				m.T.Logf("tee: %v", err)
			}
			wg.Done()
		}()
		go func() {
			if err := commonutil.TeePrefix(commonutil.OutPrefix, outPipe, &outB, Logf); err != nil {
				m.T.Logf("tee: %v", err)
			}
			wg.Done()
		}()
		err = cmd.Wait()
		wg.Wait()
		return outB.String(), errB.String(), err

	}
	return "", "", err
}

// RunCommand executes a command, optionally checking for error
func (m *MinikubeRunner) RunCommand(cmdStr string, failError bool, wait ...bool) string {
	profileArg := fmt.Sprintf("-p=%s ", m.Profile)
	cmdStr = profileArg + cmdStr
	cmdArgs := strings.Split(cmdStr, " ")
	path, _ := filepath.Abs(m.BinaryPath)

	cmd := exec.Command(path, cmdArgs...)
	Logf("Run: %s", cmd.Args)
	stdout, stderr, err := m.teeRun(cmd, wait...)
	if failError && err != nil {
		if exitError, ok := err.(*exec.ExitError); ok {
			m.T.Fatalf("Error running command: %s %s. Output: %s", cmdStr, exitError.Stderr, stdout)
		} else {
			m.T.Fatalf("Error running command: %s %v. Output: %s", cmdStr, err, stderr)
		}
	}
	return stdout
}

// RunWithContext calls the minikube command with a context, useful for timeouts.
func (m *MinikubeRunner) RunWithContext(ctx context.Context, cmdStr string, wait ...bool) (string, string, error) {
	profileArg := fmt.Sprintf("-p=%s ", m.Profile)
	cmdStr = profileArg + cmdStr
	cmdArgs := strings.Split(cmdStr, " ")
	path, _ := filepath.Abs(m.BinaryPath)

	cmd := exec.CommandContext(ctx, path, cmdArgs...)
	Logf("Run: %s", cmd.Args)
	return m.teeRun(cmd, wait...)
}

// RunDaemon executes a command, returning the stdout
func (m *MinikubeRunner) RunDaemon(cmdStr string) (*exec.Cmd, *bufio.Reader) {
	profileArg := fmt.Sprintf("-p=%s ", m.Profile)
	cmdStr = profileArg + cmdStr
	cmdArgs := strings.Split(cmdStr, " ")
	path, _ := filepath.Abs(m.BinaryPath)

	cmd := exec.Command(path, cmdArgs...)
	stdoutPipe, err := cmd.StdoutPipe()
	if err != nil {
		m.T.Fatalf("stdout pipe failed: %s %v", cmdStr, err)
	}
	stderrPipe, err := cmd.StderrPipe()
	if err != nil {
		m.T.Fatalf("stderr pipe failed: %s %v", cmdStr, err)
	}

	var errB bytes.Buffer
	go func() {
		if err := commonutil.TeePrefix(commonutil.ErrPrefix, stderrPipe, &errB, Logf); err != nil {
			m.T.Logf("tee: %v", err)
		}
	}()

	err = cmd.Start()
	if err != nil {
		m.T.Fatalf("Error running command: %s %v", cmdStr, err)
	}
	return cmd, bufio.NewReader(stdoutPipe)

}

// RunDaemon2 executes a command, returning the stdout and stderr
func (m *MinikubeRunner) RunDaemon2(cmdStr string) (*exec.Cmd, *bufio.Reader, *bufio.Reader) {
	profileArg := fmt.Sprintf("-p=%s ", m.Profile)
	cmdStr = profileArg + cmdStr
	cmdArgs := strings.Split(cmdStr, " ")
	path, _ := filepath.Abs(m.BinaryPath)
	cmd := exec.Command(path, cmdArgs...)
	stdoutPipe, err := cmd.StdoutPipe()
	if err != nil {
		m.T.Fatalf("stdout pipe failed: %s %v", cmdStr, err)
	}
	stderrPipe, err := cmd.StderrPipe()
	if err != nil {
		m.T.Fatalf("stderr pipe failed: %s %v", cmdStr, err)
	}

	err = cmd.Start()
	if err != nil {
		m.T.Fatalf("Error running command: %s %v", cmdStr, err)
	}
	return cmd, bufio.NewReader(stdoutPipe), bufio.NewReader(stderrPipe)
}

// SSH returns the output of running a command using SSH
func (m *MinikubeRunner) SSH(cmdStr string) (string, error) {
	profileArg := fmt.Sprintf("-p=%s", m.Profile)
	path, _ := filepath.Abs(m.BinaryPath)

	cmd := exec.Command(path, profileArg, "ssh", cmdStr)
	Logf("SSH: %s", cmdStr)
	stdout, err := cmd.CombinedOutput()
	Logf("Output: %s", stdout)
	if err, ok := err.(*exec.ExitError); ok {
		return string(stdout), err
	}
	return string(stdout), nil
}

// Start starts the cluster
func (m *MinikubeRunner) Start(opts ...string) string {
	cmd := fmt.Sprintf("start %s %s %s --alsologtostderr --v=2", m.StartArgs, m.GlobalArgs, strings.Join(opts, " "))
	return m.RunCommand(cmd, true)
}

// StartWithStds starts the cluster with console output without verbose log
func (m *MinikubeRunner) StartWithStds(timeout time.Duration, opts ...string) (stdout string, stderr string, err error) {
	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()
	cmd := fmt.Sprintf("start %s %s %s", m.StartArgs, m.GlobalArgs, strings.Join(opts, " "))
	return m.RunWithContext(ctx, cmd)
}

// TearDown deletes minikube without waiting for it. used to free up ram/cpu after each test
func (m *MinikubeRunner) TearDown(t *testing.T) {
	profileArg := fmt.Sprintf("-p=%s", m.Profile)
	path, _ := filepath.Abs(m.BinaryPath)
	cmd := exec.Command(path, profileArg, "delete")
	err := cmd.Start()
	if err != nil {
		t.Errorf("error tearing down minikube %s : %v", profileArg, err)
	}
}

// EnsureRunning makes sure the container runtime is running
func (m *MinikubeRunner) EnsureRunning(opts ...string) {
	if m.GetStatus() != state.Running.String() {
		m.Start(opts...)
	}
	m.CheckStatus(state.Running.String())
}

// ParseEnvCmdOutput parses the output of `env` (assumes bash)
func (m *MinikubeRunner) ParseEnvCmdOutput(out string) map[string]string {
	env := map[string]string{}
	re := regexp.MustCompile(`(\w+?) ?= ?"?(.+?)"?\n`)
	for _, m := range re.FindAllStringSubmatch(out, -1) {
		env[m[1]] = m[2]
	}
	return env
}

// GetStatus returns the status of a service
func (m *MinikubeRunner) GetStatus() string {
	return m.RunCommand(fmt.Sprintf("status --format={{.Host}} %s", m.GlobalArgs), false)
}

// GetLogs returns the logs of a service
func (m *MinikubeRunner) GetLogs() string {
	return m.RunCommand(fmt.Sprintf("logs %s", m.GlobalArgs), true)
}

// CheckStatus makes sure the service has the desired status, or cause fatal error
func (m *MinikubeRunner) CheckStatus(desired string) {
	if err := m.CheckStatusNoFail(desired); err != nil {
		m.T.Fatalf("%v", err)
	}
}

// CheckStatusNoFail makes sure the service has the desired status, returning error
func (m *MinikubeRunner) CheckStatusNoFail(desired string) error {
	s := m.GetStatus()
	if s != desired {
		return fmt.Errorf("got state: %q, expected %q", s, desired)
	}
	return nil
}