kubernetes
/
minikube
mirror of https://github.com/kubernetes/minikube.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'postmortem' of github.com:sharifelgamal/minikube into restart

pull/7973/head
Sharif Elgamal 2020-05-21 14:54:21 -07:00
parent c4122377c4 3fc412bcc2
commit bde6624365
14 changed files with 166 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -33,8 +33,6 @@ matrix:
script: make
after_success:
- bash <(curl -s https://codecov.io/bash)
travisBuddy:
regex: (FAIL:|\.go:\d+:|^panic:|failed$)
notifications:
webhooks:
urls:

2
cmd/minikube/cmd/node_start.go
View File

@ -40,7 +40,7 @@ var nodeStartCmd = &cobra.Command{
api, cc := mustload.Partial(ClusterFlagValue())
name := args[0]
n, _, err := node.Retrieve(cc, name)
n, _, err := node.Retrieve(*cc, name)
if err != nil {
exit.WithError("retrieving node", err)
}

2
cmd/minikube/cmd/node_stop.go
View File

@ -38,7 +38,7 @@ var nodeStopCmd = &cobra.Command{
name := args[0]
api, cc := mustload.Partial(ClusterFlagValue())
n, _, err := node.Retrieve(cc, name)
n, _, err := node.Retrieve(*cc, name)
if err != nil {
exit.WithError("retrieving node", err)
}

2
cmd/minikube/cmd/ssh.go
View File

@ -52,7 +52,7 @@ var sshCmd = &cobra.Command{
if nodeName == "" {
n = co.CP.Node
} else {
n, _, err = node.Retrieve(co.Config, nodeName)
n, _, err = node.Retrieve(*co.Config, nodeName)
if err != nil {
exit.WithCodeT(exit.Unavailable, "Node {{.nodeName}} does not exist.", out.V{"nodeName": nodeName})
}

32
cmd/minikube/cmd/status.go
View File

@ -37,6 +37,7 @@ import (
"k8s.io/minikube/pkg/minikube/kubeconfig"
"k8s.io/minikube/pkg/minikube/machine"
"k8s.io/minikube/pkg/minikube/mustload"
"k8s.io/minikube/pkg/minikube/node"
)
var statusFormat string
@ -105,19 +106,33 @@ var statusCmd = &cobra.Command{
api, cc := mustload.Partial(cname)
var statuses []*Status
for _, n := range cc.Nodes {
glog.Infof("checking status of %s ...", n.Name)
machineName := driver.MachineName(*cc, n)
st, err := status(api, *cc, n)
glog.Infof("%s status: %+v", machineName, st)
if nodeName != "" || statusFormat != defaultStatusFormat && len(cc.Nodes) > 1 {
n, _, err := node.Retrieve(*cc, nodeName)
if err != nil {
exit.WithError("retrieving node", err)
}
st, err := status(api, *cc, *n)
if err != nil {
glog.Errorf("status error: %v", err)
}
if st.Host == Nonexistent {
glog.Errorf("The %q host does not exist!", machineName)
}
statuses = append(statuses, st)
} else {
for _, n := range cc.Nodes {
glog.Infof("checking status of %s ...", n.Name)
machineName := driver.MachineName(*cc, n)
st, err := status(api, *cc, n)
glog.Infof("%s status: %+v", machineName, st)
if err != nil {
glog.Errorf("status error: %v", err)
}
if st.Host == Nonexistent {
glog.Errorf("The %q host does not exist!", machineName)
}
statuses = append(statuses, st)
}
}
switch strings.ToLower(output) {
@ -253,6 +268,7 @@ func init() {
For the list accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd#Status`)
statusCmd.Flags().StringVarP(&output, "output", "o", "text",
`minikube status --output OUTPUT. json, text`)
statusCmd.Flags().StringVarP(&nodeName, "node", "n", "", "The node to check status for. Defaults to control plane. Leave blank with default format for status on all nodes.")
}
func statusText(st *Status, w io.Writer) error {

8
pkg/minikube/image/image.go
View File

@ -103,6 +103,14 @@ func WriteImageToDaemon(img string) error {
glog.V(3).Infof("Getting image %v", ref)
i, err := remote.Image(ref)
if err != nil {
if strings.Contains(err.Error(), "GitHub Docker Registry needs login") {
ErrGithubNeedsLogin = errors.New(err.Error())
return ErrGithubNeedsLogin
} else if strings.Contains(err.Error(), "UNAUTHORIZED") {
ErrNeedsLogin = errors.New(err.Error())
return ErrNeedsLogin
}
return errors.Wrap(err, "getting remote image")
}
tag, err := name.NewTag(strings.Split(img, "@")[0])

23
pkg/minikube/image/types.go Normal file
View File

@ -0,0 +1,23 @@
/*
Copyright 2020 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package image
// ErrNeedsLogin is thrown when registry needs login (a general error)
var ErrNeedsLogin error
// ErrGithubNeedsLogin is thrown when user needs to login specifically to github packages)
var ErrGithubNeedsLogin error

20
pkg/minikube/node/cache.go
View File

@ -19,6 +19,7 @@ package node
import (
"os"
"runtime"
"strings"
"github.com/golang/glog"
"github.com/pkg/errors"
@ -126,8 +127,23 @@ func beginDownloadKicArtifacts(g *errgroup.Group, cc *config.ClusterConfig) {
// WaitDownloadKicArtifacts blocks until the required artifacts for KIC are downloaded.
func waitDownloadKicArtifacts(g *errgroup.Group) {
if err := g.Wait(); err != nil {
glog.Errorln("Error downloading kic artifacts: ", err)
return
if err != nil {
if errors.Is(err, image.ErrGithubNeedsLogin) {
glog.Warningf("Error downloading kic artifacts: %v", err)
out.ErrT(out.Connectivity, "Unfortunately, could not download the base image {{.image_name}} ", out.V{"image_name": strings.Split(kic.BaseImage, "@")[0]})
out.WarningT("In order to use the fall back image, you need to log in to the github packages registry")
out.T(out.Documentation, `Please visit the following link for documentation around this:
https://help.github.com/en/packages/using-github-packages-with-your-projects-ecosystem/configuring-docker-for-use-with-github-packages#authenticating-to-github-packages
`)
}
if errors.Is(err, image.ErrGithubNeedsLogin) || errors.Is(err, image.ErrNeedsLogin) {
exit.UsageT(`Please either authenticate to the registry or use --base-image flag to use a different registry.`)
} else {
glog.Errorln("Error downloading kic artifacts: ", err)
}
}
}
glog.Info("Successfully downloaded all kic artifacts")
}

12
pkg/minikube/node/node.go
View File

@ -19,6 +19,7 @@ package node
import (
"fmt"
"github.com/golang/glog"
"github.com/pkg/errors"
"github.com/spf13/viper"
@ -59,7 +60,7 @@ func Add(cc *config.ClusterConfig, n config.Node) error {
// Delete stops and deletes the given node from the given cluster
func Delete(cc config.ClusterConfig, name string) (*config.Node, error) {
n, index, err := Retrieve(&cc, name)
n, index, err := Retrieve(cc, name)
if err != nil {
return n, errors.Wrap(err, "retrieve")
}
@ -79,11 +80,18 @@ func Delete(cc config.ClusterConfig, name string) (*config.Node, error) {
}
// Retrieve finds the node by name in the given cluster
func Retrieve(cc *config.ClusterConfig, name string) (*config.Node, int, error) {
func Retrieve(cc config.ClusterConfig, name string) (*config.Node, int, error) {
for i, n := range cc.Nodes {
if n.Name == name {
return &n, i, nil
}
// Accept full machine name as well as just node name
if driver.MachineName(cc, n) == name {
glog.Infof("Couldn't find node name %s, but found it as a machine name, returning it anyway.", name)
return &n, i, nil
}
}
return nil, -1, errors.New("Could not find node " + name)

1
site/content/en/docs/commands/status.md
View File

@ -26,6 +26,7 @@ minikube status [flags]
-f, --format string Go template format string for the status output. The format for Go templates can be found here: https://golang.org/pkg/text/template/
For the list accessible variables for the template, see the struct values here: https://godoc.org/k8s.io/minikube/cmd/minikube/cmd#Status (default "{{.Name}}\ntype: Control Plane\nhost: {{.Host}}\nkubelet: {{.Kubelet}}\napiserver: {{.APIServer}}\nkubeconfig: {{.Kubeconfig}}\n\n")
-h, --help help for status
-n, --node string The node to check status for. Defaults to control plane. Leave blank with default format for status on all nodes.
-o, --output string minikube status --output OUTPUT. json, text (default "text")
```

109
test/integration/helpers.go
View File

@ -191,66 +191,83 @@ func CleanupWithLogs(t *testing.T, profile string, cancel context.CancelFunc) {
}
// PostMortemLogs shows logs for debugging a failed cluster
func PostMortemLogs(t *testing.T, profile string) {
func PostMortemLogs(t *testing.T, profile string, multinode ...bool) {
if !t.Failed() {
return
}
if !*postMortemLogs {
t.Logf("post-mortem logs disabled, oh-well!")
t.Logf("post-mortem logs disabled, oh well!")
return
}
m := false
if len(multinode) > 0 {
m = multinode[0]
}
nodes := []string{profile}
if m {
nodes = append(nodes, SecondNodeName, ThirdNodeName)
}
t.Logf("-----------------------post-mortem--------------------------------")
if DockerDriver() {
t.Logf("======> post-mortem[%s]: docker inspect <======", t.Name())
rr, err := Run(t, exec.Command("docker", "inspect", profile))
if err != nil {
t.Logf("failed to get docker inspect: %v", err)
} else {
t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output())
for _, n := range nodes {
machine := profile
if n != profile {
machine = fmt.Sprintf("%s-%s", profile, n)
}
if DockerDriver() {
t.Logf("======> post-mortem[%s]: docker inspect <======", t.Name())
rr, err := Run(t, exec.Command("docker", "inspect", machine))
if err != nil {
t.Logf("failed to get docker inspect: %v", err)
} else {
t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output())
}
}
}
st := Status(context.Background(), t, Target(), profile, "Host")
if st != state.Running.String() {
t.Logf("%q host is not running, skipping log retrieval (state=%q)", profile, st)
return
}
t.Logf("<<< %s FAILED: start of post-mortem logs <<<", t.Name())
t.Logf("======> post-mortem[%s]: minikube logs <======", t.Name())
st := Status(context.Background(), t, Target(), profile, "Host", n)
if st != state.Running.String() {
t.Logf("%q host is not running, skipping log retrieval (state=%q)", profile, st)
return
}
t.Logf("<<< %s FAILED: start of post-mortem logs <<<", t.Name())
t.Logf("======> post-mortem[%s]: minikube logs <======", t.Name())
rr, err := Run(t, exec.Command(Target(), "-p", profile, "logs", "-n", "25"))
if err != nil {
t.Logf("failed logs error: %v", err)
return
}
t.Logf("%s logs: %s", t.Name(), rr.Output())
rr, err := Run(t, exec.Command(Target(), "-p", profile, "logs", "-n", "25"))
if err != nil {
t.Logf("failed logs error: %v", err)
return
}
t.Logf("%s logs: %s", t.Name(), rr.Output())
st = Status(context.Background(), t, Target(), profile, "APIServer")
if st != state.Running.String() {
t.Logf("%q apiserver is not running, skipping kubectl commands (state=%q)", profile, st)
return
}
st = Status(context.Background(), t, Target(), profile, "APIServer", n)
if st != state.Running.String() {
t.Logf("%q apiserver is not running, skipping kubectl commands (state=%q)", profile, st)
return
}
// Get non-running pods. NOTE: This does not yet contain pods which are "running", but not "ready"
rr, rerr := Run(t, exec.Command("kubectl", "--context", profile, "get", "po", "-o=jsonpath={.items[*].metadata.name}", "-A", "--field-selector=status.phase!=Running"))
if rerr != nil {
t.Logf("%s: %v", rr.Command(), rerr)
return
}
notRunning := strings.Split(rr.Stdout.String(), " ")
t.Logf("non-running pods: %s", strings.Join(notRunning, " "))
// Get non-running pods. NOTE: This does not yet contain pods which are "running", but not "ready"
rr, rerr := Run(t, exec.Command("kubectl", "--context", profile, "get", "po", "-o=jsonpath={.items[*].metadata.name}", "-A", "--field-selector=status.phase!=Running"))
if rerr != nil {
t.Logf("%s: %v", rr.Command(), rerr)
return
}
notRunning := strings.Split(rr.Stdout.String(), " ")
t.Logf("non-running pods: %s", strings.Join(notRunning, " "))
t.Logf("======> post-mortem[%s]: describe non-running pods <======", t.Name())
t.Logf("======> post-mortem[%s]: describe non-running pods <======", t.Name())
args := append([]string{"--context", profile, "describe", "pod"}, notRunning...)
rr, rerr = Run(t, exec.Command("kubectl", args...))
if rerr != nil {
t.Logf("%s: %v", rr.Command(), rerr)
return
args := append([]string{"--context", profile, "describe", "pod"}, notRunning...)
rr, rerr = Run(t, exec.Command("kubectl", args...))
if rerr != nil {
t.Logf("%s: %v", rr.Command(), rerr)
return
}
t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output())
}
t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output())
t.Logf("<<< %s FAILED: end of post-mortem logs <<<", t.Name())
t.Logf("---------------------/post-mortem---------------------------------")
@ -355,10 +372,10 @@ func PodWait(ctx context.Context, t *testing.T, profile string, ns string, selec
}
// Status returns a minikube component status as a string
func Status(ctx context.Context, t *testing.T, path string, profile string, key string) string {
func Status(ctx context.Context, t *testing.T, path string, profile string, key string, node string) string {
t.Helper()
// Reminder of useful keys: "Host", "Kubelet", "APIServer"
rr, err := Run(t, exec.CommandContext(ctx, path, "status", fmt.Sprintf("--format={{.%s}}", key), "-p", profile))
rr, err := Run(t, exec.CommandContext(ctx, path, "status", fmt.Sprintf("--format={{.%s}}", key), "-p", profile, "-n", node))
if err != nil {
t.Logf("status error: %v (may be ok)", err)
}
@ -368,7 +385,7 @@ func Status(ctx context.Context, t *testing.T, path string, profile string, key
// showPodLogs logs debug info for pods
func showPodLogs(ctx context.Context, t *testing.T, profile string, ns string, names []string) {
t.Helper()
st := Status(context.Background(), t, Target(), profile, "APIServer")
st := Status(context.Background(), t, Target(), profile, "APIServer", profile)
if st != state.Running.String() {
t.Logf("%q apiserver is not running, skipping kubectl commands (state=%q)", profile, st)
return

6
test/integration/main.go
View File

@ -40,6 +40,12 @@ var timeOutMultiplier = flag.Float64("timeout-multiplier", 1, "multiply the time
var binaryPath = flag.String("binary", "../../out/minikube", "path to minikube binary")
var testdataDir = flag.String("testdata-dir", "testdata", "the directory relative to test/integration where the testdata lives")
// Node names are consistent, let's store these for easy access later
const (
SecondNodeName = "m02"
ThirdNodeName = "m03"
)
// TestMain is the test main
func TestMain(m *testing.M) {
flag.Parse()

15
test/integration/multinode_test.go
View File

@ -105,11 +105,8 @@ func validateAddNodeToMultiNode(ctx context.Context, t *testing.T, profile strin
}
func validateStopRunningNode(ctx context.Context, t *testing.T, profile string) {
// Names are autogenerated using the node.Name() function
name := "m03"
// Run minikube node stop on that node
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "stop", name))
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "stop", ThirdNodeName))
if err != nil {
t.Errorf("node stop returned an error. args %q: %v", rr.Command(), err)
}
@ -151,11 +148,8 @@ func validateStartNodeAfterStop(ctx context.Context, t *testing.T, profile strin
}
}
// Grab the stopped node
name := "m03"
// Start the node back up
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "start", name))
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "start", ThirdNodeName))
if err != nil {
t.Errorf("node start returned an error. args %q: %v", rr.Command(), err)
}
@ -182,10 +176,9 @@ func validateStartNodeAfterStop(ctx context.Context, t *testing.T, profile strin
}
func validateDeleteNodeFromMultiNode(ctx context.Context, t *testing.T, profile string) {
name := "m03"
// Start the node back up
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "delete", name))
rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "delete", ThirdNodeName))
if err != nil {
t.Errorf("node stop returned an error. args %q: %v", rr.Command(), err)
}
@ -209,7 +202,7 @@ func validateDeleteNodeFromMultiNode(ctx context.Context, t *testing.T, profile
if err != nil {
t.Errorf("failed to run %q : %v", rr.Command(), err)
}
if strings.Contains(rr.Stdout.String(), fmt.Sprintf("%s-%s", profile, name)) {
if strings.Contains(rr.Stdout.String(), fmt.Sprintf("%s-%s", profile, ThirdNodeName)) {
t.Errorf("docker volume was not properly deleted: %s", rr.Stdout.String())
}
}

12
test/integration/start_stop_delete_test.go
View File

@ -171,7 +171,7 @@ func validateEnableAddonAfterStop(ctx context.Context, t *testing.T, profile str
defer PostMortemLogs(t, profile)
// The none driver never really stops
if !NoneDriver() {
got := Status(ctx, t, Target(), profile, "Host")
got := Status(ctx, t, Target(), profile, "Host", profile)
if got != state.Stopped.String() {
t.Errorf("expected post-stop host status to be -%q- but got *%q*", state.Stopped, got)
}
@ -193,7 +193,7 @@ func validateSecondStart(ctx context.Context, t *testing.T, profile string, tcNa
t.Fatalf("failed to start minikube post-stop. args %q: %v", rr.Command(), err)
}
got := Status(ctx, t, Target(), profile, "Host")
got := Status(ctx, t, Target(), profile, "Host", profile)
if got != state.Running.String() {
t.Errorf("expected host status after start-stop-start to be -%q- but got *%q*", state.Running, got)
}
@ -323,12 +323,12 @@ func testPause(ctx context.Context, t *testing.T, profile string) {
t.Fatalf("%s failed: %v", rr.Command(), err)
}
got := Status(ctx, t, Target(), profile, "APIServer")
got := Status(ctx, t, Target(), profile, "APIServer", profile)
if got != state.Paused.String() {
t.Errorf("post-pause apiserver status = %q; want = %q", got, state.Paused)
}
got = Status(ctx, t, Target(), profile, "Kubelet")
got = Status(ctx, t, Target(), profile, "Kubelet", profile)
if got != state.Stopped.String() {
t.Errorf("post-pause kubelet status = %q; want = %q", got, state.Stopped)
}
@ -338,12 +338,12 @@ func testPause(ctx context.Context, t *testing.T, profile string) {
t.Fatalf("%s failed: %v", rr.Command(), err)
}
got = Status(ctx, t, Target(), profile, "APIServer")
got = Status(ctx, t, Target(), profile, "APIServer", profile)
if got != state.Running.String() {
t.Errorf("post-unpause apiserver status = %q; want = %q", got, state.Running)
}
got = Status(ctx, t, Target(), profile, "Kubelet")
got = Status(ctx, t, Target(), profile, "Kubelet", profile)
if got != state.Running.String() {
t.Errorf("post-unpause kubelet status = %q; want = %q", got, state.Running)
}