Merge branch 'master' into docker-stress-test

cmd/minikube/cmd/start.go

@@ -811,7 +811,6 @@ func suggestMemoryAllocation(sysLimit int, containerLimit int, nodes int) int {
 
 // validateMemorySize validates the memory size matches the minimum recommended
 func validateMemorySize(req int, drvName string) {
-
 	sysLimit, containerLimit, err := memoryLimits(drvName)
 	if err != nil {
 		glog.Warningf("Unable to query memory limits: %v", err)
@@ -822,7 +821,7 @@ func validateMemorySize(req int, drvName string) {
 			out.V{"requested": req, "mininum": minUsableMem})
 	}
 	if req < minRecommendedMem && !viper.GetBool(force) {
-		out.T(out.Notice, "Requested memory allocation ({{.requested}}MB) is less than the recommended minimum {{.recommended}}MB. Kubernetes may crash unexpectedly.",
+		out.WarningT("Requested memory allocation ({{.requested}}MB) is less than the recommended minimum {{.recommended}}MB. Kubernetes may crash unexpectedly.",
 			out.V{"requested": req, "recommended": minRecommendedMem})
 	}
 
@@ -830,13 +829,13 @@ func validateMemorySize(req int, drvName string) {
 		// in Docker Desktop if you allocate 2 GB the docker info shows:  Total Memory: 1.945GiB which becomes 1991 when we calculate the MBs
 		// thats why it is not same number as other drivers which is 2 GB
 		if containerLimit < 1991 {
-			out.T(out.Tip, `Increase Docker for Desktop memory to at least 2.5GB or more:
+			out.WarningT(`Increase Docker for Desktop memory to at least 2.5GB or more:
 			
 	Docker for Desktop > Settings > Resources > Memory
 
 `)
 		} else if containerLimit < 2997 && sysLimit > 8000 { // for users with more than 8 GB advice 3 GB
-			out.T(out.Tip, `Your system has {{.system_limit}}MB memory but Docker has only {{.container_limit}}MB. For a better performance increase to at least 3GB.
+			out.WarningT(`Your system has {{.system_limit}}MB memory but Docker has only {{.container_limit}}MB. For a better performance increase to at least 3GB.
 
 	Docker for Desktop  > Settings > Resources > Memory
 

deploy/iso/minikube-iso/board/coreos/minikube/linux_defconfig

@@ -274,9 +274,12 @@ CONFIG_BRIDGE_EBT_LOG=m
 CONFIG_BRIDGE_EBT_NFLOG=m
 CONFIG_BRIDGE=m
 CONFIG_NET_SCHED=y
+CONFIG_NET_SCH_PRIO=y
+CONFIG_NET_SCH_SFQ=y
 CONFIG_NET_SCH_TBF=y
 CONFIG_NET_SCH_NETEM=y
 CONFIG_NET_SCH_INGRESS=m
+CONFIG_NET_CLS_BASIC=m
 CONFIG_NET_CLS_U32=m
 CONFIG_NET_CLS_CGROUP=y
 CONFIG_NET_CLS_BPF=m

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -554,6 +554,9 @@ func (k *Bootstrapper) restartControlPlane(cfg config.ClusterConfig) error {
 		return nil
 	}
 
+	if err := k.stopKubeSystem(cfg); err != nil {
+		glog.Warningf("Failed to stop kube-system containers: port conflicts may arise: %v", err)
+	}
 	if err := k.clearStaleConfigs(cfg); err != nil {
 		return errors.Wrap(err, "clearing stale configs")
 	}
@@ -905,6 +908,27 @@ func (k *Bootstrapper) elevateKubeSystemPrivileges(cfg config.ClusterConfig) err
 	return nil
 }
 
+// stopKubeSystem stops all the containers in the kube-system to prevent #8740 when doing hot upgrade
+func (k *Bootstrapper) stopKubeSystem(cfg config.ClusterConfig) error {
+	glog.Info("stopping kube-system containers ...")
+	cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: k.c})
+	if err != nil {
+		return errors.Wrap(err, "new cruntime")
+	}
+
+	ids, err := cr.ListContainers(cruntime.ListOptions{Namespaces: []string{"kube-system"}})
+	if err != nil {
+		return errors.Wrap(err, "list")
+	}
+
+	if len(ids) > 0 {
+		if err := cr.StopContainers(ids); err != nil {
+			return errors.Wrap(err, "stop")
+		}
+	}
+	return nil
+}
+
 // adviseNodePressure will advise the user what to do with difference pressure errors based on their environment
 func adviseNodePressure(err error, name string, drv string) {
 	if diskErr, ok := err.(*kverify.ErrDiskPressure); ok {

pkg/minikube/exit/exit.go

@@ -43,13 +43,13 @@ const (
 
 // UsageT outputs a templated usage error and exits with error code 64
 func UsageT(format string, a ...out.V) {
-	out.ErrT(out.Usage, format, a...)
+	out.ErrWithExitCode(out.Usage, format, BadUsage, a...)
 	os.Exit(BadUsage)
 }
 
 // WithCodeT outputs a templated fatal error message and exits with the supplied error code.
 func WithCodeT(code int, format string, a ...out.V) {
-	out.FatalT(format, a...)
+	out.ErrWithExitCode(out.FatalType, format, code, a...)
 	os.Exit(code)
 }
 
@@ -57,8 +57,12 @@ func WithCodeT(code int, format string, a ...out.V) {
 func WithError(msg string, err error) {
 	glog.Infof("WithError(%s)=%v called from:\n%s", msg, err, debug.Stack())
 	p := problem.FromError(err, runtime.GOOS)
-	if p != nil {
+	if p != nil && out.JSON {
+		p.DisplayJSON(Config)
+		os.Exit(Config)
+	} else {
 		WithProblem(msg, err, p)
+		os.Exit(Config)
 	}
 	out.DisplayError(msg, err)
 	os.Exit(Software)
@@ -74,5 +78,4 @@ func WithProblem(msg string, err error, p *problem.Problem) {
 		out.ErrT(out.Sad, "If the above advice does not help, please let us know: ")
 		out.ErrT(out.URL, "https://github.com/kubernetes/minikube/issues/new/choose")
 	}
-	os.Exit(Config)
 }

pkg/minikube/machine/info.go

@@ -27,6 +27,7 @@ import (
 	"github.com/shirou/gopsutil/mem"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/out"
+	"k8s.io/minikube/pkg/minikube/out/register"
 )
 
 type hostInfo struct {
@@ -77,6 +78,7 @@ func showLocalOsRelease() {
 		return
 	}
 
+	register.Reg.SetStep(register.LocalOSRelease)
 	out.T(out.Provisioner, "OS release is {{.pretty_name}}", out.V{"pretty_name": osReleaseInfo.PrettyName})
 }
 

pkg/minikube/machine/start.go

@@ -253,6 +253,7 @@ func showHostInfo(cfg config.ClusterConfig) {
 	if driver.BareMetal(cfg.Driver) {
 		info, err := getHostInfo()
 		if err == nil {
+			register.Reg.SetStep(register.RunningLocalhost)
 			out.T(out.StartingNone, "Running on localhost (CPUs={{.number_of_cpus}}, Memory={{.memory_size}}MB, Disk={{.disk_size}}MB) ...", out.V{"number_of_cpus": info.CPUs, "memory_size": info.Memory, "disk_size": info.DiskSize})
 		}
 		return

pkg/minikube/node/start.go

@@ -510,6 +510,7 @@ func tryRegistry(r command.Runner, driverName string, imageRepository string) {
 
 // prepareNone prepares the user and host for the joy of the "none" driver
 func prepareNone() {
+	register.Reg.SetStep(register.ConfiguringLHEnv)
 	out.T(out.StartingNone, "Configuring local host environment ...")
 	if viper.GetBool(config.WantNoneDriverWarning) {
 		out.ErrT(out.Empty, "")

pkg/minikube/out/out.go

@@ -115,6 +115,16 @@ func Ln(format string, a ...interface{}) {
 	String(format+"\n", a...)
 }
 
+// ErrWithExitCode includes the exit code in JSON output
+func ErrWithExitCode(style StyleEnum, format string, exitcode int, a ...V) {
+	if JSON {
+		errStyled := ApplyTemplateFormatting(style, useColor, format, a...)
+		register.PrintErrorExitCode(errStyled, exitcode)
+		return
+	}
+	ErrT(style, format, a...)
+}
+
 // ErrT writes a stylized and templated error message to stderr
 func ErrT(style StyleEnum, format string, a ...V) {
 	errStyled := ApplyTemplateFormatting(style, useColor, format, a...)
@@ -123,6 +133,10 @@ func ErrT(style StyleEnum, format string, a ...V) {
 
 // Err writes a basic formatted string to stderr
 func Err(format string, a ...interface{}) {
+	if JSON {
+		register.PrintError(format)
+		return
+	}
 	if errFile == nil {
 		glog.Errorf("[unset errFile]: %s", fmt.Sprintf(format, a...))
 		return
@@ -232,8 +246,12 @@ func LogEntries(msg string, err error, entries map[string][]string) {
 
 // DisplayError prints the error and displays the standard minikube error messaging
 func DisplayError(msg string, err error) {
-	// use Warning because Error will display a duplicate message to stderr
 	glog.Warningf(fmt.Sprintf("%s: %v", msg, err))
+	if JSON {
+		FatalT("{{.msg}}: {{.err}}", V{"msg": translate.T(msg), "err": err})
+		return
+	}
+	// use Warning because Error will display a duplicate message to stderr
 	ErrT(Empty, "")
 	FatalT("{{.msg}}: {{.err}}", V{"msg": translate.T(msg), "err": err})
 	ErrT(Empty, "")

pkg/minikube/out/register/cloud_events.go

@@ -31,8 +31,8 @@ const (
 )
 
 var (
-	outputFile io.Writer = os.Stdout
+	OutputFile io.Writer = os.Stdout
-	getUUID              = randomID
+	GetUUID              = randomID
 )
 
 func printAsCloudEvent(log Log, data map[string]string) {
@@ -43,12 +43,12 @@ func printAsCloudEvent(log Log, data map[string]string) {
 	if err := event.SetData(cloudevents.ApplicationJSON, data); err != nil {
 		glog.Warningf("error setting data: %v", err)
 	}
-	event.SetID(getUUID())
+	event.SetID(GetUUID())
 	json, err := event.MarshalJSON()
 	if err != nil {
 		glog.Warningf("error marashalling event: %v", err)
 	}
-	fmt.Fprintln(outputFile, string(json))
+	fmt.Fprintln(OutputFile, string(json))
 }
 
 func randomID() string {

pkg/minikube/out/register/json.go

@@ -40,6 +40,18 @@ func PrintDownloadProgress(artifact, progress string) {
 	printAsCloudEvent(s, s.data)
 }
 
+// PrintError prints an Error type in JSON format
+func PrintError(err string) {
+	e := NewError(err)
+	printAsCloudEvent(e, e.data)
+}
+
+// PrintErrorExitCode prints an error in JSON format and includes an exit code
+func PrintErrorExitCode(err string, exitcode int, additionalArgs ...map[string]string) {
+	e := NewErrorExitCode(err, exitcode, additionalArgs...)
+	printAsCloudEvent(e, e.data)
+}
+
 // PrintWarning prints a Warning type in JSON format
 func PrintWarning(warning string) {
 	w := NewWarning(warning)

pkg/minikube/out/register/json_test.go

@@ -29,10 +29,10 @@ func TestPrintStep(t *testing.T) {
 	expected += "\n"
 
 	buf := bytes.NewBuffer([]byte{})
-	outputFile = buf
+	OutputFile = buf
-	defer func() { outputFile = os.Stdout }()
+	defer func() { OutputFile = os.Stdout }()
 
-	getUUID = func() string {
+	GetUUID = func() string {
 		return "random-id"
 	}
 
@@ -49,10 +49,10 @@ func TestPrintInfo(t *testing.T) {
 	expected += "\n"
 
 	buf := bytes.NewBuffer([]byte{})
-	outputFile = buf
+	OutputFile = buf
-	defer func() { outputFile = os.Stdout }()
+	defer func() { OutputFile = os.Stdout }()
 
-	getUUID = func() string {
+	GetUUID = func() string {
 		return "random-id"
 	}
 
@@ -64,15 +64,53 @@ func TestPrintInfo(t *testing.T) {
 	}
 }
 
+func TestError(t *testing.T) {
+	expected := `{"data":{"message":"error"},"datacontenttype":"application/json","id":"random-id","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.error"}`
+	expected += "\n"
+
+	buf := bytes.NewBuffer([]byte{})
+	OutputFile = buf
+	defer func() { OutputFile = os.Stdout }()
+
+	GetUUID = func() string {
+		return "random-id"
+	}
+
+	PrintError("error")
+	actual := buf.String()
+
+	if actual != expected {
+		t.Fatalf("expected didn't match actual:\nExpected:\n%v\n\nActual:\n%v", expected, actual)
+	}
+}
+
+func TestErrorExitCode(t *testing.T) {
+	expected := `{"data":{"a":"b","c":"d","exitcode":"5","message":"error"},"datacontenttype":"application/json","id":"random-id","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.error"}`
+	expected += "\n"
+
+	buf := bytes.NewBuffer([]byte{})
+	OutputFile = buf
+	defer func() { OutputFile = os.Stdout }()
+
+	GetUUID = func() string {
+		return "random-id"
+	}
+
+	PrintErrorExitCode("error", 5, map[string]string{"a": "b"}, map[string]string{"c": "d"})
+	actual := buf.String()
+	if actual != expected {
+		t.Fatalf("expected didn't match actual:\nExpected:\n%v\n\nActual:\n%v", expected, actual)
+	}
+}
 func TestWarning(t *testing.T) {
 	expected := `{"data":{"message":"warning"},"datacontenttype":"application/json","id":"random-id","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.warning"}`
 	expected += "\n"
 
 	buf := bytes.NewBuffer([]byte{})
-	outputFile = buf
+	OutputFile = buf
-	defer func() { outputFile = os.Stdout }()
+	defer func() { OutputFile = os.Stdout }()
 
-	getUUID = func() string {
+	GetUUID = func() string {
 		return "random-id"
 	}
 

pkg/minikube/out/register/log.go

@@ -16,6 +16,8 @@ limitations under the License.
 
 package register
 
+import "fmt"
+
 // Log represents the different types of logs that can be output as JSON
 // This includes: Step, Download, DownloadProgress, Warning, Info, Error
 type Log interface {
@@ -120,6 +122,27 @@ func NewInfo(message string) *Info {
 
 // Error will be used to notify the user of errors
 type Error struct {
+	data map[string]string
+}
+
+func NewError(err string) *Error {
+	return &Error{
+		map[string]string{
+			"message": err,
+		},
+	}
+}
+
+// NewErrorExitCode returns an error that has an associated exit code
+func NewErrorExitCode(err string, exitcode int, additionalData ...map[string]string) *Error {
+	e := NewError(err)
+	e.data["exitcode"] = fmt.Sprintf("%v", exitcode)
+	for _, a := range additionalData {
+		for k, v := range a {
+			e.data[k] = v
+		}
+	}
+	return e
 }
 
 func (s *Error) Type() string {

pkg/minikube/out/register/register.go

@@ -26,8 +26,11 @@ const (
 	SelectingDriver      RegStep = "Selecting Driver"
 	DownloadingArtifacts RegStep = "Downloading Artifacts"
 	StartingNode         RegStep = "Starting Node"
+	RunningLocalhost     RegStep = "Running on Localhost"
+	LocalOSRelease       RegStep = "Local OS Release"
 	CreatingContainer    RegStep = "Creating Container"
 	CreatingVM           RegStep = "Creating VM"
+	ConfiguringLHEnv     RegStep = "Configuring Localhost Environment"
 	PreparingKubernetes  RegStep = "Preparing Kubernetes"
 	VerifyingKubernetes  RegStep = "Verifying Kubernetes"
 	EnablingAddons       RegStep = "Enabling Addons"
@@ -54,9 +57,12 @@ func init() {
 			SelectingDriver,
 			DownloadingArtifacts,
 			StartingNode,
+			RunningLocalhost,
+			LocalOSRelease,
 			CreatingContainer,
 			CreatingVM,
 			PreparingKubernetes,
+			ConfiguringLHEnv,
 			VerifyingKubernetes,
 			EnablingAddons,
 			Done,

pkg/minikube/out/register/register_test.go

@@ -32,10 +32,10 @@ func TestSetCurrentStep(t *testing.T) {
 	expected += "\n"
 
 	buf := bytes.NewBuffer([]byte{})
-	outputFile = buf
+	OutputFile = buf
-	defer func() { outputFile = os.Stdout }()
+	defer func() { OutputFile = os.Stdout }()
 
-	getUUID = func() string {
+	GetUUID = func() string {
 		return "random-id"
 	}
 

pkg/minikube/problem/problem.go

@@ -22,6 +22,7 @@ import (
 	"regexp"
 
 	"k8s.io/minikube/pkg/minikube/out"
+	"k8s.io/minikube/pkg/minikube/out/register"
 	"k8s.io/minikube/pkg/minikube/translate"
 )
 
@@ -80,6 +81,21 @@ func (p *Problem) Display() {
 	}
 }
 
+// DisplayJSON displays problem metadata in JSON format
+func (p *Problem) DisplayJSON(exitcode int) {
+	var issues string
+	for _, i := range p.Issues {
+		issues += fmt.Sprintf("https://github.com/kubernetes/minikube/issues/%v,", i)
+	}
+	extraArgs := map[string]string{
+		"name":   p.ID,
+		"advice": p.Advice,
+		"url":    p.URL,
+		"issues": issues,
+	}
+	register.PrintErrorExitCode(p.Err.Error(), exitcode, extraArgs)
+}
+
 // FromError returns a known problem from an error on an OS
 func FromError(err error, goos string) *Problem {
 	maps := []map[string]match{

pkg/minikube/problem/problem_test.go

@@ -19,10 +19,12 @@ package problem
 import (
 	"bytes"
 	"fmt"
+	"os"
 	"strings"
 	"testing"
 
 	"k8s.io/minikube/pkg/minikube/out"
+	"k8s.io/minikube/pkg/minikube/out/register"
 )
 
 type buffFd struct {
@@ -96,6 +98,45 @@ func TestDisplay(t *testing.T) {
 	}
 }
 
+func TestDisplayJSON(t *testing.T) {
+	defer out.SetJSON(false)
+	out.SetJSON(true)
+
+	tcs := []struct {
+		p        *Problem
+		expected string
+	}{
+		{
+			p: &Problem{
+				Err:    fmt.Errorf("my error"),
+				Advice: "fix me!",
+				Issues: []int{1, 2},
+				URL:    "url",
+				ID:     "BUG",
+			},
+			expected: `{"data":{"advice":"fix me!","exitcode":"4","issues":"https://github.com/kubernetes/minikube/issues/1,https://github.com/kubernetes/minikube/issues/2,","message":"my error","name":"BUG","url":"url"},"datacontenttype":"application/json","id":"random-id","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.error"}
+`,
+		},
+	}
+	for _, tc := range tcs {
+		t.Run(tc.p.ID, func(t *testing.T) {
+			buf := bytes.NewBuffer([]byte{})
+			register.OutputFile = buf
+			defer func() { register.OutputFile = os.Stdout }()
+
+			register.GetUUID = func() string {
+				return "random-id"
+			}
+
+			tc.p.DisplayJSON(4)
+			actual := buf.String()
+			if actual != tc.expected {
+				t.Fatalf("expected didn't match actual:\nExpected:\n%v\n\nActual:\n%v", tc.expected, actual)
+			}
+		})
+	}
+}
+
 func TestFromError(t *testing.T) {
 	var tests = []struct {
 		issue int

pkg/minikube/registry/registry.go

@@ -109,7 +109,7 @@ func (d DriverDef) String() string {
 
 type driverRegistry struct {
 	drivers map[string]DriverDef
-	lock    sync.Mutex
+	lock    sync.RWMutex
 }
 
 func newRegistry() *driverRegistry {
@@ -133,8 +133,8 @@ func (r *driverRegistry) Register(def DriverDef) error {
 
 // List returns a list of registered drivers
 func (r *driverRegistry) List() []DriverDef {
-	r.lock.Lock()
+	r.lock.RLock()
-	defer r.lock.Unlock()
+	defer r.lock.RUnlock()
 
 	result := make([]DriverDef, 0, len(r.drivers))
 
@@ -147,7 +147,7 @@ func (r *driverRegistry) List() []DriverDef {
 
 // Driver returns a driver given a name
 func (r *driverRegistry) Driver(name string) DriverDef {
-	r.lock.Lock()
+	r.lock.RLock()
-	defer r.lock.Unlock()
+	defer r.lock.RUnlock()
 	return r.drivers[name]
 }

site/content/en/docs/contrib/json_output.en.md

@@ -0,0 +1,73 @@
+---
+title: "Minikube JSON Output"
+date: 2019-07-31
+weight: 4
+description: >
+  How to add logs to facilitate JSON output
+---
+
+This document is written for minikube contributors who need to add logs to the minikube log registry for sucessful JSON output.
+You may need to add logs to the registry if the `TestJSONOutput` integration test is failing on your PR.
+
+### Background
+minikube provides JSON output for `minikube start`, accesible via the `--output` flag:
+
+```
+minikube start --output json
+```
+
+This converts regular output:
+
+```
+$ minikube start
+
+😄  minikube v1.12.1 on Darwin 10.14.6
+✨  Automatically selected the hyperkit driver
+👍  Starting control plane node minikube in cluster minikube
+🔥  Creating hyperkit VM (CPUs=2, Memory=6000MB, Disk=20000MB) ...
+```
+
+into a [Cloud Events](https://cloudevents.io/) compatible JSON output:
+
+```
+$ minikube start --output json
+
+{"data":{"currentstep":"0","message":"minikube v1.12.1 on Darwin 10.14.6\n","name":"Initial Minikube Setup","totalsteps":"10"},"datacontenttype":"application/json","id":"68ff70ae-202b-4b13-8351-e9f060e8c56e","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.step"}
+{"data":{"currentstep":"1","message":"Automatically selected the hyperkit driver\n","name":"Selecting Driver","totalsteps":"10"},"datacontenttype":"application/json","id":"39bed8e9-3c1a-444e-997c-2ec19bdb1ca1","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.step"}
+{"data":{"currentstep":"3","message":"Starting control plane node minikube in cluster minikube\n","name":"Starting Node","totalsteps":"10"},"datacontenttype":"application/json","id":"7c80bc53-3ac4-4a42-a493-92e269cc56c9","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.step"}
+{"data":{"currentstep":"6","message":"Creating hyperkit VM (CPUs=2, Memory=6000MB, Disk=20000MB) ...\n","name":"Creating VM","totalsteps":"10"},"datacontenttype":"application/json","id":"7f5f23a4-9a09-4954-8abc-d29bda2cc569","source":"https://minikube.sigs.k8s.io/","specversion":"1.0","type":"io.k8s.sigs.minikube.step"}
+```
+
+There are a few key points to note in the above output:
+1. Each log of type `io.k8s.sigs.minikube.step` indicates a distinct step in the `minikube start` process
+1. Each step has a `currentstep` field which allows clients to track `minikube start` progress
+1. Each `currentstep` is distinct and increasing in order
+
+To achieve this output, minikube maintains a registry of logs.
+This way, minikube knows how many expected `totalsteps` there are at the beginning of the process, and what the current step is.
+
+If you change logs, or add a new log, you need to update the minikube registry to pass integration tests.
+
+
+### Adding a Log to the Registry
+
+There are three steps to adding a log to the registry, which exists in [register.go](https://github.com/kubernetes/minikube/blob/master/pkg/minikube/out/register/register.go).
+
+You will need to add your new log in two places:
+1. As a constant of type `RegStep` [here](https://github.com/kubernetes/minikube/blob/master/pkg/minikube/out/register/register.go#L24)
+1. In the register itself in the `init()` function, [here](https://github.com/kubernetes/minikube/blob/master/pkg/minikube/out/register/register.go#L52)
+
+**Note: It's important that the order of steps matches the expected order they will be called in. So, if you add a step that is supposed to be called after "Preparing Kubernetes", the new step should be place after "Preparing Kubernetes".
+
+Finally, set your new step in the cod by placing this line before you call `out.T`:
+
+```
+register.Reg.SetStep(register.MyNewStep)
+```
+
+You can see an example of setting the registry step in the code in [config.go](https://github.com/kubernetes/minikube/blob/master/pkg/minikube/node/config.go):
+
+```go
+	register.Reg.SetStep(register.PreparingKubernetes)
+	out.T(cr.Style(), "Preparing Kubernetes {{.k8sVersion}} on {{.runtime}} {{.runtimeVersion}} ...", out.V{"k8sVersion": k8sVersion, "runtime": cr.Name(), "runtimeVersion": version})
+```

site/content/en/docs/contrib/translations.md

@@ -16,7 +16,7 @@ All translations are stored in the top-level `translations` directory.
 	~/minikube$ ls translations/
 	de.json		es.json		fr.json		ja.json		ko.json		pl.json		zh-CN.json
 	```
-* Run `make extract` from root to populate that file with the strings to translate in json
+* Run `make extract` from root directory to populate that file with the strings to translate in json
   form.
 	```
 	~/minikube$ make extract

site/content/en/docs/tutorials/openid_connect_auth.md

@@ -25,6 +25,8 @@ minikube start \
   --extra-config=apiserver.oidc-client-id=kubernetes-local
 ```
 
+Note that as stated in the Kubernetes [documentation](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#configuring-the-api-server), for `--extra-config=apiserver.oidc-issuer-url` flag, only URLs which use the `https://` scheme are accepted. Otherwise `kube-apiserver` will not start.
+
 ## Configuring kubectl
 
 You can use the kubectl `oidc` authenticator to create a kubeconfig as shown in the Kubernetes docs: <https://kubernetes.io/docs/reference/access-authn-authz/authentication/#option-1-oidc-authenticator>

test/integration/error_spam_test.go

@@ -36,6 +36,10 @@ var stderrAllow = []string{
 	`cache_images.go:.*error getting status`,
 	// don't care if we can't push images to other profiles which are deleted.
 	`cache_images.go:.*Failed to load profile`,
+	// ! 'docker' driver reported a issue that could affect the performance."
+	`docker.*issue.*performance`,
+	// "* Suggestion: enable overlayfs kernel module on your Linux"
+	`Suggestion.*overlayfs`,
 }
 
 // stderrAllowRe combines rootCauses into a single regex
@@ -64,14 +68,10 @@ func TestErrorSpam(t *testing.T) {
 	stderr := rr.Stderr.String()
 
 	for _, line := range strings.Split(stderr, "\n") {
-		if strings.HasPrefix(line, "E") {
 		if stderrAllowRe.MatchString(line) {
 			t.Logf("acceptable stderr: %q", line)
 			continue
 		}
-			t.Errorf("unexpected error log: %q", line)
-			continue
-		}
 
 		if len(strings.TrimSpace(line)) > 0 {
 			t.Errorf("unexpected stderr: %q", line)

test/integration/json_output_test.go

@@ -19,17 +19,19 @@ package integration
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"os/exec"
+	"runtime"
+	"strconv"
 	"strings"
 	"testing"
 
 	cloudevents "github.com/cloudevents/sdk-go/v2"
+	"k8s.io/minikube/pkg/minikube/exit"
+	"k8s.io/minikube/pkg/minikube/out/register"
 )
 
 func TestJSONOutput(t *testing.T) {
-	if NoneDriver() || DockerDriver() {
-		t.Skipf("skipping: test drivers once all JSON output is enabled")
-	}
 	profile := UniqueProfileName("json-output")
 	ctx, cancel := context.WithTimeout(context.Background(), Minutes(40))
 	defer Cleanup(t, profile, cancel)
@@ -42,25 +44,117 @@ func TestJSONOutput(t *testing.T) {
 		t.Errorf("failed to clean up: args %q: %v", rr.Command(), err)
 	}
 
-	type validateJSONOutputFunc func(context.Context, *testing.T, *RunResult)
+	ces, err := cloudEvents(t, rr)
+	if err != nil {
+		t.Fatalf("converting to cloud events: %v\n", err)
+	}
+
+	type validateJSONOutputFunc func(context.Context, *testing.T, []*cloudEvent)
 	t.Run("serial", func(t *testing.T) {
 		serialTests := []struct {
 			name      string
 			validator validateJSONOutputFunc
 		}{
-			{"CloudEvents", validateCloudEvents},
+			{"DistinctCurrentSteps", validateDistinctCurrentSteps},
+			{"IncreasingCurrentSteps", validateIncreasingCurrentSteps},
 		}
 		for _, stc := range serialTests {
 			t.Run(stc.name, func(t *testing.T) {
-				stc.validator(ctx, t, rr)
+				stc.validator(ctx, t, ces)
 			})
 		}
 	})
-
 }
 
-//  make sure all output can be marshaled as a cloud event
+//  make sure each step has a distinct step number
-func validateCloudEvents(ctx context.Context, t *testing.T, rr *RunResult) {
+func validateDistinctCurrentSteps(ctx context.Context, t *testing.T, ces []*cloudEvent) {
+	steps := map[string]string{}
+	for _, ce := range ces {
+		currentStep, exists := ce.data["currentstep"]
+		if !exists {
+			continue
+		}
+		if msg, alreadySeen := steps[currentStep]; alreadySeen {
+			t.Fatalf("step %v has already been assigned to another step:\n%v\nCannot use for:\n%v\n%v", currentStep, msg, ce.data["message"], ces)
+		}
+		steps[currentStep] = ce.data["message"]
+	}
+}
+
+// for successful minikube start, 'current step' should be increasing
+func validateIncreasingCurrentSteps(ctx context.Context, t *testing.T, ces []*cloudEvent) {
+	step := -1
+	for _, ce := range ces {
+		currentStep, exists := ce.data["currentstep"]
+		if !exists {
+			continue
+		}
+		cs, err := strconv.Atoi(currentStep)
+		if err != nil {
+			t.Fatalf("current step is not an integer: %v\n%v", currentStep, ce)
+		}
+		if cs <= step {
+			t.Fatalf("current step is not in increasing order: %v", ces)
+		}
+		step = cs
+	}
+}
+
+func TestJSONOutputError(t *testing.T) {
+	profile := UniqueProfileName("json-output-error")
+	ctx, cancel := context.WithTimeout(context.Background(), Minutes(2))
+	defer Cleanup(t, profile, cancel)
+
+	// force a failure via --driver=fail so that we can make sure errors
+	// are printed as expected
+	startArgs := []string{"start", "-p", profile, "--memory=2200", "--output=json", "--wait=true", "--driver=fail"}
+
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), startArgs...))
+	if err == nil {
+		t.Errorf("expected failure: args %q: %v", rr.Command(), err)
+	}
+	ces, err := cloudEvents(t, rr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// we want the last cloud event to be of type error and have the expected exit code and message
+	last := ces[len(ces)-1]
+	if last.Type() != register.NewError("").Type() {
+		t.Fatalf("last cloud event is not of type error: %v", last)
+	}
+	last.validateData(t, "exitcode", fmt.Sprintf("%v", exit.Unavailable))
+	last.validateData(t, "message", fmt.Sprintf("The driver 'fail' is not supported on %s\n", runtime.GOOS))
+}
+
+type cloudEvent struct {
+	cloudevents.Event
+	data map[string]string
+}
+
+func newCloudEvent(t *testing.T, ce cloudevents.Event) *cloudEvent {
+	m := map[string]string{}
+	data := ce.Data()
+	if err := json.Unmarshal(data, &m); err != nil {
+		t.Fatalf("marshalling cloud event: %v", err)
+	}
+	return &cloudEvent{
+		Event: ce,
+		data:  m,
+	}
+}
+
+func (c *cloudEvent) validateData(t *testing.T, key, value string) {
+	v, ok := c.data[key]
+	if !ok {
+		t.Fatalf("expected key %s does not exist in cloud event", key)
+	}
+	if v != value {
+		t.Fatalf("values in cloud events do not match:\nActual:\n%v\nExpected:\n%v\n", v, value)
+	}
+}
+
+func cloudEvents(t *testing.T, rr *RunResult) ([]*cloudEvent, error) {
+	ces := []*cloudEvent{}
 	stdout := strings.Split(rr.Stdout.String(), "\n")
 	for _, s := range stdout {
 		if s == "" {
@@ -68,7 +162,10 @@ func validateCloudEvents(ctx context.Context, t *testing.T, rr *RunResult) {
 		}
 		event := cloudevents.NewEvent()
 		if err := json.Unmarshal([]byte(s), &event); err != nil {
-			t.Fatalf("unable to unmarshal output: %v\n%s", err, s)
+			t.Logf("unable to marshal output: %v", s)
+			return nil, err
 		}
+		ces = append(ces, newCloudEvent(t, event))
 	}
+	return ces, nil
 }

third_party/go9p/srv_file.go

@@ -81,7 +81,7 @@ const (
 
 // The srvFile type represents a file (or directory) served by the file server.
 type srvFile struct {
-	sync.Mutex
+	sync.RWMutex
 	Dir
 	flags FFlags
 
@@ -239,13 +239,13 @@ func (f *srvFile) Rename(name string) error {
 func (p *srvFile) Find(name string) *srvFile {
 	var f *srvFile
 
-	p.Lock()
+	p.RLock()
 	for f = p.cfirst; f != nil; f = f.next {
 		if name == f.Name {
 			break
 		}
 	}
-	p.Unlock()
+	p.RUnlock()
 	return f
 }
 
@@ -496,13 +496,13 @@ func (*Fsrv) Clunk(req *SrvReq) {
 func (*Fsrv) Remove(req *SrvReq) {
 	fid := req.Fid.Aux.(*FFid)
 	f := fid.F
-	f.Lock()
+	f.RLock()
 	if f.cfirst != nil {
-		f.Unlock()
+		f.RUnlock()
 		req.RespondError(Enotempty)
 		return
 	}
-	f.Unlock()
+	f.RUnlock()
 
 	if rop, ok := (f.ops).(FRemoveOp); ok {
 		err := rop.Remove(fid)