# This is a combination of 8 commits.

# This is the 1st commit message:

Fix doc comment version.gitCommitID doc


# This is the commit message #2:

Add hyperkit doc


# This is the commit message #3:

Add commit id to docker-machine-driver-kvm2 version


# This is the commit message #4:

removed label selector for registry-proxy daemonset

# This is the commit message #5:

Add support to custom qemu uri on kvm2 driver


# This is the commit message #6:

Improve hyperkit vm stop


# This is the commit message #7:

Make virtualbox DNS settings configurable 


# This is the commit message #8:

added integration tests for registry addon

Makefile

@@ -83,6 +83,9 @@ CMD_SOURCE_DIRS = cmd pkg
 SOURCE_DIRS = $(CMD_SOURCE_DIRS) test
 SOURCE_PACKAGES = ./cmd/... ./pkg/... ./test/...
 
+# kvm2 ldflags
+KVM2_LDFLAGS := -X k8s.io/minikube/pkg/drivers/kvm.version=$(VERSION) -X k8s.io/minikube/pkg/drivers/kvm.gitCommitID=$(COMMIT) 
+
 # $(call DOCKER, image, command)
 define DOCKER
 	docker run --rm -e GOCACHE=/app/.cache -e IN_DOCKER=1 --user $(shell id -u):$(shell id -g) -w /app -v $(PWD):/app -v $(GOPATH):/go --entrypoint /bin/bash $(1) -c '$(2)'
@@ -387,11 +390,11 @@ release-minikube: out/minikube checksum
 	gsutil cp out/minikube-$(GOOS)-$(GOARCH).sha256 $(MINIKUBE_UPLOAD_LOCATION)/$(MINIKUBE_VERSION)/minikube-$(GOOS)-$(GOARCH).sha256
 
 out/docker-machine-driver-kvm2:
-	go build 																		\
-		-installsuffix "static" 													\
-		-ldflags "-X k8s.io/minikube/pkg/drivers/kvm.version=$(VERSION)" 	\
+	go build 																					\
+		-installsuffix "static" 												\
+		-ldflags="$(KVM2_LDFLAGS)" 											\
 		-tags libvirt.1.3.1 														\
-		-o $(BUILD_DIR)/docker-machine-driver-kvm2 									\
+		-o $(BUILD_DIR)/docker-machine-driver-kvm2 			\
 		k8s.io/minikube/cmd/drivers/kvm
 	chmod +X $@
 

cmd/drivers/kvm/main.go

@@ -27,8 +27,9 @@ import (
 )
 
 func main() {
-	if len(os.Args) > 1 && os.Args[1] == "--version" {
-		fmt.Println(kvm.GetVersion())
+	if len(os.Args) > 1 && os.Args[1] == "version" {
+		fmt.Println("version:", kvm.GetVersion())
+		fmt.Println("commit:", kvm.GetGitCommitID())
 		return
 	}
 

cmd/minikube/cmd/start.go

@@ -77,6 +77,7 @@ const (
 	enableDefaultCNI      = "enable-default-cni"
 	hypervVirtualSwitch   = "hyperv-virtual-switch"
 	kvmNetwork            = "kvm-network"
+	kvmQemuURI            = "kvm-qemu-uri"
 	keepContext           = "keep-context"
 	createMount           = "mount"
 	featureGates          = "feature-gates"
@@ -97,6 +98,8 @@ const (
 	embedCerts            = "embed-certs"
 	noVTXCheck            = "no-vtx-check"
 	downloadOnly          = "download-only"
+	dnsProxy              = "dns-proxy"
+	hostDNSResolver       = "host-dns-resolver"
 )
 
 var (
@@ -122,6 +125,7 @@ func init() {
 	startCmd.Flags().String(hostOnlyCIDR, "192.168.99.1/24", "The CIDR to be used for the minikube VM (only supported with Virtualbox driver)")
 	startCmd.Flags().String(hypervVirtualSwitch, "", "The hyperv virtual switch name. Defaults to first found. (only supported with HyperV driver)")
 	startCmd.Flags().String(kvmNetwork, "default", "The KVM network name. (only supported with KVM driver)")
+	startCmd.Flags().String(kvmQemuURI, "qemu:///system", "The KVM QEMU connection URI. (works only with kvm2 driver on linux)")
 	startCmd.Flags().String(xhyveDiskDriver, "ahci-hd", "The disk driver to use [ahci-hd|virtio-blk] (only supported with xhyve driver)")
 	startCmd.Flags().StringSlice(nfsShare, []string{}, "Local folders to share with Guest via NFS mounts (Only supported on with hyperkit now)")
 	startCmd.Flags().String(nfsSharesRoot, "/nfsshares", "Where to root the NFS Shares (defaults to /nfsshares, only supported with hyperkit now)")
@@ -156,6 +160,8 @@ func init() {
 	startCmd.Flags().Bool(gpu, false, "Enable experimental NVIDIA GPU support in minikube (works only with kvm2 driver on Linux)")
 	startCmd.Flags().Bool(hidden, false, "Hide the hypervisor signature from the guest in minikube (works only with kvm2 driver on Linux)")
 	startCmd.Flags().Bool(noVTXCheck, false, "Disable checking for the availability of hardware virtualization before the vm is started (virtualbox)")
+	startCmd.Flags().Bool(dnsProxy, false, "Enable proxy for NAT DNS requests (virtualbox)")
+	startCmd.Flags().Bool(hostDNSResolver, true, "Enable host resolver for NAT DNS requests (virtualbox)")
 	if err := viper.BindPFlags(startCmd.Flags()); err != nil {
 		exit.WithError("unable to bind flags", err)
 	}
@@ -520,13 +526,16 @@ func generateConfig(cmd *cobra.Command, k8sVersion string) (cfg.Config, error) {
 			RegistryMirror:      registryMirror,
 			HostOnlyCIDR:        viper.GetString(hostOnlyCIDR),
 			HypervVirtualSwitch: viper.GetString(hypervVirtualSwitch),
-			KvmNetwork:          viper.GetString(kvmNetwork),
+			KVMNetwork:          viper.GetString(kvmNetwork),
+			KVMQemuURI:          viper.GetString(kvmQemuURI),
 			Downloader:          pkgutil.DefaultDownloader{},
 			DisableDriverMounts: viper.GetBool(disableDriverMounts),
 			UUID:                viper.GetString(uuid),
 			GPU:                 viper.GetBool(gpu),
 			Hidden:              viper.GetBool(hidden),
 			NoVTXCheck:          viper.GetBool(noVTXCheck),
+			DNSProxy:            viper.GetBool(dnsProxy),
+			HostDNSResolver:     viper.GetBool(hostDNSResolver),
 		},
 		KubernetesConfig: cfg.KubernetesConfig{
 			KubernetesVersion:      k8sVersion,

deploy/addons/registry/registry-proxy.yaml.tmpl

@@ -2,7 +2,7 @@ apiVersion: extensions/v1beta1
 kind: DaemonSet
 metadata:
   labels:
-    kubernetes.io/minikube-addons: registry
+    kubernetes.io/minikube-addons: registry-proxy
     addonmanager.kubernetes.io/mode: Reconcile
   name: registry-proxy
   namespace: kube-system
@@ -10,7 +10,7 @@ spec:
   template:
     metadata:
       labels:
-        kubernetes.io/minikube-addons: registry
+        kubernetes.io/minikube-addons: registry-proxy
         addonmanager.kubernetes.io/mode: Reconcile
     spec:
       containers:

docs/drivers.md

@@ -99,7 +99,7 @@ virsh net-start default
 Make sure you are running the lastest version of your driver.
 
  ```shell
-docker-machine-driver-kvm2 --version
+docker-machine-driver-kvm2 version
 ```
 
 ## Hyperkit driver

pkg/drivers/hyperkit/driver.go

@@ -54,6 +54,7 @@ const (
 		"sudo chown root:wheel %s && sudo chmod u+s %s"
 )
 
+// Driver is the machine driver for Hyperkit
 type Driver struct {
 	*drivers.BaseDriver
 	*pkgdrivers.CommonDriver
@@ -69,6 +70,7 @@ type Driver struct {
 	VSockPorts     []string
 }
 
+// NewDriver creates a new driver for a host
 func NewDriver(hostName, storePath string) *Driver {
 	return &Driver{
 		BaseDriver: &drivers.BaseDriver{
@@ -98,6 +100,7 @@ func (d *Driver) verifyRootPermissions() error {
 	return nil
 }
 
+// Create a host using the driver's config
 func (d *Driver) Create() error {
 	if err := d.verifyRootPermissions(); err != nil {
 		return err
@@ -194,6 +197,7 @@ func (d *Driver) Remove() error {
 	return nil
 }
 
+// Restart a host
 func (d *Driver) Restart() error {
 	return pkgdrivers.Restart(d)
 }
@@ -342,7 +346,26 @@ func (d *Driver) Stop() error {
 		return err
 	}
 	d.cleanupNfsExports()
-	return d.sendSignal(syscall.SIGTERM)
+	err := d.sendSignal(syscall.SIGTERM)
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("hyperkit sigterm failed"))
+	}
+
+	// wait 5s for graceful shutdown
+	for i := 0; i < 5; i++ {
+		log.Debug("waiting for graceful shutdown")
+		time.Sleep(time.Second * 1)
+		s, err := d.GetState()
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("hyperkit waiting graceful shutdown failed"))
+		}
+		if s == state.Stopped {
+			return nil
+		}
+	}
+
+	log.Debug("sending sigkill")
+	return d.Kill()
 }
 
 func (d *Driver) extractKernel(isoPath string) error {

pkg/drivers/kvm/domain.go

@@ -128,7 +128,7 @@ func randomMAC() (net.HardwareAddr, error) {
 }
 
 func (d *Driver) getDomain() (*libvirt.Domain, *libvirt.Connect, error) {
-	conn, err := getConnection()
+	conn, err := getConnection(d.ConnectionURI)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "getting domain")
 	}
@@ -141,8 +141,8 @@ func (d *Driver) getDomain() (*libvirt.Domain, *libvirt.Connect, error) {
 	return dom, conn, nil
 }
 
-func getConnection() (*libvirt.Connect, error) {
-	conn, err := libvirt.NewConnect(qemusystem)
+func getConnection(connectionURI string) (*libvirt.Connect, error) {
+	conn, err := libvirt.NewConnect(connectionURI)
 	if err != nil {
 		return nil, errors.Wrap(err, connectionErrorText)
 	}
@@ -186,7 +186,7 @@ func (d *Driver) createDomain() (*libvirt.Domain, error) {
 		return nil, errors.Wrap(err, "executing domain xml")
 	}
 
-	conn, err := getConnection()
+	conn, err := getConnection(d.ConnectionURI)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error getting libvirt connection")
 	}

pkg/drivers/kvm/kvm.go

@@ -82,6 +82,9 @@ type Driver struct {
 
 	// XML that needs to be added to passthrough GPU devices.
 	DevicesXML string
+
+	// QEMU Connection URI
+	ConnectionURI string
 }
 
 const (
@@ -107,12 +110,13 @@ func NewDriver(hostName, storePath string) *Driver {
 		Network:        defaultNetworkName,
 		DiskPath:       filepath.Join(constants.GetMinipath(), "machines", config.GetMachineName(), fmt.Sprintf("%s.rawdisk", config.GetMachineName())),
 		ISO:            filepath.Join(constants.GetMinipath(), "machines", config.GetMachineName(), "boot2docker.iso"),
+		ConnectionURI:  qemusystem,
 	}
 }
 
 // PreCommandCheck checks the connection before issuing a command
 func (d *Driver) PreCommandCheck() error {
-	conn, err := getConnection()
+	conn, err := getConnection(d.ConnectionURI)
 	if err != nil {
 		return errors.Wrap(err, "Error connecting to libvirt socket.  Have you added yourself to the libvirtd group?")
 	}
@@ -424,7 +428,7 @@ func (d *Driver) Stop() (err error) {
 // Remove a host
 func (d *Driver) Remove() error {
 	log.Debug("Removing machine...")
-	conn, err := getConnection()
+	conn, err := getConnection(d.ConnectionURI)
 	if err != nil {
 		return errors.Wrap(err, "getting connection")
 	}

pkg/drivers/kvm/network.go

@@ -81,7 +81,7 @@ func setupNetwork(conn *libvirt.Connect, name string) error {
 
 // ensureNetwork is called on start of the VM
 func (d *Driver) ensureNetwork() error {
-	conn, err := getConnection()
+	conn, err := getConnection(d.ConnectionURI)
 	if err != nil {
 		return errors.Wrap(err, "getting libvirt connection")
 	}
@@ -108,12 +108,11 @@ func (d *Driver) ensureNetwork() error {
 
 // createNetwork is called during creation of the VM only (and not on start)
 func (d *Driver) createNetwork() error {
-
 	if d.Network == defaultPrivateNetworkName {
 		return fmt.Errorf("KVM network can't be named %s. This is the name of the private network created by minikube", defaultPrivateNetworkName)
 	}
 
-	conn, err := getConnection()
+	conn, err := getConnection(d.ConnectionURI)
 	if err != nil {
 		return errors.Wrap(err, "getting libvirt connection")
 	}
@@ -151,7 +150,7 @@ func (d *Driver) createNetwork() error {
 }
 
 func (d *Driver) deleteNetwork() error {
-	conn, err := getConnection()
+	conn, err := getConnection(d.ConnectionURI)
 	if err != nil {
 		return errors.Wrap(err, "getting libvirt connection")
 	}
@@ -269,7 +268,7 @@ func (d *Driver) checkDomains(conn *libvirt.Connect) error {
 }
 
 func (d *Driver) lookupIP() (string, error) {
-	conn, err := getConnection()
+	conn, err := getConnection(d.ConnectionURI)
 	if err != nil {
 		return "", errors.Wrap(err, "getting connection and domain")
 	}

pkg/drivers/kvm/version.go

@@ -21,7 +21,15 @@ package kvm
 // version is a private field and should be set when compiling with --ldflags="-X k8s.io/minikube/pkg/drivers/kvm.version=vX.Y.Z"
 var version = "v0.0.0-unset"
 
+// gitCommitID is a private field and should be set when compiling with --ldflags="-X k8s.io/minikube/pkg/drivers/kvm.gitCommitID=<commit-id>"
+var gitCommitID = ""
+
 // GetVersion returns the current docker-machine-driver-kvm2 version
 func GetVersion() string {
 	return version
 }
+
+// GetGitCommitID returns the git commit id from which it is being built
+func GetGitCommitID() string {
+	return gitCommitID
+}

pkg/minikube/config/types.go

@@ -45,7 +45,8 @@ type MachineConfig struct {
 	RegistryMirror      []string
 	HostOnlyCIDR        string // Only used by the virtualbox driver
 	HypervVirtualSwitch string
-	KvmNetwork          string             // Only used by the KVM driver
+	KVMNetwork          string             // Only used by the KVM driver
+	KVMQemuURI          string             // Only used by kvm2
 	Downloader          util.ISODownloader `json:"-"`
 	DockerOpt           []string           // Each entry is formatted as KEY=VALUE.
 	DisableDriverMounts bool               // Only used by virtualbox and xhyve
@@ -55,6 +56,8 @@ type MachineConfig struct {
 	GPU                 bool   // Only used by kvm2
 	Hidden              bool   // Only used by kvm2
 	NoVTXCheck          bool   // Only used by virtualbox
+	DNSProxy            bool   // Only used by virtualbox
+	HostDNSResolver     bool   // Only used by virtualbox
 }
 
 // KubernetesConfig contains the parameters used to configure the VM Kubernetes.

pkg/minikube/drivers/kvm/driver.go

@@ -64,7 +64,7 @@ func createKVMHost(config cfg.MachineConfig) interface{} {
 		},
 		Memory:         config.Memory,
 		CPU:            config.CPUs,
-		Network:        config.KvmNetwork,
+		Network:        config.KVMNetwork,
 		PrivateNetwork: "docker-machines",
 		Boot2DockerURL: config.Downloader.GetISOFileURI(config.MinikubeISO),
 		DiskSize:       config.DiskSize,

pkg/minikube/drivers/kvm2/driver.go

@@ -53,6 +53,7 @@ type kvmDriver struct {
 	DiskPath       string
 	GPU            bool
 	Hidden         bool
+	ConnectionURI  string
 }
 
 func createKVM2Host(config cfg.MachineConfig) interface{} {
@@ -64,7 +65,7 @@ func createKVM2Host(config cfg.MachineConfig) interface{} {
 		},
 		Memory:         config.Memory,
 		CPU:            config.CPUs,
-		Network:        config.KvmNetwork,
+		Network:        config.KVMNetwork,
 		PrivateNetwork: "minikube-net",
 		Boot2DockerURL: config.Downloader.GetISOFileURI(config.MinikubeISO),
 		DiskSize:       config.DiskSize,
@@ -72,5 +73,6 @@ func createKVM2Host(config cfg.MachineConfig) interface{} {
 		ISO:            filepath.Join(constants.GetMinipath(), "machines", cfg.GetMachineName(), "boot2docker.iso"),
 		GPU:            config.GPU,
 		Hidden:         config.Hidden,
+		ConnectionURI:  config.KVMQemuURI,
 	}
 }

pkg/minikube/drivers/virtualbox/driver.go

@@ -54,8 +54,8 @@ func createVirtualboxHost(config cfg.MachineConfig) interface{} {
 	d.NoVTXCheck = config.NoVTXCheck
 	d.NatNicType = defaultVirtualboxNicType
 	d.HostOnlyNicType = defaultVirtualboxNicType
-	d.DNSProxy = false
-	d.HostDNSResolver = true
+	d.DNSProxy = config.DNSProxy
+	d.HostDNSResolver = config.HostDNSResolver
 
 	return d
 }

pkg/version/version.go

@@ -30,7 +30,7 @@ const VersionPrefix = "v"
 // version is a private field and should be set when compiling with --ldflags="-X k8s.io/minikube/pkg/version.version=vX.Y.Z"
 var version = "v0.0.0-unset"
 
-// version is a private field and should be set when compiling with --ldflags="-X k8s.io/minikube/pkg/version.gitCommitID=<commit-id>"
+// gitCommitID is a private field and should be set when compiling with --ldflags="-X k8s.io/minikube/pkg/version.gitCommitID=<commit-id>"
 var gitCommitID = ""
 
 // isoVersion is a private field and should be set when compiling with --ldflags="-X k8s.io/minikube/pkg/version.isoVersion=vX.Y.Z"

test/integration/addons_test.go

@@ -190,7 +190,57 @@ func testServicesList(t *testing.T) {
 		t.Fatalf(err.Error())
 	}
 }
+func testRegistry(t *testing.T) {
+	t.Parallel()
+	minikubeRunner := NewMinikubeRunner(t)
+	kubectlRunner := util.NewKubectlRunner(t)
+	minikubeRunner.RunCommand("addons enable registry", true)
+	t.Log("wait for registry to come up")
 
+	if err := util.WaitForDockerRegistryRunning(t); err != nil {
+		t.Fatalf("waiting for registry to be up: %v", err)	
+	}
+	
+	// Check access from outside the cluster on port 5000, validing connectivity via registry-proxy
+	checkExternalAccess := func() error {
+		t.Log("checking registry access from outside cluster")
+		expectedStr := "200"
+		runCmd := fmt.Sprintf("curl -sS -o /dev/null -w '%%{http_code}' http://127.0.0.1:5000")
+		externalCheckOutput, _ := minikubeRunner.SSH(runCmd)
+		if !strings.Contains(externalCheckOutput, expectedStr) {
+			return fmt.Errorf("ExpectedStr externalCheckOutput to be: %s. Output was: %s", expectedStr, externalCheckOutput)
+		}
+		return nil
+	}
+
+	if err := util.Retry(t, checkExternalAccess, 2*time.Second, 5); err != nil {
+		t.Fatalf(err.Error())
+	}
+	// check access from inside the cluster via a busybox container running inside cluster
+	t.Log("checking registry access from inside cluster")
+	expectedStr := "200"
+	out, _ := kubectlRunner.RunCommand([]string{
+		"run",
+		"registry-test",
+		"--restart=Never",
+		"--image=busybox",
+		"-it",
+		"--",
+		"sh",
+		"-c",
+		"wget --spider -S 'http://registry.kube-system.svc.cluster.local' 2>&1 | grep 'HTTP/' | awk '{print $2}'"})
+	internalCheckOutput := string(out)
+	if !strings.Contains(internalCheckOutput, expectedStr) {
+		t.Fatalf("ExpectedStr internalCheckOutput to be: %s. Output was: %s", expectedStr, internalCheckOutput)
+	}
+	
+	defer func() {
+		if _, err := kubectlRunner.RunCommand([]string{"delete", "pod", "registry-test"}); err != nil {
+			t.Fatalf("failed to delete pod registry-test")
+		}	
+	}()
+	minikubeRunner.RunCommand("addons disable registry", true)
+}
 func testGvisor(t *testing.T) {
 	minikubeRunner := NewMinikubeRunner(t)
 	minikubeRunner.RunCommand("addons enable gvisor", true)

test/integration/functional_test.go

@@ -36,6 +36,7 @@ func TestFunctional(t *testing.T) {
 	t.Run("DNS", testClusterDNS)
 	t.Run("Logs", testClusterLogs)
 	t.Run("Addons", testAddons)
+	t.Run("Registry", testRegistry)
 	t.Run("Dashboard", testDashboard)
 	t.Run("ServicesList", testServicesList)
 	t.Run("Provisioning", testProvisioning)

test/integration/start_stop_delete_test.go

@@ -41,6 +41,7 @@ func TestStartStop(t *testing.T) {
 			// default is the network created by libvirt, if we change the name minikube won't boot
 			// because the given network doesn't exist
 			"--kvm-network=default",
+			"--kvm-qemu-uri=qemu:///system",
 		}},
 		{"feature_gates_newest_cni", []string{
 			"--feature-gates",

test/integration/util/util.go

@@ -359,6 +359,28 @@ func WaitForIngressControllerRunning(t *testing.T) error {
 	return nil
 }
 
+// WaitForDockerRegistryRunning waits until docker registry pod to be running
+func WaitForDockerRegistryRunning(t *testing.T) error {
+	client, err := commonutil.GetClient()
+	if err != nil {
+		return errors.Wrap(err, "getting kubernetes client")
+	}
+
+	if err := commonutil.WaitForRCToStabilize(client, "kube-system", "registry", time.Minute*10); err != nil {
+		return errors.Wrap(err, "waiting for registry replicacontroller to stabilize")
+	}
+
+	registrySelector := labels.SelectorFromSet(labels.Set(map[string]string{"kubernetes.io/minikube-addons": "registry"}))
+	if err := commonutil.WaitForPodsWithLabelRunning(client, "kube-system", registrySelector); err != nil {
+		return errors.Wrap(err, "waiting for registry pods")
+	}
+	proxySelector := labels.SelectorFromSet(labels.Set(map[string]string{"kubernetes.io/minikube-addons": "registry-proxy"}))
+	if err := commonutil.WaitForPodsWithLabelRunning(client, "kube-system", proxySelector); err != nil {
+		return errors.Wrap(err, "waiting for registry-proxy pods")
+	}
+	return nil
+}
+
 // WaitForIngressDefaultBackendRunning waits until ingress default backend pod to be running
 func WaitForIngressDefaultBackendRunning(t *testing.T) error {
 	client, err := commonutil.GetClient()