diff --git a/cmd/minikube/cmd/docker-env.go b/cmd/minikube/cmd/docker-env.go index 2a4efb31c0..9f41275188 100644 --- a/cmd/minikube/cmd/docker-env.go +++ b/cmd/minikube/cmd/docker-env.go @@ -38,6 +38,7 @@ import ( "k8s.io/klog/v2" "k8s.io/minikube/pkg/drivers/kic/oci" + "k8s.io/minikube/pkg/drivers/qemu" "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/constants" @@ -290,11 +291,13 @@ var dockerEnvCmd = &cobra.Command{ d := co.CP.Host.Driver port := constants.DockerDaemonPort - if driver.NeedsPortForward(driverName) { + if driver.NeedsPortForward(driverName) && driver.IsKIC(driverName) { port, err = oci.ForwardedPort(driverName, cname, port) if err != nil { exit.Message(reason.DrvPortForward, "Error getting port binding for '{{.driver_name}} driver: {{.error}}", out.V{"driver_name": driverName, "error": err}) } + } else if driver.NeedsPortForward(driverName) && driverName == driver.QEMU2 { + port = d.(*qemu.Driver).EnginePort } hostname, err := d.GetSSHHostname() diff --git a/cmd/minikube/cmd/service.go b/cmd/minikube/cmd/service.go index 2f15ca38ad..59e4bfee16 100644 --- a/cmd/minikube/cmd/service.go +++ b/cmd/minikube/cmd/service.go @@ -140,8 +140,10 @@ You may select another namespace by using 'minikube service {{.service}} -n %d", minPort, maxPort) + if err != nil { + return err + } + d.SSHPort, err = getAvailableTCPPortFromRange(minPort, maxPort) + if err != nil { + return err + } + + for { + d.EnginePort, err = getAvailableTCPPortFromRange(minPort, maxPort) + if err != nil { + return err + } + if d.EnginePort == d.SSHPort { + // can't have both on same port + continue + } + break + } + } + b2dutils := mcnutils.NewB2dUtils(d.StorePath) + if err := b2dutils.CopyIsoToMachineDir(d.Boot2DockerURL, d.MachineName); err != nil { + return err + } + + log.Infof("Creating SSH key...") + if err := ssh.GenerateSSHKey(d.sshKeyPath()); err != nil { + return err + } + + log.Infof("Creating Disk image...") + if err := d.generateDiskImage(d.DiskSize); err != nil { + return err + } + + if d.UserDataFile != "" { + log.Infof("Creating Userdata Disk...") + if d.CloudConfigRoot, err = d.generateUserdataDisk(d.UserDataFile); err != nil { + return err + } + } + + log.Infof("Starting QEMU VM...") + return d.Start() +} + +func parsePortRange(rawPortRange string) (int, int, error) { + if rawPortRange == "" { + return 0, 65535, nil + } + + portRange := strings.Split(rawPortRange, "-") + + minPort, err := strconv.Atoi(portRange[0]) + if err != nil { + return 0, 0, fmt.Errorf("Invalid port range") + } + maxPort, err := strconv.Atoi(portRange[1]) + if err != nil { + return 0, 0, fmt.Errorf("Invalid port range") + } + + if maxPort < minPort { + return 0, 0, fmt.Errorf("Invalid port range") + } + + if maxPort-minPort < 2 { + return 0, 0, fmt.Errorf("Port range must be minimum 2 ports") + } + + return minPort, maxPort, nil +} + +func getRandomPortNumberInRange(min int, max int) int { + return rand.Intn(max-min) + min +} + +func getAvailableTCPPortFromRange(minPort int, maxPort int) (int, error) { + port := 0 + for i := 0; i <= 10; i++ { + var ln net.Listener + var err error + if minPort == 0 && maxPort == 65535 { + ln, err = net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + return 0, err + } + } else { + port = getRandomPortNumberInRange(minPort, maxPort) + log.Debugf("testing port: %d", port) + ln, err = net.Listen("tcp4", fmt.Sprintf("127.0.0.1:%d", port)) + if err != nil { + log.Debugf("port already in use: %d", port) + continue + } + } + defer ln.Close() + addr := ln.Addr().String() + addrParts := strings.SplitN(addr, ":", 2) + p, err := strconv.Atoi(addrParts[1]) + if err != nil { + return 0, err + } + if p != 0 { + port = p + return port, nil + } + time.Sleep(1) + } + return 0, fmt.Errorf("unable to allocate tcp port") +} + +func (d *Driver) Start() error { + // fmt.Printf("Init qemu %s\n", i.VM) + machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) + + var startCmd []string + + if d.MachineType != "" { + machineType := d.MachineType + if runtime.GOOS == "darwin" { + // highmem=off needed, see https://patchwork.kernel.org/project/qemu-devel/patch/20201126215017.41156-9-agraf@csgraf.de/#23800615 for details + machineType += ",accel=hvf,highmem=off" + } + startCmd = append(startCmd, + "-M", machineType, + ) + } + if d.CPUType != "" { + startCmd = append(startCmd, + "-cpu", d.CPUType, + ) + } + + if !d.BIOS { + if d.Firmware != "" { + startCmd = append(startCmd, + "-drive", fmt.Sprintf("file=%s,readonly=on,format=raw,if=pflash", d.Firmware)) + } else { + return fmt.Errorf("unknown firmware") + } + } + + if d.Display { + if d.DisplayType != "" { + startCmd = append(startCmd, + "-display", d.DisplayType, + ) + } else { + // Use the default graphic output + } + } else { + if d.Nographic { + startCmd = append(startCmd, + "-nographic", + ) + } else { + startCmd = append(startCmd, + "-display", "none", + ) + } + } + + startCmd = append(startCmd, + "-m", fmt.Sprintf("%d", d.Memory), + "-smp", fmt.Sprintf("%d", d.CPU), + "-boot", "d") + var isoPath = filepath.Join(machineDir, isoFilename) + if d.VirtioDrives { + startCmd = append(startCmd, + "-drive", fmt.Sprintf("file=%s,index=2,media=cdrom,if=virtio", isoPath)) + } else { + startCmd = append(startCmd, + "-cdrom", isoPath) + } + startCmd = append(startCmd, + "-qmp", fmt.Sprintf("unix:%s,server,nowait", d.monitorPath()), + "-pidfile", d.pidfilePath(), + ) + + if d.Network == "user" { + startCmd = append(startCmd, + "-nic", fmt.Sprintf("user,model=virtio,hostfwd=tcp::%d-:22,hostfwd=tcp::%d-:2376,hostname=%s", d.SSHPort, d.EnginePort, d.GetMachineName()), + ) + } else if d.Network == "tap" { + startCmd = append(startCmd, + "-nic", fmt.Sprintf("tap,model=virtio,ifname=%s,script=no,downscript=no", d.NetworkInterface), + ) + } else if d.Network == "vde" { + startCmd = append(startCmd, + "-nic", fmt.Sprintf("vde,model=virtio,sock=%s", d.NetworkSocket), + ) + } else if d.Network == "bridge" { + startCmd = append(startCmd, + "-nic", fmt.Sprintf("bridge,model=virtio,br=%s", d.NetworkBridge), + ) + } else { + log.Errorf("Unknown network: %s", d.Network) + } + + startCmd = append(startCmd, "-daemonize") + + // other options + // "-enable-kvm" if its available + if _, err := os.Stat("/dev/kvm"); err == nil { + startCmd = append(startCmd, "-enable-kvm") + } + + if d.CloudConfigRoot != "" { + startCmd = append(startCmd, + "-fsdev", + fmt.Sprintf("local,security_model=passthrough,readonly,id=fsdev0,path=%s", d.CloudConfigRoot)) + startCmd = append(startCmd, "-device", "virtio-9p-pci,id=fs0,fsdev=fsdev0,mount_tag=config-2") + } + + if d.VirtioDrives { + startCmd = append(startCmd, + "-drive", fmt.Sprintf("file=%s,index=0,media=disk,if=virtio", d.diskPath())) + } else { + // last argument is always the name of the disk image + startCmd = append(startCmd, d.diskPath()) + } + + if stdout, stderr, err := cmdOutErr(d.Program, startCmd...); err != nil { + fmt.Printf("OUTPUT: %s\n", stdout) + fmt.Printf("ERROR: %s\n", stderr) + return err + } + log.Infof("Waiting for VM to start (ssh -p %d docker@localhost)...", d.SSHPort) + + return WaitForTCPWithDelay(fmt.Sprintf("localhost:%d", d.SSHPort), time.Second) +} + +func cmdOutErr(cmdStr string, args ...string) (string, string, error) { + cmd := exec.Command(cmdStr, args...) + log.Debugf("executing: %v %v", cmdStr, strings.Join(args, " ")) + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + stderrStr := stderr.String() + log.Debugf("STDOUT: %v", stdout.String()) + log.Debugf("STDERR: %v", stderrStr) + if err != nil { + if ee, ok := err.(*exec.Error); ok && ee == exec.ErrNotFound { + err = fmt.Errorf("mystery error: %s", ee) + } + } else { + // also catch error messages in stderr, even if the return code + // looks OK + if strings.Contains(stderrStr, "error:") { + err = fmt.Errorf("%v %v failed: %v", cmdStr, strings.Join(args, " "), stderrStr) + } + } + return stdout.String(), stderrStr, err +} + +func (d *Driver) Stop() error { + // _, err := d.RunQMPCommand("stop") + _, err := d.RunQMPCommand("system_powerdown") + if err != nil { + return err + } + return nil +} + +func (d *Driver) Remove() error { + s, err := d.GetState() + if err != nil { + return err + } + if s == state.Running { + if err := d.Kill(); err != nil { + return err + } + } + if s != state.Stopped { + _, err = d.RunQMPCommand("quit") + if err != nil { + return err + } + } + return nil +} + +func (d *Driver) Restart() error { + s, err := d.GetState() + if err != nil { + return err + } + + if s == state.Running { + if err := d.Stop(); err != nil { + return err + } + } + return d.Start() +} + +func (d *Driver) Kill() error { + // _, err := d.RunQMPCommand("quit") + _, err := d.RunQMPCommand("system_powerdown") + if err != nil { + return err + } + return nil +} + +func (d *Driver) StartDocker() error { + return fmt.Errorf("hosts without a driver cannot start docker") +} + +func (d *Driver) StopDocker() error { + return fmt.Errorf("hosts without a driver cannot stop docker") +} + +func (d *Driver) GetDockerConfigDir() string { + return "" +} + +func (d *Driver) Upgrade() error { + return fmt.Errorf("hosts without a driver cannot be upgraded") +} + +func (d *Driver) sshKeyPath() string { + machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) + return filepath.Join(machineDir, "id_rsa") +} + +func (d *Driver) publicSSHKeyPath() string { + return d.sshKeyPath() + ".pub" +} + +func (d *Driver) diskPath() string { + machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) + return filepath.Join(machineDir, "disk.qcow2") +} + +func (d *Driver) monitorPath() string { + machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) + return filepath.Join(machineDir, "monitor") +} + +func (d *Driver) pidfilePath() string { + machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) + return filepath.Join(machineDir, "qemu.pid") +} + +// Make a boot2docker VM disk image. +func (d *Driver) generateDiskImage(size int) error { + log.Debugf("Creating %d MB hard disk image...", size) + + magicString := "boot2docker, please format-me" + + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + + // magicString first so the automount script knows to format the disk + file := &tar.Header{Name: magicString, Size: int64(len(magicString))} + if err := tw.WriteHeader(file); err != nil { + return err + } + if _, err := tw.Write([]byte(magicString)); err != nil { + return err + } + // .ssh/key.pub => authorized_keys + file = &tar.Header{Name: ".ssh", Typeflag: tar.TypeDir, Mode: 0700} + if err := tw.WriteHeader(file); err != nil { + return err + } + pubKey, err := ioutil.ReadFile(d.publicSSHKeyPath()) + if err != nil { + return err + } + file = &tar.Header{Name: ".ssh/authorized_keys", Size: int64(len(pubKey)), Mode: 0644} + if err := tw.WriteHeader(file); err != nil { + return err + } + if _, err := tw.Write(pubKey); err != nil { + return err + } + file = &tar.Header{Name: ".ssh/authorized_keys2", Size: int64(len(pubKey)), Mode: 0644} + if err := tw.WriteHeader(file); err != nil { + return err + } + if _, err := tw.Write(pubKey); err != nil { + return err + } + if err := tw.Close(); err != nil { + return err + } + rawFile := fmt.Sprintf("%s.raw", d.diskPath()) + if err := ioutil.WriteFile(rawFile, buf.Bytes(), 0644); err != nil { + return nil + } + if stdout, stderr, err := cmdOutErr("qemu-img", "convert", "-f", "raw", "-O", "qcow2", rawFile, d.diskPath()); err != nil { + fmt.Printf("OUTPUT: %s\n", stdout) + fmt.Printf("ERROR: %s\n", stderr) + return err + } + if stdout, stderr, err := cmdOutErr("qemu-img", "resize", d.diskPath(), fmt.Sprintf("+%dM", size)); err != nil { + fmt.Printf("OUTPUT: %s\n", stdout) + fmt.Printf("ERROR: %s\n", stderr) + return err + } + log.Debugf("DONE writing to %s and %s", rawFile, d.diskPath()) + + return nil +} + +func (d *Driver) generateUserdataDisk(userdataFile string) (string, error) { + // Start with virtio, add ISO & FAT format later + // Start with local file, add wget/fetct URL? (or if URL, use datasource..) + userdata, err := ioutil.ReadFile(userdataFile) + if err != nil { + return "", err + } + + machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName()) + ccRoot := filepath.Join(machineDir, "cloud-config") + err = os.MkdirAll(ccRoot, 0755) + if err != nil { + return "", err + } + + userDataDir := filepath.Join(ccRoot, "openstack/latest") + err = os.MkdirAll(userDataDir, 0755) + if err != nil { + return "", err + } + + writeFile := filepath.Join(userDataDir, "user_data") + if err := ioutil.WriteFile(writeFile, userdata, 0644); err != nil { + return "", err + } + + return ccRoot, nil + +} + +func (d *Driver) RunQMPCommand(command string) (map[string]interface{}, error) { + + // connect to monitor + conn, err := net.Dial("unix", d.monitorPath()) + if err != nil { + return nil, err + } + defer conn.Close() + + // initial QMP response + var buf [1024]byte + nr, err := conn.Read(buf[:]) + if err != nil { + return nil, err + } + type qmpInitialResponse struct { + QMP struct { + Version struct { + QEMU struct { + Micro int `json:"micro"` + Minor int `json:"minor"` + Major int `json:"major"` + } `json:"qemu"` + Package string `json:"package"` + } `json:"version"` + Capabilities []string `json:"capabilities"` + } `jason:"QMP"` + } + + var initialResponse qmpInitialResponse + err = json.Unmarshal(buf[:nr], &initialResponse) + if err != nil { + return nil, err + } + + // run 'qmp_capabilities' to switch to command mode + // { "execute": "qmp_capabilities" } + type qmpCommand struct { + Command string `json:"execute"` + } + jsonCommand, err := json.Marshal(qmpCommand{Command: "qmp_capabilities"}) + if err != nil { + return nil, err + } + _, err = conn.Write(jsonCommand) + if err != nil { + return nil, err + } + nr, err = conn.Read(buf[:]) + if err != nil { + return nil, err + } + type qmpResponse struct { + Return map[string]interface{} `json:"return"` + } + var response qmpResponse + err = json.Unmarshal(buf[:nr], &response) + if err != nil { + return nil, err + } + // expecting empty response + if len(response.Return) != 0 { + return nil, fmt.Errorf("qmp_capabilities failed: %v", response.Return) + } + + // { "execute": command } + jsonCommand, err = json.Marshal(qmpCommand{Command: command}) + if err != nil { + return nil, err + } + _, err = conn.Write(jsonCommand) + if err != nil { + return nil, err + } + nr, err = conn.Read(buf[:]) + if err != nil { + return nil, err + } + err = json.Unmarshal(buf[:nr], &response) + if err != nil { + return nil, err + } + if strings.HasPrefix(command, "query-") { + return response.Return, nil + } + // non-query commands should return an empty response + if len(response.Return) != 0 { + return nil, fmt.Errorf("%s failed: %v", command, response.Return) + } + return response.Return, nil +} + +func WaitForTCPWithDelay(addr string, duration time.Duration) error { + for { + conn, err := net.Dial("tcp", addr) + if err != nil { + continue + } + defer conn.Close() + if _, err = conn.Read(make([]byte, 1)); err != nil { + time.Sleep(duration) + continue + } + break + } + return nil +} diff --git a/pkg/minikube/cluster/ip.go b/pkg/minikube/cluster/ip.go index bb73ca8956..313fb72656 100644 --- a/pkg/minikube/cluster/ip.go +++ b/pkg/minikube/cluster/ip.go @@ -58,6 +58,10 @@ func HostIP(host *host.Host, clusterName string) (net.IP, error) { return []byte{}, errors.Wrap(err, "Error converting VM/Host IP address to IPv4 address") } return net.IPv4(vmIP[0], vmIP[1], vmIP[2], byte(1)), nil + case driver.QEMU2: + return net.ParseIP("10.0.2.2"), nil + case driver.QEMU: + return net.ParseIP("10.0.2.2"), nil case driver.HyperV: v := reflect.ValueOf(host.Driver).Elem() var hypervVirtualSwitch string @@ -147,6 +151,9 @@ func DriverIP(api libmachine.API, machineName string) (net.IP, error) { if driver.IsKIC(host.DriverName) { ipStr = oci.DefaultBindIPV4 } + if driver.IsQEMU(host.DriverName) { + ipStr = "127.0.0.1" + } ip := net.ParseIP(ipStr) if ip == nil { return nil, fmt.Errorf("parsing IP: %s", ipStr) diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index c1af9ace7a..4566aeb27b 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -52,11 +52,12 @@ type ClusterConfig struct { HypervVirtualSwitch string HypervUseExternalSwitch bool HypervExternalAdapter string - KVMNetwork string // Only used by the KVM2 driver - KVMQemuURI string // Only used by the KVM2 driver - KVMGPU bool // Only used by the KVM2 driver - KVMHidden bool // Only used by the KVM2 driver - KVMNUMACount int // Only used by the KVM2 driver + KVMNetwork string // Only used by the KVM2 driver + KVMQemuURI string // Only used by the KVM2 driver + KVMGPU bool // Only used by the KVM2 driver + KVMHidden bool // Only used by the KVM2 driver + KVMNUMACount int // Only used by the KVM2 driver + APIServerPort int DockerOpt []string // Each entry is formatted as KEY=VALUE. DisableDriverMounts bool // Only used by virtualbox NFSShare []string diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index 0b2e317c93..cae2cdd08e 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -46,6 +46,10 @@ const ( SSH = "ssh" // KVM2 driver KVM2 = "kvm2" + // QEMU2 driver + QEMU2 = "qemu2" + // QEMU driver + QEMU = "qemu" // VirtualBox driver VirtualBox = "virtualbox" // HyperKit driver @@ -156,6 +160,11 @@ func IsKVM(name string) bool { return name == KVM2 || name == AliasKVM } +// IsQEMU checks if the driver is a QEMU[2] +func IsQEMU(name string) bool { + return name == QEMU2 || name == QEMU +} + // IsVM checks if the driver is a VM func IsVM(name string) bool { if IsKIC(name) || BareMetal(name) { @@ -181,6 +190,9 @@ func AllowsPreload(driverName string) bool { // NeedsPortForward returns true if driver is unable provide direct IP connectivity func NeedsPortForward(name string) bool { + if IsQEMU(name) { + return true + } if !IsKIC(name) { return false } diff --git a/pkg/minikube/driver/driver_darwin.go b/pkg/minikube/driver/driver_darwin.go index 195f9841c0..cfeba75bbf 100644 --- a/pkg/minikube/driver/driver_darwin.go +++ b/pkg/minikube/driver/driver_darwin.go @@ -27,6 +27,7 @@ var supportedDrivers = func() []string { if runtime.GOARCH == "arm64" { // on darwin/arm64 only docker and ssh are supported yet return []string{ + QEMU2, Docker, Podman, SSH, @@ -50,6 +51,7 @@ var supportedDrivers = func() []string { VMwareFusion, HyperKit, VMware, + QEMU2, Docker, Podman, SSH, diff --git a/pkg/minikube/driver/driver_linux.go b/pkg/minikube/driver/driver_linux.go index a428a9a2bf..e12c23d702 100644 --- a/pkg/minikube/driver/driver_linux.go +++ b/pkg/minikube/driver/driver_linux.go @@ -25,6 +25,8 @@ var supportedDrivers = []string{ VirtualBox, VMwareFusion, KVM2, + QEMU2, + QEMU, VMware, None, Docker, diff --git a/pkg/minikube/driver/driver_test.go b/pkg/minikube/driver/driver_test.go index 021f78c74c..0b7806610e 100644 --- a/pkg/minikube/driver/driver_test.go +++ b/pkg/minikube/driver/driver_test.go @@ -66,6 +66,8 @@ func TestMachineType(t *testing.T) { None: "bare metal machine", SSH: "bare metal machine", KVM2: "VM", + QEMU2: "VM", + QEMU: "VM", VirtualBox: "VM", HyperKit: "VM", VMware: "VM", diff --git a/pkg/minikube/driver/driver_windows.go b/pkg/minikube/driver/driver_windows.go index 8352366ae2..51defd6228 100644 --- a/pkg/minikube/driver/driver_windows.go +++ b/pkg/minikube/driver/driver_windows.go @@ -32,6 +32,7 @@ var supportedDrivers = []string{ VMwareFusion, HyperV, VMware, + QEMU2, Docker, Podman, SSH, diff --git a/pkg/minikube/driver/endpoint.go b/pkg/minikube/driver/endpoint.go index 15203db100..aa02afc9af 100644 --- a/pkg/minikube/driver/endpoint.go +++ b/pkg/minikube/driver/endpoint.go @@ -28,7 +28,7 @@ import ( // ControlPlaneEndpoint returns the location where callers can reach this cluster func ControlPlaneEndpoint(cc *config.ClusterConfig, cp *config.Node, driverName string) (string, net.IP, int, error) { - if NeedsPortForward(driverName) { + if NeedsPortForward(driverName) && IsKIC(driverName) { port, err := oci.ForwardedPort(cc.Driver, cc.Name, cp.Port) if err != nil { klog.Warningf("failed to get forwarded control plane port %v", err) @@ -45,6 +45,8 @@ func ControlPlaneEndpoint(cc *config.ClusterConfig, cp *config.Node, driverName hostname = cc.KubernetesConfig.APIServerName } return hostname, ips[0], port, err + } else if NeedsPortForward(driverName) && IsQEMU(driverName) { + return "localhost", net.IPv4(127, 0, 0, 1), cc.APIServerPort, nil } // https://github.com/kubernetes/minikube/issues/3878 diff --git a/pkg/minikube/machine/machine.go b/pkg/minikube/machine/machine.go index a465694e0b..9f96600fa3 100644 --- a/pkg/minikube/machine/machine.go +++ b/pkg/minikube/machine/machine.go @@ -122,6 +122,9 @@ func saveHost(api libmachine.API, h *host.Host, cfg *config.ClusterConfig, n *co if err != nil { return err } + if ip == "127.0.0.1" && driver.IsQEMU(h.Driver.DriverName()) { + ip = "10.0.2.15" + } n.IP = ip return config.SaveNode(cfg, n) } diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 5a8bbaa185..b721f04361 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -253,6 +253,15 @@ func handleAPIServer(starter Starter, cr cruntime.Manager, hostIP net.IP) (*kube return nil, bs, err } + // Tunnel apiserver to guest, if needed + if starter.Cfg.APIServerPort != 0 { + args := []string{"-f", "-NTL", fmt.Sprintf("%d:localhost:8443", starter.Cfg.APIServerPort)} + err := machine.CreateSSHShell(starter.MachineAPI, *starter.Cfg, *starter.Node, args, false) + if err != nil { + klog.Warningf("apiserver tunnel failed: %v", err) + } + } + // Write the kubeconfig to the file system after everything required (like certs) are created by the bootstrapper. if err := kubeconfig.Update(kcs); err != nil { return nil, bs, errors.Wrap(err, "Failed kubeconfig update") @@ -558,6 +567,14 @@ func startMachine(cfg *config.ClusterConfig, node *config.Node, delOnFail bool) return runner, preExists, m, host, errors.Wrap(err, "Failed to validate network") } + if driver.IsQEMU(host.Driver.DriverName()) { + apiServerPort, err := getPort() + if err != nil { + return runner, preExists, m, host, errors.Wrap(err, "Failed to find apiserver port") + } + cfg.APIServerPort = apiServerPort + } + // Bypass proxy for minikube's vm host ip err = proxy.ExcludeIP(ip) if err != nil { @@ -567,6 +584,21 @@ func startMachine(cfg *config.ClusterConfig, node *config.Node, delOnFail bool) return runner, preExists, m, host, err } +// getPort asks the kernel for a free open port that is ready to use +func getPort() (int, error) { + addr, err := net.ResolveTCPAddr("tcp", "localhost:0") + if err != nil { + panic(err) + } + + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return -1, errors.Errorf("Error accessing port %d", addr.Port) + } + defer l.Close() + return l.Addr().(*net.TCPAddr).Port, nil +} + // startHostInternal starts a new minikube host using a VM or None func startHostInternal(api libmachine.API, cc *config.ClusterConfig, n *config.Node, delOnFail bool) (*host.Host, bool, error) { host, exists, err := machine.StartHost(api, cc, n) @@ -638,7 +670,7 @@ func validateNetwork(h *host.Host, r command.Runner, imageRepository string) (st } } - if !driver.BareMetal(h.Driver.DriverName()) && !driver.IsKIC(h.Driver.DriverName()) { + if !driver.BareMetal(h.Driver.DriverName()) && !driver.IsKIC(h.Driver.DriverName()) && !driver.IsQEMU(h.Driver.DriverName()) { if err := trySSH(h, ip); err != nil { return ip, err } diff --git a/pkg/minikube/registry/drvs/init.go b/pkg/minikube/registry/drvs/init.go index 06b1115827..6176d892ef 100644 --- a/pkg/minikube/registry/drvs/init.go +++ b/pkg/minikube/registry/drvs/init.go @@ -25,6 +25,8 @@ import ( _ "k8s.io/minikube/pkg/minikube/registry/drvs/none" _ "k8s.io/minikube/pkg/minikube/registry/drvs/parallels" _ "k8s.io/minikube/pkg/minikube/registry/drvs/podman" + _ "k8s.io/minikube/pkg/minikube/registry/drvs/qemu" + _ "k8s.io/minikube/pkg/minikube/registry/drvs/qemu2" _ "k8s.io/minikube/pkg/minikube/registry/drvs/ssh" _ "k8s.io/minikube/pkg/minikube/registry/drvs/virtualbox" _ "k8s.io/minikube/pkg/minikube/registry/drvs/vmware" diff --git a/pkg/minikube/registry/drvs/qemu/doc.go b/pkg/minikube/registry/drvs/qemu/doc.go new file mode 100644 index 0000000000..ea4425ce8d --- /dev/null +++ b/pkg/minikube/registry/drvs/qemu/doc.go @@ -0,0 +1,17 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qemu diff --git a/pkg/minikube/registry/drvs/qemu/qemu.go b/pkg/minikube/registry/drvs/qemu/qemu.go new file mode 100644 index 0000000000..24f15693eb --- /dev/null +++ b/pkg/minikube/registry/drvs/qemu/qemu.go @@ -0,0 +1,75 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qemu + +import ( + "fmt" + "os/exec" + "path/filepath" + + "github.com/docker/machine/libmachine/drivers" + drvqemu "github.com/machine-drivers/docker-machine-driver-qemu" + + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" + "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/minikube/registry" +) + +const ( + docURL = "https://minikube.sigs.k8s.io/docs/reference/drivers/qemu/" +) + +func init() { + if err := registry.Register(registry.DriverDef{ + Name: driver.QEMU, + Config: configure, + Status: status, + Default: true, + Priority: registry.Experimental, + }); err != nil { + panic(fmt.Sprintf("register failed: %v", err)) + } +} + +func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { + name := config.MachineName(cc, n) + return drvqemu.Driver{ + BaseDriver: &drivers.BaseDriver{ + MachineName: name, + StorePath: localpath.MiniPath(), + SSHUser: "docker", + }, + Boot2DockerURL: download.LocalISOResource(cc.MinikubeISO), + DiskSize: cc.DiskSize, + Memory: cc.Memory, + CPU: cc.CPUs, + EnginePort: 2376, + FirstQuery: true, + DiskPath: filepath.Join(localpath.MiniPath(), "machines", name, fmt.Sprintf("%s.img", name)), + }, nil +} + +func status() registry.State { + _, err := exec.LookPath("qemu-system-x86_64") + if err != nil { + return registry.State{Error: err, Fix: "Install qemu-system", Doc: docURL} + } + + return registry.State{Installed: true, Healthy: true, Running: true} +} diff --git a/pkg/minikube/registry/drvs/qemu2/doc.go b/pkg/minikube/registry/drvs/qemu2/doc.go new file mode 100644 index 0000000000..c059331503 --- /dev/null +++ b/pkg/minikube/registry/drvs/qemu2/doc.go @@ -0,0 +1,17 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qemu2 diff --git a/pkg/minikube/registry/drvs/qemu2/qemu2.go b/pkg/minikube/registry/drvs/qemu2/qemu2.go new file mode 100644 index 0000000000..229f3ba3cc --- /dev/null +++ b/pkg/minikube/registry/drvs/qemu2/qemu2.go @@ -0,0 +1,152 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package qemu2 + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + + "github.com/docker/machine/libmachine/drivers" + "k8s.io/minikube/pkg/drivers/qemu" + + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/download" + "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/minikube/registry" +) + +const ( + docURL = "https://minikube.sigs.k8s.io/docs/reference/drivers/qemu2/" +) + +func init() { + if err := registry.Register(registry.DriverDef{ + Name: driver.QEMU2, + Init: func() drivers.Driver { return qemu.NewDriver("", "") }, + Config: configure, + Status: status, + Default: true, + Priority: registry.Experimental, + }); err != nil { + panic(fmt.Sprintf("register failed: %v", err)) + } +} + +func qemuSystemProgram() (string, error) { + arch := runtime.GOARCH + switch arch { + case "amd64": + return "qemu-system-x86_64", nil + case "arm64": + return "qemu-system-aarch64", nil + default: + return "", fmt.Errorf("unknown arch: %s", arch) + } +} + +func qemuFirmwarePath() (string, error) { + arch := runtime.GOARCH + switch arch { + case "amd64": + // on macOS, we assume qemu is installed via homebrew for simplicity + if runtime.GOOS == "darwin" { + return "/usr/local/Cellar/qemu/6.2.0_1/share/qemu/edk2-x86_64-code.fd", nil + } + return "/usr/share/OVMF/OVMF_CODE.fd", nil + case "arm64": + if runtime.GOOS == "darwin" { + return "/opt/homebrew/Cellar/qemu/6.2.0_1/share/qemu/edk2-aarch64-code.fd", nil + } + return "/usr/share/AAVMF/AAVMF_CODE.fd", nil + default: + return "", fmt.Errorf("unknown arch: %s", arch) + } +} + +func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { + name := config.MachineName(cc, n) + qemuSystem, err := qemuSystemProgram() + if err != nil { + return nil, err + } + var qemuMachine string + var qemuCPU string + switch runtime.GOARCH { + case "amd64": + qemuMachine = "" // default + qemuCPU = "" // default + case "arm64": + qemuMachine = "virt" + qemuCPU = "cortex-a72" + default: + return nil, fmt.Errorf("unknown arch: %s", runtime.GOARCH) + } + qemuFirmware, err := qemuFirmwarePath() + if err != nil { + return nil, err + } + return qemu.Driver{ + BaseDriver: &drivers.BaseDriver{ + MachineName: name, + StorePath: localpath.MiniPath(), + SSHUser: "docker", + }, + Boot2DockerURL: download.LocalISOResource(cc.MinikubeISO), + DiskSize: cc.DiskSize, + Memory: cc.Memory, + CPU: cc.CPUs, + EnginePort: 2376, + FirstQuery: true, + DiskPath: filepath.Join(localpath.MiniPath(), "machines", name, fmt.Sprintf("%s.img", name)), + Program: qemuSystem, + BIOS: runtime.GOARCH != "arm64", + MachineType: qemuMachine, + CPUType: qemuCPU, + Firmware: qemuFirmware, + VirtioDrives: false, + Network: "user", + CacheMode: "default", + IOMode: "threads", + }, nil +} + +func status() registry.State { + qemuSystem, err := qemuSystemProgram() + if err != nil { + return registry.State{Error: err, Doc: docURL} + } + + _, err = exec.LookPath(qemuSystem) + if err != nil { + return registry.State{Error: err, Fix: "Install qemu-system", Doc: docURL} + } + + qemuFirmware, err := qemuFirmwarePath() + if err != nil { + return registry.State{Error: err, Doc: docURL} + } + + if _, err := os.Stat(qemuFirmware); err != nil && runtime.GOARCH == "arm64" { + return registry.State{Error: err, Fix: "Install uefi firmware", Doc: docURL} + } + + return registry.State{Installed: true, Healthy: true, Running: true} +} diff --git a/site/content/en/docs/drivers/_index.md b/site/content/en/docs/drivers/_index.md index daf996e84f..6744c03ff3 100644 --- a/site/content/en/docs/drivers/_index.md +++ b/site/content/en/docs/drivers/_index.md @@ -17,6 +17,7 @@ To do so, we use the [Docker Machine](https://github.com/docker/machine) library * [Docker]({{}}) - container-based (preferred) * [KVM2]({{}}) - VM-based (preferred) * [VirtualBox]({{}}) - VM +* [QEMU]({{}}) - VM (experimental) * [None]({{}}) - bare-metal * [Podman]({{}}) - container (experimental) * [SSH]({{}}) - remote ssh @@ -29,6 +30,7 @@ To do so, we use the [Docker Machine](https://github.com/docker/machine) library * [VirtualBox]({{}}) - VM * [Parallels]({{}}) - VM * [VMware Fusion]({{}}) - VM +* [QEMU]({{}}) - VM (experimental) * [SSH]({{}}) - remote ssh ## Windows @@ -37,4 +39,5 @@ To do so, we use the [Docker Machine](https://github.com/docker/machine) library * [Docker]({{}}) - VM + Container (preferred) * [VirtualBox]({{}}) - VM * [VMware Workstation]({{}}) - VM +* [QEMU]({{}}) - VM (experimental) * [SSH]({{}}) - remote ssh diff --git a/site/content/en/docs/drivers/qemu.md b/site/content/en/docs/drivers/qemu.md new file mode 100644 index 0000000000..23167b1e5b --- /dev/null +++ b/site/content/en/docs/drivers/qemu.md @@ -0,0 +1,22 @@ +--- +title: "qemu" +weight: 3 +description: > + QEMU driver +aliases: + - /docs/reference/drivers/qemu +--- + +## Overview + +The `qemu` driver users QEMU (system) for VM creation. + + + +## Issues + +* [Full list of open 'qemu' driver issues](https://github.com/kubernetes/minikube/labels/co%2Fqemu-driver) + +## Troubleshooting + +* Run `minikube start --alsologtostderr -v=4` to debug crashes