From b66888d9724edd15f6c9a8ddbe505a320fa1f6e2 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Tue, 10 Jun 2025 20:21:10 +0300 Subject: [PATCH] vfkit: Use EFI booloader (#20833) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * vfkit: Log serial console to file To make debugging easier, add virtio-serial device logging serial console to file: ~/.minikube/machines/NAME/serial.log To enable logging, we need to enable the console in the kernel command line, since we still use direct kernel boot. Example log: % cat /Users/nir/.minikube/machines/vfkit/vfkig.log [ 0.896094] cacheinfo: Unable to detect cache hierarchy for CPU 0 [ 0.897186] loop: module loaded [ 0.897670] virtio_blk virtio2: [vda] 840488 512-byte logical blocks (430 MB/410 MiB) [ 0.897733] vda: detected capacity change from 0 to 430329856 [ 0.898460] virtio_blk virtio3: [vdb] 40960000 512-byte logical blocks (21.0 GB/19.5 GiB) [ 0.898533] vdb: detected capacity change from 0 to 20971520000 ... [ 1.794714] systemd[1]: Detected virtualization vm-other. [ 1.794752] systemd[1]: Detected architecture arm64. Welcome to Buildroot 2025.02! [ 1.794944] systemd[1]: Hostname set to . [ 1.795011] systemd[1]: Initializing machine ID from random generator. ... [ OK ] Started Container Runtime Interface for OCI (CRI-O). [ OK ] Reached target Multi-User System. Welcome to minikube vfkit login: [ 6.681578] systemd-ssh-generator[630]: Binding SSH to AF_UNIX socket /run/ssh-unix-local/socket. * vfkit: Use EFI bootloader With the fixed iso, we can simplify the driver using the EFI bootloader option[1] instead of the legacy and deprecated --kernel, --kernel-cmdline, and --initrd options[2]. Example run: % minikube start -p vfkit --driver vfkit --container-runtime containerd --network vmnet-shared 😄 [vfkit] minikube v1.36.0 on Darwin 15.5 (arm64) ✨ Using the vfkit driver based on user configuration 👍 Starting "vfkit" primary control-plane node in "vfkit" cluster 🔥 Creating vfkit VM (CPUs=2, Memory=6000MB, Disk=20000MB) ... 📦 Preparing Kubernetes v1.33.1 on containerd 1.7.23 ... ▪ Generating certificates and keys ... ▪ Booting up control plane ... ▪ Configuring RBAC rules ... 🔗 Configuring bridge CNI (Container Networking Interface) ... 🔎 Verifying Kubernetes components... ▪ Using image gcr.io/k8s-minikube/storage-provisioner:v5 🌟 Enabled addons: default-storageclass, storage-provisioner 🏄 Done! kubectl is now configured to use "vfkit" cluster and "default" namespace by default Comparing direct kernel boot and --bootloader efi shows that it is little bit faster and boot time is more consistent. % hyperfine -r 10 -C "minikube delete" \ "vfkit-efi/out/minikube start --driver vfkit --network vmnet-shared --container-runtime containerd --no-kubernetes" \ "vfkit-direct/out/minikube start --driver vfkit --network vmnet-shared --container-runtime containerd --no-kubernetes" Benchmark 1: vfkit-efi/out/minikube start --driver vfkit --network vmnet-shared --container-runtime containerd --no-kubernetes Time (mean ± σ): 10.205 s ± 0.656 s [User: 0.381 s, System: 0.266 s] Range (min … max): 9.106 s … 11.254 s 10 runs Benchmark 2: vfkit-direct/out/minikube start --driver vfkit --network vmnet-shared --container-runtime containerd --no-kubernetes Time (mean ± σ): 10.933 s ± 1.616 s [User: 0.402 s, System: 0.406 s] Range (min … max): 9.155 s … 14.168 s 10 runs Summary vfkit-efi/out/minikube start --driver vfkit --network vmnet-shared --container-runtime containerd --no-kubernetes ran 1.07 ± 0.17 times faster than vfkit-direct/out/minikube start --driver vfkit --network vmnet-shared --container-runtime containerd --no-kubernetes [1] https://github.com/crc-org/vfkit/blob/main/doc/usage.md#efi-bootloader [2] https://github.com/crc-org/vfkit/blob/main/doc/usage.md#deprecated-options * docs: Update vfkit driver documentation - Separate vfkit requirements and vmnet-shared requirements - Update minimal macOS version required for --bootloader efi - Simplify vfkit upgrade, it is available in brew now --- pkg/drivers/vfkit/vfkit.go | 49 +++++++---------------- pkg/minikube/registry/drvs/vfkit/vfkit.go | 1 - site/content/en/docs/drivers/vfkit.md | 18 ++++----- 3 files changed, 24 insertions(+), 44 deletions(-) diff --git a/pkg/drivers/vfkit/vfkit.go b/pkg/drivers/vfkit/vfkit.go index e37408ba06..9c7e4fb551 100644 --- a/pkg/drivers/vfkit/vfkit.go +++ b/pkg/drivers/vfkit/vfkit.go @@ -53,10 +53,12 @@ import ( ) const ( - isoFilename = "boot2docker.iso" - pidFileName = "vfkit.pid" - sockFilename = "vfkit.sock" - defaultSSHUser = "docker" + isoFilename = "boot2docker.iso" + pidFileName = "vfkit.pid" + sockFilename = "vfkit.sock" + serialFileName = "serial.log" + efiVarsFileName = "vfkit.efivars" + defaultSSHUser = "docker" ) // Driver is the machine driver for vfkit (Virtualization.framework) @@ -67,7 +69,6 @@ type Driver struct { DiskSize int CPU int Memory int - Cmdline string ExtraDisks int Network string // "", "nat", "vmnet-shared" MACAddress string // For network=nat, network="" @@ -189,12 +190,6 @@ func (d *Driver) Create() error { if err := b2dutils.CopyIsoToMachineDir(d.Boot2DockerURL, d.MachineName); err != nil { return err } - isoPath := d.ResolveStorePath(isoFilename) - - log.Info("Extracting Kernel...") - if err := d.extractKernel(isoPath); err != nil { - return err - } log.Info("Creating SSH key...") if err := ssh.GenerateSSHKey(d.sshKeyPath()); err != nil { @@ -256,9 +251,10 @@ func (d *Driver) startVfkit(socketPath string) error { "--memory", fmt.Sprintf("%d", d.Memory), "--cpus", fmt.Sprintf("%d", d.CPU), "--restful-uri", fmt.Sprintf("unix://%s", d.sockfilePath())) - var isoPath = filepath.Join(machineDir, isoFilename) + + efiVarsPath := d.ResolveStorePath(efiVarsFileName) startCmd = append(startCmd, - "--device", fmt.Sprintf("virtio-blk,path=%s", isoPath)) + "--bootloader", fmt.Sprintf("efi,variable-store=%s,create", efiVarsPath)) if socketPath != "" { // The guest will be able to access other guests in the vmnet network. @@ -273,20 +269,21 @@ func (d *Driver) startVfkit(socketPath string) error { startCmd = append(startCmd, "--device", "virtio-rng") + var isoPath = filepath.Join(machineDir, isoFilename) startCmd = append(startCmd, - "--kernel", d.ResolveStorePath("bzimage")) + "--device", fmt.Sprintf("virtio-blk,path=%s", isoPath)) + startCmd = append(startCmd, - "--kernel-cmdline", d.Cmdline) - startCmd = append(startCmd, - "--initrd", d.ResolveStorePath("initrd")) + "--device", fmt.Sprintf("virtio-blk,path=%s", d.diskPath())) for i := 0; i < d.ExtraDisks; i++ { startCmd = append(startCmd, "--device", fmt.Sprintf("virtio-blk,path=%s", pkgdrivers.ExtraDiskPath(d.BaseDriver, i))) } + serialPath := d.ResolveStorePath(serialFileName) startCmd = append(startCmd, - "--device", fmt.Sprintf("virtio-blk,path=%s", d.diskPath())) + "--device", fmt.Sprintf("virtio-serial,logFilePath=%s", serialPath)) log.Debugf("executing: vfkit %s", strings.Join(startCmd, " ")) os.Remove(d.sockfilePath()) @@ -410,22 +407,6 @@ func (d *Driver) Restart() error { return d.Start() } -func (d *Driver) extractKernel(isoPath string) error { - for _, f := range []struct { - pathInIso string - destPath string - }{ - {"/boot/bzimage", "bzimage"}, - {"/boot/initrd", "initrd"}, - } { - fullDestPath := d.ResolveStorePath(f.destPath) - if err := pkgdrivers.ExtractFile(isoPath, f.pathInIso, fullDestPath); err != nil { - return err - } - } - return nil -} - func (d *Driver) killVfkit() error { if err := d.SetVFKitState("HardStop"); err != nil { // Typically fails with EOF due to https://github.com/crc-org/vfkit/issues/277. diff --git a/pkg/minikube/registry/drvs/vfkit/vfkit.go b/pkg/minikube/registry/drvs/vfkit/vfkit.go index b6fe05fd70..06e7693547 100644 --- a/pkg/minikube/registry/drvs/vfkit/vfkit.go +++ b/pkg/minikube/registry/drvs/vfkit/vfkit.go @@ -98,7 +98,6 @@ func configure(cfg config.ClusterConfig, n config.Node) (interface{}, error) { DiskSize: cfg.DiskSize, Memory: cfg.Memory, CPU: cfg.CPUs, - Cmdline: "", ExtraDisks: cfg.ExtraDisks, Network: cfg.Network, MACAddress: mac, diff --git a/site/content/en/docs/drivers/vfkit.md b/site/content/en/docs/drivers/vfkit.md index 14a6b00114..a99aee8246 100644 --- a/site/content/en/docs/drivers/vfkit.md +++ b/site/content/en/docs/drivers/vfkit.md @@ -11,6 +11,11 @@ aliases: macOS virtualization, optimized for lightweight virtual machines and container deployment. +## Requirements + +- Requires macOS 13 or later. +- Requires minikube version 1.36.0 or later. + ## Networking The vfkit driver has two networking options: `nat` and `vmnet-shared`. @@ -25,8 +30,6 @@ installation instructions bellow. ### Requirements -- Requires macOS 10.15 or later -- Requires minikube version 1.36.0 or later. - Requires [vmnet-helper](https://github.com/nirs/vmnet-helper). ### Install vment-helper @@ -94,13 +97,10 @@ Run `minikube start --driver vfkit --alsologtostderr -v=7` to debug crashes ### Upgrade VFKit -New updates to macOS often require an updated vfkit driver. To upgrade: - -* If Podman Desktop is installed, it also bundles `vfkit` -* If you have Brew Package Manager, run: `brew upgrade vfkit` -* As a final alternative, you install the latest VFKit from [GitHub](https://github.com/crc-org/vfkit/releases) -* To check your current version, run: `vfkit -v` -* If the version didn't change after upgrading verify the correct VFKit is in the path. run: `which vfkit` +```shell +brew update +brew upgrade vfkit +``` ### Troubleshooting the vmnet-shared network