Merge pull request #10471 from phantooom/master

kvm2 driver: Add flag --kvm-numa-count" support topology-manager simulate numa

cmd/minikube/cmd/start_flags.go

@@ -65,6 +65,7 @@ const (
 	kvmQemuURI              = "kvm-qemu-uri"
 	kvmGPU                  = "kvm-gpu"
 	kvmHidden               = "kvm-hidden"
+	kvmNUMACount            = "kvm-numa-count"
 	minikubeEnvPrefix       = "MINIKUBE"
 	installAddons           = "install-addons"
 	defaultDiskSize         = "20000mb"
@@ -193,6 +194,7 @@ func initDriverFlags() {
 	startCmd.Flags().String(kvmQemuURI, "qemu:///system", "The KVM QEMU connection URI. (kvm2 driver only)")
 	startCmd.Flags().Bool(kvmGPU, false, "Enable experimental NVIDIA GPU support in minikube")
 	startCmd.Flags().Bool(kvmHidden, false, "Hide the hypervisor signature from the guest in minikube (kvm2 driver only)")
+	startCmd.Flags().Int(kvmNUMACount, 1, "Simulate numa node count in minikube, supported numa node count range is 1-8 (kvm2 driver only)")
 
 	// virtualbox
 	startCmd.Flags().String(hostOnlyCIDR, "192.168.99.1/24", "The CIDR to be used for the minikube VM (virtualbox driver only)")
@@ -311,6 +313,8 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
 			out.WarningT("--network flag is only valid with the docker/podman drivers, it will be ignored")
 		}
 
+		checkNumaCount(k8sVersion)
+
 		cc = config.ClusterConfig{
 			Name:                    ClusterFlagValue(),
 			KeepContext:             viper.GetBool(keepContext),
@@ -338,6 +342,7 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
 			KVMQemuURI:              viper.GetString(kvmQemuURI),
 			KVMGPU:                  viper.GetBool(kvmGPU),
 			KVMHidden:               viper.GetBool(kvmHidden),
+			KVMNUMACount:            viper.GetInt(kvmNUMACount),
 			DisableDriverMounts:     viper.GetBool(disableDriverMounts),
 			UUID:                    viper.GetString(uuid),
 			NoVTXCheck:              viper.GetBool(noVTXCheck),
@@ -407,6 +412,22 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k
 	return createNode(cc, kubeNodeName, existing)
 }
 
+func checkNumaCount(k8sVersion string) {
+	if viper.GetInt(kvmNUMACount) < 1 || viper.GetInt(kvmNUMACount) > 8 {
+		exit.Message(reason.Usage, "--kvm-numa-count range is 1-8")
+	}
+
+	if viper.GetInt(kvmNUMACount) > 1 {
+		v, err := pkgutil.ParseKubernetesVersion(k8sVersion)
+		if err != nil {
+			exit.Message(reason.Usage, "invalid kubernetes version")
+		}
+		if v.LT(semver.Version{Major: 1, Minor: 18}) {
+			exit.Message(reason.Usage, "numa node is only supported on k8s v1.18 and later")
+		}
+	}
+}
+
 // upgradeExistingConfig upgrades legacy configuration files
 func upgradeExistingConfig(cc *config.ClusterConfig) {
 	if cc == nil {
@@ -545,6 +566,10 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC
 		cc.KVMHidden = viper.GetBool(kvmHidden)
 	}
 
+	if cmd.Flags().Changed(kvmNUMACount) {
+		cc.KVMNUMACount = viper.GetInt(kvmNUMACount)
+	}
+
 	if cmd.Flags().Changed(disableDriverMounts) {
 		cc.DisableDriverMounts = viper.GetBool(disableDriverMounts)
 	}

cmd/minikube/cmd/start_test.go

@@ -133,6 +133,7 @@ func TestMirrorCountry(t *testing.T) {
 			cmd := &cobra.Command{}
 			viper.SetDefault(imageRepository, test.imageRepository)
 			viper.SetDefault(imageMirrorCountry, test.mirrorCountry)
+			viper.SetDefault(kvmNUMACount, 1)
 			config, _, err := generateClusterConfig(cmd, nil, k8sVersion, "none")
 			if err != nil {
 				t.Fatalf("Got unexpected error %v during config generation", err)

pkg/drivers/kvm/domain.go

@@ -44,7 +44,11 @@ const domainTmpl = `
     </kvm>
     {{end}}
   </features>
-  <cpu mode='host-passthrough'/>
+  <cpu mode='host-passthrough'>
+  {{if gt .NUMANodeCount 1}}
+  {{.NUMANodeXML}}
+  {{end}}
+  </cpu>
   <os>
     <type>hvm</type>
     <boot dev='cdrom'/>
@@ -158,14 +162,12 @@ func (d *Driver) createDomain() (*libvirt.Domain, error) {
 		}
 		d.PrivateMAC = mac.String()
 	}
-
 	// create the XML for the domain using our domainTmpl template
 	tmpl := template.Must(template.New("domain").Parse(domainTmpl))
 	var domainXML bytes.Buffer
 	if err := tmpl.Execute(&domainXML, d); err != nil {
 		return nil, errors.Wrap(err, "executing domain xml")
 	}
-
 	conn, err := getConnection(d.ConnectionURI)
 	if err != nil {
 		return nil, errors.Wrap(err, "error getting libvirt connection")

pkg/drivers/kvm/kvm.go

@@ -81,6 +81,12 @@ type Driver struct {
 
 	// QEMU Connection URI
 	ConnectionURI string
+
+	// NUMA node count default value is 1
+	NUMANodeCount int
+
+	// NUMA XML
+	NUMANodeXML string
 }
 
 const (
@@ -301,7 +307,6 @@ func (d *Driver) Start() (err error) {
 func (d *Driver) Create() (err error) {
 	log.Info("Creating KVM machine...")
 	defer log.Infof("KVM machine creation complete!")
-
 	err = d.createNetwork()
 	if err != nil {
 		return errors.Wrap(err, "creating network")
@@ -314,6 +319,14 @@ func (d *Driver) Create() (err error) {
 		}
 	}
 
+	if d.NUMANodeCount > 1 {
+		numaXML, err := numaXML(d.CPU, d.Memory, d.NUMANodeCount)
+		if err != nil {
+			return errors.Wrap(err, "creating NUMA XML")
+		}
+		d.NUMANodeXML = numaXML
+	}
+
 	store := d.ResolveStorePath(".")
 	log.Infof("Setting up store path in %s ...", store)
 	// 0755 because it must be accessible by libvirt/qemu across a variety of configs

pkg/drivers/kvm/numa.go

@@ -0,0 +1,93 @@
+/*
+Copyright 2021 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kvm
+
+import (
+	"bytes"
+	"fmt"
+	"strconv"
+	"strings"
+	"text/template"
+)
+
+// numaTmpl NUMA XML Template
+const numaTmpl = `
+<numa>
+  {{- range $idx,$val :=. }}
+  <cell id='{{$idx}}' cpus='{{$val.CPUTopology}}' memory='{{$val.Memory}}' unit='MiB'/>
+  {{- end }}
+</numa>
+`
+
+// NUMA this struct use for numaTmpl
+type NUMA struct {
+	// cpu count on numa node
+	CPUCount int
+	// memory on numa node
+	Memory int
+	// cpu sequence on numa node eg: 0,1,2,3
+	CPUTopology string
+}
+
+// numaXML generate numa xml
+// evenly distributed cpu core & memory to each numa node
+func numaXML(cpu, memory, numaCount int) (string, error) {
+	if numaCount < 1 {
+		return "", fmt.Errorf("numa node count must >= 1")
+	}
+	if cpu < numaCount {
+		return "", fmt.Errorf("cpu count must >= numa node count")
+	}
+	numaNodes := make([]*NUMA, numaCount)
+	CPUSeq := 0
+	cpuBaseCount := cpu / numaCount
+	cpuExtraCount := cpu % numaCount
+
+	for i := range numaNodes {
+		numaNodes[i] = &NUMA{CPUCount: cpuBaseCount}
+	}
+
+	for i := 0; i < cpuExtraCount; i++ {
+		numaNodes[i].CPUCount++
+	}
+	for i := range numaNodes {
+		CPUTopologySlice := make([]string, 0)
+		for seq := CPUSeq; seq < CPUSeq+numaNodes[i].CPUCount; seq++ {
+			CPUTopologySlice = append(CPUTopologySlice, strconv.Itoa(seq))
+		}
+		numaNodes[i].CPUTopology = strings.Join(CPUTopologySlice, ",")
+		CPUSeq += numaNodes[i].CPUCount
+	}
+
+	memoryBaseCount := memory / numaCount
+	memoryExtraCount := memory % numaCount
+
+	for i := range numaNodes {
+		numaNodes[i].Memory = memoryBaseCount
+	}
+
+	for i := 0; i < memoryExtraCount; i++ {
+		numaNodes[i].Memory++
+	}
+
+	tmpl := template.Must(template.New("numa").Parse(numaTmpl))
+	var numaXML bytes.Buffer
+	if err := tmpl.Execute(&numaXML, numaNodes); err != nil {
+		return "", fmt.Errorf("couldn't generate numa XML: %v", err)
+	}
+	return numaXML.String(), nil
+}

pkg/drivers/kvm/numa_test.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2021 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kvm
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestNumaXml(t *testing.T) {
+	_, err := numaXML(1, 1024, 0)
+	if err == nil {
+		t.Errorf("check invalid numa count failed: %s", err)
+	}
+
+	xml, err := numaXML(10, 10240, 8)
+	expXML := `<numa>
+  <cell id='0' cpus='0,1' memory='1280' unit='MiB'/>
+  <cell id='1' cpus='2,3' memory='1280' unit='MiB'/>
+  <cell id='2' cpus='4' memory='1280' unit='MiB'/>
+  <cell id='3' cpus='5' memory='1280' unit='MiB'/>
+  <cell id='4' cpus='6' memory='1280' unit='MiB'/>
+  <cell id='5' cpus='7' memory='1280' unit='MiB'/>
+  <cell id='6' cpus='8' memory='1280' unit='MiB'/>
+  <cell id='7' cpus='9' memory='1280' unit='MiB'/>
+</numa>`
+	if err != nil {
+		t.Errorf("gen xml failed: %s", err)
+	}
+	if strings.TrimSpace(xml) != expXML {
+		t.Errorf("gen xml: %s not match expect xml: %s", xml, expXML)
+	}
+
+}

pkg/minikube/config/types.go

@@ -56,6 +56,7 @@ type ClusterConfig struct {
 	KVMQemuURI              string   // Only used by kvm2
 	KVMGPU                  bool     // Only used by kvm2
 	KVMHidden               bool     // Only used by kvm2
+	KVMNUMACount            int      // Only used by kvm2
 	DockerOpt               []string // Each entry is formatted as KEY=VALUE.
 	DisableDriverMounts     bool     // Only used by virtualbox
 	NFSShare                []string

pkg/minikube/registry/drvs/kvm2/kvm2.go

@@ -68,6 +68,7 @@ type kvmDriver struct {
 	GPU            bool
 	Hidden         bool
 	ConnectionURI  string
+	NUMANodeCount  int
 }
 
 func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) {
@@ -89,6 +90,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) {
 		GPU:            cc.KVMGPU,
 		Hidden:         cc.KVMHidden,
 		ConnectionURI:  cc.KVMQemuURI,
+		NUMANodeCount:  cc.KVMNUMACount,
 	}, nil
 }
 

site/content/en/docs/commands/start.md

@@ -70,6 +70,7 @@ minikube start [flags]
       --kvm-gpu                           Enable experimental NVIDIA GPU support in minikube
       --kvm-hidden                        Hide the hypervisor signature from the guest in minikube (kvm2 driver only)
       --kvm-network string                The KVM network name. (kvm2 driver only) (default "default")
+      --kvm-numa-count int                Simulate numa node count in minikube, supported numa node count range is 1-8 (kvm2 driver only) (default 1)
       --kvm-qemu-uri string               The KVM QEMU connection URI. (kvm2 driver only) (default "qemu:///system")
       --memory string                     Amount of RAM to allocate to Kubernetes (format: <number>[<unit>], where unit = b, k, m or g).
       --mount                             This will start the mount daemon and automatically mount files into minikube.