Update gvisor runsc version

- Updates the gvisor addon to use containerd shim v2 - Updates the version of runsc - Auto-installs a gvisor RuntimeClass Issue #4482
2019-03-29 11:16:02 +00:00 · 2019-03-29 11:16:02 +00:00 · d14bdbced7
parent b37d1d286a
commit d14bdbced7
11 changed files with 154 additions and 52 deletions
--- a/deploy/addons/gvisor/README.md
+++ b/deploy/addons/gvisor/README.md
@ -1,5 +1,5 @@
 ## gVisor Addon
-[gVisor](https://github.com/google/gvisor/blob/master/README.md), a sandboxed container runtime, allows users to securely run pods with untrusted workloads within Minikube.
+[gVisor](https://gvisor.dev/), a sandboxed container runtime, allows users to securely run pods with untrusted workloads within Minikube.

 ### Starting Minikube
 gVisor depends on the containerd runtime to run in Minikube.
@ -17,21 +17,27 @@ To enable this addon, simply run:
 $ minikube addons enable gvisor
 ```

-Within one minute, the addon manager should pick up the change and you should see the `gvisor` pod:
+Within one minute, the addon manager should pick up the change and you should
+see the `gvisor` pod and `gvisor` [Runtime Class](https://kubernetes.io/docs/concepts/containers/runtime-class/):

 ```
-$ kubectl get pod gvisor -n kube-system
-NAME      READY     STATUS    RESTARTS   AGE
-gvisor    1/1       Running   0          3m
+$ kubectl get pod,runtimeclass gvisor -n kube-system
+NAME         READY   STATUS    RESTARTS   AGE
+pod/gvisor   1/1     Running   0          2m52s
+
+NAME                              CREATED AT
+runtimeclass.node.k8s.io/gvisor   2019-06-15T04:35:09Z
 ```

-Once the pod has status `Running`, gVisor is enabled in Minikube. 
+Once the pod has status `Running`, gVisor is enabled in Minikube.

 ### Running pods in gVisor
-To run a pod in gVisor, add this annotation to the Kubernetes yaml:
+
+To run a pod in gVisor, add the `gvisor` runtime class to the Pod spec in your
+Kubernetes yaml:

 ```
-io.kubernetes.cri.untrusted-workload: "true"
+runtimeClassName: gvisor
 ```

 An example Pod is shown below:
@ -41,17 +47,15 @@ apiVersion: v1
 kind: Pod
 metadata:
  name: nginx-untrusted
-  annotations:
-    io.kubernetes.cri.untrusted-workload: "true"
 spec:
+  runtimeClassName: gvisor
  containers:
  - name: nginx
    image: nginx
 ```

-_Note: this annotation will not be necessary once the RuntimeClass Kubernetes feature is available broadly._
-
 ### Disabling gVisor
+
 To disable gVisor, run:

 ```
@ -67,4 +71,4 @@ NAME      READY     STATUS        RESTARTS   AGE
 gvisor    1/1       Terminating   0          5m
 ```

-_Note: Once gVisor is disabled, any pod with the `io.kubernetes.cri.untrusted-workload` annotation will fail with a FailedCreatePodSandBox error._
+_Note: Once gVisor is disabled, any pod with the `gvisor` Runtime Class or `io.kubernetes.cri.untrusted-workload` annotation will fail with a FailedCreatePodSandBox error._
--- a/deploy/addons/gvisor/gvisor-config.toml
+++ b/deploy/addons/gvisor/gvisor-config.toml
@ -41,10 +41,10 @@ oom_score = 0
        runtime_type = "io.containerd.runtime.v1.linux"
        runtime_engine = ""
        runtime_root = ""
-      [plugins.cri.containerd.untrusted_workload_runtime]
-        runtime_type = "io.containerd.runtime.v1.linux"
-        runtime_engine = "/usr/local/bin/runsc"
-        runtime_root = "/run/containerd/runsc"
+      [plugins.cri.containerd.runtimes.untrusted]
+        runtime_type = "io.containerd.runsc.v1"
+      [plugins.cri.containerd.runtimes.runsc]
+        runtime_type = "io.containerd.runsc.v1"
    [plugins.cri.cni]
      bin_dir = "/opt/cni/bin"
      conf_dir = "/etc/cni/net.d"
@ -56,7 +56,6 @@ oom_score = 0
  [plugins.diff-service]
    default = ["walking"]
  [plugins.linux]
-    shim = "gvisor-containerd-shim"
    runtime = "runc"
    runtime_root = ""
    no_shim = false
@ -66,4 +65,4 @@ oom_score = 0
    deletion_threshold = 0
    mutation_threshold = 100
    schedule_delay = "0s"
-    startup_delay = "100ms"
+    startup_delay = "100ms"
--- a/deploy/addons/gvisor/gvisor-containerd-shim.toml
+++ b/deploy/addons/gvisor/gvisor-containerd-shim.toml
@ -1,3 +0,0 @@
-runc_shim = "/bin/containerd-shim"
-[runsc_config]
- user-log="/tmp/runsc/user-log-%ID%.log"
--- a/deploy/addons/gvisor/gvisor-pod.yaml.tmpl
+++ b/deploy/addons/gvisor/gvisor-pod.yaml.tmpl
@ -14,15 +14,15 @@

 apiVersion: v1
 kind: Pod
-metadata: 
+metadata:
  name: gvisor
  namespace: kube-system
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
    kubernetes.io/minikube-addons: gvisor
-spec: 
+spec:
  hostPID: true
-  containers: 
+  containers:
    - name: gvisor
      image: {{default "gcr.io/k8s-minikube" .ImageRepository}}/gvisor-addon:latest
      securityContext:
--- a/deploy/addons/gvisor/gvisor-runtimeclass.yaml
+++ b/deploy/addons/gvisor/gvisor-runtimeclass.yaml
@ -0,0 +1,22 @@
+# Copyright 2018 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: node.k8s.io/v1beta1
+kind: RuntimeClass
+metadata:
+  name: gvisor
+  labels:
+    kubernetes.io/minikube-addons: gvisor
+    addonmanager.kubernetes.io/mode: Reconcile
+handler: runsc
--- a/pkg/gvisor/enable.go
+++ b/pkg/gvisor/enable.go
@ -80,12 +80,6 @@ func makeGvisorDirs() error {
 		return errors.Wrap(err, "creating runsc dir")
 	}

-	// Make /usr/local/bin to store the runsc binary
-	fp = filepath.Join(nodeDir, "usr/local/bin")
-	if err := os.MkdirAll(fp, 0755); err != nil {
-		return errors.Wrap(err, "creating usr/local/bin dir")
-	}
-
 	// Make /tmp/runsc to also hold logs
 	fp = filepath.Join(nodeDir, "tmp/runsc")
 	if err := os.MkdirAll(fp, 0755); err != nil {
@ -107,13 +101,13 @@ func downloadBinaries() error {

 // downloads the gvisor-containerd-shim
 func gvisorContainerdShim() error {
-	dest := filepath.Join(nodeDir, "usr/bin/gvisor-containerd-shim")
+	dest := filepath.Join(nodeDir, "usr/bin/containerd-shim-runsc-v1")
 	return downloadFileToDest(constants.GvisorContainerdShimURL, dest)
 }

 // downloads the runsc binary and returns a path to the binary
 func runsc() error {
-	dest := filepath.Join(nodeDir, "usr/local/bin/runsc")
+	dest := filepath.Join(nodeDir, "usr/bin/runsc")
 	return downloadFileToDest(constants.GvisorURL, dest)
 }

@ -159,10 +153,6 @@ func copyConfigFiles() error {
 	if err := mcnutils.CopyFile(filepath.Join(nodeDir, constants.ContainerdConfigTomlPath), filepath.Join(nodeDir, constants.StoredContainerdConfigTomlPath)); err != nil {
 		return errors.Wrap(err, "copying default config.toml")
 	}
-	log.Print("Copying gvisor-containerd-shim.toml...")
-	if err := copyAssetToDest(constants.GvisorContainerdShimTargetName, filepath.Join(nodeDir, constants.GvisorContainerdShimTomlPath)); err != nil {
-		return errors.Wrap(err, "copying gvisor-containerd-shim.toml")
-	}
 	log.Print("Copying containerd config.toml with gvisor...")
 	if err := copyAssetToDest(constants.GvisorConfigTomlTargetName, filepath.Join(nodeDir, constants.ContainerdConfigTomlPath)); err != nil {
 		return errors.Wrap(err, "copying gvisor version of config.toml")
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@ -314,18 +314,18 @@ var Addons = map[string]*Addon{
 			"gvisor-pod.yaml",
 			"0640",
 			true),
+		MustBinAsset(
+			"deploy/addons/gvisor/gvisor-runtimeclass.yaml",
+			constants.GuestAddonsDir,
+			"gvisor-runtimeclass.yaml",
+			"0640",
+			false),
 		MustBinAsset(
 			"deploy/addons/gvisor/gvisor-config.toml",
 			constants.GvisorFilesPath,
 			constants.GvisorConfigTomlTargetName,
 			"0640",
 			true),
-		MustBinAsset(
-			"deploy/addons/gvisor/gvisor-containerd-shim.toml",
-			constants.GvisorFilesPath,
-			constants.GvisorContainerdShimTargetName,
-			"0640",
-			false),
 	}, false, "gvisor"),
 }

--- a/pkg/minikube/constants/constants.go
+++ b/pkg/minikube/constants/constants.go
@ -384,20 +384,16 @@ const (
 	GvisorFilesPath = "/tmp/gvisor"
 	// ContainerdConfigTomlPath is the path to the containerd config.toml
 	ContainerdConfigTomlPath = "/etc/containerd/config.toml"
-	// GvisorContainerdShimTomlPath is the path to gvisor-containerd-shim.toml
-	GvisorContainerdShimTomlPath = "/etc/containerd/gvisor-containerd-shim.toml"
 	// StoredContainerdConfigTomlPath is the path where the default config.toml will be stored
 	StoredContainerdConfigTomlPath = "/tmp/config.toml"

 	// GvisorConfigTomlTargetName is the go-bindata target name for the gvisor config.toml
 	GvisorConfigTomlTargetName = "gvisor-config.toml"
-	// GvisorContainerdShimTargetName is the go-bindata target name for gvisor-containerd-shim
-	GvisorContainerdShimTargetName = "gvisor-containerd-shim.toml"

 	// GvisorContainerdShimURL is the url to download gvisor-containerd-shim
-	GvisorContainerdShimURL = "https://github.com/google/gvisor-containerd-shim/releases/download/v0.0.1-rc.0/gvisor-containerd-shim-v0.0.1-rc.0.linux-amd64"
+	GvisorContainerdShimURL = "https://github.com/google/gvisor-containerd-shim/releases/download/v0.0.3/containerd-shim-runsc-v1.linux-amd64"
 	// GvisorURL is the url to download gvisor
-	GvisorURL = "https://storage.googleapis.com/gvisor/releases/nightly/2018-12-07/runsc"
+	GvisorURL = "https://storage.googleapis.com/gvisor/releases/nightly/2019-01-14/runsc"
 )

 const (
--- a/test/integration/containerd_test.go
+++ b/test/integration/containerd_test.go
@ -37,9 +37,57 @@ func TestContainerd(t *testing.T) {
 		t.Parallel()
 	}

+	t.Run("GvisorUntrustedWorkload", testGvisorUntrustedWorkload)
+	t.Run("GvisorRuntimeClass", testGvisorRuntimeClass)
 	t.Run("GvisorRestart", testGvisorRestart)
 }

+func testGvisorUntrustedWorkload(t *testing.T) {
+	p := profileName(t)
+	if shouldRunInParallel(t) {
+		t.Parallel()
+	}
+	mk := NewMinikubeRunner(t, p, "--wait=false")
+	defer mk.TearDown(t)
+
+	mk.MustRun("addons enable gvisor", true)
+
+	t.Log("waiting for gvisor controller to come up")
+	if err := waitForGvisorControllerRunning(p); err != nil {
+		t.Fatalf("waiting for gvisor controller to be up: %v", err)
+	}
+
+	createUntrustedWorkload(t, p)
+	t.Log("making sure untrusted workload is Running")
+	if err := waitForUntrustedNginxRunning(p); err != nil {
+		t.Fatalf("waiting for nginx to be up: %v", err)
+	}
+	deleteUntrustedWorkload(t, p)
+}
+
+func testGvisorRuntimeClass(t *testing.T) {
+	p := profileName(t)
+	if shouldRunInParallel(t) {
+		t.Parallel()
+	}
+	mk := NewMinikubeRunner(t, p, "--wait=false")
+	defer mk.TearDown(t)
+
+	mk.MustRun("addons enable gvisor", true)
+
+	t.Log("waiting for gvisor controller to come up")
+	if err := waitForGvisorControllerRunning(p); err != nil {
+		t.Fatalf("waiting for gvisor controller to be up: %v", err)
+	}
+
+	createGvisorWorkload(t, p)
+	t.Log("making sure gvisor workload is Running")
+	if err := waitForGvisorNginxRunning(p); err != nil {
+		t.Fatalf("waiting for nginx to be up: %v", err)
+	}
+	deleteGvisorWorkload(t, p)
+}
+
 func testGvisorRestart(t *testing.T) {
 	p := profileName(t)
 	if shouldRunInParallel(t) {
@ -98,7 +146,24 @@ func deleteUntrustedWorkload(t *testing.T, profile string) {
 	}
 }

-// waitForGvisorControllerRunning waits for the gvisor controller pod to be running
+func createGvisorWorkload(t *testing.T, profile string) {
+	kr := util.NewKubectlRunner(t, profile)
+	gvisorPath := filepath.Join(*testdataDir, "nginx-gvisor.yaml")
+	t.Log("creating pod with gvisor workload annotation")
+	if _, err := kr.RunCommand([]string{"replace", "-f", gvisorPath, "--force"}); err != nil {
+		t.Fatalf("creating gvisor nginx resource: %v", err)
+	}
+}
+
+func deleteGvisorWorkload(t *testing.T, profile string) {
+	kr := util.NewKubectlRunner(t, profile)
+	gvisorPath := filepath.Join(*testdataDir, "nginx-gvisor.yaml")
+	if _, err := kr.RunCommand([]string{"delete", "-f", gvisorPath}); err != nil {
+		t.Logf("error deleting gvisor nginx resource: %v", err)
+	}
+}
+
+// waitForGvisorControllerRunning waits for the gvisor controller pod to be running.
 func waitForGvisorControllerRunning(p string) error {
 	client, err := kapi.Client(p)
 	if err != nil {
@ -112,14 +177,30 @@ func waitForGvisorControllerRunning(p string) error {
 	return nil
 }

-// waitForUntrustedNginxRunning waits for the untrusted nginx pod to start running
+// waitForUntrustedNginxRunning waits for the untrusted nginx pod to start
+// running.
 func waitForUntrustedNginxRunning(miniProfile string) error {
 	client, err := kapi.Client(miniProfile)
 	if err != nil {
 		return errors.Wrap(err, "getting kubernetes client")
 	}

-	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx"}))
+	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx", "untrusted": "true"}))
+	if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
+		return errors.Wrap(err, "waiting for nginx pods")
+	}
+	return nil
+}
+
+// waitForGvisorNginxRunning waits for the nginx pod with gvisor runtime class
+// to start running.
+func waitForGvisorNginxRunning(miniProfile string) error {
+	client, err := kapi.Client(miniProfile)
+	if err != nil {
+		return errors.Wrap(err, "getting kubernetes client")
+	}
+
+	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx", "runtime": "gvisor"}))
 	if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
 		return errors.Wrap(err, "waiting for nginx pods")
 	}
--- a/test/integration/testdata/nginx-gvisor.yaml
+++ b/test/integration/testdata/nginx-gvisor.yaml
@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-gvisor
+  labels:
+    run: nginx
+    runtime: gvisor
+spec:
+  runtimeClassName: gvisor
+  containers:
+  - name: nginx
+    image: nginx
--- a/test/integration/testdata/nginx-untrusted.yaml
+++ b/test/integration/testdata/nginx-untrusted.yaml
@ -4,6 +4,7 @@ metadata:
  name: nginx-untrusted
  labels:
    run: nginx
+    untrusted: "true"
  annotations:
    io.kubernetes.cri.untrusted-workload: "true"
 spec: