Enable gvisor addon in minikube (#3399)

This PR adds the code for enabling gvisor in minikube. It adds the pod that will run when the addon is enabled, and the code for the image which will run when this happens. When gvisor is enabled, the pod will download runsc and the gvisor-containerd-shim. It will replace the containerd config.toml and restart containerd. When gvisor is disabled, the pod will be deleted by the addon manager. This will trigger a pre-stop hook which will revert the config.toml to it's original state and restart containerd.
2018-12-07 15:27:22 -08:00 · 2018-12-07 15:27:22 -08:00 · 8f128a7d10
parent ad415e252d
commit 8f128a7d10
19 changed files with 786 additions and 6 deletions
--- a/12
+++ b/12
@ -303,6 +303,18 @@ storage-provisioner-image: out/storage-provisioner
 push-storage-provisioner-image: storage-provisioner-image
 	gcloud docker -- push $(REGISTRY)/storage-provisioner:$(STORAGE_PROVISIONER_TAG)

+.PHONY: out/gvisor-addon
+out/gvisor-addon:
+	GOOS=linux CGO_ENABLED=0 go build -o $@ cmd/gvisor/gvisor.go
+
+.PHONY: gvisor-addon-image
+gvisor-addon-image: out/gvisor-addon
+	docker build -t $(REGISTRY)/gvisor-addon:latest -f deploy/gvisor/Dockerfile .
+
+.PHONY: push-gvisor-addon-image
+push-gvisor-addon-image: gvisor-addon-image
+	gcloud docker -- push $(REGISTRY)/gvisor-addon:latest
+
 .PHONY: release-iso
 release-iso: minikube_iso checksum
 	gsutil cp out/minikube.iso gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION).iso
--- a/cmd/gvisor/gvisor.go
+++ b/cmd/gvisor/gvisor.go
@ -0,0 +1,31 @@
+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"log"
+	"os"
+
+	"k8s.io/minikube/pkg/gvisor"
+)
+
+func main() {
+	if err := gvisor.Enable(); err != nil {
+		log.Print(err)
+		os.Exit(1)
+	}
+}
--- a/cmd/minikube/cmd/config/config.go
+++ b/cmd/minikube/cmd/config/config.go
@ -212,6 +212,12 @@ var settings = []Setting{
 		validations: []setFn{IsValidAddon},
 		callbacks:   []setFn{EnableOrDisableAddon},
 	},
+	{
+		name:        "gvisor",
+		set:         SetBool,
+		validations: []setFn{IsValidAddon, IsContainerdRuntime},
+		callbacks:   []setFn{EnableOrDisableAddon},
+	},
 	{
 		name: "hyperv-virtual-switch",
 		set:  SetString,
--- a/cmd/minikube/cmd/config/validations.go
+++ b/cmd/minikube/cmd/config/validations.go
@ -18,15 +18,17 @@ package config

 import (
 	"fmt"
-	"github.com/docker/go-units"
-	"github.com/pkg/errors"
-	"k8s.io/minikube/pkg/minikube/assets"
-	"k8s.io/minikube/pkg/minikube/constants"
 	"net"
 	"net/url"
 	"os"
 	"strconv"
 	"strings"
+
+	"github.com/docker/go-units"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
 )

 func IsValidDriver(string, driver string) error {
@ -125,3 +127,23 @@ func IsValidAddon(name string, val string) error {
 	}
 	return errors.Errorf("Cannot enable/disable invalid addon %s", name)
 }
+
+func IsContainerdRuntime(_, _ string) error {
+	config, err := config.Load()
+	if err != nil {
+		return fmt.Errorf("error getting cluster config: %v", err)
+	}
+	if config.KubernetesConfig.ContainerRuntime != constants.ContainerdRuntime {
+		return fmt.Errorf(`This addon can only be enabled with the containerd runtime backend.
+
+To enable this backend, please first stop minikube with:
+
+minikube stop
+
+and then start minikube again with the following flags:
+
+minikube start --container-runtime=containerd  --docker-opt containerd=/var/run/containerd/containerd.sock --network-plugin=cni`)
+	}
+
+	return nil
+}
--- a/deploy/addons/gvisor/README.md
+++ b/deploy/addons/gvisor/README.md
@ -0,0 +1,71 @@
+## gVisor Addon
+[gVisor](https://github.com/google/gvisor/blob/master/README.md), a sandboxed container runtime, allows users to securely run pods with untrusted workloads within Minikube.
+
+### Starting Minikube
+gVisor depends on the containerd runtime to run in Minikube.
+When starting minikube, specify the following flags, along with any additional desired flags:
+
+```shell
+$ minikube start --container-runtime=containerd  \
+    --docker-opt containerd=/var/run/containerd/containerd.sock \
+    --network-plugin=cni
+```
+
+### Enabling gVisor
+To enable this addon, simply run:
+
+```
+$ minikube addons enable gvisor
+```
+
+Within one minute, the addon manager should pick up the change and you should see the `gvisor` pod:
+
+```
+$ kubectl get pod gvisor -n kube-system
+NAME      READY     STATUS    RESTARTS   AGE
+gvisor    1/1       Running   0          3m
+```
+
+Once the pod has status `Running`, gVisor is enabled in Minikube. 
+
+### Running pods in gVisor
+To run a pod in gVisor, add this annotation to the Kubernetes yaml:
+
+```
+io.kubernetes.cri.untrusted-workload: "true"
+```
+
+An example Pod is shown below:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-untrusted
+  annotations:
+    io.kubernetes.cri.untrusted-workload: "true"
+spec:
+  containers:
+  - name: nginx
+    image: nginx
+```
+
+_Note: this annotation will not be necessary once the RuntimeClass Kubernetes feature is available broadly._
+
+### Disabling gVisor
+To disable gVisor, run:
+
+```
+$ minikube addons disable gvisor
+```
+
+Within one minute, the addon manager should pick up the change.
+Once the `gvisor` pod has status `Terminating`, or has been deleted, the gvisor addon should be disabled.
+
+```
+$ kubectl get pod gvisor -n kube-system
+NAME      READY     STATUS        RESTARTS   AGE
+gvisor    1/1       Terminating   0          5m
+```
+
+_Note: Once gVisor is disabled, any pod with the `io.kubernetes.cri.untrusted-workload` annotation will fail with a FailedCreatePodSandBox error._
--- a/deploy/addons/gvisor/gvisor-config.toml
+++ b/deploy/addons/gvisor/gvisor-config.toml
@ -0,0 +1,69 @@
+root = "/var/lib/containerd"
+state = "/run/containerd"
+oom_score = 0
+
+[grpc]
+  address = "/run/containerd/containerd.sock"
+  uid = 0
+  gid = 0
+  max_recv_message_size = 16777216
+  max_send_message_size = 16777216
+
+[debug]
+  address = ""
+  uid = 0
+  gid = 0
+  level = ""
+
+[metrics]
+  address = ""
+  grpc_histogram = false
+
+[cgroup]
+  path = ""
+
+[plugins]
+  [plugins.cgroups]
+    no_prometheus = false
+  [plugins.cri]
+    stream_server_address = ""
+    stream_server_port = "10010"
+    enable_selinux = false
+    sandbox_image = "k8s.gcr.io/pause:3.1"
+    stats_collect_period = 10
+    systemd_cgroup = false
+    enable_tls_streaming = false
+    max_container_log_line_size = 16384
+    [plugins.cri.containerd]
+      snapshotter = "overlayfs"
+      no_pivot = true
+      [plugins.cri.containerd.default_runtime]
+        runtime_type = "io.containerd.runtime.v1.linux"
+        runtime_engine = ""
+        runtime_root = ""
+      [plugins.cri.containerd.untrusted_workload_runtime]
+        runtime_type = "io.containerd.runtime.v1.linux"
+        runtime_engine = "/usr/local/bin/runsc"
+        runtime_root = "/run/containerd/runsc"
+    [plugins.cri.cni]
+      bin_dir = "/opt/cni/bin"
+      conf_dir = "/etc/cni/net.d"
+      conf_template = ""
+    [plugins.cri.registry]
+      [plugins.cri.registry.mirrors]
+        [plugins.cri.registry.mirrors."docker.io"]
+          endpoint = ["https://registry-1.docker.io"]
+  [plugins.diff-service]
+    default = ["walking"]
+  [plugins.linux]
+    shim = "gvisor-containerd-shim"
+    runtime = "runc"
+    runtime_root = ""
+    no_shim = false
+    shim_debug = true
+  [plugins.scheduler]
+    pause_threshold = 0.02
+    deletion_threshold = 0
+    mutation_threshold = 100
+    schedule_delay = "0s"
+    startup_delay = "100ms"
--- a/deploy/addons/gvisor/gvisor-containerd-shim.toml
+++ b/deploy/addons/gvisor/gvisor-containerd-shim.toml
@ -0,0 +1,3 @@
+runc_shim = "/bin/containerd-shim"
+[runsc_config]
+ user-log="/tmp/runsc/user-log-%ID%.log"
--- a/deploy/addons/gvisor/gvisor-pod.yaml
+++ b/deploy/addons/gvisor/gvisor-pod.yaml
@ -0,0 +1,72 @@
+# Copyright 2018 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Pod
+metadata: 
+  name: gvisor
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/minikube-addons: gvisor
+spec: 
+  hostPID: true
+  containers: 
+    - name: gvisor
+      image: gcr.io/k8s-minikube/gvisor-addon:latest
+      securityContext:
+        privileged: true
+      volumeMounts:
+      - mountPath: /node/
+        name: node
+      - mountPath: /usr/libexec/sudo
+        name: sudo
+      - mountPath: /var/run
+        name: varrun
+      - mountPath: /usr/bin
+        name: usrbin
+      - mountPath: /usr/lib
+        name: usrlib
+      - mountPath: /bin
+        name: bin
+      - mountPath: /tmp/gvisor
+        name: gvisor
+      env:
+        - name: PATH
+          value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/node/bin
+        - name: SYSTEMD_IGNORE_CHROOT
+          value: "yes"
+  volumes:
+  - name: node
+    hostPath:
+      path: /
+  - name: sudo
+    hostPath:
+      path: /usr/libexec/sudo
+  - name: varrun
+    hostPath:
+      path: /var/run
+  - name: usrlib
+    hostPath:
+      path: /usr/lib
+  - name: usrbin
+    hostPath:
+      path: /usr/bin
+  - name: bin
+    hostPath:
+      path: /bin
+  - name: gvisor
+    hostPath:
+      path: /tmp/gvisor
+  restartPolicy: Never
--- a/deploy/gvisor/Dockerfile
+++ b/deploy/gvisor/Dockerfile
@ -0,0 +1,20 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:18.04
+RUN apt-get update && \
+    apt-get install -y kmod gcc wget xz-utils libc6-dev bc libelf-dev bison flex openssl libssl-dev libidn2-0 sudo libcap2 && \
+    rm -rf /var/lib/apt/lists/*
+COPY out/gvisor-addon /gvisor-addon
+CMD ["/gvisor-addon"]
--- a/docs/addons.md
+++ b/docs/addons.md
@ -36,6 +36,7 @@ The currently supported addons include:
 * [Freshpod](https://github.com/GoogleCloudPlatform/freshpod)
 * [nvidia-driver-installer](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/nvidia-driver-installer/minikube)
 * [nvidia-gpu-device-plugin](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/cmd/nvidia_gpu)
+* [gvisor](../deploy/addons/gvisor/README.md)

 If you would like to have minikube properly start/restart custom addons, place the addon(s) you wish to be launched with minikube in the `.minikube/addons` directory. Addons in this folder will be moved to the minikube VM and launched each time minikube is started/restarted.

--- a/pkg/gvisor/disable.go
+++ b/pkg/gvisor/disable.go
@ -0,0 +1,45 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gvisor
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/machine/libmachine/mcnutils"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/constants"
+)
+
+// Disable reverts containerd config files and restarts containerd
+func Disable() error {
+	log.Print("Disabling gvisor...")
+	if err := os.Remove(filepath.Join(nodeDir, constants.ContainerdConfigTomlPath)); err != nil {
+		return errors.Wrapf(err, "removing %s", constants.ContainerdConfigTomlPath)
+	}
+	log.Printf("Restoring default config.toml at %s", constants.ContainerdConfigTomlPath)
+	if err := mcnutils.CopyFile(filepath.Join(nodeDir, constants.StoredContainerdConfigTomlPath), filepath.Join(nodeDir, constants.ContainerdConfigTomlPath)); err != nil {
+		return errors.Wrap(err, "reverting back to default config.toml")
+	}
+	// restart containerd
+	if err := restartContainerd(); err != nil {
+		return errors.Wrap(err, "restarting containerd")
+	}
+	log.Print("Successfully disabled gvisor")
+	return nil
+}
--- a/pkg/gvisor/enable.go
+++ b/pkg/gvisor/enable.go
@ -0,0 +1,230 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gvisor
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+
+	"github.com/docker/machine/libmachine/mcnutils"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/constants"
+)
+
+const (
+	nodeDir = "/node"
+)
+
+// Enable follows these steps for enabling gvisor in minikube:
+//   1. creates necessary directories for storing binaries and runsc logs
+//   2. downloads runsc and gvisor-containerd-shim
+//   3. copies necessary containerd config files
+//   4. restarts containerd
+func Enable() error {
+	if err := makeGvisorDirs(); err != nil {
+		return errors.Wrap(err, "creating directories on node")
+	}
+	if err := downloadBinaries(); err != nil {
+		return errors.Wrap(err, "downloading binaries")
+	}
+	if err := copyConfigFiles(); err != nil {
+		return errors.Wrap(err, "copying config files")
+	}
+	if err := restartContainerd(); err != nil {
+		return errors.Wrap(err, "restarting containerd")
+	}
+	// When pod is terminated, disable gvisor and exit
+	c := make(chan os.Signal)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-c
+		if err := Disable(); err != nil {
+			log.Printf("Error disabling gvisor: %v", err)
+			os.Exit(1)
+		}
+		os.Exit(0)
+	}()
+	log.Print("gvisor successfully enabled in cluster")
+	// sleep for one year so the pod continuously runs
+	select {}
+}
+
+// makeGvisorDirs creates necessary directories on the node
+func makeGvisorDirs() error {
+	// Make /run/containerd/runsc to hold logs
+	fp := filepath.Join(nodeDir, "run/containerd/runsc")
+	if err := os.MkdirAll(fp, 0755); err != nil {
+		return errors.Wrap(err, "creating runsc dir")
+	}
+
+	// Make /usr/local/bin to store the runsc binary
+	fp = filepath.Join(nodeDir, "usr/local/bin")
+	if err := os.MkdirAll(fp, 0755); err != nil {
+		return errors.Wrap(err, "creating usr/local/bin dir")
+	}
+
+	// Make /tmp/runsc to also hold logs
+	fp = filepath.Join(nodeDir, "tmp/runsc")
+	if err := os.MkdirAll(fp, 0755); err != nil {
+		return errors.Wrap(err, "creating runsc logs dir")
+	}
+
+	return nil
+}
+
+func downloadBinaries() error {
+	if err := runsc(); err != nil {
+		return errors.Wrap(err, "downloading runsc")
+	}
+	if err := gvisorContainerdShim(); err != nil {
+		return errors.Wrap(err, "downloading gvisor-containerd-shim")
+	}
+	return nil
+}
+
+// downloads the gvisor-containerd-shim
+func gvisorContainerdShim() error {
+	dest := filepath.Join(nodeDir, "usr/bin/gvisor-containerd-shim")
+	return downloadFileToDest(constants.GvisorContainerdShimURL, dest)
+}
+
+// downloads the runsc binary and returns a path to the binary
+func runsc() error {
+	dest := filepath.Join(nodeDir, "usr/local/bin/runsc")
+	return downloadFileToDest(constants.GvisorURL, dest)
+}
+
+// downloadFileToDest downlaods the given file to the dest
+// if something already exists at dest, first remove it
+func downloadFileToDest(url, dest string) error {
+	client := &http.Client{}
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return errors.Wrapf(err, "creating request for %s", url)
+	}
+	req.Header.Set("User-Agent", "minikube")
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if _, err := os.Stat(dest); err == nil {
+		if err := os.Remove(dest); err != nil {
+			return errors.Wrapf(err, "removing %s for overwrite", dest)
+		}
+	}
+	fi, err := os.Create(dest)
+	if err != nil {
+		return errors.Wrapf(err, "creating %s", dest)
+	}
+	defer fi.Close()
+	if _, err := io.Copy(fi, resp.Body); err != nil {
+		return errors.Wrap(err, "copying binary")
+	}
+	if err := fi.Chmod(0777); err != nil {
+		return errors.Wrap(err, "fixing perms")
+	}
+	return nil
+}
+
+// Must write the following files:
+//    1. gvisor-containerd-shim.toml
+//    2. gvisor containerd config.toml
+// and save the default version of config.toml
+func copyConfigFiles() error {
+	log.Printf("Storing default config.toml at %s", constants.StoredContainerdConfigTomlPath)
+	if err := mcnutils.CopyFile(filepath.Join(nodeDir, constants.ContainerdConfigTomlPath), filepath.Join(nodeDir, constants.StoredContainerdConfigTomlPath)); err != nil {
+		return errors.Wrap(err, "copying default config.toml")
+	}
+	log.Print("Copying gvisor-containerd-shim.toml...")
+	if err := copyAssetToDest(constants.GvisorContainerdShimTargetName, filepath.Join(nodeDir, constants.GvisorContainerdShimTomlPath)); err != nil {
+		return errors.Wrap(err, "copying gvisor-containerd-shim.toml")
+	}
+	log.Print("Copying containerd config.toml with gvisor...")
+	if err := copyAssetToDest(constants.GvisorConfigTomlTargetName, filepath.Join(nodeDir, constants.ContainerdConfigTomlPath)); err != nil {
+		return errors.Wrap(err, "copying gvisor version of config.toml")
+	}
+	return nil
+}
+
+func copyAssetToDest(targetName, dest string) error {
+	var asset *assets.BinDataAsset
+	for _, a := range assets.Addons["gvisor"].Assets {
+		if a.GetTargetName() == targetName {
+			asset = a
+		}
+	}
+	// Now, copy the data from this asset to dest
+	src := filepath.Join(constants.GvisorFilesPath, asset.GetTargetName())
+	contents, err := ioutil.ReadFile(src)
+	if err != nil {
+		return errors.Wrapf(err, "getting contents of %s", asset.GetAssetName())
+	}
+	if _, err := os.Stat(dest); err == nil {
+		if err := os.Remove(dest); err != nil {
+			return errors.Wrapf(err, "removing %s", dest)
+		}
+	}
+	f, err := os.Create(dest)
+	if err != nil {
+		return errors.Wrapf(err, "creating %s", dest)
+	}
+	if _, err := f.Write(contents); err != nil {
+		return errors.Wrapf(err, "writing contents to %s", f.Name())
+	}
+	return nil
+}
+
+func restartContainerd() error {
+	dir := filepath.Join(nodeDir, "usr/libexec/sudo")
+	if err := os.Setenv("LD_LIBRARY_PATH", dir); err != nil {
+		return errors.Wrap(err, dir)
+	}
+
+	log.Print("Stopping rpc-statd.service...")
+	// first, stop  rpc-statd.service
+	cmd := exec.Command("sudo", "-E", "systemctl", "stop", "rpc-statd.service")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		fmt.Println(string(out))
+		return errors.Wrap(err, "stopping rpc-statd.service")
+	}
+	// restart containerd
+	log.Print("Restarting containerd...")
+	cmd = exec.Command("sudo", "-E", "systemctl", "restart", "containerd")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		log.Print(string(out))
+		return errors.Wrap(err, "restarting containerd")
+	}
+	// start rpc-statd.service
+	log.Print("Starting rpc-statd...")
+	cmd = exec.Command("sudo", "-E", "systemctl", "start", "rpc-statd.service")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		log.Print(string(out))
+		return errors.Wrap(err, "restarting rpc-statd.service")
+	}
+	log.Print("containerd restart complete")
+	return nil
+}
--- a/pkg/minikube/assets/addons.go
+++ b/pkg/minikube/assets/addons.go
@ -227,6 +227,23 @@ var Addons = map[string]*Addon{
 			"nvidia-gpu-device-plugin.yaml",
 			"0640"),
 	}, false, "nvidia-gpu-device-plugin"),
+	"gvisor": NewAddon([]*BinDataAsset{
+		NewBinDataAsset(
+			"deploy/addons/gvisor/gvisor-pod.yaml",
+			constants.AddonsPath,
+			"gvisor-pod.yaml",
+			"0640"),
+		NewBinDataAsset(
+			"deploy/addons/gvisor/gvisor-config.toml",
+			constants.GvisorFilesPath,
+			constants.GvisorConfigTomlTargetName,
+			"0640"),
+		NewBinDataAsset(
+			"deploy/addons/gvisor/gvisor-containerd-shim.toml",
+			constants.GvisorFilesPath,
+			constants.GvisorContainerdShimTargetName,
+			"0640"),
+	}, false, "gvisor"),
 }

 func AddMinikubeDirAssets(assets *[]CopyableFile) error {
--- a/pkg/minikube/constants/constants.go
+++ b/pkg/minikube/constants/constants.go
@ -22,13 +22,13 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"time"

 	"github.com/blang/semver"
 	"github.com/golang/glog"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/util/homedir"
 	minikubeVersion "k8s.io/minikube/pkg/version"
-	"time"
 )

 // APIServerPort is the port that the API server should listen on.
@ -263,3 +263,24 @@ func GetKubeadmCachedImages(kubernetesVersionStr string) []string {
 }

 var ImageCacheDir = MakeMiniPath("cache", "images")
+
+const (
+	// GvisorFilesPath is the path to the gvisor files saved by go-bindata
+	GvisorFilesPath = "/tmp/gvisor"
+	// ContainerdConfigTomlPath is the path to the containerd config.toml
+	ContainerdConfigTomlPath = "/etc/containerd/config.toml"
+	// GvisorContainerdShimTomlPath is the path to givosr-containerd-shim.toml
+	GvisorContainerdShimTomlPath = "/etc/containerd/gvisor-containerd-shim.toml"
+	// StoredContainerdConfigTomlPath is the path where the default config.toml will be stored
+	StoredContainerdConfigTomlPath = "/tmp/config.toml"
+
+	//GvisorConfigTomlTargetName is the go-bindata target name for the gvisor config.toml
+	GvisorConfigTomlTargetName = "gvisor-config.toml"
+	// GvisorContainerdShimTargetName is the go-bindata target name for gvisor-containerd-shim
+	GvisorContainerdShimTargetName = "gvisor-containerd-shim.toml"
+
+	// GvisorContainerdShimURL is the url to download gvisor-containerd-shim
+	GvisorContainerdShimURL = "https://github.com/google/gvisor-containerd-shim/releases/download/v0.0.1-rc.0/gvisor-containerd-shim-v0.0.1-rc.0.linux-amd64"
+	// GvisorURL is the url to download gvisor
+	GvisorURL = "https://storage.googleapis.com/gvisor/releases/nightly/2018-12-07/runsc"
+)
--- a/pkg/util/kubernetes.go
+++ b/pkg/util/kubernetes.go
@ -139,6 +139,36 @@ func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels
 	})
 }

+// Wait up to 10 minutes for a pod to be deleted
+func WaitForPodDelete(c kubernetes.Interface, ns string, label labels.Selector) error {
+	return wait.PollImmediate(constants.APICallRetryInterval, time.Minute*10, func() (bool, error) {
+		listOpts := metav1.ListOptions{LabelSelector: label.String()}
+		pods, err := c.CoreV1().Pods(ns).List(listOpts)
+		if err != nil {
+			glog.Infof("error getting Pods with label selector %q [%v]\n", label.String(), err)
+			return false, nil
+		}
+		return len(pods.Items) == 0, nil
+	})
+}
+
+// Wait up to 10 minutes for the given event to appear
+func WaitForEvent(c kubernetes.Interface, ns string, reason string) error {
+	return wait.PollImmediate(constants.APICallRetryInterval, time.Minute*10, func() (bool, error) {
+		events, err := c.Events().Events("default").List(metav1.ListOptions{})
+		if err != nil {
+			glog.Infof("error getting events: %v", err)
+			return false, nil
+		}
+		for _, e := range events.Items {
+			if e.Reason == reason {
+				return true, nil
+			}
+		}
+		return false, nil
+	})
+}
+
 // WaitForRCToStabilize waits till the RC has a matching generation/replica count between spec and status.
 func WaitForRCToStabilize(c kubernetes.Interface, ns, name string, timeout time.Duration) error {
 	options := metav1.ListOptions{FieldSelector: fields.Set{
--- a/test/integration/addons_test.go
+++ b/test/integration/addons_test.go
@ -183,3 +183,46 @@ func testServicesList(t *testing.T) {
 		t.Fatalf(err.Error())
 	}
 }
+
+func testGvisor(t *testing.T) {
+	minikubeRunner := NewMinikubeRunner(t)
+	kubectlRunner := util.NewKubectlRunner(t)
+	minikubeRunner.RunCommand("addons enable gvisor", true)
+
+	t.Log("waiting for gvisor controller to come up")
+	if err := util.WaitForGvisorControllerRunning(t); err != nil {
+		t.Fatalf("waiting for gvisor controller to be up: %v", err)
+	}
+
+	untrustedPath, _ := filepath.Abs("testdata/nginx-untrusted.yaml")
+	t.Log("creating pod with untrusted workload annotation")
+	if _, err := kubectlRunner.RunCommand([]string{"create", "-f", untrustedPath}); err != nil {
+		t.Fatalf("creating untrusted nginx resource: %v", err)
+	}
+
+	t.Log("making sure untrusted workload is Running")
+	if err := util.WaitForUntrustedNginxRunning(); err != nil {
+		t.Fatalf("waiting for nginx to be up: %v", err)
+	}
+
+	t.Log("disabling gvisor addon")
+	minikubeRunner.RunCommand("addons disable gvisor", true)
+	t.Log("waiting for gvisor controller pod to be deleted")
+	if err := util.WaitForGvisorControllerDeleted(); err != nil {
+		t.Fatalf("waiting for gvisor controller to be deleted: %v", err)
+	}
+
+	t.Log("recreating untrusted workload pod")
+	if _, err := kubectlRunner.RunCommand([]string{"replace", "-f", untrustedPath, "--force"}); err != nil {
+		t.Fatalf("replacing untrusted nginx resource: %v", err)
+	}
+
+	t.Log("waiting for FailedCreatePodSandBox event")
+	if err := util.WaitForFailedCreatePodSandBoxEvent(); err != nil {
+		t.Fatalf("waiting for FailedCreatePodSandBox event: %v", err)
+	}
+
+	if _, err := kubectlRunner.RunCommand([]string{"delete", "-f", untrustedPath}); err != nil {
+		t.Logf("error deleting untrusted nginx resource: %v", err)
+	}
+}
--- a/test/integration/functional_test.go
+++ b/test/integration/functional_test.go
@ -22,6 +22,8 @@ import (
 	"strings"
 	"testing"

+	"github.com/docker/machine/libmachine/state"
+	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/test/integration/util"
 )

@ -48,6 +50,24 @@ func TestFunctional(t *testing.T) {
 	}
 }

+func TestFunctionalContainerd(t *testing.T) {
+	minikubeRunner := NewMinikubeRunner(t)
+
+	if usingNoneDriver(minikubeRunner) {
+		t.Skip("Can't run containerd backend with none driver")
+	}
+
+	if minikubeRunner.GetStatus() != state.None.String() {
+		minikubeRunner.RunCommand("delete", true)
+	}
+
+	minikubeRunner.SetRuntime(constants.ContainerdRuntime)
+	minikubeRunner.EnsureRunning()
+
+	t.Run("Gvisor", testGvisor)
+	minikubeRunner.RunCommand("delete", true)
+}
+
 // usingNoneDriver returns true if using the none driver
 func usingNoneDriver(runner util.MinikubeRunner) bool {
 	return strings.Contains(runner.StartArgs, "--vm-driver=none")
--- a/test/integration/testdata/nginx-untrusted.yaml
+++ b/test/integration/testdata/nginx-untrusted.yaml
@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-untrusted
+  labels:
+    run: nginx
+  annotations:
+    io.kubernetes.cri.untrusted-workload: "true"
+spec:
+  containers:
+  - name: nginx
+    image: nginx
--- a/test/integration/util/util.go
+++ b/test/integration/util/util.go
@ -126,7 +126,8 @@ func (m *MinikubeRunner) SSH(command string) (string, error) {
 func (m *MinikubeRunner) Start() {
 	switch r := m.Runtime; r {
 	case constants.ContainerdRuntime:
-		containerdFlags := "--container-runtime=containerd --network-plugin=cni --docker-opt containerd=/var/run/containerd/containerd.sock"
+		// TODO: priyawadhwa@ remove iso url once updated iso is being used in integration tests
+		containerdFlags := "--container-runtime=containerd --network-plugin=cni --docker-opt containerd=/var/run/containerd/containerd.sock --iso-url=https://storage.googleapis.com/k8s-minikube/gvisor-preview.iso"
 		m.RunCommand(fmt.Sprintf("start %s %s %s", m.StartArgs, m.Args, containerdFlags), true)
 	default:
 		m.RunCommand(fmt.Sprintf("start %s %s", m.StartArgs, m.Args), true)
@ -298,6 +299,60 @@ func WaitForIngressDefaultBackendRunning(t *testing.T) error {
 	return nil
 }

+// WaitForGvisorControllerRunning waits for the gvisor controller pod to be running
+func WaitForGvisorControllerRunning(t *testing.T) error {
+	client, err := commonutil.GetClient()
+	if err != nil {
+		return errors.Wrap(err, "getting kubernetes client")
+	}
+
+	selector := labels.SelectorFromSet(labels.Set(map[string]string{"kubernetes.io/minikube-addons": "gvisor"}))
+	if err := commonutil.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
+		return errors.Wrap(err, "waiting for gvisor controller pod to stabilize")
+	}
+	return nil
+}
+
+// WaitForGvisorControllerDeleted waits for the gvisor controller pod to be deleted
+func WaitForGvisorControllerDeleted() error {
+	client, err := commonutil.GetClient()
+	if err != nil {
+		return errors.Wrap(err, "getting kubernetes client")
+	}
+
+	selector := labels.SelectorFromSet(labels.Set(map[string]string{"kubernetes.io/minikube-addons": "gvisor"}))
+	if err := commonutil.WaitForPodDelete(client, "kube-system", selector); err != nil {
+		return errors.Wrap(err, "waiting for gvisor controller pod deletion")
+	}
+	return nil
+}
+
+// WaitForUntrustedNginxRunning waits for the untrusted nginx pod to start running
+func WaitForUntrustedNginxRunning() error {
+	client, err := commonutil.GetClient()
+	if err != nil {
+		return errors.Wrap(err, "getting kubernetes client")
+	}
+
+	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx"}))
+	if err := commonutil.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
+		return errors.Wrap(err, "waiting for nginx pods")
+	}
+	return nil
+}
+
+// WaitForFailedCreatePodSandBoxEvent waits for a FailedCreatePodSandBox event to appear
+func WaitForFailedCreatePodSandBoxEvent() error {
+	client, err := commonutil.GetClient()
+	if err != nil {
+		return errors.Wrap(err, "getting kubernetes client")
+	}
+	if err := commonutil.WaitForEvent(client, "default", "FailedCreatePodSandBox"); err != nil {
+		return errors.Wrap(err, "waiting for FailedCreatePodSandBox event")
+	}
+	return nil
+}
+
 func WaitForNginxRunning(t *testing.T) error {
 	client, err := commonutil.GetClient()