Use chroot instead of LD_LIBRARY_PATH for containerd restart

cmd/gvisor/gvisor.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"flag"
 	"log"
 	"os"
 
@@ -24,6 +25,8 @@ import (
 )
 
 func main() {
+	flag.Parse()
+
 	if err := gvisor.Enable(); err != nil {
 		log.Print(err)
 		os.Exit(1)

deploy/addons/gvisor/gvisor-pod.yaml.tmpl

@@ -29,45 +29,23 @@ spec:
         privileged: true
       volumeMounts:
       - mountPath: /node/
-        name: node
-      - mountPath: /usr/libexec/sudo
-        name: sudo
-      - mountPath: /var/run
-        name: varrun
-      - mountPath: /usr/bin
-        name: usrbin
-      - mountPath: /usr/lib
-        name: usrlib
-      - mountPath: /bin
-        name: bin
+        name: node-root
+      - mountPath: /node/run
+        name: node-run
       - mountPath: /tmp/gvisor
-        name: gvisor
+        name: node-tmp
       env:
-        - name: PATH
-          value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/node/bin
         - name: SYSTEMD_IGNORE_CHROOT
           value: "yes"
       imagePullPolicy: IfNotPresent
   volumes:
-  - name: node
+  - name: node-root
     hostPath:
       path: /
-  - name: sudo
+  - name: node-run
     hostPath:
-      path: /usr/libexec/sudo
-  - name: varrun
-    hostPath:
-      path: /var/run
-  - name: usrlib
-    hostPath:
-      path: /usr/lib
-  - name: usrbin
-    hostPath:
-      path: /usr/bin
-  - name: bin
-    hostPath:
-      path: /bin
-  - name: gvisor
+      path: /run
+  - name: node-tmp
     hostPath:
       path: /tmp/gvisor
   restartPolicy: Always

deploy/gvisor/Dockerfile

@@ -12,9 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM ubuntu:18.04
-RUN apt-get update && \
-    apt-get install -y kmod gcc wget xz-utils libc6-dev bc libelf-dev bison flex openssl libssl-dev libidn2-0 sudo libcap2 && \
-    rm -rf /var/lib/apt/lists/*
+# Need an image with chroot
+FROM alpine:3
 COPY out/gvisor-addon /gvisor-addon
 CMD ["/gvisor-addon"]

pkg/gvisor/enable.go

@@ -157,7 +157,7 @@ func copyConfigFiles() error {
 	if err := mcnutils.CopyFile(filepath.Join(nodeDir, containerdConfigTomlPath), filepath.Join(nodeDir, storedContainerdConfigTomlPath)); err != nil {
 		return errors.Wrap(err, "copying default config.toml")
 	}
-	log.Print("Copying containerd config.toml with gvisor...")
+	log.Printf("Copying %s asset to %s", constants.GvisorConfigTomlTargetName, filepath.Join(nodeDir, containerdConfigTomlPath))
 	if err := copyAssetToDest(constants.GvisorConfigTomlTargetName, filepath.Join(nodeDir, containerdConfigTomlPath)); err != nil {
 		return errors.Wrap(err, "copying gvisor version of config.toml")
 	}
@@ -171,8 +171,13 @@ func copyAssetToDest(targetName, dest string) error {
 			asset = a
 		}
 	}
+	if asset == nil {
+		return fmt.Errorf("no asset matching target %s among %+v", targetName, assets.Addons["gvisor"])
+	}
+
 	// Now, copy the data from this asset to dest
 	src := filepath.Join(constants.GvisorFilesPath, asset.GetTargetName())
+	log.Printf("%s asset path: %s", targetName, src)
 	contents, err := ioutil.ReadFile(src)
 	if err != nil {
 		return errors.Wrapf(err, "getting contents of %s", asset.GetAssetName())
@@ -182,6 +187,8 @@ func copyAssetToDest(targetName, dest string) error {
 			return errors.Wrapf(err, "removing %s", dest)
 		}
 	}
+
+	log.Printf("creating %s", dest)
 	f, err := os.Create(dest)
 	if err != nil {
 		return errors.Wrapf(err, "creating %s", dest)
@@ -193,28 +200,24 @@ func copyAssetToDest(targetName, dest string) error {
 }
 
 func restartContainerd() error {
-	dir := filepath.Join(nodeDir, "usr/libexec/sudo")
-	if err := os.Setenv("LD_LIBRARY_PATH", dir); err != nil {
-		return errors.Wrap(err, dir)
-	}
+	log.Print("restartContainerd black magic happening")
 
 	log.Print("Stopping rpc-statd.service...")
-	// first, stop  rpc-statd.service
-	cmd := exec.Command("sudo", "-E", "systemctl", "stop", "rpc-statd.service")
+	cmd := exec.Command("/usr/sbin/chroot", "/node", "sudo", "systemctl", "stop", "rpc-statd.service")
 	if out, err := cmd.CombinedOutput(); err != nil {
 		fmt.Println(string(out))
 		return errors.Wrap(err, "stopping rpc-statd.service")
 	}
-	// restart containerd
+
 	log.Print("Restarting containerd...")
-	cmd = exec.Command("sudo", "-E", "systemctl", "restart", "containerd")
+	cmd = exec.Command("/usr/sbin/chroot", "/node", "sudo", "systemctl", "restart", "containerd")
 	if out, err := cmd.CombinedOutput(); err != nil {
 		log.Print(string(out))
 		return errors.Wrap(err, "restarting containerd")
 	}
-	// start rpc-statd.service
+
 	log.Print("Starting rpc-statd...")
-	cmd = exec.Command("sudo", "-E", "systemctl", "start", "rpc-statd.service")
+	cmd = exec.Command("/usr/sbin/chroot", "/node", "sudo", "systemctl", "start", "rpc-statd.service")
 	if out, err := cmd.CombinedOutput(); err != nil {
 		log.Print(string(out))
 		return errors.Wrap(err, "restarting rpc-statd.service")

test/integration/gvisor_addon_test.go

@@ -35,6 +35,13 @@ func TestGvisorAddon(t *testing.T) {
 	profile := UniqueProfileName("gvisor")
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
 	defer func() {
+		if t.Failed() {
+			rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "logs", "gvisor", "-n", "kube-system"))
+			if err != nil {
+				t.Logf("failed to get gvisor post-mortem logs: %v", err)
+			}
+			t.Logf("gvisor post-mortem: %s:\n%s\n", rr.Command(), rr.Output())
+		}
 		CleanupWithLogs(t, profile, cancel)
 	}()