Fixes containerd configuration issue with insecure registry

- Updates containerd configuration to use the new format for specifying
  container registry mirrors.
- Updates the start code to produce files in the correct location for
  registry mirrors specified with --insecure-registry

deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/config.toml

@@ -57,9 +57,8 @@ oom_score = 0
       conf_dir = "/etc/cni/net.mk"
       conf_template = ""
     [plugins."io.containerd.grpc.v1.cri".registry]
-      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+      config_path = "/etc/containerd/certs.d"
-        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
+
-          endpoint = ["https://registry-1.docker.io"]
   [plugins."io.containerd.service.v1.diff-service"]
     default = ["walking"]
   [plugins."io.containerd.gc.v1.scheduler"]

deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/config.toml.default

@@ -100,9 +100,7 @@ oom_score = 0
       max_conf_num = 1
       conf_template = ""
     [plugins."io.containerd.grpc.v1.cri".registry]
-      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+      config_path = "/etc/containerd/certs.d"
-        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
-          endpoint = ["https://registry-1.docker.io"]
     [plugins."io.containerd.grpc.v1.cri".image_decryption]
       key_model = ""
     [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]

deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/containerd-bin.mk

@@ -53,6 +53,9 @@ define CONTAINERD_BIN_AARCH64_INSTALL_TARGET_CMDS
 	$(INSTALL) -Dm644 \
 		$(CONTAINERD_BIN_AARCH64_PKGDIR)/config.toml \
 		$(TARGET_DIR)/etc/containerd/config.toml
+	$(INSTALL) -Dm644 \
+		$(CONTAINERD_BIN_AARCH64_PKGDIR)/containerd_docker_io_hosts.toml \
+		$(TARGET_DIR)/etc/containerd/docker.io/hosts.toml
 endef
 
 define CONTAINERD_BIN_AARCH64_INSTALL_INIT_SYSTEMD

deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/containerd_docker_io_hosts.toml

@@ -0,0 +1 @@
+server = "https://registry-1.docker.io"

deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/config.toml

@@ -57,9 +57,8 @@ oom_score = 0
       conf_dir = "/etc/cni/net.mk"
       conf_template = ""
     [plugins."io.containerd.grpc.v1.cri".registry]
-      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+      config_path = "/etc/containerd/certs.d"
-        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
+      
-          endpoint = ["https://registry-1.docker.io"]
   [plugins."io.containerd.service.v1.diff-service"]
     default = ["walking"]
   [plugins."io.containerd.gc.v1.scheduler"]

deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/config.toml.default

@@ -100,9 +100,7 @@ oom_score = 0
       max_conf_num = 1
       conf_template = ""
     [plugins."io.containerd.grpc.v1.cri".registry]
-      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+      config_path = "/etc/containerd/certs.d"
-        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
-          endpoint = ["https://registry-1.docker.io"]
     [plugins."io.containerd.grpc.v1.cri".image_decryption]
       key_model = ""
     [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]

deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/containerd-bin.mk

@@ -54,6 +54,9 @@ define CONTAINERD_BIN_INSTALL_TARGET_CMDS
 	$(INSTALL) -Dm644 \
 		$(CONTAINERD_BIN_PKGDIR)/config.toml \
 		$(TARGET_DIR)/etc/containerd/config.toml
+	$(INSTALL) -Dm644 \
+		$(CONTAINERD_BIN_PKGDIR)/containerd_docker_io_hosts.toml \
+		$(TARGET_DIR)/etc/containerd/certs.d/docker.io/hosts.toml
 endef
 
 define CONTAINERD_BIN_INSTALL_INIT_SYSTEMD

deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/containerd_docker_io_hosts.toml

@@ -0,0 +1 @@
+server = "https://registry-1.docker.io"

deploy/kicbase/Dockerfile

@@ -50,6 +50,7 @@ COPY deploy/kicbase/10-network-security.conf /etc/sysctl.d/10-network-security.c
 COPY deploy/kicbase/11-tcp-mtu-probing.conf /etc/sysctl.d/11-tcp-mtu-probing.conf
 COPY deploy/kicbase/02-crio.conf /etc/crio/crio.conf.d/02-crio.conf
 COPY deploy/kicbase/containerd.toml /etc/containerd/config.toml
+COPY deploy/kicbase/containerd_docker_io_hosts.toml /etc/containerd/certs.d/docker.io/hosts.toml
 COPY deploy/kicbase/clean-install /usr/local/bin/clean-install
 COPY deploy/kicbase/entrypoint /usr/local/bin/entrypoint
 COPY --from=auto-pause /src/cmd/auto-pause/auto-pause-${TARGETARCH} /bin/auto-pause

deploy/kicbase/containerd.toml

@@ -57,9 +57,8 @@ oom_score = 0
       conf_dir = "/etc/cni/net.mk"
       conf_template = ""
     [plugins."io.containerd.grpc.v1.cri".registry]
-      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+      config_path = "/etc/containerd/certs.d"
-        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
+
-          endpoint = ["https://registry-1.docker.io"]
   [plugins."io.containerd.service.v1.diff-service"]
     default = ["walking"]
   [plugins."io.containerd.gc.v1.scheduler"]

deploy/kicbase/containerd_docker_io_hosts.toml

@@ -0,0 +1 @@
+server = "https://registry-1.docker.io"

pkg/minikube/cruntime/containerd.go

@@ -21,12 +21,12 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"html/template"
 	"net/url"
 	"os"
 	"os/exec"
 	"path"
 	"strings"
-	"text/template"
 	"time"
 
 	"github.com/blang/semver/v4"
@@ -46,12 +46,11 @@ const (
 	containerdNamespaceRoot = "/run/containerd/runc/k8s.io"
 	// ContainerdConfFile is the path to the containerd configuration
 	containerdConfigFile               = "/etc/containerd/config.toml"
-	containerdImportedConfigFile = "/etc/containerd/containerd.conf.d/02-containerd.conf"
+	containerdMirrorsRoot              = "/etc/containerd/certs.d"
-	containerdConfigTemplate     = `version = 2
+	containerdInsecureRegistryTemplate = `server = "{{.InsecureRegistry -}}"
-{{ range .InsecureRegistry -}}
+
-[plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{. -}}"]
+[host."{{.InsecureRegistry -}}"]
-  endpoint = ["http://{{. -}}"]
+  skip_verify = true
-{{ end -}}
 `
 )
 
@@ -142,28 +141,35 @@ func generateContainerdConfig(cr CommandRunner, imageRepository string, kv semve
 	if _, err := cr.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo sed -e 's|^.*conf_dir = .*$|conf_dir = \"%s\"|' -i %s", cni.ConfDir, containerdConfigFile))); err != nil {
 		return errors.Wrap(err, "update conf_dir")
 	}
-	imports := `imports = ["/etc/containerd/containerd.conf.d/02-containerd.conf"]`
+
-	if _, err := cr.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo sed -e 's|^# imports|%s|' -i %s", imports, containerdConfigFile))); err != nil {
+	for _, registry := range insecureRegistry {
-		return errors.Wrap(err, "update conf_dir")
+		addr := registry
+		if strings.HasPrefix(strings.ToLower(registry), "http://") || strings.HasPrefix(strings.ToLower(registry), "https://") {
+			i := strings.Index(addr, "//")
+			addr = addr[i+2:]
+		} else {
+			registry = "http://" + registry
 		}
 
-	cPath := containerdImportedConfigFile
+		t, err := template.New("hosts.toml").Parse(containerdInsecureRegistryTemplate)
-	t, err := template.New("02-containerd.conf").Parse(containerdConfigTemplate)
 		if err != nil {
-		return err
+			return errors.Wrap(err, "unable to parse insecure registry template")
 		}
 		opts := struct {
-		InsecureRegistry []string
+			InsecureRegistry string
 		}{
-		InsecureRegistry: insecureRegistry,
+			InsecureRegistry: registry,
 		}
 		var b bytes.Buffer
 		if err := t.Execute(&b, opts); err != nil {
-		return err
+			return errors.Wrap(err, "unable to create insecure registry template")
 		}
-	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | base64 -d | sudo tee %s", path.Dir(cPath), base64.StdEncoding.EncodeToString(b.Bytes()), cPath))
+		regRootPath := path.Join(containerdMirrorsRoot, addr)
+
+		c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | base64 -d | sudo tee %s", regRootPath, base64.StdEncoding.EncodeToString(b.Bytes()), path.Join(regRootPath, "hosts.toml")))
 		if _, err := cr.RunCmd(c); err != nil {
-		return errors.Wrap(err, "generate containerd cfg")
+			return errors.Wrap(err, "unable to generate insecure registry cfg")
+		}
 	}
 	return nil
 }