From 70ca2de554b0c1474644f74e207c26fa63b22849 Mon Sep 17 00:00:00 2001 From: Kamesh Sampath Date: Sun, 16 Feb 2020 19:18:45 +0530 Subject: [PATCH 1/3] (fix) Create addon to configure registry helper - Service Account and binding to run the job - Registry aliases ConfigMap - Registry aliases daemonset to update the node etc/hosts fixes: 4604 Signed-off-by: Kamesh Sampath --- .../node-etc-hosts-update.tmpl | 51 +++++++++++++++++++ .../registry-aliases/patch-coredns-job.tmpl | 26 ++++++++++ .../registry-aliases-config.tmpl | 18 +++++++ .../registry-aliases-sa-crb.tmpl | 12 +++++ .../registry-aliases/registry-aliases-sa.tmpl | 5 ++ pkg/addons/config.go | 7 +++ pkg/minikube/assets/addons.go | 32 ++++++++++++ 7 files changed, 151 insertions(+) create mode 100644 deploy/addons/registry-aliases/node-etc-hosts-update.tmpl create mode 100644 deploy/addons/registry-aliases/patch-coredns-job.tmpl create mode 100644 deploy/addons/registry-aliases/registry-aliases-config.tmpl create mode 100644 deploy/addons/registry-aliases/registry-aliases-sa-crb.tmpl create mode 100644 deploy/addons/registry-aliases/registry-aliases-sa.tmpl diff --git a/deploy/addons/registry-aliases/node-etc-hosts-update.tmpl b/deploy/addons/registry-aliases/node-etc-hosts-update.tmpl new file mode 100644 index 0000000000..0ef938876b --- /dev/null +++ b/deploy/addons/registry-aliases/node-etc-hosts-update.tmpl @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: registry-aliases-hosts-update + namespace: kube-system + labels: + kubernetes.io/minikube-addons: registry-aliases + addonmanager.kubernetes.io/mode: Reconcile +spec: + selector: + matchLabels: + app: registry-aliases-hosts-update + template: + metadata: + labels: + app: registry-aliases-hosts-update + spec: + initContainers: + - name: update + image: registry.fedoraproject.org/fedora + volumeMounts: + - name: etchosts + mountPath: /host-etc/hosts + readOnly: false + env: + - name: REGISTRY_ALIASES + valueFrom: + configMapKeyRef: + name: registry-aliases + key: registryAliases + command: + - bash + - -ce + - | + NL=$'\n' + TAB=$'\t' + HOSTS="$(cat /host-etc/hosts)" + [ -z "$REGISTRY_SERVICE_HOST" ] && echo "Failed to get hosts entry for default registry" && exit 1; + for H in $REGISTRY_ALIASES; do + echo "$HOSTS" | grep "$H" || HOSTS="$HOSTS$NL$REGISTRY_SERVICE_HOST$TAB$H"; + done; + echo "$HOSTS" | diff -u /host-etc/hosts - || echo "$HOSTS" > /host-etc/hosts + echo "Done." + containers: + - name: pause-for-update + image: gcr.io/google_containers/pause-amd64:3.1 + terminationGracePeriodSeconds: 30 + volumes: + - name: etchosts + hostPath: + path: /etc/hosts diff --git a/deploy/addons/registry-aliases/patch-coredns-job.tmpl b/deploy/addons/registry-aliases/patch-coredns-job.tmpl new file mode 100644 index 0000000000..cdda3bc7e5 --- /dev/null +++ b/deploy/addons/registry-aliases/patch-coredns-job.tmpl @@ -0,0 +1,26 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: registry-aliases-patch-core-dns + namespace: kube-system +spec: + ttlSecondsAfterFinished: 100 + template: + spec: + serviceAccountName: registry-aliases-sa + volumes: + - name: minikube + hostPath: + path: /var/lib/minikube/binaries + containers: + - name: core-dns-patcher + image: quay.io/rhdevelopers/core-dns-patcher + imagePullPolicy: IfNotPresent + # using the kubectl from the minikube instance + volumeMounts: + - mountPath: /var/lib/minikube/binaries + name: minikube + readOnly: true + restartPolicy: Never + backoffLimit: 4 \ No newline at end of file diff --git a/deploy/addons/registry-aliases/registry-aliases-config.tmpl b/deploy/addons/registry-aliases/registry-aliases-config.tmpl new file mode 100644 index 0000000000..0dacc4ed9c --- /dev/null +++ b/deploy/addons/registry-aliases/registry-aliases-config.tmpl @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: registry-aliases + namespace: kube-system + labels: + kubernetes.io/minikube-addons: registry-aliases + addonmanager.kubernetes.io/mode: Reconcile +data: + # Add additonal hosts seperated by new-line + registryAliases: >- + example.org + example.com + test.com + test.org + # default registry address in minikube when enabled via minikube addons enable registry + registrySvc: registry.kube-system.svc.cluster.local + diff --git a/deploy/addons/registry-aliases/registry-aliases-sa-crb.tmpl b/deploy/addons/registry-aliases/registry-aliases-sa-crb.tmpl new file mode 100644 index 0000000000..1ca1b60cc8 --- /dev/null +++ b/deploy/addons/registry-aliases/registry-aliases-sa-crb.tmpl @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: registry-aliases-crb +subjects: +- kind: ServiceAccount + name: registry-aliases-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/deploy/addons/registry-aliases/registry-aliases-sa.tmpl b/deploy/addons/registry-aliases/registry-aliases-sa.tmpl new file mode 100644 index 0000000000..a40fc37999 --- /dev/null +++ b/deploy/addons/registry-aliases/registry-aliases-sa.tmpl @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: registry-aliases-sa + namespace: kube-system \ No newline at end of file diff --git a/pkg/addons/config.go b/pkg/addons/config.go index 46c713d69f..f3ed918240 100644 --- a/pkg/addons/config.go +++ b/pkg/addons/config.go @@ -112,6 +112,13 @@ var Addons = []*Addon{ set: SetBool, callbacks: []setFn{enableOrDisableAddon}, }, + { + name: "registry-aliases", + set: SetBool, + callbacks: []setFn{enableOrDisableAddon}, + //TODO - add other settings + //TODO check if registry addon is enabled + }, { name: "storage-provisioner", set: SetBool, diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index b25446e8f6..4e20974058 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -246,6 +246,38 @@ var Addons = map[string]*Addon{ "0640", false), }, false, "registry-creds"), + "registry-aliases": NewAddon([]*BinAsset{ + MustBinAsset( + "deploy/addons/registry-aliases/registry-aliases-sa.tmpl", + vmpath.GuestAddonsDir, + "registry-aliases-sa.yaml", + "0640", + false), + MustBinAsset( + "deploy/addons/registry-aliases/registry-aliases-sa-crb.tmpl", + vmpath.GuestAddonsDir, + "registry-aliases-sa-crb.yaml", + "0640", + false), + MustBinAsset( + "deploy/addons/registry-aliases/registry-aliases-config.tmpl", + vmpath.GuestAddonsDir, + "registry-aliases-config.yaml", + "0640", + false), + MustBinAsset( + "deploy/addons/registry-aliases/node-etc-hosts-update.tmpl", + vmpath.GuestAddonsDir, + "node-etc-hosts-update.yaml", + "0640", + false), + MustBinAsset( + "deploy/addons/registry-aliases/patch-coredns-job.tmpl", + vmpath.GuestAddonsDir, + "patch-coredns-job.yaml", + "0640", + false), + }, false, "registry-aliases"), "freshpod": NewAddon([]*BinAsset{ MustBinAsset( "deploy/addons/freshpod/freshpod-rc.yaml.tmpl", From 28ffcb13a1e3e4dde9e6dd70d0af867bc04bd711 Mon Sep 17 00:00:00 2001 From: Kamesh Sampath Date: Thu, 20 Feb 2020 19:32:19 +0530 Subject: [PATCH 2/3] updated readme with usage instructions Signed-off-by: Kamesh Sampath --- deploy/addons/registry-aliases/README.md | 150 +++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 deploy/addons/registry-aliases/README.md diff --git a/deploy/addons/registry-aliases/README.md b/deploy/addons/registry-aliases/README.md new file mode 100644 index 0000000000..a111524bde --- /dev/null +++ b/deploy/addons/registry-aliases/README.md @@ -0,0 +1,150 @@ +# Minikube Registry Aliases Addon + +An addon to minikube that can help push and pull from the minikube registry using custom domain names. The custom domain names will be made resolveable from with in cluster and at minikube node. + +## How to use ? + +### Start minikube + +```shell +minikube profile demo +minikube start -p demo +``` +This addon depends on `registry` addon, it need to be enabled before the alias addon is installed: + +### Enable internal registry + +```shell +minikube addons enable registry +``` + +Verifying the registry deployment + +```shell +watch kubectl get pods -n kube-system +``` + +```shell +NAME READY STATUS RESTARTS AGE +coredns-6955765f44-kpbzt 1/1 Running 0 16m +coredns-6955765f44-lzlsv 1/1 Running 0 16m +etcd-demo 1/1 Running 0 16m +kube-apiserver-demo 1/1 Running 0 16m +kube-controller-manager-demo 1/1 Running 0 16m +kube-proxy-q8rb9 1/1 Running 0 16m +kube-scheduler-demo 1/1 Running 0 16m +*registry-4k8zs* 1/1 Running 0 40s +registry-proxy-vs8jt 1/1 Running 0 40s +storage-provisioner 1/1 Running 0 16m +``` + +```shell +kubectl get svc -n kube-system +``` + +```shell +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +kube-dns ClusterIP 10.96.0.10 53/UDP,53/TCP,9153/TCP 17m +registry ClusterIP 10.97.247.75 80/TCP 94s +``` + +> +> **NOTE:** +> Please make a note of the CLUSTER-IP of `registry` service + +### Enable registry aliases addon + +```shell +minikube addons enable registry-aliases +🌟 The 'registry-aliases' addon is enabled +``` + +You can check the mikikube vm's `/etc/hosts` file for the registry aliases entries: + +```shell +watch minikube ssh -- cat /etc/hosts +``` + +```shell +127.0.0.1 localhost +127.0.1.1 demo +10.97.247.75 example.org +10.97.247.75 example.com +10.97.247.75 test.com +10.97.247.75 test.org +``` + +The above output shows that the Daemonset has added the `registryAliases` from the ConfigMap pointing to the internal registry's __CLUSTER-IP__. + +### Update CoreDNS + +The coreDNS would have been automatically updated by the patch-coredns. A successful job run will have coredns ConfigMap updated like: + +```yaml +apiVersion: v1 +data: + Corefile: |- + .:53 { + errors + health + rewrite name example.com registry.kube-system.svc.cluster.local + rewrite name example.org registry.kube-system.svc.cluster.local + rewrite name test.com registry.kube-system.svc.cluster.local + rewrite name test.org registry.kube-system.svc.cluster.local + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + upstream + fallthrough in-addr.arpa ip6.arpa + } + prometheus :9153 + proxy . /etc/resolv.conf + cache 30 + loop + reload + loadbalance + } +kind: ConfigMap +metadata: + name: coredns +``` + +To verify it run the following command: + +```shell +kubectl get cm -n kube-system coredns -o yaml +``` + +Once you have successfully patched you can now push and pull from the registry using suffix `example.com`, `example.org`,`test.com` and `test.org`. + +The successful run will show the following extra pods (Daemonset, Job) in `kube-system` namespace: + +```shell +NAME READY STATUS RESTARTS AGE +registry-aliases-hosts-update-995vx 1/1 Running 0 47s +registry-aliases-patch-core-dns-zsxfc 0/1 Completed 0 47s +``` + +## Verify with sample application + +You can verify the deployment end to end using the example [application](https://github.com/kameshsampath/minikube-registry-aliases-demo). + +```shell +git clone https://github.com/kameshsampath/minikube-registry-aliases-demo +cd minikube-registry-aliases-demo +``` + +Make sure you set the docker context using `eval $(minikube -p demo docker-env)` + +Deploy the application using [Skaffold](https://skaffold.dev): + +```shell +skaffold dev --port-forward +``` + +Once the application is running try doing `curl localhost:8080` to see the `Hello World` response + +You can also update [skaffold.yaml](./skaffold.yaml) and [app.yaml](.k8s/app.yaml), to use `test.org`, `test.com` or `example.org` as container registry urls, and see all the container image names resolves to internal registry, resulting in successful build and deployment. + +> **NOTE**: +> +> You can also update [skaffold.yaml](./skaffold.yaml) and [app. yaml](.k8s/app.yaml), to use `test.org`, `test.com` or > `example.org` as container registry urls, and see all the > container image names resolves to internal registry, resulting in successful build and deployment. From db27b59e075180085b199948855889179707aa2c Mon Sep 17 00:00:00 2001 From: Kamesh Sampath Date: Sun, 8 Mar 2020 09:36:21 +0530 Subject: [PATCH 3/3] (chore) fix README to use new profile format --- deploy/addons/registry-aliases/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/deploy/addons/registry-aliases/README.md b/deploy/addons/registry-aliases/README.md index a111524bde..1cebc54759 100644 --- a/deploy/addons/registry-aliases/README.md +++ b/deploy/addons/registry-aliases/README.md @@ -7,7 +7,6 @@ An addon to minikube that can help push and pull from the minikube registry usin ### Start minikube ```shell -minikube profile demo minikube start -p demo ``` This addon depends on `registry` addon, it need to be enabled before the alias addon is installed: