diff --git a/cmd/minikube/cmd/config/config.go b/cmd/minikube/cmd/config/config.go index 577a66073c..d76fe8a9f9 100644 --- a/cmd/minikube/cmd/config/config.go +++ b/cmd/minikube/cmd/config/config.go @@ -254,6 +254,12 @@ var settings = []Setting{ validations: []setFn{IsValidAddon}, callbacks: []setFn{EnableOrDisableAddon}, }, + { + name: "ingress-dns", + set: SetBool, + validations: []setFn{IsValidAddon}, + callbacks: []setFn{EnableOrDisableAddon}, + }, { name: "hyperv-virtual-switch", set: SetString, diff --git a/deploy/addons/ingress-dns/README.md b/deploy/addons/ingress-dns/README.md new file mode 100644 index 0000000000..f95133067d --- /dev/null +++ b/deploy/addons/ingress-dns/README.md @@ -0,0 +1,168 @@ +# Minikube Ingress DNS +![Build Status](https://gitlab.com/cryptexlabs/public/development/minikube-ingress-dns/badges/master/pipeline.svg) + +DNS service for ingress controllers running on your minikube server + +## Overview + +### Problem +When running minikube locally you are highly likely to want to run your services on an ingress controller so that you +don't have to use minikube tunnel or NodePorts to access your services. While NodePort might be ok in a lot of +circumstances in order to test some features an ingress is necessary. Ingress controllers are great because you can +define your entire architecture in something like a helm chart and all your services will be available. There is only +1 problem. That is that your ingress controller works basically off of dns and while running minikube that means that +your local dns names like `myservice.test` will have to resolve to `$(minikube ip)` not really a big deal except the +only real way to do this is to add an entry for every service in your `/etc/hosts` file. This gets messy for obvious +reasons. If you have a lot of services running that each have their own dns entry then you have to set those up +manually. Even if you automate it you then need to rely on the host operating system storing configurations instead of +storing them in your cluster. To make it worse it has to be constantly maintained and updated as services are added, +remove, and renamed. I call it the `/ets/hosts` pollution problem. + +### Solution +What if you could just access your local services magically without having to edit your `/etc/hosts` file? Well now you +can. This project acts as a DNS service that runs inside your kubernetes cluster. All you have to do is install the +service and add the `$(minikube ip)` as a DNS server on your host machine. Each time the dns service is queried an +API call is made to the kubernetes master service for a list of all the ingresses. If a match is found for the name a +response is given with an IP address as the `$(minikube ip)` which was provided when the helm chart was installed. So +for example lets say my minikube ip address is `192.168.99.106` and I have an ingress controller with the name of +`myservice.test` then I would get a result like so: + +```text +#bash:~$ nslookup myservice.test $(minikube ip) +Server: 192.168.99.169 +Address: 192.168.99.169#53 + +Non-authoritative answer: +Name: myservice.test $(minikube ip) +Address: 192.168.99.169 +``` + +## Installation + +### Start minikube +``` +minikube start +``` + +### Install the kubernetes resources +```bash +minikube addons enable ingress-dns +``` + +### Add the minikube ip as a dns server + +#### Mac OS +Create a file in `/etc/resolver/minikube-profilename-test` +``` +domain test +nameserver 192.168.99.169 +search_order 1 +timeout 5 +``` +Replace `192.168.99.169` with your minikube ip and `profilename` is the name of the minikube profile for the +corresponding ip address + +See https://www.unix.com/man-page/opendarwin/5/resolver/ +Note that even though the `port` feature is documented. It does not actually work. + +#### Linux +Update the file `/etc/resolvconf/resolv.conf.d/base` to have the following contents +``` +search test +nameserver 192.168.99.169 +timeout 5 +``` +Replace `192.168.99.169` with your minikube ip and `profilename` is the name of the minikube profile for the +corresponding ip address + +If your linux OS uses `systemctl` run the following commands +```bash +sudo resolvconf -u +systemctl disable --now resolvconf.service +``` + +If your linux does not use `systemctl` run the following commands + +TODO add supporting docs for Linux OS that do not use `systemctl` + +See https://linux.die.net/man/5/resolver + +#### Windows + +TODO + +## Testing + +### Add the test ingress +```bash +kubectl apply -f https://github.com/kubernetes/minikube/blob/master/deploy/addons/ingress-dns/example/example.yaml +``` +Note: Minimum Kubernetes version for example ingress is 1.14.7 + +### Validate DNS queries are returning A records +```bash +nslookup hello-john.test $(minikube ip) +nslookup hello-jane.test $(minikube ip) +``` + +### Validate domain names are resolving on host OS +```bash +ping hello-john.test +``` +Expected results: +```text +PING hello-john.test (192.168.99.169): 56 data bytes +64 bytes from 192.168.99.169: icmp_seq=0 ttl=64 time=0.361 ms +``` +```bash +ping hello-jane.test +``` +```text +PING hello-jane.test (192.168.99.169): 56 data bytes +64 bytes from 192.168.99.169: icmp_seq=0 ttl=64 time=0.262 ms +``` + +### Curl the example server +```bash +curl http://hello-john.test +``` +Expected result: +```text +Hello, world! +Version: 1.0.0 +Hostname: hello-world-app-557ff7dbd8-64mtv +``` +```bash +curl http://hello-jane.test +``` +Expected result: +```text +Hello, world! +Version: 1.0.0 +Hostname: hello-world-app-557ff7dbd8-64mtv +``` + +## Known issues + +### .localhost domains will not resolve on chromium +.localhost domains will not correctly resolve on chromium since it is used as a loopback address. Instead use .test, .example, or .invalid + +### .local is a reserved TLD +Do not use .local as this is a reserved TLD for mDNS and bind9 DNS servers + +### Mac OS + +#### mDNS reloading +Each time a file is created or a change is made to a file in `/etc/resolver` you may need to run the following to reload Mac OS mDNS resolver. +```bash +sudo launchctl unload -w /System/Library/LaunchDaemons/com.apple.mDNSResponder.plist +sudo launchctl load -w /System/Library/LaunchDaemons/com.apple.mDNSResponder.plist +``` + +## TODO +- Add a service that runs on the host OS which will update the files in `/etc/resolver` automatically +- Start this service when running `minikube addons enable ingress-dns` and stop the service when running +`minikube addons disable ingress-dns` + +## Contributors +- [Josh Woodcock](https://github.com/woodcockjosh) \ No newline at end of file diff --git a/deploy/addons/ingress-dns/example/example.yaml b/deploy/addons/ingress-dns/example/example.yaml new file mode 100644 index 0000000000..59a7ed8b9b --- /dev/null +++ b/deploy/addons/ingress-dns/example/example.yaml @@ -0,0 +1,82 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hello-world-app + namespace: default +spec: + selector: + matchLabels: + app: hello-world-app + template: + metadata: + labels: + app: hello-world-app + spec: + containers: + - name: hello-world-app + image: gcr.io/google-samples/hello-app:1.0 + ports: + - containerPort: 8080 +--- +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: example-ingress + namespace: kube-system + annotations: + nginx.ingress.kubernetes.io/rewrite-target: /$1 +spec: + rules: + - host: hello-john.test + http: + paths: + - path: /|/(.+) + backend: + serviceName: hello-world-app + servicePort: 80 + - host: hello-jane.test + http: + paths: + - path: /|/(.+) + backend: + serviceName: hello-world-app + servicePort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: hello-world-app + namespace: kube-system +spec: + type: ExternalName + externalName: hello-world-app.default.svc.cluster.local +--- +apiVersion: v1 +kind: Service +metadata: + name: hello-world-app + namespace: default +spec: + ports: + - name: http + port: 80 + targetPort: 8080 + protocol: TCP + type: NodePort + selector: + app: hello-world-app \ No newline at end of file diff --git a/deploy/addons/ingress-dns/ingress-dns-configmap.yaml b/deploy/addons/ingress-dns/ingress-dns-configmap.yaml new file mode 100644 index 0000000000..4e2fb5a6e1 --- /dev/null +++ b/deploy/addons/ingress-dns/ingress-dns-configmap.yaml @@ -0,0 +1,51 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v1 +data: + map-hash-bucket-size: "128" + hsts: "false" +kind: ConfigMap +metadata: + name: minikube-ingress-dns-nginx-load-balancer-conf + namespace: kube-system + labels: + app: minikube-ingress-dns + app.kubernetes.io/part-of: kube-system + addonmanager.kubernetes.io/mode: EnsureExists +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: minikube-ingress-dns-tcp-services + namespace: kube-system + labels: + app: minikube-ingress-dns + app.kubernetes.io/part-of: kube-system + addonmanager.kubernetes.io/mode: EnsureExists +data: + 53: "kube-system/kube-ingress-dns-minikube:5353" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: minikube-ingress-dns-udp-services + namespace: kube-system + labels: + app: minikube-ingress-dns + app.kubernetes.io/part-of: kube-system + addonmanager.kubernetes.io/mode: EnsureExists +data: + 53: "kube-system/kube-ingress-dns-minikube:5353" \ No newline at end of file diff --git a/deploy/addons/ingress-dns/ingress-dns-dns-server-pod.yaml b/deploy/addons/ingress-dns/ingress-dns-dns-server-pod.yaml new file mode 100644 index 0000000000..aaaea7425b --- /dev/null +++ b/deploy/addons/ingress-dns/ingress-dns-dns-server-pod.yaml @@ -0,0 +1,102 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: minikube-ingress-dns + namespace: kube-system + labels: + app: minikube-ingress-dns + kubernetes.io/bootstrapping: rbac-defaults + app.kubernetes.io/part-of: kube-system + addonmanager.kubernetes.io/mode: Reconcile +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: minikube-ingress-dns + namespace: kube-system + labels: + app: minikube-ingress-dns + kubernetes.io/bootstrapping: rbac-defaults + app.kubernetes.io/part-of: kube-system + addonmanager.kubernetes.io/mode: Reconcile +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - patch + resourceNames: + - tcp-services + - udp-services + - apiGroups: + - "" + - "extensions" + - "networking.k8s.io" + resources: + - ingresses + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: minikube-ingress-dns + namespace: kube-system + labels: + app: minikube-ingress-dns + kubernetes.io/bootstrapping: rbac-defaults + app.kubernetes.io/part-of: kube-system + addonmanager.kubernetes.io/mode: EnsureExists +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: minikube-ingress-dns + namespace: kube-system +--- +apiVersion: v1 +kind: Pod +metadata: + name: kube-ingress-dns-minikube + namespace: kube-system + labels: + app: minikube-ingress-dns + app.kubernetes.io/part-of: kube-system + addonmanager.kubernetes.io/mode: EnsureExists +spec: + serviceAccountName: minikube-ingress-dns + containers: + - name: minikube-ingress-dns + image: "cryptexlabs/minikube-ingress-dns:0.1.1" + imagePullPolicy: IfNotPresent + ports: + - containerPort: 5353 + hostPort: 5353 + protocol: TCP + - containerPort: 5353 + hostPort: 5353 + protocol: UDP + env: + - name: DNS_PORT + value: "5353" \ No newline at end of file diff --git a/deploy/addons/ingress-dns/ingress-dns-nginx-pod.yaml.tmpl b/deploy/addons/ingress-dns/ingress-dns-nginx-pod.yaml.tmpl new file mode 100644 index 0000000000..bb936ebf02 --- /dev/null +++ b/deploy/addons/ingress-dns/ingress-dns-nginx-pod.yaml.tmpl @@ -0,0 +1,229 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: minikube-ingress-dns-nginx-ingress + namespace: kube-system + labels: + kubernetes.io/bootstrapping: rbac-defaults + app: minikube-ingress-dns-nginx-ingress-controller + app.kubernetes.io/part-of: kube-system + addonmanager.kubernetes.io/mode: Reconcile +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: system:minikube-ingress-dns-nginx-ingress + labels: + kubernetes.io/bootstrapping: rbac-defaults + app: minikube-ingress-dns-nginx-ingress-controller + app.kubernetes.io/part-of: kube-system + addonmanager.kubernetes.io/mode: Reconcile +rules: + - apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses/status + verbs: + - update +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: system::minikube-ingress-dns-nginx-ingress-role + namespace: kube-system + labels: + kubernetes.io/bootstrapping: rbac-defaults + app: minikube-ingress-dns-nginx-ingress-controller + app.kubernetes.io/part-of: kube-system + addonmanager.kubernetes.io/mode: Reconcile +rules: + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - namespaces + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + # Defaults to "-" + # Here: "-" + # This has to be adapted if you change either parameter + # when launching the nginx-ingress-controller. + - ingress-controller-leader-nginx + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: system::minikube-ingress-dns-nginx-ingress-role-binding + namespace: kube-system + labels: + kubernetes.io/bootstrapping: rbac-defaults + app: minikube-ingress-dns-nginx-ingress-controller + app.kubernetes.io/part-of: kube-system + addonmanager.kubernetes.io/mode: EnsureExists +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: system::minikube-ingress-dns-nginx-ingress-role +subjects: + - kind: ServiceAccount + name: minikube-ingress-dns-nginx-ingress + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: system:minikube-ingress-dns-nginx-ingress + labels: + kubernetes.io/bootstrapping: rbac-defaults + app: minikube-ingress-dns-nginx-ingress-controller + app.kubernetes.io/part-of: kube-system + addonmanager.kubernetes.io/mode: EnsureExists +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:minikube-ingress-dns-nginx-ingress +subjects: + - kind: ServiceAccount + name: minikube-ingress-dns-nginx-ingress + namespace: kube-system +--- +apiVersion: v1 +kind: Pod +metadata: + name: minikube-ingress-dns-nginx-ingress-controller + namespace: kube-system + labels: + app: minikube-ingress-dns-nginx-ingress-controller + app.kubernetes.io/part-of: kube-system + addonmanager.kubernetes.io/mode: EnsureExists +spec: + serviceAccountName: minikube-ingress-dns-nginx-ingress + terminationGracePeriodSeconds: 60 + hostNetwork: true + containers: + - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller{{.ExoticArch}}:0.25.1 + name: nginx-ingress-controller + imagePullPolicy: IfNotPresent + readinessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + livenessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 1 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 53 + hostPort: 53 + - containerPort: 8008 + - containerPort: 4333 + args: + - /nginx-ingress-controller + - --configmap=$(POD_NAMESPACE)/minikube-ingress-dns-nginx-load-balancer-conf + - --tcp-services-configmap=$(POD_NAMESPACE)/minikube-ingress-dns-tcp-services + - --udp-services-configmap=$(POD_NAMESPACE)/minikube-ingress-dns-udp-services + - --annotations-prefix=nginx.ingress.kubernetes.io + - --http-port=8008 + - --https-port=4333 + # use minikube IP address in ingress status field + - --report-node-internal-ip-address + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + # www-data -> 33 + runAsUser: 33 \ No newline at end of file diff --git a/deploy/addons/ingress-dns/ingress-dns-svc.yaml b/deploy/addons/ingress-dns/ingress-dns-svc.yaml new file mode 100644 index 0000000000..41187d95d8 --- /dev/null +++ b/deploy/addons/ingress-dns/ingress-dns-svc.yaml @@ -0,0 +1,37 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-ingress-dns-minikube + namespace: kube-system + labels: + app: minikube-ingress-dns + app.kubernetes.io/part-of: kube-system + addonmanager.kubernetes.io/mode: EnsureExists +spec: + selector: + app: minikube-ingress-dns + clusterIP: None + ports: + - name: tcp-port + port: 5353 + targetPort: 5353 + protocol: TCP + - name: udp-port + port: 5353 + targetPort: 5353 + protocol: UDP \ No newline at end of file diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index 60afb00da2..f2c73b9813 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -348,6 +348,32 @@ var Addons = map[string]*Addon{ "0640", true), }, false, "helm-tiller"), + "ingress-dns": NewAddon([]*BinAsset{ + MustBinAsset( + "deploy/addons/ingress-dns/ingress-dns-configmap.yaml", + vmpath.GuestAddonsDir, + "ingress-dns-configmap.yaml", + "0640", + false), + MustBinAsset( + "deploy/addons/ingress-dns/ingress-dns-dns-server-pod.yaml", + vmpath.GuestAddonsDir, + "ingress-dns-dns-server-pod.yaml", + "0640", + false), + MustBinAsset( + "deploy/addons/ingress-dns/ingress-dns-nginx-pod.yaml.tmpl", + vmpath.GuestAddonsDir, + "ingress-dns-nginx-pod.yaml", + "0640", + true), + MustBinAsset( + "deploy/addons/ingress-dns/ingress-dns-svc.yaml", + vmpath.GuestAddonsDir, + "ingress-dns-svc.yaml", + "0640", + false), + }, false, "ingress-dns"), } // AddMinikubeDirAssets adds all addons and files to the list diff --git a/site/content/en/docs/Tasks/addons.md b/site/content/en/docs/Tasks/addons.md index f097466f8b..0082b19f11 100644 --- a/site/content/en/docs/Tasks/addons.md +++ b/site/content/en/docs/Tasks/addons.md @@ -23,6 +23,7 @@ minikube has a set of built-in addons that, when enabled, can be used within Kub * [gvisor](../deploy/addons/gvisor/README.md) * [storage-provisioner-gluster](../deploy/addons/storage-provisioner-gluster/README.md) * [helm-tiller](../deploy/addons/helm-tiller/README.md) +* [ingress-dns](../deploy/addons/ingress-dns/README.md) ## Listing available addons