Merge pull request #48 from dlorenc/auth

Add certificate based authentication to localkube/minikube.

Makefile

@@ -24,9 +24,8 @@ clean:
 	rm -rf $(BUILD_DIR)
 
 gopath:
-	rm -rf $(GOPATH)
 	mkdir -p $(shell dirname $(GOPATH)/src/$(REPOPATH))
-	ln -s $(shell pwd) $(GOPATH)/src/$(REPOPATH)
+	ln -s -f $(shell pwd) $(GOPATH)/src/$(REPOPATH)
 
 .PHONY: minikube
 minikube: minikube-$(GOOS)-$(GOARCH)
@@ -46,6 +45,12 @@ localkube-$(GOOS)-$(GOARCH): gopath
 integration: minikube
 	go test -v ./test/integration --tags=integration
 
+localkube-incremental:
+	GOPATH=/go CGO_ENABLED=1 GOBIN=$(shell pwd)/$(BUILD_DIR) go install ./cmd/localkube
+
+docker/localkube:
+	docker run -w /go/src/k8s.io/minikube -v $(shell pwd):/go/src/k8s.io/minikube golang:1.6 make localkube-incremental
+
 .PHONY: test
 test: gopath
 	./test.sh

cmd/minikube/cmd/root_test.go

@@ -17,24 +17,13 @@ limitations under the License.
 package cmd
 
 import (
-	"io/ioutil"
-	"log"
 	"os"
 	"testing"
 
 	"github.com/spf13/cobra"
-	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/tests"
 )
 
-func makeTempDir() string {
-	tempDir, err := ioutil.TempDir("", "minipath")
-	if err != nil {
-		log.Fatal(err)
-	}
-	constants.Minipath = tempDir
-	return tempDir
-}
-
 func runCommand(f func(*cobra.Command, []string)) {
 	cmd := cobra.Command{}
 	var args []string
@@ -43,7 +32,7 @@ func runCommand(f func(*cobra.Command, []string)) {
 
 func TestPreRunDirectories(t *testing.T) {
 	// Make sure we create the required directories.
-	tempDir := makeTempDir()
+	tempDir := tests.MakeTempDir()
 	defer os.RemoveAll(tempDir)
 
 	runCommand(RootCmd.PersistentPreRun)

cmd/minikube/cmd/start.go

@@ -37,6 +37,10 @@ assumes you already have Virtualbox installed.`,
 	Run: runStart,
 }
 
+var (
+	localkubeURL string
+)
+
 func runStart(cmd *cobra.Command, args []string) {
 
 	fmt.Println("Starting local Kubernetes cluster...")
@@ -48,7 +52,11 @@ func runStart(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	if err := cluster.StartCluster(host); err != nil {
+	config := cluster.KubernetesConfig{
+		LocalkubeURL: localkubeURL,
+	}
+
+	if err := cluster.StartCluster(host, config); err != nil {
 		log.Println("Error starting cluster: ", err)
 		os.Exit(1)
 	}
@@ -57,13 +65,23 @@ func runStart(cmd *cobra.Command, args []string) {
 	if err != nil {
 		log.Println("Error connecting to cluster: ", err)
 	}
-	kubeHost = strings.Replace(kubeHost, "tcp://", "http://", -1)
-	kubeHost = strings.Replace(kubeHost, ":2376", ":8080", -1)
-	log.Printf("Kubernetes is available at %s.\n", kubeHost)
-	log.Println("Run this command to use the cluster: ")
-	log.Printf("kubectl config set-cluster minikube --insecure-skip-tls-verify=true --server=%s\n", kubeHost)
+	kubeHost = strings.Replace(kubeHost, "tcp://", "https://", -1)
+	kubeHost = strings.Replace(kubeHost, ":2376", ":443", -1)
+	fmt.Printf("Kubernetes is available at %s.\n", kubeHost)
+	fmt.Println("Run this command to use the cluster: ")
+	fmt.Printf("kubectl config set-cluster minikube --server=%s --certificate-authority=$HOME/.minikube/ca.crt\n", kubeHost)
+	fmt.Println("kubectl config set-credentials minikube --client-certificate=$HOME/.minikube/kubecfg.crt --client-key=$HOME/.minikube/kubecfg.key")
+	fmt.Println("kubectl config set-context minikube --cluster=minikube --user=minikube")
+	fmt.Println("kubectl config use-context minikube")
+
+	if err := cluster.GetCreds(host); err != nil {
+		log.Println("Error configuring authentication: ", err)
+		os.Exit(1)
+	}
 }
 
 func init() {
+	startCmd.Flags().StringVarP(&localkubeURL, "localkube-url", "", "https://storage.googleapis.com/tinykube/localkube", "Location of the localkube binary")
+	startCmd.Flags().MarkHidden("localkube-url")
 	RootCmd.AddCommand(startCmd)
 }

pkg/localkube/apiserver.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -29,9 +30,12 @@ import (
 )
 
 const (
-	APIServerName = "apiserver"
-	APIServerHost = "0.0.0.0"
-	APIServerPort = 8080
+	APIServerName       = "apiserver"
+	APIServerHost       = "127.0.0.1"
+	APIServerPort       = 8080
+	APIServerSecureHost = "0.0.0.0"
+	APIServerSecurePort = 443
+	certPath            = "/srv/kubernetes/certs/"
 )
 
 var (
@@ -62,9 +66,16 @@ func StartAPIServer() {
 	config := options.NewAPIServer()
 
 	// use host/port from vars
+	config.BindAddress = net.ParseIP(APIServerSecureHost)
+	config.SecurePort = APIServerSecurePort
 	config.InsecureBindAddress = net.ParseIP(APIServerHost)
 	config.InsecurePort = APIServerPort
 
+	config.ClientCAFile = filepath.Join(certPath, "ca.crt")
+	config.TLSCertFile = filepath.Join(certPath, "kubernetes-master.crt")
+	config.TLSPrivateKeyFile = filepath.Join(certPath, "kubernetes-master.key")
+	config.AdmissionControl = "NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
+
 	// use localkube etcd
 	config.EtcdConfig = etcdstorage.EtcdConfig{
 		ServerList: KubeEtcdClientURLs,

pkg/localkube/controller-manager.go

@@ -18,6 +18,7 @@ package localkube
 
 import (
 	"os"
+	"path/filepath"
 	"time"
 
 	controllerManager "k8s.io/kubernetes/cmd/kube-controller-manager/app"
@@ -50,6 +51,7 @@ func StartControllerManagerServer() {
 	config.DeletingPodsQps = 0.1
 	config.DeletingPodsBurst = 10
 	config.EnableProfiling = true
+	config.ServiceAccountKeyFile = filepath.Join(certPath, "kubernetes-master.key")
 
 	fn := func() error {
 		return controllerManager.Run(config)

pkg/localkube/kubelet.go

@@ -29,8 +29,7 @@ const (
 )
 
 var (
-	WeaveProxySock = "unix:///var/run/weave/weave.sock"
-	KubeletStop    chan struct{}
+	KubeletStop chan struct{}
 )
 
 func NewKubeletServer(clusterDomain, clusterDNS string, containerized bool) Server {
@@ -50,9 +49,7 @@ func StartKubeletServer(clusterDomain, clusterDNS string, containerized bool) fu
 	// master details
 	config.APIServerList = []string{APIServerURL}
 
-	// Docker
 	config.Containerized = containerized
-	config.DockerEndpoint = WeaveProxySock
 
 	// Networking
 	config.ClusterDomain = clusterDomain

pkg/minikube/cluster/cluster.go

@@ -19,7 +19,9 @@ package cluster
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"log"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -30,6 +32,14 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
+const (
+	remotePath = "/srv/kubernetes/certs"
+)
+
+var (
+	certs = []string{"ca.crt", "kubecfg.key", "kubecfg.crt"}
+)
+
 // StartHost starts a host VM.
 func StartHost(api libmachine.API) (*host.Host, error) {
 	if exists, err := api.Exists(constants.MachineName); err != nil {
@@ -128,35 +138,37 @@ type sshAble interface {
 	RunSSHCommand(string) (string, error)
 }
 
-// StartCluster starts as k8s cluster on the specified Host.
-func StartCluster(h sshAble) error {
-	for _, cmd := range []string{
-		// Download and install weave, if it doesn't exist.
-		`if [ ! -e /usr/local/bin/weave ]; then
-		   sudo curl -L git.io/weave -o /usr/local/bin/weave
-		   sudo chmod a+x /usr/local/bin/weave;
-		 fi`,
-		// Download and install localkube, if it doesn't exist yet.
-		`if [ ! -e /usr/local/bin/localkube ];
-		 then
-		   sudo curl -L https://github.com/redspread/localkube/releases/download/v1.2.1-v1/localkube-linux -o /usr/local/bin/localkube
-		   sudo chmod a+x /usr/local/bin/localkube;
-		 fi`,
-		// Start weave.
-		"weave launch-router",
-		"weave launch-proxy --without-dns --rewrite-inspect",
-		"weave expose -h \"localkube.weave.local\"",
-		// Localkube assumes containerized kubelet, which looks at /rootfs.
-		"if [ ! -e /rootfs ]; then sudo ln -s / /rootfs; fi",
-		// Run with nohup so it stays up. Redirect logs to useful places.
-		"PATH=/usr/local/sbin:$PATH nohup sudo /usr/local/bin/localkube start > /var/log/localkube.out 2> /var/log/localkube.err < /dev/null &"} {
-		output, err := h.RunSSHCommand(cmd)
-		log.Println(output)
+// KubernetesConfig contains the parameters used to start a cluster.
+type KubernetesConfig struct {
+	LocalkubeURL string
+}
+
+// StartCluster starts a k8s cluster on the specified Host.
+func StartCluster(h sshAble, config KubernetesConfig) error {
+	output, err := h.RunSSHCommand(getStartCommand(config.LocalkubeURL))
+	log.Println(output)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// GetCreds gets the generated credentials required to talk to the APIServer.
+func GetCreds(h sshAble) error {
+	localPath := constants.Minipath
+
+	for _, cert := range certs {
+		remoteCertPath := filepath.Join(remotePath, cert)
+		localCertPath := filepath.Join(localPath, cert)
+		data, err := h.RunSSHCommand(fmt.Sprintf("cat %s", remoteCertPath))
 		if err != nil {
 			return err
 		}
+		if err := ioutil.WriteFile(localCertPath, []byte(data), 0644); err != nil {
+			return err
+		}
 	}
-
 	return nil
 }
 

pkg/minikube/cluster/cluster_test.go

@@ -18,6 +18,10 @@ package cluster
 
 import (
 	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
 	"strings"
 	"testing"
 
@@ -57,32 +61,36 @@ func TestCreateHost(t *testing.T) {
 	}
 }
 
+// Mock Host used for testing. When commands are run, the output from CommandOutput
+// is used, if present. Then the output from Error is used, if present. Finally,
+// "", nil is returned.
 type mockHost struct {
-	Commands []string
+	CommandOutput map[string]string
+	Error         string
 }
 
 func (m mockHost) RunSSHCommand(cmd string) (string, error) {
-	m.Commands = append(m.Commands, cmd)
+	output, ok := m.CommandOutput[cmd]
+	if ok {
+		return output, nil
+	}
+	if m.Error != "" {
+		return "", fmt.Errorf(m.Error)
+	}
 	return "", nil
 }
 
 func TestStartCluster(t *testing.T) {
 	h := mockHost{}
-	err := StartCluster(h)
+	err := StartCluster(h, KubernetesConfig{})
 	if err != nil {
 		t.Fatalf("Error starting cluster: %s", err)
 	}
 }
 
-type mockHostError struct{}
-
-func (m mockHostError) RunSSHCommand(cmd string) (string, error) {
-	return "", fmt.Errorf("Error calling command: %s", cmd)
-}
-
 func TestStartClusterError(t *testing.T) {
-	h := mockHostError{}
-	err := StartCluster(h)
+	h := mockHost{Error: "error"}
+	err := StartCluster(h, KubernetesConfig{})
 	if err == nil {
 		t.Fatal("Error not thrown starting cluster.")
 	}
@@ -271,3 +279,53 @@ func TestGetHostStatus(t *testing.T) {
 	StopHost(api)
 	checkState(state.Stopped.String())
 }
+
+func TestGetCreds(t *testing.T) {
+	m := make(map[string]string)
+	for _, cert := range certs {
+		m[fmt.Sprintf("cat %s/%s", remotePath, cert)] = cert
+	}
+
+	h := mockHost{CommandOutput: m}
+
+	tempDir := tests.MakeTempDir()
+	defer os.RemoveAll(tempDir)
+
+	if err := GetCreds(h); err != nil {
+		t.Fatalf("Error starting cluster: %s", err)
+	}
+
+	for _, cert := range certs {
+		// Files should be created with contents matching the output.
+		certPath := filepath.Join(tempDir, cert)
+		contents, err := ioutil.ReadFile(certPath)
+		if err != nil {
+			t.Fatalf("Error %s reading file: %s", err, certPath)
+		}
+		if !reflect.DeepEqual(contents, []byte(cert)) {
+			t.Fatalf("Contents of file are: %s, should be %s", contents, cert)
+		}
+	}
+}
+
+func TestGetCredsError(t *testing.T) {
+	h := mockHost{
+		Error: "error getting creds",
+	}
+	tempDir := tests.MakeTempDir()
+	defer os.RemoveAll(tempDir)
+
+	if err := GetCreds(h); err == nil {
+		t.Fatalf("Error should have been thrown, but was not.")
+	}
+
+	// No files should have been created.
+	for _, cert := range certs {
+		certPath := filepath.Join(tempDir, cert)
+		_, err := os.Stat(certPath)
+		if !os.IsNotExist(err) {
+			t.Fatalf("File %s should not exist.", certPath)
+		}
+	}
+
+}

pkg/minikube/cluster/commands.go

@@ -0,0 +1,39 @@
+package cluster
+
+import "fmt"
+
+var startCommand = `sudo killall localkube || true
+# Download and install localkube, if it doesn't exist yet.
+if [ ! -e /usr/local/bin/localkube ]; then
+	sudo curl --compressed -L %s -o /usr/local/bin/localkube
+	sudo chmod a+x /usr/local/bin/localkube;
+fi
+# Fetch easy-rsa.
+sudo mkdir -p /srv/kubernetes/certs && sudo chmod -R 777 /srv
+if [ ! -e easy-rsa.tar.gz ]; then
+	curl -L -O https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz
+fi
+rm -rf easy-rsa-master
+tar xzf easy-rsa.tar.gz
+# Create certs.
+cert_ip=$(ip addr show ${interface} | grep 192.168 | sed -nEe 's/^[ \t]*inet[ \t]*([0-9.]+)\/.*$/\1/p')
+ts=$(date +%%s)
+if ! grep $cert_ip /srv/kubernetes/certs/kubernetes-master.crt; then
+	cd easy-rsa-master/easyrsa3
+	./easyrsa init-pki
+	./easyrsa --batch "--req-cn=$cert_ip@$ts" build-ca nopass
+	./easyrsa --subject-alt-name="IP:$cert_ip" build-server-full kubernetes-master nopass
+	./easyrsa build-client-full kubecfg nopass
+	cp -p pki/ca.crt /srv/kubernetes/certs/
+	cp -p pki/issued/kubecfg.crt /srv/kubernetes/certs/
+	cp -p pki/private/kubecfg.key /srv/kubernetes/certs/
+	cp -p pki/issued/kubernetes-master.crt /srv/kubernetes/certs/
+	cp -p pki/private/kubernetes-master.key /srv/kubernetes/certs/
+fi
+# Run with nohup so it stays up. Redirect logs to useful places.
+PATH=/usr/local/sbin:$PATH nohup sudo /usr/local/bin/localkube --containerized=false start > /var/log/localkube.out 2> /var/log/localkube.err < /dev/null &
+`
+
+func getStartCommand(localkubeURL string) string {
+	return fmt.Sprintf(startCommand, localkubeURL)
+}

pkg/minikube/tests/dir_utils.go

@@ -0,0 +1,17 @@
+package tests
+
+import (
+	"io/ioutil"
+	"log"
+
+	"k8s.io/minikube/pkg/minikube/constants"
+)
+
+func MakeTempDir() string {
+	tempDir, err := ioutil.TempDir("", "minipath")
+	if err != nil {
+		log.Fatal(err)
+	}
+	constants.Minipath = tempDir
+	return tempDir
+}