diff --git a/Makefile b/Makefile index facf4127a3..6c55509749 100644 --- a/Makefile +++ b/Makefile @@ -24,9 +24,8 @@ clean: rm -rf $(BUILD_DIR) gopath: - rm -rf $(GOPATH) mkdir -p $(shell dirname $(GOPATH)/src/$(REPOPATH)) - ln -s $(shell pwd) $(GOPATH)/src/$(REPOPATH) + ln -s -f $(shell pwd) $(GOPATH)/src/$(REPOPATH) .PHONY: minikube minikube: minikube-$(GOOS)-$(GOARCH) @@ -46,6 +45,12 @@ localkube-$(GOOS)-$(GOARCH): gopath integration: minikube go test -v ./test/integration --tags=integration +localkube-incremental: + GOPATH=/go CGO_ENABLED=1 GOBIN=$(shell pwd)/$(BUILD_DIR) go install ./cmd/localkube + +docker/localkube: + docker run -w /go/src/k8s.io/minikube -v $(shell pwd):/go/src/k8s.io/minikube golang:1.6 make localkube-incremental + .PHONY: test test: gopath ./test.sh diff --git a/cmd/minikube/cmd/root_test.go b/cmd/minikube/cmd/root_test.go index 2694d5de08..d4e8c13317 100644 --- a/cmd/minikube/cmd/root_test.go +++ b/cmd/minikube/cmd/root_test.go @@ -17,24 +17,13 @@ limitations under the License. package cmd import ( - "io/ioutil" - "log" "os" "testing" "github.com/spf13/cobra" - "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/tests" ) -func makeTempDir() string { - tempDir, err := ioutil.TempDir("", "minipath") - if err != nil { - log.Fatal(err) - } - constants.Minipath = tempDir - return tempDir -} - func runCommand(f func(*cobra.Command, []string)) { cmd := cobra.Command{} var args []string @@ -43,7 +32,7 @@ func runCommand(f func(*cobra.Command, []string)) { func TestPreRunDirectories(t *testing.T) { // Make sure we create the required directories. - tempDir := makeTempDir() + tempDir := tests.MakeTempDir() defer os.RemoveAll(tempDir) runCommand(RootCmd.PersistentPreRun) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 000202d48c..0148989d36 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -37,6 +37,10 @@ assumes you already have Virtualbox installed.`, Run: runStart, } +var ( + localkubeURL string +) + func runStart(cmd *cobra.Command, args []string) { fmt.Println("Starting local Kubernetes cluster...") @@ -48,7 +52,11 @@ func runStart(cmd *cobra.Command, args []string) { os.Exit(1) } - if err := cluster.StartCluster(host); err != nil { + config := cluster.KubernetesConfig{ + LocalkubeURL: localkubeURL, + } + + if err := cluster.StartCluster(host, config); err != nil { log.Println("Error starting cluster: ", err) os.Exit(1) } @@ -57,13 +65,23 @@ func runStart(cmd *cobra.Command, args []string) { if err != nil { log.Println("Error connecting to cluster: ", err) } - kubeHost = strings.Replace(kubeHost, "tcp://", "http://", -1) - kubeHost = strings.Replace(kubeHost, ":2376", ":8080", -1) - log.Printf("Kubernetes is available at %s.\n", kubeHost) - log.Println("Run this command to use the cluster: ") - log.Printf("kubectl config set-cluster minikube --insecure-skip-tls-verify=true --server=%s\n", kubeHost) + kubeHost = strings.Replace(kubeHost, "tcp://", "https://", -1) + kubeHost = strings.Replace(kubeHost, ":2376", ":443", -1) + fmt.Printf("Kubernetes is available at %s.\n", kubeHost) + fmt.Println("Run this command to use the cluster: ") + fmt.Printf("kubectl config set-cluster minikube --server=%s --certificate-authority=$HOME/.minikube/ca.crt\n", kubeHost) + fmt.Println("kubectl config set-credentials minikube --client-certificate=$HOME/.minikube/kubecfg.crt --client-key=$HOME/.minikube/kubecfg.key") + fmt.Println("kubectl config set-context minikube --cluster=minikube --user=minikube") + fmt.Println("kubectl config use-context minikube") + + if err := cluster.GetCreds(host); err != nil { + log.Println("Error configuring authentication: ", err) + os.Exit(1) + } } func init() { + startCmd.Flags().StringVarP(&localkubeURL, "localkube-url", "", "https://storage.googleapis.com/tinykube/localkube", "Location of the localkube binary") + startCmd.Flags().MarkHidden("localkube-url") RootCmd.AddCommand(startCmd) } diff --git a/pkg/localkube/apiserver.go b/pkg/localkube/apiserver.go index 9447ee1562..364e0e9d5e 100644 --- a/pkg/localkube/apiserver.go +++ b/pkg/localkube/apiserver.go @@ -20,6 +20,7 @@ import ( "fmt" "net" "os" + "path/filepath" "strings" "time" @@ -29,9 +30,12 @@ import ( ) const ( - APIServerName = "apiserver" - APIServerHost = "0.0.0.0" - APIServerPort = 8080 + APIServerName = "apiserver" + APIServerHost = "127.0.0.1" + APIServerPort = 8080 + APIServerSecureHost = "0.0.0.0" + APIServerSecurePort = 443 + certPath = "/srv/kubernetes/certs/" ) var ( @@ -62,9 +66,16 @@ func StartAPIServer() { config := options.NewAPIServer() // use host/port from vars + config.BindAddress = net.ParseIP(APIServerSecureHost) + config.SecurePort = APIServerSecurePort config.InsecureBindAddress = net.ParseIP(APIServerHost) config.InsecurePort = APIServerPort + config.ClientCAFile = filepath.Join(certPath, "ca.crt") + config.TLSCertFile = filepath.Join(certPath, "kubernetes-master.crt") + config.TLSPrivateKeyFile = filepath.Join(certPath, "kubernetes-master.key") + config.AdmissionControl = "NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota" + // use localkube etcd config.EtcdConfig = etcdstorage.EtcdConfig{ ServerList: KubeEtcdClientURLs, diff --git a/pkg/localkube/controller-manager.go b/pkg/localkube/controller-manager.go index d85512d613..32d12c489d 100644 --- a/pkg/localkube/controller-manager.go +++ b/pkg/localkube/controller-manager.go @@ -18,6 +18,7 @@ package localkube import ( "os" + "path/filepath" "time" controllerManager "k8s.io/kubernetes/cmd/kube-controller-manager/app" @@ -50,6 +51,7 @@ func StartControllerManagerServer() { config.DeletingPodsQps = 0.1 config.DeletingPodsBurst = 10 config.EnableProfiling = true + config.ServiceAccountKeyFile = filepath.Join(certPath, "kubernetes-master.key") fn := func() error { return controllerManager.Run(config) diff --git a/pkg/localkube/kubelet.go b/pkg/localkube/kubelet.go index b438888616..d95fd760c2 100644 --- a/pkg/localkube/kubelet.go +++ b/pkg/localkube/kubelet.go @@ -29,8 +29,7 @@ const ( ) var ( - WeaveProxySock = "unix:///var/run/weave/weave.sock" - KubeletStop chan struct{} + KubeletStop chan struct{} ) func NewKubeletServer(clusterDomain, clusterDNS string, containerized bool) Server { @@ -50,9 +49,7 @@ func StartKubeletServer(clusterDomain, clusterDNS string, containerized bool) fu // master details config.APIServerList = []string{APIServerURL} - // Docker config.Containerized = containerized - config.DockerEndpoint = WeaveProxySock // Networking config.ClusterDomain = clusterDomain diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go index 183d18ff93..ec37a2fcf3 100644 --- a/pkg/minikube/cluster/cluster.go +++ b/pkg/minikube/cluster/cluster.go @@ -19,7 +19,9 @@ package cluster import ( "encoding/json" "fmt" + "io/ioutil" "log" + "path/filepath" "strings" "time" @@ -30,6 +32,14 @@ import ( "k8s.io/minikube/pkg/minikube/constants" ) +const ( + remotePath = "/srv/kubernetes/certs" +) + +var ( + certs = []string{"ca.crt", "kubecfg.key", "kubecfg.crt"} +) + // StartHost starts a host VM. func StartHost(api libmachine.API) (*host.Host, error) { if exists, err := api.Exists(constants.MachineName); err != nil { @@ -128,35 +138,37 @@ type sshAble interface { RunSSHCommand(string) (string, error) } -// StartCluster starts as k8s cluster on the specified Host. -func StartCluster(h sshAble) error { - for _, cmd := range []string{ - // Download and install weave, if it doesn't exist. - `if [ ! -e /usr/local/bin/weave ]; then - sudo curl -L git.io/weave -o /usr/local/bin/weave - sudo chmod a+x /usr/local/bin/weave; - fi`, - // Download and install localkube, if it doesn't exist yet. - `if [ ! -e /usr/local/bin/localkube ]; - then - sudo curl -L https://github.com/redspread/localkube/releases/download/v1.2.1-v1/localkube-linux -o /usr/local/bin/localkube - sudo chmod a+x /usr/local/bin/localkube; - fi`, - // Start weave. - "weave launch-router", - "weave launch-proxy --without-dns --rewrite-inspect", - "weave expose -h \"localkube.weave.local\"", - // Localkube assumes containerized kubelet, which looks at /rootfs. - "if [ ! -e /rootfs ]; then sudo ln -s / /rootfs; fi", - // Run with nohup so it stays up. Redirect logs to useful places. - "PATH=/usr/local/sbin:$PATH nohup sudo /usr/local/bin/localkube start > /var/log/localkube.out 2> /var/log/localkube.err < /dev/null &"} { - output, err := h.RunSSHCommand(cmd) - log.Println(output) +// KubernetesConfig contains the parameters used to start a cluster. +type KubernetesConfig struct { + LocalkubeURL string +} + +// StartCluster starts a k8s cluster on the specified Host. +func StartCluster(h sshAble, config KubernetesConfig) error { + output, err := h.RunSSHCommand(getStartCommand(config.LocalkubeURL)) + log.Println(output) + if err != nil { + return err + } + + return nil +} + +// GetCreds gets the generated credentials required to talk to the APIServer. +func GetCreds(h sshAble) error { + localPath := constants.Minipath + + for _, cert := range certs { + remoteCertPath := filepath.Join(remotePath, cert) + localCertPath := filepath.Join(localPath, cert) + data, err := h.RunSSHCommand(fmt.Sprintf("cat %s", remoteCertPath)) if err != nil { return err } + if err := ioutil.WriteFile(localCertPath, []byte(data), 0644); err != nil { + return err + } } - return nil } diff --git a/pkg/minikube/cluster/cluster_test.go b/pkg/minikube/cluster/cluster_test.go index c122d1d1cc..7cdfd71c36 100644 --- a/pkg/minikube/cluster/cluster_test.go +++ b/pkg/minikube/cluster/cluster_test.go @@ -18,6 +18,10 @@ package cluster import ( "fmt" + "io/ioutil" + "os" + "path/filepath" + "reflect" "strings" "testing" @@ -57,32 +61,36 @@ func TestCreateHost(t *testing.T) { } } +// Mock Host used for testing. When commands are run, the output from CommandOutput +// is used, if present. Then the output from Error is used, if present. Finally, +// "", nil is returned. type mockHost struct { - Commands []string + CommandOutput map[string]string + Error string } func (m mockHost) RunSSHCommand(cmd string) (string, error) { - m.Commands = append(m.Commands, cmd) + output, ok := m.CommandOutput[cmd] + if ok { + return output, nil + } + if m.Error != "" { + return "", fmt.Errorf(m.Error) + } return "", nil } func TestStartCluster(t *testing.T) { h := mockHost{} - err := StartCluster(h) + err := StartCluster(h, KubernetesConfig{}) if err != nil { t.Fatalf("Error starting cluster: %s", err) } } -type mockHostError struct{} - -func (m mockHostError) RunSSHCommand(cmd string) (string, error) { - return "", fmt.Errorf("Error calling command: %s", cmd) -} - func TestStartClusterError(t *testing.T) { - h := mockHostError{} - err := StartCluster(h) + h := mockHost{Error: "error"} + err := StartCluster(h, KubernetesConfig{}) if err == nil { t.Fatal("Error not thrown starting cluster.") } @@ -271,3 +279,53 @@ func TestGetHostStatus(t *testing.T) { StopHost(api) checkState(state.Stopped.String()) } + +func TestGetCreds(t *testing.T) { + m := make(map[string]string) + for _, cert := range certs { + m[fmt.Sprintf("cat %s/%s", remotePath, cert)] = cert + } + + h := mockHost{CommandOutput: m} + + tempDir := tests.MakeTempDir() + defer os.RemoveAll(tempDir) + + if err := GetCreds(h); err != nil { + t.Fatalf("Error starting cluster: %s", err) + } + + for _, cert := range certs { + // Files should be created with contents matching the output. + certPath := filepath.Join(tempDir, cert) + contents, err := ioutil.ReadFile(certPath) + if err != nil { + t.Fatalf("Error %s reading file: %s", err, certPath) + } + if !reflect.DeepEqual(contents, []byte(cert)) { + t.Fatalf("Contents of file are: %s, should be %s", contents, cert) + } + } +} + +func TestGetCredsError(t *testing.T) { + h := mockHost{ + Error: "error getting creds", + } + tempDir := tests.MakeTempDir() + defer os.RemoveAll(tempDir) + + if err := GetCreds(h); err == nil { + t.Fatalf("Error should have been thrown, but was not.") + } + + // No files should have been created. + for _, cert := range certs { + certPath := filepath.Join(tempDir, cert) + _, err := os.Stat(certPath) + if !os.IsNotExist(err) { + t.Fatalf("File %s should not exist.", certPath) + } + } + +} diff --git a/pkg/minikube/cluster/commands.go b/pkg/minikube/cluster/commands.go new file mode 100644 index 0000000000..7add988389 --- /dev/null +++ b/pkg/minikube/cluster/commands.go @@ -0,0 +1,39 @@ +package cluster + +import "fmt" + +var startCommand = `sudo killall localkube || true +# Download and install localkube, if it doesn't exist yet. +if [ ! -e /usr/local/bin/localkube ]; then + sudo curl --compressed -L %s -o /usr/local/bin/localkube + sudo chmod a+x /usr/local/bin/localkube; +fi +# Fetch easy-rsa. +sudo mkdir -p /srv/kubernetes/certs && sudo chmod -R 777 /srv +if [ ! -e easy-rsa.tar.gz ]; then + curl -L -O https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz +fi +rm -rf easy-rsa-master +tar xzf easy-rsa.tar.gz +# Create certs. +cert_ip=$(ip addr show ${interface} | grep 192.168 | sed -nEe 's/^[ \t]*inet[ \t]*([0-9.]+)\/.*$/\1/p') +ts=$(date +%%s) +if ! grep $cert_ip /srv/kubernetes/certs/kubernetes-master.crt; then + cd easy-rsa-master/easyrsa3 + ./easyrsa init-pki + ./easyrsa --batch "--req-cn=$cert_ip@$ts" build-ca nopass + ./easyrsa --subject-alt-name="IP:$cert_ip" build-server-full kubernetes-master nopass + ./easyrsa build-client-full kubecfg nopass + cp -p pki/ca.crt /srv/kubernetes/certs/ + cp -p pki/issued/kubecfg.crt /srv/kubernetes/certs/ + cp -p pki/private/kubecfg.key /srv/kubernetes/certs/ + cp -p pki/issued/kubernetes-master.crt /srv/kubernetes/certs/ + cp -p pki/private/kubernetes-master.key /srv/kubernetes/certs/ +fi +# Run with nohup so it stays up. Redirect logs to useful places. +PATH=/usr/local/sbin:$PATH nohup sudo /usr/local/bin/localkube --containerized=false start > /var/log/localkube.out 2> /var/log/localkube.err < /dev/null & +` + +func getStartCommand(localkubeURL string) string { + return fmt.Sprintf(startCommand, localkubeURL) +} diff --git a/pkg/minikube/tests/dir_utils.go b/pkg/minikube/tests/dir_utils.go new file mode 100644 index 0000000000..7dcb5b3651 --- /dev/null +++ b/pkg/minikube/tests/dir_utils.go @@ -0,0 +1,17 @@ +package tests + +import ( + "io/ioutil" + "log" + + "k8s.io/minikube/pkg/minikube/constants" +) + +func MakeTempDir() string { + tempDir, err := ioutil.TempDir("", "minipath") + if err != nil { + log.Fatal(err) + } + constants.Minipath = tempDir + return tempDir +}