kubernetes
/
minikube
mirror of https://github.com/kubernetes/minikube.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge

pull/3686/head
Thomas Stromberg 2019-02-15 07:06:22 -08:00
parent 5013af7917 71971667a0
commit 1d0733e1e4
25 changed files with 680 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
cmd/minikube/cmd/logs.go
View File

@ -17,17 +17,28 @@ limitations under the License.
package cmd
import (
"os"
"github.com/spf13/cobra"
"github.com/spf13/viper"
cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
"k8s.io/minikube/pkg/minikube/config"
"k8s.io/minikube/pkg/minikube/cruntime"
"k8s.io/minikube/pkg/minikube/exit"
"k8s.io/minikube/pkg/minikube/logs"
"k8s.io/minikube/pkg/minikube/machine"
)
const (
// number of problems per log to output
numberOfProblems = 5
)
var (
follow bool
// followLogs triggers tail -f mode
followLogs bool
// numberOfLines is how many lines to output, set via -n
numberOfLines int
// showProblems only shows lines that match known issues
showProblems bool
)
// logsCmd represents the logs command
@ -36,17 +47,47 @@ var logsCmd = &cobra.Command{
Short: "Gets the logs of the running instance, used for debugging minikube, not user code",
Long: `Gets the logs of the running instance, used for debugging minikube, not user code.`,
Run: func(cmd *cobra.Command, args []string) {
cfg, err := config.Load()
if err != nil {
exit.WithError("Error getting config", err)
}
api, err := machine.NewAPIClient()
if err != nil {
exit.WithError("Error getting client", err)
}
defer api.Close()
clusterBootstrapper, err := GetClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper))
h, err := api.Load(config.GetMachineName())
if err != nil {
exit.WithError("api load", err)
}
runner, err := machine.CommandRunner(h)
if err != nil {
exit.WithError("command runner", err)
}
bs, err := GetClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper))
if err != nil {
exit.WithError("Error getting cluster bootstrapper", err)
}
err = clusterBootstrapper.GetClusterLogsTo(follow, os.Stdout)
cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: runner})
if err != nil {
exit.WithError("Unable to get runtime", err)
}
if followLogs {
err := logs.Follow(cr, bs, runner)
if err != nil {
exit.WithError("Follow", err)
}
return
}
if showProblems {
problems := logs.FindProblems(cr, bs, runner)
logs.OutputProblems(problems, numberOfProblems)
return
}
err = logs.Output(cr, bs, runner, numberOfLines)
if err != nil {
exit.WithError("Error getting machine logs", err)
}
@ -54,6 +95,8 @@ var logsCmd = &cobra.Command{
}
func init() {
logsCmd.Flags().BoolVarP(&follow, "follow", "f", false, "Show only the most recent journal entries, and continuously print new entries as they are appended to the journal.")
logsCmd.Flags().BoolVarP(&followLogs, "follow", "f", false, "Show only the most recent journal entries, and continuously print new entries as they are appended to the journal.")
logsCmd.Flags().BoolVar(&showProblems, "problems", false, "Show only log entries which point to known problems")
logsCmd.Flags().IntVarP(&numberOfLines, "length", "n", 50, "Number of lines back to go within the log")
RootCmd.AddCommand(logsCmd)
}

32
cmd/minikube/cmd/start.go
View File

@ -46,6 +46,7 @@ import (
"k8s.io/minikube/pkg/minikube/constants"
"k8s.io/minikube/pkg/minikube/cruntime"
"k8s.io/minikube/pkg/minikube/exit"
"k8s.io/minikube/pkg/minikube/logs"
"k8s.io/minikube/pkg/minikube/machine"
pkgutil "k8s.io/minikube/pkg/util"
"k8s.io/minikube/pkg/util/kubeconfig"
@ -187,15 +188,19 @@ func runStart(cmd *cobra.Command, args []string) {
if err := saveConfig(config); err != nil {
exit.WithError("Failed to save config", err)
}
runner, err := machine.CommandRunner(host)
if err != nil {
exit.WithError("Failed to get command runner", err)
}
configureRuntimes(host)
cr := configureRuntimes(host, runner)
bs := prepareHostEnvironment(m, config.KubernetesConfig)
waitCacheImages(&cacheGroup)
// The kube config must be update must come before bootstrapping, otherwise health checks may use a stale IP
kubeconfig := updateKubeConfig(host, &config)
bootstrapCluster(bs, config.KubernetesConfig, preexisting)
validateCluster(bs, ip)
bootstrapCluster(bs, cr, runner, config.KubernetesConfig, preexisting)
validateCluster(bs, cr, runner, ip)
configureMounts()
if err = LoadCachedImagesInConfigFile(); err != nil {
console.Failure("Unable to load cached images from config file.")
@ -446,12 +451,7 @@ func updateKubeConfig(h *host.Host, c *cfg.Config) *kubeconfig.KubeConfigSetup {
}
// configureRuntimes does what needs to happen to get a runtime going.
func configureRuntimes(h *host.Host) {
runner, err := machine.CommandRunner(h)
if err != nil {
exit.WithError("Failed to get command runner", err)
}
func configureRuntimes(h *host.Host, runner bootstrapper.CommandRunner) cruntime.Manager {
config := cruntime.Config{Type: viper.GetString(containerRuntime), Runner: runner}
cr, err := cruntime.New(config)
if err != nil {
@ -469,7 +469,7 @@ func configureRuntimes(h *host.Host) {
if err != nil {
exit.WithError("Failed to enable container runtime", err)
}
return cr
}
// waitCacheImages blocks until the image cache jobs complete
@ -484,7 +484,7 @@ func waitCacheImages(g *errgroup.Group) {
}
// bootstrapCluster starts Kubernetes using the chosen bootstrapper
func bootstrapCluster(bs bootstrapper.Bootstrapper, kc cfg.KubernetesConfig, preexisting bool) {
func bootstrapCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner bootstrapper.CommandRunner, kc cfg.KubernetesConfig, preexisting bool) {
console.OutStyle("pulling", "Pulling images used by Kubernetes %s ...", kc.KubernetesVersion)
if err := bs.PullImages(kc); err != nil {
console.OutStyle("failure", "Unable to pull images, which may be OK: %v", err)
@ -495,19 +495,19 @@ func bootstrapCluster(bs bootstrapper.Bootstrapper, kc cfg.KubernetesConfig, pre
if preexisting {
console.OutStyle("restarting", "Relaunching Kubernetes %s using %s ... ", kc.KubernetesVersion, bsName)
if err := bs.RestartCluster(kc); err != nil {
exit.WithError("Error restarting cluster", err)
exit.WithProblems("Error restarting cluster", err, logs.FindProblems(r, bs, runner))
}
return
}
console.OutStyle("launch", "Launching Kubernetes %s using %s ... ", kc.KubernetesVersion, bsName)
if err := bs.StartCluster(kc); err != nil {
exit.WithError("Error starting cluster", err)
exit.WithProblems("Error starting cluster", err, logs.FindProblems(r, bs, runner))
}
}
// validateCluster validates that the cluster is well-configured and healthy
func validateCluster(bs bootstrapper.Bootstrapper, ip string) {
func validateCluster(bs bootstrapper.Bootstrapper, r cruntime.Manager, runner bootstrapper.CommandRunner, ip string) {
console.OutStyle("verifying-noline", "Verifying component health ...")
kStat := func() (err error) {
st, err := bs.GetKubeletStatus()
@ -519,7 +519,7 @@ func validateCluster(bs bootstrapper.Bootstrapper, ip string) {
}
err := pkgutil.RetryAfter(20, kStat, 3*time.Second)
if err != nil {
exit.WithError("kubelet checks failed", err)
exit.WithProblems("kubelet checks failed", err, logs.FindProblems(r, bs, runner))
}
aStat := func() (err error) {
st, err := bs.GetApiServerStatus(net.ParseIP(ip))
@ -532,7 +532,7 @@ func validateCluster(bs bootstrapper.Bootstrapper, ip string) {
err = pkgutil.RetryAfter(30, aStat, 10*time.Second)
if err != nil {
exit.WithError("apiserver checks failed", err)
exit.WithProblems("apiserver checks failed", err, logs.FindProblems(r, bs, runner))
}
console.OutLn("")
}

2
docs/README.md
View File

@ -14,6 +14,8 @@
* **GPUs** ([gpu.md](gpu.md)): Using NVIDIA GPUs on minikube
* **OpenID Connect Authentication** ([openid_connect_auth](openid_connect_auth)): Using OIDC Authentication on minikube
### Installation and debugging
* **Driver installation** ([drivers.md](drivers.md)): In depth instructions for installing the various hypervisor drivers

6
docs/drivers.md
View File

@ -154,3 +154,9 @@ export LATEST_VERSION=$(curl -L -s -H 'Accept: application/json' https://github.
&& chmod +x docker-machine-driver-vmware \
&& mv docker-machine-driver-vmware /usr/local/bin/
```
To use the driver you would do:
```shell
minikube start --vm-driver vmware
```

34
docs/openid_connect_auth.md Normal file
View File

@ -0,0 +1,34 @@
# OpenID Connect Authentication
Minikube `kube-apiserver` can be configured to support OpenID Connect Authentication.
Read more about OpenID Connect Authentication for Kubernetes here: https://kubernetes.io/docs/reference/access-authn-authz/authentication/#openid-connect-tokens
## Configuring the API Server
Configuration values can be passed to the API server using the `--extra-config` flag on the `minikube start` command. See [configuring_kubernetes.md](https://github.com/kubernetes/minikube/blob/master/docs/configuring_kubernetes.md) for more details.
The following example configures your Minikube cluster to support RBAC and OIDC:
```shell
minikube start \
--extra-config=apiserver.authorization-mode=RBAC \
--extra-config=apiserver.oidc-issuer-url=https://example.com \
--extra-config=apiserver.oidc-username-claim=email \
--extra-config=apiserver.oidc-client-id=kubernetes-local
```
## Configuring kubectl
You can use the kubectl `oidc` authenticator to create a kubeconfig as shown in the Kubernetes docs: https://kubernetes.io/docs/reference/access-authn-authz/authentication/#option-1-oidc-authenticator
`minikube start` already creates a kubeconfig that includes a `cluster`, in order to use it with your `oidc` authenticator kubeconfig, you can run:
```shell
kubectl config set-context kubernetes-local-oidc --cluster=minikube --user username@example.com
Context "kubernetes-local-oidc" created.
kubectl config use-context kubernetes-local-oidc
```
For the new context to work you will need to create, at the very minimum, a `Role` and a `RoleBinding` in your cluster to grant permissions to the `subjects` included in your `oidc-username-claim`.

2
hack/jenkins/linux_integration_tests_none.sh
View File

@ -42,6 +42,8 @@ sudo kubeadm reset || sudo kubeadm reset -f || true
sudo rm -rf /data/*
# Cleanup old Kubernetes configs
sudo rm -rf /etc/kubernetes/*
# Cleanup old minikube files
sudo rm -rf /var/lib/minikube/*
# Stop any leftover kubelets
systemctl is-active --quiet kubelet \
&& echo "stopping kubelet" \

116
pkg/drivers/hyperkit/driver.go
View File

@ -22,6 +22,7 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
golog "log"
"os"
"os/user"
"path"
@ -79,19 +80,28 @@ func NewDriver(hostName, storePath string) *Driver {
// PreCreateCheck is called to enforce pre-creation steps
func (d *Driver) PreCreateCheck() error {
return d.verifyRootPermissions()
}
// verifyRootPermissions is called before any step which needs root access
func (d *Driver) verifyRootPermissions() error {
exe, err := os.Executable()
if err != nil {
return err
}
if syscall.Geteuid() != 0 {
euid := syscall.Geteuid()
log.Debugf("exe=%s uid=%d", exe, euid)
if euid != 0 {
return fmt.Errorf(permErr, filepath.Base(exe), exe, exe)
}
return nil
}
func (d *Driver) Create() error {
if err := d.verifyRootPermissions(); err != nil {
return err
}
// TODO: handle different disk types.
if err := pkgdrivers.MakeDiskImage(d.BaseDriver, d.Boot2DockerURL, d.DiskSize); err != nil {
return errors.Wrap(err, "making disk image")
@ -125,37 +135,55 @@ func (d *Driver) GetURL() (string, error) {
return fmt.Sprintf("tcp://%s:2376", ip), nil
}
// GetState returns the state that the host is in (running, stopped, etc)
func (d *Driver) GetState() (state.State, error) {
pid := d.getPid()
// Return the state of the hyperkit pid
func pidState(pid int) (state.State, error) {
if pid == 0 {
return state.Stopped, nil
}
p, err := os.FindProcess(pid)
p, err := ps.FindProcess(pid)
if err != nil {
return state.Error, err
}
// Sending a signal of 0 can be used to check the existence of a process.
if err := p.Signal(syscall.Signal(0)); err != nil {
if p == nil {
log.Debugf("hyperkit pid %d missing from process table", pid)
return state.Stopped, nil
}
if p == nil {
// hyperkit or com.docker.hyper
if !strings.Contains(p.Executable(), "hyper") {
log.Debugf("pid %d is stale, and is being used by %s", pid, p.Executable())
return state.Stopped, nil
}
return state.Running, nil
}
// GetState returns the state that the host is in (running, stopped, etc)
func (d *Driver) GetState() (state.State, error) {
if err := d.verifyRootPermissions(); err != nil {
return state.Error, err
}
pid := d.getPid()
log.Debugf("hyperkit pid from json: %d", pid)
return pidState(pid)
}
// Kill stops a host forcefully
func (d *Driver) Kill() error {
if err := d.verifyRootPermissions(); err != nil {
return err
}
return d.sendSignal(syscall.SIGKILL)
}
// Remove a host
func (d *Driver) Remove() error {
if err := d.verifyRootPermissions(); err != nil {
return err
}
s, err := d.GetState()
if err != nil || s == state.Error {
log.Infof("Error checking machine status: %v, assuming it has been removed already", err)
log.Debugf("Error checking machine status: %v, assuming it has been removed already", err)
}
if s == state.Running {
if err := d.Stop(); err != nil {
@ -171,6 +199,10 @@ func (d *Driver) Restart() error {
// Start a host
func (d *Driver) Start() error {
if err := d.verifyRootPermissions(); err != nil {
return err
}
stateDir := filepath.Join(d.StorePath, "machines", d.MachineName)
if err := d.recoverFromUncleanShutdown(); err != nil {
return err
@ -189,6 +221,9 @@ func (d *Driver) Start() error {
h.CPUs = d.CPU
h.Memory = d.Memory
h.UUID = d.UUID
// This should stream logs from hyperkit, but doesn't seem to work.
logger := golog.New(os.Stderr, "hyperkit", golog.LstdFlags)
h.SetLogger(logger)
if vsockPorts, err := d.extractVSockPorts(); err != nil {
return err
@ -197,7 +232,7 @@ func (d *Driver) Start() error {
h.VSockPorts = vsockPorts
}
log.Infof("Using UUID %s", h.UUID)
log.Debugf("Using UUID %s", h.UUID)
mac, err := GetMACAddressFromUUID(h.UUID)
if err != nil {
return errors.Wrap(err, "getting MAC address from UUID")
@ -205,7 +240,7 @@ func (d *Driver) Start() error {
// Need to strip 0's
mac = trimMacAddress(mac)
log.Infof("Generated MAC %s", mac)
log.Debugf("Generated MAC %s", mac)
h.Disks = []hyperkit.DiskConfig{
{
Path: pkgdrivers.GetDiskPath(d.BaseDriver),
@ -213,13 +248,20 @@ func (d *Driver) Start() error {
Driver: "virtio-blk",
},
}
log.Infof("Starting with cmdline: %s", d.Cmdline)
log.Debugf("Starting with cmdline: %s", d.Cmdline)
if err := h.Start(d.Cmdline); err != nil {
return errors.Wrapf(err, "starting with cmd line: %s", d.Cmdline)
}
getIP := func() error {
var err error
st, err := d.GetState()
if err != nil {
return errors.Wrap(err, "get state")
}
if st == state.Error || st == state.Stopped {
return fmt.Errorf("hyperkit crashed! command line:\n hyperkit %s", d.Cmdline)
}
d.IPAddress, err = GetIPAddressByMACAddress(mac)
if err != nil {
return &commonutil.RetriableError{Err: err}
@ -230,6 +272,7 @@ func (d *Driver) Start() error {
if err := commonutil.RetryAfter(30, getIP, 2*time.Second); err != nil {
return fmt.Errorf("IP address never found in dhcp leases file %v", err)
}
log.Debugf("IP: %s", d.IPAddress)
if len(d.NFSShares) > 0 {
log.Info("Setting up NFS mounts")
@ -257,47 +300,46 @@ func (d *Driver) recoverFromUncleanShutdown() error {
stateDir := filepath.Join(d.StorePath, "machines", d.MachineName)
pidFile := filepath.Join(stateDir, pidFileName)
_, err := os.Stat(pidFile)
if os.IsNotExist(err) {
log.Infof("clean start, hyperkit pid file doesn't exist: %s", pidFile)
return nil
}
if err != nil {
return errors.Wrap(err, "checking hyperkit pid file existence")
if _, err := os.Stat(pidFile); err != nil {
if os.IsNotExist(err) {
log.Debugf("clean start, hyperkit pid file doesn't exist: %s", pidFile)
return nil
}
return errors.Wrap(err, "stat")
}
log.Warnf("minikube might have been shutdown in an unclean way, the hyperkit pid file still exists: %s", pidFile)
content, err := ioutil.ReadFile(pidFile)
bs, err := ioutil.ReadFile(pidFile)
if err != nil {
return errors.Wrapf(err, "reading pidfile %s", pidFile)
}
pid, err := strconv.Atoi(string(content))
content := strings.TrimSpace(string(bs))
pid, err := strconv.Atoi(content)
if err != nil {
return errors.Wrapf(err, "parsing pidfile %s", pidFile)
}
p, err := ps.FindProcess(pid)
st, err := pidState(pid)
if err != nil {
return errors.Wrapf(err, "trying to find process for PID %d", pid)
return errors.Wrap(err, "pidState")
}
if p != nil && !strings.Contains(p.Executable(), "hyperkit") {
return fmt.Errorf("something is not right...please stop all minikube instances, seemingly a hyperkit server is already running with pid %d, executable: %s", pid, p.Executable())
log.Debugf("pid %d is in state %q", pid, st)
if st == state.Running {
return nil
}
log.Infof("No running hyperkit process found with PID %d, removing %s...", pid, pidFile)
log.Debugf("Removing stale pid file %s...", pidFile)
if err := os.Remove(pidFile); err != nil {
return errors.Wrap(err, fmt.Sprintf("removing pidFile %s", pidFile))
}
return nil
}
// Stop a host gracefully
func (d *Driver) Stop() error {
if err := d.verifyRootPermissions(); err != nil {
return err
}
d.cleanupNfsExports()
return d.sendSignal(syscall.SIGTERM)
}
@ -334,9 +376,7 @@ func (d *Driver) extractVSockPorts() ([]int, error) {
for _, port := range d.VSockPorts {
p, err := strconv.Atoi(port)
if err != nil {
var err InvalidPortNumberError
err = InvalidPortNumberError(port)
return nil, err
return nil, InvalidPortNumberError(port)
}
vsockPorts = append(vsockPorts, p)
}

1
pkg/drivers/hyperkit/iso.go
View File

@ -21,7 +21,6 @@ import (
"io"
"io/ioutil"
"os"
"strings"
"github.com/hooklift/iso9660"

31
pkg/drivers/hyperkit/network.go
View File

@ -25,12 +25,14 @@ import (
"os/exec"
"regexp"
"strings"
"github.com/docker/machine/libmachine/log"
)
const (
DHCPLeasesFile = "/var/db/dhcpd_leases"
CONFIG_PLIST = "/Library/Preferences/SystemConfiguration/com.apple.vmnet"
NET_ADDR_KEY = "Shared_Net_Address"
LeasesPath = "/var/db/dhcpd_leases"
VMNetDomain = "/Library/Preferences/SystemConfiguration/com.apple.vmnet"
SharedNetAddrKey = "Shared_Net_Address"
)
type DHCPEntry struct {
@ -42,10 +44,11 @@ type DHCPEntry struct {
}
func GetIPAddressByMACAddress(mac string) (string, error) {
return getIpAddressFromFile(mac, DHCPLeasesFile)
return getIPAddressFromFile(mac, LeasesPath)
}
func getIpAddressFromFile(mac, path string) (string, error) {
func getIPAddressFromFile(mac, path string) (string, error) {
log.Debugf("Searching for %s in %s ...", mac, path)
file, err := os.Open(path)
if err != nil {
return "", err
@ -56,12 +59,15 @@ func getIpAddressFromFile(mac, path string) (string, error) {
if err != nil {
return "", err
}
log.Debugf("Found %d entries in %s!", len(dhcpEntries), path)
for _, dhcpEntry := range dhcpEntries {
log.Debugf("dhcp entry: %+v", dhcpEntry)
if dhcpEntry.HWAddress == mac {
log.Debugf("Found match: %s", mac)
return dhcpEntry.IPAddress, nil
}
}
return "", fmt.Errorf("Could not find an IP address for %s", mac)
return "", fmt.Errorf("could not find an IP address for %s", mac)
}
func parseDHCPdLeasesFile(file io.Reader) ([]DHCPEntry, error) {
@ -99,7 +105,7 @@ func parseDHCPdLeasesFile(file io.Reader) ([]DHCPEntry, error) {
case "lease":
dhcpEntry.Lease = val
default:
return dhcpEntries, fmt.Errorf("Unable to parse line: %s", line)
return dhcpEntries, fmt.Errorf("unable to parse line: %s", line)
}
}
return dhcpEntries, scanner.Err()
@ -114,18 +120,17 @@ func trimMacAddress(rawUUID string) string {
}
func GetNetAddr() (net.IP, error) {
_, err := os.Stat(CONFIG_PLIST + ".plist")
if err != nil {
return nil, fmt.Errorf("Does not exist %s", CONFIG_PLIST+".plist")
plistPath := VMNetDomain + ".plist"
if _, err := os.Stat(plistPath); err != nil {
return nil, fmt.Errorf("stat: %v", err)
}
out, err := exec.Command("defaults", "read", CONFIG_PLIST, NET_ADDR_KEY).Output()
out, err := exec.Command("defaults", "read", VMNetDomain, SharedNetAddrKey).Output()
if err != nil {
return nil, err
}
ip := net.ParseIP(strings.TrimSpace(string(out)))
if ip == nil {
return nil, fmt.Errorf("Could not get the network address for vmnet")
return nil, fmt.Errorf("could not get the network address for vmnet")
}
return ip, nil
}

6
pkg/drivers/hyperkit/network_test.go
View File

@ -88,13 +88,13 @@ func Test_getIpAddressFromFile(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := getIpAddressFromFile(tt.args.mac, tt.args.path)
got, err := getIPAddressFromFile(tt.args.mac, tt.args.path)
if (err != nil) != tt.wantErr {
t.Errorf("getIpAddressFromFile() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("getIPAddressFromFile() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("getIpAddressFromFile() = %v, want %v", got, tt.want)
t.Errorf("getIPAddressFromFile() = %v, want %v", got, tt.want)
}
})
}

7
pkg/minikube/assets/addons.go
View File

@ -302,14 +302,15 @@ func addMinikubeDirToAssets(basedir, vmpath string, assets *[]CopyableFile) erro
return errors.Wrapf(err, "checking if %s is directory", hostpath)
}
if !isDir {
if vmpath == "" {
vmdir := vmpath
if vmdir == "" {
rPath, err := filepath.Rel(basedir, hostpath)
if err != nil {
return errors.Wrap(err, "generating relative path")
}
rPath = filepath.Dir(rPath)
rPath = filepath.ToSlash(rPath)
vmpath = path.Join("/", rPath)
vmdir = path.Join("/", rPath)
}
permString := fmt.Sprintf("%o", info.Mode().Perm())
// The conversion will strip the leading 0 if present, so add it back
@ -318,7 +319,7 @@ func addMinikubeDirToAssets(basedir, vmpath string, assets *[]CopyableFile) erro
permString = fmt.Sprintf("0%s", permString)
}
f, err := NewFileAsset(hostpath, vmpath, filepath.Base(hostpath), permString)
f, err := NewFileAsset(hostpath, vmdir, filepath.Base(hostpath), permString)
if err != nil {
return errors.Wrapf(err, "creating file asset for %s", hostpath)
}

159
pkg/minikube/assets/addons_test.go Normal file
View File

@ -0,0 +1,159 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package assets
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/google/go-cmp/cmp"
"k8s.io/minikube/pkg/minikube/constants"
)
func setupTestDir() (string, error) {
path, err := ioutil.TempDir("", "minipath")
if err != nil {
return "", err
}
os.Setenv(constants.MinikubeHome, path)
return path, err
}
func TestAddMinikubeDirAssets(t *testing.T) {
tests := []struct {
description string
baseDir string
files []struct {
relativePath string
expectedPath string
}
vmPath string
expectedCfg string
}{
{
description: "relative path assets",
baseDir: "/files",
files: []struct {
relativePath string
expectedPath string
}{
{
relativePath: "/dir1/file1.txt",
expectedPath: constants.AddonsPath,
},
{
relativePath: "/dir1/file2.txt",
expectedPath: constants.AddonsPath,
},
{
relativePath: "/dir2/file1.txt",
expectedPath: constants.AddonsPath,
},
},
vmPath: constants.AddonsPath,
},
{
description: "absolute path assets",
baseDir: "/files",
files: []struct {
relativePath string
expectedPath string
}{
{
relativePath: "/dir1/file1.txt",
expectedPath: "/dir1",
},
{
relativePath: "/dir1/file2.txt",
expectedPath: "/dir1",
},
{
relativePath: "/dir2/file1.txt",
expectedPath: "/dir2",
},
},
vmPath: "",
},
}
var testDirs = make([]string, 0)
defer func() {
for _, testDir := range testDirs {
err := os.RemoveAll(testDir)
if err != nil {
t.Logf("got unexpected error removing test dir: %v", err)
}
}
}()
for _, test := range tests {
t.Run(test.description, func(t *testing.T) {
testDir, err := setupTestDir()
if err != nil {
t.Errorf("got unexpected error creating test dir: %v", err)
return
}
testDirs = append(testDirs, testDir)
testFileBaseDir := filepath.Join(testDir, test.baseDir)
want := make(map[string]string, 0)
for _, fileDef := range test.files {
err := func() error {
path := filepath.Join(testFileBaseDir, fileDef.relativePath)
err := os.MkdirAll(filepath.Dir(path), 0755)
want[path] = fileDef.expectedPath
if err != nil {
return err
}
file, err := os.Create(path)
if err != nil {
return err
}
defer file.Close()
_, err = file.WriteString("test")
return err
}()
if err != nil {
t.Errorf("unable to create file on fs: %v", err)
return
}
}
var actualFiles []CopyableFile
err = addMinikubeDirToAssets(testFileBaseDir, test.vmPath, &actualFiles)
if err != nil {
t.Errorf("got unexpected error adding minikube dir assets: %v", err)
return
}
got := make(map[string]string, 0)
for _, actualFile := range actualFiles {
got[actualFile.GetAssetName()] = actualFile.GetTargetDir()
}
if diff := cmp.Diff(want, got); diff != "" {
t.Errorf("files differ: (-want +got)\n%s", diff)
}
})
}
}

12
pkg/minikube/bootstrapper/bootstrapper.go
View File

@ -17,13 +17,20 @@ limitations under the License.
package bootstrapper
import (
"io"
"net"
"k8s.io/minikube/pkg/minikube/config"
"k8s.io/minikube/pkg/minikube/constants"
)
// LogOptions are options to be passed to LogCommands
type LogOptions struct {
// Lines is the number of recent log lines to include, as in tail -n.
Lines int
// Follow is whether or not to actively follow the logs, as in tail -f.
Follow bool
}
// Bootstrapper contains all the methods needed to bootstrap a kubernetes cluster
type Bootstrapper interface {
// PullImages pulls images necessary for a cluster. Success should not be required.
@ -32,7 +39,8 @@ type Bootstrapper interface {
UpdateCluster(config.KubernetesConfig) error
RestartCluster(config.KubernetesConfig) error
DeleteCluster(config.KubernetesConfig) error
GetClusterLogsTo(follow bool, out io.Writer) error
// LogCommands returns a map of log type to a command which will display that log.
LogCommands(LogOptions) map[string]string
SetupCerts(cfg config.KubernetesConfig) error
GetKubeletStatus() (string, error)
GetApiServerStatus(net.IP) (string, error)

30
pkg/minikube/bootstrapper/kubeadm/kubeadm.go
View File

@ -21,7 +21,6 @@ import (
"crypto"
"crypto/tls"
"fmt"
"io"
"net"
"net/http"
"os"
@ -121,28 +120,17 @@ func (k *KubeadmBootstrapper) GetApiServerStatus(ip net.IP) (string, error) {
return state.Running.String(), nil
}
// TODO(r2d4): Should this aggregate all the logs from the control plane?
// Maybe subcommands for each component? minikube logs apiserver?
func (k *KubeadmBootstrapper) GetClusterLogsTo(follow bool, out io.Writer) error {
var flags []string
if follow {
flags = append(flags, "-f")
// LogCommands returns a map of log type to a command which will display that log.
func (k *KubeadmBootstrapper) LogCommands(o bootstrapper.LogOptions) map[string]string {
var kcmd strings.Builder
kcmd.WriteString("journalctl -u kubelet")
if o.Lines > 0 {
kcmd.WriteString(fmt.Sprintf(" -n %d", o.Lines))
}
logsCommand := fmt.Sprintf("sudo journalctl %s -u kubelet", strings.Join(flags, " "))
if follow {
if err := k.c.CombinedOutputTo(logsCommand, out); err != nil {
return errors.Wrap(err, "getting cluster logs")
}
} else {
logs, err := k.c.CombinedOutput(logsCommand)
if err != nil {
return errors.Wrap(err, "getting cluster logs")
}
fmt.Fprint(out, logs)
if o.Follow {
kcmd.WriteString(" -f")
}
return nil
return map[string]string{"kubelet": kcmd.String()}
}
func (k *KubeadmBootstrapper) StartCluster(k8s config.KubernetesConfig) error {

5
pkg/minikube/console/style.go
View File

@ -60,8 +60,11 @@ var styles = map[string]style{
"sad": {Prefix: "😿 ", LowPrefix: "* "},
"thumbs-up": {Prefix: "👍 "},
"option": {Prefix: " ▪ "}, // Indented bullet
"url": {Prefix: "👉 "},
"log-entry": {Prefix: " "}, // Indent
"crushed": {Prefix: "💔 "},
"running": {Prefix: "🏃 "},
"provisioning": {Prefix: "🌱 "},
"url": {Prefix: "👉 "},
// Specialized purpose styles
"iso-download": {Prefix: "💿 ", LowPrefix: "@ "},

5
pkg/minikube/cruntime/containerd.go
View File

@ -107,3 +107,8 @@ func (r *Containerd) KillContainers(ids []string) error {
func (r *Containerd) StopContainers(ids []string) error {
return stopCRIContainers(r.Runner, ids)
}
// ContainerLogCmd returns the command to retrieve the log for a container based on ID
func (r *Containerd) ContainerLogCmd(id string, len int, follow bool) string {
return criContainerLogCmd(id, len, follow)
}

15
pkg/minikube/cruntime/cri.go
View File

@ -60,3 +60,18 @@ image-endpoint: unix://{{.Socket}}
}
return cr.Run(fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | sudo tee %s", path.Dir(cPath), b.String(), cPath))
}
// criContainerLogCmd returns the command to retrieve the log for a container based on ID
func criContainerLogCmd(id string, len int, follow bool) string {
var cmd strings.Builder
cmd.WriteString("crictl logs ")
if len > 0 {
cmd.WriteString(fmt.Sprintf("--tail %d ", len))
}
if follow {
cmd.WriteString("--follow ")
}
cmd.WriteString(id)
return cmd.String()
}

5
pkg/minikube/cruntime/crio.go
View File

@ -106,3 +106,8 @@ func (r *CRIO) KillContainers(ids []string) error {
func (r *CRIO) StopContainers(ids []string) error {
return stopCRIContainers(r.Runner, ids)
}
// ContainerLogCmd returns the command to retrieve the log for a container based on ID
func (r *CRIO) ContainerLogCmd(id string, len int, follow bool) string {
return criContainerLogCmd(id, len, follow)
}

2
pkg/minikube/cruntime/cruntime.go
View File

@ -61,6 +61,8 @@ type Manager interface {
KillContainers([]string) error
// StopContainers stops containers based on ID
StopContainers([]string) error
// ContainerLogCmd returns the command to retrieve the log for a container based on ID
ContainerLogCmd(string, int, bool) string
}
// Config is runtime configuration

15
pkg/minikube/cruntime/docker.go
View File

@ -101,3 +101,18 @@ func (r *Docker) KillContainers(ids []string) error {
func (r *Docker) StopContainers(ids []string) error {
return r.Runner.Run(fmt.Sprintf("docker stop %s", strings.Join(ids, " ")))
}
// ContainerLogCmd returns the command to retrieve the log for a container based on ID
func (r *Docker) ContainerLogCmd(id string, len int, follow bool) string {
var cmd strings.Builder
cmd.WriteString("docker logs ")
if len > 0 {
cmd.WriteString(fmt.Sprintf("--tail %d ", len))
}
if follow {
cmd.WriteString("--follow ")
}
cmd.WriteString(id)
return cmd.String()
}

36
pkg/minikube/exit/exit.go
View File

@ -18,6 +18,7 @@ limitations under the License.
package exit
import (
"fmt"
"os"
"github.com/golang/glog"
@ -35,6 +36,9 @@ const (
IO = 74 // IO represents an I/O error
Config = 78 // Config represents an unconfigured or miscon­figured state
Permissions = 77 // Permissions represents a permissions error
// MaxProblems controls the number of problems to show for each source
MaxProblems = 3
)
// Usage outputs a usage error and exits with error code 64
@ -53,14 +57,34 @@ func WithCode(code int, format string, a ...interface{}) {
// WithError outputs an error and exits.
func WithError(msg string, err error) {
console.Fatal(msg+": %v", err)
console.Err("\n")
console.ErrStyle("sad", "Sorry that minikube crashed. If this was unexpected, we would love to hear from you:")
console.ErrStyle("url", "https://github.com/kubernetes/minikube/issues/new")
// use Warning because Error will display a duplicate message to stderr
glog.Warningf(msg)
displayError(msg, err)
// Here is where we would insert code to optionally upload a stack trace.
// We can be smarter about guessing exit codes, but EX_SOFTWARE should suffice.
os.Exit(Software)
}
// WithProblems outputs an error along with any autodetected problems, and exits.
func WithProblems(msg string, err error, problems map[string][]string) {
displayError(msg, err)
for name, lines := range problems {
console.OutStyle("failure", "Problems detected in %q:", name)
if len(lines) > MaxProblems {
lines = lines[:MaxProblems]
}
for _, l := range lines {
console.OutStyle("log-entry", l)
}
}
os.Exit(Software)
}
func displayError(msg string, err error) {
// use Warning because Error will display a duplicate message to stderr
glog.Warningf(fmt.Sprintf("%s: %v", msg, err))
console.Fatal(msg+": %v", err)
console.Err("\n")
console.ErrStyle("sad", "Sorry that minikube crashed. If this was unexpected, we would love to hear from you:")
console.ErrStyle("url", "https://github.com/kubernetes/minikube/issues/new")
}

152
pkg/minikube/logs/logs.go Normal file
View File

@ -0,0 +1,152 @@
/*
Copyright 2019 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// package logs are convenience methods for fetching logs from a minikube cluster
package logs
import (
"bufio"
"bytes"
"fmt"
"os"
"regexp"
"sort"
"strings"
"github.com/golang/glog"
"k8s.io/minikube/pkg/minikube/bootstrapper"
"k8s.io/minikube/pkg/minikube/console"
"k8s.io/minikube/pkg/minikube/cruntime"
)
// rootCauseRe is a regular expression that matches known failure root causes
var rootCauseRe = regexp.MustCompile(`^error: |eviction manager: pods.* evicted|unknown flag: --`)
// importantPods are a list of pods to retrieve logs for, in addition to the bootstrapper logs.
var importantPods = []string{
"k8s_kube-apiserver",
"k8s_coredns_coredns",
"k8s_kube-scheduler",
}
// lookbackwardsCount is how far back to look in a log for problems. This should be large enough to
// include usage messages from a failed binary, but small enough to not include irrelevant problems.
const lookBackwardsCount = 200
// Follow follows logs from multiple files in tail(1) format
func Follow(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner bootstrapper.CommandRunner) error {
cs := []string{}
for _, v := range logCommands(r, bs, 0, true) {
cs = append(cs, v+" &")
}
cs = append(cs, "wait")
return runner.CombinedOutputTo(strings.Join(cs, " "), os.Stdout)
}
// IsProblem returns whether this line matches a known problem
func IsProblem(line string) bool {
return rootCauseRe.MatchString(line)
}
// FindProblems finds possible root causes among the logs
func FindProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner bootstrapper.CommandRunner) map[string][]string {
pMap := map[string][]string{}
cmds := logCommands(r, bs, lookBackwardsCount, false)
for name, cmd := range cmds {
glog.Infof("Gathering logs for %s ...", name)
var b bytes.Buffer
err := runner.CombinedOutputTo(cmds[name], &b)
if err != nil {
glog.Warningf("failed %s: %s: %v", name, cmd, err)
continue
}
scanner := bufio.NewScanner(&b)
problems := []string{}
for scanner.Scan() {
l := scanner.Text()
if IsProblem(l) {
glog.Warningf("Found %s problem: %s", name, l)
problems = append(problems, l)
}
}
if len(problems) > 0 {
pMap[name] = problems
}
}
return pMap
}
// OutputProblems outputs discovered problems.
func OutputProblems(problems map[string][]string, maxLines int) {
for name, lines := range problems {
console.OutStyle("failure", "Problems detected in %q:", name)
if len(lines) > maxLines {
lines = lines[len(lines)-maxLines:]
}
for _, l := range lines {
console.OutStyle("log-entry", l)
}
}
}
// Output displays logs from multiple sources in tail(1) format
func Output(r cruntime.Manager, bs bootstrapper.Bootstrapper, runner bootstrapper.CommandRunner, lines int) error {
cmds := logCommands(r, bs, lines, false)
names := []string{}
for k := range cmds {
names = append(names, k)
}
sort.Strings(names)
failed := []string{}
for _, name := range names {
console.OutLn("==> %s <==", name)
var b bytes.Buffer
err := runner.CombinedOutputTo(cmds[name], &b)
if err != nil {
glog.Errorf("failed: %v", err)
failed = append(failed, name)
continue
}
scanner := bufio.NewScanner(&b)
for scanner.Scan() {
console.OutLn(scanner.Text())
}
}
if len(failed) > 0 {
return fmt.Errorf("unable to fetch logs for: %s", strings.Join(failed, ", "))
}
return nil
}
// logCommands returns a list of commands that would be run to receive the anticipated logs
func logCommands(r cruntime.Manager, bs bootstrapper.Bootstrapper, length int, follow bool) map[string]string {
cmds := bs.LogCommands(bootstrapper.LogOptions{Lines: length, Follow: follow})
for _, pod := range importantPods {
ids, err := r.ListContainers(pod)
if err != nil {
glog.Errorf("Failed to list containers for %q: %v", pod, err)
continue
}
glog.Infof("%d containers: %s", len(ids), ids)
if len(ids) == 0 {
cmds[pod] = fmt.Sprintf("No container was found matching %q", pod)
continue
}
cmds[pod] = r.ContainerLogCmd(ids[0], length, follow)
}
return cmds
}

44
pkg/minikube/logs/logs_test.go Normal file
View File

@ -0,0 +1,44 @@
/*
Copyright 2019 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package logs
import (
"testing"
)
func TestIsProblem(t *testing.T) {
var tests = []struct {
name string
want bool
input string
}{
{"almost", false, "F2350 I would love to be an unknown flag, but I am not -- :( --"},
{"apiserver-required-flag #1962", true, "error: [service-account-issuer is a required flag when BoundServiceAccountTokenVolume is enabled, --service-account-signing-key-file and --service-account-issuer are required flags"},
{"kubelet-eviction #", true, "I0213 07:16:44.041623 2410 eviction_manager.go:187] eviction manager: pods kube-apiserver-minikube_kube-system(87f41e2e0629c3deb5c2239e08d8045d) evicted, waiting for pod to be cleaned up"},
{"kubelet-unknown-flag #3655", true, "F0212 14:55:46.443031 2693 server.go:148] unknown flag: --AllowedUnsafeSysctls"},
{"apiserver-auth-mode #2852", true, `{"log":"Error: unknown flag: --Authorization.Mode\n","stream":"stderr","time":"2018-06-17T22:16:35.134161966Z"}`},
{"apiserver-admission #3524", true, "error: unknown flag: --GenericServerRunOptions.AdmissionControl"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := IsProblem(tc.input)
if got != tc.want {
t.Fatalf("IsProblem(%s)=%v, want %v", tc.input, got, tc.want)
}
})
}
}

32
pkg/util/kubernetes.go
View File

@ -20,8 +20,11 @@ import (
"fmt"
"time"
"github.com/golang/glog"
"github.com/pkg/errors"
appsv1 "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
@ -29,16 +32,15 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
appsv1 "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/client-go/kubernetes"
"github.com/golang/glog"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
)
var (
ReasonableMutateTime = time.Minute * 1
ReasonableStartTime = time.Minute * 5
)
type PodStore struct {
@ -108,11 +110,11 @@ func StartPods(c kubernetes.Interface, namespace string, pod v1.Pod, waitForRunn
return nil
}
// Wait up to 10 minutes for all matching pods to become Running and at least one
// matching pod exists.
// WaitForPodsWithLabelRunning waits for all matching pods to become Running and at least one matching pod exists.
func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels.Selector) error {
glog.Infof("Waiting for pod with label %q in ns %q ...", ns, label)
lastKnownPodNumber := -1
return wait.PollImmediate(constants.APICallRetryInterval, time.Minute*10, func() (bool, error) {
return wait.PollImmediate(constants.APICallRetryInterval, ReasonableStartTime, func() (bool, error) {
listOpts := metav1.ListOptions{LabelSelector: label.String()}
pods, err := c.CoreV1().Pods(ns).List(listOpts)
if err != nil {
@ -139,9 +141,9 @@ func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels
})
}
// Wait up to 10 minutes for a pod to be deleted
// WaitForPodDelete waits for a pod to be deleted
func WaitForPodDelete(c kubernetes.Interface, ns string, label labels.Selector) error {
return wait.PollImmediate(constants.APICallRetryInterval, time.Minute*10, func() (bool, error) {
return wait.PollImmediate(constants.APICallRetryInterval, ReasonableMutateTime, func() (bool, error) {
listOpts := metav1.ListOptions{LabelSelector: label.String()}
pods, err := c.CoreV1().Pods(ns).List(listOpts)
if err != nil {
@ -152,9 +154,9 @@ func WaitForPodDelete(c kubernetes.Interface, ns string, label labels.Selector)
})
}
// Wait up to 10 minutes for the given event to appear
// WaitForEvent waits for the given event to appear
func WaitForEvent(c kubernetes.Interface, ns string, reason string) error {
return wait.PollImmediate(constants.APICallRetryInterval, time.Minute*10, func() (bool, error) {
return wait.PollImmediate(constants.APICallRetryInterval, ReasonableMutateTime, func() (bool, error) {
events, err := c.Events().Events("default").List(metav1.ListOptions{})
if err != nil {
glog.Infof("error getting events: %v", err)

2
test/integration/start_stop_delete_test.go
View File

@ -40,7 +40,7 @@ func TestStartStop(t *testing.T) {
for _, test := range tests {
t.Run(test.runtime, func(t *testing.T) {
runner := NewMinikubeRunner(t)
if test.runtime != "" && usingNoneDriver(runner) {
if test.runtime != "docker" && usingNoneDriver(runner) {
t.Skipf("skipping, can't use %s with none driver", test.runtime)
}