kubernetes
/
minikube
mirror of https://github.com/kubernetes/minikube.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #1985 from r2d4/kubelet-config-kubeadm 

Implement extra-config for kubeadm components
pull/2033/head
Aaron Prindle 2017-10-04 16:16:50 -07:00 committed by GitHub
parent d23b7fd7cd a139ccd0ab
commit 8dbe63d697
6 changed files with 522 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -150,6 +150,10 @@ endif
test-iso:
go test -v $(REPOPATH)/test/integration --tags=iso --minikube-args="--iso-url=file://$(shell pwd)/out/buildroot/output/images/rootfs.iso9660"
.PHONY: test-pkg
test-pkg/%:
go test -v -test.timeout=30m $(REPOPATH)/$* --tags="$(MINIKUBE_BUILD_TAGS)"
.PHONY: integration
integration: out/minikube
go test -v -test.timeout=30m $(REPOPATH)/test/integration --tags="$(MINIKUBE_INTEGRATION_BUILD_TAGS)" $(TEST_ARGS)

102
pkg/minikube/bootstrapper/kubeadm/kubeadm.go
View File

@ -20,7 +20,6 @@ import (
"bytes"
"crypto"
"fmt"
"html/template"
"os"
"path/filepath"
"strings"
@ -44,48 +43,6 @@ type KubeadmBootstrapper struct {
c bootstrapper.CommandRunner
}
// TODO(r2d4): template this with bootstrapper.KubernetesConfig
const kubeletSystemdConf = `
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true"
Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true"
Environment="KUBELET_DNS_ARGS=--cluster-dns=10.0.0.10 --cluster-domain=cluster.local"
Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0"
Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs"
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS $KUBELET_EXTRA_ARGS
`
const kubeletService = `
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=http://kubernetes.io/docs/
[Service]
ExecStart=/usr/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10
[Install]
WantedBy=multi-user.target
`
const kubeadmConfigTmpl = `
apiVersion: kubeadm.k8s.io/v1alpha1
kind: MasterConfiguration
api:
advertiseAddress: {{.AdvertiseAddress}}
bindPort: {{.APIServerPort}}
kubernetesVersion: {{.KubernetesVersion}}
certificatesDir: {{.CertDir}}
networking:
serviceSubnet: {{.ServiceCIDR}}
etcd:
dataDir: {{.EtcdDataDir}}
nodeName: {{.NodeName}}
`
func NewKubeadmBootstrapper(api libmachine.API) (*KubeadmBootstrapper, error) {
h, err := api.Load(config.GetMachineName())
if err != nil {
@ -147,10 +104,8 @@ func (k *KubeadmBootstrapper) GetClusterLogs(follow bool) (string, error) {
func (k *KubeadmBootstrapper) StartCluster(k8s bootstrapper.KubernetesConfig) error {
// We use --skip-preflight-checks since we have our own custom addons
// that we also stick in /etc/kubernetes/manifests
kubeadmTmpl := "sudo /usr/bin/kubeadm init --config {{.KubeadmConfigFile}} --skip-preflight-checks"
t := template.Must(template.New("kubeadmTmpl").Parse(kubeadmTmpl))
b := bytes.Buffer{}
if err := t.Execute(&b, struct{ KubeadmConfigFile string }{constants.KubeadmConfigFile}); err != nil {
if err := kubeadmInitTemplate.Execute(&b, struct{ KubeadmConfigFile string }{constants.KubeadmConfigFile}); err != nil {
return err
}
@ -197,14 +152,6 @@ func addAddons(files *[]assets.CopyableFile) error {
}
func (k *KubeadmBootstrapper) RestartCluster(k8s bootstrapper.KubernetesConfig) error {
restoreTmpl := `
sudo kubeadm alpha phase certs all --config {{.KubeadmConfigFile}} &&
sudo /usr/bin/kubeadm alpha phase kubeconfig all --config {{.KubeadmConfigFile}} &&
sudo /usr/bin/kubeadm alpha phase controlplane all --config {{.KubeadmConfigFile}} &&
sudo /usr/bin/kubeadm alpha phase etcd local --config {{.KubeadmConfigFile}}
`
t := template.Must(template.New("restoreTmpl").Parse(restoreTmpl))
opts := struct {
KubeadmConfigFile string
}{
@ -212,7 +159,7 @@ func (k *KubeadmBootstrapper) RestartCluster(k8s bootstrapper.KubernetesConfig)
}
b := bytes.Buffer{}
if err := t.Execute(&b, opts); err != nil {
if err := kubeadmRestoreTemplate.Execute(&b, opts); err != nil {
return err
}
@ -231,19 +178,44 @@ func (k *KubeadmBootstrapper) SetupCerts(k8s bootstrapper.KubernetesConfig) erro
return bootstrapper.SetupCerts(k.c, k8s)
}
func NewKubeletConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
if err != nil {
return "", errors.Wrap(err, "parsing kubernetes version")
}
extraOpts, err := ExtraConfigForComponent(Kubelet, k8s.ExtraOptions, version)
if err != nil {
return "", errors.Wrap(err, "generating extra configuration for kubelet")
}
extraFlags := convertToFlags(extraOpts)
b := bytes.Buffer{}
if err := kubeletSystemdTemplate.Execute(&b, map[string]string{"ExtraOptions": extraFlags}); err != nil {
return "", err
}
return b.String(), nil
}
func (k *KubeadmBootstrapper) UpdateCluster(cfg bootstrapper.KubernetesConfig) error {
if cfg.ShouldLoadCachedImages {
// Make best effort to load any cached images
go machine.LoadImages(k.c, constants.GetKubeadmCachedImages(cfg.KubernetesVersion), constants.ImageCacheDir)
}
kubeadmCfg, err := k.generateConfig(cfg)
kubeadmCfg, err := generateConfig(cfg)
if err != nil {
return errors.Wrap(err, "generating kubeadm cfg")
}
kubeletCfg, err := NewKubeletConfig(cfg)
if err != nil {
return errors.Wrap(err, "generating kubelet config")
}
files := []assets.CopyableFile{
assets.NewMemoryAssetTarget([]byte(kubeletService), constants.KubeletServiceFile, "0640"),
assets.NewMemoryAssetTarget([]byte(kubeletSystemdConf), constants.KubeletSystemdConfFile, "0640"),
assets.NewMemoryAssetTarget([]byte(kubeletCfg), constants.KubeletSystemdConfFile, "0640"),
assets.NewMemoryAssetTarget([]byte(kubeadmCfg), constants.KubeadmConfigFile, "0640"),
}
@ -290,9 +262,17 @@ sudo systemctl start kubelet
return nil
}
func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
t := template.Must(template.New("kubeadmConfigTmpl").Parse(kubeadmConfigTmpl))
func generateConfig(k8s bootstrapper.KubernetesConfig) (string, error) {
version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
if err != nil {
return "", errors.Wrap(err, "parsing kubernetes version")
}
// generates a map of component to extra args for apiserver, controller-manager, and scheduler
extraComponentConfig, err := NewComponentExtraArgs(k8s.ExtraOptions, version)
if err != nil {
return "", errors.Wrap(err, "generating extra component config for kubeadm")
}
opts := struct {
CertDir string
ServiceCIDR string
@ -301,6 +281,7 @@ func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig)
KubernetesVersion string
EtcdDataDir string
NodeName string
ExtraArgs []ComponentExtraArgs
}{
CertDir: util.DefaultCertPath,
ServiceCIDR: util.DefaultInsecureRegistry,
@ -309,10 +290,11 @@ func (k *KubeadmBootstrapper) generateConfig(k8s bootstrapper.KubernetesConfig)
KubernetesVersion: k8s.KubernetesVersion,
EtcdDataDir: "/data", //TODO(r2d4): change to something else persisted
NodeName: k8s.NodeName,
ExtraArgs: extraComponentConfig,
}
b := bytes.Buffer{}
if err := t.Execute(&b, opts); err != nil {
if err := kubeadmConfigTemplate.Execute(&b, opts); err != nil {
return "", err
}

134
pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go Normal file
View File

@ -0,0 +1,134 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubeadm
import (
"testing"
"k8s.io/minikube/pkg/minikube/bootstrapper"
"k8s.io/minikube/pkg/util"
)
func TestGenerateConfig(t *testing.T) {
tests := []struct {
description string
cfg bootstrapper.KubernetesConfig
expectedCfg string
shouldErr bool
}{
{
description: "no extra args",
cfg: bootstrapper.KubernetesConfig{
NodeIP: "192.168.1.100",
KubernetesVersion: "v1.8.0",
NodeName: "minikube",
},
expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
kind: MasterConfiguration
api:
advertiseAddress: 192.168.1.100
bindPort: 8443
kubernetesVersion: v1.8.0
certificatesDir: /var/lib/localkube/certs/
networking:
serviceSubnet: 10.0.0.0/24
etcd:
dataDir: /data
nodeName: minikube
`,
},
{
description: "extra args all components",
cfg: bootstrapper.KubernetesConfig{
NodeIP: "192.168.1.101",
KubernetesVersion: "v1.8.0-alpha.0",
NodeName: "extra-args-minikube",
ExtraOptions: util.ExtraOptionSlice{
util.ExtraOption{
Component: Apiserver,
Key: "fail-no-swap",
Value: "true",
},
util.ExtraOption{
Component: ControllerManager,
Key: "kube-api-burst",
Value: "32",
},
util.ExtraOption{
Component: Scheduler,
Key: "scheduler-name",
Value: "mini-scheduler",
},
},
},
expectedCfg: `apiVersion: kubeadm.k8s.io/v1alpha1
kind: MasterConfiguration
api:
advertiseAddress: 192.168.1.101
bindPort: 8443
kubernetesVersion: v1.8.0-alpha.0
certificatesDir: /var/lib/localkube/certs/
networking:
serviceSubnet: 10.0.0.0/24
etcd:
dataDir: /data
nodeName: extra-args-minikube
apiServerExtraArgs:
fail-no-swap: true
controllerManagerExtraArgs:
kube-api-burst: 32
schedulerExtraArgs:
scheduler-name: mini-scheduler
`,
},
{
// Unknown components should fail silently
description: "unknown component",
cfg: bootstrapper.KubernetesConfig{
NodeIP: "192.168.1.101",
KubernetesVersion: "v1.8.0-alpha.0",
NodeName: "extra-args-minikube",
ExtraOptions: util.ExtraOptionSlice{
util.ExtraOption{
Component: "not-a-real-component",
Key: "killswitch",
Value: "true",
},
},
},
shouldErr: true,
},
}
for _, test := range tests {
t.Run(test.description, func(t *testing.T) {
actualCfg, err := generateConfig(test.cfg)
if err != nil && !test.shouldErr {
t.Errorf("got unexpected error generating config: %s", err)
return
}
if err == nil && test.shouldErr {
t.Errorf("expected error but got none, config: %s", actualCfg)
return
}
if actualCfg != test.expectedCfg {
t.Errorf("actual config does not match expected. actual:\n%sexpected:\n%s", actualCfg, test.expectedCfg)
return
}
})
}
}

70
pkg/minikube/bootstrapper/kubeadm/templates.go Normal file
View File

@ -0,0 +1,70 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubeadm
import "html/template"
var kubeadmConfigTemplate = template.Must(template.New("kubeadmConfigTemplate").Parse(`apiVersion: kubeadm.k8s.io/v1alpha1
kind: MasterConfiguration
api:
advertiseAddress: {{.AdvertiseAddress}}
bindPort: {{.APIServerPort}}
kubernetesVersion: {{.KubernetesVersion}}
certificatesDir: {{.CertDir}}
networking:
serviceSubnet: {{.ServiceCIDR}}
etcd:
dataDir: {{.EtcdDataDir}}
nodeName: {{.NodeName}}
{{range .ExtraArgs}}{{.Component}}:{{range $key, $value := .Options}}
{{$key}}: {{$value}}
{{end}}{{end}}`))
var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(`
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true"
Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true"
Environment="KUBELET_DNS_ARGS=--cluster-dns=10.0.0.10 --cluster-domain=cluster.local"
Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0"
Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs"
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_DNS_ARGS $KUBELET_CADVISOR_ARGS $KUBELET_CGROUP_ARGS {{.ExtraOptions}}
`))
const kubeletService = `
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=http://kubernetes.io/docs/
[Service]
ExecStart=/usr/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10
[Install]
WantedBy=multi-user.target
`
var kubeadmRestoreTemplate = template.Must(template.New("kubeadmRestoreTemplate").Parse(`
sudo kubeadm alpha phase certs all --config {{.KubeadmConfigFile}} &&
sudo /usr/bin/kubeadm alpha phase kubeconfig all --config {{.KubeadmConfigFile}} &&
sudo /usr/bin/kubeadm alpha phase controlplane all --config {{.KubeadmConfigFile}} &&
sudo /usr/bin/kubeadm alpha phase etcd local --config {{.KubeadmConfigFile}}
`))
var kubeadmInitTemplate = template.Must(template.New("kubeadmInitTemplate").Parse("sudo /usr/bin/kubeadm init --config {{.KubeadmConfigFile}} --skip-preflight-checks"))

169
pkg/minikube/bootstrapper/kubeadm/versions.go Normal file
View File

@ -0,0 +1,169 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubeadm
import (
"fmt"
"strings"
"github.com/blang/semver"
"github.com/golang/glog"
"github.com/pkg/errors"
"k8s.io/minikube/pkg/util"
)
// These are the components that can be configured
// through the "extra-config"
const (
Kubelet = "kubelet"
Apiserver = "apiserver"
Scheduler = "scheduler"
ControllerManager = "controller-manager"
)
// ExtraConfigForComponent generates a map of flagname-value pairs for a k8s
// component.
func ExtraConfigForComponent(component string, opts util.ExtraOptionSlice, version semver.Version) (map[string]string, error) {
versionedOpts, err := DefaultOptionsForComponentAndVersion(component, version)
if err != nil {
return nil, errors.Wrapf(err, "setting version specific options for %s", component)
}
for _, opt := range opts {
if opt.Component == component {
if val, ok := versionedOpts[opt.Key]; ok {
glog.Infof("Overwriting default %s=%s with user provided %s=%s for component %s", opt.Key, val, opt.Key, opt.Value, component)
}
versionedOpts[opt.Key] = opt.Value
}
}
return versionedOpts, nil
}
type ComponentExtraArgs struct {
Component string
Options map[string]string
}
var componentToKubeadmConfigKey = map[string]string{
Apiserver: "apiServerExtraArgs",
Scheduler: "schedulerExtraArgs",
ControllerManager: "controllerManagerExtraArgs",
}
func NewComponentExtraArgs(opts util.ExtraOptionSlice, version semver.Version) ([]ComponentExtraArgs, error) {
var kubeadmExtraArgs []ComponentExtraArgs
for _, extraOpt := range opts {
kubeadmKey, ok := componentToKubeadmConfigKey[extraOpt.Component]
if !ok {
return nil, fmt.Errorf("Unknown component %s. Valid components and kubeadm config are %v", componentToKubeadmConfigKey, componentToKubeadmConfigKey)
}
extraConfig, err := ExtraConfigForComponent(extraOpt.Component, opts, version)
if err != nil {
return nil, errors.Wrapf(err, "getting kubeadm extra args for %s", extraOpt.Component)
}
if len(extraConfig) > 0 {
kubeadmExtraArgs = append(kubeadmExtraArgs, ComponentExtraArgs{
Component: kubeadmKey,
Options: extraConfig,
})
}
}
return kubeadmExtraArgs, nil
}
func ParseKubernetesVersion(version string) (semver.Version, error) {
// Strip leading 'v' prefix from version for semver parsing
v, err := semver.Make(version[1:])
if err != nil {
return semver.Version{}, errors.Wrap(err, "parsing kubernetes version")
}
return v, nil
}
func convertToFlags(opts map[string]string) string {
var flags []string
for k, v := range opts {
flags = append(flags, fmt.Sprintf("--%s=%s", k, v))
}
return strings.Join(flags, " ")
}
// VersionedExtraOption holds information on flags to apply to a specific range
// of versions
type VersionedExtraOption struct {
// Special Cases:
//
// If LessThanOrEqual and GreaterThanOrEqual are both nil, the flag will be applied
// to all versions
//
// If LessThanOrEqual == GreaterThanOrEqual, the flag will only be applied to that
// specific version
// The flag and component that will be set
Option util.ExtraOption
// This flag will only be applied to versions before or equal to this version
// If it is the default value, it will have no upper bound on versions the
// flag is applied to
LessThanOrEqual semver.Version
// The flag will only be applied to versions after or equal to this version
// If it is the default value, it will have no lower bound on versions the
// flag is applied to
GreaterThanOrEqual semver.Version
}
var versionSpecificOpts = []VersionedExtraOption{
{
Option: util.ExtraOption{
Component: Kubelet,
Key: "fail-swap-on",
Value: "false",
},
GreaterThanOrEqual: semver.MustParse("1.8.0-alpha.0"),
},
}
func VersionIsBetween(version, gte, lte semver.Version) bool {
if gte.NE(semver.Version{}) && !version.GTE(gte) {
return false
}
if lte.NE(semver.Version{}) && !version.LTE(lte) {
return false
}
return true
}
func DefaultOptionsForComponentAndVersion(component string, version semver.Version) (map[string]string, error) {
versionedOpts := map[string]string{}
for _, opts := range versionSpecificOpts {
if opts.Option.Component == component {
if VersionIsBetween(version, opts.GreaterThanOrEqual, opts.LessThanOrEqual) {
if val, ok := versionedOpts[opts.Option.Key]; ok {
return nil, fmt.Errorf("Flag %s=%s already set %s=%s", opts.Option.Key, opts.Option.Value, opts.Option.Key, val)
}
versionedOpts[opts.Option.Key] = opts.Option.Value
}
}
}
return versionedOpts, nil
}

103
pkg/minikube/bootstrapper/kubeadm/versions_test.go Normal file
View File

@ -0,0 +1,103 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubeadm
import (
"testing"
"github.com/blang/semver"
)
func TestVersionIsBetween(t *testing.T) {
tests := []struct {
description string
ver semver.Version
gte semver.Version
lte semver.Version
expected bool
}{
{
description: "between",
ver: semver.MustParse("1.8.0"),
gte: semver.MustParse("1.7.0"),
lte: semver.MustParse("1.9.0"),
expected: true,
},
{
description: "less than minimum version",
ver: semver.MustParse("1.6.0"),
gte: semver.MustParse("1.7.0"),
lte: semver.MustParse("1.9.0"),
expected: false,
},
{
description: "greather than max version",
ver: semver.MustParse("2.8.0"),
gte: semver.MustParse("1.7.0"),
lte: semver.MustParse("1.9.0"),
expected: true,
},
{
description: "equal to max version",
ver: semver.MustParse("1.9.0"),
gte: semver.MustParse("1.7.0"),
lte: semver.MustParse("1.9.0"),
expected: true,
},
{
description: "equal to min version",
ver: semver.MustParse("1.7.0"),
gte: semver.MustParse("1.7.0"),
lte: semver.MustParse("1.9.0"),
expected: true,
},
{
description: "alpha between",
ver: semver.MustParse("1.8.0-alpha.0"),
gte: semver.MustParse("1.8.0"),
lte: semver.MustParse("1.9.0"),
expected: true,
},
{
description: "beta greater than alpha",
ver: semver.MustParse("1.8.0-beta.1"),
gte: semver.MustParse("1.8.0"),
lte: semver.MustParse("1.8.0-alpha.0"),
expected: false,
},
}
for _, test := range tests {
t.Run(test.description, func(t *testing.T) {
t.Parallel()
between := VersionIsBetween(test.ver, test.gte, test.lte)
if between != test.expected {
t.Errorf("Expected: %t, Actual: %t", test.expected, between)
}
})
}
}
func TestParseKubernetesVersion(t *testing.T) {
version, err := ParseKubernetesVersion("v1.8.0-alpha.5")
if err != nil {
t.Fatalf("Error parsing version: %s", err)
}
if version.NE(semver.MustParse("1.8.0-alpha.5")) {
t.Errorf("Expected: %s, Actual:%s", "1.8.0-alpha.5", version)
}
}