diff --git a/pkg/agent/cridockerd/cridockerd.go b/pkg/agent/cridockerd/cridockerd.go index 9b910f3f24..d105a932f7 100644 --- a/pkg/agent/cridockerd/cridockerd.go +++ b/pkg/agent/cridockerd/cridockerd.go @@ -16,6 +16,7 @@ import ( "github.com/k3s-io/k3s/pkg/agent/cri" "github.com/k3s-io/k3s/pkg/cgroups" "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/util" "github.com/sirupsen/logrus" utilsnet "k8s.io/utils/net" @@ -90,5 +91,5 @@ func getDockerCRIArgs(cfg *config.Node) []string { argsMap["runtime-cgroups"] = runtimeRoot } - return config.GetArgs(argsMap, nil) + return util.GetArgs(argsMap, nil) } diff --git a/pkg/agent/run.go b/pkg/agent/run.go index 44c04b5921..8aa3e2ff58 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -235,7 +235,7 @@ func getConntrackConfig(nodeConfig *daemonconfig.Node) (*kubeproxyconfig.KubePro cmd := app2.NewProxyCommand() globalflag.AddGlobalFlags(cmd.Flags(), cmd.Name(), logs.SkipLoggingConfigurationFlags()) - if err := cmd.ParseFlags(daemonconfig.GetArgs(map[string]string{}, nodeConfig.AgentConfig.ExtraKubeProxyArgs)); err != nil { + if err := cmd.ParseFlags(util.GetArgs(map[string]string{}, nodeConfig.AgentConfig.ExtraKubeProxyArgs)); err != nil { return nil, err } maxPerCore, err := cmd.Flags().GetInt32("conntrack-max-per-core") diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 78613bfda1..2f0805f55b 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -423,7 +423,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont serverConfig.ControlConfig.Disables["ccm"] = true } - tlsMinVersionArg := getArgValueFromList("tls-min-version", serverConfig.ControlConfig.ExtraAPIArgs) + tlsMinVersionArg := util.ArgValue("tls-min-version", serverConfig.ControlConfig.ExtraAPIArgs) serverConfig.ControlConfig.MinTLSVersion = tlsMinVersionArg serverConfig.ControlConfig.TLSMinVersion, err = kubeapiserverflag.TLSVersion(tlsMinVersionArg) if err != nil { @@ -438,7 +438,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont // TLS config based on mozilla ssl-config generator // https://ssl-config.mozilla.org/#server=golang&version=1.13.6&config=intermediate&guideline=5.4 // Need to disable the TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 Cipher for TLS1.2 - tlsCipherSuitesArg := getArgValueFromList("tls-cipher-suites", serverConfig.ControlConfig.ExtraAPIArgs) + tlsCipherSuitesArg := util.ArgValue("tls-cipher-suites", serverConfig.ControlConfig.ExtraAPIArgs) tlsCipherSuites := strings.Split(tlsCipherSuitesArg, ",") for i := range tlsCipherSuites { tlsCipherSuites[i] = strings.TrimSpace(tlsCipherSuites[i]) @@ -633,19 +633,6 @@ func validateNetworkConfiguration(serverConfig server.Config) error { return nil } -func getArgValueFromList(searchArg string, argList []string) string { - var value string - for _, arg := range argList { - splitArg := strings.SplitN(arg, "=", 2) - if splitArg[0] == searchArg { - value = splitArg[1] - // break if we found our value - break - } - } - return value -} - func getAPIAddressFromEtcd(ctx context.Context, serverConfig server.Config, agentConfig cmds.Agent) { defer close(agentConfig.APIAddressCh) for { diff --git a/pkg/daemons/agent/agent.go b/pkg/daemons/agent/agent.go index 89e10b9e33..84fd0a850d 100644 --- a/pkg/daemons/agent/agent.go +++ b/pkg/daemons/agent/agent.go @@ -10,6 +10,7 @@ import ( "github.com/k3s-io/k3s/pkg/agent/proxy" daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/daemons/executor" + "github.com/k3s-io/k3s/pkg/util" pkgerrors "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/component-base/logs" @@ -41,7 +42,7 @@ func Agent(ctx context.Context, nodeConfig *daemonconfig.Node, proxy proxy.Proxy func startKubeProxy(ctx context.Context, cfg *daemonconfig.Agent) error { argsMap := kubeProxyArgs(cfg) - args := daemonconfig.GetArgs(argsMap, cfg.ExtraKubeProxyArgs) + args := util.GetArgs(argsMap, cfg.ExtraKubeProxyArgs) logrus.Infof("Running kube-proxy %s", daemonconfig.ArgString(args)) return executor.KubeProxy(ctx, args) } @@ -49,7 +50,7 @@ func startKubeProxy(ctx context.Context, cfg *daemonconfig.Agent) error { func startKubelet(ctx context.Context, cfg *daemonconfig.Agent) error { argsMap := kubeletArgs(cfg) - args := daemonconfig.GetArgs(argsMap, cfg.ExtraKubeletArgs) + args := util.GetArgs(argsMap, cfg.ExtraKubeletArgs) logrus.Infof("Running kubelet %s", daemonconfig.ArgString(args)) return executor.Kubelet(ctx, args) diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 7d74f43950..d952a0b1f2 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -5,7 +5,6 @@ import ( "fmt" "net" "net/http" - "sort" "strings" "sync" @@ -409,79 +408,3 @@ func (a ArgString) String() string { } return b.String() } - -// GetArgs appends extra arguments to existing arguments with logic to override any default -// arguments whilst also allowing to prefix and suffix default string slice arguments. -func GetArgs(initialArgs map[string]string, extraArgs []string) []string { - const hyphens = "--" - - multiArgs := make(map[string][]string) - - for _, unsplitArg := range extraArgs { - splitArg := strings.SplitN(strings.TrimPrefix(unsplitArg, hyphens), "=", 2) - arg := splitArg[0] - value := "true" - if len(splitArg) > 1 { - value = splitArg[1] - } - - // After the first iteration, initial args will be empty when handling - // duplicate arguments as they will form part of existingValues - cleanedArg := strings.TrimRight(arg, "-+") - initialValue, initialValueExists := initialArgs[cleanedArg] - existingValues, existingValuesFound := multiArgs[cleanedArg] - - newValues := make([]string, 0) - if strings.HasSuffix(arg, "+") { // Append value to initial args - if initialValueExists { - newValues = append(newValues, initialValue) - } - if existingValuesFound { - newValues = append(newValues, existingValues...) - } - newValues = append(newValues, value) - - } else if strings.HasSuffix(arg, "-") { // Prepend value to initial args - newValues = append(newValues, value) - if initialValueExists { - newValues = append(newValues, initialValue) - } - if existingValuesFound { - newValues = append(newValues, existingValues...) - } - } else { // Append value ignoring initial args - if existingValuesFound { - newValues = append(newValues, existingValues...) - } - newValues = append(newValues, value) - } - - delete(initialArgs, cleanedArg) - multiArgs[cleanedArg] = newValues - - } - - // Add any remaining initial args to the map - for arg, value := range initialArgs { - multiArgs[arg] = []string{value} - } - - // Get args so we can output them sorted whilst preserving the order of - // repeated keys - var keys []string - for arg := range multiArgs { - keys = append(keys, arg) - } - sort.Strings(keys) - - var args []string - for _, arg := range keys { - values := multiArgs[arg] - for _, value := range values { - cmd := fmt.Sprintf("%s%s=%s", hyphens, strings.TrimPrefix(arg, hyphens), value) - args = append(args, cmd) - } - } - - return args -} diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 97b71e8bde..cd7b682f81 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -142,7 +142,7 @@ func controllerManager(ctx context.Context, cfg *config.Control) error { argsMap["vmodule"] = cfg.VModule } - args := config.GetArgs(argsMap, cfg.ExtraControllerArgs) + args := util.GetArgs(argsMap, cfg.ExtraControllerArgs) logrus.Infof("Running kube-controller-manager %s", config.ArgString(args)) return executor.ControllerManager(ctx, args) @@ -169,7 +169,7 @@ func scheduler(ctx context.Context, cfg *config.Control) error { argsMap["vmodule"] = cfg.VModule } - args := config.GetArgs(argsMap, cfg.ExtraSchedulerAPIArgs) + args := util.GetArgs(argsMap, cfg.ExtraSchedulerAPIArgs) nodeReady := make(chan struct{}) @@ -205,7 +205,9 @@ func apiServer(ctx context.Context, cfg *config.Control) error { argsMap["cert-dir"] = certDir argsMap["allow-privileged"] = "true" argsMap["enable-bootstrap-token-auth"] = "true" - argsMap["authorization-mode"] = strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ",") + if authConfigFile := util.ArgValue("authorization-config", cfg.ExtraAPIArgs); authConfigFile == "" { + argsMap["authorization-mode"] = strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ",") + } argsMap["service-account-signing-key-file"] = runtime.ServiceCurrentKey argsMap["service-cluster-ip-range"] = util.JoinIPNets(cfg.ServiceIPRanges) argsMap["service-node-port-range"] = cfg.ServiceNodePortRange.String() @@ -258,7 +260,7 @@ func apiServer(ctx context.Context, cfg *config.Control) error { argsMap["vmodule"] = cfg.VModule } - args := config.GetArgs(argsMap, cfg.ExtraAPIArgs) + args := util.GetArgs(argsMap, cfg.ExtraAPIArgs) logrus.Infof("Running kube-apiserver %s", config.ArgString(args)) @@ -373,7 +375,7 @@ func cloudControllerManager(ctx context.Context, cfg *config.Control) error { argsMap["vmodule"] = cfg.VModule } - args := config.GetArgs(argsMap, cfg.ExtraCloudControllerArgs) + args := util.GetArgs(argsMap, cfg.ExtraCloudControllerArgs) logrus.Infof("Running cloud-controller-manager %s", config.ArgString(args)) diff --git a/pkg/daemons/executor/embed.go b/pkg/daemons/executor/embed.go index 682d80f379..f4674394ee 100644 --- a/pkg/daemons/executor/embed.go +++ b/pkg/daemons/executor/embed.go @@ -95,7 +95,7 @@ func (e *Embedded) Kubelet(ctx context.Context, args []string) error { func (e *Embedded) KubeProxy(ctx context.Context, args []string) error { command := proxy.NewProxyCommand() - command.SetArgs(daemonconfig.GetArgs(platformKubeProxyArgs(e.nodeConfig), args)) + command.SetArgs(util.GetArgs(platformKubeProxyArgs(e.nodeConfig), args)) go func() { <-e.APIServerReadyChan() diff --git a/pkg/util/args.go b/pkg/util/args.go new file mode 100644 index 0000000000..02838dd95d --- /dev/null +++ b/pkg/util/args.go @@ -0,0 +1,106 @@ +package util + +import ( + "fmt" + "sort" + "strings" +) + +const hyphens = "--" + +// ArgValue returns the value of the first matching arg in the provided list. +func ArgValue(searchArg string, extraArgs []string) string { + var value string + for _, unsplitArg := range extraArgs { + splitArg := strings.SplitN(strings.TrimPrefix(unsplitArg, hyphens), "=", 2) + if splitArg[0] == searchArg { + value = splitArg[1] + // break if we found our value + break + } + } + return value +} + +// GetArgs appends extra arguments to existing arguments with logic to override any default +// arguments whilst also allowing to prefix and suffix default string slice arguments. +func GetArgs(initialArgs map[string]string, extraArgs []string) []string { + + multiArgs := make(map[string][]string) + + for _, unsplitArg := range extraArgs { + splitArg := strings.SplitN(strings.TrimPrefix(unsplitArg, hyphens), "=", 2) + arg := splitArg[0] + value := "true" + if len(splitArg) > 1 { + value = splitArg[1] + } + + // After the first iteration, initial args will be empty when handling + // duplicate arguments as they will form part of existingValues + cleanedArg := strings.TrimRight(arg, "-+") + initialValue, initialValueExists := initialArgs[cleanedArg] + existingValues, existingValuesFound := multiArgs[cleanedArg] + + newValues := make([]string, 0) + if strings.HasSuffix(arg, "+") { // Append value to initial args + if initialValueExists { + newValues = append(newValues, initialValue) + } + if existingValuesFound { + newValues = append(newValues, existingValues...) + } + newValues = append(newValues, value) + + } else if strings.HasSuffix(arg, "-") { // Prepend value to initial args + newValues = append(newValues, value) + if initialValueExists { + newValues = append(newValues, initialValue) + } + if existingValuesFound { + newValues = append(newValues, existingValues...) + } + } else { // Append value ignoring initial args + if existingValuesFound { + newValues = append(newValues, existingValues...) + } + newValues = append(newValues, value) + } + + delete(initialArgs, cleanedArg) + multiArgs[cleanedArg] = newValues + + } + + // Add any remaining initial args to the map + for arg, value := range initialArgs { + multiArgs[arg] = []string{value} + } + + // Get args so we can output them sorted whilst preserving the order of + // repeated keys + var keys []string + for arg := range multiArgs { + keys = append(keys, arg) + } + sort.Strings(keys) + + var args []string + for _, arg := range keys { + values := multiArgs[arg] + for _, value := range values { + cmd := fmt.Sprintf("%s%s=%s", hyphens, strings.TrimPrefix(arg, hyphens), value) + args = append(args, cmd) + } + } + + return args +} + +// AddFeatureGate correctly appends a feature gate key pair to the feature gates CLI switch. +func AddFeatureGate(current, new string) string { + if current == "" { + return new + } + return current + "," + new +} diff --git a/pkg/daemons/config/types_test.go b/pkg/util/args_test.go similarity index 73% rename from pkg/daemons/config/types_test.go rename to pkg/util/args_test.go index 9744abe2b7..40f91106c9 100644 --- a/pkg/daemons/config/types_test.go +++ b/pkg/util/args_test.go @@ -1,10 +1,48 @@ -package config +package util import ( "reflect" "testing" ) +func Test_UnitAddFeatureGate(t *testing.T) { + type args struct { + currentArg string + featureGate string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "Feature gate added to empty arg", + args: args{ + currentArg: "", + featureGate: "SupportPodPidsLimit=false", + }, + want: "SupportPodPidsLimit=false", + }, + { + name: "Feature gate added to existing arg", + args: args{ + currentArg: "SupportPodPidsLimit=false", + featureGate: "DevicePlugins=false", + }, + want: "SupportPodPidsLimit=false,DevicePlugins=false", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := AddFeatureGate(tt.args.currentArg, tt.args.featureGate) + if got != tt.want { + t.Errorf("error, should be " + tt.want + ", but got " + got) + } + }) + } +} + func Test_UnitGetArgs(t *testing.T) { type args struct { argsMap map[string]string diff --git a/pkg/util/gates.go b/pkg/util/gates.go deleted file mode 100644 index d7498b52d6..0000000000 --- a/pkg/util/gates.go +++ /dev/null @@ -1,9 +0,0 @@ -package util - -// AddFeatureGate correctly appends a feature gate key pair to the feature gates CLI switch. -func AddFeatureGate(current, new string) string { - if current == "" { - return new - } - return current + "," + new -} diff --git a/pkg/util/gates_test.go b/pkg/util/gates_test.go deleted file mode 100644 index bbe0c18dff..0000000000 --- a/pkg/util/gates_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package util - -import ( - "testing" -) - -func TestAddFeatureGate(t *testing.T) { - type args struct { - currentArg string - featureGate string - } - tests := []struct { - name string - args args - want string - }{ - { - name: "Feature gate added to empty arg", - args: args{ - currentArg: "", - featureGate: "SupportPodPidsLimit=false", - }, - want: "SupportPodPidsLimit=false", - }, - { - name: "Feature gate added to existing arg", - args: args{ - currentArg: "SupportPodPidsLimit=false", - featureGate: "DevicePlugins=false", - }, - want: "SupportPodPidsLimit=false,DevicePlugins=false", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := AddFeatureGate(tt.args.currentArg, tt.args.featureGate) - if got != tt.want { - t.Errorf("error, should be " + tt.want + ", but got " + got) - } - }) - } -}