2019-12-18 19:31:29 +00:00
/ *
Copyright 2019 The Kubernetes Authors All rights reserved .
Licensed under the Apache License , Version 2.0 ( the "License" ) ;
you may not use this file except in compliance with the License .
You may obtain a copy of the License at
http : //www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing , software
distributed under the License is distributed on an "AS IS" BASIS ,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
See the License for the specific language governing permissions and
limitations under the License .
* /
package addons
import (
2023-03-01 21:07:46 +00:00
"context"
2019-12-19 05:35:46 +00:00
"fmt"
2021-08-11 16:58:03 +00:00
"os/exec"
2020-01-30 23:54:04 +00:00
"path"
2020-06-25 22:35:58 +00:00
"runtime"
2020-02-03 18:49:40 +00:00
"sort"
2019-12-18 19:31:29 +00:00
"strconv"
2020-01-31 01:46:25 +00:00
"strings"
2020-04-08 19:29:23 +00:00
"sync"
2020-01-31 01:46:25 +00:00
"time"
2019-12-18 19:31:29 +00:00
2021-08-22 02:58:34 +00:00
"github.com/blang/semver/v4"
2023-07-13 17:09:08 +00:00
"github.com/docker/machine/libmachine/state"
2019-12-18 19:31:29 +00:00
"github.com/pkg/errors"
2020-06-24 23:31:46 +00:00
"github.com/spf13/viper"
2020-04-15 14:30:10 +00:00
2020-09-29 22:49:41 +00:00
"k8s.io/klog/v2"
2020-04-17 00:49:26 +00:00
"k8s.io/minikube/pkg/drivers/kic/oci"
2020-06-24 23:31:46 +00:00
"k8s.io/minikube/pkg/kapi"
2019-12-18 19:31:29 +00:00
"k8s.io/minikube/pkg/minikube/assets"
2023-02-16 15:45:03 +00:00
"k8s.io/minikube/pkg/minikube/cluster"
2019-12-18 19:31:29 +00:00
"k8s.io/minikube/pkg/minikube/command"
"k8s.io/minikube/pkg/minikube/config"
2020-04-17 00:49:26 +00:00
"k8s.io/minikube/pkg/minikube/constants"
2023-02-16 15:45:03 +00:00
"k8s.io/minikube/pkg/minikube/cruntime"
2020-02-28 23:58:05 +00:00
"k8s.io/minikube/pkg/minikube/driver"
2019-12-18 19:31:29 +00:00
"k8s.io/minikube/pkg/minikube/exit"
"k8s.io/minikube/pkg/minikube/machine"
"k8s.io/minikube/pkg/minikube/out"
2020-07-07 18:25:27 +00:00
"k8s.io/minikube/pkg/minikube/out/register"
2020-08-31 00:25:11 +00:00
"k8s.io/minikube/pkg/minikube/reason"
"k8s.io/minikube/pkg/minikube/style"
2021-02-24 20:48:39 +00:00
"k8s.io/minikube/pkg/minikube/sysinit"
2021-08-22 02:58:34 +00:00
"k8s.io/minikube/pkg/util"
2020-03-25 20:44:28 +00:00
"k8s.io/minikube/pkg/util/retry"
2019-12-18 19:31:29 +00:00
)
2021-03-09 21:32:09 +00:00
// Force is used to override checks for addons
2021-08-13 01:11:16 +00:00
var Force = false
2019-12-18 19:31:29 +00:00
2021-05-26 16:26:10 +00:00
// Refresh is used to refresh pods in specific cases when an addon is enabled
// Currently only used for gcp-auth
2021-08-13 01:11:16 +00:00
var Refresh = false
2021-05-26 16:26:10 +00:00
2021-08-13 20:42:02 +00:00
// ErrSkipThisAddon is a special error that tells us to not error out, but to also not mark the addon as enabled
var ErrSkipThisAddon = errors . New ( "skipping this addon" )
2020-04-09 04:23:35 +00:00
// RunCallbacks runs all actions associated to an addon, but does not set it (thread-safe)
func RunCallbacks ( cc * config . ClusterConfig , name string , value string ) error {
2020-09-29 22:49:41 +00:00
klog . Infof ( "Setting %s=%s in profile %q" , name , value , cc . Name )
2019-12-19 06:21:53 +00:00
a , valid := isAddonValid ( name )
2019-12-19 05:35:46 +00:00
if ! valid {
return errors . Errorf ( "%s is not a valid addon" , name )
}
2020-03-11 00:49:47 +00:00
// Run any additional validations for this property
if err := run ( cc , name , value , a . validations ) ; err != nil {
2021-08-13 20:42:02 +00:00
if errors . Is ( err , ErrSkipThisAddon ) {
return err
}
2020-03-11 00:49:47 +00:00
return errors . Wrap ( err , "running validations" )
}
2022-11-03 23:38:59 +00:00
preStartMessages ( name , value )
2019-12-19 05:35:46 +00:00
// Run any callbacks for this property
2020-03-11 00:49:47 +00:00
if err := run ( cc , name , value , a . callbacks ) ; err != nil {
2021-08-13 20:42:02 +00:00
if errors . Is ( err , ErrSkipThisAddon ) {
return err
}
2020-01-06 21:42:09 +00:00
return errors . Wrap ( err , "running callbacks" )
2019-12-19 05:35:46 +00:00
}
2022-11-03 23:38:59 +00:00
postStartMessages ( cc , name , value )
2020-04-09 04:23:35 +00:00
return nil
}
2022-11-03 23:38:59 +00:00
func preStartMessages ( name , value string ) {
if value != "true" {
return
}
switch name {
case "ambassador" :
out . Styled ( style . Warning , "The ambassador addon has stopped working as of v1.23.0, for more details visit: https://github.com/datawire/ambassador-operator/issues/73" )
case "olm" :
out . Styled ( style . Warning , "The OLM addon has stopped working, for more details visit: https://github.com/operator-framework/operator-lifecycle-manager/issues/2534" )
}
}
func postStartMessages ( cc * config . ClusterConfig , name , value string ) {
if value != "true" {
return
}
clusterName := cc . Name
tipProfileArg := ""
if clusterName != constants . DefaultClusterName {
tipProfileArg = fmt . Sprintf ( " -p %s" , clusterName )
}
switch name {
case "dashboard" :
out . Styled ( style . Tip , ` Some dashboard features require the metrics - server addon . To enable all features please run :
2023-12-20 19:55:54 +00:00
minikube { { . profileArg } } addons enable metrics - server
2022-11-03 23:38:59 +00:00
` , out . V { "profileArg" : tipProfileArg } )
case "headlamp" :
out . Styled ( style . Tip , ` To access Headlamp , use the following command :
2023-12-20 19:55:54 +00:00
minikube { { . profileArg } } service headlamp - n headlamp
2023-12-12 20:52:47 +00:00
` , out . V { "profileArg" : tipProfileArg } )
2022-11-03 23:38:59 +00:00
tokenGenerationTip := "To authenticate in Headlamp, fetch the Authentication Token using the following command:"
createSvcAccountToken := "kubectl create token headlamp --duration 24h -n headlamp"
getSvcAccountToken := ` export SECRET = $ ( kubectl get secrets -- namespace headlamp - o custom - columns = ":metadata.name" | grep "headlamp-token" )
kubectl get secret $ SECRET -- namespace headlamp -- template = \ { \ { . data . token \ } \ } | base64 -- decode `
clusterVersion := cc . KubernetesConfig . KubernetesVersion
parsedClusterVersion , err := util . ParseKubernetesVersion ( clusterVersion )
if err != nil {
tokenGenerationTip = fmt . Sprintf ( "%s\nIf Kubernetes Version is <1.24:\n%s\n\nIf Kubernetes Version is >=1.24:\n%s\n" , tokenGenerationTip , createSvcAccountToken , getSvcAccountToken )
} else {
if parsedClusterVersion . GTE ( semver . Version { Major : 1 , Minor : 24 } ) {
2023-12-20 19:55:54 +00:00
tokenGenerationTip = fmt . Sprintf ( "%s\n\n %s" , tokenGenerationTip , createSvcAccountToken )
2022-11-03 23:38:59 +00:00
} else {
2023-12-20 19:55:54 +00:00
tokenGenerationTip = fmt . Sprintf ( "%s\n\n %s" , tokenGenerationTip , getSvcAccountToken )
2022-11-03 23:38:59 +00:00
}
}
out . Styled ( style . Tip , fmt . Sprintf ( "%s\n" , tokenGenerationTip ) )
out . Styled ( style . Tip , ` Headlamp can display more detailed information when metrics - server is installed . To install it , run :
2023-12-20 19:55:54 +00:00
minikube { { . profileArg } } addons enable metrics - server
2022-11-03 23:38:59 +00:00
` , out . V { "profileArg" : tipProfileArg } )
2023-12-12 18:30:09 +00:00
case "yakd" :
out . Styled ( style . Tip , ` To access YAKD - Kubernetes Dashboard , wait for Pod to be ready and run the following command :
2023-12-13 14:53:34 +00:00
minikube { { . profileArg } } service yakd - dashboard - n yakd - dashboard
` , out . V { "profileArg" : tipProfileArg } )
2022-11-03 23:38:59 +00:00
}
}
// Deprecations if the selected addon is deprecated return the replacement addon, otherwise return the passed in addon
func Deprecations ( name string ) ( bool , string , string ) {
switch name {
case "heapster" :
return true , "metrics-server" , "using metrics-server addon, heapster is deprecated"
case "efk" :
return true , "" , "The current images used in the efk addon contain Log4j vulnerabilities, the addon will be disabled until images are updated, see: https://github.com/kubernetes/minikube/issues/15280"
}
return false , "" , ""
}
2020-04-09 04:23:35 +00:00
// Set sets a value in the config (not threadsafe)
func Set ( cc * config . ClusterConfig , name string , value string ) error {
a , valid := isAddonValid ( name )
if ! valid {
return errors . Errorf ( "%s is not a valid addon" , name )
}
return a . set ( cc , name , value )
}
// SetAndSave sets a value and saves the config
func SetAndSave ( profile string , name string , value string ) error {
cc , err := config . Load ( profile )
if err != nil {
return errors . Wrap ( err , "loading profile" )
}
if err := RunCallbacks ( cc , name , value ) ; err != nil {
2021-08-13 20:42:02 +00:00
if errors . Is ( err , ErrSkipThisAddon ) {
return err
}
2020-04-09 04:23:35 +00:00
return errors . Wrap ( err , "run callbacks" )
}
if err := Set ( cc , name , value ) ; err != nil {
return errors . Wrap ( err , "set" )
}
2019-12-19 05:35:46 +00:00
2020-09-29 22:49:41 +00:00
klog . Infof ( "Writing out %q config to set %s=%v..." , profile , name , value )
2020-03-11 00:49:47 +00:00
return config . Write ( profile , cc )
2019-12-19 05:35:46 +00:00
}
// Runs all the validation or callback functions and collects errors
2020-03-11 00:49:47 +00:00
func run ( cc * config . ClusterConfig , name string , value string , fns [ ] setFn ) error {
2021-08-13 20:42:02 +00:00
var errs [ ] error
2019-12-19 05:35:46 +00:00
for _ , fn := range fns {
2020-03-11 00:49:47 +00:00
err := fn ( cc , name , value )
2019-12-19 05:35:46 +00:00
if err != nil {
2021-08-13 20:42:02 +00:00
if errors . Is ( err , ErrSkipThisAddon ) {
return ErrSkipThisAddon
}
errs = append ( errs , err )
2019-12-19 05:35:46 +00:00
}
}
2021-08-13 20:42:02 +00:00
if len ( errs ) > 0 {
return fmt . Errorf ( "%v" , errs )
2019-12-19 05:35:46 +00:00
}
return nil
}
2020-04-09 04:23:35 +00:00
// SetBool sets a bool value in the config (not threadsafe)
2020-03-11 00:49:47 +00:00
func SetBool ( cc * config . ClusterConfig , name string , val string ) error {
2019-12-18 19:31:29 +00:00
b , err := strconv . ParseBool ( val )
if err != nil {
return err
}
2020-03-11 00:49:47 +00:00
if cc . Addons == nil {
cc . Addons = map [ string ] bool { }
2019-12-19 05:35:46 +00:00
}
2020-03-11 00:49:47 +00:00
cc . Addons [ name ] = b
2019-12-18 19:31:29 +00:00
return nil
}
2021-02-07 14:43:05 +00:00
// EnableOrDisableAddon updates addon status executing any commands necessary
func EnableOrDisableAddon ( cc * config . ClusterConfig , name string , val string ) error {
2020-09-29 22:49:41 +00:00
klog . Infof ( "Setting addon %s=%s in %q" , name , val , cc . Name )
2019-12-18 19:31:29 +00:00
enable , err := strconv . ParseBool ( val )
if err != nil {
return errors . Wrapf ( err , "parsing bool: %s" , name )
}
addon := assets . Addons [ name ]
// check addon status before enabling/disabling it
2020-04-09 04:23:35 +00:00
if isAddonAlreadySet ( cc , addon , enable ) {
2020-09-29 22:49:41 +00:00
klog . Warningf ( "addon %s should already be in state %v" , name , val )
2020-02-27 10:21:08 +00:00
if ! enable {
return nil
}
2019-12-18 19:31:29 +00:00
}
api , err := machine . NewAPIClient ( )
if err != nil {
return errors . Wrap ( err , "machine client" )
}
defer api . Close ( )
2024-01-07 21:36:17 +00:00
cp , err := config . ControlPlane ( * cc )
2020-03-11 00:49:47 +00:00
if err != nil {
2024-01-07 21:36:17 +00:00
exit . Error ( reason . GuestCpConfig , "Error getting control-plane node" , err )
2019-12-18 19:31:29 +00:00
}
2021-10-28 02:50:06 +00:00
// maintain backwards compatibility for ingress and ingress-dns addons with k8s < v1.19
2021-10-27 01:36:41 +00:00
if strings . HasPrefix ( name , "ingress" ) && enable {
if err := supportLegacyIngress ( addon , * cc ) ; err != nil {
return err
}
}
2021-05-14 22:29:36 +00:00
// Persist images even if the machine is running so starting gets the correct images.
images , customRegistries , err := assets . SelectAndPersistImages ( addon , cc )
if err != nil {
exit . Error ( reason . HostSaveProfile , "Failed to persist images" , err )
}
2022-03-14 10:42:46 +00:00
if cc . KubernetesConfig . ImageRepository == constants . AliyunMirror {
2021-09-26 07:36:36 +00:00
images , customRegistries = assets . FixAddonImagesAndRegistries ( addon , images , customRegistries )
}
2021-01-08 19:26:10 +00:00
mName := config . MachineName ( * cc , cp )
2020-03-13 22:51:03 +00:00
host , err := machine . LoadHost ( api , mName )
if err != nil || ! machine . IsRunning ( api , mName ) {
2020-09-29 22:49:41 +00:00
klog . Warningf ( "%q is not running, setting %s=%v and skipping enablement (err=%v)" , mName , addon . Name ( ) , enable , err )
2020-01-31 20:19:25 +00:00
return nil
2019-12-18 19:31:29 +00:00
}
2021-02-22 06:41:38 +00:00
runner , err := machine . CommandRunner ( host )
2019-12-18 19:31:29 +00:00
if err != nil {
return errors . Wrap ( err , "command runner" )
}
2021-08-12 21:13:51 +00:00
bail , err := addonSpecificChecks ( cc , name , enable , runner )
if err != nil {
return err
2021-02-22 06:41:38 +00:00
}
2021-08-12 21:13:51 +00:00
if bail {
return nil
2021-08-11 16:58:03 +00:00
}
2021-03-17 18:52:55 +00:00
var networkInfo assets . NetworkInfo
if len ( cc . Nodes ) >= 1 {
2021-03-23 07:03:42 +00:00
networkInfo . ControlPlaneNodeIP = cc . Nodes [ 0 ] . IP
2021-05-18 01:30:18 +00:00
networkInfo . ControlPlaneNodePort = cc . Nodes [ 0 ] . Port
2021-03-17 18:52:55 +00:00
} else {
out . WarningT ( "At least needs control plane nodes to enable addon" )
}
2022-08-01 16:23:08 +00:00
data := assets . GenerateTemplateData ( addon , cc , networkInfo , images , customRegistries , enable )
2021-02-22 06:41:38 +00:00
return enableOrDisableAddonInternal ( cc , addon , runner , data , enable )
2019-12-18 19:31:29 +00:00
}
2021-08-12 21:13:51 +00:00
func addonSpecificChecks ( cc * config . ClusterConfig , name string , enable bool , runner command . Runner ) ( bool , error ) {
2021-08-10 22:54:12 +00:00
// to match both ingress and ingress-dns addons
if strings . HasPrefix ( name , "ingress" ) && enable {
if driver . IsKIC ( cc . Driver ) {
2021-08-23 18:11:16 +00:00
if runtime . GOOS == "windows" || runtime . GOOS == "darwin" {
2021-08-10 22:54:12 +00:00
out . Styled ( style . Tip , ` After the addon is enabled, please run "minikube tunnel" and your ingress resources would be available at "127.0.0.1" ` )
}
}
}
if strings . HasPrefix ( name , "istio" ) && enable {
minMem := 8192
minCPUs := 4
if cc . Memory < minMem {
out . WarningT ( "Istio needs {{.minMem}}MB of memory -- your configuration only allocates {{.memory}}MB" , out . V { "minMem" : minMem , "memory" : cc . Memory } )
}
if cc . CPUs < minCPUs {
out . WarningT ( "Istio needs {{.minCPUs}} CPUs -- your configuration only allocates {{.cpus}} CPUs" , out . V { "minCPUs" : minCPUs , "cpus" : cc . CPUs } )
}
}
if name == "registry" {
2022-09-26 20:58:48 +00:00
if driver . NeedsPortForward ( cc . Driver ) {
2021-08-10 22:54:12 +00:00
port , err := oci . ForwardedPort ( cc . Driver , cc . Name , constants . RegistryAddonPort )
if err != nil {
2021-08-12 21:13:51 +00:00
return false , errors . Wrap ( err , "registry port" )
2021-08-10 22:54:12 +00:00
}
if enable {
out . Boxed ( ` Registry addon with {{ .driver }} driver uses port {{ .port }} please use that instead of default port 5000 ` , out . V { "driver" : cc . Driver , "port" : port } )
}
out . Styled ( style . Documentation , ` For more information see: https://minikube.sigs.k8s.io/docs/drivers/ {{ .driver }} ` , out . V { "driver" : cc . Driver } )
}
2021-08-12 21:13:51 +00:00
return false , nil
2021-08-10 22:54:12 +00:00
}
2021-08-12 21:13:51 +00:00
if name == "auto-pause" && ! enable { // needs to be disabled before deleting the service file in the internal disable
if err := sysinit . New ( runner ) . DisableNow ( "auto-pause" ) ; err != nil {
klog . ErrorS ( err , "failed to disable" , "service" , "auto-pause" )
}
return false , nil
}
// If the gcp-auth credentials haven't been mounted in, don't start the pods
2021-10-25 22:32:13 +00:00
if name == "gcp-auth" && enable {
2021-08-12 21:13:51 +00:00
rr , err := runner . RunCmd ( exec . Command ( "cat" , credentialsPath ) )
if err != nil || rr . Stdout . String ( ) == "" {
return true , nil
}
}
2024-04-20 15:52:03 +00:00
// we cannot use volcano for crio
if name == "volcano" && cc . KubernetesConfig . ContainerRuntime == constants . CRIO && enable {
return false , fmt . Errorf ( "volcano addon does not support crio" )
}
2021-08-12 21:13:51 +00:00
return false , nil
2021-08-10 22:54:12 +00:00
}
2020-04-09 04:23:35 +00:00
func isAddonAlreadySet ( cc * config . ClusterConfig , addon * assets . Addon , enable bool ) bool {
enabled := addon . IsEnabled ( cc )
if enabled && enable {
return true
2019-12-18 19:31:29 +00:00
}
2020-04-09 04:23:35 +00:00
if ! enabled && ! enable {
return true
2019-12-18 19:31:29 +00:00
}
2020-04-09 04:23:35 +00:00
return false
2019-12-18 19:31:29 +00:00
}
2021-10-28 02:50:06 +00:00
// maintain backwards compatibility for ingress and ingress-dns addons with k8s < v1.19 by replacing default addons' images with compatible versions
2021-10-27 01:36:41 +00:00
func supportLegacyIngress ( addon * assets . Addon , cc config . ClusterConfig ) error {
2021-08-22 02:58:34 +00:00
v , err := util . ParseKubernetesVersion ( cc . KubernetesConfig . KubernetesVersion )
if err != nil {
return errors . Wrap ( err , "parsing Kubernetes version" )
}
if semver . MustParseRange ( "<1.19.0" ) ( v ) {
2021-10-28 02:50:06 +00:00
if addon . Name ( ) == "ingress" {
addon . Images = map [ string ] string {
// https://github.com/kubernetes/ingress-nginx/blob/0a2ec01eb4ec0e1b29c4b96eb838a2e7bfe0e9f6/deploy/static/provider/kind/deploy.yaml#L328
"IngressController" : "ingress-nginx/controller:v0.49.3@sha256:35fe394c82164efa8f47f3ed0be981b3f23da77175bbb8268a9ae438851c8324" ,
// issues: https://github.com/kubernetes/ingress-nginx/issues/7418 and https://github.com/jet/kube-webhook-certgen/issues/30
"KubeWebhookCertgenCreate" : "docker.io/jettech/kube-webhook-certgen:v1.5.1@sha256:950833e19ade18cd389d647efb88992a7cc077abedef343fa59e012d376d79b7" ,
"KubeWebhookCertgenPatch" : "docker.io/jettech/kube-webhook-certgen:v1.5.1@sha256:950833e19ade18cd389d647efb88992a7cc077abedef343fa59e012d376d79b7" ,
}
addon . Registries = map [ string ] string {
2023-03-30 16:45:52 +00:00
"IngressController" : "registry.k8s.io" ,
2021-10-28 02:50:06 +00:00
}
return nil
2021-08-22 02:58:34 +00:00
}
2021-10-28 02:50:06 +00:00
if addon . Name ( ) == "ingress-dns" {
addon . Images = map [ string ] string {
"IngressDNS" : "cryptexlabs/minikube-ingress-dns:0.3.0@sha256:e252d2a4c704027342b303cc563e95d2e71d2a0f1404f55d676390e28d5093ab" ,
}
addon . Registries = nil
return nil
}
return fmt . Errorf ( "supportLegacyIngress called for unexpected addon %q - nothing to do here" , addon . Name ( ) )
2021-08-22 02:58:34 +00:00
}
2021-10-28 02:50:06 +00:00
2021-08-22 02:58:34 +00:00
return nil
}
2021-02-22 06:41:38 +00:00
func enableOrDisableAddonInternal ( cc * config . ClusterConfig , addon * assets . Addon , runner command . Runner , data interface { } , enable bool ) error {
2020-02-05 22:41:05 +00:00
deployFiles := [ ] string { }
2020-01-06 21:42:09 +00:00
for _ , addon := range addon . Assets {
2020-01-30 23:54:04 +00:00
var f assets . CopyableFile
2020-01-22 22:18:20 +00:00
var err error
2020-01-06 21:42:09 +00:00
if addon . IsTemplate ( ) {
2020-01-30 23:54:04 +00:00
f , err = addon . Evaluate ( data )
2020-01-06 21:42:09 +00:00
if err != nil {
2020-04-09 20:24:14 +00:00
return errors . Wrapf ( err , "evaluate bundled addon %s asset" , addon . GetSourcePath ( ) )
2019-12-18 19:31:29 +00:00
}
2020-01-06 21:42:09 +00:00
} else {
2020-01-30 23:54:04 +00:00
f = addon
2019-12-18 19:31:29 +00:00
}
2020-01-30 23:54:04 +00:00
fPath := path . Join ( f . GetTargetDir ( ) , f . GetTargetName ( ) )
2020-01-22 22:18:20 +00:00
if enable {
2020-09-29 22:49:41 +00:00
klog . Infof ( "installing %s" , fPath )
2021-02-22 06:41:38 +00:00
if err := runner . Copy ( f ) ; err != nil {
2020-01-22 22:18:20 +00:00
return err
}
} else {
2020-09-29 22:49:41 +00:00
klog . Infof ( "Removing %+v" , fPath )
2020-01-22 23:11:33 +00:00
defer func ( ) {
2021-02-22 06:41:38 +00:00
if err := runner . Remove ( f ) ; err != nil {
2020-09-29 22:49:41 +00:00
klog . Warningf ( "error removing %s; addon should still be disabled as expected" , fPath )
2020-01-22 23:11:33 +00:00
}
} ( )
2019-12-18 19:31:29 +00:00
}
2020-02-05 22:41:05 +00:00
if strings . HasSuffix ( fPath , ".yaml" ) {
deployFiles = append ( deployFiles , fPath )
}
2019-12-18 19:31:29 +00:00
}
2020-02-05 22:41:05 +00:00
2023-03-06 22:15:28 +00:00
// on the first attempt try without force, but on subsequent attempts use force
2023-03-01 21:07:46 +00:00
force := false
2020-03-25 20:44:28 +00:00
// Retry, because sometimes we race against an apiserver restart
apply := func ( ) error {
2023-04-27 05:01:25 +00:00
ctx , cancel := context . WithTimeout ( context . Background ( ) , 20 * time . Second )
2023-03-01 21:07:46 +00:00
defer cancel ( )
_ , err := runner . RunCmd ( kubectlCommand ( ctx , cc , deployFiles , enable , force ) )
2020-03-25 20:44:28 +00:00
if err != nil {
2020-09-29 22:49:41 +00:00
klog . Warningf ( "apply failed, will retry: %v" , err )
2023-03-01 21:07:46 +00:00
force = true
2020-03-25 20:44:28 +00:00
}
return err
2020-01-22 22:18:20 +00:00
}
2020-03-25 20:44:28 +00:00
2020-05-04 19:44:30 +00:00
return retry . Expo ( apply , 250 * time . Millisecond , 2 * time . Minute )
2019-12-18 19:31:29 +00:00
}
2020-06-26 22:08:38 +00:00
func verifyAddonStatus ( cc * config . ClusterConfig , name string , val string ) error {
2021-03-19 19:07:21 +00:00
ns := "kube-system"
if name == "ingress" {
ns = "ingress-nginx"
}
return verifyAddonStatusInternal ( cc , name , val , ns )
2020-07-21 21:25:58 +00:00
}
func verifyAddonStatusInternal ( cc * config . ClusterConfig , name string , val string , ns string ) error {
2020-09-29 22:49:41 +00:00
klog . Infof ( "Verifying addon %s=%s in %q" , name , val , cc . Name )
2020-06-25 17:40:32 +00:00
enable , err := strconv . ParseBool ( val )
if err != nil {
return errors . Wrapf ( err , "parsing bool: %s" , name )
}
2020-06-26 21:22:17 +00:00
2020-06-26 22:08:38 +00:00
label , ok := addonPodLabels [ name ]
if ok && enable {
2021-02-25 23:32:03 +00:00
out . Step ( style . HealthCheck , "Verifying {{.addon_name}} addon..." , out . V { "addon_name" : name } )
2020-06-26 22:08:38 +00:00
client , err := kapi . Client ( viper . GetString ( config . ProfileName ) )
2020-06-25 22:43:44 +00:00
if err != nil {
2020-06-26 22:08:38 +00:00
return errors . Wrapf ( err , "get kube-client to validate %s addon: %v" , name , err )
2020-06-25 17:40:32 +00:00
}
2020-06-26 21:22:17 +00:00
2020-10-21 16:44:39 +00:00
// This timeout includes image pull time, which can take a few minutes. 3 is not enough.
2020-10-21 16:54:01 +00:00
err = kapi . WaitForPods ( client , ns , label , time . Minute * 6 )
2020-06-26 21:22:17 +00:00
if err != nil {
2020-10-21 16:44:39 +00:00
return errors . Wrapf ( err , "waiting for %s pods" , label )
2020-06-26 21:22:17 +00:00
}
2020-06-25 17:40:32 +00:00
}
return nil
}
2023-01-25 16:05:35 +00:00
// Enable tries to enable the default addons for a profile plus any additional, and returns a single slice of all successfully enabled addons via channel (thread-safe).
// Since Enable is called asynchronously (so is not thread-safe for concurrent addons map updating/reading), to avoid race conditions,
// ToEnable should be called synchronously before Enable to get complete list of addons to enable, and
// UpdateConfig should be called synchronously after Enable to update the config with successfully enabled addons.
func Enable ( wg * sync . WaitGroup , cc * config . ClusterConfig , toEnable map [ string ] bool , enabled chan <- [ ] string ) {
2020-04-08 22:17:14 +00:00
defer wg . Done ( )
2020-04-08 19:29:23 +00:00
2020-01-31 01:46:25 +00:00
start := time . Now ( )
2023-01-25 16:05:35 +00:00
klog . Infof ( "enable addons start: toEnable=%v" , toEnable )
var enabledAddons [ ] string
2020-01-31 01:46:25 +00:00
defer func ( ) {
2024-01-07 21:36:17 +00:00
klog . Infof ( "duration metric: took %s for enable addons: enabled=%v" , time . Since ( start ) , enabledAddons )
2020-01-31 01:46:25 +00:00
} ( )
2020-02-03 18:49:40 +00:00
toEnableList := [ ] string { }
for k , v := range toEnable {
if v {
toEnableList = append ( toEnableList , k )
}
2020-01-31 01:46:25 +00:00
}
2020-02-03 18:49:40 +00:00
sort . Strings ( toEnableList )
2020-01-31 01:46:25 +00:00
2020-04-09 04:23:35 +00:00
var awg sync . WaitGroup
2020-09-29 20:29:14 +00:00
defer func ( ) { // making it show after verifications (see #7613)
2020-07-07 18:25:27 +00:00
register . Reg . SetStep ( register . EnablingAddons )
2020-11-12 12:45:33 +00:00
out . Step ( style . AddonEnable , "Enabled addons: {{.addons}}" , out . V { "addons" : strings . Join ( enabledAddons , ", " ) } )
2020-04-16 20:38:17 +00:00
} ( )
2020-02-03 18:49:40 +00:00
for _ , a := range toEnableList {
2020-04-09 04:23:35 +00:00
awg . Add ( 1 )
2020-04-08 22:20:30 +00:00
go func ( name string ) {
2020-04-09 04:23:35 +00:00
err := RunCallbacks ( cc , name , "true" )
2021-08-31 19:21:09 +00:00
if err != nil && ! errors . Is ( err , ErrSkipThisAddon ) {
2020-04-08 22:17:14 +00:00
out . WarningT ( "Enabling '{{.name}}' returned an error: {{.error}}" , out . V { "name" : name , "error" : err } )
2020-09-29 20:29:14 +00:00
} else {
enabledAddons = append ( enabledAddons , name )
2020-04-08 19:29:23 +00:00
}
2020-04-09 04:23:35 +00:00
awg . Done ( )
2020-04-08 22:20:30 +00:00
} ( a )
2020-01-31 01:46:25 +00:00
}
2020-04-09 04:23:35 +00:00
2023-01-25 16:05:35 +00:00
// Wait until all of the addons are enabled
2020-04-09 04:23:35 +00:00
awg . Wait ( )
2020-09-29 20:29:14 +00:00
2023-01-25 16:05:35 +00:00
// send the slice of all successfully enabled addons to channel and close
enabled <- enabledAddons
close ( enabled )
}
// ToEnable returns the final list of addons to enable (not thread-safe).
func ToEnable ( cc * config . ClusterConfig , existing map [ string ] bool , additional [ ] string ) map [ string ] bool {
// start from existing
enable := map [ string ] bool { }
for k , v := range existing {
enable [ k ] = v
}
// Get the default values of any addons not saved to our config
for name , a := range assets . Addons {
if _ , exists := existing [ name ] ; ! exists {
2023-02-17 16:00:22 +00:00
enable [ name ] = a . IsEnabledOrDefault ( cc )
2023-01-25 16:05:35 +00:00
}
}
// Apply new addons
for _ , name := range additional {
isDeprecated , replacement , msg := Deprecations ( name )
if isDeprecated && replacement == "" {
out . FailureT ( msg )
continue
} else if isDeprecated {
out . Styled ( style . Waiting , msg )
name = replacement
}
// if the specified addon doesn't exist, skip enabling
if _ , e := isAddonValid ( name ) ; e {
enable [ name ] = true
}
}
return enable
}
// UpdateConfig tries to update config with all enabled addons (not thread-safe).
// Any error will be logged and it will continue.
2023-01-31 19:18:48 +00:00
func UpdateConfigToEnable ( cc * config . ClusterConfig , enabled [ ] string ) {
2023-01-25 16:05:35 +00:00
for _ , a := range enabled {
2020-04-09 04:23:35 +00:00
if err := Set ( cc , a , "true" ) ; err != nil {
2020-09-29 22:49:41 +00:00
klog . Errorf ( "store failed: %v" , err )
2020-04-09 04:23:35 +00:00
}
}
2020-01-31 01:46:25 +00:00
}
2023-01-31 19:18:48 +00:00
func UpdateConfigToDisable ( cc * config . ClusterConfig ) {
for name := range assets . Addons {
if err := Set ( cc , name , "false" ) ; err != nil {
klog . Errorf ( "store failed: %v" , err )
}
}
}
2023-02-17 19:31:24 +00:00
2023-03-06 14:35:43 +00:00
// VerifyNotPaused verifies the cluster is not paused before enable/disable an addon.
func VerifyNotPaused ( profile string , enable bool ) error {
2023-02-17 19:31:24 +00:00
klog . Info ( "checking whether the cluster is paused" )
cc , err := config . Load ( profile )
if err != nil {
return errors . Wrap ( err , "loading profile" )
}
api , err := machine . NewAPIClient ( )
if err != nil {
return errors . Wrap ( err , "machine client" )
}
defer api . Close ( )
2024-01-07 21:36:17 +00:00
cp , err := config . ControlPlane ( * cc )
2023-02-17 19:31:24 +00:00
if err != nil {
2024-01-07 21:36:17 +00:00
return errors . Wrap ( err , "get control-plane node" )
2023-02-17 19:31:24 +00:00
}
host , err := machine . LoadHost ( api , config . MachineName ( * cc , cp ) )
if err != nil {
return errors . Wrap ( err , "get host" )
}
2023-07-13 17:09:08 +00:00
s , err := host . Driver . GetState ( )
if err != nil {
return errors . Wrap ( err , "get state" )
}
if s != state . Running {
// can't check the status of pods on a non-running cluster
return nil
}
2023-02-17 19:31:24 +00:00
runner , err := machine . CommandRunner ( host )
if err != nil {
return errors . Wrap ( err , "command runner" )
}
crName := cc . KubernetesConfig . ContainerRuntime
cr , err := cruntime . New ( cruntime . Config { Type : crName , Runner : runner } )
if err != nil {
return errors . Wrap ( err , "container runtime" )
}
runtimePaused , err := cluster . CheckIfPaused ( cr , [ ] string { "kube-system" } )
if err != nil {
return errors . Wrap ( err , "check paused" )
}
2023-07-13 20:08:52 +00:00
if ! runtimePaused {
return nil
2023-02-17 19:31:24 +00:00
}
2023-07-13 20:08:52 +00:00
action := "disable"
if enable {
action = "enable"
}
msg := fmt . Sprintf ( "Can't %s addon on a paused cluster, please unpause the cluster first." , action )
out . Styled ( style . Shrug , msg )
return errors . New ( msg )
2023-02-17 19:31:24 +00:00
}