From 7364fe8cc6636a0a28334451c56e7d49110e302a Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Wed, 8 Sep 2021 10:56:18 -0700 Subject: [PATCH] Replace klog with non-exiting fork Signed-off-by: Brad Davidson (cherry picked from commit 29c8b238e5d669c04b44753183520d97ccd7a2d8) --- cmd/agent/main.go | 5 +++-- cmd/etcdsnapshot/main.go | 4 +++- cmd/k3s/main.go | 5 +++-- cmd/server/main.go | 5 +++-- go.mod | 2 ++ go.sum | 14 ++++---------- main.go | 4 +++- pkg/agent/flannel/setup.go | 4 +++- pkg/cluster/https.go | 5 ++++- pkg/daemons/executor/embed.go | 30 ++++++++++++++++++++++++++++++ pkg/server/server.go | 4 +++- vendor/k8s.io/klog/klog.go | 25 +------------------------ vendor/k8s.io/klog/klog_file.go | 31 +++++++++++++++++++++++++------ vendor/k8s.io/klog/v2/klog.go | 23 +---------------------- vendor/modules.txt | 6 ++++-- 15 files changed, 92 insertions(+), 75 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index dfb75dfe30..268995d32c 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -1,6 +1,8 @@ package main import ( + "context" + "errors" "os" "github.com/rancher/k3s/pkg/cli/agent" @@ -16,8 +18,7 @@ func main() { cmds.NewAgentCommand(agent.Run), } - err := app.Run(configfilearg.MustParse(os.Args)) - if err != nil { + if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) { logrus.Fatal(err) } } diff --git a/cmd/etcdsnapshot/main.go b/cmd/etcdsnapshot/main.go index 3d55ab6181..0ad08000f6 100644 --- a/cmd/etcdsnapshot/main.go +++ b/cmd/etcdsnapshot/main.go @@ -1,6 +1,8 @@ package main import ( + "context" + "errors" "os" "github.com/rancher/k3s/pkg/cli/cmds" @@ -22,7 +24,7 @@ func main() { ), } - if err := app.Run(configfilearg.MustParse(os.Args)); err != nil { + if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) { logrus.Fatal(err) } } diff --git a/cmd/k3s/main.go b/cmd/k3s/main.go index 036cea3da9..4d7014985e 100644 --- a/cmd/k3s/main.go +++ b/cmd/k3s/main.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "os" "os/exec" "path/filepath" @@ -53,7 +54,7 @@ func main() { ), } - if err := app.Run(os.Args); err != nil { + if err := app.Run(os.Args); err != nil && !errors.Is(err, context.Canceled) { logrus.Fatal(err) } } @@ -90,7 +91,7 @@ func runCLIs(dataDir string) bool { progName := filepath.Base(os.Args[0]) switch progName { case "crictl", "ctr", "kubectl": - if err := externalCLI(progName, dataDir, os.Args[1:]); err != nil { + if err := externalCLI(progName, dataDir, os.Args[1:]); err != nil && !errors.Is(err, context.Canceled) { logrus.Fatal(err) } return true diff --git a/cmd/server/main.go b/cmd/server/main.go index bd01b22be2..9cc8fdbc88 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,6 +1,8 @@ package main import ( + "context" + "errors" "os" "path/filepath" @@ -52,8 +54,7 @@ func main() { ), } - err := app.Run(configfilearg.MustParse(os.Args)) - if err != nil { + if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) { logrus.Fatal(err) } } diff --git a/go.mod b/go.mod index ffff97eb02..cd924f1325 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,8 @@ replace ( k8s.io/controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.21.6-k3s1 k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.21.6-k3s1 k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.21.6-k3s1 + k8s.io/klog => github.com/k3s-io/klog v1.0.0-k3s1 // k3s-release-1.x + k8s.io/klog/v2 => github.com/k3s-io/klog/v2 v2.9.0-k3s1 // k3s-main k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.21.6-k3s1 k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.21.6-k3s1 k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.21.6-k3s1 diff --git a/go.sum b/go.sum index f74a463a5d..59586968e9 100644 --- a/go.sum +++ b/go.sum @@ -312,7 +312,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= @@ -553,6 +552,10 @@ github.com/k3s-io/helm-controller v0.11.7 h1:fNpBImB3h5aHvPf3zwU9sFWmeVQh0nTG1sL github.com/k3s-io/helm-controller v0.11.7/go.mod h1:z0ExsRRIkTO/QC//3/Esn5ItTD6AiQSluwzMaS7RI/4= github.com/k3s-io/kine v0.6.2 h1:1aJTPfB8HG4exqMKFVE5H0z4bepF05tJHtYNXotWXa4= github.com/k3s-io/kine v0.6.2/go.mod h1:rzCs93+rQHZGOiewMd84PDrER92QeZ6eeHbWkfEy4+w= +github.com/k3s-io/klog v1.0.0-k3s1 h1:Bg+gRta3s4sfbaYUSWbHcMEyVdxdaU1cJCRtWcaxjBE= +github.com/k3s-io/klog v1.0.0-k3s1/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +github.com/k3s-io/klog/v2 v2.9.0-k3s1 h1:q4DqcZgBG+D2TSTGx6JcP1cuXeoxSoixSLE3casDcSw= +github.com/k3s-io/klog/v2 v2.9.0-k3s1/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= github.com/k3s-io/kubernetes v1.21.6-k3s1 h1:HsiQ/OtXP0H72QkQJu5AqL6lohRDzx4VuEVtkSErvRs= github.com/k3s-io/kubernetes v1.21.6-k3s1/go.mod h1:fhjjdD6kyZTCtggL22q32PwAzTyE5kv7CbzSYa5r/cQ= github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.21.6-k3s1 h1:GVn1gZtWdB7psjWKG8Tcq4UiI3YInZd6S/AyT8gYOVk= @@ -1249,15 +1252,6 @@ k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027 h1:Uusb3oh8XcdzDF/ndlI4ToKTYVlkC k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/heapster v1.2.0-beta.1 h1:lUsE/AHOMHpi3MLlBEkaU8Esxm5QhdyCrv1o7ot0s84= k8s.io/heapster v1.2.0-beta.1/go.mod h1:h1uhptVXMwC8xtZBYsPXKVi8fpdlYkTs6k949KozGrM= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= -k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/system-validators v1.4.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q= diff --git a/main.go b/main.go index 8f9de71b21..d7afe195fe 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,8 @@ package main import ( + "context" + "errors" "os" "github.com/rancher/k3s/pkg/cli/agent" @@ -36,7 +38,7 @@ func main() { ), } - if err := app.Run(configfilearg.MustParse(os.Args)); err != nil { + if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) { logrus.Fatal(err) } } diff --git a/pkg/agent/flannel/setup.go b/pkg/agent/flannel/setup.go index 14223f75e8..495d0aac91 100644 --- a/pkg/agent/flannel/setup.go +++ b/pkg/agent/flannel/setup.go @@ -86,7 +86,9 @@ func Run(ctx context.Context, nodeConfig *config.Node, nodes typedcorev1.NodeInt go func() { err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigKubelet) - logrus.Fatalf("flannel exited: %v", err) + if err != nil && !errors.Is(err, context.Canceled) { + logrus.Fatalf("flannel exited: %v", err) + } }() return nil diff --git a/pkg/cluster/https.go b/pkg/cluster/https.go index fff6746386..c14588da73 100644 --- a/pkg/cluster/https.go +++ b/pkg/cluster/https.go @@ -3,6 +3,7 @@ package cluster import ( "context" "crypto/tls" + "errors" "io/ioutil" "log" "net" @@ -91,7 +92,9 @@ func (c *Cluster) initClusterAndHTTPS(ctx context.Context) error { // Start the supervisor http server on the tls listener go func() { err := server.Serve(listener) - logrus.Fatalf("server stopped: %v", err) + if err != nil && !errors.Is(err, http.ErrServerClosed) { + logrus.Fatalf("server stopped: %v", err) + } }() // Shutdown the http server when the context is closed diff --git a/pkg/daemons/executor/embed.go b/pkg/daemons/executor/embed.go index accc8f79ec..047863213d 100644 --- a/pkg/daemons/executor/embed.go +++ b/pkg/daemons/executor/embed.go @@ -55,6 +55,11 @@ func (*Embedded) Kubelet(args []string) error { command.SetArgs(args) go func() { + defer func() { + if err := recover(); err != nil { + logrus.Fatalf("kubelet panic: %v", err) + } + }() logrus.Fatalf("kubelet exited: %v", command.Execute()) }() @@ -66,6 +71,11 @@ func (*Embedded) KubeProxy(args []string) error { command.SetArgs(args) go func() { + defer func() { + if err := recover(); err != nil { + logrus.Fatalf("kube-proxy panic: %v", err) + } + }() logrus.Fatalf("kube-proxy exited: %v", command.Execute()) }() @@ -83,6 +93,11 @@ func (*Embedded) APIServer(ctx context.Context, etcdReady <-chan struct{}, args go func() { <-etcdReady + defer func() { + if err := recover(); err != nil { + logrus.Fatalf("apiserver panic: %v", err) + } + }() logrus.Fatalf("apiserver exited: %v", command.Execute()) }() @@ -107,6 +122,11 @@ func (e *Embedded) Scheduler(apiReady <-chan struct{}, args []string) error { logrus.Fatalf("failed to wait for untained node: %v", err) } } + defer func() { + if err := recover(); err != nil { + logrus.Fatalf("scheduler panic: %v", err) + } + }() logrus.Fatalf("scheduler exited: %v", command.Execute()) }() @@ -119,6 +139,11 @@ func (*Embedded) ControllerManager(apiReady <-chan struct{}, args []string) erro go func() { <-apiReady + defer func() { + if err := recover(); err != nil { + logrus.Fatalf("controller-manager panic: %v", err) + } + }() logrus.Fatalf("controller-manager exited: %v", command.Execute()) }() @@ -157,6 +182,11 @@ func (*Embedded) CloudControllerManager(ccmRBACReady <-chan struct{}, args []str go func() { <-ccmRBACReady + defer func() { + if err := recover(); err != nil { + logrus.Fatalf("cloud-controller-manager panic: %v", err) + } + }() logrus.Fatalf("cloud-controller-manager exited: %v", command.Execute()) }() diff --git a/pkg/server/server.go b/pkg/server/server.go index 31827ec410..8ead793db6 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -177,7 +177,9 @@ func runControllers(ctx context.Context, wg *sync.WaitGroup, config *Config) err go func() { start(ctx) <-ctx.Done() - logrus.Fatal("controllers exited") + if err := ctx.Err(); err != nil && !errors.Is(err, context.Canceled) { + logrus.Fatalf("controllers exited: %v", err) + } }() } else { go leader.RunOrDie(ctx, "", version.Program, sc.K8s, start) diff --git a/vendor/k8s.io/klog/klog.go b/vendor/k8s.io/klog/klog.go index 2712ce0afc..4227983522 100644 --- a/vendor/k8s.io/klog/klog.go +++ b/vendor/k8s.io/klog/klog.go @@ -812,30 +812,7 @@ func (l *loggingT) output(s severity, buf *buffer, file string, line int, alsoTo } } if s == fatalLog { - // If we got here via Exit rather than Fatal, print no stacks. - if atomic.LoadUint32(&fatalNoStacks) > 0 { - l.mu.Unlock() - timeoutFlush(10 * time.Second) - os.Exit(1) - } - // Dump all goroutine stacks before exiting. - // First, make sure we see the trace for the current goroutine on standard error. - // If -logtostderr has been specified, the loop below will do that anyway - // as the first stack in the full dump. - if !l.toStderr { - os.Stderr.Write(stacks(false)) - } - // Write the stack trace for all goroutines to the files. - trace := stacks(true) - logExitFunc = func(error) {} // If we get a write error, we'll still exit below. - for log := fatalLog; log >= infoLog; log-- { - if f := l.file[log]; f != nil { // Can be nil if -logtostderr is set. - f.Write(trace) - } - } - l.mu.Unlock() - timeoutFlush(10 * time.Second) - os.Exit(255) // C++ uses -1, which is silly because it's anded with 255 anyway. + panic(string(data)) } l.putBuffer(buf) l.mu.Unlock() diff --git a/vendor/k8s.io/klog/klog_file.go b/vendor/k8s.io/klog/klog_file.go index e4010ad4df..458456a4a5 100644 --- a/vendor/k8s.io/klog/klog_file.go +++ b/vendor/k8s.io/klog/klog_file.go @@ -24,6 +24,7 @@ import ( "os" "os/user" "path/filepath" + "runtime" "strings" "sync" "time" @@ -55,13 +56,31 @@ func init() { host = shortHostname(h) } - current, err := user.Current() - if err == nil { - userName = current.Username - } + // On Windows, the Go 'user' package requires netapi32.dll. + // This affects Windows Nano Server: + // https://github.com/golang/go/issues/21867 + // Fallback to using environment variables. + if runtime.GOOS == "windows" { + u := os.Getenv("USERNAME") + if len(u) == 0 { + return + } + // Sanitize the USERNAME since it may contain filepath separators. + u = strings.Replace(u, `\`, "_", -1) - // Sanitize userName since it may contain filepath separators on Windows. - userName = strings.Replace(userName, `\`, "_", -1) + // user.Current().Username normally produces something like 'USERDOMAIN\USERNAME' + d := os.Getenv("USERDOMAIN") + if len(d) != 0 { + userName = d + "_" + u + } else { + userName = u + } + } else { + current, err := user.Current() + if err == nil { + userName = current.Username + } + } } // shortHostname returns its argument, truncating at the first period. diff --git a/vendor/k8s.io/klog/v2/klog.go b/vendor/k8s.io/klog/v2/klog.go index 1e187f7635..5d3993e63c 100644 --- a/vendor/k8s.io/klog/v2/klog.go +++ b/vendor/k8s.io/klog/v2/klog.go @@ -965,28 +965,7 @@ func (l *loggingT) output(s severity, log logr.Logger, buf *buffer, depth int, f } } if s == fatalLog { - // If we got here via Exit rather than Fatal, print no stacks. - if atomic.LoadUint32(&fatalNoStacks) > 0 { - l.mu.Unlock() - timeoutFlush(10 * time.Second) - os.Exit(1) - } - // Dump all goroutine stacks before exiting. - trace := stacks(true) - // Write the stack trace for all goroutines to the stderr. - if l.toStderr || l.alsoToStderr || s >= l.stderrThreshold.get() || alsoToStderr { - os.Stderr.Write(trace) - } - // Write the stack trace for all goroutines to the files. - logExitFunc = func(error) {} // If we get a write error, we'll still exit below. - for log := fatalLog; log >= infoLog; log-- { - if f := l.file[log]; f != nil { // Can be nil if -logtostderr is set. - f.Write(trace) - } - } - l.mu.Unlock() - timeoutFlush(10 * time.Second) - os.Exit(255) // C++ uses -1, which is silly because it's anded with 255 anyway. + panic(string(data)) } l.putBuffer(buf) l.mu.Unlock() diff --git a/vendor/modules.txt b/vendor/modules.txt index 4f7e4f4d4a..848e90185d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -2225,10 +2225,10 @@ k8s.io/gengo/parser k8s.io/gengo/types # k8s.io/heapster v1.2.0-beta.1 k8s.io/heapster/metrics/api/v1/types -# k8s.io/klog v1.0.0 +# k8s.io/klog v1.0.0 => github.com/k3s-io/klog v1.0.0-k3s1 ## explicit k8s.io/klog -# k8s.io/klog/v2 v2.9.0 +# k8s.io/klog/v2 v2.9.0 => github.com/k3s-io/klog/v2 v2.9.0-k3s1 ## explicit k8s.io/klog/v2 # k8s.io/kube-aggregator v0.18.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.21.6-k3s1 @@ -3312,6 +3312,8 @@ sigs.k8s.io/yaml # k8s.io/controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.21.6-k3s1 # k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.21.6-k3s1 # k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.21.6-k3s1 +# k8s.io/klog => github.com/k3s-io/klog v1.0.0-k3s1 +# k8s.io/klog/v2 => github.com/k3s-io/klog/v2 v2.9.0-k3s1 # k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.21.6-k3s1 # k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.21.6-k3s1 # k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.21.6-k3s1