fix: Properly shutdown http server on Close() (#20171)

services/httpd/handler.go

@@ -316,6 +316,10 @@ func (h *Handler) Open() {
 }
 
 func (h *Handler) Close() {
+
+	// lets gracefully shut down http connections.  we'll give them 10 seconds
+	// before we shut them down "with extreme predjudice".
+
 	if h.accessLog != nil {
 		h.accessLog.Close()
 		h.accessLog = nil

services/httpd/service.go

@@ -2,6 +2,7 @@
 package httpd // import "github.com/influxdata/influxdb/services/httpd"
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
 	"net"
@@ -10,6 +11,7 @@ import (
 	"path"
 	"runtime"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 
@@ -57,6 +59,9 @@ type Service struct {
 	tlsConfig *tls.Config
 	err       chan error
 
+	httpServerMutex sync.Mutex
+	httpServer      []*http.Server
+
 	unixSocket         bool
 	unixSocketPerm     uint32
 	unixSocketGroup    int
@@ -192,6 +197,23 @@ func (s *Service) Open() error {
 func (s *Service) Close() error {
 	s.Handler.Close()
 
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	s.httpServerMutex.Lock()
+	errCh := make(chan error, len(s.httpServer))
+	// shut down all running http servers
+	for _, server := range s.httpServer {
+		server := server
+		go func() { errCh <- server.Shutdown(ctx) }()
+	}
+
+	// wait for shutdowns to complete
+	for i := 0; i < len(s.httpServer); i++ {
+		<-errCh
+	}
+	s.httpServerMutex.Unlock()
+
 	if s.ln != nil {
 		if err := s.ln.Close(); err != nil {
 			return err
@@ -245,10 +267,16 @@ func (s *Service) serveUnixSocket() {
 
 // serve serves the handler from the listener.
 func (s *Service) serve(listener net.Listener) {
+	svr := &http.Server{
+		Handler: s.Handler,
+	}
+	s.httpServerMutex.Lock()
+	s.httpServer = append(s.httpServer, svr)
+	s.httpServerMutex.Unlock()
+
 	// The listener was closed so exit
 	// See https://github.com/golang/go/issues/4373
-	err := http.Serve(listener, s.Handler)
-	if err != nil && !strings.Contains(err.Error(), "closed") {
+	if err := svr.Serve(listener); err != nil && !strings.Contains(err.Error(), "closed") {
 		s.err <- fmt.Errorf("listener failed: addr=%s, err=%s", s.Addr(), err)
 	}
 }