fix: cp.Mux.Serve() closes all net.Listener instances silently on error (#20295)

* fix: cp.Mux.Serve() closes all net.Listener instances silently on error. (#20278)

A customer has seen a rash of "connection refused" errors to the meta node.
This fix ensures that when net.Listener instances are closed because of an
error in Accept(), influxdb logs the error which caused the closures, as well
as any errors in closing the Listeners.

Fixes https://github.com/influxdata/influxdb/issues/20256

(cherry picked from commit 5b98166b05)+

CHANGELOG.md

@@ -11,6 +11,7 @@ v1.8.4 [unreleased]
 -	[#19696](https://github.com/influxdata/influxdb/pull/19697): fix(flux): add durations to Flux logging.
 -	[#20276](https://github.com/influxdata/influxdb/pull/20276): fix(error): unsupported value: +Inf" error not handled gracefully.
 -	[#20277](https://github.com/influxdata/influxdb/pull/20277): fix(query): Group By queries with offset that crosses a DST boundary can fail.
+-	[#202955](https://github.com/influxdata/influxdb/pull/20295): fix: cp.Mux.Serve() closes all net.Listener instances silently on error.
 
 v1.8.3 [2020-09-30]
 -------------------

tcp/mux.go

@@ -79,6 +79,7 @@ func (mux *Mux) Serve(ln net.Listener) error {
 			continue
 		}
 		if err != nil {
+			mux.Logger.Printf("tcp.Mux: Listener at %s failed failed to accept a connection, closing all listeners - %s", ln.Addr(), err)
 			// Wait for all connections to be demux
 			mux.wg.Wait()
 
@@ -90,7 +91,9 @@ func (mux *Mux) Serve(ln net.Listener) error {
 				wg.Add(1)
 				go func(ln *listener) {
 					defer wg.Done()
-					ln.Close()
+					if err := ln.Close(); err != nil {
+						mux.Logger.Printf("tcp.Mux: Closing the listener at %s failed - %s", ln.Addr().String(), err)
+					}
 				}(ln)
 			}
 			mux.mu.RUnlock()
@@ -100,9 +103,10 @@ func (mux *Mux) Serve(ln net.Listener) error {
 			dl := mux.defaultListener
 			mux.mu.RUnlock()
 			if dl != nil {
-				dl.Close()
+				if closeErr := dl.Close(); closeErr != nil {
+					mux.Logger.Printf("tcp.Mux: Closing the default listener at %s failed - %s", ln.Addr().String(), closeErr)
+				}
 			}
-
 			return err
 		}