Add an access log filter for the access log

The access log filter allows the access log to be filtered by a status code pattern. The pattern is a list of strings of the form `NXX`. At least one number must be specified and up to 2 Xs can be used. That means the filter can be an exact status code or it can be a range of them. For example, `500` would only match the 500 http status code while `5XX` would match any status code beginning with the number 5 (categorized as server errors). The pattern `50X` would also be accepted. Both uppercase and lowercase Xs are allowed. Multiple filters can be specified and the log line will be printed if one of them matches. If there are no filters specified, all status codes are printed.
2018-03-05 09:20:42 -06:00 · 2018-03-05 09:20:42 -06:00 · b3472a54ee
parent 7d9830ab24
commit b3472a54ee
4 changed files with 333 additions and 32 deletions
--- a/etc/config.sample.toml
+++ b/etc/config.sample.toml
@ -255,6 +255,12 @@
  # the request log to stderr.
  # access-log-path = ""

+  # Filters which requests should be logged. Each filter is of the pattern NNN, NNX, or NXX where N is
+  # a number and X is a wildcard for any number. To filter all 5xx responses, use the string 5xx.
+  # If multiple filters are used, then only one has to match. The default is to have no filters which
+  # will cause every request to be printed.
+  # access-log-status-filters = []
+
  # Determines whether detailed write logging is enabled.
  # write-tracing = false

--- a/services/httpd/config.go
+++ b/services/httpd/config.go
@ -1,7 +1,12 @@
 package httpd

 import (
+	"bytes"
 	"crypto/tls"
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
 	"time"

 	"github.com/influxdata/influxdb/monitor/diagnostics"
@ -27,32 +32,33 @@ const (

 // Config represents a configuration for a HTTP service.
 type Config struct {
-	Enabled                 bool          `toml:"enabled"`
-	BindAddress             string        `toml:"bind-address"`
-	AuthEnabled             bool          `toml:"auth-enabled"`
-	LogEnabled              bool          `toml:"log-enabled"`
-	SuppressWriteLog        bool          `toml:"suppress-write-log"`
-	WriteTracing            bool          `toml:"write-tracing"`
-	FluxEnabled             bool          `toml:"flux-enabled"`
-	PprofEnabled            bool          `toml:"pprof-enabled"`
-	DebugPprofEnabled       bool          `toml:"debug-pprof-enabled"`
-	HTTPSEnabled            bool          `toml:"https-enabled"`
-	HTTPSCertificate        string        `toml:"https-certificate"`
-	HTTPSPrivateKey         string        `toml:"https-private-key"`
-	MaxRowLimit             int           `toml:"max-row-limit"`
-	MaxConnectionLimit      int           `toml:"max-connection-limit"`
-	SharedSecret            string        `toml:"shared-secret"`
-	Realm                   string        `toml:"realm"`
-	UnixSocketEnabled       bool          `toml:"unix-socket-enabled"`
-	UnixSocketGroup         *toml.Group   `toml:"unix-socket-group"`
-	UnixSocketPermissions   toml.FileMode `toml:"unix-socket-permissions"`
-	BindSocket              string        `toml:"bind-socket"`
-	MaxBodySize             int           `toml:"max-body-size"`
-	AccessLogPath           string        `toml:"access-log-path"`
-	MaxConcurrentWriteLimit int           `toml:"max-concurrent-write-limit"`
-	MaxEnqueuedWriteLimit   int           `toml:"max-enqueued-write-limit"`
-	EnqueuedWriteTimeout    time.Duration `toml:"enqueued-write-timeout"`
-	TLS                     *tls.Config   `toml:"-"`
+	Enabled                 bool           `toml:"enabled"`
+	BindAddress             string         `toml:"bind-address"`
+	AuthEnabled             bool           `toml:"auth-enabled"`
+	LogEnabled              bool           `toml:"log-enabled"`
+	SuppressWriteLog        bool           `toml:"suppress-write-log"`
+	WriteTracing            bool           `toml:"write-tracing"`
+	FluxEnabled             bool           `toml:"flux-enabled"`
+	PprofEnabled            bool           `toml:"pprof-enabled"`
+	DebugPprofEnabled       bool           `toml:"debug-pprof-enabled"`
+	HTTPSEnabled            bool           `toml:"https-enabled"`
+	HTTPSCertificate        string         `toml:"https-certificate"`
+	HTTPSPrivateKey         string         `toml:"https-private-key"`
+	MaxRowLimit             int            `toml:"max-row-limit"`
+	MaxConnectionLimit      int            `toml:"max-connection-limit"`
+	SharedSecret            string         `toml:"shared-secret"`
+	Realm                   string         `toml:"realm"`
+	UnixSocketEnabled       bool           `toml:"unix-socket-enabled"`
+	UnixSocketGroup         *toml.Group    `toml:"unix-socket-group"`
+	UnixSocketPermissions   toml.FileMode  `toml:"unix-socket-permissions"`
+	BindSocket              string         `toml:"bind-socket"`
+	MaxBodySize             int            `toml:"max-body-size"`
+	AccessLogPath           string         `toml:"access-log-path"`
+	AccessLogStatusFilters  []StatusFilter `toml:"access-log-status-filters"`
+	MaxConcurrentWriteLimit int            `toml:"max-concurrent-write-limit"`
+	MaxEnqueuedWriteLimit   int            `toml:"max-enqueued-write-limit"`
+	EnqueuedWriteTimeout    time.Duration  `toml:"enqueued-write-timeout"`
+	TLS                     *tls.Config    `toml:"-"`
 }

 // NewConfig returns a new Config with default settings.
@ -93,3 +99,94 @@ func (c Config) Diagnostics() (*diagnostics.Diagnostics, error) {
 		"access-log-path":      c.AccessLogPath,
 	}), nil
 }
+
+// StatusFilter will check if an http status code matches a certain pattern.
+type StatusFilter struct {
+	base    int
+	divisor int
+}
+
+// reStatusFilter ensures that the format is digits optionally followed by X values.
+var reStatusFilter = regexp.MustCompile(`^([1-5]\d*)([xX]*)$`)
+
+// ParseStatusFilter will create a new status filter from the string.
+func ParseStatusFilter(s string) (StatusFilter, error) {
+	m := reStatusFilter.FindStringSubmatch(s)
+	if m == nil {
+		return StatusFilter{}, fmt.Errorf("status filter must be a digit that starts with 1-5 optionally followed by X characters")
+	} else if len(s) != 3 {
+		return StatusFilter{}, fmt.Errorf("status filter must be exactly 3 characters long")
+	}
+
+	// Compute the divisor and the expected value. If we have one X, we divide by 10 so we are only comparing
+	// the first two numbers. If we have two Xs, we divide by 100 so we only compare the first number. We
+	// then check if the result is equal to the remaining number.
+	base, err := strconv.Atoi(m[1])
+	if err != nil {
+		return StatusFilter{}, err
+	}
+
+	divisor := 1
+	switch len(m[2]) {
+	case 1:
+		divisor = 10
+	case 2:
+		divisor = 100
+	}
+	return StatusFilter{
+		base:    base,
+		divisor: divisor,
+	}, nil
+}
+
+// Match will check if the status code matches this filter.
+func (sf StatusFilter) Match(statusCode int) bool {
+	if sf.divisor == 0 {
+		return false
+	}
+	return statusCode/sf.divisor == sf.base
+}
+
+// UnmarshalText parses a TOML value into a duration value.
+func (sf *StatusFilter) UnmarshalText(text []byte) error {
+	f, err := ParseStatusFilter(string(text))
+	if err != nil {
+		return err
+	}
+	*sf = f
+	return nil
+}
+
+// MarshalText converts a duration to a string for decoding toml
+func (sf StatusFilter) MarshalText() (text []byte, err error) {
+	var buf bytes.Buffer
+	if sf.base != 0 {
+		buf.WriteString(strconv.Itoa(sf.base))
+	}
+
+	switch sf.divisor {
+	case 1:
+	case 10:
+		buf.WriteString("X")
+	case 100:
+		buf.WriteString("XX")
+	default:
+		return nil, errors.New("invalid status filter")
+	}
+	return buf.Bytes(), nil
+}
+
+type StatusFilters []StatusFilter
+
+func (filters StatusFilters) Match(statusCode int) bool {
+	if len(filters) == 0 {
+		return true
+	}
+
+	for _, sf := range filters {
+		if sf.Match(statusCode) {
+			return true
+		}
+	}
+	return false
+}
--- a/services/httpd/config_test.go
+++ b/services/httpd/config_test.go
@ -56,3 +56,195 @@ func TestConfig_WriteTracing(t *testing.T) {
 		t.Fatalf("write tracing was not set")
 	}
 }
+
+func TestConfig_StatusFilter(t *testing.T) {
+	for i, tt := range []struct {
+		cfg     string
+		status  int
+		matches bool
+	}{
+		{
+			cfg:     ``,
+			status:  200,
+			matches: true,
+		},
+		{
+			cfg:     ``,
+			status:  404,
+			matches: true,
+		},
+		{
+			cfg:     ``,
+			status:  500,
+			matches: true,
+		},
+		{
+			cfg: `
+access-log-status-filters = []
+`,
+			status:  200,
+			matches: true,
+		},
+		{
+			cfg: `
+access-log-status-filters = []
+`,
+			status:  404,
+			matches: true,
+		},
+		{
+			cfg: `
+access-log-status-filters = []
+`,
+			status:  500,
+			matches: true,
+		},
+		{
+			cfg: `
+access-log-status-filters = ["4xx"]
+`,
+			status:  200,
+			matches: false,
+		},
+		{
+			cfg: `
+access-log-status-filters = ["4xx"]
+`,
+			status:  404,
+			matches: true,
+		},
+		{
+			cfg: `
+access-log-status-filters = ["4xx"]
+`,
+			status:  400,
+			matches: true,
+		},
+		{
+			cfg: `
+access-log-status-filters = ["4xx"]
+`,
+			status:  500,
+			matches: false,
+		},
+		{
+			cfg: `
+access-log-status-filters = ["4xx", "5xx"]
+`,
+			status:  200,
+			matches: false,
+		},
+		{
+			cfg: `
+access-log-status-filters = ["4xx", "5xx"]
+`,
+			status:  404,
+			matches: true,
+		},
+		{
+			cfg: `
+access-log-status-filters = ["4xx", "5xx"]
+`,
+			status:  400,
+			matches: true,
+		},
+		{
+			cfg: `
+access-log-status-filters = ["4xx", "5xx"]
+`,
+			status:  500,
+			matches: true,
+		},
+		{
+			cfg: `
+access-log-status-filters = ["400"]
+`,
+			status:  400,
+			matches: true,
+		},
+		{
+			cfg: `
+access-log-status-filters = ["400"]
+`,
+			status:  404,
+			matches: false,
+		},
+		{
+			cfg: `
+access-log-status-filters = ["40x"]
+`,
+			status:  400,
+			matches: true,
+		},
+		{
+			cfg: `
+access-log-status-filters = ["40x"]
+`,
+			status:  404,
+			matches: true,
+		},
+		{
+			cfg: `
+access-log-status-filters = ["40x"]
+`,
+			status:  419,
+			matches: false,
+		},
+	} {
+		// Parse configuration.
+		var c httpd.Config
+		if _, err := toml.Decode(tt.cfg, &c); err != nil {
+			t.Fatal(err)
+		}
+
+		if got, want := httpd.StatusFilters(c.AccessLogStatusFilters).Match(tt.status), tt.matches; got != want {
+			t.Errorf("%d. status was not filtered correctly: got=%v want=%v", i, got, want)
+		}
+	}
+}
+
+func TestConfig_StatusFilter_Error(t *testing.T) {
+	for i, tt := range []struct {
+		cfg string
+		err string
+	}{
+		{
+			cfg: `
+access-log-status-filters = ["xxx"]
+`,
+			err: "status filter must be a digit that starts with 1-5 optionally followed by X characters",
+		},
+		{
+			cfg: `
+access-log-status-filters = ["4x4"]
+`,
+			err: "status filter must be a digit that starts with 1-5 optionally followed by X characters",
+		},
+		{
+			cfg: `
+access-log-status-filters = ["6xx"]
+`,
+			err: "status filter must be a digit that starts with 1-5 optionally followed by X characters",
+		},
+		{
+			cfg: `
+access-log-status-filters = ["0xx"]
+`,
+			err: "status filter must be a digit that starts with 1-5 optionally followed by X characters",
+		},
+		{
+			cfg: `
+access-log-status-filters = ["4xxx"]
+`,
+			err: "status filter must be exactly 3 characters long",
+		},
+	} {
+		// Parse configuration.
+		var c httpd.Config
+		if _, err := toml.Decode(tt.cfg, &c); err == nil {
+			t.Errorf("%d. expected error", i)
+		} else if got, want := err.Error(), tt.err; got != want {
+			t.Errorf("%d. config parsing error was not correct: got=%q want=%q", i, got, want)
+		}
+	}
+}
--- a/services/httpd/handler.go
+++ b/services/httpd/handler.go
@ -120,11 +120,12 @@ type Handler struct {
 	CompilerMappings flux.CompilerMappings
 	registered       bool

-	Config    *Config
-	Logger    *zap.Logger
-	CLFLogger *log.Logger
-	accessLog *os.File
-	stats     *Statistics
+	Config           *Config
+	Logger           *zap.Logger
+	CLFLogger        *log.Logger
+	accessLog        *os.File
+	accessLogFilters StatusFilters
+	stats            *Statistics

 	requestTracker *RequestTracker
 	writeThrottler *Throttler
@ -235,6 +236,7 @@ func (h *Handler) Open() {
 		}
 		h.Logger.Info("opened HTTP access log", zap.String("path", path))
 	}
+	h.accessLogFilters = StatusFilters(h.Config.AccessLogStatusFilters)

 	if h.Config.FluxEnabled {
 		h.registered = true
@ -246,6 +248,7 @@ func (h *Handler) Close() {
 	if h.accessLog != nil {
 		h.accessLog.Close()
 		h.accessLog = nil
+		h.accessLogFilters = nil
 	}

 	if h.registered {
@ -1651,7 +1654,10 @@ func (h *Handler) logging(inner http.Handler, name string) http.Handler {
 		start := time.Now()
 		l := &responseLogger{w: w}
 		inner.ServeHTTP(l, r)
-		h.CLFLogger.Println(buildLogLine(l, r, start))
+
+		if h.accessLogFilters.Match(l.Status()) {
+			h.CLFLogger.Println(buildLogLine(l, r, start))
+		}

 		// Log server errors.
 		if l.Status()/100 == 5 {