From b3472a54ee5dd73232ef7d4eed8978df7fdb0dc8 Mon Sep 17 00:00:00 2001 From: "Jonathan A. Sternberg" Date: Mon, 5 Mar 2018 09:20:42 -0600 Subject: [PATCH] Add an access log filter for the access log The access log filter allows the access log to be filtered by a status code pattern. The pattern is a list of strings of the form `NXX`. At least one number must be specified and up to 2 Xs can be used. That means the filter can be an exact status code or it can be a range of them. For example, `500` would only match the 500 http status code while `5XX` would match any status code beginning with the number 5 (categorized as server errors). The pattern `50X` would also be accepted. Both uppercase and lowercase Xs are allowed. Multiple filters can be specified and the log line will be printed if one of them matches. If there are no filters specified, all status codes are printed. --- etc/config.sample.toml | 6 ++ services/httpd/config.go | 149 +++++++++++++++++++++----- services/httpd/config_test.go | 192 ++++++++++++++++++++++++++++++++++ services/httpd/handler.go | 18 ++-- 4 files changed, 333 insertions(+), 32 deletions(-) diff --git a/etc/config.sample.toml b/etc/config.sample.toml index d51864b501..80aaea83d0 100644 --- a/etc/config.sample.toml +++ b/etc/config.sample.toml @@ -255,6 +255,12 @@ # the request log to stderr. # access-log-path = "" + # Filters which requests should be logged. Each filter is of the pattern NNN, NNX, or NXX where N is + # a number and X is a wildcard for any number. To filter all 5xx responses, use the string 5xx. + # If multiple filters are used, then only one has to match. The default is to have no filters which + # will cause every request to be printed. + # access-log-status-filters = [] + # Determines whether detailed write logging is enabled. # write-tracing = false diff --git a/services/httpd/config.go b/services/httpd/config.go index 1a1583c78b..2212263109 100644 --- a/services/httpd/config.go +++ b/services/httpd/config.go @@ -1,7 +1,12 @@ package httpd import ( + "bytes" "crypto/tls" + "errors" + "fmt" + "regexp" + "strconv" "time" "github.com/influxdata/influxdb/monitor/diagnostics" @@ -27,32 +32,33 @@ const ( // Config represents a configuration for a HTTP service. type Config struct { - Enabled bool `toml:"enabled"` - BindAddress string `toml:"bind-address"` - AuthEnabled bool `toml:"auth-enabled"` - LogEnabled bool `toml:"log-enabled"` - SuppressWriteLog bool `toml:"suppress-write-log"` - WriteTracing bool `toml:"write-tracing"` - FluxEnabled bool `toml:"flux-enabled"` - PprofEnabled bool `toml:"pprof-enabled"` - DebugPprofEnabled bool `toml:"debug-pprof-enabled"` - HTTPSEnabled bool `toml:"https-enabled"` - HTTPSCertificate string `toml:"https-certificate"` - HTTPSPrivateKey string `toml:"https-private-key"` - MaxRowLimit int `toml:"max-row-limit"` - MaxConnectionLimit int `toml:"max-connection-limit"` - SharedSecret string `toml:"shared-secret"` - Realm string `toml:"realm"` - UnixSocketEnabled bool `toml:"unix-socket-enabled"` - UnixSocketGroup *toml.Group `toml:"unix-socket-group"` - UnixSocketPermissions toml.FileMode `toml:"unix-socket-permissions"` - BindSocket string `toml:"bind-socket"` - MaxBodySize int `toml:"max-body-size"` - AccessLogPath string `toml:"access-log-path"` - MaxConcurrentWriteLimit int `toml:"max-concurrent-write-limit"` - MaxEnqueuedWriteLimit int `toml:"max-enqueued-write-limit"` - EnqueuedWriteTimeout time.Duration `toml:"enqueued-write-timeout"` - TLS *tls.Config `toml:"-"` + Enabled bool `toml:"enabled"` + BindAddress string `toml:"bind-address"` + AuthEnabled bool `toml:"auth-enabled"` + LogEnabled bool `toml:"log-enabled"` + SuppressWriteLog bool `toml:"suppress-write-log"` + WriteTracing bool `toml:"write-tracing"` + FluxEnabled bool `toml:"flux-enabled"` + PprofEnabled bool `toml:"pprof-enabled"` + DebugPprofEnabled bool `toml:"debug-pprof-enabled"` + HTTPSEnabled bool `toml:"https-enabled"` + HTTPSCertificate string `toml:"https-certificate"` + HTTPSPrivateKey string `toml:"https-private-key"` + MaxRowLimit int `toml:"max-row-limit"` + MaxConnectionLimit int `toml:"max-connection-limit"` + SharedSecret string `toml:"shared-secret"` + Realm string `toml:"realm"` + UnixSocketEnabled bool `toml:"unix-socket-enabled"` + UnixSocketGroup *toml.Group `toml:"unix-socket-group"` + UnixSocketPermissions toml.FileMode `toml:"unix-socket-permissions"` + BindSocket string `toml:"bind-socket"` + MaxBodySize int `toml:"max-body-size"` + AccessLogPath string `toml:"access-log-path"` + AccessLogStatusFilters []StatusFilter `toml:"access-log-status-filters"` + MaxConcurrentWriteLimit int `toml:"max-concurrent-write-limit"` + MaxEnqueuedWriteLimit int `toml:"max-enqueued-write-limit"` + EnqueuedWriteTimeout time.Duration `toml:"enqueued-write-timeout"` + TLS *tls.Config `toml:"-"` } // NewConfig returns a new Config with default settings. @@ -93,3 +99,94 @@ func (c Config) Diagnostics() (*diagnostics.Diagnostics, error) { "access-log-path": c.AccessLogPath, }), nil } + +// StatusFilter will check if an http status code matches a certain pattern. +type StatusFilter struct { + base int + divisor int +} + +// reStatusFilter ensures that the format is digits optionally followed by X values. +var reStatusFilter = regexp.MustCompile(`^([1-5]\d*)([xX]*)$`) + +// ParseStatusFilter will create a new status filter from the string. +func ParseStatusFilter(s string) (StatusFilter, error) { + m := reStatusFilter.FindStringSubmatch(s) + if m == nil { + return StatusFilter{}, fmt.Errorf("status filter must be a digit that starts with 1-5 optionally followed by X characters") + } else if len(s) != 3 { + return StatusFilter{}, fmt.Errorf("status filter must be exactly 3 characters long") + } + + // Compute the divisor and the expected value. If we have one X, we divide by 10 so we are only comparing + // the first two numbers. If we have two Xs, we divide by 100 so we only compare the first number. We + // then check if the result is equal to the remaining number. + base, err := strconv.Atoi(m[1]) + if err != nil { + return StatusFilter{}, err + } + + divisor := 1 + switch len(m[2]) { + case 1: + divisor = 10 + case 2: + divisor = 100 + } + return StatusFilter{ + base: base, + divisor: divisor, + }, nil +} + +// Match will check if the status code matches this filter. +func (sf StatusFilter) Match(statusCode int) bool { + if sf.divisor == 0 { + return false + } + return statusCode/sf.divisor == sf.base +} + +// UnmarshalText parses a TOML value into a duration value. +func (sf *StatusFilter) UnmarshalText(text []byte) error { + f, err := ParseStatusFilter(string(text)) + if err != nil { + return err + } + *sf = f + return nil +} + +// MarshalText converts a duration to a string for decoding toml +func (sf StatusFilter) MarshalText() (text []byte, err error) { + var buf bytes.Buffer + if sf.base != 0 { + buf.WriteString(strconv.Itoa(sf.base)) + } + + switch sf.divisor { + case 1: + case 10: + buf.WriteString("X") + case 100: + buf.WriteString("XX") + default: + return nil, errors.New("invalid status filter") + } + return buf.Bytes(), nil +} + +type StatusFilters []StatusFilter + +func (filters StatusFilters) Match(statusCode int) bool { + if len(filters) == 0 { + return true + } + + for _, sf := range filters { + if sf.Match(statusCode) { + return true + } + } + return false +} diff --git a/services/httpd/config_test.go b/services/httpd/config_test.go index 03f5992c86..0b931b20f2 100644 --- a/services/httpd/config_test.go +++ b/services/httpd/config_test.go @@ -56,3 +56,195 @@ func TestConfig_WriteTracing(t *testing.T) { t.Fatalf("write tracing was not set") } } + +func TestConfig_StatusFilter(t *testing.T) { + for i, tt := range []struct { + cfg string + status int + matches bool + }{ + { + cfg: ``, + status: 200, + matches: true, + }, + { + cfg: ``, + status: 404, + matches: true, + }, + { + cfg: ``, + status: 500, + matches: true, + }, + { + cfg: ` +access-log-status-filters = [] +`, + status: 200, + matches: true, + }, + { + cfg: ` +access-log-status-filters = [] +`, + status: 404, + matches: true, + }, + { + cfg: ` +access-log-status-filters = [] +`, + status: 500, + matches: true, + }, + { + cfg: ` +access-log-status-filters = ["4xx"] +`, + status: 200, + matches: false, + }, + { + cfg: ` +access-log-status-filters = ["4xx"] +`, + status: 404, + matches: true, + }, + { + cfg: ` +access-log-status-filters = ["4xx"] +`, + status: 400, + matches: true, + }, + { + cfg: ` +access-log-status-filters = ["4xx"] +`, + status: 500, + matches: false, + }, + { + cfg: ` +access-log-status-filters = ["4xx", "5xx"] +`, + status: 200, + matches: false, + }, + { + cfg: ` +access-log-status-filters = ["4xx", "5xx"] +`, + status: 404, + matches: true, + }, + { + cfg: ` +access-log-status-filters = ["4xx", "5xx"] +`, + status: 400, + matches: true, + }, + { + cfg: ` +access-log-status-filters = ["4xx", "5xx"] +`, + status: 500, + matches: true, + }, + { + cfg: ` +access-log-status-filters = ["400"] +`, + status: 400, + matches: true, + }, + { + cfg: ` +access-log-status-filters = ["400"] +`, + status: 404, + matches: false, + }, + { + cfg: ` +access-log-status-filters = ["40x"] +`, + status: 400, + matches: true, + }, + { + cfg: ` +access-log-status-filters = ["40x"] +`, + status: 404, + matches: true, + }, + { + cfg: ` +access-log-status-filters = ["40x"] +`, + status: 419, + matches: false, + }, + } { + // Parse configuration. + var c httpd.Config + if _, err := toml.Decode(tt.cfg, &c); err != nil { + t.Fatal(err) + } + + if got, want := httpd.StatusFilters(c.AccessLogStatusFilters).Match(tt.status), tt.matches; got != want { + t.Errorf("%d. status was not filtered correctly: got=%v want=%v", i, got, want) + } + } +} + +func TestConfig_StatusFilter_Error(t *testing.T) { + for i, tt := range []struct { + cfg string + err string + }{ + { + cfg: ` +access-log-status-filters = ["xxx"] +`, + err: "status filter must be a digit that starts with 1-5 optionally followed by X characters", + }, + { + cfg: ` +access-log-status-filters = ["4x4"] +`, + err: "status filter must be a digit that starts with 1-5 optionally followed by X characters", + }, + { + cfg: ` +access-log-status-filters = ["6xx"] +`, + err: "status filter must be a digit that starts with 1-5 optionally followed by X characters", + }, + { + cfg: ` +access-log-status-filters = ["0xx"] +`, + err: "status filter must be a digit that starts with 1-5 optionally followed by X characters", + }, + { + cfg: ` +access-log-status-filters = ["4xxx"] +`, + err: "status filter must be exactly 3 characters long", + }, + } { + // Parse configuration. + var c httpd.Config + if _, err := toml.Decode(tt.cfg, &c); err == nil { + t.Errorf("%d. expected error", i) + } else if got, want := err.Error(), tt.err; got != want { + t.Errorf("%d. config parsing error was not correct: got=%q want=%q", i, got, want) + } + } +} diff --git a/services/httpd/handler.go b/services/httpd/handler.go index 90f291d19e..bc16b63abc 100644 --- a/services/httpd/handler.go +++ b/services/httpd/handler.go @@ -120,11 +120,12 @@ type Handler struct { CompilerMappings flux.CompilerMappings registered bool - Config *Config - Logger *zap.Logger - CLFLogger *log.Logger - accessLog *os.File - stats *Statistics + Config *Config + Logger *zap.Logger + CLFLogger *log.Logger + accessLog *os.File + accessLogFilters StatusFilters + stats *Statistics requestTracker *RequestTracker writeThrottler *Throttler @@ -235,6 +236,7 @@ func (h *Handler) Open() { } h.Logger.Info("opened HTTP access log", zap.String("path", path)) } + h.accessLogFilters = StatusFilters(h.Config.AccessLogStatusFilters) if h.Config.FluxEnabled { h.registered = true @@ -246,6 +248,7 @@ func (h *Handler) Close() { if h.accessLog != nil { h.accessLog.Close() h.accessLog = nil + h.accessLogFilters = nil } if h.registered { @@ -1651,7 +1654,10 @@ func (h *Handler) logging(inner http.Handler, name string) http.Handler { start := time.Now() l := &responseLogger{w: w} inner.ServeHTTP(l, r) - h.CLFLogger.Println(buildLogLine(l, r, start)) + + if h.accessLogFilters.Match(l.Status()) { + h.CLFLogger.Println(buildLogLine(l, r, start)) + } // Log server errors. if l.Status()/100 == 5 {