Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add an access log filter for the access log 

The access log filter allows the access log to be filtered by a status
code pattern. The pattern is a list of strings of the form `NXX`. At
least one number must be specified and up to 2 Xs can be used. That
means the filter can be an exact status code or it can be a range of
them. For example, `500` would only match the 500 http status code while
`5XX` would match any status code beginning with the number 5
(categorized as server errors). The pattern `50X` would also be
accepted. Both uppercase and lowercase Xs are allowed.

Multiple filters can be specified and the log line will be printed if
one of them matches. If there are no filters specified, all status codes
are printed.
pull/9509/head
Jonathan A. Sternberg 2018-03-05 09:20:42 -06:00
parent 7d9830ab24
commit b3472a54ee
No known key found for this signature in database
GPG Key ID: 4A0C1200CB8B9D2E
4 changed files with 333 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
etc/config.sample.toml
View File

@ -255,6 +255,12 @@
# the request log to stderr.
# access-log-path = ""
# Filters which requests should be logged. Each filter is of the pattern NNN, NNX, or NXX where N is
# a number and X is a wildcard for any number. To filter all 5xx responses, use the string 5xx.
# If multiple filters are used, then only one has to match. The default is to have no filters which
# will cause every request to be printed.
# access-log-status-filters = []
# Determines whether detailed write logging is enabled.
# write-tracing = false

149
services/httpd/config.go
View File

@ -1,7 +1,12 @@
package httpd
import (
"bytes"
"crypto/tls"
"errors"
"fmt"
"regexp"
"strconv"
"time"
"github.com/influxdata/influxdb/monitor/diagnostics"
@ -27,32 +32,33 @@ const (
// Config represents a configuration for a HTTP service.
type Config struct {
Enabled bool `toml:"enabled"`
BindAddress string `toml:"bind-address"`
AuthEnabled bool `toml:"auth-enabled"`
LogEnabled bool `toml:"log-enabled"`
SuppressWriteLog bool `toml:"suppress-write-log"`
WriteTracing bool `toml:"write-tracing"`
FluxEnabled bool `toml:"flux-enabled"`
PprofEnabled bool `toml:"pprof-enabled"`
DebugPprofEnabled bool `toml:"debug-pprof-enabled"`
HTTPSEnabled bool `toml:"https-enabled"`
HTTPSCertificate string `toml:"https-certificate"`
HTTPSPrivateKey string `toml:"https-private-key"`
MaxRowLimit int `toml:"max-row-limit"`
MaxConnectionLimit int `toml:"max-connection-limit"`
SharedSecret string `toml:"shared-secret"`
Realm string `toml:"realm"`
UnixSocketEnabled bool `toml:"unix-socket-enabled"`
UnixSocketGroup *toml.Group `toml:"unix-socket-group"`
UnixSocketPermissions toml.FileMode `toml:"unix-socket-permissions"`
BindSocket string `toml:"bind-socket"`
MaxBodySize int `toml:"max-body-size"`
AccessLogPath string `toml:"access-log-path"`
MaxConcurrentWriteLimit int `toml:"max-concurrent-write-limit"`
MaxEnqueuedWriteLimit int `toml:"max-enqueued-write-limit"`
EnqueuedWriteTimeout time.Duration `toml:"enqueued-write-timeout"`
TLS *tls.Config `toml:"-"`
Enabled bool `toml:"enabled"`
BindAddress string `toml:"bind-address"`
AuthEnabled bool `toml:"auth-enabled"`
LogEnabled bool `toml:"log-enabled"`
SuppressWriteLog bool `toml:"suppress-write-log"`
WriteTracing bool `toml:"write-tracing"`
FluxEnabled bool `toml:"flux-enabled"`
PprofEnabled bool `toml:"pprof-enabled"`
DebugPprofEnabled bool `toml:"debug-pprof-enabled"`
HTTPSEnabled bool `toml:"https-enabled"`
HTTPSCertificate string `toml:"https-certificate"`
HTTPSPrivateKey string `toml:"https-private-key"`
MaxRowLimit int `toml:"max-row-limit"`
MaxConnectionLimit int `toml:"max-connection-limit"`
SharedSecret string `toml:"shared-secret"`
Realm string `toml:"realm"`
UnixSocketEnabled bool `toml:"unix-socket-enabled"`
UnixSocketGroup *toml.Group `toml:"unix-socket-group"`
UnixSocketPermissions toml.FileMode `toml:"unix-socket-permissions"`
BindSocket string `toml:"bind-socket"`
MaxBodySize int `toml:"max-body-size"`
AccessLogPath string `toml:"access-log-path"`
AccessLogStatusFilters []StatusFilter `toml:"access-log-status-filters"`
MaxConcurrentWriteLimit int `toml:"max-concurrent-write-limit"`
MaxEnqueuedWriteLimit int `toml:"max-enqueued-write-limit"`
EnqueuedWriteTimeout time.Duration `toml:"enqueued-write-timeout"`
TLS *tls.Config `toml:"-"`
}
// NewConfig returns a new Config with default settings.
@ -93,3 +99,94 @@ func (c Config) Diagnostics() (*diagnostics.Diagnostics, error) {
"access-log-path": c.AccessLogPath,
}), nil
}
// StatusFilter will check if an http status code matches a certain pattern.
type StatusFilter struct {
base int
divisor int
}
// reStatusFilter ensures that the format is digits optionally followed by X values.
var reStatusFilter = regexp.MustCompile(`^([1-5]\d*)([xX]*)$`)
// ParseStatusFilter will create a new status filter from the string.
func ParseStatusFilter(s string) (StatusFilter, error) {
m := reStatusFilter.FindStringSubmatch(s)
if m == nil {
return StatusFilter{}, fmt.Errorf("status filter must be a digit that starts with 1-5 optionally followed by X characters")
} else if len(s) != 3 {
return StatusFilter{}, fmt.Errorf("status filter must be exactly 3 characters long")
}
// Compute the divisor and the expected value. If we have one X, we divide by 10 so we are only comparing
// the first two numbers. If we have two Xs, we divide by 100 so we only compare the first number. We
// then check if the result is equal to the remaining number.
base, err := strconv.Atoi(m[1])
if err != nil {
return StatusFilter{}, err
}
divisor := 1
switch len(m[2]) {
case 1:
divisor = 10
case 2:
divisor = 100
}
return StatusFilter{
base: base,
divisor: divisor,
}, nil
}
// Match will check if the status code matches this filter.
func (sf StatusFilter) Match(statusCode int) bool {
if sf.divisor == 0 {
return false
}
return statusCode/sf.divisor == sf.base
}
// UnmarshalText parses a TOML value into a duration value.
func (sf *StatusFilter) UnmarshalText(text []byte) error {
f, err := ParseStatusFilter(string(text))
if err != nil {
return err
}
*sf = f
return nil
}
// MarshalText converts a duration to a string for decoding toml
func (sf StatusFilter) MarshalText() (text []byte, err error) {
var buf bytes.Buffer
if sf.base != 0 {
buf.WriteString(strconv.Itoa(sf.base))
}
switch sf.divisor {
case 1:
case 10:
buf.WriteString("X")
case 100:
buf.WriteString("XX")
default:
return nil, errors.New("invalid status filter")
}
return buf.Bytes(), nil
}
type StatusFilters []StatusFilter
func (filters StatusFilters) Match(statusCode int) bool {
if len(filters) == 0 {
return true
}
for _, sf := range filters {
if sf.Match(statusCode) {
return true
}
}
return false
}

192
services/httpd/config_test.go
View File

@ -56,3 +56,195 @@ func TestConfig_WriteTracing(t *testing.T) {
t.Fatalf("write tracing was not set")
}
}
func TestConfig_StatusFilter(t *testing.T) {
for i, tt := range []struct {
cfg string
status int
matches bool
}{
{
cfg: ``,
status: 200,
matches: true,
},
{
cfg: ``,
status: 404,
matches: true,
},
{
cfg: ``,
status: 500,
matches: true,
},
{
cfg: `
access-log-status-filters = []
`,
status: 200,
matches: true,
},
{
cfg: `
access-log-status-filters = []
`,
status: 404,
matches: true,
},
{
cfg: `
access-log-status-filters = []
`,
status: 500,
matches: true,
},
{
cfg: `
access-log-status-filters = ["4xx"]
`,
status: 200,
matches: false,
},
{
cfg: `
access-log-status-filters = ["4xx"]
`,
status: 404,
matches: true,
},
{
cfg: `
access-log-status-filters = ["4xx"]
`,
status: 400,
matches: true,
},
{
cfg: `
access-log-status-filters = ["4xx"]
`,
status: 500,
matches: false,
},
{
cfg: `
access-log-status-filters = ["4xx", "5xx"]
`,
status: 200,
matches: false,
},
{
cfg: `
access-log-status-filters = ["4xx", "5xx"]
`,
status: 404,
matches: true,
},
{
cfg: `
access-log-status-filters = ["4xx", "5xx"]
`,
status: 400,
matches: true,
},
{
cfg: `
access-log-status-filters = ["4xx", "5xx"]
`,
status: 500,
matches: true,
},
{
cfg: `
access-log-status-filters = ["400"]
`,
status: 400,
matches: true,
},
{
cfg: `
access-log-status-filters = ["400"]
`,
status: 404,
matches: false,
},
{
cfg: `
access-log-status-filters = ["40x"]
`,
status: 400,
matches: true,
},
{
cfg: `
access-log-status-filters = ["40x"]
`,
status: 404,
matches: true,
},
{
cfg: `
access-log-status-filters = ["40x"]
`,
status: 419,
matches: false,
},
} {
// Parse configuration.
var c httpd.Config
if _, err := toml.Decode(tt.cfg, &c); err != nil {
t.Fatal(err)
}
if got, want := httpd.StatusFilters(c.AccessLogStatusFilters).Match(tt.status), tt.matches; got != want {
t.Errorf("%d. status was not filtered correctly: got=%v want=%v", i, got, want)
}
}
}
func TestConfig_StatusFilter_Error(t *testing.T) {
for i, tt := range []struct {
cfg string
err string
}{
{
cfg: `
access-log-status-filters = ["xxx"]
`,
err: "status filter must be a digit that starts with 1-5 optionally followed by X characters",
},
{
cfg: `
access-log-status-filters = ["4x4"]
`,
err: "status filter must be a digit that starts with 1-5 optionally followed by X characters",
},
{
cfg: `
access-log-status-filters = ["6xx"]
`,
err: "status filter must be a digit that starts with 1-5 optionally followed by X characters",
},
{
cfg: `
access-log-status-filters = ["0xx"]
`,
err: "status filter must be a digit that starts with 1-5 optionally followed by X characters",
},
{
cfg: `
access-log-status-filters = ["4xxx"]
`,
err: "status filter must be exactly 3 characters long",
},
} {
// Parse configuration.
var c httpd.Config
if _, err := toml.Decode(tt.cfg, &c); err == nil {
t.Errorf("%d. expected error", i)
} else if got, want := err.Error(), tt.err; got != want {
t.Errorf("%d. config parsing error was not correct: got=%q want=%q", i, got, want)
}
}
}

18
services/httpd/handler.go
View File

@ -120,11 +120,12 @@ type Handler struct {
CompilerMappings flux.CompilerMappings
registered bool
Config *Config
Logger *zap.Logger
CLFLogger *log.Logger
accessLog *os.File
stats *Statistics
Config *Config
Logger *zap.Logger
CLFLogger *log.Logger
accessLog *os.File
accessLogFilters StatusFilters
stats *Statistics
requestTracker *RequestTracker
writeThrottler *Throttler
@ -235,6 +236,7 @@ func (h *Handler) Open() {
}
h.Logger.Info("opened HTTP access log", zap.String("path", path))
}
h.accessLogFilters = StatusFilters(h.Config.AccessLogStatusFilters)
if h.Config.FluxEnabled {
h.registered = true
@ -246,6 +248,7 @@ func (h *Handler) Close() {
if h.accessLog != nil {
h.accessLog.Close()
h.accessLog = nil
h.accessLogFilters = nil
}
if h.registered {
@ -1651,7 +1654,10 @@ func (h *Handler) logging(inner http.Handler, name string) http.Handler {
start := time.Now()
l := &responseLogger{w: w}
inner.ServeHTTP(l, r)
h.CLFLogger.Println(buildLogLine(l, r, start))
if h.accessLogFilters.Match(l.Status()) {
h.CLFLogger.Println(buildLogLine(l, r, start))
}
// Log server errors.
if l.Status()/100 == 5 {