From 8dee88403875758abd42d850cedbd9e5c6cb1a61 Mon Sep 17 00:00:00 2001 From: "Jonathan A. Sternberg" Date: Thu, 11 Oct 2018 09:43:29 -0500 Subject: [PATCH] refactor(toml): copy the toml utility package from influxdb to platform --- go.mod | 3 - go.sum | 6 - toml/toml.go | 279 ++++++++++++++++++++++++++++++++++++++++++++ toml/toml_test.go | 244 ++++++++++++++++++++++++++++++++++++++ tsdb/config.go | 2 +- tsdb/tsi1/config.go | 2 +- 6 files changed, 525 insertions(+), 11 deletions(-) create mode 100644 toml/toml.go create mode 100644 toml/toml_test.go diff --git a/go.mod b/go.mod index 61b77bc322..8306e2e074 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,6 @@ module github.com/influxdata/platform require ( - collectd.org v0.3.0 // indirect github.com/BurntSushi/toml v0.3.1 // indirect github.com/Masterminds/semver v1.4.2 // indirect github.com/NYTimes/gziphandler v1.0.1 @@ -14,7 +13,6 @@ require ( github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect github.com/aws/aws-sdk-go v1.15.50 // indirect github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2 // indirect - github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 // indirect github.com/boltdb/bolt v1.3.1 // indirect github.com/bouk/httprouter v0.0.0-20160817010721-ee8b3818a7f5 github.com/caarlos0/ctrlc v1.0.0 // indirect @@ -45,7 +43,6 @@ require ( github.com/influxdata/influxdb v0.0.0-20181009160823-86ac358448ec github.com/influxdata/influxql v0.0.0-20180925231337-1cbfca8e56b6 github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e - github.com/influxdata/roaring v0.4.12 // indirect github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368 github.com/jessevdk/go-flags v1.4.0 github.com/jsternberg/zap-logfmt v1.2.0 diff --git a/go.sum b/go.sum index e322733260..c40f9311c7 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,4 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -collectd.org v0.3.0 h1:iNBHGw1VvPJxH2B6RiFWFZ+vsjo1lCdRszBeOuwGi00= -collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc= @@ -29,8 +27,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLM github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2 h1:oMCHnXa6CCCafdPDbMh/lWRhRByN0VFLvv+g+ayx1SI= github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= -github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 h1:y4B3+GPxKlrigF1ha5FFErxK+sr6sWxQovRMzwMhejo= -github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bouk/httprouter v0.0.0-20160817010721-ee8b3818a7f5 h1:kS0dw4K730x7cxT+bVyTyYJZHuSoH7ofSr/Ijit56Qw= @@ -130,8 +126,6 @@ github.com/influxdata/influxql v0.0.0-20180925231337-1cbfca8e56b6 h1:CFx+pP90q/q github.com/influxdata/influxql v0.0.0-20180925231337-1cbfca8e56b6/go.mod h1:KpVI7okXjK6PRi3Z5B+mtKZli+R1DnZgb3N+tzevNgo= github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e h1:/o3vQtpWJhvnIbXley4/jwzzqNeigJK9z+LZcJZ9zfM= github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= -github.com/influxdata/roaring v0.4.12 h1:3DzTjKHcXFs4P3D7xRLpCqVrfK6eFRQT0c8BG99M3Ms= -github.com/influxdata/roaring v0.4.12/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= github.com/influxdata/tdigest v0.0.0-20180711151920-a7d76c6f093a h1:vMqgISSVkIqWxCIZs8m1L4096temR7IbYyNdMiBxSPA= github.com/influxdata/tdigest v0.0.0-20180711151920-a7d76c6f093a/go.mod h1:9GkyshztGufsdPQWjH+ifgnIr3xNUL5syI70g2dzU1o= github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368 h1:+TUUmaFa4YD1Q+7bH9o5NCHQGPMqZCYJiNW6lIIS9z4= diff --git a/toml/toml.go b/toml/toml.go new file mode 100644 index 0000000000..93c9217d39 --- /dev/null +++ b/toml/toml.go @@ -0,0 +1,279 @@ +// Package toml adds support to marshal and unmarshal types not in the official TOML spec. +package toml + +import ( + "encoding" + "errors" + "fmt" + "math" + "os" + "os/user" + "reflect" + "strconv" + "strings" + "time" + "unicode" +) + +// Duration is a TOML wrapper type for time.Duration. +type Duration time.Duration + +// String returns the string representation of the duration. +func (d Duration) String() string { + return time.Duration(d).String() +} + +// UnmarshalText parses a TOML value into a duration value. +func (d *Duration) UnmarshalText(text []byte) error { + // Ignore if there is no value set. + if len(text) == 0 { + return nil + } + + // Otherwise parse as a duration formatted string. + duration, err := time.ParseDuration(string(text)) + if err != nil { + return err + } + + // Set duration and return. + *d = Duration(duration) + return nil +} + +// MarshalText converts a duration to a string for decoding toml +func (d Duration) MarshalText() (text []byte, err error) { + return []byte(d.String()), nil +} + +// Size represents a TOML parseable file size. +// Users can specify size using "k" or "K" for kibibytes, "m" or "M" for mebibytes, +// and "g" or "G" for gibibytes. If a size suffix isn't specified then bytes are assumed. +type Size uint64 + +// UnmarshalText parses a byte size from text. +func (s *Size) UnmarshalText(text []byte) error { + if len(text) == 0 { + return fmt.Errorf("size was empty") + } + + // The multiplier defaults to 1 in case the size has + // no suffix (and is then just raw bytes) + mult := uint64(1) + + // Preserve the original text for error messages + sizeText := text + + // Parse unit of measure + suffix := text[len(sizeText)-1] + if !unicode.IsDigit(rune(suffix)) { + switch suffix { + case 'k', 'K': + mult = 1 << 10 // KiB + case 'm', 'M': + mult = 1 << 20 // MiB + case 'g', 'G': + mult = 1 << 30 // GiB + default: + return fmt.Errorf("unknown size suffix: %c (expected k, m, or g)", suffix) + } + sizeText = sizeText[:len(sizeText)-1] + } + + // Parse numeric portion of value. + size, err := strconv.ParseUint(string(sizeText), 10, 64) + if err != nil { + return fmt.Errorf("invalid size: %s", string(text)) + } + + if math.MaxUint64/mult < size { + return fmt.Errorf("size would overflow the max size (%d) of a uint: %s", uint64(math.MaxUint64), string(text)) + } + + size *= mult + + *s = Size(size) + return nil +} + +type FileMode uint32 + +func (m *FileMode) UnmarshalText(text []byte) error { + // Ignore if there is no value set. + if len(text) == 0 { + return nil + } + + mode, err := strconv.ParseUint(string(text), 8, 32) + if err != nil { + return err + } else if mode == 0 { + return errors.New("file mode cannot be zero") + } + *m = FileMode(mode) + return nil +} + +func (m FileMode) MarshalText() (text []byte, err error) { + if m != 0 { + return []byte(fmt.Sprintf("%04o", m)), nil + } + return nil, nil +} + +type Group int + +func (g *Group) UnmarshalTOML(data interface{}) error { + if grpName, ok := data.(string); ok { + group, err := user.LookupGroup(grpName) + if err != nil { + return err + } + + gid, err := strconv.Atoi(group.Gid) + if err != nil { + return err + } + *g = Group(gid) + return nil + } else if gid, ok := data.(int64); ok { + *g = Group(gid) + return nil + } + return errors.New("group must be a name (string) or id (int)") +} + +func ApplyEnvOverrides(getenv func(string) string, prefix string, val interface{}) error { + if getenv == nil { + getenv = os.Getenv + } + return applyEnvOverrides(getenv, prefix, reflect.ValueOf(val), "") +} + +func applyEnvOverrides(getenv func(string) string, prefix string, spec reflect.Value, structKey string) error { + element := spec + // If spec is a named type and is addressable, + // check the address to see if it implements encoding.TextUnmarshaler. + if spec.Kind() != reflect.Ptr && spec.Type().Name() != "" && spec.CanAddr() { + v := spec.Addr() + if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { + value := getenv(prefix) + return u.UnmarshalText([]byte(value)) + } + } + // If we have a pointer, dereference it + if spec.Kind() == reflect.Ptr { + element = spec.Elem() + } + + value := getenv(prefix) + + switch element.Kind() { + case reflect.String: + if len(value) == 0 { + return nil + } + element.SetString(value) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + intValue, err := strconv.ParseInt(value, 0, element.Type().Bits()) + if err != nil { + return fmt.Errorf("failed to apply %v to %v using type %v and value '%v': %s", prefix, structKey, element.Type().String(), value, err) + } + element.SetInt(intValue) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + intValue, err := strconv.ParseUint(value, 0, element.Type().Bits()) + if err != nil { + return fmt.Errorf("failed to apply %v to %v using type %v and value '%v': %s", prefix, structKey, element.Type().String(), value, err) + } + element.SetUint(intValue) + case reflect.Bool: + boolValue, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("failed to apply %v to %v using type %v and value '%v': %s", prefix, structKey, element.Type().String(), value, err) + } + element.SetBool(boolValue) + case reflect.Float32, reflect.Float64: + floatValue, err := strconv.ParseFloat(value, element.Type().Bits()) + if err != nil { + return fmt.Errorf("failed to apply %v to %v using type %v and value '%v': %s", prefix, structKey, element.Type().String(), value, err) + } + element.SetFloat(floatValue) + case reflect.Slice: + // If the type is s slice, apply to each using the index as a suffix, e.g. GRAPHITE_0, GRAPHITE_0_TEMPLATES_0 or GRAPHITE_0_TEMPLATES="item1,item2" + for j := 0; j < element.Len(); j++ { + f := element.Index(j) + if err := applyEnvOverrides(getenv, prefix, f, structKey); err != nil { + return err + } + + if err := applyEnvOverrides(getenv, fmt.Sprintf("%s_%d", prefix, j), f, structKey); err != nil { + return err + } + } + + // If the type is s slice but have value not parsed as slice e.g. GRAPHITE_0_TEMPLATES="item1,item2" + if element.Len() == 0 && len(value) > 0 { + rules := strings.Split(value, ",") + + for _, rule := range rules { + element.Set(reflect.Append(element, reflect.ValueOf(rule))) + } + } + case reflect.Struct: + typeOfSpec := element.Type() + for i := 0; i < element.NumField(); i++ { + field := element.Field(i) + + // Skip any fields that we cannot set + if !field.CanSet() && field.Kind() != reflect.Slice { + continue + } + + structField := typeOfSpec.Field(i) + fieldName := structField.Name + + configName := structField.Tag.Get("toml") + if configName == "-" { + // Skip fields with tag `toml:"-"`. + continue + } + + if configName == "" && structField.Anonymous { + // Embedded field without a toml tag. + // Don't modify prefix. + if err := applyEnvOverrides(getenv, prefix, field, fieldName); err != nil { + return err + } + continue + } + + // Replace hyphens with underscores to avoid issues with shells + configName = strings.Replace(configName, "-", "_", -1) + + envKey := strings.ToUpper(configName) + if prefix != "" { + envKey = strings.ToUpper(fmt.Sprintf("%s_%s", prefix, configName)) + } + + // If it's a sub-config, recursively apply + if field.Kind() == reflect.Struct || field.Kind() == reflect.Ptr || + field.Kind() == reflect.Slice || field.Kind() == reflect.Array { + if err := applyEnvOverrides(getenv, envKey, field, fieldName); err != nil { + return err + } + continue + } + + value := getenv(envKey) + // Skip any fields we don't have a value to set + if len(value) == 0 { + continue + } + + if err := applyEnvOverrides(getenv, envKey, field, fieldName); err != nil { + return err + } + } + } + return nil +} diff --git a/toml/toml_test.go b/toml/toml_test.go new file mode 100644 index 0000000000..89fe771981 --- /dev/null +++ b/toml/toml_test.go @@ -0,0 +1,244 @@ +package toml_test + +import ( + "fmt" + "math" + "os/user" + "runtime" + "strconv" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + itoml "github.com/influxdata/platform/toml" +) + +func TestSize_UnmarshalText(t *testing.T) { + var s itoml.Size + for _, test := range []struct { + str string + want uint64 + }{ + {"1", 1}, + {"10", 10}, + {"100", 100}, + {"1k", 1 << 10}, + {"10k", 10 << 10}, + {"100k", 100 << 10}, + {"1K", 1 << 10}, + {"10K", 10 << 10}, + {"100K", 100 << 10}, + {"1m", 1 << 20}, + {"10m", 10 << 20}, + {"100m", 100 << 20}, + {"1M", 1 << 20}, + {"10M", 10 << 20}, + {"100M", 100 << 20}, + {"1g", 1 << 30}, + {"1G", 1 << 30}, + {fmt.Sprint(uint64(math.MaxUint64) - 1), math.MaxUint64 - 1}, + } { + if err := s.UnmarshalText([]byte(test.str)); err != nil { + t.Fatalf("unexpected error: %s", err) + } + if s != itoml.Size(test.want) { + t.Fatalf("wanted: %d got: %d", test.want, s) + } + } + + for _, str := range []string{ + fmt.Sprintf("%dk", uint64(math.MaxUint64-1)), + "10000000000000000000g", + "abcdef", + "1KB", + "√m", + "a1", + "", + } { + if err := s.UnmarshalText([]byte(str)); err == nil { + t.Fatalf("input should have failed: %s", str) + } + } +} + +func TestFileMode_MarshalText(t *testing.T) { + for _, test := range []struct { + mode int + want string + }{ + {mode: 0755, want: `0755`}, + {mode: 0777, want: `0777`}, + {mode: 01777, want: `1777`}, + } { + mode := itoml.FileMode(test.mode) + if got, err := mode.MarshalText(); err != nil { + t.Errorf("unexpected error: %s", err) + } else if test.want != string(got) { + t.Errorf("wanted: %v got: %v", test.want, string(got)) + } + } +} + +func TestFileMode_UnmarshalText(t *testing.T) { + for _, test := range []struct { + str string + want uint32 + }{ + {str: ``, want: 0}, + {str: `0777`, want: 0777}, + {str: `777`, want: 0777}, + {str: `1777`, want: 01777}, + {str: `0755`, want: 0755}, + } { + var mode itoml.FileMode + if err := mode.UnmarshalText([]byte(test.str)); err != nil { + t.Errorf("unexpected error: %s", err) + } else if mode != itoml.FileMode(test.want) { + t.Errorf("wanted: %04o got: %04o", test.want, mode) + } + } +} + +func TestGroup_UnmarshalTOML(t *testing.T) { + // Skip this test on windows since it does not support setting the group anyway. + if runtime.GOOS == "windows" { + t.Skip("unsupported on windows") + } + + // Find the current user ID so we can use that group name. + u, err := user.Current() + if err != nil { + t.Skipf("unable to find the current user: %s", err) + } + + // Lookup the group by the group id. + gr, err := user.LookupGroupId(u.Gid) + if err == nil { + var group itoml.Group + if err := group.UnmarshalTOML(gr.Name); err != nil { + t.Fatalf("unexpected error: %s", err) + } else if got, want := u.Gid, strconv.Itoa(int(group)); got != want { + t.Fatalf("unexpected group id: %s != %s", got, want) + } + } + + // Attempt to convert the group to an integer so we can test reading an integer. + gid, err := strconv.Atoi(u.Gid) + if err != nil { + t.Fatalf("group id is not an integer: %s", err) + } + + var group itoml.Group + if err := group.UnmarshalTOML(int64(gid)); err != nil { + t.Fatalf("unexpected error: %s", err) + } else if int(group) != gid { + t.Fatalf("unexpected group id: %d != %d", gid, int(group)) + } +} + +func TestConfig_Encode(t *testing.T) { + t.Skip("TODO(jsternberg): rewrite this test to use something from platform") + //var c run.Config + //c.Coordinator.WriteTimeout = itoml.Duration(time.Minute) + //buf := new(bytes.Buffer) + //if err := toml.NewEncoder(buf).Encode(&c); err != nil { + // t.Fatal("Failed to encode: ", err) + //} + //got, search := buf.String(), `write-timeout = "1m0s"` + //if !strings.Contains(got, search) { + // t.Fatalf("Encoding config failed.\nfailed to find %s in:\n%s\n", search, got) + //} +} + +func TestEnvOverride_Builtins(t *testing.T) { + envMap := map[string]string{ + "X_STRING": "a string", + "X_DURATION": "1m1s", + "X_INT": "1", + "X_INT8": "2", + "X_INT16": "3", + "X_INT32": "4", + "X_INT64": "5", + "X_UINT": "6", + "X_UINT8": "7", + "X_UINT16": "8", + "X_UINT32": "9", + "X_UINT64": "10", + "X_BOOL": "true", + "X_FLOAT32": "11.5", + "X_FLOAT64": "12.5", + "X_NESTED_STRING": "a nested string", + "X_NESTED_INT": "13", + "X_ES": "an embedded string", + "X__": "-1", // This value should not be applied to the "ignored" field with toml tag -. + } + + env := func(s string) string { + return envMap[s] + } + + type nested struct { + Str string `toml:"string"` + Int int `toml:"int"` + } + type Embedded struct { + ES string `toml:"es"` + } + type all struct { + Str string `toml:"string"` + Dur itoml.Duration `toml:"duration"` + Int int `toml:"int"` + Int8 int8 `toml:"int8"` + Int16 int16 `toml:"int16"` + Int32 int32 `toml:"int32"` + Int64 int64 `toml:"int64"` + Uint uint `toml:"uint"` + Uint8 uint8 `toml:"uint8"` + Uint16 uint16 `toml:"uint16"` + Uint32 uint32 `toml:"uint32"` + Uint64 uint64 `toml:"uint64"` + Bool bool `toml:"bool"` + Float32 float32 `toml:"float32"` + Float64 float64 `toml:"float64"` + Nested nested `toml:"nested"` + + Embedded + + Ignored int `toml:"-"` + } + + var got all + if err := itoml.ApplyEnvOverrides(env, "X", &got); err != nil { + t.Fatal(err) + } + + exp := all{ + Str: "a string", + Dur: itoml.Duration(time.Minute + time.Second), + Int: 1, + Int8: 2, + Int16: 3, + Int32: 4, + Int64: 5, + Uint: 6, + Uint8: 7, + Uint16: 8, + Uint32: 9, + Uint64: 10, + Bool: true, + Float32: 11.5, + Float64: 12.5, + Nested: nested{ + Str: "a nested string", + Int: 13, + }, + Embedded: Embedded{ + ES: "an embedded string", + }, + Ignored: 0, + } + + if diff := cmp.Diff(got, exp); diff != "" { + t.Fatal(diff) + } +} diff --git a/tsdb/config.go b/tsdb/config.go index 5ad7ffb6d9..83f8728084 100644 --- a/tsdb/config.go +++ b/tsdb/config.go @@ -6,7 +6,7 @@ import ( "github.com/influxdata/influxdb/monitor/diagnostics" "github.com/influxdata/influxdb/query" - "github.com/influxdata/influxdb/toml" + "github.com/influxdata/platform/toml" "github.com/influxdata/platform/tsdb/defaults" ) diff --git a/tsdb/tsi1/config.go b/tsdb/tsi1/config.go index f58753288a..d5cd4a7801 100644 --- a/tsdb/tsi1/config.go +++ b/tsdb/tsi1/config.go @@ -1,6 +1,6 @@ package tsi1 -import "github.com/influxdata/influxdb/toml" +import "github.com/influxdata/platform/toml" // DefaultMaxIndexLogFileSize is the default threshold, in bytes, when an index // write-ahead log file will compact into an index file.