Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #1044 from influxdata/js-move-toml-package 

refactor(toml): copy the toml utility package from influxdb to platform
pull/10616/head
Jonathan A. Sternberg 2018-10-11 13:03:15 -05:00 committed by GitHub
parent 883ad88ac1 8dee884038
commit 8a67febf35
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 525 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
go.mod
View File

@ -1,7 +1,6 @@
module github.com/influxdata/platform
require (
collectd.org v0.3.0 // indirect
github.com/BurntSushi/toml v0.3.1 // indirect
github.com/Masterminds/semver v1.4.2 // indirect
github.com/NYTimes/gziphandler v1.0.1
@ -14,7 +13,6 @@ require (
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect
github.com/aws/aws-sdk-go v1.15.50 // indirect
github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2 // indirect
github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 // indirect
github.com/boltdb/bolt v1.3.1 // indirect
github.com/bouk/httprouter v0.0.0-20160817010721-ee8b3818a7f5
github.com/caarlos0/ctrlc v1.0.0 // indirect
@ -45,7 +43,6 @@ require (
github.com/influxdata/influxdb v0.0.0-20181009160823-86ac358448ec
github.com/influxdata/influxql v0.0.0-20180925231337-1cbfca8e56b6
github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e
github.com/influxdata/roaring v0.4.12 // indirect
github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368
github.com/jessevdk/go-flags v1.4.0
github.com/jsternberg/zap-logfmt v1.2.0

6
go.sum
View File

@ -1,6 +1,4 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
collectd.org v0.3.0 h1:iNBHGw1VvPJxH2B6RiFWFZ+vsjo1lCdRszBeOuwGi00=
collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc=
@ -29,8 +27,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLM
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2 h1:oMCHnXa6CCCafdPDbMh/lWRhRByN0VFLvv+g+ayx1SI=
github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 h1:y4B3+GPxKlrigF1ha5FFErxK+sr6sWxQovRMzwMhejo=
github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
github.com/bouk/httprouter v0.0.0-20160817010721-ee8b3818a7f5 h1:kS0dw4K730x7cxT+bVyTyYJZHuSoH7ofSr/Ijit56Qw=
@ -130,8 +126,6 @@ github.com/influxdata/influxql v0.0.0-20180925231337-1cbfca8e56b6 h1:CFx+pP90q/q
github.com/influxdata/influxql v0.0.0-20180925231337-1cbfca8e56b6/go.mod h1:KpVI7okXjK6PRi3Z5B+mtKZli+R1DnZgb3N+tzevNgo=
github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e h1:/o3vQtpWJhvnIbXley4/jwzzqNeigJK9z+LZcJZ9zfM=
github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE=
github.com/influxdata/roaring v0.4.12 h1:3DzTjKHcXFs4P3D7xRLpCqVrfK6eFRQT0c8BG99M3Ms=
github.com/influxdata/roaring v0.4.12/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE=
github.com/influxdata/tdigest v0.0.0-20180711151920-a7d76c6f093a h1:vMqgISSVkIqWxCIZs8m1L4096temR7IbYyNdMiBxSPA=
github.com/influxdata/tdigest v0.0.0-20180711151920-a7d76c6f093a/go.mod h1:9GkyshztGufsdPQWjH+ifgnIr3xNUL5syI70g2dzU1o=
github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368 h1:+TUUmaFa4YD1Q+7bH9o5NCHQGPMqZCYJiNW6lIIS9z4=

279
toml/toml.go Normal file
View File

@ -0,0 +1,279 @@
// Package toml adds support to marshal and unmarshal types not in the official TOML spec.
package toml
import (
"encoding"
"errors"
"fmt"
"math"
"os"
"os/user"
"reflect"
"strconv"
"strings"
"time"
"unicode"
)
// Duration is a TOML wrapper type for time.Duration.
type Duration time.Duration
// String returns the string representation of the duration.
func (d Duration) String() string {
return time.Duration(d).String()
}
// UnmarshalText parses a TOML value into a duration value.
func (d *Duration) UnmarshalText(text []byte) error {
// Ignore if there is no value set.
if len(text) == 0 {
return nil
}
// Otherwise parse as a duration formatted string.
duration, err := time.ParseDuration(string(text))
if err != nil {
return err
}
// Set duration and return.
*d = Duration(duration)
return nil
}
// MarshalText converts a duration to a string for decoding toml
func (d Duration) MarshalText() (text []byte, err error) {
return []byte(d.String()), nil
}
// Size represents a TOML parseable file size.
// Users can specify size using "k" or "K" for kibibytes, "m" or "M" for mebibytes,
// and "g" or "G" for gibibytes. If a size suffix isn't specified then bytes are assumed.
type Size uint64
// UnmarshalText parses a byte size from text.
func (s *Size) UnmarshalText(text []byte) error {
if len(text) == 0 {
return fmt.Errorf("size was empty")
}
// The multiplier defaults to 1 in case the size has
// no suffix (and is then just raw bytes)
mult := uint64(1)
// Preserve the original text for error messages
sizeText := text
// Parse unit of measure
suffix := text[len(sizeText)-1]
if !unicode.IsDigit(rune(suffix)) {
switch suffix {
case 'k', 'K':
mult = 1 << 10 // KiB
case 'm', 'M':
mult = 1 << 20 // MiB
case 'g', 'G':
mult = 1 << 30 // GiB
default:
return fmt.Errorf("unknown size suffix: %c (expected k, m, or g)", suffix)
}
sizeText = sizeText[:len(sizeText)-1]
}
// Parse numeric portion of value.
size, err := strconv.ParseUint(string(sizeText), 10, 64)
if err != nil {
return fmt.Errorf("invalid size: %s", string(text))
}
if math.MaxUint64/mult < size {
return fmt.Errorf("size would overflow the max size (%d) of a uint: %s", uint64(math.MaxUint64), string(text))
}
size *= mult
*s = Size(size)
return nil
}
type FileMode uint32
func (m *FileMode) UnmarshalText(text []byte) error {
// Ignore if there is no value set.
if len(text) == 0 {
return nil
}
mode, err := strconv.ParseUint(string(text), 8, 32)
if err != nil {
return err
} else if mode == 0 {
return errors.New("file mode cannot be zero")
}
*m = FileMode(mode)
return nil
}
func (m FileMode) MarshalText() (text []byte, err error) {
if m != 0 {
return []byte(fmt.Sprintf("%04o", m)), nil
}
return nil, nil
}
type Group int
func (g *Group) UnmarshalTOML(data interface{}) error {
if grpName, ok := data.(string); ok {
group, err := user.LookupGroup(grpName)
if err != nil {
return err
}
gid, err := strconv.Atoi(group.Gid)
if err != nil {
return err
}
*g = Group(gid)
return nil
} else if gid, ok := data.(int64); ok {
*g = Group(gid)
return nil
}
return errors.New("group must be a name (string) or id (int)")
}
func ApplyEnvOverrides(getenv func(string) string, prefix string, val interface{}) error {
if getenv == nil {
getenv = os.Getenv
}
return applyEnvOverrides(getenv, prefix, reflect.ValueOf(val), "")
}
func applyEnvOverrides(getenv func(string) string, prefix string, spec reflect.Value, structKey string) error {
element := spec
// If spec is a named type and is addressable,
// check the address to see if it implements encoding.TextUnmarshaler.
if spec.Kind() != reflect.Ptr && spec.Type().Name() != "" && spec.CanAddr() {
v := spec.Addr()
if u, ok := v.Interface().(encoding.TextUnmarshaler); ok {
value := getenv(prefix)
return u.UnmarshalText([]byte(value))
}
}
// If we have a pointer, dereference it
if spec.Kind() == reflect.Ptr {
element = spec.Elem()
}
value := getenv(prefix)
switch element.Kind() {
case reflect.String:
if len(value) == 0 {
return nil
}
element.SetString(value)
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
intValue, err := strconv.ParseInt(value, 0, element.Type().Bits())
if err != nil {
return fmt.Errorf("failed to apply %v to %v using type %v and value '%v': %s", prefix, structKey, element.Type().String(), value, err)
}
element.SetInt(intValue)
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
intValue, err := strconv.ParseUint(value, 0, element.Type().Bits())
if err != nil {
return fmt.Errorf("failed to apply %v to %v using type %v and value '%v': %s", prefix, structKey, element.Type().String(), value, err)
}
element.SetUint(intValue)
case reflect.Bool:
boolValue, err := strconv.ParseBool(value)
if err != nil {
return fmt.Errorf("failed to apply %v to %v using type %v and value '%v': %s", prefix, structKey, element.Type().String(), value, err)
}
element.SetBool(boolValue)
case reflect.Float32, reflect.Float64:
floatValue, err := strconv.ParseFloat(value, element.Type().Bits())
if err != nil {
return fmt.Errorf("failed to apply %v to %v using type %v and value '%v': %s", prefix, structKey, element.Type().String(), value, err)
}
element.SetFloat(floatValue)
case reflect.Slice:
// If the type is s slice, apply to each using the index as a suffix, e.g. GRAPHITE_0, GRAPHITE_0_TEMPLATES_0 or GRAPHITE_0_TEMPLATES="item1,item2"
for j := 0; j < element.Len(); j++ {
f := element.Index(j)
if err := applyEnvOverrides(getenv, prefix, f, structKey); err != nil {
return err
}
if err := applyEnvOverrides(getenv, fmt.Sprintf("%s_%d", prefix, j), f, structKey); err != nil {
return err
}
}
// If the type is s slice but have value not parsed as slice e.g. GRAPHITE_0_TEMPLATES="item1,item2"
if element.Len() == 0 && len(value) > 0 {
rules := strings.Split(value, ",")
for _, rule := range rules {
element.Set(reflect.Append(element, reflect.ValueOf(rule)))
}
}
case reflect.Struct:
typeOfSpec := element.Type()
for i := 0; i < element.NumField(); i++ {
field := element.Field(i)
// Skip any fields that we cannot set
if !field.CanSet() && field.Kind() != reflect.Slice {
continue
}
structField := typeOfSpec.Field(i)
fieldName := structField.Name
configName := structField.Tag.Get("toml")
if configName == "-" {
// Skip fields with tag `toml:"-"`.
continue
}
if configName == "" && structField.Anonymous {
// Embedded field without a toml tag.
// Don't modify prefix.
if err := applyEnvOverrides(getenv, prefix, field, fieldName); err != nil {
return err
}
continue
}
// Replace hyphens with underscores to avoid issues with shells
configName = strings.Replace(configName, "-", "_", -1)
envKey := strings.ToUpper(configName)
if prefix != "" {
envKey = strings.ToUpper(fmt.Sprintf("%s_%s", prefix, configName))
}
// If it's a sub-config, recursively apply
if field.Kind() == reflect.Struct || field.Kind() == reflect.Ptr ||
field.Kind() == reflect.Slice || field.Kind() == reflect.Array {
if err := applyEnvOverrides(getenv, envKey, field, fieldName); err != nil {
return err
}
continue
}
value := getenv(envKey)
// Skip any fields we don't have a value to set
if len(value) == 0 {
continue
}
if err := applyEnvOverrides(getenv, envKey, field, fieldName); err != nil {
return err
}
}
}
return nil
}

244
toml/toml_test.go Normal file
View File

@ -0,0 +1,244 @@
package toml_test
import (
"fmt"
"math"
"os/user"
"runtime"
"strconv"
"testing"
"time"
"github.com/google/go-cmp/cmp"
itoml "github.com/influxdata/platform/toml"
)
func TestSize_UnmarshalText(t *testing.T) {
var s itoml.Size
for _, test := range []struct {
str string
want uint64
}{
{"1", 1},
{"10", 10},
{"100", 100},
{"1k", 1 << 10},
{"10k", 10 << 10},
{"100k", 100 << 10},
{"1K", 1 << 10},
{"10K", 10 << 10},
{"100K", 100 << 10},
{"1m", 1 << 20},
{"10m", 10 << 20},
{"100m", 100 << 20},
{"1M", 1 << 20},
{"10M", 10 << 20},
{"100M", 100 << 20},
{"1g", 1 << 30},
{"1G", 1 << 30},
{fmt.Sprint(uint64(math.MaxUint64) - 1), math.MaxUint64 - 1},
} {
if err := s.UnmarshalText([]byte(test.str)); err != nil {
t.Fatalf("unexpected error: %s", err)
}
if s != itoml.Size(test.want) {
t.Fatalf("wanted: %d got: %d", test.want, s)
}
}
for _, str := range []string{
fmt.Sprintf("%dk", uint64(math.MaxUint64-1)),
"10000000000000000000g",
"abcdef",
"1KB",
"√m",
"a1",
"",
} {
if err := s.UnmarshalText([]byte(str)); err == nil {
t.Fatalf("input should have failed: %s", str)
}
}
}
func TestFileMode_MarshalText(t *testing.T) {
for _, test := range []struct {
mode int
want string
}{
{mode: 0755, want: `0755`},
{mode: 0777, want: `0777`},
{mode: 01777, want: `1777`},
} {
mode := itoml.FileMode(test.mode)
if got, err := mode.MarshalText(); err != nil {
t.Errorf("unexpected error: %s", err)
} else if test.want != string(got) {
t.Errorf("wanted: %v got: %v", test.want, string(got))
}
}
}
func TestFileMode_UnmarshalText(t *testing.T) {
for _, test := range []struct {
str string
want uint32
}{
{str: ``, want: 0},
{str: `0777`, want: 0777},
{str: `777`, want: 0777},
{str: `1777`, want: 01777},
{str: `0755`, want: 0755},
} {
var mode itoml.FileMode
if err := mode.UnmarshalText([]byte(test.str)); err != nil {
t.Errorf("unexpected error: %s", err)
} else if mode != itoml.FileMode(test.want) {
t.Errorf("wanted: %04o got: %04o", test.want, mode)
}
}
}
func TestGroup_UnmarshalTOML(t *testing.T) {
// Skip this test on windows since it does not support setting the group anyway.
if runtime.GOOS == "windows" {
t.Skip("unsupported on windows")
}
// Find the current user ID so we can use that group name.
u, err := user.Current()
if err != nil {
t.Skipf("unable to find the current user: %s", err)
}
// Lookup the group by the group id.
gr, err := user.LookupGroupId(u.Gid)
if err == nil {
var group itoml.Group
if err := group.UnmarshalTOML(gr.Name); err != nil {
t.Fatalf("unexpected error: %s", err)
} else if got, want := u.Gid, strconv.Itoa(int(group)); got != want {
t.Fatalf("unexpected group id: %s != %s", got, want)
}
}
// Attempt to convert the group to an integer so we can test reading an integer.
gid, err := strconv.Atoi(u.Gid)
if err != nil {
t.Fatalf("group id is not an integer: %s", err)
}
var group itoml.Group
if err := group.UnmarshalTOML(int64(gid)); err != nil {
t.Fatalf("unexpected error: %s", err)
} else if int(group) != gid {
t.Fatalf("unexpected group id: %d != %d", gid, int(group))
}
}
func TestConfig_Encode(t *testing.T) {
t.Skip("TODO(jsternberg): rewrite this test to use something from platform")
//var c run.Config
//c.Coordinator.WriteTimeout = itoml.Duration(time.Minute)
//buf := new(bytes.Buffer)
//if err := toml.NewEncoder(buf).Encode(&c); err != nil {
// t.Fatal("Failed to encode: ", err)
//}
//got, search := buf.String(), `write-timeout = "1m0s"`
//if !strings.Contains(got, search) {
// t.Fatalf("Encoding config failed.\nfailed to find %s in:\n%s\n", search, got)
//}
}
func TestEnvOverride_Builtins(t *testing.T) {
envMap := map[string]string{
"X_STRING": "a string",
"X_DURATION": "1m1s",
"X_INT": "1",
"X_INT8": "2",
"X_INT16": "3",
"X_INT32": "4",
"X_INT64": "5",
"X_UINT": "6",
"X_UINT8": "7",
"X_UINT16": "8",
"X_UINT32": "9",
"X_UINT64": "10",
"X_BOOL": "true",
"X_FLOAT32": "11.5",
"X_FLOAT64": "12.5",
"X_NESTED_STRING": "a nested string",
"X_NESTED_INT": "13",
"X_ES": "an embedded string",
"X__": "-1", // This value should not be applied to the "ignored" field with toml tag -.
}
env := func(s string) string {
return envMap[s]
}
type nested struct {
Str string `toml:"string"`
Int int `toml:"int"`
}
type Embedded struct {
ES string `toml:"es"`
}
type all struct {
Str string `toml:"string"`
Dur itoml.Duration `toml:"duration"`
Int int `toml:"int"`
Int8 int8 `toml:"int8"`
Int16 int16 `toml:"int16"`
Int32 int32 `toml:"int32"`
Int64 int64 `toml:"int64"`
Uint uint `toml:"uint"`
Uint8 uint8 `toml:"uint8"`
Uint16 uint16 `toml:"uint16"`
Uint32 uint32 `toml:"uint32"`
Uint64 uint64 `toml:"uint64"`
Bool bool `toml:"bool"`
Float32 float32 `toml:"float32"`
Float64 float64 `toml:"float64"`
Nested nested `toml:"nested"`
Embedded
Ignored int `toml:"-"`
}
var got all
if err := itoml.ApplyEnvOverrides(env, "X", &got); err != nil {
t.Fatal(err)
}
exp := all{
Str: "a string",
Dur: itoml.Duration(time.Minute + time.Second),
Int: 1,
Int8: 2,
Int16: 3,
Int32: 4,
Int64: 5,
Uint: 6,
Uint8: 7,
Uint16: 8,
Uint32: 9,
Uint64: 10,
Bool: true,
Float32: 11.5,
Float64: 12.5,
Nested: nested{
Str: "a nested string",
Int: 13,
},
Embedded: Embedded{
ES: "an embedded string",
},
Ignored: 0,
}
if diff := cmp.Diff(got, exp); diff != "" {
t.Fatal(diff)
}
}

2
tsdb/config.go
View File

@ -6,7 +6,7 @@ import (
"github.com/influxdata/influxdb/monitor/diagnostics"
"github.com/influxdata/influxdb/query"
"github.com/influxdata/influxdb/toml"
"github.com/influxdata/platform/toml"
"github.com/influxdata/platform/tsdb/defaults"
)

2
tsdb/tsi1/config.go
View File

@ -1,6 +1,6 @@
package tsi1
import "github.com/influxdata/influxdb/toml"
import "github.com/influxdata/platform/toml"
// DefaultMaxIndexLogFileSize is the default threshold, in bytes, when an index
// write-ahead log file will compact into an index file.