Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #9043 from influxdata/js-8965-handle-utf16-bom 

Handle utf16 files when reading the configuration file
pull/9054/head
Jonathan A. Sternberg 2017-10-31 14:16:09 -05:00 committed by GitHub
parent 9a43c14653 9dc60962c7
commit fade2ba9a8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 204 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -90,6 +90,7 @@
- [#9005](https://github.com/influxdata/influxdb/pull/9005): Return `query.ErrQueryInterrupted` for successful read on `InterruptCh`.
- [#8989](https://github.com/influxdata/influxdb/issues/8989): Fix race inside Measurement index.
- [#8819](https://github.com/influxdata/influxdb/issues/8819): Ensure retention service always removes local shards.
- [#8965](https://github.com/influxdata/influxdb/issues/8965): Handle utf16 files when reading the configuration file.
## v1.3.7 [2017-10-26]

1
Godeps
View File

@ -26,3 +26,4 @@ github.com/uber-go/zap fbae0281ffd546fa6d1959fec6075ac5da7fb577
github.com/xlab/treeprint 06dfc6fa17cdde904617990a0c2d89e3e332dbb3
golang.org/x/crypto 9477e0b78b9ac3d0b03822fd95422e2fe07627cd
golang.org/x/sys 062cd7e4e68206d8bab9b18396626e855c992658
golang.org/x/text a71fd10341b064c10f4a81ceac72bcf70f26ea34

1
LICENSE_OF_DEPENDENCIES.md
View File

@ -24,5 +24,6 @@
- github.com/uber-go/atomic [MIT LICENSE](https://github.com/uber-go/atomic/blob/master/LICENSE.txt)
- github.com/uber-go/zap [MIT LICENSE](https://github.com/uber-go/zap/blob/master/LICENSE.txt)
- golang.org/x/crypto [BSD LICENSE](https://github.com/golang/crypto/blob/master/LICENSE)
- golang.org/x/text [BSD LICENSE](https://github.com/golang/text/blob/master/LICENSE)
- jquery 2.1.4 [MIT LICENSE](https://github.com/jquery/jquery/blob/master/LICENSE.txt)
- github.com/xlab/treeprint [MIT LICENSE](https://github.com/xlab/treeprint/blob/master/LICENSE)

21
cmd/influxd/run/config.go
View File

@ -1,7 +1,6 @@
package run
import (
"bytes"
"fmt"
"io/ioutil"
"log"
@ -30,6 +29,8 @@ import (
"github.com/influxdata/influxdb/services/subscriber"
"github.com/influxdata/influxdb/services/udp"
"github.com/influxdata/influxdb/tsdb"
"golang.org/x/text/encoding/unicode"
"golang.org/x/text/transform"
)
const (
@ -110,20 +111,22 @@ func NewDemoConfig() (*Config, error) {
return c, nil
}
// trimBOM trims the Byte-Order-Marks from the beginning of the file.
// This is for Windows compatability only.
// See https://github.com/influxdata/telegraf/issues/1378.
func trimBOM(f []byte) []byte {
return bytes.TrimPrefix(f, []byte("\xef\xbb\xbf"))
}
// FromTomlFile loads the config from a TOML file.
func (c *Config) FromTomlFile(fpath string) error {
bs, err := ioutil.ReadFile(fpath)
if err != nil {
return err
}
bs = trimBOM(bs)
// Handle any potential Byte-Order-Marks that may be in the config file.
// This is for Windows compatibility only.
// See https://github.com/influxdata/telegraf/issues/1378 and
// https://github.com/influxdata/influxdb/issues/8965.
bom := unicode.BOMOverride(transform.Nop)
bs, _, err = transform.Bytes(bom, bs)
if err != nil {
return err
}
return c.FromToml(string(bs))
}

189
cmd/influxd/run/config_test.go
View File

@ -2,11 +2,15 @@ package run_test
import (
"fmt"
"io"
"io/ioutil"
"os"
"testing"
"github.com/BurntSushi/toml"
"github.com/influxdata/influxdb/cmd/influxd/run"
"golang.org/x/text/encoding/unicode"
"golang.org/x/text/transform"
)
// Ensure the configuration can be parsed.
@ -310,3 +314,188 @@ func TestConfig_InvalidSubsections(t *testing.T) {
}
}
}
// Ensure the configuration can be parsed when a Byte-Order-Mark is present.
func TestConfig_Parse_UTF8_ByteOrderMark(t *testing.T) {
// Parse configuration.
var c run.Config
f, err := ioutil.TempFile("", "influxd")
if err != nil {
t.Fatal(err)
}
defer os.Remove(f.Name())
f.WriteString("\ufeff")
f.WriteString(`
[meta]
dir = "/tmp/meta"
[data]
dir = "/tmp/data"
[coordinator]
[http]
bind-address = ":8087"
[[graphite]]
protocol = "udp"
[[graphite]]
protocol = "tcp"
[[collectd]]
bind-address = ":1000"
[[collectd]]
bind-address = ":1010"
[[opentsdb]]
bind-address = ":2000"
[[opentsdb]]
bind-address = ":2010"
[[opentsdb]]
bind-address = ":2020"
[[udp]]
bind-address = ":4444"
[monitoring]
enabled = true
[subscriber]
enabled = true
[continuous_queries]
enabled = true
`)
if err := c.FromTomlFile(f.Name()); err != nil {
t.Fatal(err)
}
// Validate configuration.
if c.Meta.Dir != "/tmp/meta" {
t.Fatalf("unexpected meta dir: %s", c.Meta.Dir)
} else if c.Data.Dir != "/tmp/data" {
t.Fatalf("unexpected data dir: %s", c.Data.Dir)
} else if c.HTTPD.BindAddress != ":8087" {
t.Fatalf("unexpected api bind address: %s", c.HTTPD.BindAddress)
} else if len(c.GraphiteInputs) != 2 {
t.Fatalf("unexpected graphiteInputs count: %d", len(c.GraphiteInputs))
} else if c.GraphiteInputs[0].Protocol != "udp" {
t.Fatalf("unexpected graphite protocol(0): %s", c.GraphiteInputs[0].Protocol)
} else if c.GraphiteInputs[1].Protocol != "tcp" {
t.Fatalf("unexpected graphite protocol(1): %s", c.GraphiteInputs[1].Protocol)
} else if c.CollectdInputs[0].BindAddress != ":1000" {
t.Fatalf("unexpected collectd bind address: %s", c.CollectdInputs[0].BindAddress)
} else if c.CollectdInputs[1].BindAddress != ":1010" {
t.Fatalf("unexpected collectd bind address: %s", c.CollectdInputs[1].BindAddress)
} else if c.OpenTSDBInputs[0].BindAddress != ":2000" {
t.Fatalf("unexpected opentsdb bind address: %s", c.OpenTSDBInputs[0].BindAddress)
} else if c.OpenTSDBInputs[1].BindAddress != ":2010" {
t.Fatalf("unexpected opentsdb bind address: %s", c.OpenTSDBInputs[1].BindAddress)
} else if c.OpenTSDBInputs[2].BindAddress != ":2020" {
t.Fatalf("unexpected opentsdb bind address: %s", c.OpenTSDBInputs[2].BindAddress)
} else if c.UDPInputs[0].BindAddress != ":4444" {
t.Fatalf("unexpected udp bind address: %s", c.UDPInputs[0].BindAddress)
} else if c.Subscriber.Enabled != true {
t.Fatalf("unexpected subscriber enabled: %v", c.Subscriber.Enabled)
} else if c.ContinuousQuery.Enabled != true {
t.Fatalf("unexpected continuous query enabled: %v", c.ContinuousQuery.Enabled)
}
}
// Ensure the configuration can be parsed when a Byte-Order-Mark is present.
func TestConfig_Parse_UTF16_ByteOrderMark(t *testing.T) {
// Parse configuration.
var c run.Config
f, err := ioutil.TempFile("", "influxd")
if err != nil {
t.Fatal(err)
}
defer os.Remove(f.Name())
utf16 := unicode.UTF16(unicode.BigEndian, unicode.UseBOM)
w := transform.NewWriter(f, utf16.NewEncoder())
io.WriteString(w, `
[meta]
dir = "/tmp/meta"
[data]
dir = "/tmp/data"
[coordinator]
[http]
bind-address = ":8087"
[[graphite]]
protocol = "udp"
[[graphite]]
protocol = "tcp"
[[collectd]]
bind-address = ":1000"
[[collectd]]
bind-address = ":1010"
[[opentsdb]]
bind-address = ":2000"
[[opentsdb]]
bind-address = ":2010"
[[opentsdb]]
bind-address = ":2020"
[[udp]]
bind-address = ":4444"
[monitoring]
enabled = true
[subscriber]
enabled = true
[continuous_queries]
enabled = true
`)
if err := c.FromTomlFile(f.Name()); err != nil {
t.Fatal(err)
}
// Validate configuration.
if c.Meta.Dir != "/tmp/meta" {
t.Fatalf("unexpected meta dir: %s", c.Meta.Dir)
} else if c.Data.Dir != "/tmp/data" {
t.Fatalf("unexpected data dir: %s", c.Data.Dir)
} else if c.HTTPD.BindAddress != ":8087" {
t.Fatalf("unexpected api bind address: %s", c.HTTPD.BindAddress)
} else if len(c.GraphiteInputs) != 2 {
t.Fatalf("unexpected graphiteInputs count: %d", len(c.GraphiteInputs))
} else if c.GraphiteInputs[0].Protocol != "udp" {
t.Fatalf("unexpected graphite protocol(0): %s", c.GraphiteInputs[0].Protocol)
} else if c.GraphiteInputs[1].Protocol != "tcp" {
t.Fatalf("unexpected graphite protocol(1): %s", c.GraphiteInputs[1].Protocol)
} else if c.CollectdInputs[0].BindAddress != ":1000" {
t.Fatalf("unexpected collectd bind address: %s", c.CollectdInputs[0].BindAddress)
} else if c.CollectdInputs[1].BindAddress != ":1010" {
t.Fatalf("unexpected collectd bind address: %s", c.CollectdInputs[1].BindAddress)
} else if c.OpenTSDBInputs[0].BindAddress != ":2000" {
t.Fatalf("unexpected opentsdb bind address: %s", c.OpenTSDBInputs[0].BindAddress)
} else if c.OpenTSDBInputs[1].BindAddress != ":2010" {
t.Fatalf("unexpected opentsdb bind address: %s", c.OpenTSDBInputs[1].BindAddress)
} else if c.OpenTSDBInputs[2].BindAddress != ":2020" {
t.Fatalf("unexpected opentsdb bind address: %s", c.OpenTSDBInputs[2].BindAddress)
} else if c.UDPInputs[0].BindAddress != ":4444" {
t.Fatalf("unexpected udp bind address: %s", c.UDPInputs[0].BindAddress)
} else if c.Subscriber.Enabled != true {
t.Fatalf("unexpected subscriber enabled: %v", c.Subscriber.Enabled)
} else if c.ContinuousQuery.Enabled != true {
t.Fatalf("unexpected continuous query enabled: %v", c.ContinuousQuery.Enabled)
}
}