feat(upgrade): upgrade config file

pull/19685/head
Ales Pour 2020-10-02 11:30:39 +02:00 committed by Stuart Carnie
parent feb41819ec
commit 1e4650913d
2 changed files with 695 additions and 5 deletions

View File

@ -1,12 +1,208 @@
package upgrade
import (
"errors"
// Configuration file upgrade implementation.
// The strategy is to transform only those entries for which rule exists.
import (
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/BurntSushi/toml"
"go.uber.org/zap"
"golang.org/x/text/encoding/unicode"
"golang.org/x/text/transform"
)
// upgradeConfig backups existing config file and updates it with upgraded config.
func upgradeConfig(configFile string, targetOptions optionsV2, log *zap.Logger) (*configV1, error) {
return nil, errors.New("not implemented")
// configMapRules is a map of transformation rules
var configMapRules = map[string]string{
"reporting-disabled": "reporting-disabled",
"data.dir": "engine-path",
"data.wal-fsync-delay": "storage-wal-fsync-delay",
"data.validate-keys": "storage-validate-keys",
"data.cache-max-memory-size": "storage-cache-max-memory-size",
"data.cache-snapshot-memory-size": "storage-cache-snapshot-memory-size",
"data.cache-snapshot-write-cold-duration": "storage-cache-snapshot-write-cold-duration",
"data.compact-full-write-cold-duration": "storage-compact-full-write-cold-duration",
"data.compact-throughput-burst": "storage-compact-throughput-burst",
"data.max-concurrent-compactions": "storage-max-concurrent-compactions",
"data.max-index-log-file-size": "storage-max-index-log-file-size",
"data.series-id-set-cache-size": "storage-series-id-set-cache-size",
"data.series-file-max-concurrent-snapshot-compactions": "storage-series-file-max-concurrent-snapshot-compactions",
"data.tsm-use-madv-willneed": "storage-tsm-use-madv-willneed",
"retention.check-interval": "storage-retention-check-interval",
"shard-precreation.check-interval": "storage-shard-precreator-check-interval",
"shard-precreation.advance-period": "storage-shard-precreator-advance-period",
"coordinator.max-concurrent-queries": "query-concurrency",
"coordinator.max-select-point": "influxql-max-select-point",
"coordinator.max-select-series": "influxql-max-select-series",
"coordinator.max-select-buckets": "influxql-max-select-buckets",
"logging.level": "log-level",
"http.bind-address": "http-bind-address",
"http.https-certificate": "tls-cert",
"http.https-private-key": "tls-key",
}
// upgradeConfig upgrades existing 1.x (ie. typically influxdb.conf) configuration file to 2.x influxdb.toml file.
func upgradeConfig(configFile string, targetOptions optionsV2, log *zap.Logger) (*configV1, error) {
// create and initialize helper
cu := &configUpgrader{
rules: configMapRules,
log: log,
}
// load 1.x config content into byte array
bs, err := cu.load(configFile)
if err != nil {
return nil, err
}
// parse it into simplified v1 config used as return value
var configV1 configV1
_, err = toml.Decode(string(bs), &configV1)
if err != nil {
return nil, err
}
// parse into a generic config map
var cAny map[string]interface{}
_, err = toml.Decode(string(bs), &cAny)
if err != nil {
return nil, err
}
// transform the config according to rules
cTransformed := cu.transform(cAny)
if err != nil {
return nil, err
}
// update new config with upgrade command options
cu.updateV2Config(cTransformed, targetOptions)
// backup existing 2.x config if already exists (it should not)
configFileV2 := strings.TrimSuffix(configFile, filepath.Ext(configFile)) + ".toml"
err = cu.backupIfExists(configFileV2)
if err != nil {
return nil, err
}
// save new config
err = cu.save(cTransformed, configFileV2)
if err != nil {
return nil, err
}
log.Info("Config file upgraded.",
zap.String("1.x config", configFile),
zap.String("2.x config", configFileV2))
return &configV1, nil
}
// configUpgrader is a helper used by `upgrade-config` command.
type configUpgrader struct {
rules map[string]string
log *zap.Logger
}
func (cu *configUpgrader) backupIfExists(path string) error {
if _, err := os.Stat(path); os.IsNotExist(err) {
return nil
}
source, err := os.Open(path)
if err != nil {
return err
}
defer source.Close()
backupFile := path + "~"
if _, err := os.Stat(backupFile); !os.IsNotExist(err) {
errMsg := fmt.Sprintf("upgrade: config file backup %s already exist", backupFile)
return errors.New(errMsg)
}
destination, err := os.Create(backupFile)
if err != nil {
return err
}
defer destination.Close()
_, err = io.Copy(destination, source)
return err
}
func (cu *configUpgrader) updateV2Config(config map[string]interface{}, targetOptions optionsV2) {
if targetOptions.enginePath != "" {
config["engine-path"] = targetOptions.enginePath
}
if targetOptions.boltPath != "" {
config["bolt-path"] = targetOptions.boltPath
}
}
func (cu *configUpgrader) load(path string) ([]byte, error) {
bs, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
// From master-1.x/cmd/influxd/run/config.go:
// Handle any potential Byte-Order-Marks that may be in the config file.
// This is for Windows compatibility only.
// See https://github.com/influxdata/telegraf/issues/1378 and
// https://github.com/influxdata/influxdb/issues/8965.
bom := unicode.BOMOverride(transform.Nop)
bs, _, err = transform.Bytes(bom, bs)
return bs, err
}
func (cu *configUpgrader) save(config map[string]interface{}, path string) error {
buf := new(bytes.Buffer)
if err := toml.NewEncoder(buf).Encode(&config); err != nil {
return err
}
return ioutil.WriteFile(path, buf.Bytes(), 0666)
}
// Credits: @rogpeppe (Roger Peppe)
func (cu *configUpgrader) transform(x map[string]interface{}) map[string]interface{} {
res := make(map[string]interface{})
for old, new := range cu.rules {
val, ok := cu.lookup(x, old)
if ok {
res[new] = val
}
}
return res
}
func (cu *configUpgrader) lookup(x map[string]interface{}, path string) (interface{}, bool) {
for {
elem := path
rest := ""
if i := strings.Index(path, "."); i != -1 {
elem, rest = path[0:i], path[i+1:]
}
val, ok := x[elem]
if rest == "" {
return val, ok
}
child, ok := val.(map[string]interface{})
if !ok {
return nil, false
}
path, x = rest, child
}
}

View File

@ -0,0 +1,494 @@
package upgrade
import (
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"github.com/BurntSushi/toml"
"github.com/google/go-cmp/cmp"
"go.uber.org/zap"
)
func testCreateTempFile(t *testing.T, pattern, content string) string {
f, err := ioutil.TempFile("", pattern)
if err != nil {
t.Fatal(err)
}
_, err = f.WriteString(content)
if err != nil {
t.Fatal(err)
}
if err = f.Close(); err != nil {
t.Fatal(err)
}
return f.Name()
}
func TestConfigUpgrade(t *testing.T) {
targetOtions := optionsV2{
boltPath: "/db/.influxdbv2/influxd.bolt",
enginePath: "/db/.influxdbv2/engine",
}
var typicalRetval, emptyRetval configV1
_, err := toml.Decode("[meta]\ndir=\"/var/lib/influxdb/meta\"\n[data]\ndir=\"/var/lib/influxdb/data\"\nwal-dir=\"/var/lib/influxdb/wal\"\n",
&typicalRetval)
if err != nil {
t.Fatal(err)
}
type testCase struct {
name string
config1x string
config2x string
retval *configV1
}
var testCases = []testCase{
{
name: "minimal",
config1x: testConfigV1minimal,
config2x: testConfigV2minimal,
retval: &typicalRetval,
},
{
name: "default",
config1x: testConfigV1default,
config2x: testConfigV2default,
retval: &typicalRetval,
},
{
name: "empty",
config1x: testConfigV1empty,
config2x: testConfigV2empty,
retval: &emptyRetval,
},
{
name: "obsolete / arrays",
config1x: testConfigV1obsoleteArrays,
config2x: testConfigV2obsoleteArrays,
retval: &typicalRetval,
},
}
for _, tc := range testCases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
configFile := testCreateTempFile(t, "influxdb-*.conf", tc.config1x)
configFileV2 := strings.TrimSuffix(configFile, filepath.Ext(configFile)) + ".toml"
defer func() {
os.Remove(configFile)
os.Remove(configFileV2)
}()
retval, err := upgradeConfig(configFile, targetOtions, zap.NewNop())
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(tc.retval, retval); diff != "" {
t.Fatal(diff)
}
var actual, expected map[string]interface{}
if _, err = toml.Decode(tc.config2x, &expected); err != nil {
t.Fatal(err)
}
if _, err = toml.DecodeFile(configFileV2, &actual); err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestConfigUpgradeWithBackup(t *testing.T) {
targetOtions := optionsV2{
boltPath: "/db/.influxdbv2/influxd.bolt",
enginePath: "/db/.influxdbv2/engine",
}
type testCase struct {
name string
config1x string
config2x string
}
var testCases = []testCase{
{
name: "default",
config1x: testConfigV1default,
config2x: testConfigV2default,
},
}
for _, tc := range testCases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
configFile := testCreateTempFile(t, "influxdb-*.conf", tc.config1x)
configFileV2 := strings.TrimSuffix(configFile, filepath.Ext(configFile)) + ".toml"
configFileV2Backup := strings.TrimSuffix(configFile, filepath.Ext(configFile)) + ".toml~"
defer func() {
os.Remove(configFile)
os.Remove(configFileV2)
os.Remove(configFileV2Backup)
}()
// upgrade
_, err := upgradeConfig(configFile, targetOtions, zap.NewNop())
if err != nil {
t.Fatal(err)
}
// upgrade again, backup of existing newly created config should get created
_, err = upgradeConfig(configFile, targetOtions, zap.NewNop())
if err != nil {
t.Fatal(err)
}
// validate new config
var actual, expected map[string]interface{}
if _, err = toml.Decode(tc.config2x, &expected); err != nil {
t.Fatal(err)
}
if _, err = toml.DecodeFile(configFileV2, &actual); err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(expected, actual); diff != "" {
t.Fatal(diff)
}
// validate backup of the new config, it should exists and be the same
var actualBackup, expectedBackup map[string]interface{}
if _, err = toml.Decode(tc.config2x, &expectedBackup); err != nil {
t.Fatal(err)
}
if _, err = toml.DecodeFile(configFileV2Backup, &actualBackup); err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(expectedBackup, actualBackup); diff != "" {
t.Fatal(diff)
}
})
}
}
func TestConfigUpgradeFileNotExists(t *testing.T) {
targetOtions := optionsV2{
}
configFile := "/there/is/no/such/path/influxdb.conf"
// try upgrade
_, err := upgradeConfig(configFile, targetOtions, zap.NewNop())
if err == nil {
t.Fatal("error expected")
}
}
// 1.x test configs
var testConfigV1minimal = `### Welcome to the InfluxDB configuration file.
# Change this option to true to disable reporting.
reporting-disabled = false
# Bind address to use for the RPC service for backup and restore.
bind-address = "127.0.0.1:8088"
[meta]
dir = "/var/lib/influxdb/meta"
[data]
dir = "/var/lib/influxdb/data"
wal-dir = "/var/lib/influxdb/wal"
wal-fsync-delay = "100s"
index-version = "inmem"
[coordinator]
max-select-point = 0
[retention]
check-interval = "30m"
[shard-precreation]
check-interval = "5m"
[monitor]
store-enabled = true
[http]
flux-enabled = false
bind-address = ":8086"
https-certificate = "/etc/ssl/influxdb.pem"
https-private-key = "/etc/ssl/influxdb-key.pem"
[logging]
level = "debug"
[subscriber]
[[graphite]]
[[collectd]]
[[opentsdb]]
[[udp]]
[continuous_queries]
query-stats-enabled = true
[tls]
`
var testConfigV1default = `reporting-disabled = false
bind-address = "127.0.0.1:8088"
[meta]
dir = "/var/lib/influxdb/meta"
retention-autocreate = true
logging-enabled = true
[data]
dir = "/var/lib/influxdb/data"
wal-dir = "/var/lib/influxdb/wal"
wal-fsync-delay = "0s"
validate-keys = false
index-version = "inmem"
query-log-enabled = true
cache-max-memory-size = 1073741824
cache-snapshot-memory-size = 26214400
cache-snapshot-write-cold-duration = "10m0s"
compact-full-write-cold-duration = "4h0m0s"
compact-throughput = 50331648
compact-throughput-burst = 50331648
max-series-per-database = 1000000
max-values-per-tag = 100000
max-concurrent-compactions = 0
max-index-log-file-size = 1048576
series-id-set-cache-size = 100
series-file-max-concurrent-snapshot-compactions = 0
trace-logging-enabled = false
tsm-use-madv-willneed = false
[coordinator]
write-timeout = "10s"
max-concurrent-queries = 0
query-timeout = "0s"
log-queries-after = "0s"
max-select-point = 0
max-select-series = 0
max-select-buckets = 0
[retention]
enabled = true
check-interval = "30m0s"
[shard-precreation]
enabled = true
check-interval = "10m0s"
advance-period = "30m0s"
[monitor]
store-enabled = true
store-database = "_internal"
store-interval = "10s"
[subscriber]
enabled = true
http-timeout = "30s"
insecure-skip-verify = false
ca-certs = ""
write-concurrency = 40
write-buffer-size = 1000
[http]
enabled = true
bind-address = ":8086"
auth-enabled = false
log-enabled = true
suppress-write-log = false
write-tracing = false
flux-enabled = false
flux-log-enabled = false
pprof-enabled = true
pprof-auth-enabled = false
debug-pprof-enabled = false
ping-auth-enabled = false
prom-read-auth-enabled = false
https-enabled = false
https-certificate = "/etc/ssl/influxdb.pem"
https-private-key = ""
max-row-limit = 0
max-connection-limit = 0
shared-secret = ""
realm = "InfluxDB"
unix-socket-enabled = false
unix-socket-permissions = "0777"
bind-socket = "/var/run/influxdb.sock"
max-body-size = 25000000
access-log-path = ""
max-concurrent-write-limit = 0
max-enqueued-write-limit = 0
enqueued-write-timeout = 30000000000
[logging]
format = "auto"
level = "info"
suppress-logo = false
[[graphite]]
enabled = false
bind-address = ":2003"
database = "graphite"
retention-policy = ""
protocol = "tcp"
batch-size = 5000
batch-pending = 10
batch-timeout = "1s"
consistency-level = "one"
separator = "."
udp-read-buffer = 0
[[collectd]]
enabled = false
bind-address = ":25826"
database = "collectd"
retention-policy = ""
batch-size = 5000
batch-pending = 10
batch-timeout = "10s"
read-buffer = 0
typesdb = "/usr/share/collectd/types.db"
security-level = "none"
auth-file = "/etc/collectd/auth_file"
parse-multivalue-plugin = "split"
[[opentsdb]]
enabled = false
bind-address = ":4242"
database = "opentsdb"
retention-policy = ""
consistency-level = "one"
tls-enabled = false
certificate = "/etc/ssl/influxdb.pem"
batch-size = 1000
batch-pending = 5
batch-timeout = "1s"
log-point-errors = true
[[udp]]
enabled = false
bind-address = ":8089"
database = "udp"
retention-policy = ""
batch-size = 5000
batch-pending = 10
read-buffer = 0
batch-timeout = "1s"
precision = ""
[continuous_queries]
log-enabled = true
enabled = true
query-stats-enabled = false
run-interval = "1s"
[tls]
min-version = "tls1.2"
max-version = "tls1.3"
`
var testConfigV1obsoleteArrays =`
reporting-disabled = true
[meta]
dir = "/var/lib/influxdb/meta"
[data]
dir = "/var/lib/influxdb/data"
wal-dir = "/var/lib/influxdb/wal"
[[udp]]
enabled = false
bind-address = ":8089"
database = "udp"
retention-policy = ""
batch-size = 5000
batch-pending = 10
read-buffer = 0
batch-timeout = "1s"
precision = ""
[[udp]]
enabled = false
bind-address = ":8090"
database = "udp2"
retention-policy = ""
batch-size = 5000
batch-pending = 10
read-buffer = 0
batch-timeout = "1s"
precision = ""
`
var testConfigV1empty = `
`
// 2.x test configs
var testConfigV2minimal = `reporting-disabled = false
bolt-path = "/db/.influxdbv2/influxd.bolt"
engine-path = "/db/.influxdbv2/engine"
http-bind-address = ":8086"
influxql-max-select-point = 0
log-level = "debug"
storage-retention-check-interval = "30m"
storage-shard-precreator-check-interval = "5m"
storage-wal-fsync-delay = "100s"
tls-cert = "/etc/ssl/influxdb.pem"
tls-key = "/etc/ssl/influxdb-key.pem"
`
var testConfigV2default = `reporting-disabled = false
bolt-path = "/db/.influxdbv2/influxd.bolt"
engine-path = "/db/.influxdbv2/engine"
http-bind-address = ":8086"
influxql-max-select-buckets = 0
influxql-max-select-point = 0
influxql-max-select-series = 0
log-level = "info"
query-concurrency = 0
storage-cache-max-memory-size = 1073741824
storage-cache-snapshot-memory-size = 26214400
storage-cache-snapshot-write-cold-duration = "10m0s"
storage-compact-full-write-cold-duration = "4h0m0s"
storage-compact-throughput-burst = 50331648
storage-max-concurrent-compactions = 0
storage-max-index-log-file-size = 1048576
storage-retention-check-interval = "30m0s"
storage-series-file-max-concurrent-snapshot-compactions = 0
storage-series-id-set-cache-size = 100
storage-shard-precreator-advance-period = "30m0s"
storage-shard-precreator-check-interval = "10m0s"
storage-tsm-use-madv-willneed = false
storage-validate-keys = false
storage-wal-fsync-delay = "0s"
tls-cert = "/etc/ssl/influxdb.pem"
tls-key = ""
`
var testConfigV2obsoleteArrays =`reporting-disabled = true
bolt-path = "/db/.influxdbv2/influxd.bolt"
engine-path = "/db/.influxdbv2/engine"
`
var testConfigV2empty = `
bolt-path = "/db/.influxdbv2/influxd.bolt"
engine-path = "/db/.influxdbv2/engine"
`