From 1e4650913d06e4a97d34582550c4e2b3bcdded13 Mon Sep 17 00:00:00 2001
From: Ales Pour <ales.pour@bonitoo.io>
Date: Fri, 2 Oct 2020 11:30:39 +0200
Subject: [PATCH] feat(upgrade): upgrade config file

---
 cmd/influxd/upgrade/config.go      | 206 +++++++++++-
 cmd/influxd/upgrade/config_test.go | 494 +++++++++++++++++++++++++++++
 2 files changed, 695 insertions(+), 5 deletions(-)
 create mode 100644 cmd/influxd/upgrade/config_test.go

diff --git a/cmd/influxd/upgrade/config.go b/cmd/influxd/upgrade/config.go
index 73906a051e..7b8395cd2e 100644
--- a/cmd/influxd/upgrade/config.go
+++ b/cmd/influxd/upgrade/config.go
@@ -1,12 +1,208 @@
 package upgrade
 
-import (
-	"errors"
+// Configuration file upgrade implementation.
+// The strategy is to transform only those entries for which rule exists.
 
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/BurntSushi/toml"
 	"go.uber.org/zap"
+	"golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
 )
 
-// upgradeConfig backups existing config file and updates it with upgraded config.
-func upgradeConfig(configFile string, targetOptions optionsV2, log *zap.Logger) (*configV1, error) {
-	return nil, errors.New("not implemented")
+// configMapRules is a map of transformation rules
+var configMapRules = map[string]string{
+	"reporting-disabled":                                   "reporting-disabled",
+	"data.dir":                                             "engine-path",
+	"data.wal-fsync-delay":                                 "storage-wal-fsync-delay",
+	"data.validate-keys":                                   "storage-validate-keys",
+	"data.cache-max-memory-size":                           "storage-cache-max-memory-size",
+	"data.cache-snapshot-memory-size":                      "storage-cache-snapshot-memory-size",
+	"data.cache-snapshot-write-cold-duration":              "storage-cache-snapshot-write-cold-duration",
+	"data.compact-full-write-cold-duration":                "storage-compact-full-write-cold-duration",
+	"data.compact-throughput-burst":                        "storage-compact-throughput-burst",
+	"data.max-concurrent-compactions":                      "storage-max-concurrent-compactions",
+	"data.max-index-log-file-size":                         "storage-max-index-log-file-size",
+	"data.series-id-set-cache-size":                        "storage-series-id-set-cache-size",
+	"data.series-file-max-concurrent-snapshot-compactions": "storage-series-file-max-concurrent-snapshot-compactions",
+	"data.tsm-use-madv-willneed":                           "storage-tsm-use-madv-willneed",
+	"retention.check-interval":                             "storage-retention-check-interval",
+	"shard-precreation.check-interval":                     "storage-shard-precreator-check-interval",
+	"shard-precreation.advance-period":                     "storage-shard-precreator-advance-period",
+	"coordinator.max-concurrent-queries":                   "query-concurrency",
+	"coordinator.max-select-point":                         "influxql-max-select-point",
+	"coordinator.max-select-series":                        "influxql-max-select-series",
+	"coordinator.max-select-buckets":                       "influxql-max-select-buckets",
+	"logging.level":                                        "log-level",
+	"http.bind-address":                                    "http-bind-address",
+	"http.https-certificate":                               "tls-cert",
+	"http.https-private-key":                               "tls-key",
+}
+
+// upgradeConfig upgrades existing 1.x (ie. typically influxdb.conf) configuration file to 2.x influxdb.toml file.
+func upgradeConfig(configFile string, targetOptions optionsV2, log *zap.Logger) (*configV1, error) {
+	// create and initialize helper
+	cu := &configUpgrader{
+		rules: configMapRules,
+		log:   log,
+	}
+
+	// load 1.x config content into byte array
+	bs, err := cu.load(configFile)
+	if err != nil {
+		return nil, err
+	}
+
+	// parse it into simplified v1 config used as return value
+	var configV1 configV1
+	_, err = toml.Decode(string(bs), &configV1)
+	if err != nil {
+		return nil, err
+	}
+
+	// parse into a generic config map
+	var cAny map[string]interface{}
+	_, err = toml.Decode(string(bs), &cAny)
+	if err != nil {
+		return nil, err
+	}
+
+	// transform the config according to rules
+	cTransformed := cu.transform(cAny)
+	if err != nil {
+		return nil, err
+	}
+
+	// update new config with upgrade command options
+	cu.updateV2Config(cTransformed, targetOptions)
+
+	// backup existing 2.x config if already exists (it should not)
+	configFileV2 := strings.TrimSuffix(configFile, filepath.Ext(configFile)) + ".toml"
+	err = cu.backupIfExists(configFileV2)
+	if err != nil {
+		return nil, err
+	}
+
+	// save new config
+	err = cu.save(cTransformed, configFileV2)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Info("Config file upgraded.",
+		zap.String("1.x config", configFile),
+		zap.String("2.x config", configFileV2))
+
+	return &configV1, nil
+}
+
+// configUpgrader is a helper used by `upgrade-config` command.
+type configUpgrader struct {
+	rules map[string]string
+	log   *zap.Logger
+}
+
+func (cu *configUpgrader) backupIfExists(path string) error {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return nil
+	}
+
+	source, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	backupFile := path + "~"
+	if _, err := os.Stat(backupFile); !os.IsNotExist(err) {
+		errMsg := fmt.Sprintf("upgrade: config file backup %s already exist", backupFile)
+		return errors.New(errMsg)
+	}
+
+	destination, err := os.Create(backupFile)
+	if err != nil {
+		return err
+	}
+	defer destination.Close()
+
+	_, err = io.Copy(destination, source)
+
+	return err
+}
+
+func (cu *configUpgrader) updateV2Config(config map[string]interface{}, targetOptions optionsV2) {
+	if targetOptions.enginePath != "" {
+		config["engine-path"] = targetOptions.enginePath
+	}
+	if targetOptions.boltPath != "" {
+		config["bolt-path"] = targetOptions.boltPath
+	}
+}
+
+func (cu *configUpgrader) load(path string) ([]byte, error) {
+	bs, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	// From master-1.x/cmd/influxd/run/config.go:
+	// Handle any potential Byte-Order-Marks that may be in the config file.
+	// This is for Windows compatibility only.
+	// See https://github.com/influxdata/telegraf/issues/1378 and
+	// https://github.com/influxdata/influxdb/issues/8965.
+	bom := unicode.BOMOverride(transform.Nop)
+	bs, _, err = transform.Bytes(bom, bs)
+
+	return bs, err
+}
+
+func (cu *configUpgrader) save(config map[string]interface{}, path string) error {
+	buf := new(bytes.Buffer)
+	if err := toml.NewEncoder(buf).Encode(&config); err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(path, buf.Bytes(), 0666)
+}
+
+// Credits: @rogpeppe (Roger Peppe)
+
+func (cu *configUpgrader) transform(x map[string]interface{}) map[string]interface{} {
+	res := make(map[string]interface{})
+	for old, new := range cu.rules {
+		val, ok := cu.lookup(x, old)
+		if ok {
+			res[new] = val
+		}
+	}
+
+	return res
+}
+
+func (cu *configUpgrader) lookup(x map[string]interface{}, path string) (interface{}, bool) {
+	for {
+		elem := path
+		rest := ""
+		if i := strings.Index(path, "."); i != -1 {
+			elem, rest = path[0:i], path[i+1:]
+		}
+		val, ok := x[elem]
+		if rest == "" {
+			return val, ok
+		}
+		child, ok := val.(map[string]interface{})
+		if !ok {
+			return nil, false
+		}
+		path, x = rest, child
+	}
 }
diff --git a/cmd/influxd/upgrade/config_test.go b/cmd/influxd/upgrade/config_test.go
new file mode 100644
index 0000000000..1a1c910a27
--- /dev/null
+++ b/cmd/influxd/upgrade/config_test.go
@@ -0,0 +1,494 @@
+package upgrade
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/BurntSushi/toml"
+	"github.com/google/go-cmp/cmp"
+	"go.uber.org/zap"
+)
+
+func testCreateTempFile(t *testing.T, pattern, content string) string {
+	f, err := ioutil.TempFile("", pattern)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = f.WriteString(content)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = f.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	return f.Name()
+}
+
+func TestConfigUpgrade(t *testing.T) {
+	targetOtions := optionsV2{
+		boltPath:   "/db/.influxdbv2/influxd.bolt",
+		enginePath: "/db/.influxdbv2/engine",
+	}
+
+	var typicalRetval, emptyRetval configV1
+	_, err := toml.Decode("[meta]\ndir=\"/var/lib/influxdb/meta\"\n[data]\ndir=\"/var/lib/influxdb/data\"\nwal-dir=\"/var/lib/influxdb/wal\"\n",
+		&typicalRetval)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	type testCase struct {
+		name     string
+		config1x string
+		config2x string
+		retval   *configV1
+	}
+
+	var testCases = []testCase{
+		{
+			name:     "minimal",
+			config1x: testConfigV1minimal,
+			config2x: testConfigV2minimal,
+			retval:   &typicalRetval,
+		},
+		{
+			name:     "default",
+			config1x: testConfigV1default,
+			config2x: testConfigV2default,
+			retval:   &typicalRetval,
+		},
+		{
+			name:     "empty",
+			config1x: testConfigV1empty,
+			config2x: testConfigV2empty,
+			retval:   &emptyRetval,
+		},
+		{
+			name:     "obsolete / arrays",
+			config1x: testConfigV1obsoleteArrays,
+			config2x: testConfigV2obsoleteArrays,
+			retval:   &typicalRetval,
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			configFile := testCreateTempFile(t, "influxdb-*.conf", tc.config1x)
+			configFileV2 := strings.TrimSuffix(configFile, filepath.Ext(configFile)) + ".toml"
+			defer func() {
+				os.Remove(configFile)
+				os.Remove(configFileV2)
+			}()
+			retval, err := upgradeConfig(configFile, targetOtions, zap.NewNop())
+			if err != nil {
+				t.Fatal(err)
+			}
+			if diff := cmp.Diff(tc.retval, retval); diff != "" {
+				t.Fatal(diff)
+			}
+			var actual, expected map[string]interface{}
+			if _, err = toml.Decode(tc.config2x, &expected); err != nil {
+				t.Fatal(err)
+			}
+			if _, err = toml.DecodeFile(configFileV2, &actual); err != nil {
+				t.Fatal(err)
+			}
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}
+
+func TestConfigUpgradeWithBackup(t *testing.T) {
+	targetOtions := optionsV2{
+		boltPath:   "/db/.influxdbv2/influxd.bolt",
+		enginePath: "/db/.influxdbv2/engine",
+	}
+
+	type testCase struct {
+		name     string
+		config1x string
+		config2x string
+	}
+
+	var testCases = []testCase{
+		{
+			name:     "default",
+			config1x: testConfigV1default,
+			config2x: testConfigV2default,
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			configFile := testCreateTempFile(t, "influxdb-*.conf", tc.config1x)
+			configFileV2 := strings.TrimSuffix(configFile, filepath.Ext(configFile)) + ".toml"
+			configFileV2Backup := strings.TrimSuffix(configFile, filepath.Ext(configFile)) + ".toml~"
+			defer func() {
+				os.Remove(configFile)
+				os.Remove(configFileV2)
+				os.Remove(configFileV2Backup)
+			}()
+
+			// upgrade
+			_, err := upgradeConfig(configFile, targetOtions, zap.NewNop())
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// upgrade again, backup of existing newly created config should get created
+			_, err = upgradeConfig(configFile, targetOtions, zap.NewNop())
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// validate new config
+			var actual, expected map[string]interface{}
+			if _, err = toml.Decode(tc.config2x, &expected); err != nil {
+				t.Fatal(err)
+			}
+			if _, err = toml.DecodeFile(configFileV2, &actual); err != nil {
+				t.Fatal(err)
+			}
+			if diff := cmp.Diff(expected, actual); diff != "" {
+				t.Fatal(diff)
+			}
+
+			// validate backup of the new config, it should exists and be the same
+			var actualBackup, expectedBackup map[string]interface{}
+			if _, err = toml.Decode(tc.config2x, &expectedBackup); err != nil {
+				t.Fatal(err)
+			}
+			if _, err = toml.DecodeFile(configFileV2Backup, &actualBackup); err != nil {
+				t.Fatal(err)
+			}
+			if diff := cmp.Diff(expectedBackup, actualBackup); diff != "" {
+				t.Fatal(diff)
+			}
+		})
+	}
+}
+
+func TestConfigUpgradeFileNotExists(t *testing.T) {
+	targetOtions := optionsV2{
+	}
+	configFile := "/there/is/no/such/path/influxdb.conf"
+
+	// try upgrade
+	_, err := upgradeConfig(configFile, targetOtions, zap.NewNop())
+	if err == nil {
+		t.Fatal("error expected")
+	}
+}
+
+// 1.x test configs
+
+var testConfigV1minimal = `### Welcome to the InfluxDB configuration file.
+
+# Change this option to true to disable reporting.
+reporting-disabled = false
+
+# Bind address to use for the RPC service for backup and restore.
+bind-address = "127.0.0.1:8088"
+
+[meta]
+  dir = "/var/lib/influxdb/meta"
+
+[data]
+  dir = "/var/lib/influxdb/data"
+  wal-dir = "/var/lib/influxdb/wal"
+  wal-fsync-delay = "100s"
+  index-version = "inmem"
+
+[coordinator]
+  max-select-point = 0
+
+[retention]
+  check-interval = "30m"
+
+[shard-precreation]
+  check-interval = "5m"
+
+[monitor]
+  store-enabled = true
+
+[http]
+  flux-enabled = false
+  bind-address = ":8086"
+  https-certificate = "/etc/ssl/influxdb.pem"
+  https-private-key = "/etc/ssl/influxdb-key.pem"
+
+[logging]
+  level = "debug"
+
+[subscriber]
+
+[[graphite]]
+
+[[collectd]]
+
+[[opentsdb]]
+
+[[udp]]
+
+[continuous_queries]
+  query-stats-enabled = true
+
+[tls]
+`
+
+var testConfigV1default = `reporting-disabled = false
+bind-address = "127.0.0.1:8088"
+
+[meta]
+  dir = "/var/lib/influxdb/meta"
+  retention-autocreate = true
+  logging-enabled = true
+
+[data]
+  dir = "/var/lib/influxdb/data"
+  wal-dir = "/var/lib/influxdb/wal"
+  wal-fsync-delay = "0s"
+  validate-keys = false
+  index-version = "inmem"
+  query-log-enabled = true
+  cache-max-memory-size = 1073741824
+  cache-snapshot-memory-size = 26214400
+  cache-snapshot-write-cold-duration = "10m0s"
+  compact-full-write-cold-duration = "4h0m0s"
+  compact-throughput = 50331648
+  compact-throughput-burst = 50331648
+  max-series-per-database = 1000000
+  max-values-per-tag = 100000
+  max-concurrent-compactions = 0
+  max-index-log-file-size = 1048576
+  series-id-set-cache-size = 100
+  series-file-max-concurrent-snapshot-compactions = 0
+  trace-logging-enabled = false
+  tsm-use-madv-willneed = false
+
+[coordinator]
+  write-timeout = "10s"
+  max-concurrent-queries = 0
+  query-timeout = "0s"
+  log-queries-after = "0s"
+  max-select-point = 0
+  max-select-series = 0
+  max-select-buckets = 0
+
+[retention]
+  enabled = true
+  check-interval = "30m0s"
+
+[shard-precreation]
+  enabled = true
+  check-interval = "10m0s"
+  advance-period = "30m0s"
+
+[monitor]
+  store-enabled = true
+  store-database = "_internal"
+  store-interval = "10s"
+
+[subscriber]
+  enabled = true
+  http-timeout = "30s"
+  insecure-skip-verify = false
+  ca-certs = ""
+  write-concurrency = 40
+  write-buffer-size = 1000
+
+[http]
+  enabled = true
+  bind-address = ":8086"
+  auth-enabled = false
+  log-enabled = true
+  suppress-write-log = false
+  write-tracing = false
+  flux-enabled = false
+  flux-log-enabled = false
+  pprof-enabled = true
+  pprof-auth-enabled = false
+  debug-pprof-enabled = false
+  ping-auth-enabled = false
+  prom-read-auth-enabled = false
+  https-enabled = false
+  https-certificate = "/etc/ssl/influxdb.pem"
+  https-private-key = ""
+  max-row-limit = 0
+  max-connection-limit = 0
+  shared-secret = ""
+  realm = "InfluxDB"
+  unix-socket-enabled = false
+  unix-socket-permissions = "0777"
+  bind-socket = "/var/run/influxdb.sock"
+  max-body-size = 25000000
+  access-log-path = ""
+  max-concurrent-write-limit = 0
+  max-enqueued-write-limit = 0
+  enqueued-write-timeout = 30000000000
+
+[logging]
+  format = "auto"
+  level = "info"
+  suppress-logo = false
+
+[[graphite]]
+  enabled = false
+  bind-address = ":2003"
+  database = "graphite"
+  retention-policy = ""
+  protocol = "tcp"
+  batch-size = 5000
+  batch-pending = 10
+  batch-timeout = "1s"
+  consistency-level = "one"
+  separator = "."
+  udp-read-buffer = 0
+
+[[collectd]]
+  enabled = false
+  bind-address = ":25826"
+  database = "collectd"
+  retention-policy = ""
+  batch-size = 5000
+  batch-pending = 10
+  batch-timeout = "10s"
+  read-buffer = 0
+  typesdb = "/usr/share/collectd/types.db"
+  security-level = "none"
+  auth-file = "/etc/collectd/auth_file"
+  parse-multivalue-plugin = "split"
+
+[[opentsdb]]
+  enabled = false
+  bind-address = ":4242"
+  database = "opentsdb"
+  retention-policy = ""
+  consistency-level = "one"
+  tls-enabled = false
+  certificate = "/etc/ssl/influxdb.pem"
+  batch-size = 1000
+  batch-pending = 5
+  batch-timeout = "1s"
+  log-point-errors = true
+
+[[udp]]
+  enabled = false
+  bind-address = ":8089"
+  database = "udp"
+  retention-policy = ""
+  batch-size = 5000
+  batch-pending = 10
+  read-buffer = 0
+  batch-timeout = "1s"
+  precision = ""
+
+[continuous_queries]
+  log-enabled = true
+  enabled = true
+  query-stats-enabled = false
+  run-interval = "1s"
+
+[tls]
+  min-version = "tls1.2"
+  max-version = "tls1.3"
+`
+
+var testConfigV1obsoleteArrays =`
+reporting-disabled = true
+
+[meta]
+  dir = "/var/lib/influxdb/meta"
+
+[data]
+  dir = "/var/lib/influxdb/data"
+  wal-dir = "/var/lib/influxdb/wal"
+
+[[udp]]
+  enabled = false
+  bind-address = ":8089"
+  database = "udp"
+  retention-policy = ""
+  batch-size = 5000
+  batch-pending = 10
+  read-buffer = 0
+  batch-timeout = "1s"
+  precision = ""
+
+[[udp]]
+  enabled = false
+  bind-address = ":8090"
+  database = "udp2"
+  retention-policy = ""
+  batch-size = 5000
+  batch-pending = 10
+  read-buffer = 0
+  batch-timeout = "1s"
+  precision = ""
+`
+
+var testConfigV1empty = `
+`
+
+// 2.x test configs
+
+var testConfigV2minimal = `reporting-disabled = false
+bolt-path = "/db/.influxdbv2/influxd.bolt"
+engine-path = "/db/.influxdbv2/engine"
+http-bind-address = ":8086"
+influxql-max-select-point = 0
+log-level = "debug"
+storage-retention-check-interval = "30m"
+storage-shard-precreator-check-interval = "5m"
+storage-wal-fsync-delay = "100s"
+tls-cert = "/etc/ssl/influxdb.pem"
+tls-key = "/etc/ssl/influxdb-key.pem"
+`
+
+var testConfigV2default = `reporting-disabled = false
+bolt-path = "/db/.influxdbv2/influxd.bolt"
+engine-path = "/db/.influxdbv2/engine"
+http-bind-address = ":8086"
+influxql-max-select-buckets = 0
+influxql-max-select-point = 0
+influxql-max-select-series = 0
+log-level = "info"
+query-concurrency = 0
+storage-cache-max-memory-size = 1073741824
+storage-cache-snapshot-memory-size = 26214400
+storage-cache-snapshot-write-cold-duration = "10m0s"
+storage-compact-full-write-cold-duration = "4h0m0s"
+storage-compact-throughput-burst = 50331648
+storage-max-concurrent-compactions = 0
+storage-max-index-log-file-size = 1048576
+storage-retention-check-interval = "30m0s"
+storage-series-file-max-concurrent-snapshot-compactions = 0
+storage-series-id-set-cache-size = 100
+storage-shard-precreator-advance-period = "30m0s"
+storage-shard-precreator-check-interval = "10m0s"
+storage-tsm-use-madv-willneed = false
+storage-validate-keys = false
+storage-wal-fsync-delay = "0s"
+tls-cert = "/etc/ssl/influxdb.pem"
+tls-key = ""
+`
+
+var testConfigV2obsoleteArrays =`reporting-disabled = true
+bolt-path = "/db/.influxdbv2/influxd.bolt"
+engine-path = "/db/.influxdbv2/engine"
+`
+
+var testConfigV2empty = `
+bolt-path = "/db/.influxdbv2/influxd.bolt"
+engine-path = "/db/.influxdbv2/engine"
+`