From 1a612ffa2218c613bb77b5a571b566746b35f98f Mon Sep 17 00:00:00 2001 From: greg linton Date: Mon, 3 Feb 2020 13:35:47 -0700 Subject: [PATCH] feat: add migrate command to chronoctl adds a command to migrate a db from bolt to etcd and visa versa. there is no promise that the options/functionality will not change, hence the 'beta' note. obviously there is also no guarantee that there will never be any data loss. after several manual tests, i do have a high level of confidence in the functionality presented. --- chronograf.go | 2 + cmd/chronoctl/README.md | 50 ++++++++ cmd/chronoctl/migrate.go | 226 ++++++++++++++++++++++++++++++++++++ kv/org_config.go | 27 +++++ mocks/org_config.go | 5 + noop/org_config.go | 8 ++ organizations/org_config.go | 9 +- 7 files changed, 325 insertions(+), 2 deletions(-) create mode 100644 cmd/chronoctl/README.md create mode 100644 cmd/chronoctl/migrate.go diff --git a/chronograf.go b/chronograf.go index 155d89594..01883d035 100644 --- a/chronograf.go +++ b/chronograf.go @@ -917,6 +917,8 @@ type ColumnEncoding struct { // OrganizationConfigStore is the storage and retrieval of organization Configs type OrganizationConfigStore interface { + // All lists all org configs in the OrganizationConfigStore + All(ctx context.Context) ([]OrganizationConfig, error) // FindOrCreate gets an existing OrganizationConfig and creates one if none exists FindOrCreate(ctx context.Context, orgID string) (*OrganizationConfig, error) // Put replaces the whole organization config in the OrganizationConfigStore diff --git a/cmd/chronoctl/README.md b/cmd/chronoctl/README.md new file mode 100644 index 000000000..949ad0233 --- /dev/null +++ b/cmd/chronoctl/README.md @@ -0,0 +1,50 @@ +## Chronoctl + +Chronoctl is a tool to interact with an instance of a chronograf's bolt database. + +``` +Available commands: + add-superadmin Creates a new superadmin user (bolt specific) + list-users Lists users (bolt specific) + migrate Migrate db (beta) +``` + + +### Migrate + +The `migrate` command allows you to migrate your chronograf configuration store. It is highly recommended that you make a backup of all databases involved before running a migration as there is no guarantee that there will be no data loss. When specifying an etcd endpoint, the URI must begin with `etcd://`. It is preferred that you prefix `bolt://` to an absolute path when specifying a local bolt db file, but a lone relative path is also accepted without the prefix. If there is authentication on etcd, use the standard URI format to define a username/password: `[scheme:][//[userinfo@]host][/]path`. +There is currently no cleanup for a failed migration, so keep that in mind before migrating to a db that contains other important data. + + +##### Usage +``` +chronoctl migrate [OPTIONS] + +OPTIONS + -f, --from= Full path to boltDB file or etcd (e.g. 'bolt:///path/to/chronograf-v1.db' or 'etcd://user:pass@localhost:2379 (default: chronograf-v1.db) + -t, --to= Full path to boltDB file or etcd (e.g. 'bolt:///path/to/chronograf-v1.db' or 'etcd://user:pass@localhost:2379 (default: etcd://localhost:2379) +``` + + +##### Example +```sh +$ chronoctl migrate -f etcd://localhost:2379 -t bolt:///tmp/chronograf.db +# Performing non-idempotent db migration from "etcd://localhost:2379" to "bolt:///tmp/chronograf.db"... +# Saved 1 organizations. +# Saved 1 organization configs. +# Saved 1 dashboards. +# Saved 3 mappings. +# Saved 0 servers. +# Saved 1 sources. +# Migration successful! + +$ chronoctl migrate -f ./chronograf-v1.db -t etcd://localhost:2379 +# Performing non-idempotent db migration from "./chronograf-v1.db" to "etcd://localhost:2379"... +# Saved 1 organizations. +# Saved 1 organization configs. +# Saved 1 dashboards. +# Saved 3 mappings. +# Saved 0 servers. +# Saved 1 sources. +# Migration successful! +``` diff --git a/cmd/chronoctl/migrate.go b/cmd/chronoctl/migrate.go new file mode 100644 index 000000000..dcc7b277c --- /dev/null +++ b/cmd/chronoctl/migrate.go @@ -0,0 +1,226 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + + "github.com/influxdata/chronograf" + "github.com/influxdata/chronograf/kv" + "github.com/influxdata/chronograf/kv/bolt" + "github.com/influxdata/chronograf/kv/etcd" +) + +func init() { + parser.AddCommand("migrate", + "Migrate db (beta)", + "The migrate command (beta) will copy one db to another", + &migrateCommand{}) +} + +type migrateCommand struct { + From string `short:"f" long:"from" description:"Full path to boltDB file or etcd (e.g. 'bolt:///path/to/chronograf-v1.db' or 'etcd://user:pass@localhost:2379" default:"chronograf-v1.db"` + To string `short:"t" long:"to" description:"Full path to boltDB file or etcd (e.g. 'bolt:///path/to/chronograf-v1.db' or 'etcd://user:pass@localhost:2379" default:"etcd://localhost:2379"` +} + +func (m *migrateCommand) Execute(args []string) error { + fmt.Printf("Performing non-idempotent db migration from %q to %q...\n", m.From, m.To) + if m.From == m.To { + errExit(errors.New("Cannot migrate to original source")) + } + + ctx := context.TODO() + + datas, err := getData(ctx, m.From) + errExit(err) + + errExit(saveData(ctx, m.To, datas)) + + fmt.Println("Migration successful!") + return nil +} + +func openService(ctx context.Context, s string) (*kv.Service, error) { + u, err := url.Parse(s) + if err != nil { + return nil, err + } + + var db kv.Store + + switch u.Scheme { + case "bolt", "boltdb", "": + if u.Host != "" { + return nil, errors.New("ambiguous uri") + } + + db, err = bolt.NewClient(ctx, + bolt.WithPath(u.Path), + ) + if err != nil { + return nil, fmt.Errorf("unable to create bolt client: %s", err) + } + case "etcd": + pw, _ := u.User.Password() + db, err = etcd.NewClient(ctx, + etcd.WithEndpoints([]string{u.Host}), + etcd.WithLogin(u.User.Username(), pw), + ) + if err != nil { + return nil, fmt.Errorf("unable to create etcd client: %s", err) + } + default: + return nil, fmt.Errorf("invalid uri scheme '%s'", u.Scheme) + } + + return kv.NewService(ctx, db) +} + +func getData(ctx context.Context, fromURI string) (*datas, error) { + from, err := openService(ctx, fromURI) + if err != nil { + return nil, err + } + defer from.Close() + + cfg, err := from.ConfigStore().Get(ctx) + if err != nil { + return nil, err + } + + dashboards, err := from.DashboardsStore().All(ctx) + if err != nil { + return nil, err + } + + mappings, err := from.MappingsStore().All(ctx) + if err != nil { + return nil, err + } + + orgCfgs, err := from.OrganizationConfigStore().All(ctx) + if err != nil { + return nil, err + } + + orgs, err := from.OrganizationsStore().All(ctx) + if err != nil { + return nil, err + } + + servers, err := from.ServersStore().All(ctx) + if err != nil { + return nil, err + } + + srcs, err := from.SourcesStore().All(ctx) + if err != nil { + return nil, err + } + + users, err := from.UsersStore().All(ctx) + if err != nil { + return nil, err + } + + return &datas{ + config: *cfg, + dashboards: dashboards, + mappings: mappings, + orgConfigs: orgCfgs, + organizations: orgs, + servers: servers, + sources: srcs, + users: users, + }, nil +} + +func saveData(ctx context.Context, t string, datas *datas) error { + to, err := openService(ctx, t) + if err != nil { + return fmt.Errorf("failed to open service '%s': %s", t, err) + } + defer to.Close() + + err = to.ConfigStore().Update(ctx, &datas.config) + if err != nil { + return err + } + + for _, org := range datas.organizations { + _, err = to.OrganizationsStore().Add(ctx, &org) + if err != nil { + if err == chronograf.ErrOrganizationAlreadyExists { + err = to.OrganizationsStore().Update(ctx, &org) + if err == nil { + continue + } + } + return fmt.Errorf("failed to add to OrganizationsStore: %s", err) + } + } + fmt.Printf(" Saved %d organizations.\n", len(datas.organizations)) + + for _, orgCfg := range datas.orgConfigs { + err = to.OrganizationConfigStore().Put(ctx, &orgCfg) + if err != nil { + return fmt.Errorf("failed to add to OrganizationConfigStore: %s", err) + } + } + fmt.Printf(" Saved %d organization configs.\n", len(datas.orgConfigs)) + + for _, dash := range datas.dashboards { + _, err = to.DashboardsStore().Add(ctx, dash) + if err != nil { + return fmt.Errorf("failed to add to DashboardsStore: %s", err) + } + } + fmt.Printf(" Saved %d dashboards.\n", len(datas.dashboards)) + + for _, mapping := range datas.mappings { + _, err = to.MappingsStore().Add(ctx, &mapping) + if err != nil { + return fmt.Errorf("failed to add to MappingsStore: %s", err) + } + } + fmt.Printf(" Saved %d mappings.\n", len(datas.mappings)) + + for _, server := range datas.servers { + _, err = to.ServersStore().Add(ctx, server) + if err != nil { + return fmt.Errorf("failed to add to ServersStore: %s", err) + } + } + fmt.Printf(" Saved %d servers.\n", len(datas.servers)) + + for _, source := range datas.sources { + _, err = to.SourcesStore().Add(ctx, source) + if err != nil { + return fmt.Errorf("failed to add to SourcesStore: %s", err) + } + } + fmt.Printf(" Saved %d sources.\n", len(datas.sources)) + + return nil +} + +type datas struct { + config chronograf.Config + dashboards []chronograf.Dashboard + mappings []chronograf.Mapping + orgConfigs []chronograf.OrganizationConfig + organizations []chronograf.Organization + servers []chronograf.Server + sources []chronograf.Source + users []chronograf.User +} + +func errExit(err error) { + if err == nil { + return + } + fmt.Println(err.Error()) + os.Exit(1) +} diff --git a/kv/org_config.go b/kv/org_config.go index 515ae8ae1..7d1347552 100644 --- a/kv/org_config.go +++ b/kv/org_config.go @@ -16,6 +16,33 @@ type organizationConfigStore struct { client *Service } +// All returns all known organization configurations. +func (s *organizationConfigStore) All(ctx context.Context) ([]chronograf.OrganizationConfig, error) { + var orgCfgs []chronograf.OrganizationConfig + err := s.each(ctx, func(o *chronograf.OrganizationConfig) { + orgCfgs = append(orgCfgs, *o) + }) + + if err != nil { + return nil, err + } + + return orgCfgs, nil +} + +func (s *organizationConfigStore) each(ctx context.Context, fn func(*chronograf.OrganizationConfig)) error { + return s.client.kv.View(ctx, func(tx Tx) error { + return tx.Bucket(organizationConfigBucket).ForEach(func(k, v []byte) error { + var orgCfg chronograf.OrganizationConfig + if err := internal.UnmarshalOrganizationConfig(v, &orgCfg); err != nil { + return err + } + fn(&orgCfg) + return nil + }) + }) +} + func (s *organizationConfigStore) get(ctx context.Context, tx Tx, orgID string, c *chronograf.OrganizationConfig) error { v, err := tx.Bucket(organizationConfigBucket).Get([]byte(orgID)) if len(v) == 0 || err != nil { diff --git a/mocks/org_config.go b/mocks/org_config.go index 66f16cb8e..9fc3aeb11 100644 --- a/mocks/org_config.go +++ b/mocks/org_config.go @@ -9,10 +9,15 @@ import ( var _ chronograf.OrganizationConfigStore = &OrganizationConfigStore{} type OrganizationConfigStore struct { + AllF func(ctx context.Context) ([]chronograf.OrganizationConfig, error) FindOrCreateF func(ctx context.Context, id string) (*chronograf.OrganizationConfig, error) PutF func(ctx context.Context, c *chronograf.OrganizationConfig) error } +func (s *OrganizationConfigStore) All(ctx context.Context) ([]chronograf.OrganizationConfig, error) { + return s.AllF(ctx) +} + func (s *OrganizationConfigStore) FindOrCreate(ctx context.Context, id string) (*chronograf.OrganizationConfig, error) { return s.FindOrCreateF(ctx, id) } diff --git a/noop/org_config.go b/noop/org_config.go index 6aa8a3351..3c8396477 100644 --- a/noop/org_config.go +++ b/noop/org_config.go @@ -10,12 +10,20 @@ import ( // ensure OrganizationConfigStore implements chronograf.OrganizationConfigStore var _ chronograf.OrganizationConfigStore = &OrganizationConfigStore{} +// OrganizationConfigStore is an empty struct for satisfying an interface and returning errors. type OrganizationConfigStore struct{} +// All returns an error +func (s *OrganizationConfigStore) All(context.Context) ([]chronograf.OrganizationConfig, error) { + return nil, chronograf.ErrOrganizationConfigNotFound +} + +// FindOrCreate returns an error func (s *OrganizationConfigStore) FindOrCreate(context.Context, string) (*chronograf.OrganizationConfig, error) { return nil, chronograf.ErrOrganizationConfigNotFound } +// Put returns an error func (s *OrganizationConfigStore) Put(context.Context, *chronograf.OrganizationConfig) error { return fmt.Errorf("cannot replace config") } diff --git a/organizations/org_config.go b/organizations/org_config.go index fe35c4c3d..35480de6c 100644 --- a/organizations/org_config.go +++ b/organizations/org_config.go @@ -31,12 +31,17 @@ func (s *OrganizationConfigStore) FindOrCreate(ctx context.Context, orgID string return nil, err } - oc, err := s.store.FindOrCreate(ctx, orgID) + return s.store.FindOrCreate(ctx, orgID) +} + +// All returns all organization configs from the store. +func (s *OrganizationConfigStore) All(ctx context.Context) ([]chronograf.OrganizationConfig, error) { + var err = validOrganization(ctx) if err != nil { return nil, err } - return oc, nil + return s.store.All(ctx) } // Put the OrganizationConfig in OrganizationConfigStore.