feat: add migrate command to chronoctl
adds a command to migrate a db from bolt to etcd and visa versa. there is no promise that the options/functionality will not change, hence the 'beta' note. obviously there is also no guarantee that there will never be any data loss. after several manual tests, i do have a high level of confidence in the functionality presented.pull/5363/head
greg linton
Greg
parent
749ebf76f2
commit
1a612ffa22
|
|
@ -917,6 +917,8 @@ type ColumnEncoding struct {
|
|||
|
||||
// OrganizationConfigStore is the storage and retrieval of organization Configs
|
||||
type OrganizationConfigStore interface {
|
||||
// All lists all org configs in the OrganizationConfigStore
|
||||
All(ctx context.Context) ([]OrganizationConfig, error)
|
||||
// FindOrCreate gets an existing OrganizationConfig and creates one if none exists
|
||||
FindOrCreate(ctx context.Context, orgID string) (*OrganizationConfig, error)
|
||||
// Put replaces the whole organization config in the OrganizationConfigStore
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
## Chronoctl
|
||||
|
||||
Chronoctl is a tool to interact with an instance of a chronograf's bolt database.
|
||||
|
||||
```
|
||||
Available commands:
|
||||
add-superadmin Creates a new superadmin user (bolt specific)
|
||||
list-users Lists users (bolt specific)
|
||||
migrate Migrate db (beta)
|
||||
```
|
||||
|
||||
|
||||
### Migrate
|
||||
|
||||
The `migrate` command allows you to migrate your chronograf configuration store. It is highly recommended that you make a backup of all databases involved before running a migration as there is no guarantee that there will be no data loss. When specifying an etcd endpoint, the URI must begin with `etcd://`. It is preferred that you prefix `bolt://` to an absolute path when specifying a local bolt db file, but a lone relative path is also accepted without the prefix. If there is authentication on etcd, use the standard URI format to define a username/password: `[scheme:][//[userinfo@]host][/]path`.
|
||||
There is currently no cleanup for a failed migration, so keep that in mind before migrating to a db that contains other important data.
|
||||
|
||||
|
||||
##### Usage
|
||||
```
|
||||
chronoctl migrate [OPTIONS]
|
||||
|
||||
OPTIONS
|
||||
-f, --from= Full path to boltDB file or etcd (e.g. 'bolt:///path/to/chronograf-v1.db' or 'etcd://user:pass@localhost:2379 (default: chronograf-v1.db)
|
||||
-t, --to= Full path to boltDB file or etcd (e.g. 'bolt:///path/to/chronograf-v1.db' or 'etcd://user:pass@localhost:2379 (default: etcd://localhost:2379)
|
||||
```
|
||||
|
||||
|
||||
##### Example
|
||||
```sh
|
||||
$ chronoctl migrate -f etcd://localhost:2379 -t bolt:///tmp/chronograf.db
|
||||
# Performing non-idempotent db migration from "etcd://localhost:2379" to "bolt:///tmp/chronograf.db"...
|
||||
# Saved 1 organizations.
|
||||
# Saved 1 organization configs.
|
||||
# Saved 1 dashboards.
|
||||
# Saved 3 mappings.
|
||||
# Saved 0 servers.
|
||||
# Saved 1 sources.
|
||||
# Migration successful!
|
||||
|
||||
$ chronoctl migrate -f ./chronograf-v1.db -t etcd://localhost:2379
|
||||
# Performing non-idempotent db migration from "./chronograf-v1.db" to "etcd://localhost:2379"...
|
||||
# Saved 1 organizations.
|
||||
# Saved 1 organization configs.
|
||||
# Saved 1 dashboards.
|
||||
# Saved 3 mappings.
|
||||
# Saved 0 servers.
|
||||
# Saved 1 sources.
|
||||
# Migration successful!
|
||||
```
|
||||
|
|
@ -0,0 +1,226 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
|
||||
"github.com/influxdata/chronograf"
|
||||
"github.com/influxdata/chronograf/kv"
|
||||
"github.com/influxdata/chronograf/kv/bolt"
|
||||
"github.com/influxdata/chronograf/kv/etcd"
|
||||
)
|
||||
|
||||
func init() {
|
||||
parser.AddCommand("migrate",
|
||||
"Migrate db (beta)",
|
||||
"The migrate command (beta) will copy one db to another",
|
||||
&migrateCommand{})
|
||||
}
|
||||
|
||||
type migrateCommand struct {
|
||||
From string `short:"f" long:"from" description:"Full path to boltDB file or etcd (e.g. 'bolt:///path/to/chronograf-v1.db' or 'etcd://user:pass@localhost:2379" default:"chronograf-v1.db"`
|
||||
To string `short:"t" long:"to" description:"Full path to boltDB file or etcd (e.g. 'bolt:///path/to/chronograf-v1.db' or 'etcd://user:pass@localhost:2379" default:"etcd://localhost:2379"`
|
||||
}
|
||||
|
||||
func (m *migrateCommand) Execute(args []string) error {
|
||||
fmt.Printf("Performing non-idempotent db migration from %q to %q...\n", m.From, m.To)
|
||||
if m.From == m.To {
|
||||
errExit(errors.New("Cannot migrate to original source"))
|
||||
}
|
||||
|
||||
ctx := context.TODO()
|
||||
|
||||
datas, err := getData(ctx, m.From)
|
||||
errExit(err)
|
||||
|
||||
errExit(saveData(ctx, m.To, datas))
|
||||
|
||||
fmt.Println("Migration successful!")
|
||||
return nil
|
||||
}
|
||||
|
||||
func openService(ctx context.Context, s string) (*kv.Service, error) {
|
||||
u, err := url.Parse(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var db kv.Store
|
||||
|
||||
switch u.Scheme {
|
||||
case "bolt", "boltdb", "":
|
||||
if u.Host != "" {
|
||||
return nil, errors.New("ambiguous uri")
|
||||
}
|
||||
|
||||
db, err = bolt.NewClient(ctx,
|
||||
bolt.WithPath(u.Path),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create bolt client: %s", err)
|
||||
}
|
||||
case "etcd":
|
||||
pw, _ := u.User.Password()
|
||||
db, err = etcd.NewClient(ctx,
|
||||
etcd.WithEndpoints([]string{u.Host}),
|
||||
etcd.WithLogin(u.User.Username(), pw),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to create etcd client: %s", err)
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("invalid uri scheme '%s'", u.Scheme)
|
||||
}
|
||||
|
||||
return kv.NewService(ctx, db)
|
||||
}
|
||||
|
||||
func getData(ctx context.Context, fromURI string) (*datas, error) {
|
||||
from, err := openService(ctx, fromURI)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer from.Close()
|
||||
|
||||
cfg, err := from.ConfigStore().Get(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dashboards, err := from.DashboardsStore().All(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
mappings, err := from.MappingsStore().All(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
orgCfgs, err := from.OrganizationConfigStore().All(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
orgs, err := from.OrganizationsStore().All(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
servers, err := from.ServersStore().All(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
srcs, err := from.SourcesStore().All(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
users, err := from.UsersStore().All(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &datas{
|
||||
config: *cfg,
|
||||
dashboards: dashboards,
|
||||
mappings: mappings,
|
||||
orgConfigs: orgCfgs,
|
||||
organizations: orgs,
|
||||
servers: servers,
|
||||
sources: srcs,
|
||||
users: users,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func saveData(ctx context.Context, t string, datas *datas) error {
|
||||
to, err := openService(ctx, t)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open service '%s': %s", t, err)
|
||||
}
|
||||
defer to.Close()
|
||||
|
||||
err = to.ConfigStore().Update(ctx, &datas.config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, org := range datas.organizations {
|
||||
_, err = to.OrganizationsStore().Add(ctx, &org)
|
||||
if err != nil {
|
||||
if err == chronograf.ErrOrganizationAlreadyExists {
|
||||
err = to.OrganizationsStore().Update(ctx, &org)
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("failed to add to OrganizationsStore: %s", err)
|
||||
}
|
||||
}
|
||||
fmt.Printf(" Saved %d organizations.\n", len(datas.organizations))
|
||||
|
||||
for _, orgCfg := range datas.orgConfigs {
|
||||
err = to.OrganizationConfigStore().Put(ctx, &orgCfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add to OrganizationConfigStore: %s", err)
|
||||
}
|
||||
}
|
||||
fmt.Printf(" Saved %d organization configs.\n", len(datas.orgConfigs))
|
||||
|
||||
for _, dash := range datas.dashboards {
|
||||
_, err = to.DashboardsStore().Add(ctx, dash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add to DashboardsStore: %s", err)
|
||||
}
|
||||
}
|
||||
fmt.Printf(" Saved %d dashboards.\n", len(datas.dashboards))
|
||||
|
||||
for _, mapping := range datas.mappings {
|
||||
_, err = to.MappingsStore().Add(ctx, &mapping)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add to MappingsStore: %s", err)
|
||||
}
|
||||
}
|
||||
fmt.Printf(" Saved %d mappings.\n", len(datas.mappings))
|
||||
|
||||
for _, server := range datas.servers {
|
||||
_, err = to.ServersStore().Add(ctx, server)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add to ServersStore: %s", err)
|
||||
}
|
||||
}
|
||||
fmt.Printf(" Saved %d servers.\n", len(datas.servers))
|
||||
|
||||
for _, source := range datas.sources {
|
||||
_, err = to.SourcesStore().Add(ctx, source)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to add to SourcesStore: %s", err)
|
||||
}
|
||||
}
|
||||
fmt.Printf(" Saved %d sources.\n", len(datas.sources))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type datas struct {
|
||||
config chronograf.Config
|
||||
dashboards []chronograf.Dashboard
|
||||
mappings []chronograf.Mapping
|
||||
orgConfigs []chronograf.OrganizationConfig
|
||||
organizations []chronograf.Organization
|
||||
servers []chronograf.Server
|
||||
sources []chronograf.Source
|
||||
users []chronograf.User
|
||||
}
|
||||
|
||||
func errExit(err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
fmt.Println(err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
|
@ -16,6 +16,33 @@ type organizationConfigStore struct {
|
|||
client *Service
|
||||
}
|
||||
|
||||
// All returns all known organization configurations.
|
||||
func (s *organizationConfigStore) All(ctx context.Context) ([]chronograf.OrganizationConfig, error) {
|
||||
var orgCfgs []chronograf.OrganizationConfig
|
||||
err := s.each(ctx, func(o *chronograf.OrganizationConfig) {
|
||||
orgCfgs = append(orgCfgs, *o)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return orgCfgs, nil
|
||||
}
|
||||
|
||||
func (s *organizationConfigStore) each(ctx context.Context, fn func(*chronograf.OrganizationConfig)) error {
|
||||
return s.client.kv.View(ctx, func(tx Tx) error {
|
||||
return tx.Bucket(organizationConfigBucket).ForEach(func(k, v []byte) error {
|
||||
var orgCfg chronograf.OrganizationConfig
|
||||
if err := internal.UnmarshalOrganizationConfig(v, &orgCfg); err != nil {
|
||||
return err
|
||||
}
|
||||
fn(&orgCfg)
|
||||
return nil
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func (s *organizationConfigStore) get(ctx context.Context, tx Tx, orgID string, c *chronograf.OrganizationConfig) error {
|
||||
v, err := tx.Bucket(organizationConfigBucket).Get([]byte(orgID))
|
||||
if len(v) == 0 || err != nil {
|
||||
|
|
|
|||
|
|
@ -9,10 +9,15 @@ import (
|
|||
var _ chronograf.OrganizationConfigStore = &OrganizationConfigStore{}
|
||||
|
||||
type OrganizationConfigStore struct {
|
||||
AllF func(ctx context.Context) ([]chronograf.OrganizationConfig, error)
|
||||
FindOrCreateF func(ctx context.Context, id string) (*chronograf.OrganizationConfig, error)
|
||||
PutF func(ctx context.Context, c *chronograf.OrganizationConfig) error
|
||||
}
|
||||
|
||||
func (s *OrganizationConfigStore) All(ctx context.Context) ([]chronograf.OrganizationConfig, error) {
|
||||
return s.AllF(ctx)
|
||||
}
|
||||
|
||||
func (s *OrganizationConfigStore) FindOrCreate(ctx context.Context, id string) (*chronograf.OrganizationConfig, error) {
|
||||
return s.FindOrCreateF(ctx, id)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,12 +10,20 @@ import (
|
|||
// ensure OrganizationConfigStore implements chronograf.OrganizationConfigStore
|
||||
var _ chronograf.OrganizationConfigStore = &OrganizationConfigStore{}
|
||||
|
||||
// OrganizationConfigStore is an empty struct for satisfying an interface and returning errors.
|
||||
type OrganizationConfigStore struct{}
|
||||
|
||||
// All returns an error
|
||||
func (s *OrganizationConfigStore) All(context.Context) ([]chronograf.OrganizationConfig, error) {
|
||||
return nil, chronograf.ErrOrganizationConfigNotFound
|
||||
}
|
||||
|
||||
// FindOrCreate returns an error
|
||||
func (s *OrganizationConfigStore) FindOrCreate(context.Context, string) (*chronograf.OrganizationConfig, error) {
|
||||
return nil, chronograf.ErrOrganizationConfigNotFound
|
||||
}
|
||||
|
||||
// Put returns an error
|
||||
func (s *OrganizationConfigStore) Put(context.Context, *chronograf.OrganizationConfig) error {
|
||||
return fmt.Errorf("cannot replace config")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,12 +31,17 @@ func (s *OrganizationConfigStore) FindOrCreate(ctx context.Context, orgID string
|
|||
return nil, err
|
||||
}
|
||||
|
||||
oc, err := s.store.FindOrCreate(ctx, orgID)
|
||||
return s.store.FindOrCreate(ctx, orgID)
|
||||
}
|
||||
|
||||
// All returns all organization configs from the store.
|
||||
func (s *OrganizationConfigStore) All(ctx context.Context) ([]chronograf.OrganizationConfig, error) {
|
||||
var err = validOrganization(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return oc, nil
|
||||
return s.store.All(ctx)
|
||||
}
|
||||
|
||||
// Put the OrganizationConfig in OrganizationConfigStore.
|
||||
|
|
|
|||
Loading…
Reference in New Issue