Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'feature/dashboard-filestore' of github.com:influxdata/chronograf into feature/dashboard-filestore

pull/10616/head
Chris Goller 2017-12-19 15:21:45 -06:00
parent a91953ecfb ca055fe410
commit a8beb49ec3
11 changed files with 680 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bolt/users_test.go
View File

@ -2,7 +2,6 @@ package bolt_test
import (
"context"
"fmt"
"testing"
"github.com/google/go-cmp/cmp"
@ -303,7 +302,6 @@ func TestUsersStore_Delete(t *testing.T) {
if tt.addFirst {
var err error
tt.args.user, err = s.Add(tt.args.ctx, tt.args.user)
fmt.Println(err)
}
if err := s.Delete(tt.args.ctx, tt.args.user); (err != nil) != tt.wantErr {
t.Errorf("%q. UsersStore.Delete() error = %v, wantErr %v", tt.name, err, tt.wantErr)

2
chronograf.go
View File

@ -17,6 +17,8 @@ const (
ErrUserNotFound = Error("user not found")
ErrLayoutInvalid = Error("layout is invalid")
ErrDashboardInvalid = Error("dashboard is invalid")
ErrSourceInvalid = Error("source is invalid")
ErrServerInvalid = Error("server is invalid")
ErrAlertNotFound = Error("alert not found")
ErrAuthentication = Error("user not authenticated")
ErrUninitialized = Error("client uninitialized. Call Open() method")

184
filestore/kapacitors.go Normal file
View File

@ -0,0 +1,184 @@
package filestore
import (
"context"
"fmt"
"io/ioutil"
"os"
"path"
"strconv"
"github.com/influxdata/chronograf"
)
// KapExt is the the file extension searched for in the directory for kapacitor files
const KapExt = ".kap"
var _ chronograf.ServersStore = &Kapacitors{}
// Kapacitors are JSON kapacitors stored in the filesystem
type Kapacitors struct {
Dir string // Dir is the directory containing the kapacitors.
Load func(string, interface{}) error // Load loads string name and dashbaord passed in as interface
Create func(string, interface{}) error // Create will write kapacitor to file.
ReadDir func(dirname string) ([]os.FileInfo, error) // ReadDir reads the directory named by dirname and returns a list of directory entries sorted by filename.
Remove func(name string) error // Remove file
IDs chronograf.ID // IDs generate unique ids for new kapacitors
Logger chronograf.Logger
}
// NewKapacitors constructs a kapacitor store wrapping a file system directory
func NewKapacitors(dir string, ids chronograf.ID, logger chronograf.Logger) chronograf.ServersStore {
return &Kapacitors{
Dir: dir,
Load: load,
Create: create,
ReadDir: ioutil.ReadDir,
Remove: os.Remove,
IDs: ids,
Logger: logger,
}
}
func kapacitorFile(dir string, kapacitor chronograf.Server) string {
base := fmt.Sprintf("%s%s", kapacitor.Name, KapExt)
return path.Join(dir, base)
}
// All returns all kapacitors from the directory
func (d *Kapacitors) All(ctx context.Context) ([]chronograf.Server, error) {
files, err := d.ReadDir(d.Dir)
if err != nil {
return nil, err
}
kapacitors := []chronograf.Server{}
for _, file := range files {
if path.Ext(file.Name()) != KapExt {
continue
}
var kapacitor chronograf.Server
if err := d.Load(path.Join(d.Dir, file.Name()), &kapacitor); err != nil {
continue // We want to load all files we can.
} else {
kapacitors = append(kapacitors, kapacitor)
}
}
return kapacitors, nil
}
// Add creates a new kapacitor within the directory
func (d *Kapacitors) Add(ctx context.Context, kapacitor chronograf.Server) (chronograf.Server, error) {
genID, err := d.IDs.Generate()
if err != nil {
d.Logger.
WithField("component", "kapacitor").
Error("Unable to generate ID")
return chronograf.Server{}, err
}
id, err := strconv.Atoi(genID)
if err != nil {
d.Logger.
WithField("component", "kapacitor").
Error("Unable to convert ID")
return chronograf.Server{}, err
}
kapacitor.ID = id
file := kapacitorFile(d.Dir, kapacitor)
if err = d.Create(file, kapacitor); err != nil {
if err == chronograf.ErrServerInvalid {
d.Logger.
WithField("component", "kapacitor").
WithField("name", file).
Error("Invalid Server: ", err)
} else {
d.Logger.
WithField("component", "kapacitor").
WithField("name", file).
Error("Unable to write kapacitor:", err)
}
return chronograf.Server{}, err
}
return kapacitor, nil
}
// Delete removes a kapacitor file from the directory
func (d *Kapacitors) Delete(ctx context.Context, kapacitor chronograf.Server) error {
_, file, err := d.idToFile(kapacitor.ID)
if err != nil {
return err
}
if err := d.Remove(file); err != nil {
d.Logger.
WithField("component", "kapacitor").
WithField("name", file).
Error("Unable to remove kapacitor:", err)
return err
}
return nil
}
// Get returns a kapacitor file from the kapacitor directory
func (d *Kapacitors) Get(ctx context.Context, id int) (chronograf.Server, error) {
board, file, err := d.idToFile(id)
if err != nil {
if err == chronograf.ErrServerNotFound {
d.Logger.
WithField("component", "kapacitor").
WithField("name", file).
Error("Unable to read file")
} else if err == chronograf.ErrServerInvalid {
d.Logger.
WithField("component", "kapacitor").
WithField("name", file).
Error("File is not a kapacitor")
}
return chronograf.Server{}, err
}
return board, nil
}
// Update replaces a kapacitor from the file system directory
func (d *Kapacitors) Update(ctx context.Context, kapacitor chronograf.Server) error {
board, _, err := d.idToFile(kapacitor.ID)
if err != nil {
return err
}
if err := d.Delete(ctx, board); err != nil {
return err
}
file := kapacitorFile(d.Dir, kapacitor)
return d.Create(file, kapacitor)
}
// idToFile takes an id and finds the associated filename
func (d *Kapacitors) idToFile(id int) (chronograf.Server, string, error) {
// Because the entire kapacitor information is not known at this point, we need
// to try to find the name of the file through matching the ID in the kapacitor
// content with the ID passed.
files, err := d.ReadDir(d.Dir)
if err != nil {
return chronograf.Server{}, "", err
}
for _, f := range files {
if path.Ext(f.Name()) != KapExt {
continue
}
file := path.Join(d.Dir, f.Name())
var kapacitor chronograf.Server
if err := d.Load(file, &kapacitor); err != nil {
return chronograf.Server{}, "", err
}
if kapacitor.ID == id {
return kapacitor, file, nil
}
}
return chronograf.Server{}, "", chronograf.ErrServerNotFound
}

184
filestore/sources.go Normal file
View File

@ -0,0 +1,184 @@
package filestore
import (
"context"
"fmt"
"io/ioutil"
"os"
"path"
"strconv"
"github.com/influxdata/chronograf"
)
// SrcExt is the the file extension searched for in the directory for source files
const SrcExt = ".src"
var _ chronograf.SourcesStore = &Sources{}
// Sources are JSON sources stored in the filesystem
type Sources struct {
Dir string // Dir is the directory containing the sources.
Load func(string, interface{}) error // Load loads string name and dashbaord passed in as interface
Create func(string, interface{}) error // Create will write source to file.
ReadDir func(dirname string) ([]os.FileInfo, error) // ReadDir reads the directory named by dirname and returns a list of directory entries sorted by filename.
Remove func(name string) error // Remove file
IDs chronograf.ID // IDs generate unique ids for new sources
Logger chronograf.Logger
}
// NewSources constructs a source store wrapping a file system directory
func NewSources(dir string, ids chronograf.ID, logger chronograf.Logger) chronograf.SourcesStore {
return &Sources{
Dir: dir,
Load: load,
Create: create,
ReadDir: ioutil.ReadDir,
Remove: os.Remove,
IDs: ids,
Logger: logger,
}
}
func sourceFile(dir string, source chronograf.Source) string {
base := fmt.Sprintf("%s%s", source.Name, SrcExt)
return path.Join(dir, base)
}
// All returns all sources from the directory
func (d *Sources) All(ctx context.Context) ([]chronograf.Source, error) {
files, err := d.ReadDir(d.Dir)
if err != nil {
return nil, err
}
sources := []chronograf.Source{}
for _, file := range files {
if path.Ext(file.Name()) != SrcExt {
continue
}
var source chronograf.Source
if err := d.Load(path.Join(d.Dir, file.Name()), &source); err != nil {
continue // We want to load all files we can.
} else {
sources = append(sources, source)
}
}
return sources, nil
}
// Add creates a new source within the directory
func (d *Sources) Add(ctx context.Context, source chronograf.Source) (chronograf.Source, error) {
genID, err := d.IDs.Generate()
if err != nil {
d.Logger.
WithField("component", "source").
Error("Unable to generate ID")
return chronograf.Source{}, err
}
id, err := strconv.Atoi(genID)
if err != nil {
d.Logger.
WithField("component", "source").
Error("Unable to convert ID")
return chronograf.Source{}, err
}
source.ID = id
file := sourceFile(d.Dir, source)
if err = d.Create(file, source); err != nil {
if err == chronograf.ErrSourceInvalid {
d.Logger.
WithField("component", "source").
WithField("name", file).
Error("Invalid Source: ", err)
} else {
d.Logger.
WithField("component", "source").
WithField("name", file).
Error("Unable to write source:", err)
}
return chronograf.Source{}, err
}
return source, nil
}
// Delete removes a source file from the directory
func (d *Sources) Delete(ctx context.Context, source chronograf.Source) error {
_, file, err := d.idToFile(source.ID)
if err != nil {
return err
}
if err := d.Remove(file); err != nil {
d.Logger.
WithField("component", "source").
WithField("name", file).
Error("Unable to remove source:", err)
return err
}
return nil
}
// Get returns a source file from the source directory
func (d *Sources) Get(ctx context.Context, id int) (chronograf.Source, error) {
board, file, err := d.idToFile(id)
if err != nil {
if err == chronograf.ErrSourceNotFound {
d.Logger.
WithField("component", "source").
WithField("name", file).
Error("Unable to read file")
} else if err == chronograf.ErrSourceInvalid {
d.Logger.
WithField("component", "source").
WithField("name", file).
Error("File is not a source")
}
return chronograf.Source{}, err
}
return board, nil
}
// Update replaces a source from the file system directory
func (d *Sources) Update(ctx context.Context, source chronograf.Source) error {
board, _, err := d.idToFile(source.ID)
if err != nil {
return err
}
if err := d.Delete(ctx, board); err != nil {
return err
}
file := sourceFile(d.Dir, source)
return d.Create(file, source)
}
// idToFile takes an id and finds the associated filename
func (d *Sources) idToFile(id int) (chronograf.Source, string, error) {
// Because the entire source information is not known at this point, we need
// to try to find the name of the file through matching the ID in the source
// content with the ID passed.
files, err := d.ReadDir(d.Dir)
if err != nil {
return chronograf.Source{}, "", err
}
for _, f := range files {
if path.Ext(f.Name()) != SrcExt {
continue
}
file := path.Join(d.Dir, f.Name())
var source chronograf.Source
if err := d.Load(file, &source); err != nil {
return chronograf.Source{}, "", err
}
if source.ID == id {
return source, file, nil
}
}
return chronograf.Source{}, "", chronograf.ErrSourceNotFound
}

268
integrations/server_test.go
View File

@ -53,6 +53,256 @@ func TestServer(t *testing.T) {
args args
wants wants
}{
{
name: "GET /sources/5000",
subName: "Get specific source; including Canned source",
fields: fields{
Users: []chronograf.User{
{
ID: 1, // This is artificial, but should be reflective of the users actual ID
Name: "billibob",
Provider: "github",
Scheme: "oauth2",
SuperAdmin: true,
Roles: []chronograf.Role{
{
Name: "admin",
Organization: "default",
},
{
Name: "viewer",
Organization: "howdy", // from canned testdata
},
},
},
},
},
args: args{
server: &server.Server{
GithubClientID: "not empty",
GithubClientSecret: "not empty",
},
method: "GET",
path: "/chronograf/v1/sources/5000",
principal: oauth2.Principal{
Organization: "howdy",
Subject: "billibob",
Issuer: "github",
},
},
wants: wants{
statusCode: 200,
body: `
{
"id": "5000",
"name": "Influx 1",
"type": "influx-enterprise",
"username": "user1",
"url": "http://localhost:8086",
"metaUrl": "http://metaurl.com",
"default": true,
"telegraf": "telegraf",
"organization": "howdy",
"links": {
"self": "/chronograf/v1/sources/5000",
"kapacitors": "/chronograf/v1/sources/5000/kapacitors",
"proxy": "/chronograf/v1/sources/5000/proxy",
"queries": "/chronograf/v1/sources/5000/queries",
"write": "/chronograf/v1/sources/5000/write",
"permissions": "/chronograf/v1/sources/5000/permissions",
"users": "/chronograf/v1/sources/5000/users",
"roles": "/chronograf/v1/sources/5000/roles",
"databases": "/chronograf/v1/sources/5000/dbs"
}
}
`,
},
},
{
name: "GET /sources/5000/kapacitors/5000",
subName: "Get specific kapacitors; including Canned kapacitors",
fields: fields{
Users: []chronograf.User{
{
ID: 1, // This is artificial, but should be reflective of the users actual ID
Name: "billibob",
Provider: "github",
Scheme: "oauth2",
SuperAdmin: true,
Roles: []chronograf.Role{
{
Name: "admin",
Organization: "default",
},
{
Name: "viewer",
Organization: "howdy", // from canned testdata
},
},
},
},
},
args: args{
server: &server.Server{
GithubClientID: "not empty",
GithubClientSecret: "not empty",
},
method: "GET",
path: "/chronograf/v1/sources/5000/kapacitors/5000",
principal: oauth2.Principal{
Organization: "howdy",
Subject: "billibob",
Issuer: "github",
},
},
wants: wants{
statusCode: 200,
body: `
{
"id": "5000",
"name": "Kapa 1",
"url": "http://localhost:9092",
"active": true,
"links": {
"proxy": "/chronograf/v1/sources/5000/kapacitors/5000/proxy",
"self": "/chronograf/v1/sources/5000/kapacitors/5000",
"rules": "/chronograf/v1/sources/5000/kapacitors/5000/rules",
"tasks": "/chronograf/v1/sources/5000/kapacitors/5000/proxy?path=/kapacitor/v1/tasks",
"ping": "/chronograf/v1/sources/5000/kapacitors/5000/proxy?path=/kapacitor/v1/ping"
}
}
`,
},
},
{
name: "GET /sources/5000/kapacitors",
subName: "Get all kapacitors; including Canned kapacitors",
fields: fields{
Users: []chronograf.User{
{
ID: 1, // This is artificial, but should be reflective of the users actual ID
Name: "billibob",
Provider: "github",
Scheme: "oauth2",
SuperAdmin: true,
Roles: []chronograf.Role{
{
Name: "admin",
Organization: "default",
},
{
Name: "viewer",
Organization: "howdy", // from canned testdata
},
},
},
},
},
args: args{
server: &server.Server{
GithubClientID: "not empty",
GithubClientSecret: "not empty",
},
method: "GET",
path: "/chronograf/v1/sources/5000/kapacitors",
principal: oauth2.Principal{
Organization: "howdy",
Subject: "billibob",
Issuer: "github",
},
},
wants: wants{
statusCode: 200,
body: `
{
"kapacitors": [
{
"id": "5000",
"name": "Kapa 1",
"url": "http://localhost:9092",
"active": true,
"links": {
"proxy": "/chronograf/v1/sources/5000/kapacitors/5000/proxy",
"self": "/chronograf/v1/sources/5000/kapacitors/5000",
"rules": "/chronograf/v1/sources/5000/kapacitors/5000/rules",
"tasks": "/chronograf/v1/sources/5000/kapacitors/5000/proxy?path=/kapacitor/v1/tasks",
"ping": "/chronograf/v1/sources/5000/kapacitors/5000/proxy?path=/kapacitor/v1/ping"
}
}
]
}
`,
},
},
{
name: "GET /sources",
subName: "Get all sources; including Canned sources",
fields: fields{
Users: []chronograf.User{
{
ID: 1, // This is artificial, but should be reflective of the users actual ID
Name: "billibob",
Provider: "github",
Scheme: "oauth2",
SuperAdmin: true,
Roles: []chronograf.Role{
{
Name: "admin",
Organization: "default",
},
{
Name: "viewer",
Organization: "howdy", // from canned testdata
},
},
},
},
},
args: args{
server: &server.Server{
GithubClientID: "not empty",
GithubClientSecret: "not empty",
},
method: "GET",
path: "/chronograf/v1/sources",
principal: oauth2.Principal{
Organization: "howdy",
Subject: "billibob",
Issuer: "github",
},
},
wants: wants{
statusCode: 200,
body: `
{
"sources": [
{
"id": "5000",
"name": "Influx 1",
"type": "influx-enterprise",
"username": "user1",
"url": "http://localhost:8086",
"metaUrl": "http://metaurl.com",
"default": true,
"telegraf": "telegraf",
"organization": "howdy",
"links": {
"self": "/chronograf/v1/sources/5000",
"kapacitors": "/chronograf/v1/sources/5000/kapacitors",
"proxy": "/chronograf/v1/sources/5000/proxy",
"queries": "/chronograf/v1/sources/5000/queries",
"write": "/chronograf/v1/sources/5000/write",
"permissions": "/chronograf/v1/sources/5000/permissions",
"users": "/chronograf/v1/sources/5000/users",
"roles": "/chronograf/v1/sources/5000/roles",
"databases": "/chronograf/v1/sources/5000/dbs"
}
}
]
}
`,
},
},
{
name: "GET /organizations",
subName: "Get all organizations; including Canned organization",
@ -165,7 +415,7 @@ func TestServer(t *testing.T) {
},
{
name: "GET /dashboards/1000",
subName: "Get specific in the default organization; Using Canned testdata",
subName: "Get specific in the howdy organization; Using Canned testdata",
fields: fields{
Users: []chronograf.User{
{
@ -177,7 +427,7 @@ func TestServer(t *testing.T) {
Roles: []chronograf.Role{
{
Name: "admin",
Organization: "default",
Organization: "howdy",
},
},
},
@ -191,7 +441,7 @@ func TestServer(t *testing.T) {
method: "GET",
path: "/chronograf/v1/dashboards/1000",
principal: oauth2.Principal{
Organization: "default",
Organization: "howdy",
Subject: "billibob",
Issuer: "github",
},
@ -387,7 +637,7 @@ func TestServer(t *testing.T) {
}
],
"name": "Name This Dashboard",
"organization": "default",
"organization": "howdy",
"links": {
"self": "/chronograf/v1/dashboards/1000",
"cells": "/chronograf/v1/dashboards/1000/cells",
@ -398,7 +648,7 @@ func TestServer(t *testing.T) {
},
{
name: "GET /dashboards",
subName: "Get all dashboards in the default organization; Using Canned testdata",
subName: "Get all dashboards in the howdy organization; Using Canned testdata",
fields: fields{
Users: []chronograf.User{
{
@ -412,6 +662,10 @@ func TestServer(t *testing.T) {
Name: "admin",
Organization: "default",
},
{
Name: "admin",
Organization: "howdy",
},
},
},
},
@ -424,7 +678,7 @@ func TestServer(t *testing.T) {
method: "GET",
path: "/chronograf/v1/dashboards",
principal: oauth2.Principal{
Organization: "default",
Organization: "howdy",
Subject: "billibob",
Issuer: "github",
},
@ -622,7 +876,7 @@ func TestServer(t *testing.T) {
}
],
"name": "Name This Dashboard",
"organization": "default",
"organization": "howdy",
"links": {
"self": "/chronograf/v1/dashboards/1000",
"cells": "/chronograf/v1/dashboards/1000/cells",

8
integrations/testdata/example.kap vendored Normal file
View File

@ -0,0 +1,8 @@
{
"id": 5000,
"srcID": 5000,
"name": "Kapa 1",
"url": "http://localhost:9092",
"active": true,
"organization": "howdy"
}

14
integrations/testdata/example.src vendored Normal file
View File

@ -0,0 +1,14 @@
{
"id": "5000",
"name": "Influx 1",
"username": "user1",
"password": "pass1",
"url": "http://localhost:8086",
"metaUrl": "http://metaurl.com",
"type": "influx-enterprise",
"insecureSkipVerify": false,
"default": true,
"telegraf": "telegraf",
"sharedSecret": "cubeapples",
"organization": "howdy"
}

2
integrations/testdata/mydash.dashboard vendored
View File

@ -181,5 +181,5 @@
}
],
"name": "Name This Dashboard",
"organization": "default"
"organization": "howdy"
}

22
server/builders.go
View File

@ -1,6 +1,8 @@
package server
import (
"context"
"github.com/influxdata/chronograf"
"github.com/influxdata/chronograf/canned"
"github.com/influxdata/chronograf/filestore"
@ -82,11 +84,19 @@ type MultiSourceBuilder struct {
InfluxDBURL string
InfluxDBUsername string
InfluxDBPassword string
Logger chronograf.Logger
ID chronograf.ID
Path string
}
// Build will return a MultiSourceStore
func (fs *MultiSourceBuilder) Build(db chronograf.SourcesStore) (*multistore.SourcesStore, error) {
stores := []chronograf.SourcesStore{db}
// These dashboards are those handled from a directory
files := filestore.NewSources(fs.Path, fs.ID, fs.Logger)
xs, err := files.All(context.Background())
stores := []chronograf.SourcesStore{db, files}
if fs.InfluxDBURL != "" {
influxStore := &memdb.SourcesStore{
@ -118,11 +128,19 @@ type MultiKapacitorBuilder struct {
KapacitorURL string
KapacitorUsername string
KapacitorPassword string
Logger chronograf.Logger
ID chronograf.ID
Path string
}
// Build will return a multistore facade KapacitorStore over memdb and bolt
func (builder *MultiKapacitorBuilder) Build(db chronograf.ServersStore) (*multistore.KapacitorStore, error) {
stores := []chronograf.ServersStore{db}
// These dashboards are those handled from a directory
files := filestore.NewKapacitors(builder.Path, builder.ID, builder.Logger)
stores := []chronograf.ServersStore{db, files}
if builder.KapacitorURL != "" {
memStore := &memdb.KapacitorStore{
Kapacitor: &chronograf.Server{

6
server/server.go
View File

@ -294,11 +294,17 @@ func (s *Server) newBuilders(logger chronograf.Logger) builders {
InfluxDBURL: s.InfluxDBURL,
InfluxDBUsername: s.InfluxDBUsername,
InfluxDBPassword: s.InfluxDBPassword,
Logger: logger,
ID: idgen.NewTime(),
Path: s.CannedPath,
},
Kapacitors: &MultiKapacitorBuilder{
KapacitorURL: s.KapacitorURL,
KapacitorUsername: s.KapacitorUsername,
KapacitorPassword: s.KapacitorPassword,
Logger: logger,
ID: idgen.NewTime(),
Path: s.CannedPath,
},
Organizations: &MultiOrganizationBuilder{
Logger: logger,

2
server/stores_test.go
View File

@ -2,7 +2,6 @@ package server
import (
"context"
"fmt"
"testing"
"github.com/google/go-cmp/cmp"
@ -341,7 +340,6 @@ func TestStore_OrganizationsAdd(t *testing.T) {
fields: fields{
OrganizationsStore: &mocks.OrganizationsStore{
GetF: func(ctx context.Context, q chronograf.OrganizationQuery) (*chronograf.Organization, error) {
fmt.Println(*q.ID)
return &chronograf.Organization{
ID: "22",
Name: "my sweet name",