influxdb/inmem/source.go

148 lines
3.8 KiB
Go
Raw Normal View History

feat(kv): implemented key/value store with end-to-end integration tests * feat(kv:inmem:bolt): implement user service in a kv * refactor(kv): use consistent func receiver name * feat(kv): add initial basic auth service * refactor(passwords): move auth interface into own file * refactor(passwords): rename basic auth files to passwords * refactor(passwords): rename from BasicAuth to Passwords * refactor(kv): copy bolt user test into kv Co-authored-by: Michael Desa <mjdesa@gmail.com> * feat(kv): add inmem testing to kv store * fix(kv): remove extra user index initialization * feat(kv): attempt at making errors nice * fix(http): return not found error if filter is invalid * fix(http): s/platform/influxdb/ for user service * fix(http): s/platform/influxdb/ for user service * feat(kv): initial port of telegraf configs to kv * feat(kv): first pass at migrating bolt org service to kv * feat(kv): first pass at bucket service * feat(kv): first pass at migrating kvlog to kv package * feat(kv): add resource op logs * feat(kv): first pass at user resource mapping migration * feat(kv): add urm usage to bucket and org services * feat(kv): first pass at kv authz service * feat(kv): add cascading auth delete for users * feat(kv): first pass d authorizer.OrganizationService in kv * feat(cmd/influxd/launcher): user kv services where appropriate * fix(kv): initialize authorizations * fix(influxdb): use same buckets while slowly migrating stuff * fix(kv): make staticcheck pass * feat(kv): add dashboards to kv review: make suggestions from pr review fix: use common bucket names for bolt/kv stores * test(kv): add complete password test coverage * chore(kv): fixes for staticcheck * feat(kv): implement labels generically on kv * feat(kv): implement macro service * feat(kv): add source service * feat(kv): add session service * feat(kv): add kv secret service * refactor(kv): update telegraf and urm with error messages * feat(kv): add lookup service * feat(kv): add kv onboarding service * refactor(kv): update telegraf to avoid repetition * feat(cmd/influxd): use kv lookup service * feat(kv): add telegraf to lookup service * feat(cmd/influxd): use kv telegraf service * feat(kv): initial port of scrapers in bolt to kv * feat(kv): update scraper error messaging * feat(cmd/influxd): add kv scraper * feat(kv): add inmem backend tests * refactor(kv): copy paste errors * refactor(kv): add code to password errors * fix(testing): update error messages for incorrect passwords * feat(kv:inmem:bolt): implement user service in a kv * refactor(kv): use consistent func receiver name * refactor(kv): copy bolt user test into kv Co-authored-by: Michael Desa <mjdesa@gmail.com> * feat(kv): add inmem testing to kv store * fix(kv): remove extra user index initialization * feat(kv): attempt at making errors nice * fix(http): return not found error if filter is invalid * fix(http): s/platform/influxdb/ for user service * feat(kv): first pass at migrating bolt org service to kv * feat(kv): first pass at bucket service * feat(kv): first pass at migrating kvlog to kv package * feat(kv): add resource op logs * feat(kv): first pass at user resource mapping migration * feat(kv): add urm usage to bucket and org services * feat(kv): first pass at kv authz service * feat(kv): add cascading auth delete for users * feat(kv): first pass d authorizer.OrganizationService in kv * feat(cmd/influxd/launcher): user kv services where appropriate * feat(kv): add initial basic auth service * refactor(passwords): move auth interface into own file * refactor(passwords): rename basic auth files to passwords * fix(http): s/platform/influxdb/ for user service * fix(kv): initialize authorizations * fix(influxdb): use same buckets while slowly migrating stuff * fix(kv): make staticcheck pass * feat(kv): add dashboards to kv review: make suggestions from pr review fix: use common bucket names for bolt/kv stores * feat(kv): implement labels generically on kv * refactor(passwords): rename from BasicAuth to Passwords * test(kv): add complete password test coverage * chore(kv): fixes for staticcheck * feat(kv): implement macro service * feat(kv): add source service * feat(kv): add session service * feat(kv): initial port of telegraf configs to kv * feat(kv): initial port of scrapers in bolt to kv * feat(kv): add kv secret service * refactor(kv): update telegraf and urm with error messages * feat(kv): add lookup service * feat(kv): add kv onboarding service * refactor(kv): update telegraf to avoid repetition * feat(cmd/influxd): use kv lookup service * feat(kv): add telegraf to lookup service * feat(cmd/influxd): use kv telegraf service * feat(kv): update scraper error messaging * feat(cmd/influxd): add kv scraper * feat(kv): add inmem backend tests * refactor(kv): copy paste errors * refactor(kv): add code to password errors * fix(testing): update error messages for incorrect passwords * feat(http): initial support for flushing all key/values from kv store * feat(kv): rename macro to variable * feat(cmd/influxd/launcher): user kv services where appropriate * refactor(passwords): rename from BasicAuth to Passwords * feat(kv): implement macro service * test(ui): introduce cypress * test(ui): introduce first typescript test * test(ui/e2e): add ci job * chore: update gitignore to ignore test outputs * feat(inmem): in memory influxdb * test(e2e): adding pinger that checks if influxdb is alive * hackathon * hack * hack * hack * hack * Revert "feat(inmem): in memory influxdb" This reverts commit 30ddf032003e704643b07ce80df61c3299ea7295. * hack * hack * hack * hack * hack * hack * hack * hack * hack * hack * hack * hack * hack * chore: lint ignore node_modules * hack * hack * hack * add user and flush * hack * remove unused vars * hack * hack * ci(circle): prefix e2e artifacts * change test to testid * update cypress * moar testid * fix npm warnings * remove absolte path * chore(ci): remove /home/circleci proto mkdir hack * wip: crud resources e2e * fix(inmem): use inmem kv store services * test(dashboard): add first dashboard crud tests * hack * undo hack * fix: use response from setup for orgID * chore: wip * add convenience getByTitle function * test(e2e): ui can create orgs * test(e2e): add test for org deletion and update * test(e2e): introduce task creation test * test(e2e): create and update of buckets on org view * chore: move types to declaration file * chore: use route fixture in dashboard tests * chore(ci): hack back * test(ui): update snapshots * chore: package-lock * chore: remove macros * fix: launcher rebase issues * fix: compile errors * fix: compile errors * feat(cmd/influxdb): add explicit testing, asset-path, and store flags Co-authored-by: Andrew Watkins <watts@influxdb.com> * fix(cmd/influxd): set default HTTP handler and flags Co-authored-by: Andrew Watkins <watts@influxdb.com> * build(Makefile): add run-e2e and PHONY * feat(kv:inmem:bolt): implement user service in a kv * refactor(kv): use consistent func receiver name * feat(kv): add initial basic auth service * refactor(passwords): move auth interface into own file * refactor(passwords): rename basic auth files to passwords * refactor(passwords): rename from BasicAuth to Passwords * refactor(kv): copy bolt user test into kv Co-authored-by: Michael Desa <mjdesa@gmail.com> * feat(kv): add inmem testing to kv store * fix(kv): remove extra user index initialization * feat(kv): attempt at making errors nice * fix(http): return not found error if filter is invalid * fix(http): s/platform/influxdb/ for user service * fix(http): s/platform/influxdb/ for user service * feat(kv): initial port of telegraf configs to kv * feat(kv): initial port of scrapers in bolt to kv * feat(kv): first pass at migrating bolt org service to kv * feat(kv): first pass at bucket service * feat(kv): first pass at migrating kvlog to kv package * feat(kv): add resource op logs * feat(kv): first pass at user resource mapping migration * feat(kv): add urm usage to bucket and org services * feat(kv): first pass at kv authz service * feat(kv): add cascading auth delete for users * feat(kv): first pass d authorizer.OrganizationService in kv * feat(cmd/influxd/launcher): user kv services where appropriate * fix(kv): initialize authorizations * fix(influxdb): use same buckets while slowly migrating stuff * fix(kv): make staticcheck pass * feat(kv): add dashboards to kv review: make suggestions from pr review fix: use common bucket names for bolt/kv stores * test(kv): add complete password test coverage * chore(kv): fixes for staticcheck * feat(kv): implement labels generically on kv * feat(kv): implement macro service * feat(kv): add source service * feat(kv): add session service * feat(kv): add kv secret service * refactor(kv): update telegraf and urm with error messages * feat(kv): add lookup service * feat(kv): add kv onboarding service * refactor(kv): update telegraf to avoid repetition * feat(cmd/influxd): use kv lookup service * feat(kv): add telegraf to lookup service * feat(cmd/influxd): use kv telegraf service * feat(kv): update scraper error messaging * feat(cmd/influxd): add kv scraper * feat(kv): add inmem backend tests * refactor(kv): copy paste errors * refactor(kv): add code to password errors * fix(testing): update error messages for incorrect passwords * feat(kv): rename macro to variable * refactor(kv): auth/bucket/org/user unique checks return errors now * feat(inmem): add way to get all bucket names from store * feat(inmem): Buckets to return slice of bytes rather than strings * feat(inmem): add locks around Buckets to avoid races * feat(cmd/influx): check for unauthorized error in wrapCheckSetup * chore(e2e): add video and screenshot artifcats to gitignore * docs(ci): add build instructions for e2e tests * feat(kv): add id lookup for authorized resources
2019-02-19 23:47:19 +00:00
package inmem
import (
"context"
"fmt"
platform "github.com/influxdata/influxdb"
)
// DefaultSource is the default source.
var DefaultSource = platform.Source{
Default: true,
Name: "autogen",
Type: platform.SelfSourceType,
}
const (
// DefaultSourceID it the default source identifier
DefaultSourceID = "020f755c3c082000"
// DefaultSourceOrganizationID is the default source's organization identifier
DefaultSourceOrganizationID = "50616e67652c206c"
)
func init() {
if err := DefaultSource.ID.DecodeFromString(DefaultSourceID); err != nil {
panic(fmt.Sprintf("failed to decode default source id: %v", err))
}
if err := DefaultSource.OrganizationID.DecodeFromString(DefaultSourceOrganizationID); err != nil {
panic(fmt.Sprintf("failed to decode default source organization id: %v", err))
}
}
func (s *Service) initializeSources(ctx context.Context) error {
_, pe := s.FindSourceByID(ctx, DefaultSource.ID)
if pe != nil && platform.ErrorCode(pe) != platform.ENotFound {
return pe
}
if platform.ErrorCode(pe) == platform.ENotFound {
if err := s.PutSource(ctx, &DefaultSource); err != nil {
return err
}
}
return nil
}
// DefaultSource retrieves the default source.
func (s *Service) DefaultSource(ctx context.Context) (*platform.Source, error) {
// TODO(desa): make this faster by putting the default source in an index.
srcs, _, err := s.FindSources(ctx, platform.FindOptions{})
if err != nil {
return nil, err
}
for _, src := range srcs {
if src.Default {
return src, nil
}
}
return nil, &platform.Error{
Code: platform.ENotFound,
Msg: "no default source found",
}
}
// FindSourceByID retrieves a source by id.
func (s *Service) FindSourceByID(ctx context.Context, id platform.ID) (*platform.Source, error) {
i, ok := s.sourceKV.Load(id.String())
if !ok {
return nil, &platform.Error{
Code: platform.ENotFound,
Msg: platform.ErrSourceNotFound,
}
}
src, ok := i.(*platform.Source)
if !ok {
return nil, &platform.Error{
Code: platform.EInvalid,
Msg: fmt.Sprintf("type %T is not a source", i),
}
}
return src, nil
}
// FindSources retrives all sources that match an arbitrary source filter.
// Filters using ID, or OrganizationID and source Name should be efficient.
// Other filters will do a linear scan across all sources searching for a match.
func (s *Service) FindSources(ctx context.Context, opt platform.FindOptions) ([]*platform.Source, int, error) {
var ds []*platform.Source
s.sourceKV.Range(func(k, v interface{}) bool {
d, ok := v.(*platform.Source)
if !ok {
return false
}
ds = append(ds, d)
return true
})
return ds, len(ds), nil
}
// CreateSource creates a platform source and sets s.ID.
func (s *Service) CreateSource(ctx context.Context, src *platform.Source) error {
src.ID = s.IDGenerator.ID()
if err := s.PutSource(ctx, src); err != nil {
return &platform.Error{
Err: err,
}
}
return nil
}
// PutSource will put a source without setting an ID.
func (s *Service) PutSource(ctx context.Context, src *platform.Source) error {
s.sourceKV.Store(src.ID.String(), src)
return nil
}
// UpdateSource updates a source according the parameters set on upd.
func (s *Service) UpdateSource(ctx context.Context, id platform.ID, upd platform.SourceUpdate) (*platform.Source, error) {
src, err := s.FindSourceByID(ctx, id)
if err != nil {
return nil, &platform.Error{
Err: err,
Op: OpPrefix + platform.OpUpdateView,
}
}
upd.Apply(src)
s.sourceKV.Store(src.ID.String(), src)
return src, nil
}
// DeleteSource deletes a source and prunes it from the index.
func (s *Service) DeleteSource(ctx context.Context, id platform.ID) error {
if _, err := s.FindSourceByID(ctx, id); err != nil {
return &platform.Error{
Err: err,
Op: OpPrefix + platform.OpDeleteView,
}
}
s.sourceKV.Delete(id.String())
return nil
}