Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

add telegraf store service

pull/10616/head
Kelvin Wang 2018-10-15 15:17:01 -04:00
parent ea4c08b327
commit a8ac4a931c
11 changed files with 2073 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
bolt/bbolt.go
View File

@ -108,6 +108,11 @@ func (c *Client) initialize(ctx context.Context) error {
return err
}
// Always create Telegraf Config bucket.
if err := c.initializeTelegraf(ctx, tx); err != nil {
return err
}
// Always create Source bucket.
if err := c.initializeSources(ctx, tx); err != nil {
return err

240
bolt/telegraf.go Normal file
View File

@ -0,0 +1,240 @@
package bolt
import (
"context"
"encoding/json"
"fmt"
"time"
bolt "github.com/coreos/bbolt"
"github.com/influxdata/platform"
)
var (
telegrafBucket = []byte("telegrafv1")
)
var _ platform.TelegrafConfigStore = new(Client)
func (c *Client) initializeTelegraf(ctx context.Context, tx *bolt.Tx) error {
if _, err := tx.CreateBucketIfNotExists(telegrafBucket); err != nil {
return err
}
return nil
}
// FindTelegrafConfigByID returns a single telegraf config by ID.
func (c *Client) FindTelegrafConfigByID(ctx context.Context, id platform.ID) (tc *platform.TelegrafConfig, err error) {
op := "bolt/find telegraf config by id"
err = c.db.View(func(tx *bolt.Tx) error {
var pErr *platform.Error
tc, pErr = c.findTelegrafConfigByID(ctx, tx, id)
if pErr != nil {
pErr.Op = op
err = pErr
}
return err
})
return tc, err
}
func (c *Client) findTelegrafConfigByID(ctx context.Context, tx *bolt.Tx, id platform.ID) (*platform.TelegrafConfig, *platform.Error) {
encID, err := id.Encode()
if err != nil {
return nil, &platform.Error{
Code: platform.EEmptyValue,
Err: err,
}
}
d := tx.Bucket(telegrafBucket).Get(encID)
if d == nil {
return nil, &platform.Error{
Code: platform.ENotFound,
Msg: fmt.Sprintf("telegraf config with ID %v not found", id),
}
}
tc := new(platform.TelegrafConfig)
err = json.Unmarshal(d, tc)
if err != nil {
return nil, &platform.Error{
Err: err,
}
}
return tc, nil
}
// FindTelegrafConfig returns the first telegraf config that matches filter.
func (c *Client) FindTelegrafConfig(ctx context.Context, filter platform.UserResourceMappingFilter) (*platform.TelegrafConfig, error) {
op := "bolt/find telegraf config"
tcs, n, err := c.FindTelegrafConfigs(ctx, filter)
if err != nil {
return nil, err
}
if n > 0 {
return tcs[0], nil
}
return nil, &platform.Error{
Code: platform.ENotFound,
Op: op,
}
}
func (c *Client) findTelegrafConfigs(ctx context.Context, tx *bolt.Tx, filter platform.UserResourceMappingFilter, opt ...platform.FindOptions) ([]*platform.TelegrafConfig, int, *platform.Error) {
tcs := make([]*platform.TelegrafConfig, 0)
m, err := c.findUserResourceMappings(ctx, tx, filter)
if err != nil {
return nil, 0, &platform.Error{
Err: err,
}
}
if len(m) == 0 {
return nil, 0, &platform.Error{
Code: platform.ENotFound,
}
}
for _, item := range m {
tc, err := c.findTelegrafConfigByID(ctx, tx, item.ResourceID)
if err != nil {
return nil, 0, &platform.Error{
// return internal error, for any mapping issue
Err: err,
}
}
tcs = append(tcs, tc)
}
if len(tcs) == 0 {
return nil, 0, &platform.Error{
Msg: "inconsistent user resource mapping and telegraf config",
}
}
return tcs, len(tcs), nil
}
// FindTelegrafConfigs returns a list of telegraf configs that match filter and the total count of matching telegraf configs.
// Additional options provide pagination & sorting.
func (c *Client) FindTelegrafConfigs(ctx context.Context, filter platform.UserResourceMappingFilter, opt ...platform.FindOptions) (tcs []*platform.TelegrafConfig, n int, err error) {
op := "bolt/find telegraf configs"
err = c.db.View(func(tx *bolt.Tx) error {
var pErr *platform.Error
tcs, n, pErr = c.findTelegrafConfigs(ctx, tx, filter)
if pErr != nil {
pErr.Op = op
return pErr
}
return nil
})
return tcs, len(tcs), err
}
func (c *Client) putTelegrafConfig(ctx context.Context, tx *bolt.Tx, tc *platform.TelegrafConfig) *platform.Error {
v, err := json.Marshal(tc)
if err != nil {
return &platform.Error{
Err: err,
}
}
encodedID, err := tc.ID.Encode()
if err != nil {
return &platform.Error{
Code: platform.EEmptyValue,
Err: err,
}
}
err = tx.Bucket(telegrafBucket).Put(encodedID, v)
if err != nil {
return &platform.Error{
Err: err,
}
}
return nil
}
// CreateTelegrafConfig creates a new telegraf config and sets b.ID with the new identifier.
func (c *Client) CreateTelegrafConfig(ctx context.Context, tc *platform.TelegrafConfig, userID platform.ID, now time.Time) error {
op := "bolt/create telegraf config"
return c.db.Update(func(tx *bolt.Tx) error {
tc.ID = c.IDGenerator.ID()
err := c.createUserResourceMapping(ctx, tx, &platform.UserResourceMapping{
ResourceID: tc.ID,
UserID: userID,
UserType: platform.Owner,
ResourceType: platform.TelegrafResourceType,
})
if err != nil {
return err
}
tc.Created = now
tc.LastMod = now
tc.LastModBy = userID
pErr := c.putTelegrafConfig(ctx, tx, tc)
if pErr != nil {
pErr.Op = op
err = pErr
}
return err
})
}
// UpdateTelegrafConfig updates a single telegraf config.
// Returns the new telegraf config after update.
func (c *Client) UpdateTelegrafConfig(ctx context.Context, id platform.ID, tc *platform.TelegrafConfig, userID platform.ID, now time.Time) (*platform.TelegrafConfig, error) {
op := "bolt/update telegraf config"
err := c.db.Update(func(tx *bolt.Tx) (err error) {
_, pErr := c.findTelegrafConfigByID(ctx, tx, id)
if pErr != nil {
pErr.Op = op
err = pErr
}
tc.ID = id
tc.LastMod = now
tc.LastModBy = userID
pErr = c.putTelegrafConfig(ctx, tx, tc)
if pErr != nil {
pErr.Op = op
err = pErr
}
return err
})
return tc, err
}
// DeleteTelegrafConfig removes a telegraf config by ID.
func (c *Client) DeleteTelegrafConfig(ctx context.Context, id platform.ID) error {
op := "bolt/delete telegraf config"
err := c.db.Update(func(tx *bolt.Tx) error {
encodedID, err := id.Encode()
if err != nil {
return &platform.Error{
Code: platform.EEmptyValue,
Err: err,
}
}
err = tx.Bucket(telegrafBucket).Delete(encodedID)
if err != nil {
return err
}
return c.deleteUserResourceMappings(ctx, tx, platform.UserResourceMappingFilter{
ResourceID: id,
ResourceType: platform.TelegrafResourceType,
})
})
if err != nil {
err = &platform.Error{
Code: platform.ErrorCode(err),
Op: op,
Err: err,
}
}
return err
}
// PutTelegrafConfig put a telegraf config to storage
func (c *Client) PutTelegrafConfig(ctx context.Context, tc *platform.TelegrafConfig) error {
return c.db.Update(func(tx *bolt.Tx) (err error) {
pErr := c.putTelegrafConfig(ctx, tx, tc)
if pErr != nil {
err = pErr
}
return nil
})
}

40
bolt/telegraf_test.go Normal file
View File

@ -0,0 +1,40 @@
package bolt_test
import (
"context"
"testing"
"github.com/influxdata/platform"
platformtesting "github.com/influxdata/platform/testing"
)
func initTelegrafConfigStore(f platformtesting.TelegrafConfigFields, t *testing.T) (platform.TelegrafConfigStore, func()) {
c, closeFn, err := NewTestClient()
if err != nil {
t.Fatalf("failed to create new bolt client: %v", err)
}
c.IDGenerator = f.IDGenerator
ctx := context.TODO()
for _, tc := range f.TelegrafConfigs {
if err := c.PutTelegrafConfig(ctx, tc); err != nil {
t.Fatalf("failed to populate telegraf config: %s", err.Error())
}
}
for _, m := range f.UserResourceMappings {
if err := c.CreateUserResourceMapping(ctx, m); err != nil {
t.Fatalf("failed to populate user resource mapping")
}
}
return c, func() {
defer closeFn()
for _, tc := range f.TelegrafConfigs {
if err := c.DeleteTelegrafConfig(ctx, tc.ID); err != nil {
t.Logf("failed to remove telegraf config: %v", err)
}
}
}
}
func TestTelegrafConfigStore(t *testing.T) {
platformtesting.TelegrafConfigStore(initTelegrafConfigStore, t)
}

63
bolt/user_resource_mapping.go
View File

@ -89,30 +89,34 @@ func (c *Client) findUserResourceMapping(ctx context.Context, tx *bolt.Tx, resou
func (c *Client) CreateUserResourceMapping(ctx context.Context, m *platform.UserResourceMapping) error {
return c.db.Update(func(tx *bolt.Tx) error {
unique := c.uniqueUserResourceMapping(ctx, tx, m)
if !unique {
return fmt.Errorf("mapping for user %s already exists", m.UserID.String())
}
v, err := json.Marshal(m)
if err != nil {
return err
}
key, err := userResourceKey(m)
if err != nil {
return err
}
if err := tx.Bucket(userResourceMappingBucket).Put(key, v); err != nil {
return err
}
return nil
return c.createUserResourceMapping(ctx, tx, m)
})
}
func (c *Client) createUserResourceMapping(ctx context.Context, tx *bolt.Tx, m *platform.UserResourceMapping) error {
unique := c.uniqueUserResourceMapping(ctx, tx, m)
if !unique {
return fmt.Errorf("mapping for user %s already exists", m.UserID.String())
}
v, err := json.Marshal(m)
if err != nil {
return err
}
key, err := userResourceKey(m)
if err != nil {
return err
}
if err := tx.Bucket(userResourceMappingBucket).Put(key, v); err != nil {
return err
}
return nil
}
func userResourceKey(m *platform.UserResourceMapping) ([]byte, error) {
encodedResourceID, err := m.ResourceID.Encode()
if err != nil {
@ -175,3 +179,20 @@ func (c *Client) deleteUserResourceMapping(ctx context.Context, tx *bolt.Tx, res
return tx.Bucket(userResourceMappingBucket).Delete(key)
}
func (c *Client) deleteUserResourceMappings(ctx context.Context, tx *bolt.Tx, filter platform.UserResourceMappingFilter) error {
ms, err := c.findUserResourceMappings(ctx, tx, filter)
if err != nil {
return err
}
for _, m := range ms {
key, err := userResourceKey(m)
if err != nil {
return err
}
if err = tx.Bucket(userResourceMappingBucket).Delete(key); err != nil {
return err
}
}
return nil
}

1
inmem/service.go
View File

@ -20,6 +20,7 @@ type Service struct {
dbrpMappingKV sync.Map
userResourceMappingKV sync.Map
scraperTargetKV sync.Map
telegrafConfigKV sync.Map
TokenGenerator platform.TokenGenerator
IDGenerator platform.IDGenerator

191
inmem/telegraf.go Normal file
View File

@ -0,0 +1,191 @@
package inmem
import (
"context"
"fmt"
"time"
"github.com/influxdata/platform"
)
var _ platform.TelegrafConfigStore = new(Service)
// FindTelegrafConfigByID returns a single telegraf config by ID.
func (s *Service) FindTelegrafConfigByID(ctx context.Context, id platform.ID) (tc *platform.TelegrafConfig, err error) {
op := "inmem/find telegraf config by id"
var pErr *platform.Error
tc, pErr = s.findTelegrafConfigByID(ctx, id)
if pErr != nil {
pErr.Op = op
err = pErr
}
return tc, err
}
func (s *Service) findTelegrafConfigByID(ctx context.Context, id platform.ID) (*platform.TelegrafConfig, *platform.Error) {
if !id.Valid() {
return nil, &platform.Error{
Code: platform.EEmptyValue,
Err: platform.ErrInvalidID,
}
}
result, found := s.telegrafConfigKV.Load(id)
if !found {
return nil, &platform.Error{
Code: platform.ENotFound,
Msg: fmt.Sprintf("telegraf config with ID %v not found", id),
}
}
tc := new(platform.TelegrafConfig)
*tc = result.(platform.TelegrafConfig)
return tc, nil
}
// FindTelegrafConfig returns the first telegraf config that matches filter.
func (s *Service) FindTelegrafConfig(ctx context.Context, filter platform.UserResourceMappingFilter) (*platform.TelegrafConfig, error) {
op := "inmem/find telegraf config"
tcs, n, err := s.FindTelegrafConfigs(ctx, filter)
if err != nil {
return nil, err
}
if n > 0 {
return tcs[0], nil
}
return nil, &platform.Error{
Code: platform.ENotFound,
Op: op,
}
}
func (s *Service) findTelegrafConfigs(ctx context.Context, filter platform.UserResourceMappingFilter, opt ...platform.FindOptions) ([]*platform.TelegrafConfig, int, *platform.Error) {
tcs := make([]*platform.TelegrafConfig, 0)
m, _, err := s.FindUserResourceMappings(ctx, filter)
if err != nil {
return nil, 0, &platform.Error{
Err: err,
}
}
if len(m) == 0 {
return nil, 0, &platform.Error{
Code: platform.ENotFound,
}
}
for _, item := range m {
tc, err := s.findTelegrafConfigByID(ctx, item.ResourceID)
if err != nil {
return nil, 0, &platform.Error{
// return internal error, for any mapping issue
Err: err,
}
}
tcs = append(tcs, tc)
}
if len(tcs) == 0 {
return nil, 0, &platform.Error{
Msg: "inconsistent user resource mapping and telegraf config",
}
}
return tcs, len(tcs), nil
}
// FindTelegrafConfigs returns a list of telegraf configs that match filter and the total count of matching telegraf configs.
// Additional options provide pagination & sorting.
func (s *Service) FindTelegrafConfigs(ctx context.Context, filter platform.UserResourceMappingFilter, opt ...platform.FindOptions) (tcs []*platform.TelegrafConfig, n int, err error) {
op := "inmem/find telegraf configs"
var pErr *platform.Error
tcs, n, pErr = s.findTelegrafConfigs(ctx, filter)
if pErr != nil {
pErr.Op = op
err = pErr
}
return tcs, len(tcs), err
}
func (s *Service) putTelegrafConfig(ctx context.Context, tc *platform.TelegrafConfig) *platform.Error {
if !tc.ID.Valid() {
return &platform.Error{
Code: platform.EEmptyValue,
Err: platform.ErrInvalidID,
}
}
s.telegrafConfigKV.Store(tc.ID, *tc)
return nil
}
// CreateTelegrafConfig creates a new telegraf config and sets b.ID with the new identifier.
func (s *Service) CreateTelegrafConfig(ctx context.Context, tc *platform.TelegrafConfig, userID platform.ID, now time.Time) error {
op := "inmem/create telegraf config"
tc.ID = s.IDGenerator.ID()
err := s.CreateUserResourceMapping(ctx, &platform.UserResourceMapping{
ResourceID: tc.ID,
UserID: userID,
UserType: platform.Owner,
ResourceType: platform.TelegrafResourceType,
})
if err != nil {
return err
}
tc.Created = now
tc.LastMod = now
tc.LastModBy = userID
pErr := s.putTelegrafConfig(ctx, tc)
if pErr != nil {
pErr.Op = op
err = pErr
}
return err
}
// UpdateTelegrafConfig updates a single telegraf config.
// Returns the new telegraf config after update.
func (s *Service) UpdateTelegrafConfig(ctx context.Context, id platform.ID, tc *platform.TelegrafConfig, userID platform.ID, now time.Time) (*platform.TelegrafConfig, error) {
var err error
op := "inmem/update telegraf config"
_, pErr := s.findTelegrafConfigByID(ctx, id)
if pErr != nil {
pErr.Op = op
err = pErr
}
tc.ID = id
tc.LastMod = now
tc.LastModBy = userID
pErr = s.putTelegrafConfig(ctx, tc)
if pErr != nil {
pErr.Op = op
err = pErr
}
return tc, err
}
// DeleteTelegrafConfig removes a telegraf config by ID.
func (s *Service) DeleteTelegrafConfig(ctx context.Context, id platform.ID) error {
op := "inmem/delete telegraf config"
var err error
if !id.Valid() {
return &platform.Error{
Op: op,
Code: platform.EEmptyValue,
Err: platform.ErrInvalidID,
}
}
if _, pErr := s.findTelegrafConfigByID(ctx, id); pErr != nil {
return pErr
}
s.telegrafConfigKV.Delete(id)
err = s.deleteUserResourceMapping(ctx, platform.UserResourceMappingFilter{
ResourceID: id,
ResourceType: platform.TelegrafResourceType,
})
if err != nil {
return &platform.Error{
Code: platform.ErrorCode(err),
Op: op,
Err: err,
}
}
return nil
}

30
inmem/telegraf_test.go Normal file
View File

@ -0,0 +1,30 @@
package inmem
import (
"context"
"testing"
"github.com/influxdata/platform"
platformtesting "github.com/influxdata/platform/testing"
)
func initTelegrafStore(f platformtesting.TelegrafConfigFields, t *testing.T) (platform.TelegrafConfigStore, func()) {
s := NewService()
s.IDGenerator = f.IDGenerator
ctx := context.Background()
for _, m := range f.UserResourceMappings {
if err := s.PutUserResourceMapping(ctx, m); err != nil {
t.Fatalf("failed to populate user resource mapping")
}
}
for _, tc := range f.TelegrafConfigs {
if err := s.putTelegrafConfig(ctx, tc); err != nil {
t.Fatalf("failed to populate telegraf configs")
}
}
return s, func() {}
}
func TestTelegrafStore(t *testing.T) {
platformtesting.TelegrafConfigStore(initTelegrafStore, t)
}

32
inmem/user_resource_mapping_service.go
View File

@ -75,27 +75,29 @@ func (s *Service) FindUserResourceMappings(ctx context.Context, filter platform.
return true
}
good := true
// Filter by UserID
if filter.UserID.Valid() && filter.UserID == mapping.UserID {
return true
if filter.UserID.Valid() && filter.UserID != mapping.UserID {
good = false
}
// Filter by ResourceID
if filter.ResourceID.Valid() && filter.ResourceID == mapping.ResourceID {
return true
if filter.ResourceID.Valid() && filter.ResourceID != mapping.ResourceID {
good = false
}
// Filter by user type
if filter.UserType == mapping.UserType {
return true
if filter.UserType != "" && filter.UserType != mapping.UserType {
good = false
}
// Filter by resource type
if filter.ResourceType == mapping.ResourceType {
return true
if filter.ResourceType != "" && filter.ResourceType != mapping.ResourceType {
good = false
}
return false
return good
}
mappings, err := s.filterUserResourceMappings(ctx, filterFunc)
@ -135,3 +137,15 @@ func (s *Service) DeleteUserResourceMapping(ctx context.Context, resourceID, use
s.userResourceMappingKV.Delete(encodeUserResourceMappingKey(resourceID, userID))
return nil
}
func (s *Service) deleteUserResourceMapping(ctx context.Context, filter platform.UserResourceMappingFilter) error {
mappings, _, err := s.FindUserResourceMappings(ctx, filter)
if mappings == nil && err != nil {
return err
}
for _, m := range mappings {
s.userResourceMappingKV.Delete(encodeUserResourceMappingKey(m.ResourceID, m.UserID))
}
return nil
}

4
telegraf.go
View File

@ -13,6 +13,10 @@ import (
// TelegrafConfigStore represents a service for managing telegraf config data.
type TelegrafConfigStore interface {
// UserResourceMappingService must be part of all TelegrafConfigStore service,
// for create, search, delete.
UserResourceMappingService
// FindTelegrafConfigByID returns a single telegraf config by ID.
FindTelegrafConfigByID(ctx context.Context, id ID) (*TelegrafConfig, error)

1494
testing/telegraf.go Normal file
View File

File diff suppressed because it is too large Load Diff

4
user_resource_mapping.go
View File

@ -9,6 +9,7 @@ import (
type UserType string
type ResourceType string
// available user resource types.
const (
Owner UserType = "owner"
Member UserType = "member"
@ -17,6 +18,7 @@ const (
TaskResourceType ResourceType = "task"
OrgResourceType ResourceType = "org"
ViewResourceType ResourceType = "view"
TelegrafResourceType ResourceType = "telegraf"
)
// UserResourceMappingService maps the relationships between users and resources
@ -51,7 +53,7 @@ func (m UserResourceMapping) Validate() error {
return errors.New("a valid user type is required")
}
switch m.ResourceType {
case DashboardResourceType, BucketResourceType, TaskResourceType, OrgResourceType, ViewResourceType:
case DashboardResourceType, BucketResourceType, TaskResourceType, OrgResourceType, ViewResourceType, TelegrafResourceType:
default:
return fmt.Errorf("a valid resource type is required")
}