Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat(pkger): adds label support

pull/15574/head
Deary Hudson 2019-10-24 18:59:01 -05:00 committed by Johnny Steenbergen
parent 2c9f0f2d66
commit 220309e498
8 changed files with 473 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
http/label_service.go
View File

@ -636,6 +636,8 @@ func (s *LabelService) CreateLabel(ctx context.Context, l *influxdb.Label) error
return err
}
// this is super dirty >_<
*l = lr.Label
return nil
}

28
pkger/models.go
View File

@ -40,12 +40,26 @@ type Summary struct {
influxdb.Bucket
Associations []influxdb.Label
}
Labels []struct {
influxdb.Label
}
}
type bucket struct {
ID influxdb.ID
OrgID influxdb.ID
Description string
Name string
RetentionPeriod time.Duration
}
type (
bucket struct {
ID influxdb.ID
OrgID influxdb.ID
Description string
Name string
RetentionPeriod time.Duration
}
label struct {
ID influxdb.ID
OrgID influxdb.ID
Name string
Color string
Description string
}
)

65
pkger/parser.go
View File

@ -125,6 +125,7 @@ type Pkg struct {
Resources []Resource `yaml:"resources" json:"resources"`
} `yaml:"spec" json:"spec"`
mLabels map[string]*label
mBuckets map[string]*bucket
}
@ -134,10 +135,32 @@ type Pkg struct {
func (m *Pkg) Summary() Summary {
var sum Summary
type lbl struct {
influxdb.Label
}
for _, l := range m.mLabels {
sum.Labels = append(sum.Labels, lbl{
Label: influxdb.Label{
ID: l.ID,
OrgID: l.OrgID,
Name: l.Name,
Properties: map[string]string{
"color": l.Color,
"description": l.Description,
},
},
})
}
sort.Slice(sum.Labels, func(i, j int) bool {
return sum.Labels[i].Name < sum.Labels[j].Name
})
type bkt struct {
influxdb.Bucket
Associations []influxdb.Label
}
for _, b := range m.mBuckets {
sum.Buckets = append(sum.Buckets, bkt{
Bucket: influxdb.Bucket{
@ -169,6 +192,19 @@ func (m *Pkg) buckets() []*bucket {
return buckets
}
func (m *Pkg) labels() []*label {
labels := make([]*label, 0, len(m.mLabels))
for _, b := range m.mLabels {
labels = append(labels, b)
}
sort.Slice(labels, func(i, j int) bool {
return labels[i].Name < labels[j].Name
})
return labels
}
func (m *Pkg) validMetadata() error {
var failures []*failure
if m.APIVersion != "0.1.0" {
@ -224,6 +260,7 @@ func (m *Pkg) validMetadata() error {
func (m *Pkg) graphResources() error {
graphFns := []func() error{
m.graphLabels,
m.graphBuckets,
}
@ -233,6 +270,8 @@ func (m *Pkg) graphResources() error {
}
}
//todo: make sure a resource was created....
return nil
}
@ -262,6 +301,32 @@ func (m *Pkg) graphBuckets() error {
})
}
func (m *Pkg) graphLabels() error {
m.mLabels = make(map[string]*label)
return m.eachResource(kindLabel, func(r Resource) *failure {
if r.Name() == "" {
return &failure{
field: "name",
msg: "must be a string of at least 2 chars in length",
}
}
if _, ok := m.mLabels[r.Name()]; ok {
return &failure{
field: "name",
msg: "duplicate name: " + r.Name(),
}
}
m.mLabels[r.Name()] = &label{
Name: r.Name(),
Color: r.stringShort("color"),
Description: r.stringShort("description"),
}
return nil
})
}
func (m *Pkg) eachResource(resourceKind kind, fn func(r Resource) *failure) error {
var parseErr ParseErr
for i, r := range m.Spec.Resources {

87
pkger/parser_test.go
View File

@ -140,7 +140,7 @@ spec:
})
t.Run("file with just a bucket", func(t *testing.T) {
t.Run("with valid bucket data should be valid", func(t *testing.T) {
t.Run("with valid bucket pkg should be valid", func(t *testing.T) {
testfileRunner(t, "testdata/bucket", func(t *testing.T, pkg *Pkg) {
buckets := pkg.buckets()
require.Len(t, buckets, 1)
@ -250,6 +250,91 @@ spec:
assert.Equal(t, "name", fields[0].Field)
})
})
t.Run("file with just a label", func(t *testing.T) {
t.Run("with valid label pkg should be valid", func(t *testing.T) {
testfileRunner(t, "testdata/label", func(t *testing.T, pkg *Pkg) {
labels := pkg.labels()
require.Len(t, labels, 2)
expectedLabel1 := label{
Name: "label_1",
Description: "label 1 description",
Color: "#FFFFFF",
}
assert.Equal(t, expectedLabel1, *labels[0])
expectedLabel2 := label{
Name: "label_2",
Description: "label 2 description",
Color: "#000000",
}
assert.Equal(t, expectedLabel2, *labels[1])
})
})
t.Run("with missing label name should error", func(t *testing.T) {
tests := []struct {
name string
numErrs int
in string
}{
{
name: "missing name",
numErrs: 1,
in: `apiVersion: 0.1.0
kind: Package
meta:
pkgName: first_label_pkg
pkgVersion: 1
spec:
resources:
- kind: Label
`,
},
{
name: "mixed valid and missing name",
numErrs: 1,
in: `apiVersion: 0.1.0
kind: Package
meta:
pkgName: label_pkg
pkgVersion: 1
spec:
resources:
- kind: Label
name: valid name
- kind: Label
`,
},
{
name: "multiple labels with missing name",
numErrs: 2,
in: `apiVersion: 0.1.0
kind: Package
meta:
pkgName: label_pkg
pkgVersion: 1
spec:
resources:
- kind: Label
- kind: Label
`,
},
}
for _, tt := range tests {
fn := func(t *testing.T) {
_, err := Parse(EncodingYAML, FromString(tt.in))
pErr, ok := IsParseErr(err)
require.True(t, ok)
assert.Len(t, pErr.Resources, tt.numErrs)
}
t.Run(tt.name, fn)
}
})
})
}
type baseAsserts struct {

142
pkger/service.go
View File

@ -12,13 +12,15 @@ import (
type Service struct {
logger *zap.Logger
labelSVC influxdb.LabelService
bucketSVC influxdb.BucketService
}
func NewService(l *zap.Logger, bucketSVC influxdb.BucketService) *Service {
func NewService(l *zap.Logger, bucketSVC influxdb.BucketService, labelSVC influxdb.LabelService) *Service {
svc := Service{
logger: zap.NewNop(),
bucketSVC: bucketSVC,
labelSVC: labelSVC,
}
if l != nil {
@ -28,18 +30,7 @@ func NewService(l *zap.Logger, bucketSVC influxdb.BucketService) *Service {
}
func (s *Service) Apply(ctx context.Context, orgID influxdb.ID, pkg *Pkg) (sum Summary, e error) {
type rollback struct {
resource string
fn func() error
}
var newBuckets []influxdb.Bucket
rollbacks := []rollback{
{
resource: "bucket",
fn: func() error { return s.deleteBuckets(newBuckets) },
},
}
var rollbacks []rollbacker
defer func() {
if e == nil {
return
@ -51,38 +42,69 @@ func (s *Service) Apply(ctx context.Context, orgID influxdb.ID, pkg *Pkg) (sum S
}
}()
newBuckets, err := s.applyBuckets(ctx, orgID, pkg.buckets())
if err != nil {
return Summary{}, err
runners := []applier{
s.applyLabels(pkg.labels()),
s.applyBuckets(pkg.buckets()),
}
for _, r := range runners {
rollbacks = append(rollbacks, r.rollbacker)
if err := r.creater(ctx, orgID); err != nil {
return Summary{}, err
}
}
return pkg.Summary(), nil
}
func (s *Service) applyBuckets(ctx context.Context, orgID influxdb.ID, buckets []*bucket) ([]influxdb.Bucket, error) {
ctx, cancel := context.WithTimeout(ctx, 1*time.Minute)
defer cancel()
type applier struct {
creater creater
rollbacker rollbacker
}
type rollbacker struct {
resource string
fn func() error
}
type creater func(ctx context.Context, orgID influxdb.ID) error
func (s *Service) applyBuckets(buckets []*bucket) applier {
const resource = "bucket"
influxBuckets := make([]influxdb.Bucket, 0, len(buckets))
for i, b := range buckets {
influxBucket := influxdb.Bucket{
OrgID: orgID,
Description: b.Description,
Name: b.Name,
RetentionPeriod: b.RetentionPeriod,
createFn := func(ctx context.Context, orgID influxdb.ID) error {
ctx, cancel := context.WithTimeout(ctx, 1*time.Minute)
defer cancel()
for i, b := range buckets {
influxBucket := influxdb.Bucket{
OrgID: orgID,
Description: b.Description,
Name: b.Name,
RetentionPeriod: b.RetentionPeriod,
}
err := s.bucketSVC.CreateBucket(ctx, &influxBucket)
if err != nil {
return err
}
buckets[i].ID = influxBucket.ID
buckets[i].OrgID = influxBucket.OrgID
influxBuckets = append(influxBuckets, influxBucket)
}
err := s.bucketSVC.CreateBucket(ctx, &influxBucket)
if err != nil {
return influxBuckets, err
}
buckets[i].ID = influxBucket.ID
buckets[i].OrgID = influxBucket.OrgID
influxBuckets = append(influxBuckets, influxBucket)
return nil
}
return influxBuckets, nil
return applier{
creater: createFn,
rollbacker: rollbacker{
resource: resource,
fn: func() error { return s.deleteBuckets(influxBuckets) },
},
}
}
func (s *Service) deleteBuckets(buckets []influxdb.Bucket) error {
@ -101,3 +123,55 @@ func (s *Service) deleteBuckets(buckets []influxdb.Bucket) error {
return nil
}
func (s *Service) applyLabels(labels []*label) applier {
influxLabels := make([]influxdb.Label, 0, len(labels))
createFn := func(ctx context.Context, orgID influxdb.ID) error {
ctx, cancel := context.WithTimeout(ctx, 1*time.Minute)
defer cancel()
for i, l := range labels {
influxLabel := influxdb.Label{
OrgID: orgID,
Name: l.Name,
Properties: map[string]string{
"color": l.Color,
"description": l.Description,
},
}
err := s.labelSVC.CreateLabel(ctx, &influxLabel)
if err != nil {
return err
}
labels[i].ID = influxLabel.ID
labels[i].OrgID = influxLabel.OrgID
influxLabels = append(influxLabels, influxLabel)
}
return nil
}
return applier{
creater: createFn,
rollbacker: rollbacker{
resource: "label",
fn: func() error { return s.deleteLabels(influxLabels) },
},
}
}
func (s *Service) deleteLabels(labels []influxdb.Label) error {
var errs []string
for _, l := range labels {
err := s.labelSVC.DeleteLabel(context.Background(), l.ID)
if err != nil {
errs = append(errs, l.ID.String())
}
}
if len(errs) > 0 {
return fmt.Errorf(`label_ids=[%s] err="unable to delete label"`, strings.Join(errs, ", "))
}
return nil
}

190
pkger/service_test.go
View File

@ -16,59 +16,129 @@ import (
func TestService(t *testing.T) {
t.Run("Apply", func(t *testing.T) {
t.Run("successfully creates pkg of buckets", func(t *testing.T) {
testfileRunner(t, "testdata/bucket", func(t *testing.T, pkg *Pkg) {
fakeBucketSVC := mock.NewBucketService()
fakeBucketSVC.CreateBucketFn = func(_ context.Context, b *influxdb.Bucket) error {
b.ID = influxdb.ID(b.RetentionPeriod)
return nil
}
t.Run("buckets", func(t *testing.T) {
t.Run("successfully creates pkg of buckets", func(t *testing.T) {
testfileRunner(t, "testdata/bucket", func(t *testing.T, pkg *Pkg) {
fakeBucketSVC := mock.NewBucketService()
fakeBucketSVC.CreateBucketFn = func(_ context.Context, b *influxdb.Bucket) error {
b.ID = influxdb.ID(b.RetentionPeriod)
return nil
}
svc := NewService(zap.NewNop(), fakeBucketSVC)
svc := NewService(zap.NewNop(), fakeBucketSVC, nil)
orgID := influxdb.ID(9000)
orgID := influxdb.ID(9000)
sum, err := svc.Apply(context.TODO(), orgID, pkg)
require.NoError(t, err)
sum, err := svc.Apply(context.TODO(), orgID, pkg)
require.NoError(t, err)
require.Len(t, sum.Buckets, 1)
buck1 := sum.Buckets[0]
assert.Equal(t, influxdb.ID(time.Hour), buck1.ID)
assert.Equal(t, orgID, buck1.OrgID)
assert.Equal(t, "rucket_11", buck1.Name)
assert.Equal(t, time.Hour, buck1.RetentionPeriod)
assert.Equal(t, "bucket 1 description", buck1.Description)
require.Len(t, sum.Buckets, 1)
buck1 := sum.Buckets[0]
assert.Equal(t, influxdb.ID(time.Hour), buck1.ID)
assert.Equal(t, orgID, buck1.OrgID)
assert.Equal(t, "rucket_11", buck1.Name)
assert.Equal(t, time.Hour, buck1.RetentionPeriod)
assert.Equal(t, "bucket 1 description", buck1.Description)
})
})
t.Run("rollsback all created buckets on an error", func(t *testing.T) {
testfileRunner(t, "testdata/bucket", func(t *testing.T, pkg *Pkg) {
fakeBucketSVC := mock.NewBucketService()
var c int
fakeBucketSVC.CreateBucketFn = func(_ context.Context, b *influxdb.Bucket) error {
if c == 2 {
return errors.New("blowed up ")
}
c++
return nil
}
var count int
fakeBucketSVC.DeleteBucketFn = func(_ context.Context, id influxdb.ID) error {
count++
return nil
}
pkg.mBuckets["copybuck1"] = pkg.mBuckets["rucket_11"]
pkg.mBuckets["copybuck2"] = pkg.mBuckets["rucket_11"]
svc := NewService(zap.NewNop(), fakeBucketSVC, nil)
orgID := influxdb.ID(9000)
_, err := svc.Apply(context.TODO(), orgID, pkg)
require.Error(t, err)
assert.Equal(t, 2, count)
})
})
})
t.Run("rollsback all created buckets on an error", func(t *testing.T) {
testfileRunner(t, "testdata/bucket", func(t *testing.T, pkg *Pkg) {
fakeBucketSVC := mock.NewBucketService()
var c int
fakeBucketSVC.CreateBucketFn = func(_ context.Context, b *influxdb.Bucket) error {
if c == 2 {
return errors.New("blowed up ")
t.Run("labels", func(t *testing.T) {
t.Run("successfully creates pkg of labels", func(t *testing.T) {
testfileRunner(t, "testdata/label", func(t *testing.T, pkg *Pkg) {
fakeLabelSVC := mock.NewLabelService()
id := 1
fakeLabelSVC.CreateLabelFn = func(_ context.Context, b *influxdb.Label) error {
b.ID = influxdb.ID(id)
id++
return nil
}
c++
return nil
}
var count int
fakeBucketSVC.DeleteBucketFn = func(_ context.Context, id influxdb.ID) error {
count++
return nil
}
pkg.mBuckets["copybuck1"] = pkg.mBuckets["rucket_11"]
pkg.mBuckets["copybuck2"] = pkg.mBuckets["rucket_11"]
svc := NewService(zap.NewNop(), nil, fakeLabelSVC)
svc := NewService(zap.NewNop(), fakeBucketSVC)
orgID := influxdb.ID(9000)
orgID := influxdb.ID(9000)
sum, err := svc.Apply(context.TODO(), orgID, pkg)
require.NoError(t, err)
_, err := svc.Apply(context.TODO(), orgID, pkg)
require.Error(t, err)
require.Len(t, sum.Labels, 2)
label1 := sum.Labels[0]
assert.Equal(t, influxdb.ID(1), label1.ID)
assert.Equal(t, orgID, label1.OrgID)
assert.Equal(t, "label_1", label1.Name)
assert.Equal(t, "#FFFFFF", label1.Properties["color"])
assert.Equal(t, "label 1 description", label1.Properties["description"])
assert.Equal(t, 2, count)
label2 := sum.Labels[1]
assert.Equal(t, influxdb.ID(2), label2.ID)
assert.Equal(t, orgID, label2.OrgID)
assert.Equal(t, "label_2", label2.Name)
assert.Equal(t, "#000000", label2.Properties["color"])
assert.Equal(t, "label 2 description", label2.Properties["description"])
})
t.Run("rollsback all created labels on an error", func(t *testing.T) {
testfileRunner(t, "testdata/label", func(t *testing.T, pkg *Pkg) {
fakeLabelSVC := mock.NewLabelService()
var c int
fakeLabelSVC.CreateLabelFn = func(_ context.Context, b *influxdb.Label) error {
// 4th label will return the error here, and 3 before should be rolled back
if c == 3 {
return errors.New("blowed up ")
}
c++
return nil
}
var count int
fakeLabelSVC.DeleteLabelFn = func(_ context.Context, id influxdb.ID) error {
count++
return nil
}
pkg.mLabels["copy1"] = pkg.mLabels["label_1"]
pkg.mLabels["copy2"] = pkg.mLabels["label_2"]
svc := NewService(zap.NewNop(), nil, fakeLabelSVC)
orgID := influxdb.ID(9000)
_, err := svc.Apply(context.TODO(), orgID, pkg)
require.Error(t, err)
assert.Equal(t, 3, count)
})
})
})
})
})
@ -89,8 +159,8 @@ func TestPkg(t *testing.T) {
"buck_1": {
ID: influxdb.ID(1),
OrgID: influxdb.ID(100),
Description: "desc1",
Name: "name1",
Description: "desc1",
RetentionPeriod: time.Hour,
},
},
@ -108,5 +178,43 @@ func TestPkg(t *testing.T) {
assert.Equal(t, time.Duration(i)*time.Hour, buck.RetentionPeriod)
}
})
t.Run("labels returned in asc order by name", func(t *testing.T) {
pkg := Pkg{
mLabels: map[string]*label{
"2": {
ID: influxdb.ID(2),
OrgID: influxdb.ID(100),
Name: "name2",
Description: "desc2",
Color: "blurple",
},
"1": {
ID: influxdb.ID(1),
OrgID: influxdb.ID(100),
Name: "name1",
Description: "desc1",
Color: "peru",
},
},
}
summary := pkg.Summary()
require.Len(t, summary.Labels, len(pkg.mLabels))
label1 := summary.Labels[0]
assert.Equal(t, influxdb.ID(1), label1.ID)
assert.Equal(t, influxdb.ID(100), label1.OrgID)
assert.Equal(t, "desc1", label1.Properties["description"])
assert.Equal(t, "name1", label1.Name)
assert.Equal(t, "peru", label1.Properties["color"])
label2 := summary.Labels[1]
assert.Equal(t, influxdb.ID(2), label2.ID)
assert.Equal(t, influxdb.ID(100), label2.OrgID)
assert.Equal(t, "desc2", label2.Properties["description"])
assert.Equal(t, "name2", label2.Name)
assert.Equal(t, "blurple", label2.Properties["color"])
})
})
}

26
pkger/testdata/label.json vendored Normal file
View File

@ -0,0 +1,26 @@
{
"apiVersion": "0.1.0",
"kind": "Package",
"meta": {
"pkgName": "pkg_name",
"pkgVersion": "1",
"description": "pack description"
},
"spec": {
"resources": [
{
"kind": "Label",
"name": "label_1",
"color": "#FFFFFF",
"description": "label 1 description"
},
{
"kind": "Label",
"name": "label_2",
"color": "#000000",
"description": "label 2 description"
}
]
}
}

16
pkger/testdata/label.yml vendored Normal file
View File

@ -0,0 +1,16 @@
apiVersion: 0.1.0
kind: Package
meta:
pkgName: pkg_name
pkgVersion: 1
description: pack description
spec:
resources:
- kind: Label
name: label_2
color: "#000000"
description: label 2 description
- kind: Label
name: label_1
color: "#FFFFFF"
description: label 1 description