feat(pkger): add label associations to variables

pull/15797/head
Johnny Steenbergen 2019-11-06 16:45:00 -08:00 committed by Johnny Steenbergen
parent d252b20ecc
commit 5eb29e9ed9
10 changed files with 809 additions and 128 deletions

View File

@ -2,6 +2,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
@ -17,7 +18,6 @@ import (
"github.com/olekukonko/tablewriter"
"github.com/spf13/cobra"
input "github.com/tcnksm/go-input"
"go.uber.org/zap"
)
func pkgCmd() *cobra.Command {
@ -107,7 +107,17 @@ func newPkgerSVC(f Flags) (*pkger.Service, error) {
return nil, err
}
return pkger.NewService(zap.NewNop(), bucketSVC, labelSVC, dashSVC), nil
varSVC, err := newVariableService(f)
if err != nil {
return nil, err
}
return pkger.NewService(
pkger.WithBucketSVC(bucketSVC),
pkger.WithDashboardSVC(dashSVC),
pkger.WithLabelSVC(labelSVC),
pkger.WithVariableSVC(varSVC),
), nil
}
func newDashboardService(f Flags) (influxdb.DashboardService, error) {
@ -130,6 +140,16 @@ func newLabelService(f Flags) (influxdb.LabelService, error) {
}, nil
}
func newVariableService(f Flags) (influxdb.VariableService, error) {
if f.local {
return newLocalKVService()
}
return &http.VariableService{
Addr: f.host,
Token: f.token,
}, nil
}
func pkgFromFile(path string) (*pkger.Pkg, error) {
var enc pkger.Encoding
switch ext := filepath.Ext(path); ext {
@ -184,9 +204,10 @@ func printPkgDiff(hasColor, hasTableBorders bool, diff pkger.Diff) {
return fmt.Sprintf("%s\n%s", red(o), green(n))
}
tablePrintFn := tablePrinterGen(hasColor, hasTableBorders)
if labels := diff.Labels; len(labels) > 0 {
headers := []string{"New", "ID", "Name", "Color", "Description"}
tablePrinter("LABELS", headers, len(labels), hasColor, hasTableBorders, func(w *tablewriter.Table) {
tablePrintFn("LABELS", headers, len(labels), func(w *tablewriter.Table) {
for _, l := range labels {
w.Append([]string{
boolDiff(l.IsNew()),
@ -201,7 +222,7 @@ func printPkgDiff(hasColor, hasTableBorders bool, diff pkger.Diff) {
if bkts := diff.Buckets; len(bkts) > 0 {
headers := []string{"New", "ID", "Name", "Retention Period", "Description"}
tablePrinter("BUCKETS", headers, len(bkts), hasColor, hasTableBorders, func(w *tablewriter.Table) {
tablePrintFn("BUCKETS", headers, len(bkts), func(w *tablewriter.Table) {
for _, b := range bkts {
w.Append([]string{
boolDiff(b.IsNew()),
@ -216,7 +237,7 @@ func printPkgDiff(hasColor, hasTableBorders bool, diff pkger.Diff) {
if dashes := diff.Dashboards; len(dashes) > 0 {
headers := []string{"New", "Name", "Description", "Num Charts"}
tablePrinter("DASHBOARDS", headers, len(dashes), hasColor, hasTableBorders, func(w *tablewriter.Table) {
tablePrintFn("DASHBOARDS", headers, len(dashes), func(w *tablewriter.Table) {
for _, d := range dashes {
w.Append([]string{
boolDiff(true),
@ -228,9 +249,33 @@ func printPkgDiff(hasColor, hasTableBorders bool, diff pkger.Diff) {
})
}
if vars := diff.Variables; len(vars) > 0 {
headers := []string{"New", "ID", "Name", "Description", "Arg Type", "Arg Values"}
tablePrintFn("VARIABLES", headers, len(vars), func(w *tablewriter.Table) {
for _, v := range vars {
var oldArgType string
if v.OldArgs != nil {
oldArgType = v.OldArgs.Type
}
var newArgType string
if v.NewArgs != nil {
newArgType = v.NewArgs.Type
}
w.Append([]string{
boolDiff(v.IsNew()),
v.ID.String(),
v.Name,
strDiff(v.IsNew(), v.OldDesc, v.NewDesc),
strDiff(v.IsNew(), oldArgType, newArgType),
strDiff(v.IsNew(), printVarArgs(v.OldArgs), printVarArgs(v.NewArgs)),
})
}
})
}
if len(diff.LabelMappings) > 0 {
headers := []string{"New", "Resource Type", "Resource Name", "Resource ID", "Label Name", "Label ID"}
tablePrinter("LABEL MAPPINGS", headers, len(diff.LabelMappings), hasColor, hasTableBorders, func(w *tablewriter.Table) {
tablePrintFn("LABEL MAPPINGS", headers, len(diff.LabelMappings), func(w *tablewriter.Table) {
for _, m := range diff.LabelMappings {
w.Append([]string{
boolDiff(m.IsNew),
@ -245,10 +290,43 @@ func printPkgDiff(hasColor, hasTableBorders bool, diff pkger.Diff) {
}
}
func printVarArgs(a *influxdb.VariableArguments) string {
if a == nil {
return "<nil>"
}
if a.Type == "map" {
b, err := json.Marshal(a.Values)
if err != nil {
return "{}"
}
return string(b)
}
if a.Type == "constant" {
vals, ok := a.Values.(influxdb.VariableConstantValues)
if !ok {
return "[]"
}
var out []string
for _, s := range vals {
out = append(out, fmt.Sprintf("%q", s))
}
return fmt.Sprintf("[%s]", strings.Join(out, " "))
}
if a.Type == "query" {
qVal, ok := a.Values.(influxdb.VariableQueryValues)
if !ok {
return ""
}
return fmt.Sprintf("language=%q query=%q", qVal.Language, qVal.Query)
}
return "unknown variable argument"
}
func printPkgSummary(hasColor, hasTableBorders bool, sum pkger.Summary) {
tablePrintFn := tablePrinterGen(hasColor, hasTableBorders)
if labels := sum.Labels; len(labels) > 0 {
headers := []string{"ID", "Name", "Description", "Color"}
tablePrinter("LABELS", headers, len(labels), hasColor, hasTableBorders, func(w *tablewriter.Table) {
tablePrintFn("LABELS", headers, len(labels), func(w *tablewriter.Table) {
for _, l := range labels {
w.Append([]string{
l.ID.String(),
@ -262,7 +340,7 @@ func printPkgSummary(hasColor, hasTableBorders bool, sum pkger.Summary) {
if buckets := sum.Buckets; len(buckets) > 0 {
headers := []string{"ID", "Name", "Retention", "Description"}
tablePrinter("BUCKETS", headers, len(buckets), hasColor, hasTableBorders, func(w *tablewriter.Table) {
tablePrintFn("BUCKETS", headers, len(buckets), func(w *tablewriter.Table) {
for _, bucket := range buckets {
w.Append([]string{
bucket.ID.String(),
@ -276,7 +354,7 @@ func printPkgSummary(hasColor, hasTableBorders bool, sum pkger.Summary) {
if dashes := sum.Dashboards; len(dashes) > 0 {
headers := []string{"ID", "Name", "Description"}
tablePrinter("DASHBOARDS", headers, len(dashes), hasColor, hasTableBorders, func(w *tablewriter.Table) {
tablePrintFn("DASHBOARDS", headers, len(dashes), func(w *tablewriter.Table) {
for _, d := range dashes {
w.Append([]string{
d.ID.String(),
@ -287,9 +365,25 @@ func printPkgSummary(hasColor, hasTableBorders bool, sum pkger.Summary) {
})
}
if vars := sum.Variables; len(vars) > 0 {
headers := []string{"ID", "Name", "Description", "Arg Type", "Arg Values"}
tablePrintFn("VARIABLES", headers, len(vars), func(w *tablewriter.Table) {
for _, v := range vars {
args := v.Arguments
w.Append([]string{
v.ID.String(),
v.Name,
v.Description,
args.Type,
printVarArgs(args),
})
}
})
}
if mappings := sum.LabelMappings; len(mappings) > 0 {
headers := []string{"Resource Type", "Resource Name", "Resource ID", "Label Name", "Label ID"}
tablePrinter("LABEL MAPPINGS", headers, len(mappings), hasColor, hasTableBorders, func(w *tablewriter.Table) {
tablePrintFn("LABEL MAPPINGS", headers, len(mappings), func(w *tablewriter.Table) {
for _, m := range mappings {
w.Append([]string{
string(m.ResourceType),
@ -303,6 +397,12 @@ func printPkgSummary(hasColor, hasTableBorders bool, sum pkger.Summary) {
}
}
func tablePrinterGen(hasColor, hasTableBorder bool) func(table string, headers []string, count int, appendFn func(w *tablewriter.Table)) {
return func(table string, headers []string, count int, appendFn func(w *tablewriter.Table)) {
tablePrinter(table, headers, count, hasColor, hasTableBorder, appendFn)
}
}
func tablePrinter(table string, headers []string, count int, hasColor, hasTableBorders bool, appendFn func(w *tablewriter.Table)) {
descrCol := -1
for i, h := range headers {
@ -321,7 +421,6 @@ func tablePrinter(table string, headers []string, count int, hasColor, hasTableB
alignments = append(alignments, tablewriter.ALIGN_CENTER)
}
if descrCol != -1 {
w.SetAutoWrapText(false)
w.SetColMinWidth(descrCol, 30)
alignments[descrCol] = tablewriter.ALIGN_LEFT
}

View File

@ -839,9 +839,14 @@ func (m *Launcher) run(ctx context.Context) (err error) {
var pkgSVC pkger.SVC
{
pkgLogger := m.logger.With(zap.String("service", "pkger"))
b := m.apibackend
pkgSVC = pkger.NewService(pkgLogger, b.BucketService, b.LabelService, b.DashboardService)
pkgSVC = pkger.NewService(
pkger.WithLogger(m.logger.With(zap.String("service", "pkger"))),
pkger.WithBucketSVC(b.BucketService),
pkger.WithDashboardSVC(b.DashboardService),
pkger.WithLabelSVC(b.LabelService),
pkger.WithVariableSVC(b.VariableService),
)
}
var pkgHTTPServer *http.HandlerPkg

View File

@ -7186,6 +7186,17 @@ components:
type: string
labelID:
type: string
variables:
type: array
items:
allOf:
- $ref: "#/components/schemas/Variable"
- type: object
properties:
labelAssociations:
type: array
items:
$ref: "#/components/schemas/Label"
diff:
type: object
properties:
@ -7253,6 +7264,23 @@ components:
type: string
labelName:
type: string
variables:
type: array
items:
type: object
properties:
id:
type: string
name:
type: string
oldDescription:
type: string
newDescription:
type: string
oldArgs:
$ref: "#/components/schemas/VariableProperties"
newArgs:
$ref: "#/components/schemas/VariableProperties"
PkgChart:
type: object
properties:
@ -8461,11 +8489,7 @@ components:
labels:
$ref: "#/components/schemas/Labels"
arguments:
type: object
oneOf:
- $ref: "#/components/schemas/QueryVariableProperties"
- $ref: "#/components/schemas/ConstantVariableProperties"
- $ref: "#/components/schemas/MapVariableProperties"
$ref: "#/components/schemas/VariableProperties"
createdAt:
type: string
format: date-time
@ -8511,6 +8535,12 @@ components:
type: array
items:
$ref: "#/components/schemas/Variable"
VariableProperties:
type: object
oneOf:
- $ref: "#/components/schemas/QueryVariableProperties"
- $ref: "#/components/schemas/ConstantVariableProperties"
- $ref: "#/components/schemas/MapVariableProperties"
ViewProperties:
oneOf:
- $ref: "#/components/schemas/LinePlusSingleStatProperties"

View File

@ -67,6 +67,7 @@ type Diff struct {
Dashboards []DiffDashboard `json:"dashboards"`
Labels []DiffLabel `json:"labels"`
LabelMappings []DiffLabelMapping `json:"labelMappings"`
Variables []DiffVariable `json:"variables"`
}
// DiffBucket is a diff of an individual bucket.
@ -163,6 +164,33 @@ type DiffLabelMapping struct {
LabelName string `json:"labelName"`
}
// DiffVariable is a diff of an individual variable.
type DiffVariable struct {
ID SafeID `json:"id"`
Name string `json:"name"`
OldDesc string `json:"oldDescription"`
NewDesc string `json:"newDescription"`
OldArgs *influxdb.VariableArguments `json:"oldArgs"`
NewArgs *influxdb.VariableArguments `json:"newArgs"`
}
func newDiffVariable(v *variable, iv influxdb.Variable) DiffVariable {
return DiffVariable{
ID: SafeID(iv.ID),
Name: v.Name,
OldDesc: iv.Description,
NewDesc: v.Description,
OldArgs: iv.Arguments,
NewArgs: v.influxVarArgs(),
}
}
// IsNew indicates whether a pkg variable is going to be new to the platform.
func (d DiffVariable) IsNew() bool {
return d.ID == SafeID(0)
}
// Summary is a definition of all the resources that have or
// will be created from a pkg.
type Summary struct {
@ -240,6 +268,7 @@ type SummaryLabelMapping struct {
// SummaryVariable provides a summary of a pkg variable.
type SummaryVariable struct {
influxdb.Variable
LabelAssociations []influxdb.Label `json:"labelAssociations"`
}
type bucket struct {
@ -317,14 +346,68 @@ func (l assocMapVal) dashboard() (*dashboard, bool) {
return d, ok
}
func (l assocMapVal) variable() (*variable, bool) {
if l.v == nil {
return nil, false
}
v, ok := l.v.(*variable)
return v, ok
}
type associationMapping struct {
mappings map[assocMapKey]assocMapVal
}
func (l *associationMapping) setMapping(k assocMapKey, v assocMapVal) {
if l == nil {
return
}
if l.mappings == nil {
l.mappings = make(map[assocMapKey]assocMapVal)
}
l.mappings[k] = v
}
func (l *associationMapping) setBucketMapping(b *bucket, exists bool) {
key := assocMapKey{
resType: b.ResourceType(),
name: b.Name,
}
val := assocMapVal{
exists: exists,
v: b,
}
l.setMapping(key, val)
}
func (l *associationMapping) setDashboardMapping(d *dashboard) {
key := assocMapKey{
resType: d.ResourceType(),
name: d.Name,
}
val := assocMapVal{v: d}
l.setMapping(key, val)
}
func (l *associationMapping) setVariableMapping(v *variable, exists bool) {
key := assocMapKey{
resType: v.ResourceType(),
name: v.Name,
}
val := assocMapVal{
exists: exists,
v: v,
}
l.setMapping(key, val)
}
type label struct {
id influxdb.ID
OrgID influxdb.ID
Name string
Color string
Description string
mappings map[assocMapKey]assocMapVal
associationMapping
// exists provides context for a resource that already
// exists in the platform. If a resource already exists(exists=true)
@ -387,43 +470,15 @@ func (l *label) getMappedResourceID(k assocMapKey) influxdb.ID {
if ok {
return d.ID()
}
case influxdb.VariablesResourceType:
v, ok := l.mappings[k].variable()
if ok {
return v.ID()
}
}
return 0
}
func (l *label) setBucketMapping(b *bucket, exists bool) {
if l == nil {
return
}
if l.mappings == nil {
l.mappings = make(map[assocMapKey]assocMapVal)
}
key := assocMapKey{
resType: influxdb.BucketsResourceType,
name: b.Name,
}
l.mappings[key] = assocMapVal{
exists: exists,
v: b,
}
}
func (l *label) setDashboardMapping(d *dashboard) {
if l == nil {
return
}
if l.mappings == nil {
l.mappings = make(map[assocMapKey]assocMapVal)
}
key := assocMapKey{
resType: d.ResourceType(),
name: d.Name,
}
l.mappings[key] = assocMapVal{v: d}
}
func (l *label) properties() map[string]string {
return map[string]string{
"color": l.Color,
@ -455,10 +510,47 @@ type variable struct {
ConstValues []string
MapValues map[string]string
mappings map[assocMapKey]assocMapVal
labels []*label
existing *influxdb.Variable
}
func (v *variable) ID() influxdb.ID {
if v.existing != nil {
return v.existing.ID
}
return v.id
}
func (v *variable) Exists() bool {
return v.existing != nil
}
func (v *variable) ResourceType() influxdb.ResourceType {
return influxdb.VariablesResourceType
}
func (v *variable) shouldApply() bool {
return v.existing == nil ||
v.existing.Description != v.Description ||
v.existing.Arguments == nil ||
v.existing.Arguments.Type != v.Type
}
func (v *variable) summarize() SummaryVariable {
return SummaryVariable{
Variable: influxdb.Variable{
ID: v.ID(),
OrganizationID: v.OrgID,
Name: v.Name,
Description: v.Description,
Arguments: v.influxVarArgs(),
},
LabelAssociations: toInfluxLabels(v.labels...),
}
}
func (v *variable) influxVarArgs() *influxdb.VariableArguments {
args := &influxdb.VariableArguments{
Type: v.Type,
}
@ -473,16 +565,7 @@ func (v *variable) summarize() SummaryVariable {
case "map":
args.Values = influxdb.VariableMapValues(v.MapValues)
}
return SummaryVariable{
Variable: influxdb.Variable{
ID: v.id,
OrganizationID: v.OrgID,
Name: v.Name,
Description: v.Description,
Arguments: args,
},
}
return args
}
func (v *variable) valid() []failure {

View File

@ -94,12 +94,14 @@ func TestPkg(t *testing.T) {
Name: "name2",
Description: "desc2",
Color: "blurple",
mappings: map[assocMapKey]assocMapVal{
assocMapKey{
resType: influxdb.BucketsResourceType,
name: bucket1.Name,
}: {
v: bucket1,
associationMapping: associationMapping{
mappings: map[assocMapKey]assocMapVal{
assocMapKey{
resType: influxdb.BucketsResourceType,
name: bucket1.Name,
}: {
v: bucket1,
},
},
},
}

View File

@ -513,6 +513,15 @@ func (p *Pkg) graphVariables() error {
MapValues: r.mapStrStr("values"),
}
failures := p.parseNestedLabels(r, func(l *label) error {
newVar.labels = append(newVar.labels, l)
p.mLabels[l.Name].setVariableMapping(newVar, false)
return nil
})
sort.Slice(newVar.labels, func(i, j int) bool {
return newVar.labels[i].Name < newVar.labels[j].Name
})
p.mVariables[r.Name()] = newVar
// here we set the var on the var map and return fails
@ -523,7 +532,7 @@ func (p *Pkg) graphVariables() error {
// invalid. So the mapping is correct. So we keep this
// to validate that mapping is correct, and return fails
// to indicate fails from the var.
return newVar.valid()
return append(failures, newVar.valid()...)
})
}

View File

@ -1,6 +1,8 @@
package pkger
import (
"path/filepath"
"strings"
"testing"
"time"
@ -1831,6 +1833,47 @@ spec:
}
})
})
t.Run("pkg with variable and labels associated", func(t *testing.T) {
testfileRunner(t, "testdata/variables_associates_label.yml", func(t *testing.T, pkg *Pkg) {
sum := pkg.Summary()
require.Len(t, sum.Labels, 1)
vars := sum.Variables
require.Len(t, vars, 1)
expectedLabelMappings := []struct {
varName string
labels []string
}{
{
varName: "var_1",
labels: []string{"label_1"},
},
}
for i, expected := range expectedLabelMappings {
v := vars[i]
require.Len(t, v.LabelAssociations, len(expected.labels))
for j, label := range expected.labels {
assert.Equal(t, label, v.LabelAssociations[j].Name)
}
}
expectedMappings := []SummaryLabelMapping{
{
ResourceName: "var_1",
LabelName: "label_1",
},
}
require.Len(t, sum.LabelMappings, len(expectedMappings))
for i, expected := range expectedMappings {
expected.LabelMapping.ResourceType = influxdb.VariablesResourceType
assert.Equal(t, expected, sum.LabelMappings[i])
}
})
})
}
type testPkgResourceError struct {
@ -1925,21 +1968,31 @@ func testfileRunner(t *testing.T, path string, testFn func(t *testing.T, pkg *Pk
}{
{
name: "yaml",
extension: "yml",
extension: ".yml",
encoding: EncodingYAML,
},
{
name: "json",
extension: "json",
extension: ".json",
encoding: EncodingJSON,
},
}
ext := filepath.Ext(path)
switch ext {
case ".yml":
tests = tests[:1]
case ".json":
tests = tests[1:]
}
path = strings.TrimSuffix(path, ext)
for _, tt := range tests {
fn := func(t *testing.T) {
t.Helper()
pkg := validParsedPkg(t, path+"."+tt.extension, tt.encoding, baseAsserts{
pkg := validParsedPkg(t, path+tt.extension, tt.encoding, baseAsserts{
version: "0.1.0",
kind: "Package",
description: "pack description",

View File

@ -22,6 +22,53 @@ type SVC interface {
Apply(ctx context.Context, orgID influxdb.ID, pkg *Pkg) (Summary, error)
}
type serviceOpt struct {
logger *zap.Logger
labelSVC influxdb.LabelService
bucketSVC influxdb.BucketService
dashSVC influxdb.DashboardService
varSVC influxdb.VariableService
}
// ServiceSetterFn is a means of setting dependencies on the Service type.
type ServiceSetterFn func(opt *serviceOpt)
// WithLogger sets the service logger.
func WithLogger(logger *zap.Logger) ServiceSetterFn {
return func(opt *serviceOpt) {
opt.logger = logger
}
}
// WithBucketSVC sets the bucket service.
func WithBucketSVC(bktSVC influxdb.BucketService) ServiceSetterFn {
return func(opt *serviceOpt) {
opt.bucketSVC = bktSVC
}
}
// WithDashboardSVC sets the dashboard service.
func WithDashboardSVC(dashSVC influxdb.DashboardService) ServiceSetterFn {
return func(opt *serviceOpt) {
opt.dashSVC = dashSVC
}
}
// WithLabelSVC sets the label service.
func WithLabelSVC(labelSVC influxdb.LabelService) ServiceSetterFn {
return func(opt *serviceOpt) {
opt.labelSVC = labelSVC
}
}
// WithVariableSVC sets the variable service.
func WithVariableSVC(varSVC influxdb.VariableService) ServiceSetterFn {
return func(opt *serviceOpt) {
opt.varSVC = varSVC
}
}
// Service provides the pkger business logic including all the dependencies to make
// this resource sausage.
type Service struct {
@ -30,21 +77,25 @@ type Service struct {
labelSVC influxdb.LabelService
bucketSVC influxdb.BucketService
dashSVC influxdb.DashboardService
varSVC influxdb.VariableService
}
// NewService is a constructor for a pkger Service.
func NewService(l *zap.Logger, bucketSVC influxdb.BucketService, labelSVC influxdb.LabelService, dashSVC influxdb.DashboardService) *Service {
svc := Service{
logger: zap.NewNop(),
bucketSVC: bucketSVC,
labelSVC: labelSVC,
dashSVC: dashSVC,
func NewService(opts ...ServiceSetterFn) *Service {
opt := &serviceOpt{
logger: zap.NewNop(),
}
for _, o := range opts {
o(opt)
}
if l != nil {
svc.logger = l
return &Service{
logger: opt.logger,
bucketSVC: opt.bucketSVC,
labelSVC: opt.labelSVC,
dashSVC: opt.dashSVC,
varSVC: opt.varSVC,
}
return &svc
}
// CreatePkgSetFn is a functional input for setting the pkg fields.
@ -105,6 +156,11 @@ func (s *Service) DryRun(ctx context.Context, orgID influxdb.ID, pkg *Pkg) (Summ
return Summary{}, Diff{}, err
}
diffVars, err := s.dryRunVariables(ctx, orgID, pkg)
if err != nil {
return Summary{}, Diff{}, err
}
diffLabelMappings, err := s.dryRunLabelMappings(ctx, pkg)
if err != nil {
return Summary{}, Diff{}, err
@ -120,6 +176,7 @@ func (s *Service) DryRun(ctx context.Context, orgID influxdb.ID, pkg *Pkg) (Summ
Dashboards: diffDashes,
Labels: diffLabels,
LabelMappings: diffLabelMappings,
Variables: diffVars,
}
return pkg.Summary(), diff, nil
}
@ -169,9 +226,9 @@ func (s *Service) dryRunLabels(ctx context.Context, orgID influxdb.ID, pkg *Pkg)
mExistingLabels := make(map[string]DiffLabel)
labels := pkg.labels()
for i := range labels {
l := labels[i]
pkgLabel := labels[i]
existingLabels, err := s.labelSVC.FindLabels(ctx, influxdb.LabelFilter{
Name: l.Name,
Name: pkgLabel.Name,
OrgID: &orgID,
}, influxdb.FindOptions{Limit: 1})
switch {
@ -179,10 +236,10 @@ func (s *Service) dryRunLabels(ctx context.Context, orgID influxdb.ID, pkg *Pkg)
// err isn't a not found (some other error)
case err == nil && len(existingLabels) > 0:
existingLabel := existingLabels[0]
l.existing = existingLabel
mExistingLabels[l.Name] = newDiffLabel(l, *existingLabel)
pkgLabel.existing = existingLabel
mExistingLabels[pkgLabel.Name] = newDiffLabel(pkgLabel, *existingLabel)
default:
mExistingLabels[l.Name] = newDiffLabel(l, influxdb.Label{})
mExistingLabels[pkgLabel.Name] = newDiffLabel(pkgLabel, influxdb.Label{})
}
}
@ -197,6 +254,49 @@ func (s *Service) dryRunLabels(ctx context.Context, orgID influxdb.ID, pkg *Pkg)
return diffs, nil
}
func (s *Service) dryRunVariables(ctx context.Context, orgID influxdb.ID, pkg *Pkg) ([]DiffVariable, error) {
mExistingLabels := make(map[string]DiffVariable)
variables := pkg.variables()
VarLoop:
for i := range variables {
pkgVar := variables[i]
existingLabels, err := s.varSVC.FindVariables(ctx, influxdb.VariableFilter{
OrganizationID: &orgID,
// TODO: would be ideal to extend find variables to allow for a name matcher
// since names are unique for vars within an org, meanwhile, make large limit
// returned vars, should be more than enough for the time being.
}, influxdb.FindOptions{Limit: 10000})
switch {
case err == nil && len(existingLabels) > 0:
for i := range existingLabels {
existingVar := existingLabels[i]
if existingVar.Name != pkgVar.Name {
continue
}
pkgVar.existing = existingVar
mExistingLabels[pkgVar.Name] = newDiffVariable(pkgVar, *existingVar)
continue VarLoop
}
// fallthrough here for when the variable is not found, it'll fall to the
// default case and add it as new.
fallthrough
default:
mExistingLabels[pkgVar.Name] = newDiffVariable(pkgVar, influxdb.Variable{})
}
}
diffs := make([]DiffVariable, 0, len(mExistingLabels))
for _, diff := range mExistingLabels {
diffs = append(diffs, diff)
}
sort.Slice(diffs, func(i, j int) bool {
return diffs[i].Name < diffs[j].Name
})
return diffs, nil
}
type (
labelMappingDiffFn func(labelID influxdb.ID, labelName string, isNew bool)
@ -243,6 +343,23 @@ func (s *Service) dryRunLabelMappings(ctx context.Context, pkg *Pkg) ([]DiffLabe
}
}
for _, v := range pkg.variables() {
err := s.dryRunResourceLabelMapping(ctx, v, v.labels, func(labelID influxdb.ID, labelName string, isNew bool) {
pkg.mLabels[labelName].setVariableMapping(v, !isNew)
diffs = append(diffs, DiffLabelMapping{
IsNew: isNew,
ResType: v.ResourceType(),
ResID: SafeID(v.ID()),
ResName: v.Name,
LabelID: SafeID(labelID),
LabelName: labelName,
})
})
if err != nil {
return nil, err
}
}
// sort by res type ASC, then res name ASC, then label name ASC
sort.Slice(diffs, func(i, j int) bool {
n, m := diffs[i], diffs[j]
@ -271,6 +388,7 @@ func (s *Service) dryRunResourceLabelMapping(ctx context.Context, la labelAssoci
}
return nil
}
// loop through and hit api for all labels associated with a bkt
// lookup labels in pkg, add it to the label mapping, if exists in
// the results from API, mark it exists
@ -338,6 +456,7 @@ func (s *Service) Apply(ctx context.Context, orgID influxdb.ID, pkg *Pkg) (sum S
{
// primary resources
s.applyLabels(pkg.labels()),
s.applyVariables(pkg.variables()),
s.applyBuckets(pkg.buckets()),
s.applyDashboards(pkg.dashboards()),
},
@ -589,7 +708,17 @@ func (s *Service) applyLabels(labels []*label) applier {
func (s *Service) rollbackLabels(labels []*label) error {
var errs []string
for _, l := range labels {
err := s.labelSVC.DeleteLabel(context.Background(), l.ID())
if l.existing == nil {
err := s.labelSVC.DeleteLabel(context.Background(), l.ID())
if err != nil {
errs = append(errs, l.ID().String())
}
continue
}
_, err := s.labelSVC.UpdateLabel(context.Background(), l.ID(), influxdb.LabelUpdate{
Properties: l.existing.Properties,
})
if err != nil {
errs = append(errs, l.ID().String())
}
@ -626,6 +755,97 @@ func (s *Service) applyLabel(ctx context.Context, l *label) (influxdb.Label, err
return influxLabel, nil
}
func (s *Service) applyVariables(vars []*variable) applier {
const resource = "variable"
rollBackVars := make([]*variable, 0, len(vars))
createFn := func(ctx context.Context, orgID influxdb.ID) error {
ctx, cancel := context.WithTimeout(ctx, 1*time.Minute)
defer cancel()
var errs applyErrs
for i, v := range vars {
vars[i].OrgID = orgID
if !v.shouldApply() {
continue
}
influxVar, err := s.applyVariable(ctx, v)
if err != nil {
errs = append(errs, applyErrBody{
name: v.Name,
msg: err.Error(),
})
continue
}
vars[i].id = influxVar.ID
rollBackVars = append(rollBackVars, vars[i])
}
return errs.toError(resource, "failed to create variable")
}
return applier{
creater: createFn,
rollbacker: rollbacker{
resource: resource,
fn: func() error { return s.rollbackVariables(rollBackVars) },
},
}
}
func (s *Service) rollbackVariables(variables []*variable) error {
var errs []string
for _, v := range variables {
if v.existing == nil {
err := s.varSVC.DeleteVariable(context.Background(), v.ID())
if err != nil {
errs = append(errs, v.ID().String())
}
continue
}
_, err := s.varSVC.UpdateVariable(context.Background(), v.ID(), &influxdb.VariableUpdate{
Description: v.existing.Description,
Arguments: v.existing.Arguments,
})
if err != nil {
errs = append(errs, v.ID().String())
}
}
if len(errs) > 0 {
return fmt.Errorf(`variable_ids=[%s] err="unable to delete variable"`, strings.Join(errs, ", "))
}
return nil
}
func (s *Service) applyVariable(ctx context.Context, v *variable) (influxdb.Variable, error) {
if v.existing != nil {
updatedVar, err := s.varSVC.UpdateVariable(ctx, v.ID(), &influxdb.VariableUpdate{
Description: v.Description,
Arguments: v.influxVarArgs(),
})
if err != nil {
return influxdb.Variable{}, err
}
return *updatedVar, nil
}
influxVar := influxdb.Variable{
OrganizationID: v.OrgID,
Name: v.Name,
Description: v.Description,
Arguments: v.influxVarArgs(),
}
err := s.varSVC.CreateVariable(ctx, &influxVar)
if err != nil {
return influxdb.Variable{}, err
}
return influxVar, nil
}
func (s *Service) applyLabelMappings(pkg *Pkg) applier {
var mappings []influxdb.LabelMapping
createFn := func(ctx context.Context, orgID influxdb.ID) error {

View File

@ -10,7 +10,6 @@ import (
"github.com/influxdata/influxdb/mock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
)
func TestService(t *testing.T) {
@ -28,9 +27,7 @@ func TestService(t *testing.T) {
RetentionPeriod: 30 * time.Hour,
}, nil
}
fakeLabelSVC := mock.NewLabelService()
fakeDashSVC := mock.NewDashboardService()
svc := NewService(zap.NewNop(), fakeBktSVC, fakeLabelSVC, fakeDashSVC)
svc := NewService(WithBucketSVC(fakeBktSVC), WithLabelSVC(mock.NewLabelService()))
_, diff, err := svc.DryRun(context.TODO(), influxdb.ID(100), pkg)
require.NoError(t, err)
@ -55,9 +52,7 @@ func TestService(t *testing.T) {
fakeBktSVC.FindBucketByNameFn = func(_ context.Context, orgID influxdb.ID, name string) (*influxdb.Bucket, error) {
return nil, errors.New("not found")
}
fakeLabelSVC := mock.NewLabelService()
fakeDashSVC := mock.NewDashboardService()
svc := NewService(zap.NewNop(), fakeBktSVC, fakeLabelSVC, fakeDashSVC)
svc := NewService(WithBucketSVC(fakeBktSVC), WithLabelSVC(mock.NewLabelService()))
_, diff, err := svc.DryRun(context.TODO(), influxdb.ID(100), pkg)
require.NoError(t, err)
@ -77,7 +72,6 @@ func TestService(t *testing.T) {
t.Run("labels", func(t *testing.T) {
t.Run("two labels updated", func(t *testing.T) {
testfileRunner(t, "testdata/label", func(t *testing.T, pkg *Pkg) {
fakeBktSVC := mock.NewBucketService()
fakeLabelSVC := mock.NewLabelService()
fakeLabelSVC.FindLabelsFn = func(_ context.Context, filter influxdb.LabelFilter) ([]*influxdb.Label, error) {
return []*influxdb.Label{
@ -91,8 +85,7 @@ func TestService(t *testing.T) {
},
}, nil
}
fakeDashSVC := mock.NewDashboardService()
svc := NewService(zap.NewNop(), fakeBktSVC, fakeLabelSVC, fakeDashSVC)
svc := NewService(WithLabelSVC(fakeLabelSVC))
_, diff, err := svc.DryRun(context.TODO(), influxdb.ID(100), pkg)
require.NoError(t, err)
@ -118,13 +111,11 @@ func TestService(t *testing.T) {
t.Run("two labels created", func(t *testing.T) {
testfileRunner(t, "testdata/label", func(t *testing.T, pkg *Pkg) {
fakeBktSVC := mock.NewBucketService()
fakeLabelSVC := mock.NewLabelService()
fakeLabelSVC.FindLabelsFn = func(_ context.Context, filter influxdb.LabelFilter) ([]*influxdb.Label, error) {
return nil, errors.New("no labels found")
}
fakeDashSVC := mock.NewDashboardService()
svc := NewService(zap.NewNop(), fakeBktSVC, fakeLabelSVC, fakeDashSVC)
svc := NewService(WithLabelSVC(fakeLabelSVC))
_, diff, err := svc.DryRun(context.TODO(), influxdb.ID(100), pkg)
require.NoError(t, err)
@ -145,26 +136,75 @@ func TestService(t *testing.T) {
})
})
})
t.Run("variables", func(t *testing.T) {
testfileRunner(t, "testdata/variables", func(t *testing.T, pkg *Pkg) {
fakeVarSVC := mock.NewVariableService()
fakeVarSVC.FindVariablesF = func(_ context.Context, filter influxdb.VariableFilter, opts ...influxdb.FindOptions) ([]*influxdb.Variable, error) {
return []*influxdb.Variable{
{
ID: influxdb.ID(1),
Name: "var_const",
Description: "old desc",
},
}, nil
}
fakeLabelSVC := mock.NewLabelService() // ignore mappings for now
svc := NewService(
WithLabelSVC(fakeLabelSVC),
WithVariableSVC(fakeVarSVC),
)
_, diff, err := svc.DryRun(context.TODO(), influxdb.ID(100), pkg)
require.NoError(t, err)
require.Len(t, diff.Variables, 4)
expected := DiffVariable{
ID: SafeID(1),
Name: "var_const",
OldDesc: "old desc",
NewDesc: "var_const desc",
NewArgs: &influxdb.VariableArguments{
Type: "constant",
Values: influxdb.VariableConstantValues{"first val"},
},
}
assert.Equal(t, expected, diff.Variables[0])
expected = DiffVariable{
// no ID here since this one would be new
Name: "var_map",
OldDesc: "",
NewDesc: "var_map desc",
NewArgs: &influxdb.VariableArguments{
Type: "map",
Values: influxdb.VariableMapValues{"k1": "v1"},
},
}
assert.Equal(t, expected, diff.Variables[1])
})
})
})
t.Run("Apply", func(t *testing.T) {
t.Run("buckets", func(t *testing.T) {
t.Run("successfully creates pkg of buckets", func(t *testing.T) {
testfileRunner(t, "testdata/bucket", func(t *testing.T, pkg *Pkg) {
fakeBucketSVC := mock.NewBucketService()
fakeBucketSVC.CreateBucketFn = func(_ context.Context, b *influxdb.Bucket) error {
testfileRunner(t, "testdata/bucket.yml", func(t *testing.T, pkg *Pkg) {
fakeBktSVC := mock.NewBucketService()
fakeBktSVC.CreateBucketFn = func(_ context.Context, b *influxdb.Bucket) error {
b.ID = influxdb.ID(b.RetentionPeriod)
return nil
}
fakeBucketSVC.FindBucketByNameFn = func(_ context.Context, id influxdb.ID, s string) (*influxdb.Bucket, error) {
fakeBktSVC.FindBucketByNameFn = func(_ context.Context, id influxdb.ID, s string) (*influxdb.Bucket, error) {
// forces the bucket to be created a new
return nil, errors.New("an error")
}
fakeBucketSVC.UpdateBucketFn = func(_ context.Context, id influxdb.ID, upd influxdb.BucketUpdate) (*influxdb.Bucket, error) {
fakeBktSVC.UpdateBucketFn = func(_ context.Context, id influxdb.ID, upd influxdb.BucketUpdate) (*influxdb.Bucket, error) {
return &influxdb.Bucket{ID: id}, nil
}
svc := NewService(zap.NewNop(), fakeBucketSVC, nil, nil)
svc := NewService(WithBucketSVC(fakeBktSVC))
orgID := influxdb.ID(9000)
@ -196,19 +236,19 @@ func TestService(t *testing.T) {
RetentionPeriod: pkgBkt.RetentionPeriod,
}
fakeBucketSVC := mock.NewBucketService()
fakeBktSVC := mock.NewBucketService()
var createCallCount int
fakeBucketSVC.CreateBucketFn = func(_ context.Context, b *influxdb.Bucket) error {
fakeBktSVC.CreateBucketFn = func(_ context.Context, b *influxdb.Bucket) error {
createCallCount++
return nil
}
var updateCallCount int
fakeBucketSVC.UpdateBucketFn = func(_ context.Context, id influxdb.ID, upd influxdb.BucketUpdate) (*influxdb.Bucket, error) {
fakeBktSVC.UpdateBucketFn = func(_ context.Context, id influxdb.ID, upd influxdb.BucketUpdate) (*influxdb.Bucket, error) {
updateCallCount++
return &influxdb.Bucket{ID: id}, nil
}
svc := NewService(zap.NewNop(), fakeBucketSVC, nil, nil)
svc := NewService(WithBucketSVC(fakeBktSVC))
sum, err := svc.Apply(context.TODO(), orgID, pkg)
require.NoError(t, err)
@ -227,13 +267,13 @@ func TestService(t *testing.T) {
t.Run("rolls back all created buckets on an error", func(t *testing.T) {
testfileRunner(t, "testdata/bucket", func(t *testing.T, pkg *Pkg) {
fakeBucketSVC := mock.NewBucketService()
fakeBucketSVC.FindBucketByNameFn = func(_ context.Context, id influxdb.ID, s string) (*influxdb.Bucket, error) {
fakeBktSVC := mock.NewBucketService()
fakeBktSVC.FindBucketByNameFn = func(_ context.Context, id influxdb.ID, s string) (*influxdb.Bucket, error) {
// forces the bucket to be created a new
return nil, errors.New("an error")
}
var c int
fakeBucketSVC.CreateBucketFn = func(_ context.Context, b *influxdb.Bucket) error {
fakeBktSVC.CreateBucketFn = func(_ context.Context, b *influxdb.Bucket) error {
if c == 2 {
return errors.New("blowed up ")
}
@ -241,7 +281,7 @@ func TestService(t *testing.T) {
return nil
}
var count int
fakeBucketSVC.DeleteBucketFn = func(_ context.Context, id influxdb.ID) error {
fakeBktSVC.DeleteBucketFn = func(_ context.Context, id influxdb.ID) error {
count++
return nil
}
@ -249,7 +289,7 @@ func TestService(t *testing.T) {
pkg.mBuckets["copybuck1"] = pkg.mBuckets["rucket_11"]
pkg.mBuckets["copybuck2"] = pkg.mBuckets["rucket_11"]
svc := NewService(zap.NewNop(), fakeBucketSVC, nil, nil)
svc := NewService(WithBucketSVC(fakeBktSVC))
orgID := influxdb.ID(9000)
@ -272,7 +312,7 @@ func TestService(t *testing.T) {
return nil
}
svc := NewService(zap.NewNop(), nil, fakeLabelSVC, nil)
svc := NewService(WithLabelSVC(fakeLabelSVC))
orgID := influxdb.ID(9000)
@ -313,12 +353,11 @@ func TestService(t *testing.T) {
count++
return nil
}
fakeDashSVC := mock.NewDashboardService()
pkg.mLabels["copy1"] = pkg.mLabels["label_1"]
pkg.mLabels["copy2"] = pkg.mLabels["label_2"]
svc := NewService(zap.NewNop(), nil, fakeLabelSVC, fakeDashSVC)
svc := NewService(WithLabelSVC(fakeLabelSVC))
orgID := influxdb.ID(9000)
@ -363,7 +402,7 @@ func TestService(t *testing.T) {
return &influxdb.Label{ID: id}, nil
}
svc := NewService(zap.NewNop(), nil, fakeLabelSVC, nil)
svc := NewService(WithLabelSVC(fakeLabelSVC))
sum, err := svc.Apply(context.TODO(), orgID, pkg)
require.NoError(t, err)
@ -386,12 +425,11 @@ func TestService(t *testing.T) {
assert.Equal(t, 1, createCallCount) // only called for second label
})
})
})
t.Run("dashboards", func(t *testing.T) {
t.Run("successfully creates a dashboard", func(t *testing.T) {
testfileRunner(t, "testdata/dashboard", func(t *testing.T, pkg *Pkg) {
testfileRunner(t, "testdata/dashboard.yml", func(t *testing.T, pkg *Pkg) {
fakeDashSVC := mock.NewDashboardService()
id := 1
fakeDashSVC.CreateDashboardF = func(_ context.Context, d *influxdb.Dashboard) error {
@ -405,7 +443,7 @@ func TestService(t *testing.T) {
return &influxdb.View{}, nil
}
svc := NewService(zap.NewNop(), nil, nil, fakeDashSVC)
svc := NewService(WithDashboardSVC(fakeDashSVC))
orgID := influxdb.ID(9000)
@ -422,7 +460,7 @@ func TestService(t *testing.T) {
})
t.Run("rolls back created dashboard on an error", func(t *testing.T) {
testfileRunner(t, "testdata/dashboard", func(t *testing.T, pkg *Pkg) {
testfileRunner(t, "testdata/dashboard.yml", func(t *testing.T, pkg *Pkg) {
fakeDashSVC := mock.NewDashboardService()
var c int
fakeDashSVC.CreateDashboardF = func(_ context.Context, d *influxdb.Dashboard) error {
@ -442,7 +480,7 @@ func TestService(t *testing.T) {
pkg.mDashboards["copy1"] = pkg.mDashboards["dash_1"]
svc := NewService(zap.NewNop(), nil, nil, fakeDashSVC)
svc := NewService(WithDashboardSVC(fakeDashSVC))
orgID := influxdb.ID(9000)
@ -456,7 +494,7 @@ func TestService(t *testing.T) {
t.Run("label mapping", func(t *testing.T) {
t.Run("successfully creates pkg of labels", func(t *testing.T) {
testfileRunner(t, "testdata/bucket_associates_label", func(t *testing.T, pkg *Pkg) {
testfileRunner(t, "testdata/bucket_associates_label.yml", func(t *testing.T, pkg *Pkg) {
fakeBktSVC := mock.NewBucketService()
id := 1
fakeBktSVC.CreateBucketFn = func(_ context.Context, b *influxdb.Bucket) error {
@ -482,7 +520,11 @@ func TestService(t *testing.T) {
return nil
}
fakeDashSVC := mock.NewDashboardService()
svc := NewService(zap.NewNop(), fakeBktSVC, fakeLabelSVC, fakeDashSVC)
svc := NewService(
WithBucketSVC(fakeBktSVC),
WithLabelSVC(fakeLabelSVC),
WithDashboardSVC(fakeDashSVC),
)
orgID := influxdb.ID(9000)
@ -493,6 +535,126 @@ func TestService(t *testing.T) {
})
})
})
t.Run("variables", func(t *testing.T) {
t.Run("successfully creates pkg of variables", func(t *testing.T) {
testfileRunner(t, "testdata/variables.yml", func(t *testing.T, pkg *Pkg) {
fakeVarSVC := mock.NewVariableService()
id := 1
fakeVarSVC.CreateVariableF = func(_ context.Context, v *influxdb.Variable) error {
v.ID = influxdb.ID(id)
id++
return nil
}
svc := NewService(
WithLabelSVC(mock.NewLabelService()),
WithVariableSVC(fakeVarSVC),
)
orgID := influxdb.ID(9000)
sum, err := svc.Apply(context.TODO(), orgID, pkg)
require.NoError(t, err)
require.Len(t, sum.Variables, 4)
expected := sum.Variables[0]
assert.Equal(t, influxdb.ID(1), expected.ID)
assert.Equal(t, orgID, expected.OrganizationID)
assert.Equal(t, "var_const", expected.Name)
assert.Equal(t, "var_const desc", expected.Description)
require.NotNil(t, expected.Arguments)
assert.Equal(t, influxdb.VariableConstantValues{"first val"}, expected.Arguments.Values)
for i := 1; i < 3; i++ {
expected = sum.Variables[i]
assert.Equal(t, influxdb.ID(i+1), expected.ID)
}
})
})
t.Run("rolls back all created variables on an error", func(t *testing.T) {
testfileRunner(t, "testdata/variables.yml", func(t *testing.T, pkg *Pkg) {
fakeVarSVC := mock.NewVariableService()
var c int
fakeVarSVC.CreateVariableF = func(_ context.Context, l *influxdb.Variable) error {
// 4th variable will return the error here, and 3 before should be rolled back
if c == 3 {
return errors.New("blowed up ")
}
c++
return nil
}
var count int
fakeVarSVC.DeleteVariableF = func(_ context.Context, id influxdb.ID) error {
count++
return nil
}
svc := NewService(
WithLabelSVC(mock.NewLabelService()),
WithVariableSVC(fakeVarSVC),
)
orgID := influxdb.ID(9000)
_, err := svc.Apply(context.TODO(), orgID, pkg)
require.Error(t, err)
assert.Equal(t, 3, count)
})
})
t.Run("will not apply variable if no changes to be applied", func(t *testing.T) {
testfileRunner(t, "testdata/variables.yml", func(t *testing.T, pkg *Pkg) {
orgID := influxdb.ID(9000)
pkg.isVerified = true
pkgLabel := pkg.mVariables["var_const"]
pkgLabel.existing = &influxdb.Variable{
// makes all pkg changes same as they are on the existing
ID: influxdb.ID(1),
OrganizationID: orgID,
Name: pkgLabel.Name,
Arguments: &influxdb.VariableArguments{
Type: "constant",
Values: influxdb.VariableConstantValues{"first val"},
},
}
fakeVarSVC := mock.NewVariableService()
var createCallCount int
fakeVarSVC.CreateVariableF = func(_ context.Context, l *influxdb.Variable) error {
createCallCount++
if l.Name == "var_const" {
return errors.New("shouldn't get here")
}
return nil
}
fakeVarSVC.UpdateVariableF = func(_ context.Context, id influxdb.ID, v *influxdb.VariableUpdate) (*influxdb.Variable, error) {
if id > influxdb.ID(1) {
return nil, errors.New("this id should not be updated")
}
return &influxdb.Variable{ID: id}, nil
}
svc := NewService(
WithLabelSVC(mock.NewLabelService()),
WithVariableSVC(fakeVarSVC),
)
sum, err := svc.Apply(context.TODO(), orgID, pkg)
require.NoError(t, err)
require.Len(t, sum.Variables, 4)
expected := sum.Variables[0]
assert.Equal(t, influxdb.ID(1), expected.ID)
assert.Equal(t, "var_const", expected.Name)
assert.Equal(t, 3, createCallCount) // only called for last 3 labels
})
})
})
})
t.Run("CreatePkg", func(t *testing.T) {

View File

@ -0,0 +1,18 @@
apiVersion: 0.1.0
kind: Package
meta:
pkgName: pkg_name
pkgVersion: 1
description: pack description
spec:
resources:
- kind: Label
name: label_1
- kind: Variable
name: var_1
type: constant
values:
- first val
associations:
- kind: Label
name: label_1