From b8652ee178e4e51c57a653531b7aa712c8a70d41 Mon Sep 17 00:00:00 2001 From: Johnny Steenbergen Date: Mon, 16 Dec 2019 09:39:55 -0800 Subject: [PATCH] feat(pkger): add support for secret references to notification endpoints parsing --- cmd/influxd/launcher/launcher.go | 1 + cmd/influxd/launcher/pkger_test.go | 62 +++++++++++++++ http/notification_endpoint.go | 3 +- pkger/models.go | 76 +++++++++++++------ pkger/parser.go | 51 ++++++++++++- pkger/parser_test.go | 36 +++++++-- pkger/service.go | 57 +++++++++++++- pkger/service_test.go | 14 ++++ .../notification_endpoint_secrets.yml | 14 ++++ 9 files changed, 279 insertions(+), 35 deletions(-) create mode 100644 pkger/testdata/notification_endpoint_secrets.yml diff --git a/cmd/influxd/launcher/launcher.go b/cmd/influxd/launcher/launcher.go index 9e081acc5a..242e756bc0 100644 --- a/cmd/influxd/launcher/launcher.go +++ b/cmd/influxd/launcher/launcher.go @@ -842,6 +842,7 @@ func (m *Launcher) run(ctx context.Context) (err error) { pkger.WithDashboardSVC(authorizer.NewDashboardService(b.DashboardService)), pkger.WithLabelSVC(authorizer.NewLabelService(b.LabelService)), pkger.WithNoticationEndpointSVC(authorizer.NewNotificationEndpointService(b.NotificationEndpointService, b.UserResourceMappingService, b.OrganizationService)), + pkger.WithSecretSVC(authorizer.NewSecretService(b.SecretService)), pkger.WithTelegrafSVC(authorizer.NewTelegrafConfigService(b.TelegrafService, b.UserResourceMappingService)), pkger.WithVariableSVC(authorizer.NewVariableService(b.VariableService)), ) diff --git a/cmd/influxd/launcher/pkger_test.go b/cmd/influxd/launcher/pkger_test.go index fd8b7993c0..f086454f2a 100644 --- a/cmd/influxd/launcher/pkger_test.go +++ b/cmd/influxd/launcher/pkger_test.go @@ -3,6 +3,7 @@ package launcher_test import ( "context" "errors" + "fmt" "testing" "time" @@ -266,6 +267,67 @@ func TestLauncher_Pkger(t *testing.T) { require.NotEqual(t, sum1.Dashboards, sum2.Dashboards) }) + t.Run("referenced secret values provided do not create new secrets", func(t *testing.T) { + applyPkgStr := func(t *testing.T, pkgStr string) pkger.Summary { + t.Helper() + pkg, err := pkger.Parse(pkger.EncodingYAML, pkger.FromString(pkgStr)) + require.NoError(t, err) + + sum, err := svc.Apply(ctx, l.Org.ID, l.User.ID, pkg) + require.NoError(t, err) + return sum + } + + const pkgWithSecretRaw = `apiVersion: 0.1.0 +kind: Package +meta: + pkgName: pkg_name + pkgVersion: 1 + description: pack description +spec: + resources: + - kind: NotificationEndpointPagerDuty + name: pager_duty_notification_endpoint + url: http://localhost:8080/orgs/7167eb6719fa34e5/alert-history + routingKey: secret-sauce +` + secretSum := applyPkgStr(t, pkgWithSecretRaw) + require.Len(t, secretSum.NotificationEndpoints, 1) + + id := secretSum.NotificationEndpoints[0].NotificationEndpoint.GetID() + expected := influxdb.SecretField{ + Key: id.String() + "-routing-key", + } + secrets := secretSum.NotificationEndpoints[0].NotificationEndpoint.SecretFields() + require.Len(t, secrets, 1) + assert.Equal(t, expected, secrets[0]) + + const pkgWithSecretRef = `apiVersion: 0.1.0 +kind: Package +meta: + pkgName: pkg_name + pkgVersion: 1 + description: pack description +spec: + resources: + - kind: NotificationEndpointPagerDuty + name: pager_duty_notification_endpoint + url: http://localhost:8080/orgs/7167eb6719fa34e5/alert-history + routingKey: + secretRef: + key: %s-routing-key +` + secretSum = applyPkgStr(t, fmt.Sprintf(pkgWithSecretRef, id.String())) + require.Len(t, secretSum.NotificationEndpoints, 1) + + expected = influxdb.SecretField{ + Key: id.String() + "-routing-key", + } + secrets = secretSum.NotificationEndpoints[0].NotificationEndpoint.SecretFields() + require.Len(t, secrets, 1) + assert.Equal(t, expected, secrets[0]) + }) + t.Run("exporting resources with existing ids should return a valid pkg", func(t *testing.T) { resToClone := []pkger.ResourceToClone{ { diff --git a/http/notification_endpoint.go b/http/notification_endpoint.go index 888501e013..d1af571c6e 100644 --- a/http/notification_endpoint.go +++ b/http/notification_endpoint.go @@ -410,7 +410,8 @@ func (h *NotificationEndpointHandler) handlePostNotificationEndpoint(w http.Resp return } - if err := h.NotificationEndpointService.CreateNotificationEndpoint(ctx, edp.NotificationEndpoint, auth.GetUserID()); err != nil { + err = h.NotificationEndpointService.CreateNotificationEndpoint(ctx, edp.NotificationEndpoint, auth.GetUserID()) + if err != nil { h.HandleHTTPError(ctx, err, w) return } diff --git a/pkger/models.go b/pkger/models.go index da3bf33d31..acee8d81ee 100644 --- a/pkger/models.go +++ b/pkger/models.go @@ -573,6 +573,7 @@ type SummaryVariable struct { const ( fieldAssociations = "associations" fieldDescription = "description" + fieldKey = "key" fieldKind = "kind" fieldLanguage = "language" fieldName = "name" @@ -911,13 +912,13 @@ type notificationEndpoint struct { name string description string method string - password string - routingKey string + password references + routingKey references status string - token string + token references httpType string url string - username string + username references labels sortedLabels @@ -979,34 +980,30 @@ func (n *notificationEndpoint) summarize() SummaryNotificationEndpoint { Method: n.method, } switch n.httpType { - case notificationHTTPAuthTypeNone: - e.AuthMethod = notificationHTTPAuthTypeNone + case notificationHTTPAuthTypeBasic: + e.AuthMethod = notificationHTTPAuthTypeBasic + e.Password = n.password.SecretField() + e.Username = n.username.SecretField() case notificationHTTPAuthTypeBearer: e.AuthMethod = notificationHTTPAuthTypeBearer - e.Token = influxdb.SecretField{Value: &n.token} - default: - e.AuthMethod = notificationHTTPAuthTypeBasic - e.Password = influxdb.SecretField{Value: &n.password} - e.Username = influxdb.SecretField{Value: &n.username} + e.Token = n.token.SecretField() + case notificationHTTPAuthTypeNone: + e.AuthMethod = notificationHTTPAuthTypeNone } sum.NotificationEndpoint = e case notificationKindPagerDuty: sum.NotificationEndpoint = &endpoint.PagerDuty{ Base: base, ClientURL: n.url, - RoutingKey: influxdb.SecretField{Value: &n.routingKey}, + RoutingKey: n.routingKey.SecretField(), } case notificationKindSlack: - e := &endpoint.Slack{ - Base: base, - URL: n.url, + sum.NotificationEndpoint = &endpoint.Slack{ + Base: base, + URL: n.url, + Token: n.token.SecretField(), } - if n.token != "" { - e.Token = influxdb.SecretField{Value: &n.token} - } - sum.NotificationEndpoint = e } - sum.NotificationEndpoint.BackfillSecretKeys() return sum } @@ -1038,7 +1035,7 @@ func (n *notificationEndpoint) valid() []validationErr { switch n.kind { case notificationKindPagerDuty: - if n.routingKey == "" { + if !n.routingKey.hasValue() { failures = append(failures, validationErr{ Field: fieldNotificationEndpointRoutingKey, Msg: "must be provide", @@ -1054,20 +1051,20 @@ func (n *notificationEndpoint) valid() []validationErr { switch n.httpType { case notificationHTTPAuthTypeBasic: - if n.password == "" { + if !n.password.hasValue() { failures = append(failures, validationErr{ Field: fieldNotificationEndpointPassword, Msg: "must provide non empty string", }) } - if n.username == "" { + if !n.username.hasValue() { failures = append(failures, validationErr{ Field: fieldNotificationEndpointUsername, Msg: "must provide non empty string", }) } case notificationHTTPAuthTypeBearer: - if n.token == "" { + if !n.token.hasValue() { failures = append(failures, validationErr{ Field: fieldNotificationEndpointToken, Msg: "must provide non empty string", @@ -1833,6 +1830,37 @@ func (l legend) influxLegend() influxdb.Legend { } } +const ( + fieldReferencesSecret = "secretRef" +) + +type references struct { + val interface{} + Secret string `json:"secretRef"` +} + +func (r references) hasValue() bool { + return r.Secret != "" || r.val != nil +} + +func (r references) String() string { + if r.val != nil { + s, _ := r.val.(string) + return s + } + return "" +} + +func (r references) SecretField() influxdb.SecretField { + if secret := r.Secret; secret != "" { + return influxdb.SecretField{Key: secret} + } + if str := r.String(); str != "" { + return influxdb.SecretField{Value: &str} + } + return influxdb.SecretField{} +} + func flt64Ptr(f float64) *float64 { if f != 0 { return &f diff --git a/pkger/parser.go b/pkger/parser.go index 8273a10280..005605148c 100644 --- a/pkger/parser.go +++ b/pkger/parser.go @@ -140,6 +140,8 @@ type Pkg struct { mTelegrafs []*telegraf mVariables map[string]*variable + mSecrets map[string]struct{} + isVerified bool // dry run has verified pkg resources with existing resources isParsed bool // indicates the pkg has been parsed and all resources graphed accordingly } @@ -273,6 +275,15 @@ func (p *Pkg) notificationEndpoints() []*notificationEndpoint { return endpoints } +func (p *Pkg) secrets() map[string]bool { + // copies the secrets map so we can destroy this one without concern + secrets := make(map[string]bool, len(p.mSecrets)) + for secret := range p.mSecrets { + secrets[secret] = true + } + return secrets +} + func (p *Pkg) telegrafs() []*telegraf { teles := p.mTelegrafs[:] sort.Slice(teles, func(i, j int) bool { return teles[i].Name() < teles[j].Name() }) @@ -389,6 +400,8 @@ func (p *Pkg) validResources() error { } func (p *Pkg) graphResources() error { + p.mSecrets = make(map[string]struct{}) + graphFns := []func() *parseErr{ // labels are first, this is to validate associations with other resources p.graphLabels, @@ -545,12 +558,12 @@ func (p *Pkg) graphNotificationEndpoints() *parseErr { description: r.stringShort(fieldDescription), method: strings.TrimSpace(strings.ToUpper(r.stringShort(fieldNotificationEndpointHTTPMethod))), httpType: normStr(r.stringShort(fieldType)), - password: r.stringShort(fieldNotificationEndpointPassword), - routingKey: r.stringShort(fieldNotificationEndpointRoutingKey), + password: r.references(fieldNotificationEndpointPassword), + routingKey: r.references(fieldNotificationEndpointRoutingKey), status: normStr(r.stringShort(fieldStatus)), - token: r.stringShort(fieldNotificationEndpointToken), + token: r.references(fieldNotificationEndpointToken), url: r.stringShort(fieldNotificationEndpointURL), - username: r.stringShort(fieldNotificationEndpointUsername), + username: r.references(fieldNotificationEndpointUsername), } failures := p.parseNestedLabels(r, func(l *label) error { endpoint.labels = append(endpoint.labels, l) @@ -559,6 +572,13 @@ func (p *Pkg) graphNotificationEndpoints() *parseErr { }) sort.Sort(endpoint.labels) + refs := []references{endpoint.password, endpoint.routingKey, endpoint.token, endpoint.username} + for _, ref := range refs { + if secret := ref.Secret; secret != "" { + p.mSecrets[secret] = struct{}{} + } + } + p.mNotificationEndpoints[endpoint.Name()] = endpoint return append(failures, endpoint.valid()...) }) @@ -938,6 +958,29 @@ func (r Resource) intShort(key string) int { return i } +func (r Resource) references(key string) references { + v, ok := r[key] + if !ok { + return references{} + } + + var ref references + for _, f := range []string{fieldReferencesSecret} { + resBody, ok := ifaceToResource(v) + if !ok { + continue + } + if keyRes, ok := ifaceToResource(resBody[f]); ok { + ref.Secret = keyRes.stringShort(fieldKey) + } + } + if ref.Secret != "" { + return ref + } + + return references{val: v} +} + func (r Resource) string(key string) (string, bool) { return ifaceToStr(r[key]) } diff --git a/pkger/parser_test.go b/pkger/parser_test.go index dde70c3a3f..becd52e55d 100644 --- a/pkger/parser_test.go +++ b/pkger/parser_test.go @@ -2715,8 +2715,8 @@ spec: URL: "https://www.example.com/endpoint/basicauth", AuthMethod: "basic", Method: "POST", - Username: influxdb.SecretField{Key: "-username", Value: strPtr("secret username")}, - Password: influxdb.SecretField{Key: "-password", Value: strPtr("secret password")}, + Username: influxdb.SecretField{Value: strPtr("secret username")}, + Password: influxdb.SecretField{Value: strPtr("secret password")}, }, }, { @@ -2729,7 +2729,7 @@ spec: URL: "https://www.example.com/endpoint/bearerauth", AuthMethod: "bearer", Method: "PUT", - Token: influxdb.SecretField{Key: "-token", Value: strPtr("secret token")}, + Token: influxdb.SecretField{Value: strPtr("secret token")}, }, }, { @@ -2752,7 +2752,7 @@ spec: Status: influxdb.TaskStatusActive, }, ClientURL: "http://localhost:8080/orgs/7167eb6719fa34e5/alert-history", - RoutingKey: influxdb.SecretField{Key: "-routing-key", Value: strPtr("secret routing-key")}, + RoutingKey: influxdb.SecretField{Value: strPtr("secret routing-key")}, }, }, { @@ -2763,7 +2763,7 @@ spec: Status: influxdb.TaskStatusActive, }, URL: "https://hooks.slack.com/services/bip/piddy/boppidy", - Token: influxdb.SecretField{Key: "-token", Value: strPtr("tokenval")}, + Token: influxdb.SecretField{Value: strPtr("tokenval")}, }, }, } @@ -3365,6 +3365,32 @@ spec: } }) }) + + t.Run("referencing secrets", func(t *testing.T) { + testfileRunner(t, "testdata/notification_endpoint_secrets.yml", func(t *testing.T, pkg *Pkg) { + sum := pkg.Summary() + + endpoints := sum.NotificationEndpoints + require.Len(t, endpoints, 1) + + expected := &endpoint.PagerDuty{ + Base: endpoint.Base{ + Name: "pager_duty_notification_endpoint", + Status: influxdb.TaskStatusActive, + }, + ClientURL: "http://localhost:8080/orgs/7167eb6719fa34e5/alert-history", + RoutingKey: influxdb.SecretField{Key: "-routing-key", Value: strPtr("not emtpy")}, + } + actual, ok := endpoints[0].NotificationEndpoint.(*endpoint.PagerDuty) + require.True(t, ok) + assert.Equal(t, expected.Base.Name, actual.Name) + require.Nil(t, actual.RoutingKey.Value) + assert.Equal(t, "routing-key", actual.RoutingKey.Key) + + _, ok = pkg.mSecrets["routing-key"] + require.True(t, ok) + }) + }) } func Test_PkgValidationErr(t *testing.T) { diff --git a/pkger/service.go b/pkger/service.go index 891d322f19..f1fd659e55 100644 --- a/pkger/service.go +++ b/pkger/service.go @@ -31,6 +31,7 @@ type serviceOpt struct { bucketSVC influxdb.BucketService dashSVC influxdb.DashboardService endpointSVC influxdb.NotificationEndpointService + secretSVC influxdb.SecretService teleSVC influxdb.TelegrafConfigStore varSVC influxdb.VariableService @@ -75,6 +76,13 @@ func WithLabelSVC(labelSVC influxdb.LabelService) ServiceSetterFn { } } +// WithSecretSVC sets the secret service. +func WithSecretSVC(secretSVC influxdb.SecretService) ServiceSetterFn { + return func(opt *serviceOpt) { + opt.secretSVC = secretSVC + } +} + // WithTelegrafSVC sets the telegraf service. func WithTelegrafSVC(telegrafSVC influxdb.TelegrafConfigStore) ServiceSetterFn { return func(opt *serviceOpt) { @@ -98,6 +106,7 @@ type Service struct { bucketSVC influxdb.BucketService dashSVC influxdb.DashboardService endpointSVC influxdb.NotificationEndpointService + secretSVC influxdb.SecretService teleSVC influxdb.TelegrafConfigStore varSVC influxdb.VariableService @@ -122,6 +131,7 @@ func NewService(opts ...ServiceSetterFn) *Service { labelSVC: opt.labelSVC, dashSVC: opt.dashSVC, endpointSVC: opt.endpointSVC, + secretSVC: opt.secretSVC, teleSVC: opt.teleSVC, varSVC: opt.varSVC, applyReqLimit: opt.applyReqLimit, @@ -493,6 +503,10 @@ func (s *Service) DryRun(ctx context.Context, orgID, userID influxdb.ID, pkg *Pk parseErr = err } + if err := s.dryRunSecrets(ctx, orgID, pkg); err != nil { + return Summary{}, Diff{}, err + } + diffBuckets, err := s.dryRunBuckets(ctx, orgID, pkg) if err != nil { return Summary{}, Diff{}, err @@ -641,6 +655,33 @@ func (s *Service) dryRunNotificationEndpoints(ctx context.Context, orgID influxd return diffs, nil } +func (s *Service) dryRunSecrets(ctx context.Context, orgID influxdb.ID, pkg *Pkg) error { + secrets := pkg.secrets() + if len(secrets) == 0 { + return nil + } + + existingSecrets, err := s.secretSVC.GetSecretKeys(ctx, orgID) + if err != nil { + return err + } + + for _, secret := range existingSecrets { + delete(secrets, secret) + } + + if len(secrets) == 0 { + return nil + } + + missing := make([]string, 0, len(secrets)) + for secret := range secrets { + missing = append(missing, secret) + } + sort.Strings(missing) + return fmt.Errorf("secrets to not exist for secret reference keys: %s", strings.Join(missing, ", ")) +} + func (s *Service) dryRunTelegraf(pkg *Pkg) []DiffTelegraf { var diffs []DiffTelegraf for _, t := range pkg.telegrafs() { @@ -1150,6 +1191,20 @@ func (s *Service) applyNotificationEndpoints(endpoints []*notificationEndpoint) mutex.Do(func() { endpoints[i].id = influxEndpoint.GetID() + for _, secret := range influxEndpoint.SecretFields() { + switch { + case strings.HasSuffix(secret.Key, "-routing-key"): + endpoints[i].routingKey.Secret = secret.Key + case strings.HasSuffix(secret.Key, "-token"): + endpoints[i].token.Secret = secret.Key + case strings.HasSuffix(secret.Key, "-username"): + endpoints[i].username.Secret = secret.Key + case strings.HasSuffix(secret.Key, "-password"): + endpoints[i].password.Secret = secret.Key + default: + fmt.Println("no match for key: ", secret.Key) + } + } rollbackEndpoints = append(rollbackEndpoints, endpoints[i]) }) @@ -1175,7 +1230,7 @@ func (s *Service) applyNotificationEndpoint(ctx context.Context, e notificationE // stub out userID since we're always using hte http client which will fill it in for us with the token // feels a bit broken that is required. // TODO: look into this userID requirement - updatedEndpoint, err := s.endpointSVC.UpdateNotificationEndpoint(ctx, e.ID(), e.existing, 0) + updatedEndpoint, err := s.endpointSVC.UpdateNotificationEndpoint(ctx, e.ID(), e.existing, userID) if err != nil { return nil, err } diff --git a/pkger/service_test.go b/pkger/service_test.go index 2ed42b61d2..8e958ab50d 100644 --- a/pkger/service_test.go +++ b/pkger/service_test.go @@ -35,6 +35,7 @@ func TestService(t *testing.T) { WithDashboardSVC(opt.dashSVC), WithLabelSVC(opt.labelSVC), WithNoticationEndpointSVC(opt.endpointSVC), + WithSecretSVC(opt.secretSVC), WithTelegrafSVC(opt.teleSVC), WithVariableSVC(opt.varSVC), ) @@ -240,6 +241,19 @@ func TestService(t *testing.T) { }) }) + t.Run("secrets not found returns error", func(t *testing.T) { + testfileRunner(t, "testdata/notification_endpoint_secrets.yml", func(t *testing.T, pkg *Pkg) { + fakeSecretSVC := mock.NewSecretService() + fakeSecretSVC.GetSecretKeysFn = func(ctx context.Context, orgID influxdb.ID) ([]string, error) { + return []string{"rando-1", "rando-2"}, nil + } + svc := newTestService(WithSecretSVC(fakeSecretSVC)) + + _, _, err := svc.DryRun(context.TODO(), influxdb.ID(100), 0, pkg) + require.Error(t, err) + }) + }) + t.Run("variables", func(t *testing.T) { testfileRunner(t, "testdata/variables", func(t *testing.T, pkg *Pkg) { fakeVarSVC := mock.NewVariableService() diff --git a/pkger/testdata/notification_endpoint_secrets.yml b/pkger/testdata/notification_endpoint_secrets.yml new file mode 100644 index 0000000000..3c921e1098 --- /dev/null +++ b/pkger/testdata/notification_endpoint_secrets.yml @@ -0,0 +1,14 @@ +apiVersion: 0.1.0 +kind: Package +meta: + pkgName: pkg_name + pkgVersion: 1 + description: pack description +spec: + resources: + - kind: NotificationEndpointPagerDuty + name: pager_duty_notification_endpoint + url: http://localhost:8080/orgs/7167eb6719fa34e5/alert-history + routingKey: + secretRef: + key: "routing-key"