Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat(pkger): add support for secret references to notification endpoints parsing

pull/16245/head
Johnny Steenbergen 2019-12-16 09:39:55 -08:00 committed by Johnny Steenbergen
parent a6e768dc7c
commit b8652ee178
9 changed files with 279 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/influxd/launcher/launcher.go
View File

@ -842,6 +842,7 @@ func (m *Launcher) run(ctx context.Context) (err error) {
pkger.WithDashboardSVC(authorizer.NewDashboardService(b.DashboardService)),
pkger.WithLabelSVC(authorizer.NewLabelService(b.LabelService)),
pkger.WithNoticationEndpointSVC(authorizer.NewNotificationEndpointService(b.NotificationEndpointService, b.UserResourceMappingService, b.OrganizationService)),
pkger.WithSecretSVC(authorizer.NewSecretService(b.SecretService)),
pkger.WithTelegrafSVC(authorizer.NewTelegrafConfigService(b.TelegrafService, b.UserResourceMappingService)),
pkger.WithVariableSVC(authorizer.NewVariableService(b.VariableService)),
)

62
cmd/influxd/launcher/pkger_test.go
View File

@ -3,6 +3,7 @@ package launcher_test
import (
"context"
"errors"
"fmt"
"testing"
"time"
@ -266,6 +267,67 @@ func TestLauncher_Pkger(t *testing.T) {
require.NotEqual(t, sum1.Dashboards, sum2.Dashboards)
})
t.Run("referenced secret values provided do not create new secrets", func(t *testing.T) {
applyPkgStr := func(t *testing.T, pkgStr string) pkger.Summary {
t.Helper()
pkg, err := pkger.Parse(pkger.EncodingYAML, pkger.FromString(pkgStr))
require.NoError(t, err)
sum, err := svc.Apply(ctx, l.Org.ID, l.User.ID, pkg)
require.NoError(t, err)
return sum
}
const pkgWithSecretRaw = `apiVersion: 0.1.0
kind: Package
meta:
pkgName: pkg_name
pkgVersion: 1
description: pack description
spec:
resources:
- kind: NotificationEndpointPagerDuty
name: pager_duty_notification_endpoint
url: http://localhost:8080/orgs/7167eb6719fa34e5/alert-history
routingKey: secret-sauce
`
secretSum := applyPkgStr(t, pkgWithSecretRaw)
require.Len(t, secretSum.NotificationEndpoints, 1)
id := secretSum.NotificationEndpoints[0].NotificationEndpoint.GetID()
expected := influxdb.SecretField{
Key: id.String() + "-routing-key",
}
secrets := secretSum.NotificationEndpoints[0].NotificationEndpoint.SecretFields()
require.Len(t, secrets, 1)
assert.Equal(t, expected, secrets[0])
const pkgWithSecretRef = `apiVersion: 0.1.0
kind: Package
meta:
pkgName: pkg_name
pkgVersion: 1
description: pack description
spec:
resources:
- kind: NotificationEndpointPagerDuty
name: pager_duty_notification_endpoint
url: http://localhost:8080/orgs/7167eb6719fa34e5/alert-history
routingKey:
secretRef:
key: %s-routing-key
`
secretSum = applyPkgStr(t, fmt.Sprintf(pkgWithSecretRef, id.String()))
require.Len(t, secretSum.NotificationEndpoints, 1)
expected = influxdb.SecretField{
Key: id.String() + "-routing-key",
}
secrets = secretSum.NotificationEndpoints[0].NotificationEndpoint.SecretFields()
require.Len(t, secrets, 1)
assert.Equal(t, expected, secrets[0])
})
t.Run("exporting resources with existing ids should return a valid pkg", func(t *testing.T) {
resToClone := []pkger.ResourceToClone{
{

3
http/notification_endpoint.go
View File

@ -410,7 +410,8 @@ func (h *NotificationEndpointHandler) handlePostNotificationEndpoint(w http.Resp
return
}
if err := h.NotificationEndpointService.CreateNotificationEndpoint(ctx, edp.NotificationEndpoint, auth.GetUserID()); err != nil {
err = h.NotificationEndpointService.CreateNotificationEndpoint(ctx, edp.NotificationEndpoint, auth.GetUserID())
if err != nil {
h.HandleHTTPError(ctx, err, w)
return
}

76
pkger/models.go
View File

@ -573,6 +573,7 @@ type SummaryVariable struct {
const (
fieldAssociations = "associations"
fieldDescription = "description"
fieldKey = "key"
fieldKind = "kind"
fieldLanguage = "language"
fieldName = "name"
@ -911,13 +912,13 @@ type notificationEndpoint struct {
name string
description string
method string
password string
routingKey string
password references
routingKey references
status string
token string
token references
httpType string
url string
username string
username references
labels sortedLabels
@ -979,34 +980,30 @@ func (n *notificationEndpoint) summarize() SummaryNotificationEndpoint {
Method: n.method,
}
switch n.httpType {
case notificationHTTPAuthTypeNone:
e.AuthMethod = notificationHTTPAuthTypeNone
case notificationHTTPAuthTypeBasic:
e.AuthMethod = notificationHTTPAuthTypeBasic
e.Password = n.password.SecretField()
e.Username = n.username.SecretField()
case notificationHTTPAuthTypeBearer:
e.AuthMethod = notificationHTTPAuthTypeBearer
e.Token = influxdb.SecretField{Value: &n.token}
default:
e.AuthMethod = notificationHTTPAuthTypeBasic
e.Password = influxdb.SecretField{Value: &n.password}
e.Username = influxdb.SecretField{Value: &n.username}
e.Token = n.token.SecretField()
case notificationHTTPAuthTypeNone:
e.AuthMethod = notificationHTTPAuthTypeNone
}
sum.NotificationEndpoint = e
case notificationKindPagerDuty:
sum.NotificationEndpoint = &endpoint.PagerDuty{
Base: base,
ClientURL: n.url,
RoutingKey: influxdb.SecretField{Value: &n.routingKey},
RoutingKey: n.routingKey.SecretField(),
}
case notificationKindSlack:
e := &endpoint.Slack{
Base: base,
URL: n.url,
sum.NotificationEndpoint = &endpoint.Slack{
Base: base,
URL: n.url,
Token: n.token.SecretField(),
}
if n.token != "" {
e.Token = influxdb.SecretField{Value: &n.token}
}
sum.NotificationEndpoint = e
}
sum.NotificationEndpoint.BackfillSecretKeys()
return sum
}
@ -1038,7 +1035,7 @@ func (n *notificationEndpoint) valid() []validationErr {
switch n.kind {
case notificationKindPagerDuty:
if n.routingKey == "" {
if !n.routingKey.hasValue() {
failures = append(failures, validationErr{
Field: fieldNotificationEndpointRoutingKey,
Msg: "must be provide",
@ -1054,20 +1051,20 @@ func (n *notificationEndpoint) valid() []validationErr {
switch n.httpType {
case notificationHTTPAuthTypeBasic:
if n.password == "" {
if !n.password.hasValue() {
failures = append(failures, validationErr{
Field: fieldNotificationEndpointPassword,
Msg: "must provide non empty string",
})
}
if n.username == "" {
if !n.username.hasValue() {
failures = append(failures, validationErr{
Field: fieldNotificationEndpointUsername,
Msg: "must provide non empty string",
})
}
case notificationHTTPAuthTypeBearer:
if n.token == "" {
if !n.token.hasValue() {
failures = append(failures, validationErr{
Field: fieldNotificationEndpointToken,
Msg: "must provide non empty string",
@ -1833,6 +1830,37 @@ func (l legend) influxLegend() influxdb.Legend {
}
}
const (
fieldReferencesSecret = "secretRef"
)
type references struct {
val interface{}
Secret string `json:"secretRef"`
}
func (r references) hasValue() bool {
return r.Secret != "" || r.val != nil
}
func (r references) String() string {
if r.val != nil {
s, _ := r.val.(string)
return s
}
return ""
}
func (r references) SecretField() influxdb.SecretField {
if secret := r.Secret; secret != "" {
return influxdb.SecretField{Key: secret}
}
if str := r.String(); str != "" {
return influxdb.SecretField{Value: &str}
}
return influxdb.SecretField{}
}
func flt64Ptr(f float64) *float64 {
if f != 0 {
return &f

51
pkger/parser.go
View File

@ -140,6 +140,8 @@ type Pkg struct {
mTelegrafs []*telegraf
mVariables map[string]*variable
mSecrets map[string]struct{}
isVerified bool // dry run has verified pkg resources with existing resources
isParsed bool // indicates the pkg has been parsed and all resources graphed accordingly
}
@ -273,6 +275,15 @@ func (p *Pkg) notificationEndpoints() []*notificationEndpoint {
return endpoints
}
func (p *Pkg) secrets() map[string]bool {
// copies the secrets map so we can destroy this one without concern
secrets := make(map[string]bool, len(p.mSecrets))
for secret := range p.mSecrets {
secrets[secret] = true
}
return secrets
}
func (p *Pkg) telegrafs() []*telegraf {
teles := p.mTelegrafs[:]
sort.Slice(teles, func(i, j int) bool { return teles[i].Name() < teles[j].Name() })
@ -389,6 +400,8 @@ func (p *Pkg) validResources() error {
}
func (p *Pkg) graphResources() error {
p.mSecrets = make(map[string]struct{})
graphFns := []func() *parseErr{
// labels are first, this is to validate associations with other resources
p.graphLabels,
@ -545,12 +558,12 @@ func (p *Pkg) graphNotificationEndpoints() *parseErr {
description: r.stringShort(fieldDescription),
method: strings.TrimSpace(strings.ToUpper(r.stringShort(fieldNotificationEndpointHTTPMethod))),
httpType: normStr(r.stringShort(fieldType)),
password: r.stringShort(fieldNotificationEndpointPassword),
routingKey: r.stringShort(fieldNotificationEndpointRoutingKey),
password: r.references(fieldNotificationEndpointPassword),
routingKey: r.references(fieldNotificationEndpointRoutingKey),
status: normStr(r.stringShort(fieldStatus)),
token: r.stringShort(fieldNotificationEndpointToken),
token: r.references(fieldNotificationEndpointToken),
url: r.stringShort(fieldNotificationEndpointURL),
username: r.stringShort(fieldNotificationEndpointUsername),
username: r.references(fieldNotificationEndpointUsername),
}
failures := p.parseNestedLabels(r, func(l *label) error {
endpoint.labels = append(endpoint.labels, l)
@ -559,6 +572,13 @@ func (p *Pkg) graphNotificationEndpoints() *parseErr {
})
sort.Sort(endpoint.labels)
refs := []references{endpoint.password, endpoint.routingKey, endpoint.token, endpoint.username}
for _, ref := range refs {
if secret := ref.Secret; secret != "" {
p.mSecrets[secret] = struct{}{}
}
}
p.mNotificationEndpoints[endpoint.Name()] = endpoint
return append(failures, endpoint.valid()...)
})
@ -938,6 +958,29 @@ func (r Resource) intShort(key string) int {
return i
}
func (r Resource) references(key string) references {
v, ok := r[key]
if !ok {
return references{}
}
var ref references
for _, f := range []string{fieldReferencesSecret} {
resBody, ok := ifaceToResource(v)
if !ok {
continue
}
if keyRes, ok := ifaceToResource(resBody[f]); ok {
ref.Secret = keyRes.stringShort(fieldKey)
}
}
if ref.Secret != "" {
return ref
}
return references{val: v}
}
func (r Resource) string(key string) (string, bool) {
return ifaceToStr(r[key])
}

36
pkger/parser_test.go
View File

@ -2715,8 +2715,8 @@ spec:
URL: "https://www.example.com/endpoint/basicauth",
AuthMethod: "basic",
Method: "POST",
Username: influxdb.SecretField{Key: "-username", Value: strPtr("secret username")},
Password: influxdb.SecretField{Key: "-password", Value: strPtr("secret password")},
Username: influxdb.SecretField{Value: strPtr("secret username")},
Password: influxdb.SecretField{Value: strPtr("secret password")},
},
},
{
@ -2729,7 +2729,7 @@ spec:
URL: "https://www.example.com/endpoint/bearerauth",
AuthMethod: "bearer",
Method: "PUT",
Token: influxdb.SecretField{Key: "-token", Value: strPtr("secret token")},
Token: influxdb.SecretField{Value: strPtr("secret token")},
},
},
{
@ -2752,7 +2752,7 @@ spec:
Status: influxdb.TaskStatusActive,
},
ClientURL: "http://localhost:8080/orgs/7167eb6719fa34e5/alert-history",
RoutingKey: influxdb.SecretField{Key: "-routing-key", Value: strPtr("secret routing-key")},
RoutingKey: influxdb.SecretField{Value: strPtr("secret routing-key")},
},
},
{
@ -2763,7 +2763,7 @@ spec:
Status: influxdb.TaskStatusActive,
},
URL: "https://hooks.slack.com/services/bip/piddy/boppidy",
Token: influxdb.SecretField{Key: "-token", Value: strPtr("tokenval")},
Token: influxdb.SecretField{Value: strPtr("tokenval")},
},
},
}
@ -3365,6 +3365,32 @@ spec:
}
})
})
t.Run("referencing secrets", func(t *testing.T) {
testfileRunner(t, "testdata/notification_endpoint_secrets.yml", func(t *testing.T, pkg *Pkg) {
sum := pkg.Summary()
endpoints := sum.NotificationEndpoints
require.Len(t, endpoints, 1)
expected := &endpoint.PagerDuty{
Base: endpoint.Base{
Name: "pager_duty_notification_endpoint",
Status: influxdb.TaskStatusActive,
},
ClientURL: "http://localhost:8080/orgs/7167eb6719fa34e5/alert-history",
RoutingKey: influxdb.SecretField{Key: "-routing-key", Value: strPtr("not emtpy")},
}
actual, ok := endpoints[0].NotificationEndpoint.(*endpoint.PagerDuty)
require.True(t, ok)
assert.Equal(t, expected.Base.Name, actual.Name)
require.Nil(t, actual.RoutingKey.Value)
assert.Equal(t, "routing-key", actual.RoutingKey.Key)
_, ok = pkg.mSecrets["routing-key"]
require.True(t, ok)
})
})
}
func Test_PkgValidationErr(t *testing.T) {

57
pkger/service.go
View File

@ -31,6 +31,7 @@ type serviceOpt struct {
bucketSVC influxdb.BucketService
dashSVC influxdb.DashboardService
endpointSVC influxdb.NotificationEndpointService
secretSVC influxdb.SecretService
teleSVC influxdb.TelegrafConfigStore
varSVC influxdb.VariableService
@ -75,6 +76,13 @@ func WithLabelSVC(labelSVC influxdb.LabelService) ServiceSetterFn {
}
}
// WithSecretSVC sets the secret service.
func WithSecretSVC(secretSVC influxdb.SecretService) ServiceSetterFn {
return func(opt *serviceOpt) {
opt.secretSVC = secretSVC
}
}
// WithTelegrafSVC sets the telegraf service.
func WithTelegrafSVC(telegrafSVC influxdb.TelegrafConfigStore) ServiceSetterFn {
return func(opt *serviceOpt) {
@ -98,6 +106,7 @@ type Service struct {
bucketSVC influxdb.BucketService
dashSVC influxdb.DashboardService
endpointSVC influxdb.NotificationEndpointService
secretSVC influxdb.SecretService
teleSVC influxdb.TelegrafConfigStore
varSVC influxdb.VariableService
@ -122,6 +131,7 @@ func NewService(opts ...ServiceSetterFn) *Service {
labelSVC: opt.labelSVC,
dashSVC: opt.dashSVC,
endpointSVC: opt.endpointSVC,
secretSVC: opt.secretSVC,
teleSVC: opt.teleSVC,
varSVC: opt.varSVC,
applyReqLimit: opt.applyReqLimit,
@ -493,6 +503,10 @@ func (s *Service) DryRun(ctx context.Context, orgID, userID influxdb.ID, pkg *Pk
parseErr = err
}
if err := s.dryRunSecrets(ctx, orgID, pkg); err != nil {
return Summary{}, Diff{}, err
}
diffBuckets, err := s.dryRunBuckets(ctx, orgID, pkg)
if err != nil {
return Summary{}, Diff{}, err
@ -641,6 +655,33 @@ func (s *Service) dryRunNotificationEndpoints(ctx context.Context, orgID influxd
return diffs, nil
}
func (s *Service) dryRunSecrets(ctx context.Context, orgID influxdb.ID, pkg *Pkg) error {
secrets := pkg.secrets()
if len(secrets) == 0 {
return nil
}
existingSecrets, err := s.secretSVC.GetSecretKeys(ctx, orgID)
if err != nil {
return err
}
for _, secret := range existingSecrets {
delete(secrets, secret)
}
if len(secrets) == 0 {
return nil
}
missing := make([]string, 0, len(secrets))
for secret := range secrets {
missing = append(missing, secret)
}
sort.Strings(missing)
return fmt.Errorf("secrets to not exist for secret reference keys: %s", strings.Join(missing, ", "))
}
func (s *Service) dryRunTelegraf(pkg *Pkg) []DiffTelegraf {
var diffs []DiffTelegraf
for _, t := range pkg.telegrafs() {
@ -1150,6 +1191,20 @@ func (s *Service) applyNotificationEndpoints(endpoints []*notificationEndpoint)
mutex.Do(func() {
endpoints[i].id = influxEndpoint.GetID()
for _, secret := range influxEndpoint.SecretFields() {
switch {
case strings.HasSuffix(secret.Key, "-routing-key"):
endpoints[i].routingKey.Secret = secret.Key
case strings.HasSuffix(secret.Key, "-token"):
endpoints[i].token.Secret = secret.Key
case strings.HasSuffix(secret.Key, "-username"):
endpoints[i].username.Secret = secret.Key
case strings.HasSuffix(secret.Key, "-password"):
endpoints[i].password.Secret = secret.Key
default:
fmt.Println("no match for key: ", secret.Key)
}
}
rollbackEndpoints = append(rollbackEndpoints, endpoints[i])
})
@ -1175,7 +1230,7 @@ func (s *Service) applyNotificationEndpoint(ctx context.Context, e notificationE
// stub out userID since we're always using hte http client which will fill it in for us with the token
// feels a bit broken that is required.
// TODO: look into this userID requirement
updatedEndpoint, err := s.endpointSVC.UpdateNotificationEndpoint(ctx, e.ID(), e.existing, 0)
updatedEndpoint, err := s.endpointSVC.UpdateNotificationEndpoint(ctx, e.ID(), e.existing, userID)
if err != nil {
return nil, err
}

14
pkger/service_test.go
View File

@ -35,6 +35,7 @@ func TestService(t *testing.T) {
WithDashboardSVC(opt.dashSVC),
WithLabelSVC(opt.labelSVC),
WithNoticationEndpointSVC(opt.endpointSVC),
WithSecretSVC(opt.secretSVC),
WithTelegrafSVC(opt.teleSVC),
WithVariableSVC(opt.varSVC),
)
@ -240,6 +241,19 @@ func TestService(t *testing.T) {
})
})
t.Run("secrets not found returns error", func(t *testing.T) {
testfileRunner(t, "testdata/notification_endpoint_secrets.yml", func(t *testing.T, pkg *Pkg) {
fakeSecretSVC := mock.NewSecretService()
fakeSecretSVC.GetSecretKeysFn = func(ctx context.Context, orgID influxdb.ID) ([]string, error) {
return []string{"rando-1", "rando-2"}, nil
}
svc := newTestService(WithSecretSVC(fakeSecretSVC))
_, _, err := svc.DryRun(context.TODO(), influxdb.ID(100), 0, pkg)
require.Error(t, err)
})
})
t.Run("variables", func(t *testing.T) {
testfileRunner(t, "testdata/variables", func(t *testing.T, pkg *Pkg) {
fakeVarSVC := mock.NewVariableService()

14
pkger/testdata/notification_endpoint_secrets.yml vendored Normal file
View File

@ -0,0 +1,14 @@
apiVersion: 0.1.0
kind: Package
meta:
pkgName: pkg_name
pkgVersion: 1
description: pack description
spec:
resources:
- kind: NotificationEndpointPagerDuty
name: pager_duty_notification_endpoint
url: http://localhost:8080/orgs/7167eb6719fa34e5/alert-history
routingKey:
secretRef:
key: "routing-key"