diff --git a/dbrp/http_server_dbrp_test.go b/dbrp/http_server_dbrp_test.go index 8abba254dc..67d08316c0 100644 --- a/dbrp/http_server_dbrp_test.go +++ b/dbrp/http_server_dbrp_test.go @@ -489,7 +489,7 @@ func Test_handleDeleteDBRP(t *testing.T) { BucketID: influxdbtesting.MustIDBase16("5555f7ed2a035555"), OrganizationID: influxdbtesting.MustIDBase16("059af7ed2a034000"), Database: "mydb", - RetentionPolicy: "autogen", + RetentionPolicy: "testrp", Default: true, } if err := svc.Create(ctx, d); err != nil { diff --git a/dbrp/service.go b/dbrp/service.go index 70ca7d5ae6..a024982ddf 100644 --- a/dbrp/service.go +++ b/dbrp/service.go @@ -28,6 +28,7 @@ import ( "bytes" "context" "encoding/json" + "strings" "github.com/influxdata/influxdb/v2" "github.com/influxdata/influxdb/v2/kit/platform" @@ -231,11 +232,28 @@ func (s *Service) FindByID(ctx context.Context, orgID, id platform.ID) (*influxd } return nil }); err != nil { + // if not found, fallback to virtual DBRP search + if err == ErrDBRPNotFound { + b, err := s.bucketSvc.FindBucketByID(ctx, id) + if err != nil || b == nil { + return nil, ErrDBRPNotFound + } + return bucketToMapping(b), nil + } return nil, err } return m, nil } +// parseDBRP parses DB and RP strings out of a bucket name +func parseDBRP(bucketName string) (string, string) { + db, rp, isCut := strings.Cut(bucketName, "/") + if isCut { + return db, rp + } + return bucketName, "autogen" +} + // FindMany returns a list of mappings that match filter and the total count of matching dbrp mappings. // TODO(affo): find a smart way to apply FindOptions to a list of items. func (s *Service) FindMany(ctx context.Context, filter influxdb.DBRPMappingFilter, opts ...influxdb.FindOptions) ([]*influxdb.DBRPMapping, int, error) { @@ -334,8 +352,65 @@ func (s *Service) FindMany(ctx context.Context, filter influxdb.DBRPMappingFilte } return nil }) + if err != nil { + return ms, len(ms), err + } - return ms, len(ms), err + buckets, count, err := s.bucketSvc.FindBuckets(ctx, influxdb.BucketFilter{ + ID: filter.BucketID, + Name: filter.Database, + OrganizationID: filter.OrgID, + }, opts...) + if (err != nil || count == 0) && filter.Database != nil && filter.RetentionPolicy != nil { + // if the search couldn't find a corresponding dbrp, it could be that the bucket name has a slash (like db/rp) + // instead of just bucket name being the database with "autogen" retention policy + bucketName := *filter.Database + "/" + *filter.RetentionPolicy + buckets, _, err = s.bucketSvc.FindBuckets(ctx, influxdb.BucketFilter{ + ID: filter.BucketID, + Name: &bucketName, + OrganizationID: filter.OrgID, + }, opts...) + if err != nil { + // we were unable to find any virtual mappings, so return what physical mappings we have + return ms, len(ms), nil + } + } +OUTER: + for _, bucket := range buckets { + // check if this virtual mapping has been overriden by a custom, physical mapping + for _, m := range ms { + if m.BucketID == bucket.ID { + continue OUTER + } + } + if bucket == nil { + continue + } + mapping := bucketToMapping(bucket) + if filterFunc(mapping, filter) { + ms = append(ms, mapping) + } + } + + return ms, len(ms), nil +} + +func bucketToMapping(bucket *influxdb.Bucket) *influxdb.DBRPMapping { + if bucket == nil { + return nil + } + // for now, virtual DBRPs will use the same ID as their bucket to be able to find them by ID + dbrpID := bucket.ID + db, rp := parseDBRP(bucket.Name) + return &influxdb.DBRPMapping{ + ID: dbrpID, + Default: false, + Database: db, + RetentionPolicy: rp, + OrganizationID: bucket.OrgID, + BucketID: bucket.ID, + Virtual: true, + } } // Create creates a new mapping. @@ -354,7 +429,7 @@ func (s *Service) Create(ctx context.Context, dbrp *influxdb.DBRPMapping) error } // If a dbrp with this particular ID already exists an error is returned. - if _, err := s.FindByID(ctx, dbrp.OrganizationID, dbrp.ID); err == nil { + if d, err := s.FindByID(ctx, dbrp.OrganizationID, dbrp.ID); err == nil && !d.Virtual { return ErrDBRPAlreadyExists("dbrp already exist for this particular ID. If you are trying an update use the right function .Update") } // If a dbrp with this orgID, db, and rp exists an error is returned. diff --git a/dbrp/service_test.go b/dbrp/service_test.go index 02a95ba056..0f69821c11 100644 --- a/dbrp/service_test.go +++ b/dbrp/service_test.go @@ -23,6 +23,9 @@ func initDBRPMappingService(f itesting.DBRPMappingFields, t *testing.T) (influxd Name: fmt.Sprintf("bucket-%v", id), }, nil }, + FindBucketsFn: func(ctx context.Context, bf influxdb.BucketFilter, fo ...influxdb.FindOptions) ([]*influxdb.Bucket, int, error) { + return []*influxdb.Bucket{}, 0, nil + }, } } diff --git a/dbrp_mapping.go b/dbrp_mapping.go index de21d7fd53..f7b86dd87a 100644 --- a/dbrp_mapping.go +++ b/dbrp_mapping.go @@ -35,6 +35,8 @@ type DBRPMapping struct { // Default indicates if this mapping is the default for the cluster and database. Default bool `json:"default"` + // Virtual indicates if this is a virtual mapping (tied to bucket name) or physical + Virtual bool `json:"virtual"` OrganizationID platform.ID `json:"orgID"` BucketID platform.ID `json:"bucketID"` diff --git a/influxql/v1tests/query_test.go b/influxql/v1tests/query_test.go index 21d55e67d8..4b82bf8b93 100644 --- a/influxql/v1tests/query_test.go +++ b/influxql/v1tests/query_test.go @@ -177,7 +177,7 @@ func TestServer_Query_ShowDatabases(t *testing.T) { &Query{ name: "show databases does not return duplicates", command: "SHOW DATABASES", - exp: `{"results":[{"statement_id":0,"series":[{"name":"databases","columns":["name"],"values":[["my-bucket"],["telegraf"]]}]}]}`, + exp: `{"results":[{"statement_id":0,"series":[{"name":"databases","columns":["name"],"values":[["my-bucket"],["telegraf"],["_monitoring"],["_tasks"],["db"]]}]}]}`, }, ) diff --git a/testing/dbrp_mapping.go b/testing/dbrp_mapping.go index 5a552b1024..27bce2f4c0 100644 --- a/testing/dbrp_mapping.go +++ b/testing/dbrp_mapping.go @@ -331,7 +331,9 @@ func CreateDBRPMappingV2( } return nil, nil }, - }, + FindBucketsFn: func(ctx context.Context, bf influxdb.BucketFilter, fo ...influxdb.FindOptions) ([]*influxdb.Bucket, int, error) { + return []*influxdb.Bucket{}, 0, nil + }}, DBRPMappingsV2: []*influxdb.DBRPMapping{{ ID: 100, Database: "database1", @@ -622,6 +624,112 @@ func FindManyDBRPMappingsV2( }, }, }, + { + name: "find virtual by orgID", + fields: DBRPMappingFields{ + BucketSvc: &mock.BucketService{ + FindBucketByIDFn: func(ctx context.Context, id platform.ID) (*influxdb.Bucket, error) { + if id == MustIDBase16(dbrpBucket2ID) { + return nil, &errors2.Error{ + Code: errors2.ENotFound, + Msg: "bucket not found", + } + } + return nil, nil + }, + FindBucketsFn: func(ctx context.Context, bf influxdb.BucketFilter, fo ...influxdb.FindOptions) ([]*influxdb.Bucket, int, error) { + return []*influxdb.Bucket{ + // org 3 + {ID: 100, Name: "testdb", OrgID: MustIDBase16(dbrpOrg3ID)}, + {ID: 200, Name: "testdb2/testrp2", OrgID: MustIDBase16(dbrpOrg3ID)}, + // org 2 + {ID: 300, Name: "testdb3", OrgID: MustIDBase16(dbrpOrg2ID)}, + {ID: 400, Name: "testdb4/testrp4", OrgID: MustIDBase16(dbrpOrg2ID)}, + }, 0, nil + }}, + DBRPMappingsV2: []*influxdb.DBRPMapping{}, + }, + args: args{ + filter: influxdb.DBRPMappingFilter{ + OrgID: MustIDBase16Ptr(dbrpOrg3ID), + }, + }, + wants: wants{ + dbrpMappings: []*influxdb.DBRPMapping{ + { + ID: 100, + Database: "testdb", + RetentionPolicy: "autogen", + Default: false, + Virtual: true, + OrganizationID: MustIDBase16(dbrpOrg3ID), + BucketID: 100, + }, + { + ID: 200, + Database: "testdb2", + RetentionPolicy: "testrp2", + Default: false, + Virtual: true, + OrganizationID: MustIDBase16(dbrpOrg3ID), + BucketID: 200, + }, + }, + }, + }, + { + name: "find virtual by rp", + fields: DBRPMappingFields{ + BucketSvc: &mock.BucketService{ + FindBucketByIDFn: func(ctx context.Context, id platform.ID) (*influxdb.Bucket, error) { + if id == MustIDBase16(dbrpBucket2ID) { + return nil, &errors2.Error{ + Code: errors2.ENotFound, + Msg: "bucket not found", + } + } + return nil, nil + }, + FindBucketsFn: func(ctx context.Context, bf influxdb.BucketFilter, fo ...influxdb.FindOptions) ([]*influxdb.Bucket, int, error) { + return []*influxdb.Bucket{ + // org 3 + {ID: 100, Name: "testdb", OrgID: MustIDBase16(dbrpOrg3ID)}, + {ID: 200, Name: "testdb2/testrp2", OrgID: MustIDBase16(dbrpOrg3ID)}, + // org 2 + {ID: 300, Name: "testdb3", OrgID: MustIDBase16(dbrpOrg2ID)}, + {ID: 400, Name: "testdb4/testrp4", OrgID: MustIDBase16(dbrpOrg2ID)}, + }, 0, nil + }}, + DBRPMappingsV2: []*influxdb.DBRPMapping{}, + }, + args: args{ + filter: influxdb.DBRPMappingFilter{ + RetentionPolicy: stringPtr("autogen"), + }, + }, + wants: wants{ + dbrpMappings: []*influxdb.DBRPMapping{ + { + ID: 100, + Database: "testdb", + RetentionPolicy: "autogen", + Default: false, + Virtual: true, + OrganizationID: MustIDBase16(dbrpOrg3ID), + BucketID: 100, + }, + { + ID: 300, + Database: "testdb3", + RetentionPolicy: "autogen", + Default: false, + Virtual: true, + OrganizationID: MustIDBase16(dbrpOrg2ID), + BucketID: 300, + }, + }, + }, + }, { name: "find by db", fields: DBRPMappingFields{ @@ -989,6 +1097,19 @@ func FindDBRPMappingByIDV2( { name: "find non existing dbrp", fields: DBRPMappingFields{ + BucketSvc: &mock.BucketService{ + FindBucketByIDFn: func(ctx context.Context, id platform.ID) (*influxdb.Bucket, error) { + if id == MustIDBase16(dbrpBucketAID) { + return &influxdb.Bucket{ID: id}, nil + } + return nil, &errors2.Error{ + Code: errors2.ENotFound, + Msg: "bucket not found", + } + }, + FindBucketsFn: func(ctx context.Context, bf influxdb.BucketFilter, fo ...influxdb.FindOptions) ([]*influxdb.Bucket, int, error) { + return []*influxdb.Bucket{}, 0, nil + }}, DBRPMappingsV2: []*influxdb.DBRPMapping{ { ID: 100, @@ -1008,9 +1129,90 @@ func FindDBRPMappingByIDV2( err: dbrp.ErrDBRPNotFound, }, }, + { + name: "find virtual dbrp with slash", + fields: DBRPMappingFields{ + BucketSvc: &mock.BucketService{ + FindBucketByIDFn: func(ctx context.Context, id platform.ID) (*influxdb.Bucket, error) { + if id == MustIDBase16(dbrpBucketAID) { + return &influxdb.Bucket{ID: id, Name: "testdb/testrp", OrgID: MustIDBase16(dbrpOrg3ID)}, nil + } + return nil, &errors2.Error{ + Code: errors2.ENotFound, + Msg: "bucket not found", + } + }, + FindBucketsFn: func(ctx context.Context, bf influxdb.BucketFilter, fo ...influxdb.FindOptions) ([]*influxdb.Bucket, int, error) { + return []*influxdb.Bucket{}, 0, nil + }}, + DBRPMappingsV2: []*influxdb.DBRPMapping{}, + }, + args: args{ + OrgID: MustIDBase16(dbrpOrg3ID), + ID: MustIDBase16(dbrpBucketAID), + }, + wants: wants{ + dbrpMapping: &influxdb.DBRPMapping{ + ID: MustIDBase16(dbrpBucketAID), + Database: "testdb", + RetentionPolicy: "testrp", + Default: false, + Virtual: true, + OrganizationID: MustIDBase16(dbrpOrg3ID), + BucketID: MustIDBase16(dbrpBucketAID), + }, + }, + }, + { + name: "find virtual dbrp", + fields: DBRPMappingFields{ + BucketSvc: &mock.BucketService{ + FindBucketByIDFn: func(ctx context.Context, id platform.ID) (*influxdb.Bucket, error) { + if id == MustIDBase16(dbrpBucketAID) { + return &influxdb.Bucket{ID: id, Name: "testdb", OrgID: MustIDBase16(dbrpOrg3ID)}, nil + } + return nil, &errors2.Error{ + Code: errors2.ENotFound, + Msg: "bucket not found", + } + }, + FindBucketsFn: func(ctx context.Context, bf influxdb.BucketFilter, fo ...influxdb.FindOptions) ([]*influxdb.Bucket, int, error) { + return []*influxdb.Bucket{}, 0, nil + }}, + DBRPMappingsV2: []*influxdb.DBRPMapping{}, + }, + args: args{ + OrgID: MustIDBase16(dbrpOrg3ID), + ID: MustIDBase16(dbrpBucketAID), + }, + wants: wants{ + dbrpMapping: &influxdb.DBRPMapping{ + ID: MustIDBase16(dbrpBucketAID), + Database: "testdb", + RetentionPolicy: "autogen", + Default: false, + Virtual: true, + OrganizationID: MustIDBase16(dbrpOrg3ID), + BucketID: MustIDBase16(dbrpBucketAID), + }, + }, + }, { name: "find existing dbrp but wrong orgID", fields: DBRPMappingFields{ + BucketSvc: &mock.BucketService{ + FindBucketByIDFn: func(ctx context.Context, id platform.ID) (*influxdb.Bucket, error) { + if id == MustIDBase16(dbrpBucketAID) { + return &influxdb.Bucket{ID: id}, nil + } + return nil, &errors2.Error{ + Code: errors2.ENotFound, + Msg: "bucket not found", + } + }, + FindBucketsFn: func(ctx context.Context, bf influxdb.BucketFilter, fo ...influxdb.FindOptions) ([]*influxdb.Bucket, int, error) { + return []*influxdb.Bucket{}, 0, nil + }}, DBRPMappingsV2: []*influxdb.DBRPMapping{ { ID: 100, @@ -1146,6 +1348,19 @@ func UpdateDBRPMappingV2( { name: "error dbrp not found", fields: DBRPMappingFields{ + BucketSvc: &mock.BucketService{ + FindBucketByIDFn: func(ctx context.Context, id platform.ID) (*influxdb.Bucket, error) { + if id == MustIDBase16(dbrpBucket1ID) { + return &influxdb.Bucket{ID: id}, nil + } + return nil, &errors2.Error{ + Code: errors2.ENotFound, + Msg: "bucket not found", + } + }, + FindBucketsFn: func(ctx context.Context, bf influxdb.BucketFilter, fo ...influxdb.FindOptions) ([]*influxdb.Bucket, int, error) { + return []*influxdb.Bucket{}, 0, nil + }}, DBRPMappingsV2: []*influxdb.DBRPMapping{{ ID: 100, Database: "database1", @@ -1631,7 +1846,7 @@ func DeleteDBRPMappingV2( }, args: args{ OrgID: MustIDBase16(dbrpOrg2ID), - ID: 100, + ID: 150, }, wants: wants{ dbrpMappings: []*influxdb.DBRPMapping{