diff --git a/meta/data.go b/meta/data.go index d7ef4b0947..8c05964a76 100644 --- a/meta/data.go +++ b/meta/data.go @@ -240,7 +240,14 @@ func (data *Data) ShardGroups(database, policy string) ([]ShardGroupInfo, error) } else if rpi == nil { return nil, ErrRetentionPolicyNotFound } - return rpi.ShardGroups, nil + groups := make([]ShardGroupInfo, 0, len(rpi.ShardGroups)) + for _, g := range rpi.ShardGroups { + if g.Deleted() { + continue + } + groups = append(groups, g) + } + return groups, nil } // ShardGroupByTimestamp returns the shard group on a database and policy for a given timestamp. @@ -331,10 +338,10 @@ func (data *Data) DeleteShardGroup(database, policy string, id uint64) error { return ErrRetentionPolicyNotFound } - // Find shard group by ID and remove it. + // Find shard group by ID and set its deletion timestamp. for i := range rpi.ShardGroups { if rpi.ShardGroups[i].ID == id { - rpi.ShardGroups = append(rpi.ShardGroups[:i], rpi.ShardGroups[i+1:]...) + rpi.ShardGroups[i].DeletedAt = time.Now().UTC() return nil } } @@ -568,7 +575,7 @@ func NewRetentionPolicyInfo(name string) *RetentionPolicyInfo { // ShardGroupByTimestamp returns the shard group in the policy that contains the timestamp. func (rpi *RetentionPolicyInfo) ShardGroupByTimestamp(timestamp time.Time) *ShardGroupInfo { for i := range rpi.ShardGroups { - if rpi.ShardGroups[i].Contains(timestamp) { + if rpi.ShardGroups[i].Contains(timestamp) && !rpi.ShardGroups[i].Deleted() { return &rpi.ShardGroups[i] } } @@ -609,12 +616,16 @@ func shardGroupDuration(d time.Duration) time.Duration { return 1 * time.Hour } -// ShardGroupInfo represents metadata about a shard group. +// ShardGroupInfo represents metadata about a shard group. The DeletedAt field is important +// because it makes it clear that a ShardGroup has been marked as deleted, and allow the system +// to be sure that a ShardGroup is not simply missing. If the DeletedAt is set, the system can +// safely delete any associated shards. type ShardGroupInfo struct { ID uint64 StartTime time.Time EndTime time.Time Shards []ShardInfo + DeletedAt time.Time } // Contains return true if the shard group contains data for the timestamp. @@ -627,6 +638,11 @@ func (sgi *ShardGroupInfo) Overlaps(min, max time.Time) bool { return !sgi.StartTime.After(max) && sgi.EndTime.After(min) } +// Deleted returns whether this ShardGroup has been deleted. +func (sgi *ShardGroupInfo) Deleted() bool { + return !sgi.DeletedAt.IsZero() +} + // clone returns a deep copy of sgi. func (sgi ShardGroupInfo) clone() ShardGroupInfo { other := sgi diff --git a/meta/data_test.go b/meta/data_test.go index a15cdd5d0d..4998b0a32c 100644 --- a/meta/data_test.go +++ b/meta/data_test.go @@ -365,8 +365,9 @@ func TestData_DeleteShardGroup(t *testing.T) { if err := data.DeleteShardGroup("db0", "rp0", 1); err != nil { t.Fatal(err) - } else if len(data.Databases[0].RetentionPolicies[0].ShardGroups) != 0 { - t.Fatalf("unexpected shard groups: %#v", data.Databases[0].RetentionPolicies[0].ShardGroups) + } + if sg := data.Databases[0].RetentionPolicies[0].ShardGroups[0]; !sg.Deleted() { + t.Fatalf("shard group not correctly flagged as deleted") } } diff --git a/meta/store.go b/meta/store.go index d67d2148cf..ff9ce5fa08 100644 --- a/meta/store.go +++ b/meta/store.go @@ -525,7 +525,7 @@ func (s *Store) CreateShardGroupIfNotExists(database, policy string, timestamp t // Try to find shard group locally first. if sgi, err := s.ShardGroupByTimestamp(database, policy, timestamp); err != nil { return nil, err - } else if sgi != nil { + } else if sgi != nil && !sgi.Deleted() { return sgi, nil } diff --git a/services/retention/service.go b/services/retention/service.go index 56777d09a9..883c606980 100644 --- a/services/retention/service.go +++ b/services/retention/service.go @@ -107,16 +107,17 @@ func (s *Service) deleteShards() { case <-ticker.C: s.logger.Println("retention policy shard deletion commencing") - // Build a list of all shard IDs that exist. - shardIDs := make(map[uint64]struct{}, 0) + deletedShardIDs := make(map[uint64]struct{}, 0) s.MetaStore.VisitShardGroups(func(d meta.DatabaseInfo, r meta.RetentionPolicyInfo, g meta.ShardGroupInfo) { - for _, sh := range g.Shards { - shardIDs[sh.ID] = struct{}{} + if g.Deleted() { + for _, sh := range g.Shards { + deletedShardIDs[sh.ID] = struct{}{} + } } }) for _, id := range s.TSDBStore.ShardIDs() { - if _, ok := shardIDs[id]; !ok { + if _, ok := deletedShardIDs[id]; ok { if err := s.TSDBStore.DeleteShard(id); err != nil { s.logger.Printf("failed to delete shard ID %d: %s", id, err.Error()) continue