milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: Load l0 delta for growings when using `RemoteLoad` (#37771) 

Related to #37574

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
pull/37794/head
congqixia 2024-11-19 10:38:31 +08:00 committed by GitHub
parent 113c72c02e
commit c79fbd5eab
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 30 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
internal/querynodev2/delegator/delegator_data.go
View File

@ -359,16 +359,7 @@ func (sd *shardDelegator) LoadGrowing(ctx context.Context, infos []*querypb.Segm
} }
for _, segment := range loaded { for _, segment := range loaded {
log := log.With( err = sd.addL0ForGrowing(ctx, segment)
zap.Int64("segmentID", segment.ID()),
)
deletedPks, deletedTss := sd.GetLevel0Deletions(segment.Partition(), pkoracle.NewCandidateKey(segment.ID(), segment.Partition(), segments.SegmentTypeGrowing))
if len(deletedPks) == 0 {
continue
}
log.Info("forwarding L0 delete records...", zap.Int("deletionCount", len(deletedPks)))
err = segment.Delete(ctx, deletedPks, deletedTss)
if err != nil { if err != nil {
log.Warn("failed to forward L0 deletions to growing segment", log.Warn("failed to forward L0 deletions to growing segment",
zap.Error(err), zap.Error(err),

29
internal/querynodev2/delegator/delta_forward.go
View File

@ -85,6 +85,35 @@ func (sd *shardDelegator) forwardStreamingDeletion(ctx context.Context, deleteDa
} }
} }
func (sd *shardDelegator) addL0ForGrowing(ctx context.Context, segment segments.Segment) error {
switch policy := paramtable.Get().QueryNodeCfg.StreamingDeltaForwardPolicy.GetValue(); policy {
case ForwardPolicyDefault, StreamingForwardPolicyBF:
return sd.addL0GrowingBF(ctx, segment)
case StreamingForwardPolicyDirect:
// forward streaming deletion without bf filtering
return sd.addL0ForGrowingLoad(ctx, segment)
default:
log.Fatal("unsupported streaming forward policy", zap.String("policy", policy))
}
return nil
}
func (sd *shardDelegator) addL0GrowingBF(ctx context.Context, segment segments.Segment) error {
deletedPks, deletedTss := sd.GetLevel0Deletions(segment.Partition(), pkoracle.NewCandidateKey(segment.ID(), segment.Partition(), segments.SegmentTypeGrowing))
if len(deletedPks) == 0 {
return nil
}
log.Info("forwarding L0 delete records...", zap.Int64("segmentID", segment.ID()), zap.Int("deletionCount", len(deletedPks)))
return segment.Delete(ctx, deletedPks, deletedTss)
}
func (sd *shardDelegator) addL0ForGrowingLoad(ctx context.Context, segment segments.Segment) error {
deltalogs := sd.getLevel0Deltalogs(segment.Partition())
log.Info("forwarding L0 via loader...", zap.Int64("segmentID", segment.ID()), zap.Int("deltalogsNum", len(deltalogs)))
return sd.loader.LoadDeltaLogs(ctx, segment, deltalogs)
}
func (sd *shardDelegator) forwardL0ByBF(ctx context.Context, func (sd *shardDelegator) forwardL0ByBF(ctx context.Context,
info *querypb.SegmentLoadInfo, info *querypb.SegmentLoadInfo,
candidate *pkoracle.BloomFilterSet, candidate *pkoracle.BloomFilterSet,