diff --git a/internal/core/src/segcore/SegmentSealedImpl.cpp b/internal/core/src/segcore/SegmentSealedImpl.cpp index 627e146998..cda939b582 100644 --- a/internal/core/src/segcore/SegmentSealedImpl.cpp +++ b/internal/core/src/segcore/SegmentSealedImpl.cpp @@ -1129,7 +1129,7 @@ SegmentSealedImpl::ClearData() { index_ready_bitset_.reset(); binlog_index_bitset_.reset(); system_ready_count_ = 0; - num_rows_ = 0; + num_rows_ = std::nullopt; scalar_indexings_.clear(); vector_indexings_.clear(); insert_record_.clear(); diff --git a/internal/querynodev2/segments/mock_segment.go b/internal/querynodev2/segments/mock_segment.go index 878ccca5ed..0670236a0e 100644 --- a/internal/querynodev2/segments/mock_segment.go +++ b/internal/querynodev2/segments/mock_segment.go @@ -993,6 +993,39 @@ func (_c *MockSegment_Release_Call) RunAndReturn(run func(...releaseOption)) *Mo return _c } +// ResetIndexesLazyLoad provides a mock function with given fields: lazyState +func (_m *MockSegment) ResetIndexesLazyLoad(lazyState bool) { + _m.Called(lazyState) +} + +// MockSegment_ResetIndexesLazyLoad_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ResetIndexesLazyLoad' +type MockSegment_ResetIndexesLazyLoad_Call struct { + *mock.Call +} + +// ResetIndexesLazyLoad is a helper method to define mock.On call +// - lazyState bool +func (_e *MockSegment_Expecter) ResetIndexesLazyLoad(lazyState interface{}) *MockSegment_ResetIndexesLazyLoad_Call { + return &MockSegment_ResetIndexesLazyLoad_Call{Call: _e.mock.On("ResetIndexesLazyLoad", lazyState)} +} + +func (_c *MockSegment_ResetIndexesLazyLoad_Call) Run(run func(lazyState bool)) *MockSegment_ResetIndexesLazyLoad_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(bool)) + }) + return _c +} + +func (_c *MockSegment_ResetIndexesLazyLoad_Call) Return() *MockSegment_ResetIndexesLazyLoad_Call { + _c.Call.Return() + return _c +} + +func (_c *MockSegment_ResetIndexesLazyLoad_Call) RunAndReturn(run func(bool)) *MockSegment_ResetIndexesLazyLoad_Call { + _c.Call.Return(run) + return _c +} + // ResourceGroup provides a mock function with given fields: func (_m *MockSegment) ResourceGroup() string { ret := _m.Called() diff --git a/internal/querynodev2/segments/segment.go b/internal/querynodev2/segments/segment.go index 6d5cd008ea..04a2aabf90 100644 --- a/internal/querynodev2/segments/segment.go +++ b/internal/querynodev2/segments/segment.go @@ -463,6 +463,12 @@ func (s *LocalSegment) Indexes() []*IndexedFieldInfo { return result } +func (s *LocalSegment) ResetIndexesLazyLoad(lazyState bool) { + for _, indexInfo := range s.Indexes() { + indexInfo.LazyLoad = lazyState + } +} + func (s *LocalSegment) Search(ctx context.Context, searchReq *SearchRequest) (*SearchResult, error) { /* CStatus @@ -1129,6 +1135,14 @@ func (s *LocalSegment) LoadIndex(ctx context.Context, indexInfo *querypb.FieldIn opt(options) } + log := log.Ctx(ctx).With( + zap.Int64("collectionID", s.Collection()), + zap.Int64("partitionID", s.Partition()), + zap.Int64("segmentID", s.ID()), + zap.Int64("fieldID", indexInfo.GetFieldID()), + zap.Int64("indexID", indexInfo.GetIndexID()), + ) + if options.LoadStatus == LoadStatusMeta { s.addIndex(indexInfo.GetFieldID(), &IndexedFieldInfo{ FieldBinlog: &datapb.FieldBinlog{ @@ -1149,14 +1163,6 @@ func (s *LocalSegment) LoadIndex(ctx context.Context, indexInfo *querypb.FieldIn ctx, sp := otel.Tracer(typeutil.QueryNodeRole).Start(ctx, fmt.Sprintf("LoadIndex-%d-%d", s.ID(), indexInfo.GetFieldID())) defer sp.End() - log := log.Ctx(ctx).With( - zap.Int64("collectionID", s.Collection()), - zap.Int64("partitionID", s.Partition()), - zap.Int64("segmentID", s.ID()), - zap.Int64("fieldID", indexInfo.GetFieldID()), - zap.Int64("indexID", indexInfo.GetIndexID()), - ) - loadIndexInfo, err := newLoadIndexInfo(ctx) if err != nil { return err @@ -1340,11 +1346,20 @@ func (s *LocalSegment) Release(opts ...releaseOption) { // release will never fail defer stateLockGuard.Done(nil) + log := log.With(zap.Int64("collectionID", s.Collection()), + zap.Int64("partitionID", s.Partition()), + zap.Int64("segmentID", s.ID()), + zap.String("segmentType", s.segmentType.String()), + zap.Int64("insertCount", s.InsertCount()), + ) + // wait all read ops finished ptr := s.ptr if options.Scope == ReleaseScopeData { s.loadStatus.Store(string(LoadStatusMeta)) C.ClearSegmentData(ptr) + s.ResetIndexesLazyLoad(true) + log.Debug("release segment data done and the field indexes info has been set lazy load=true") return } @@ -1365,13 +1380,7 @@ func (s *LocalSegment) Release(opts ...releaseOption) { metrics.QueryNodeDiskUsedSize.WithLabelValues(fmt.Sprint(paramtable.GetNodeID())).Set(float64(localDiskUsage) / 1024 / 1024) // in MB } - log.Info("delete segment from memory", - zap.Int64("collectionID", s.Collection()), - zap.Int64("partitionID", s.Partition()), - zap.Int64("segmentID", s.ID()), - zap.String("segmentType", s.segmentType.String()), - zap.Int64("insertCount", s.InsertCount()), - ) + log.Info("delete segment from memory") } // StartLoadData starts the loading process of the segment. diff --git a/internal/querynodev2/segments/segment_interface.go b/internal/querynodev2/segments/segment_interface.go index e36612a2a1..fa4f3695f6 100644 --- a/internal/querynodev2/segments/segment_interface.go +++ b/internal/querynodev2/segments/segment_interface.go @@ -97,4 +97,5 @@ type Segment interface { Search(ctx context.Context, searchReq *SearchRequest) (*SearchResult, error) Retrieve(ctx context.Context, plan *RetrievePlan) (*segcorepb.RetrieveResults, error) IsLazyLoad() bool + ResetIndexesLazyLoad(lazyState bool) } diff --git a/internal/querynodev2/segments/segment_l0.go b/internal/querynodev2/segments/segment_l0.go index 582c98a08d..3a004b5db9 100644 --- a/internal/querynodev2/segments/segment_l0.go +++ b/internal/querynodev2/segments/segment_l0.go @@ -117,6 +117,9 @@ func (s *L0Segment) Indexes() []*IndexedFieldInfo { return nil } +func (s *L0Segment) ResetIndexesLazyLoad(lazyState bool) { +} + func (s *L0Segment) Type() SegmentType { return s.segmentType } diff --git a/internal/querynodev2/segments/segment_loader.go b/internal/querynodev2/segments/segment_loader.go index f95035605e..2e1a303a7d 100644 --- a/internal/querynodev2/segments/segment_loader.go +++ b/internal/querynodev2/segments/segment_loader.go @@ -978,7 +978,7 @@ func (loader *segmentLoader) loadSealedSegment(ctx context.Context, loadInfo *qu if err := segment.AddFieldDataInfo(ctx, loadInfo.GetNumOfRows(), loadInfo.GetBinlogPaths()); err != nil { return err } - + log := log.Ctx(ctx).With(zap.Int64("segmentID", segment.ID())) tr := timerecord.NewTimeRecorder("segmentLoader.LoadIndex") log.Info("load fields...", zap.Int64s("indexedFields", lo.Keys(indexedFieldInfos)),