mirror of https://github.com/milvus-io/milvus.git
parent
779d677eb2
commit
168f346620
internal/querynodev2/segments
|
@ -25,7 +25,6 @@ package segments
|
||||||
import "C"
|
import "C"
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"path/filepath"
|
|
||||||
"unsafe"
|
"unsafe"
|
||||||
|
|
||||||
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
|
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
|
||||||
|
@ -57,7 +56,7 @@ func deleteLoadIndexInfo(info *LoadIndexInfo) {
|
||||||
C.DeleteLoadIndexInfo(info.cLoadIndexInfo)
|
C.DeleteLoadIndexInfo(info.cLoadIndexInfo)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (li *LoadIndexInfo) appendLoadIndexInfo(bytesIndex [][]byte, indexInfo *querypb.FieldIndexInfo, collectionID int64, partitionID int64, segmentID int64, fieldType schemapb.DataType) error {
|
func (li *LoadIndexInfo) appendLoadIndexInfo(indexInfo *querypb.FieldIndexInfo, collectionID int64, partitionID int64, segmentID int64, fieldType schemapb.DataType) error {
|
||||||
fieldID := indexInfo.FieldID
|
fieldID := indexInfo.FieldID
|
||||||
indexPaths := indexInfo.IndexFilePaths
|
indexPaths := indexInfo.IndexFilePaths
|
||||||
|
|
||||||
|
@ -87,7 +86,7 @@ func (li *LoadIndexInfo) appendLoadIndexInfo(bytesIndex [][]byte, indexInfo *que
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
err = li.appendIndexData(bytesIndex, indexPaths)
|
err = li.appendIndexData(indexPaths)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -135,7 +134,7 @@ func (li *LoadIndexInfo) appendFieldInfo(collectionID int64, partitionID int64,
|
||||||
}
|
}
|
||||||
|
|
||||||
// appendIndexData appends index path to cLoadIndexInfo and create index
|
// appendIndexData appends index path to cLoadIndexInfo and create index
|
||||||
func (li *LoadIndexInfo) appendIndexData(bytesIndex [][]byte, indexKeys []string) error {
|
func (li *LoadIndexInfo) appendIndexData(indexKeys []string) error {
|
||||||
for _, indexPath := range indexKeys {
|
for _, indexPath := range indexKeys {
|
||||||
err := li.appendIndexFile(indexPath)
|
err := li.appendIndexFile(indexPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -143,30 +142,6 @@ func (li *LoadIndexInfo) appendIndexData(bytesIndex [][]byte, indexKeys []string
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if bytesIndex != nil {
|
|
||||||
var cBinarySet C.CBinarySet
|
|
||||||
status := C.NewBinarySet(&cBinarySet)
|
|
||||||
defer C.DeleteBinarySet(cBinarySet)
|
|
||||||
if err := HandleCStatus(&status, "NewBinarySet failed"); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, byteIndex := range bytesIndex {
|
|
||||||
indexPtr := unsafe.Pointer(&byteIndex[0])
|
|
||||||
indexLen := C.int64_t(len(byteIndex))
|
|
||||||
binarySetKey := filepath.Base(indexKeys[i])
|
|
||||||
indexKey := C.CString(binarySetKey)
|
|
||||||
status = C.AppendIndexBinary(cBinarySet, indexPtr, indexLen, indexKey)
|
|
||||||
C.free(unsafe.Pointer(indexKey))
|
|
||||||
if err := HandleCStatus(&status, "LoadIndexInfo AppendIndexBinary failed"); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
status = C.AppendIndex(li.cLoadIndexInfo, cBinarySet)
|
|
||||||
return HandleCStatus(&status, "AppendIndex failed")
|
|
||||||
}
|
|
||||||
|
|
||||||
status := C.AppendIndexV2(li.cLoadIndexInfo)
|
status := C.AppendIndexV2(li.cLoadIndexInfo)
|
||||||
return HandleCStatus(&status, "AppendIndex failed")
|
return HandleCStatus(&status, "AppendIndex failed")
|
||||||
}
|
}
|
||||||
|
|
|
@ -831,37 +831,14 @@ func (s *LocalSegment) LoadDeltaData(deltaData *storage.DeleteData) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *LocalSegment) LoadIndex(bytesIndex [][]byte, indexInfo *querypb.FieldIndexInfo, fieldType schemapb.DataType) error {
|
func (s *LocalSegment) LoadIndex(indexInfo *querypb.FieldIndexInfo, fieldType schemapb.DataType) error {
|
||||||
loadIndexInfo, err := newLoadIndexInfo()
|
loadIndexInfo, err := newLoadIndexInfo()
|
||||||
defer deleteLoadIndexInfo(loadIndexInfo)
|
defer deleteLoadIndexInfo(loadIndexInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = loadIndexInfo.appendLoadIndexInfo(bytesIndex, indexInfo, s.collectionID, s.partitionID, s.segmentID, fieldType)
|
err = loadIndexInfo.appendLoadIndexInfo(indexInfo, s.collectionID, s.partitionID, s.segmentID, fieldType)
|
||||||
if err != nil {
|
|
||||||
if loadIndexInfo.cleanLocalData() != nil {
|
|
||||||
log.Warn("failed to clean cached data on disk after append index failed",
|
|
||||||
zap.Int64("buildID", indexInfo.BuildID),
|
|
||||||
zap.Int64("index version", indexInfo.IndexVersion))
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if s.Type() != SegmentTypeSealed {
|
|
||||||
errMsg := fmt.Sprintln("updateSegmentIndex failed, illegal segment type ", s.typ, "segmentID = ", s.ID())
|
|
||||||
return errors.New(errMsg)
|
|
||||||
}
|
|
||||||
return s.LoadIndexInfo(indexInfo, loadIndexInfo)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *LocalSegment) LoadIndexData(indexInfo *querypb.FieldIndexInfo, fieldType schemapb.DataType) error {
|
|
||||||
loadIndexInfo, err := newLoadIndexInfo()
|
|
||||||
defer deleteLoadIndexInfo(loadIndexInfo)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = loadIndexInfo.appendLoadIndexInfo(nil, indexInfo, s.collectionID, s.partitionID, s.segmentID, fieldType)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if loadIndexInfo.cleanLocalData() != nil {
|
if loadIndexInfo.cleanLocalData() != nil {
|
||||||
log.Warn("failed to clean cached data on disk after append index failed",
|
log.Warn("failed to clean cached data on disk after append index failed",
|
||||||
|
|
|
@ -607,7 +607,7 @@ func (loader *segmentLoader) loadFieldIndex(ctx context.Context, segment *LocalS
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
return segment.LoadIndexData(indexInfo, fieldType)
|
return segment.LoadIndex(indexInfo, fieldType)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (loader *segmentLoader) insertIntoSegment(segment *LocalSegment,
|
func (loader *segmentLoader) insertIntoSegment(segment *LocalSegment,
|
||||||
|
|
Loading…
Reference in New Issue