mirror of https://github.com/milvus-io/milvus.git
remove binlog/delta log from getRecoveryInfoV2 (#27895)
Signed-off-by: Wei Liu <wei.liu@zilliz.com>pull/27274/head
wei liu
GitHub
parent
36ddeae2cc
commit
7e0bda8b4c
|
|
@ -32,7 +32,6 @@ import (
|
|||
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
|
||||
"github.com/milvus-io/milvus/internal/metastore/kv/datacoord"
|
||||
"github.com/milvus-io/milvus/internal/proto/datapb"
|
||||
"github.com/milvus-io/milvus/internal/proto/internalpb"
|
||||
"github.com/milvus-io/milvus/internal/util/segmentutil"
|
||||
|
|
@ -818,37 +817,12 @@ func (s *Server) GetRecoveryInfoV2(ctx context.Context, req *datapb.GetRecoveryI
|
|||
rowCount = segment.NumOfRows
|
||||
}
|
||||
|
||||
// save the traffic of sending
|
||||
binLogs, err := datacoord.CompressBinLog(segment.Binlogs)
|
||||
if err != nil {
|
||||
log.Warn("failed to compress segment", zap.Int64("segmentID", id), zap.Error(err))
|
||||
resp.Status = merr.Status(err)
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
deltaLogs, err := datacoord.CompressBinLog(segment.Deltalogs)
|
||||
if err != nil {
|
||||
log.Warn("failed to compress segment", zap.Int64("segmentID", id), zap.Error(err))
|
||||
resp.Status = merr.Status(err)
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
statLogs, err := datacoord.CompressBinLog(segment.Statslogs)
|
||||
if err != nil {
|
||||
log.Warn("failed to compress segment", zap.Int64("segmentID", id), zap.Error(err))
|
||||
resp.Status = merr.Status(err)
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
segmentInfos = append(segmentInfos, &datapb.SegmentInfo{
|
||||
ID: segment.ID,
|
||||
PartitionID: segment.PartitionID,
|
||||
CollectionID: segment.CollectionID,
|
||||
InsertChannel: segment.InsertChannel,
|
||||
NumOfRows: rowCount,
|
||||
Binlogs: binLogs,
|
||||
Statslogs: statLogs,
|
||||
Deltalogs: deltaLogs,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -420,20 +420,7 @@ func TestGetRecoveryInfoV2(t *testing.T) {
|
|||
assert.EqualValues(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
|
||||
assert.EqualValues(t, 1, len(resp.GetSegments()))
|
||||
assert.EqualValues(t, 0, resp.GetSegments()[0].GetID())
|
||||
assert.EqualValues(t, 1, len(resp.GetSegments()[0].GetBinlogs()))
|
||||
assert.EqualValues(t, 1, resp.GetSegments()[0].GetBinlogs()[0].GetFieldID())
|
||||
for _, binlog := range resp.GetSegments()[0].GetBinlogs()[0].GetBinlogs() {
|
||||
assert.Equal(t, "", binlog.GetLogPath())
|
||||
assert.Equal(t, int64(801), binlog.GetLogID())
|
||||
}
|
||||
for _, binlog := range resp.GetSegments()[0].GetStatslogs()[0].GetBinlogs() {
|
||||
assert.Equal(t, "", binlog.GetLogPath())
|
||||
assert.Equal(t, int64(10000), binlog.GetLogID())
|
||||
}
|
||||
for _, binlog := range resp.GetSegments()[0].GetDeltalogs()[0].GetBinlogs() {
|
||||
assert.Equal(t, "", binlog.GetLogPath())
|
||||
assert.Equal(t, int64(100000), binlog.GetLogID())
|
||||
}
|
||||
assert.EqualValues(t, 0, len(resp.GetSegments()[0].GetBinlogs()))
|
||||
})
|
||||
t.Run("with dropped segments", func(t *testing.T) {
|
||||
svr := newTestServer(t, nil)
|
||||
|
|
@ -524,197 +511,6 @@ func TestGetRecoveryInfoV2(t *testing.T) {
|
|||
assert.NotEqual(t, 0, resp.GetChannels()[0].GetSeekPosition().GetTimestamp())
|
||||
})
|
||||
|
||||
t.Run("with failed compress", func(t *testing.T) {
|
||||
svr := newTestServer(t, nil)
|
||||
defer closeTestServer(t, svr)
|
||||
|
||||
svr.rootCoordClientCreator = func(ctx context.Context) (types.RootCoordClient, error) {
|
||||
return newMockRootCoordClient(), nil
|
||||
}
|
||||
|
||||
svr.meta.AddCollection(&collectionInfo{
|
||||
ID: 0,
|
||||
Schema: newTestSchema(),
|
||||
})
|
||||
|
||||
err := svr.meta.UpdateChannelCheckpoint("vchan1", &msgpb.MsgPosition{
|
||||
ChannelName: "vchan1",
|
||||
Timestamp: 0,
|
||||
MsgID: []byte{0, 0, 0, 0, 0, 0, 0, 0},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
svr.meta.AddCollection(&collectionInfo{
|
||||
ID: 1,
|
||||
Schema: newTestSchema(),
|
||||
})
|
||||
|
||||
err = svr.meta.UpdateChannelCheckpoint("vchan2", &msgpb.MsgPosition{
|
||||
ChannelName: "vchan2",
|
||||
Timestamp: 0,
|
||||
MsgID: []byte{0, 0, 0, 0, 0, 0, 0, 0},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
svr.meta.AddCollection(&collectionInfo{
|
||||
ID: 2,
|
||||
Schema: newTestSchema(),
|
||||
})
|
||||
|
||||
err = svr.meta.UpdateChannelCheckpoint("vchan3", &msgpb.MsgPosition{
|
||||
ChannelName: "vchan3",
|
||||
Timestamp: 0,
|
||||
MsgID: []byte{0, 0, 0, 0, 0, 0, 0, 0},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
svr.channelManager.AddNode(0)
|
||||
ch := &channel{
|
||||
Name: "vchan1",
|
||||
CollectionID: 0,
|
||||
}
|
||||
err = svr.channelManager.Watch(context.TODO(), ch)
|
||||
assert.NoError(t, err)
|
||||
|
||||
ch = &channel{
|
||||
Name: "vchan2",
|
||||
CollectionID: 1,
|
||||
}
|
||||
err = svr.channelManager.Watch(context.TODO(), ch)
|
||||
assert.NoError(t, err)
|
||||
|
||||
ch = &channel{
|
||||
Name: "vchan3",
|
||||
CollectionID: 2,
|
||||
}
|
||||
err = svr.channelManager.Watch(context.TODO(), ch)
|
||||
assert.NoError(t, err)
|
||||
|
||||
seg := createSegment(8, 0, 0, 100, 40, "vchan1", commonpb.SegmentState_Flushed)
|
||||
binLogPaths := make([]*datapb.Binlog, 1)
|
||||
// miss one field
|
||||
path := metautil.JoinIDPath(0, 0, 8, fieldID)
|
||||
path = path + "/mock"
|
||||
binLogPaths[0] = &datapb.Binlog{
|
||||
EntriesNum: 10000,
|
||||
LogPath: path,
|
||||
}
|
||||
|
||||
seg.Statslogs = append(seg.Statslogs, &datapb.FieldBinlog{
|
||||
FieldID: fieldID,
|
||||
Binlogs: binLogPaths,
|
||||
})
|
||||
|
||||
binLogPaths2 := make([]*datapb.Binlog, 1)
|
||||
pathCorrect := metautil.JoinIDPath(0, 0, 8, fieldID, 1)
|
||||
binLogPaths2[0] = &datapb.Binlog{
|
||||
EntriesNum: 10000,
|
||||
LogPath: pathCorrect,
|
||||
}
|
||||
|
||||
seg.Binlogs = append(seg.Binlogs, &datapb.FieldBinlog{
|
||||
FieldID: fieldID,
|
||||
Binlogs: binLogPaths2,
|
||||
})
|
||||
err = svr.meta.AddSegment(context.TODO(), NewSegmentInfo(seg))
|
||||
assert.NoError(t, err)
|
||||
|
||||
// make sure collection is indexed
|
||||
err = svr.meta.CreateIndex(&model.Index{
|
||||
TenantID: "",
|
||||
CollectionID: 0,
|
||||
FieldID: 2,
|
||||
IndexID: 0,
|
||||
IndexName: "_default_idx_1",
|
||||
IsDeleted: false,
|
||||
CreateTime: 0,
|
||||
TypeParams: nil,
|
||||
IndexParams: nil,
|
||||
IsAutoIndex: false,
|
||||
UserIndexParams: nil,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
svr.meta.segments.SetSegmentIndex(seg.ID, &model.SegmentIndex{
|
||||
SegmentID: seg.ID,
|
||||
CollectionID: 0,
|
||||
PartitionID: 0,
|
||||
NumRows: 100,
|
||||
IndexID: 0,
|
||||
BuildID: 0,
|
||||
NodeID: 0,
|
||||
IndexVersion: 1,
|
||||
IndexState: commonpb.IndexState_Finished,
|
||||
FailReason: "",
|
||||
IsDeleted: false,
|
||||
CreateTime: 0,
|
||||
IndexFileKeys: nil,
|
||||
IndexSize: 0,
|
||||
})
|
||||
|
||||
req := &datapb.GetRecoveryInfoRequestV2{
|
||||
CollectionID: 0,
|
||||
}
|
||||
resp, err := svr.GetRecoveryInfoV2(context.TODO(), req)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, resp.Status.ErrorCode == commonpb.ErrorCode_UnexpectedError)
|
||||
|
||||
// test bin log
|
||||
path = metautil.JoinIDPath(0, 0, 9, fieldID)
|
||||
path = path + "/mock"
|
||||
binLogPaths[0] = &datapb.Binlog{
|
||||
EntriesNum: 10000,
|
||||
LogPath: path,
|
||||
}
|
||||
|
||||
seg2 := createSegment(9, 1, 0, 100, 40, "vchan2", commonpb.SegmentState_Flushed)
|
||||
seg2.Binlogs = append(seg2.Binlogs, &datapb.FieldBinlog{
|
||||
FieldID: fieldID,
|
||||
Binlogs: binLogPaths,
|
||||
})
|
||||
err = svr.meta.AddSegment(context.TODO(), NewSegmentInfo(seg2))
|
||||
assert.NoError(t, err)
|
||||
|
||||
// make sure collection is indexed
|
||||
err = svr.meta.CreateIndex(&model.Index{
|
||||
TenantID: "",
|
||||
CollectionID: 1,
|
||||
FieldID: 2,
|
||||
IndexID: 1,
|
||||
IndexName: "_default_idx_2",
|
||||
IsDeleted: false,
|
||||
CreateTime: 0,
|
||||
TypeParams: nil,
|
||||
IndexParams: nil,
|
||||
IsAutoIndex: false,
|
||||
UserIndexParams: nil,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
svr.meta.segments.SetSegmentIndex(seg2.ID, &model.SegmentIndex{
|
||||
SegmentID: seg2.ID,
|
||||
CollectionID: 1,
|
||||
PartitionID: 0,
|
||||
NumRows: 100,
|
||||
IndexID: 1,
|
||||
BuildID: 0,
|
||||
NodeID: 0,
|
||||
IndexVersion: 1,
|
||||
IndexState: commonpb.IndexState_Finished,
|
||||
FailReason: "",
|
||||
IsDeleted: false,
|
||||
CreateTime: 0,
|
||||
IndexFileKeys: nil,
|
||||
IndexSize: 0,
|
||||
})
|
||||
req = &datapb.GetRecoveryInfoRequestV2{
|
||||
CollectionID: 1,
|
||||
}
|
||||
resp, err = svr.GetRecoveryInfoV2(context.TODO(), req)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, resp.Status.ErrorCode == commonpb.ErrorCode_UnexpectedError)
|
||||
})
|
||||
|
||||
t.Run("with continuous compaction", func(t *testing.T) {
|
||||
svr := newTestServer(t, nil)
|
||||
defer closeTestServer(t, svr)
|
||||
|
|
|
|||
Loading…
Reference in New Issue