milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: add message and msgstream msgpack adaptor (#34874) 

issue: #33285

- make message builder and message conversion type safe
- add adaptor type and function to adapt old msgstream msgpack and
message interface

---------

Signed-off-by: chyezh <chyezh@outlook.com>
pull/34909/head
chyezh 2024-07-22 20:59:42 +08:00 committed by GitHub
parent 6c19f9baf8
commit 39c7e06bc5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
42 changed files with 1759 additions and 662 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -103,4 +103,5 @@ cwrapper_rocksdb_build/
internal/proto/**/*.pb.go
internal/core/src/pb/*.pb.h
internal/core/src/pb/*.pb.cc
*.pb.go
**/legacypb/*.pb.go

44
internal/flushcommon/writebuffer/mock_manager.go
View File

@ -46,11 +46,11 @@ type MockBufferManager_BufferData_Call struct {
}
// BufferData is a helper method to define mock.On call
// - channel string
// - insertMsgs []*msgstream.InsertMsg
// - deleteMsgs []*msgstream.DeleteMsg
// - startPos *msgpb.MsgPosition
// - endPos *msgpb.MsgPosition
// - channel string
// - insertMsgs []*msgstream.InsertMsg
// - deleteMsgs []*msgstream.DeleteMsg
// - startPos *msgpb.MsgPosition
// - endPos *msgpb.MsgPosition
func (_e *MockBufferManager_Expecter) BufferData(channel interface{}, insertMsgs interface{}, deleteMsgs interface{}, startPos interface{}, endPos interface{}) *MockBufferManager_BufferData_Call {
return &MockBufferManager_BufferData_Call{Call: _e.mock.On("BufferData", channel, insertMsgs, deleteMsgs, startPos, endPos)}
}
@ -83,7 +83,7 @@ type MockBufferManager_DropChannel_Call struct {
}
// DropChannel is a helper method to define mock.On call
// - channel string
// - channel string
func (_e *MockBufferManager_Expecter) DropChannel(channel interface{}) *MockBufferManager_DropChannel_Call {
return &MockBufferManager_DropChannel_Call{Call: _e.mock.On("DropChannel", channel)}
}
@ -116,8 +116,8 @@ type MockBufferManager_DropPartitions_Call struct {
}
// DropPartitions is a helper method to define mock.On call
// - channel string
// - partitionIDs []int64
// - channel string
// - partitionIDs []int64
func (_e *MockBufferManager_Expecter) DropPartitions(channel interface{}, partitionIDs interface{}) *MockBufferManager_DropPartitions_Call {
return &MockBufferManager_DropPartitions_Call{Call: _e.mock.On("DropPartitions", channel, partitionIDs)}
}
@ -159,9 +159,9 @@ type MockBufferManager_FlushChannel_Call struct {
}
// FlushChannel is a helper method to define mock.On call
// - ctx context.Context
// - channel string
// - flushTs uint64
// - ctx context.Context
// - channel string
// - flushTs uint64
func (_e *MockBufferManager_Expecter) FlushChannel(ctx interface{}, channel interface{}, flushTs interface{}) *MockBufferManager_FlushChannel_Call {
return &MockBufferManager_FlushChannel_Call{Call: _e.mock.On("FlushChannel", ctx, channel, flushTs)}
}
@ -222,7 +222,7 @@ type MockBufferManager_GetCheckpoint_Call struct {
}
// GetCheckpoint is a helper method to define mock.On call
// - channel string
// - channel string
func (_e *MockBufferManager_Expecter) GetCheckpoint(channel interface{}) *MockBufferManager_GetCheckpoint_Call {
return &MockBufferManager_GetCheckpoint_Call{Call: _e.mock.On("GetCheckpoint", channel)}
}
@ -255,8 +255,8 @@ type MockBufferManager_NotifyCheckpointUpdated_Call struct {
}
// NotifyCheckpointUpdated is a helper method to define mock.On call
// - channel string
// - ts uint64
// - channel string
// - ts uint64
func (_e *MockBufferManager_Expecter) NotifyCheckpointUpdated(channel interface{}, ts interface{}) *MockBufferManager_NotifyCheckpointUpdated_Call {
return &MockBufferManager_NotifyCheckpointUpdated_Call{Call: _e.mock.On("NotifyCheckpointUpdated", channel, ts)}
}
@ -305,10 +305,10 @@ type MockBufferManager_Register_Call struct {
}
// Register is a helper method to define mock.On call
// - channel string
// - _a1 metacache.MetaCache
// - storageV2Cache *metacache.StorageV2Cache
// - opts ...WriteBufferOption
// - channel string
// - _a1 metacache.MetaCache
// - storageV2Cache *metacache.StorageV2Cache
// - opts ...WriteBufferOption
func (_e *MockBufferManager_Expecter) Register(channel interface{}, _a1 interface{}, storageV2Cache interface{}, opts ...interface{}) *MockBufferManager_Register_Call {
return &MockBufferManager_Register_Call{Call: _e.mock.On("Register",
append([]interface{}{channel, _a1, storageV2Cache}, opts...)...)}
@ -348,7 +348,7 @@ type MockBufferManager_RemoveChannel_Call struct {
}
// RemoveChannel is a helper method to define mock.On call
// - channel string
// - channel string
func (_e *MockBufferManager_Expecter) RemoveChannel(channel interface{}) *MockBufferManager_RemoveChannel_Call {
return &MockBufferManager_RemoveChannel_Call{Call: _e.mock.On("RemoveChannel", channel)}
}
@ -390,9 +390,9 @@ type MockBufferManager_SealSegments_Call struct {
}
// SealSegments is a helper method to define mock.On call
// - ctx context.Context
// - channel string
// - segmentIDs []int64
// - ctx context.Context
// - channel string
// - segmentIDs []int64
func (_e *MockBufferManager_Expecter) SealSegments(ctx interface{}, channel interface{}, segmentIDs interface{}) *MockBufferManager_SealSegments_Call {
return &MockBufferManager_SealSegments_Call{Call: _e.mock.On("SealSegments", ctx, channel, segmentIDs)}
}

206
internal/mocks/mock_rootcoord.go
View File

@ -68,8 +68,8 @@ type RootCoord_AllocID_Call struct {
}
// AllocID is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *rootcoordpb.AllocIDRequest
// - _a0 context.Context
// - _a1 *rootcoordpb.AllocIDRequest
func (_e *RootCoord_Expecter) AllocID(_a0 interface{}, _a1 interface{}) *RootCoord_AllocID_Call {
return &RootCoord_AllocID_Call{Call: _e.mock.On("AllocID", _a0, _a1)}
}
@ -123,8 +123,8 @@ type RootCoord_AllocTimestamp_Call struct {
}
// AllocTimestamp is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *rootcoordpb.AllocTimestampRequest
// - _a0 context.Context
// - _a1 *rootcoordpb.AllocTimestampRequest
func (_e *RootCoord_Expecter) AllocTimestamp(_a0 interface{}, _a1 interface{}) *RootCoord_AllocTimestamp_Call {
return &RootCoord_AllocTimestamp_Call{Call: _e.mock.On("AllocTimestamp", _a0, _a1)}
}
@ -178,8 +178,8 @@ type RootCoord_AlterAlias_Call struct {
}
// AlterAlias is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.AlterAliasRequest
// - _a0 context.Context
// - _a1 *milvuspb.AlterAliasRequest
func (_e *RootCoord_Expecter) AlterAlias(_a0 interface{}, _a1 interface{}) *RootCoord_AlterAlias_Call {
return &RootCoord_AlterAlias_Call{Call: _e.mock.On("AlterAlias", _a0, _a1)}
}
@ -233,8 +233,8 @@ type RootCoord_AlterCollection_Call struct {
}
// AlterCollection is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.AlterCollectionRequest
// - _a0 context.Context
// - _a1 *milvuspb.AlterCollectionRequest
func (_e *RootCoord_Expecter) AlterCollection(_a0 interface{}, _a1 interface{}) *RootCoord_AlterCollection_Call {
return &RootCoord_AlterCollection_Call{Call: _e.mock.On("AlterCollection", _a0, _a1)}
}
@ -288,8 +288,8 @@ type RootCoord_AlterDatabase_Call struct {
}
// AlterDatabase is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *rootcoordpb.AlterDatabaseRequest
// - _a0 context.Context
// - _a1 *rootcoordpb.AlterDatabaseRequest
func (_e *RootCoord_Expecter) AlterDatabase(_a0 interface{}, _a1 interface{}) *RootCoord_AlterDatabase_Call {
return &RootCoord_AlterDatabase_Call{Call: _e.mock.On("AlterDatabase", _a0, _a1)}
}
@ -343,8 +343,8 @@ type RootCoord_CheckHealth_Call struct {
}
// CheckHealth is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.CheckHealthRequest
// - _a0 context.Context
// - _a1 *milvuspb.CheckHealthRequest
func (_e *RootCoord_Expecter) CheckHealth(_a0 interface{}, _a1 interface{}) *RootCoord_CheckHealth_Call {
return &RootCoord_CheckHealth_Call{Call: _e.mock.On("CheckHealth", _a0, _a1)}
}
@ -398,8 +398,8 @@ type RootCoord_CreateAlias_Call struct {
}
// CreateAlias is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.CreateAliasRequest
// - _a0 context.Context
// - _a1 *milvuspb.CreateAliasRequest
func (_e *RootCoord_Expecter) CreateAlias(_a0 interface{}, _a1 interface{}) *RootCoord_CreateAlias_Call {
return &RootCoord_CreateAlias_Call{Call: _e.mock.On("CreateAlias", _a0, _a1)}
}
@ -453,8 +453,8 @@ type RootCoord_CreateCollection_Call struct {
}
// CreateCollection is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.CreateCollectionRequest
// - _a0 context.Context
// - _a1 *milvuspb.CreateCollectionRequest
func (_e *RootCoord_Expecter) CreateCollection(_a0 interface{}, _a1 interface{}) *RootCoord_CreateCollection_Call {
return &RootCoord_CreateCollection_Call{Call: _e.mock.On("CreateCollection", _a0, _a1)}
}
@ -508,8 +508,8 @@ type RootCoord_CreateCredential_Call struct {
}
// CreateCredential is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *internalpb.CredentialInfo
// - _a0 context.Context
// - _a1 *internalpb.CredentialInfo
func (_e *RootCoord_Expecter) CreateCredential(_a0 interface{}, _a1 interface{}) *RootCoord_CreateCredential_Call {
return &RootCoord_CreateCredential_Call{Call: _e.mock.On("CreateCredential", _a0, _a1)}
}
@ -563,8 +563,8 @@ type RootCoord_CreateDatabase_Call struct {
}
// CreateDatabase is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.CreateDatabaseRequest
// - _a0 context.Context
// - _a1 *milvuspb.CreateDatabaseRequest
func (_e *RootCoord_Expecter) CreateDatabase(_a0 interface{}, _a1 interface{}) *RootCoord_CreateDatabase_Call {
return &RootCoord_CreateDatabase_Call{Call: _e.mock.On("CreateDatabase", _a0, _a1)}
}
@ -618,8 +618,8 @@ type RootCoord_CreatePartition_Call struct {
}
// CreatePartition is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.CreatePartitionRequest
// - _a0 context.Context
// - _a1 *milvuspb.CreatePartitionRequest
func (_e *RootCoord_Expecter) CreatePartition(_a0 interface{}, _a1 interface{}) *RootCoord_CreatePartition_Call {
return &RootCoord_CreatePartition_Call{Call: _e.mock.On("CreatePartition", _a0, _a1)}
}
@ -673,8 +673,8 @@ type RootCoord_CreateRole_Call struct {
}
// CreateRole is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.CreateRoleRequest
// - _a0 context.Context
// - _a1 *milvuspb.CreateRoleRequest
func (_e *RootCoord_Expecter) CreateRole(_a0 interface{}, _a1 interface{}) *RootCoord_CreateRole_Call {
return &RootCoord_CreateRole_Call{Call: _e.mock.On("CreateRole", _a0, _a1)}
}
@ -728,8 +728,8 @@ type RootCoord_DeleteCredential_Call struct {
}
// DeleteCredential is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.DeleteCredentialRequest
// - _a0 context.Context
// - _a1 *milvuspb.DeleteCredentialRequest
func (_e *RootCoord_Expecter) DeleteCredential(_a0 interface{}, _a1 interface{}) *RootCoord_DeleteCredential_Call {
return &RootCoord_DeleteCredential_Call{Call: _e.mock.On("DeleteCredential", _a0, _a1)}
}
@ -783,8 +783,8 @@ type RootCoord_DescribeAlias_Call struct {
}
// DescribeAlias is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.DescribeAliasRequest
// - _a0 context.Context
// - _a1 *milvuspb.DescribeAliasRequest
func (_e *RootCoord_Expecter) DescribeAlias(_a0 interface{}, _a1 interface{}) *RootCoord_DescribeAlias_Call {
return &RootCoord_DescribeAlias_Call{Call: _e.mock.On("DescribeAlias", _a0, _a1)}
}
@ -838,8 +838,8 @@ type RootCoord_DescribeCollection_Call struct {
}
// DescribeCollection is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.DescribeCollectionRequest
// - _a0 context.Context
// - _a1 *milvuspb.DescribeCollectionRequest
func (_e *RootCoord_Expecter) DescribeCollection(_a0 interface{}, _a1 interface{}) *RootCoord_DescribeCollection_Call {
return &RootCoord_DescribeCollection_Call{Call: _e.mock.On("DescribeCollection", _a0, _a1)}
}
@ -893,8 +893,8 @@ type RootCoord_DescribeCollectionInternal_Call struct {
}
// DescribeCollectionInternal is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.DescribeCollectionRequest
// - _a0 context.Context
// - _a1 *milvuspb.DescribeCollectionRequest
func (_e *RootCoord_Expecter) DescribeCollectionInternal(_a0 interface{}, _a1 interface{}) *RootCoord_DescribeCollectionInternal_Call {
return &RootCoord_DescribeCollectionInternal_Call{Call: _e.mock.On("DescribeCollectionInternal", _a0, _a1)}
}
@ -948,8 +948,8 @@ type RootCoord_DescribeDatabase_Call struct {
}
// DescribeDatabase is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *rootcoordpb.DescribeDatabaseRequest
// - _a0 context.Context
// - _a1 *rootcoordpb.DescribeDatabaseRequest
func (_e *RootCoord_Expecter) DescribeDatabase(_a0 interface{}, _a1 interface{}) *RootCoord_DescribeDatabase_Call {
return &RootCoord_DescribeDatabase_Call{Call: _e.mock.On("DescribeDatabase", _a0, _a1)}
}
@ -1003,8 +1003,8 @@ type RootCoord_DropAlias_Call struct {
}
// DropAlias is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.DropAliasRequest
// - _a0 context.Context
// - _a1 *milvuspb.DropAliasRequest
func (_e *RootCoord_Expecter) DropAlias(_a0 interface{}, _a1 interface{}) *RootCoord_DropAlias_Call {
return &RootCoord_DropAlias_Call{Call: _e.mock.On("DropAlias", _a0, _a1)}
}
@ -1058,8 +1058,8 @@ type RootCoord_DropCollection_Call struct {
}
// DropCollection is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.DropCollectionRequest
// - _a0 context.Context
// - _a1 *milvuspb.DropCollectionRequest
func (_e *RootCoord_Expecter) DropCollection(_a0 interface{}, _a1 interface{}) *RootCoord_DropCollection_Call {
return &RootCoord_DropCollection_Call{Call: _e.mock.On("DropCollection", _a0, _a1)}
}
@ -1113,8 +1113,8 @@ type RootCoord_DropDatabase_Call struct {
}
// DropDatabase is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.DropDatabaseRequest
// - _a0 context.Context
// - _a1 *milvuspb.DropDatabaseRequest
func (_e *RootCoord_Expecter) DropDatabase(_a0 interface{}, _a1 interface{}) *RootCoord_DropDatabase_Call {
return &RootCoord_DropDatabase_Call{Call: _e.mock.On("DropDatabase", _a0, _a1)}
}
@ -1168,8 +1168,8 @@ type RootCoord_DropPartition_Call struct {
}
// DropPartition is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.DropPartitionRequest
// - _a0 context.Context
// - _a1 *milvuspb.DropPartitionRequest
func (_e *RootCoord_Expecter) DropPartition(_a0 interface{}, _a1 interface{}) *RootCoord_DropPartition_Call {
return &RootCoord_DropPartition_Call{Call: _e.mock.On("DropPartition", _a0, _a1)}
}
@ -1223,8 +1223,8 @@ type RootCoord_DropRole_Call struct {
}
// DropRole is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.DropRoleRequest
// - _a0 context.Context
// - _a1 *milvuspb.DropRoleRequest
func (_e *RootCoord_Expecter) DropRole(_a0 interface{}, _a1 interface{}) *RootCoord_DropRole_Call {
return &RootCoord_DropRole_Call{Call: _e.mock.On("DropRole", _a0, _a1)}
}
@ -1278,8 +1278,8 @@ type RootCoord_GetComponentStates_Call struct {
}
// GetComponentStates is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.GetComponentStatesRequest
// - _a0 context.Context
// - _a1 *milvuspb.GetComponentStatesRequest
func (_e *RootCoord_Expecter) GetComponentStates(_a0 interface{}, _a1 interface{}) *RootCoord_GetComponentStates_Call {
return &RootCoord_GetComponentStates_Call{Call: _e.mock.On("GetComponentStates", _a0, _a1)}
}
@ -1333,8 +1333,8 @@ type RootCoord_GetCredential_Call struct {
}
// GetCredential is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *rootcoordpb.GetCredentialRequest
// - _a0 context.Context
// - _a1 *rootcoordpb.GetCredentialRequest
func (_e *RootCoord_Expecter) GetCredential(_a0 interface{}, _a1 interface{}) *RootCoord_GetCredential_Call {
return &RootCoord_GetCredential_Call{Call: _e.mock.On("GetCredential", _a0, _a1)}
}
@ -1388,8 +1388,8 @@ type RootCoord_GetMetrics_Call struct {
}
// GetMetrics is a helper method to define mock.On call
// - ctx context.Context
// - req *milvuspb.GetMetricsRequest
// - ctx context.Context
// - req *milvuspb.GetMetricsRequest
func (_e *RootCoord_Expecter) GetMetrics(ctx interface{}, req interface{}) *RootCoord_GetMetrics_Call {
return &RootCoord_GetMetrics_Call{Call: _e.mock.On("GetMetrics", ctx, req)}
}
@ -1443,8 +1443,8 @@ type RootCoord_GetStatisticsChannel_Call struct {
}
// GetStatisticsChannel is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *internalpb.GetStatisticsChannelRequest
// - _a0 context.Context
// - _a1 *internalpb.GetStatisticsChannelRequest
func (_e *RootCoord_Expecter) GetStatisticsChannel(_a0 interface{}, _a1 interface{}) *RootCoord_GetStatisticsChannel_Call {
return &RootCoord_GetStatisticsChannel_Call{Call: _e.mock.On("GetStatisticsChannel", _a0, _a1)}
}
@ -1498,8 +1498,8 @@ type RootCoord_GetTimeTickChannel_Call struct {
}
// GetTimeTickChannel is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *internalpb.GetTimeTickChannelRequest
// - _a0 context.Context
// - _a1 *internalpb.GetTimeTickChannelRequest
func (_e *RootCoord_Expecter) GetTimeTickChannel(_a0 interface{}, _a1 interface{}) *RootCoord_GetTimeTickChannel_Call {
return &RootCoord_GetTimeTickChannel_Call{Call: _e.mock.On("GetTimeTickChannel", _a0, _a1)}
}
@ -1553,8 +1553,8 @@ type RootCoord_GetVChannels_Call struct {
}
// GetVChannels is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *rootcoordpb.GetVChannelsRequest
// - _a0 context.Context
// - _a1 *rootcoordpb.GetVChannelsRequest
func (_e *RootCoord_Expecter) GetVChannels(_a0 interface{}, _a1 interface{}) *RootCoord_GetVChannels_Call {
return &RootCoord_GetVChannels_Call{Call: _e.mock.On("GetVChannels", _a0, _a1)}
}
@ -1608,8 +1608,8 @@ type RootCoord_HasCollection_Call struct {
}
// HasCollection is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.HasCollectionRequest
// - _a0 context.Context
// - _a1 *milvuspb.HasCollectionRequest
func (_e *RootCoord_Expecter) HasCollection(_a0 interface{}, _a1 interface{}) *RootCoord_HasCollection_Call {
return &RootCoord_HasCollection_Call{Call: _e.mock.On("HasCollection", _a0, _a1)}
}
@ -1663,8 +1663,8 @@ type RootCoord_HasPartition_Call struct {
}
// HasPartition is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.HasPartitionRequest
// - _a0 context.Context
// - _a1 *milvuspb.HasPartitionRequest
func (_e *RootCoord_Expecter) HasPartition(_a0 interface{}, _a1 interface{}) *RootCoord_HasPartition_Call {
return &RootCoord_HasPartition_Call{Call: _e.mock.On("HasPartition", _a0, _a1)}
}
@ -1759,8 +1759,8 @@ type RootCoord_InvalidateCollectionMetaCache_Call struct {
}
// InvalidateCollectionMetaCache is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *proxypb.InvalidateCollMetaCacheRequest
// - _a0 context.Context
// - _a1 *proxypb.InvalidateCollMetaCacheRequest
func (_e *RootCoord_Expecter) InvalidateCollectionMetaCache(_a0 interface{}, _a1 interface{}) *RootCoord_InvalidateCollectionMetaCache_Call {
return &RootCoord_InvalidateCollectionMetaCache_Call{Call: _e.mock.On("InvalidateCollectionMetaCache", _a0, _a1)}
}
@ -1814,8 +1814,8 @@ type RootCoord_ListAliases_Call struct {
}
// ListAliases is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.ListAliasesRequest
// - _a0 context.Context
// - _a1 *milvuspb.ListAliasesRequest
func (_e *RootCoord_Expecter) ListAliases(_a0 interface{}, _a1 interface{}) *RootCoord_ListAliases_Call {
return &RootCoord_ListAliases_Call{Call: _e.mock.On("ListAliases", _a0, _a1)}
}
@ -1869,8 +1869,8 @@ type RootCoord_ListCredUsers_Call struct {
}
// ListCredUsers is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.ListCredUsersRequest
// - _a0 context.Context
// - _a1 *milvuspb.ListCredUsersRequest
func (_e *RootCoord_Expecter) ListCredUsers(_a0 interface{}, _a1 interface{}) *RootCoord_ListCredUsers_Call {
return &RootCoord_ListCredUsers_Call{Call: _e.mock.On("ListCredUsers", _a0, _a1)}
}
@ -1924,8 +1924,8 @@ type RootCoord_ListDatabases_Call struct {
}
// ListDatabases is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.ListDatabasesRequest
// - _a0 context.Context
// - _a1 *milvuspb.ListDatabasesRequest
func (_e *RootCoord_Expecter) ListDatabases(_a0 interface{}, _a1 interface{}) *RootCoord_ListDatabases_Call {
return &RootCoord_ListDatabases_Call{Call: _e.mock.On("ListDatabases", _a0, _a1)}
}
@ -1979,8 +1979,8 @@ type RootCoord_ListPolicy_Call struct {
}
// ListPolicy is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *internalpb.ListPolicyRequest
// - _a0 context.Context
// - _a1 *internalpb.ListPolicyRequest
func (_e *RootCoord_Expecter) ListPolicy(_a0 interface{}, _a1 interface{}) *RootCoord_ListPolicy_Call {
return &RootCoord_ListPolicy_Call{Call: _e.mock.On("ListPolicy", _a0, _a1)}
}
@ -2034,8 +2034,8 @@ type RootCoord_OperatePrivilege_Call struct {
}
// OperatePrivilege is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.OperatePrivilegeRequest
// - _a0 context.Context
// - _a1 *milvuspb.OperatePrivilegeRequest
func (_e *RootCoord_Expecter) OperatePrivilege(_a0 interface{}, _a1 interface{}) *RootCoord_OperatePrivilege_Call {
return &RootCoord_OperatePrivilege_Call{Call: _e.mock.On("OperatePrivilege", _a0, _a1)}
}
@ -2089,8 +2089,8 @@ type RootCoord_OperateUserRole_Call struct {
}
// OperateUserRole is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.OperateUserRoleRequest
// - _a0 context.Context
// - _a1 *milvuspb.OperateUserRoleRequest
func (_e *RootCoord_Expecter) OperateUserRole(_a0 interface{}, _a1 interface{}) *RootCoord_OperateUserRole_Call {
return &RootCoord_OperateUserRole_Call{Call: _e.mock.On("OperateUserRole", _a0, _a1)}
}
@ -2185,8 +2185,8 @@ type RootCoord_RenameCollection_Call struct {
}
// RenameCollection is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.RenameCollectionRequest
// - _a0 context.Context
// - _a1 *milvuspb.RenameCollectionRequest
func (_e *RootCoord_Expecter) RenameCollection(_a0 interface{}, _a1 interface{}) *RootCoord_RenameCollection_Call {
return &RootCoord_RenameCollection_Call{Call: _e.mock.On("RenameCollection", _a0, _a1)}
}
@ -2240,8 +2240,8 @@ type RootCoord_SelectGrant_Call struct {
}
// SelectGrant is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.SelectGrantRequest
// - _a0 context.Context
// - _a1 *milvuspb.SelectGrantRequest
func (_e *RootCoord_Expecter) SelectGrant(_a0 interface{}, _a1 interface{}) *RootCoord_SelectGrant_Call {
return &RootCoord_SelectGrant_Call{Call: _e.mock.On("SelectGrant", _a0, _a1)}
}
@ -2295,8 +2295,8 @@ type RootCoord_SelectRole_Call struct {
}
// SelectRole is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.SelectRoleRequest
// - _a0 context.Context
// - _a1 *milvuspb.SelectRoleRequest
func (_e *RootCoord_Expecter) SelectRole(_a0 interface{}, _a1 interface{}) *RootCoord_SelectRole_Call {
return &RootCoord_SelectRole_Call{Call: _e.mock.On("SelectRole", _a0, _a1)}
}
@ -2350,8 +2350,8 @@ type RootCoord_SelectUser_Call struct {
}
// SelectUser is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.SelectUserRequest
// - _a0 context.Context
// - _a1 *milvuspb.SelectUserRequest
func (_e *RootCoord_Expecter) SelectUser(_a0 interface{}, _a1 interface{}) *RootCoord_SelectUser_Call {
return &RootCoord_SelectUser_Call{Call: _e.mock.On("SelectUser", _a0, _a1)}
}
@ -2384,7 +2384,7 @@ type RootCoord_SetAddress_Call struct {
}
// SetAddress is a helper method to define mock.On call
// - address string
// - address string
func (_e *RootCoord_Expecter) SetAddress(address interface{}) *RootCoord_SetAddress_Call {
return &RootCoord_SetAddress_Call{Call: _e.mock.On("SetAddress", address)}
}
@ -2426,7 +2426,7 @@ type RootCoord_SetDataCoordClient_Call struct {
}
// SetDataCoordClient is a helper method to define mock.On call
// - dataCoord types.DataCoordClient
// - dataCoord types.DataCoordClient
func (_e *RootCoord_Expecter) SetDataCoordClient(dataCoord interface{}) *RootCoord_SetDataCoordClient_Call {
return &RootCoord_SetDataCoordClient_Call{Call: _e.mock.On("SetDataCoordClient", dataCoord)}
}
@ -2459,7 +2459,7 @@ type RootCoord_SetEtcdClient_Call struct {
}
// SetEtcdClient is a helper method to define mock.On call
// - etcdClient *clientv3.Client
// - etcdClient *clientv3.Client
func (_e *RootCoord_Expecter) SetEtcdClient(etcdClient interface{}) *RootCoord_SetEtcdClient_Call {
return &RootCoord_SetEtcdClient_Call{Call: _e.mock.On("SetEtcdClient", etcdClient)}
}
@ -2492,7 +2492,7 @@ type RootCoord_SetProxyCreator_Call struct {
}
// SetProxyCreator is a helper method to define mock.On call
// - _a0 func(context.Context , string , int64)(types.ProxyClient , error)
// - _a0 func(context.Context , string , int64)(types.ProxyClient , error)
func (_e *RootCoord_Expecter) SetProxyCreator(_a0 interface{}) *RootCoord_SetProxyCreator_Call {
return &RootCoord_SetProxyCreator_Call{Call: _e.mock.On("SetProxyCreator", _a0)}
}
@ -2534,7 +2534,7 @@ type RootCoord_SetQueryCoordClient_Call struct {
}
// SetQueryCoordClient is a helper method to define mock.On call
// - queryCoord types.QueryCoordClient
// - queryCoord types.QueryCoordClient
func (_e *RootCoord_Expecter) SetQueryCoordClient(queryCoord interface{}) *RootCoord_SetQueryCoordClient_Call {
return &RootCoord_SetQueryCoordClient_Call{Call: _e.mock.On("SetQueryCoordClient", queryCoord)}
}
@ -2567,7 +2567,7 @@ type RootCoord_SetTiKVClient_Call struct {
}
// SetTiKVClient is a helper method to define mock.On call
// - client *txnkv.Client
// - client *txnkv.Client
func (_e *RootCoord_Expecter) SetTiKVClient(client interface{}) *RootCoord_SetTiKVClient_Call {
return &RootCoord_SetTiKVClient_Call{Call: _e.mock.On("SetTiKVClient", client)}
}
@ -2621,8 +2621,8 @@ type RootCoord_ShowCollections_Call struct {
}
// ShowCollections is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.ShowCollectionsRequest
// - _a0 context.Context
// - _a1 *milvuspb.ShowCollectionsRequest
func (_e *RootCoord_Expecter) ShowCollections(_a0 interface{}, _a1 interface{}) *RootCoord_ShowCollections_Call {
return &RootCoord_ShowCollections_Call{Call: _e.mock.On("ShowCollections", _a0, _a1)}
}
@ -2676,8 +2676,8 @@ type RootCoord_ShowConfigurations_Call struct {
}
// ShowConfigurations is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *internalpb.ShowConfigurationsRequest
// - _a0 context.Context
// - _a1 *internalpb.ShowConfigurationsRequest
func (_e *RootCoord_Expecter) ShowConfigurations(_a0 interface{}, _a1 interface{}) *RootCoord_ShowConfigurations_Call {
return &RootCoord_ShowConfigurations_Call{Call: _e.mock.On("ShowConfigurations", _a0, _a1)}
}
@ -2731,8 +2731,8 @@ type RootCoord_ShowPartitions_Call struct {
}
// ShowPartitions is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.ShowPartitionsRequest
// - _a0 context.Context
// - _a1 *milvuspb.ShowPartitionsRequest
func (_e *RootCoord_Expecter) ShowPartitions(_a0 interface{}, _a1 interface{}) *RootCoord_ShowPartitions_Call {
return &RootCoord_ShowPartitions_Call{Call: _e.mock.On("ShowPartitions", _a0, _a1)}
}
@ -2786,8 +2786,8 @@ type RootCoord_ShowPartitionsInternal_Call struct {
}
// ShowPartitionsInternal is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.ShowPartitionsRequest
// - _a0 context.Context
// - _a1 *milvuspb.ShowPartitionsRequest
func (_e *RootCoord_Expecter) ShowPartitionsInternal(_a0 interface{}, _a1 interface{}) *RootCoord_ShowPartitionsInternal_Call {
return &RootCoord_ShowPartitionsInternal_Call{Call: _e.mock.On("ShowPartitionsInternal", _a0, _a1)}
}
@ -2841,8 +2841,8 @@ type RootCoord_ShowSegments_Call struct {
}
// ShowSegments is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *milvuspb.ShowSegmentsRequest
// - _a0 context.Context
// - _a1 *milvuspb.ShowSegmentsRequest
func (_e *RootCoord_Expecter) ShowSegments(_a0 interface{}, _a1 interface{}) *RootCoord_ShowSegments_Call {
return &RootCoord_ShowSegments_Call{Call: _e.mock.On("ShowSegments", _a0, _a1)}
}
@ -2978,8 +2978,8 @@ type RootCoord_UpdateChannelTimeTick_Call struct {
}
// UpdateChannelTimeTick is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *internalpb.ChannelTimeTickMsg
// - _a0 context.Context
// - _a1 *internalpb.ChannelTimeTickMsg
func (_e *RootCoord_Expecter) UpdateChannelTimeTick(_a0 interface{}, _a1 interface{}) *RootCoord_UpdateChannelTimeTick_Call {
return &RootCoord_UpdateChannelTimeTick_Call{Call: _e.mock.On("UpdateChannelTimeTick", _a0, _a1)}
}
@ -3033,8 +3033,8 @@ type RootCoord_UpdateCredential_Call struct {
}
// UpdateCredential is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *internalpb.CredentialInfo
// - _a0 context.Context
// - _a1 *internalpb.CredentialInfo
func (_e *RootCoord_Expecter) UpdateCredential(_a0 interface{}, _a1 interface{}) *RootCoord_UpdateCredential_Call {
return &RootCoord_UpdateCredential_Call{Call: _e.mock.On("UpdateCredential", _a0, _a1)}
}
@ -3067,7 +3067,7 @@ type RootCoord_UpdateStateCode_Call struct {
}
// UpdateStateCode is a helper method to define mock.On call
// - _a0 commonpb.StateCode
// - _a0 commonpb.StateCode
func (_e *RootCoord_Expecter) UpdateStateCode(_a0 interface{}) *RootCoord_UpdateStateCode_Call {
return &RootCoord_UpdateStateCode_Call{Call: _e.mock.On("UpdateStateCode", _a0)}
}

288
internal/mocks/mock_rootcoord_client.go
View File

@ -72,9 +72,9 @@ type MockRootCoordClient_AllocID_Call struct {
}
// AllocID is a helper method to define mock.On call
// - ctx context.Context
// - in *rootcoordpb.AllocIDRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *rootcoordpb.AllocIDRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) AllocID(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_AllocID_Call {
return &MockRootCoordClient_AllocID_Call{Call: _e.mock.On("AllocID",
append([]interface{}{ctx, in}, opts...)...)}
@ -142,9 +142,9 @@ type MockRootCoordClient_AllocTimestamp_Call struct {
}
// AllocTimestamp is a helper method to define mock.On call
// - ctx context.Context
// - in *rootcoordpb.AllocTimestampRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *rootcoordpb.AllocTimestampRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) AllocTimestamp(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_AllocTimestamp_Call {
return &MockRootCoordClient_AllocTimestamp_Call{Call: _e.mock.On("AllocTimestamp",
append([]interface{}{ctx, in}, opts...)...)}
@ -212,9 +212,9 @@ type MockRootCoordClient_AlterAlias_Call struct {
}
// AlterAlias is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.AlterAliasRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.AlterAliasRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) AlterAlias(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_AlterAlias_Call {
return &MockRootCoordClient_AlterAlias_Call{Call: _e.mock.On("AlterAlias",
append([]interface{}{ctx, in}, opts...)...)}
@ -282,9 +282,9 @@ type MockRootCoordClient_AlterCollection_Call struct {
}
// AlterCollection is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.AlterCollectionRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.AlterCollectionRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) AlterCollection(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_AlterCollection_Call {
return &MockRootCoordClient_AlterCollection_Call{Call: _e.mock.On("AlterCollection",
append([]interface{}{ctx, in}, opts...)...)}
@ -352,9 +352,9 @@ type MockRootCoordClient_AlterDatabase_Call struct {
}
// AlterDatabase is a helper method to define mock.On call
// - ctx context.Context
// - in *rootcoordpb.AlterDatabaseRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *rootcoordpb.AlterDatabaseRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) AlterDatabase(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_AlterDatabase_Call {
return &MockRootCoordClient_AlterDatabase_Call{Call: _e.mock.On("AlterDatabase",
append([]interface{}{ctx, in}, opts...)...)}
@ -422,9 +422,9 @@ type MockRootCoordClient_CheckHealth_Call struct {
}
// CheckHealth is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.CheckHealthRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.CheckHealthRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) CheckHealth(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_CheckHealth_Call {
return &MockRootCoordClient_CheckHealth_Call{Call: _e.mock.On("CheckHealth",
append([]interface{}{ctx, in}, opts...)...)}
@ -533,9 +533,9 @@ type MockRootCoordClient_CreateAlias_Call struct {
}
// CreateAlias is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.CreateAliasRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.CreateAliasRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) CreateAlias(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_CreateAlias_Call {
return &MockRootCoordClient_CreateAlias_Call{Call: _e.mock.On("CreateAlias",
append([]interface{}{ctx, in}, opts...)...)}
@ -603,9 +603,9 @@ type MockRootCoordClient_CreateCollection_Call struct {
}
// CreateCollection is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.CreateCollectionRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.CreateCollectionRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) CreateCollection(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_CreateCollection_Call {
return &MockRootCoordClient_CreateCollection_Call{Call: _e.mock.On("CreateCollection",
append([]interface{}{ctx, in}, opts...)...)}
@ -673,9 +673,9 @@ type MockRootCoordClient_CreateCredential_Call struct {
}
// CreateCredential is a helper method to define mock.On call
// - ctx context.Context
// - in *internalpb.CredentialInfo
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *internalpb.CredentialInfo
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) CreateCredential(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_CreateCredential_Call {
return &MockRootCoordClient_CreateCredential_Call{Call: _e.mock.On("CreateCredential",
append([]interface{}{ctx, in}, opts...)...)}
@ -743,9 +743,9 @@ type MockRootCoordClient_CreateDatabase_Call struct {
}
// CreateDatabase is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.CreateDatabaseRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.CreateDatabaseRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) CreateDatabase(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_CreateDatabase_Call {
return &MockRootCoordClient_CreateDatabase_Call{Call: _e.mock.On("CreateDatabase",
append([]interface{}{ctx, in}, opts...)...)}
@ -813,9 +813,9 @@ type MockRootCoordClient_CreatePartition_Call struct {
}
// CreatePartition is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.CreatePartitionRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.CreatePartitionRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) CreatePartition(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_CreatePartition_Call {
return &MockRootCoordClient_CreatePartition_Call{Call: _e.mock.On("CreatePartition",
append([]interface{}{ctx, in}, opts...)...)}
@ -883,9 +883,9 @@ type MockRootCoordClient_CreateRole_Call struct {
}
// CreateRole is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.CreateRoleRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.CreateRoleRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) CreateRole(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_CreateRole_Call {
return &MockRootCoordClient_CreateRole_Call{Call: _e.mock.On("CreateRole",
append([]interface{}{ctx, in}, opts...)...)}
@ -953,9 +953,9 @@ type MockRootCoordClient_DeleteCredential_Call struct {
}
// DeleteCredential is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.DeleteCredentialRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.DeleteCredentialRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) DeleteCredential(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DeleteCredential_Call {
return &MockRootCoordClient_DeleteCredential_Call{Call: _e.mock.On("DeleteCredential",
append([]interface{}{ctx, in}, opts...)...)}
@ -1023,9 +1023,9 @@ type MockRootCoordClient_DescribeAlias_Call struct {
}
// DescribeAlias is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.DescribeAliasRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.DescribeAliasRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) DescribeAlias(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DescribeAlias_Call {
return &MockRootCoordClient_DescribeAlias_Call{Call: _e.mock.On("DescribeAlias",
append([]interface{}{ctx, in}, opts...)...)}
@ -1093,9 +1093,9 @@ type MockRootCoordClient_DescribeCollection_Call struct {
}
// DescribeCollection is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.DescribeCollectionRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.DescribeCollectionRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) DescribeCollection(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DescribeCollection_Call {
return &MockRootCoordClient_DescribeCollection_Call{Call: _e.mock.On("DescribeCollection",
append([]interface{}{ctx, in}, opts...)...)}
@ -1163,9 +1163,9 @@ type MockRootCoordClient_DescribeCollectionInternal_Call struct {
}
// DescribeCollectionInternal is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.DescribeCollectionRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.DescribeCollectionRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) DescribeCollectionInternal(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DescribeCollectionInternal_Call {
return &MockRootCoordClient_DescribeCollectionInternal_Call{Call: _e.mock.On("DescribeCollectionInternal",
append([]interface{}{ctx, in}, opts...)...)}
@ -1233,9 +1233,9 @@ type MockRootCoordClient_DescribeDatabase_Call struct {
}
// DescribeDatabase is a helper method to define mock.On call
// - ctx context.Context
// - in *rootcoordpb.DescribeDatabaseRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *rootcoordpb.DescribeDatabaseRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) DescribeDatabase(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DescribeDatabase_Call {
return &MockRootCoordClient_DescribeDatabase_Call{Call: _e.mock.On("DescribeDatabase",
append([]interface{}{ctx, in}, opts...)...)}
@ -1303,9 +1303,9 @@ type MockRootCoordClient_DropAlias_Call struct {
}
// DropAlias is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.DropAliasRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.DropAliasRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) DropAlias(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DropAlias_Call {
return &MockRootCoordClient_DropAlias_Call{Call: _e.mock.On("DropAlias",
append([]interface{}{ctx, in}, opts...)...)}
@ -1373,9 +1373,9 @@ type MockRootCoordClient_DropCollection_Call struct {
}
// DropCollection is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.DropCollectionRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.DropCollectionRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) DropCollection(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DropCollection_Call {
return &MockRootCoordClient_DropCollection_Call{Call: _e.mock.On("DropCollection",
append([]interface{}{ctx, in}, opts...)...)}
@ -1443,9 +1443,9 @@ type MockRootCoordClient_DropDatabase_Call struct {
}
// DropDatabase is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.DropDatabaseRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.DropDatabaseRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) DropDatabase(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DropDatabase_Call {
return &MockRootCoordClient_DropDatabase_Call{Call: _e.mock.On("DropDatabase",
append([]interface{}{ctx, in}, opts...)...)}
@ -1513,9 +1513,9 @@ type MockRootCoordClient_DropPartition_Call struct {
}
// DropPartition is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.DropPartitionRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.DropPartitionRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) DropPartition(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DropPartition_Call {
return &MockRootCoordClient_DropPartition_Call{Call: _e.mock.On("DropPartition",
append([]interface{}{ctx, in}, opts...)...)}
@ -1583,9 +1583,9 @@ type MockRootCoordClient_DropRole_Call struct {
}
// DropRole is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.DropRoleRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.DropRoleRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) DropRole(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DropRole_Call {
return &MockRootCoordClient_DropRole_Call{Call: _e.mock.On("DropRole",
append([]interface{}{ctx, in}, opts...)...)}
@ -1653,9 +1653,9 @@ type MockRootCoordClient_GetComponentStates_Call struct {
}
// GetComponentStates is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.GetComponentStatesRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.GetComponentStatesRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) GetComponentStates(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_GetComponentStates_Call {
return &MockRootCoordClient_GetComponentStates_Call{Call: _e.mock.On("GetComponentStates",
append([]interface{}{ctx, in}, opts...)...)}
@ -1723,9 +1723,9 @@ type MockRootCoordClient_GetCredential_Call struct {
}
// GetCredential is a helper method to define mock.On call
// - ctx context.Context
// - in *rootcoordpb.GetCredentialRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *rootcoordpb.GetCredentialRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) GetCredential(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_GetCredential_Call {
return &MockRootCoordClient_GetCredential_Call{Call: _e.mock.On("GetCredential",
append([]interface{}{ctx, in}, opts...)...)}
@ -1793,9 +1793,9 @@ type MockRootCoordClient_GetMetrics_Call struct {
}
// GetMetrics is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.GetMetricsRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.GetMetricsRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) GetMetrics(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_GetMetrics_Call {
return &MockRootCoordClient_GetMetrics_Call{Call: _e.mock.On("GetMetrics",
append([]interface{}{ctx, in}, opts...)...)}
@ -1863,9 +1863,9 @@ type MockRootCoordClient_GetStatisticsChannel_Call struct {
}
// GetStatisticsChannel is a helper method to define mock.On call
// - ctx context.Context
// - in *internalpb.GetStatisticsChannelRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *internalpb.GetStatisticsChannelRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) GetStatisticsChannel(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_GetStatisticsChannel_Call {
return &MockRootCoordClient_GetStatisticsChannel_Call{Call: _e.mock.On("GetStatisticsChannel",
append([]interface{}{ctx, in}, opts...)...)}
@ -1933,9 +1933,9 @@ type MockRootCoordClient_GetTimeTickChannel_Call struct {
}
// GetTimeTickChannel is a helper method to define mock.On call
// - ctx context.Context
// - in *internalpb.GetTimeTickChannelRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *internalpb.GetTimeTickChannelRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) GetTimeTickChannel(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_GetTimeTickChannel_Call {
return &MockRootCoordClient_GetTimeTickChannel_Call{Call: _e.mock.On("GetTimeTickChannel",
append([]interface{}{ctx, in}, opts...)...)}
@ -2003,9 +2003,9 @@ type MockRootCoordClient_GetVChannels_Call struct {
}
// GetVChannels is a helper method to define mock.On call
// - ctx context.Context
// - in *rootcoordpb.GetVChannelsRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *rootcoordpb.GetVChannelsRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) GetVChannels(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_GetVChannels_Call {
return &MockRootCoordClient_GetVChannels_Call{Call: _e.mock.On("GetVChannels",
append([]interface{}{ctx, in}, opts...)...)}
@ -2073,9 +2073,9 @@ type MockRootCoordClient_HasCollection_Call struct {
}
// HasCollection is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.HasCollectionRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.HasCollectionRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) HasCollection(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_HasCollection_Call {
return &MockRootCoordClient_HasCollection_Call{Call: _e.mock.On("HasCollection",
append([]interface{}{ctx, in}, opts...)...)}
@ -2143,9 +2143,9 @@ type MockRootCoordClient_HasPartition_Call struct {
}
// HasPartition is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.HasPartitionRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.HasPartitionRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) HasPartition(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_HasPartition_Call {
return &MockRootCoordClient_HasPartition_Call{Call: _e.mock.On("HasPartition",
append([]interface{}{ctx, in}, opts...)...)}
@ -2213,9 +2213,9 @@ type MockRootCoordClient_InvalidateCollectionMetaCache_Call struct {
}
// InvalidateCollectionMetaCache is a helper method to define mock.On call
// - ctx context.Context
// - in *proxypb.InvalidateCollMetaCacheRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *proxypb.InvalidateCollMetaCacheRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) InvalidateCollectionMetaCache(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_InvalidateCollectionMetaCache_Call {
return &MockRootCoordClient_InvalidateCollectionMetaCache_Call{Call: _e.mock.On("InvalidateCollectionMetaCache",
append([]interface{}{ctx, in}, opts...)...)}
@ -2283,9 +2283,9 @@ type MockRootCoordClient_ListAliases_Call struct {
}
// ListAliases is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.ListAliasesRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.ListAliasesRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) ListAliases(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_ListAliases_Call {
return &MockRootCoordClient_ListAliases_Call{Call: _e.mock.On("ListAliases",
append([]interface{}{ctx, in}, opts...)...)}
@ -2353,9 +2353,9 @@ type MockRootCoordClient_ListCredUsers_Call struct {
}
// ListCredUsers is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.ListCredUsersRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.ListCredUsersRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) ListCredUsers(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_ListCredUsers_Call {
return &MockRootCoordClient_ListCredUsers_Call{Call: _e.mock.On("ListCredUsers",
append([]interface{}{ctx, in}, opts...)...)}
@ -2423,9 +2423,9 @@ type MockRootCoordClient_ListDatabases_Call struct {
}
// ListDatabases is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.ListDatabasesRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.ListDatabasesRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) ListDatabases(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_ListDatabases_Call {
return &MockRootCoordClient_ListDatabases_Call{Call: _e.mock.On("ListDatabases",
append([]interface{}{ctx, in}, opts...)...)}
@ -2493,9 +2493,9 @@ type MockRootCoordClient_ListPolicy_Call struct {
}
// ListPolicy is a helper method to define mock.On call
// - ctx context.Context
// - in *internalpb.ListPolicyRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *internalpb.ListPolicyRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) ListPolicy(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_ListPolicy_Call {
return &MockRootCoordClient_ListPolicy_Call{Call: _e.mock.On("ListPolicy",
append([]interface{}{ctx, in}, opts...)...)}
@ -2563,9 +2563,9 @@ type MockRootCoordClient_OperatePrivilege_Call struct {
}
// OperatePrivilege is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.OperatePrivilegeRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.OperatePrivilegeRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) OperatePrivilege(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_OperatePrivilege_Call {
return &MockRootCoordClient_OperatePrivilege_Call{Call: _e.mock.On("OperatePrivilege",
append([]interface{}{ctx, in}, opts...)...)}
@ -2633,9 +2633,9 @@ type MockRootCoordClient_OperateUserRole_Call struct {
}
// OperateUserRole is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.OperateUserRoleRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.OperateUserRoleRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) OperateUserRole(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_OperateUserRole_Call {
return &MockRootCoordClient_OperateUserRole_Call{Call: _e.mock.On("OperateUserRole",
append([]interface{}{ctx, in}, opts...)...)}
@ -2703,9 +2703,9 @@ type MockRootCoordClient_RenameCollection_Call struct {
}
// RenameCollection is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.RenameCollectionRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.RenameCollectionRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) RenameCollection(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_RenameCollection_Call {
return &MockRootCoordClient_RenameCollection_Call{Call: _e.mock.On("RenameCollection",
append([]interface{}{ctx, in}, opts...)...)}
@ -2773,9 +2773,9 @@ type MockRootCoordClient_SelectGrant_Call struct {
}
// SelectGrant is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.SelectGrantRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.SelectGrantRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) SelectGrant(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_SelectGrant_Call {
return &MockRootCoordClient_SelectGrant_Call{Call: _e.mock.On("SelectGrant",
append([]interface{}{ctx, in}, opts...)...)}
@ -2843,9 +2843,9 @@ type MockRootCoordClient_SelectRole_Call struct {
}
// SelectRole is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.SelectRoleRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.SelectRoleRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) SelectRole(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_SelectRole_Call {
return &MockRootCoordClient_SelectRole_Call{Call: _e.mock.On("SelectRole",
append([]interface{}{ctx, in}, opts...)...)}
@ -2913,9 +2913,9 @@ type MockRootCoordClient_SelectUser_Call struct {
}
// SelectUser is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.SelectUserRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.SelectUserRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) SelectUser(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_SelectUser_Call {
return &MockRootCoordClient_SelectUser_Call{Call: _e.mock.On("SelectUser",
append([]interface{}{ctx, in}, opts...)...)}
@ -2983,9 +2983,9 @@ type MockRootCoordClient_ShowCollections_Call struct {
}
// ShowCollections is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.ShowCollectionsRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.ShowCollectionsRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) ShowCollections(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_ShowCollections_Call {
return &MockRootCoordClient_ShowCollections_Call{Call: _e.mock.On("ShowCollections",
append([]interface{}{ctx, in}, opts...)...)}
@ -3053,9 +3053,9 @@ type MockRootCoordClient_ShowConfigurations_Call struct {
}
// ShowConfigurations is a helper method to define mock.On call
// - ctx context.Context
// - in *internalpb.ShowConfigurationsRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *internalpb.ShowConfigurationsRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) ShowConfigurations(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_ShowConfigurations_Call {
return &MockRootCoordClient_ShowConfigurations_Call{Call: _e.mock.On("ShowConfigurations",
append([]interface{}{ctx, in}, opts...)...)}
@ -3123,9 +3123,9 @@ type MockRootCoordClient_ShowPartitions_Call struct {
}
// ShowPartitions is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.ShowPartitionsRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.ShowPartitionsRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) ShowPartitions(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_ShowPartitions_Call {
return &MockRootCoordClient_ShowPartitions_Call{Call: _e.mock.On("ShowPartitions",
append([]interface{}{ctx, in}, opts...)...)}
@ -3193,9 +3193,9 @@ type MockRootCoordClient_ShowPartitionsInternal_Call struct {
}
// ShowPartitionsInternal is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.ShowPartitionsRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.ShowPartitionsRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) ShowPartitionsInternal(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_ShowPartitionsInternal_Call {
return &MockRootCoordClient_ShowPartitionsInternal_Call{Call: _e.mock.On("ShowPartitionsInternal",
append([]interface{}{ctx, in}, opts...)...)}
@ -3263,9 +3263,9 @@ type MockRootCoordClient_ShowSegments_Call struct {
}
// ShowSegments is a helper method to define mock.On call
// - ctx context.Context
// - in *milvuspb.ShowSegmentsRequest
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *milvuspb.ShowSegmentsRequest
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) ShowSegments(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_ShowSegments_Call {
return &MockRootCoordClient_ShowSegments_Call{Call: _e.mock.On("ShowSegments",
append([]interface{}{ctx, in}, opts...)...)}
@ -3333,9 +3333,9 @@ type MockRootCoordClient_UpdateChannelTimeTick_Call struct {
}
// UpdateChannelTimeTick is a helper method to define mock.On call
// - ctx context.Context
// - in *internalpb.ChannelTimeTickMsg
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *internalpb.ChannelTimeTickMsg
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) UpdateChannelTimeTick(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_UpdateChannelTimeTick_Call {
return &MockRootCoordClient_UpdateChannelTimeTick_Call{Call: _e.mock.On("UpdateChannelTimeTick",
append([]interface{}{ctx, in}, opts...)...)}
@ -3403,9 +3403,9 @@ type MockRootCoordClient_UpdateCredential_Call struct {
}
// UpdateCredential is a helper method to define mock.On call
// - ctx context.Context
// - in *internalpb.CredentialInfo
// - opts ...grpc.CallOption
// - ctx context.Context
// - in *internalpb.CredentialInfo
// - opts ...grpc.CallOption
func (_e *MockRootCoordClient_Expecter) UpdateCredential(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_UpdateCredential_Call {
return &MockRootCoordClient_UpdateCredential_Call{Call: _e.mock.On("UpdateCredential",
append([]interface{}{ctx, in}, opts...)...)}

238
internal/rootcoord/mocks/meta_table.go
View File

@ -48,8 +48,8 @@ type IMetaTable_AddCollection_Call struct {
}
// AddCollection is a helper method to define mock.On call
// - ctx context.Context
// - coll *model.Collection
// - ctx context.Context
// - coll *model.Collection
func (_e *IMetaTable_Expecter) AddCollection(ctx interface{}, coll interface{}) *IMetaTable_AddCollection_Call {
return &IMetaTable_AddCollection_Call{Call: _e.mock.On("AddCollection", ctx, coll)}
}
@ -91,7 +91,7 @@ type IMetaTable_AddCredential_Call struct {
}
// AddCredential is a helper method to define mock.On call
// - credInfo *internalpb.CredentialInfo
// - credInfo *internalpb.CredentialInfo
func (_e *IMetaTable_Expecter) AddCredential(credInfo interface{}) *IMetaTable_AddCredential_Call {
return &IMetaTable_AddCredential_Call{Call: _e.mock.On("AddCredential", credInfo)}
}
@ -133,8 +133,8 @@ type IMetaTable_AddPartition_Call struct {
}
// AddPartition is a helper method to define mock.On call
// - ctx context.Context
// - partition *model.Partition
// - ctx context.Context
// - partition *model.Partition
func (_e *IMetaTable_Expecter) AddPartition(ctx interface{}, partition interface{}) *IMetaTable_AddPartition_Call {
return &IMetaTable_AddPartition_Call{Call: _e.mock.On("AddPartition", ctx, partition)}
}
@ -176,11 +176,11 @@ type IMetaTable_AlterAlias_Call struct {
}
// AlterAlias is a helper method to define mock.On call
// - ctx context.Context
// - dbName string
// - alias string
// - collectionName string
// - ts uint64
// - ctx context.Context
// - dbName string
// - alias string
// - collectionName string
// - ts uint64
func (_e *IMetaTable_Expecter) AlterAlias(ctx interface{}, dbName interface{}, alias interface{}, collectionName interface{}, ts interface{}) *IMetaTable_AlterAlias_Call {
return &IMetaTable_AlterAlias_Call{Call: _e.mock.On("AlterAlias", ctx, dbName, alias, collectionName, ts)}
}
@ -222,10 +222,10 @@ type IMetaTable_AlterCollection_Call struct {
}
// AlterCollection is a helper method to define mock.On call
// - ctx context.Context
// - oldColl *model.Collection
// - newColl *model.Collection
// - ts uint64
// - ctx context.Context
// - oldColl *model.Collection
// - newColl *model.Collection
// - ts uint64
func (_e *IMetaTable_Expecter) AlterCollection(ctx interface{}, oldColl interface{}, newColl interface{}, ts interface{}) *IMetaTable_AlterCollection_Call {
return &IMetaTable_AlterCollection_Call{Call: _e.mock.On("AlterCollection", ctx, oldColl, newColl, ts)}
}
@ -267,7 +267,7 @@ type IMetaTable_AlterCredential_Call struct {
}
// AlterCredential is a helper method to define mock.On call
// - credInfo *internalpb.CredentialInfo
// - credInfo *internalpb.CredentialInfo
func (_e *IMetaTable_Expecter) AlterCredential(credInfo interface{}) *IMetaTable_AlterCredential_Call {
return &IMetaTable_AlterCredential_Call{Call: _e.mock.On("AlterCredential", credInfo)}
}
@ -309,10 +309,10 @@ type IMetaTable_AlterDatabase_Call struct {
}
// AlterDatabase is a helper method to define mock.On call
// - ctx context.Context
// - oldDB *model.Database
// - newDB *model.Database
// - ts uint64
// - ctx context.Context
// - oldDB *model.Database
// - newDB *model.Database
// - ts uint64
func (_e *IMetaTable_Expecter) AlterDatabase(ctx interface{}, oldDB interface{}, newDB interface{}, ts interface{}) *IMetaTable_AlterDatabase_Call {
return &IMetaTable_AlterDatabase_Call{Call: _e.mock.On("AlterDatabase", ctx, oldDB, newDB, ts)}
}
@ -354,10 +354,10 @@ type IMetaTable_ChangeCollectionState_Call struct {
}
// ChangeCollectionState is a helper method to define mock.On call
// - ctx context.Context
// - collectionID int64
// - state etcdpb.CollectionState
// - ts uint64
// - ctx context.Context
// - collectionID int64
// - state etcdpb.CollectionState
// - ts uint64
func (_e *IMetaTable_Expecter) ChangeCollectionState(ctx interface{}, collectionID interface{}, state interface{}, ts interface{}) *IMetaTable_ChangeCollectionState_Call {
return &IMetaTable_ChangeCollectionState_Call{Call: _e.mock.On("ChangeCollectionState", ctx, collectionID, state, ts)}
}
@ -399,11 +399,11 @@ type IMetaTable_ChangePartitionState_Call struct {
}
// ChangePartitionState is a helper method to define mock.On call
// - ctx context.Context
// - collectionID int64
// - partitionID int64
// - state etcdpb.PartitionState
// - ts uint64
// - ctx context.Context
// - collectionID int64
// - partitionID int64
// - state etcdpb.PartitionState
// - ts uint64
func (_e *IMetaTable_Expecter) ChangePartitionState(ctx interface{}, collectionID interface{}, partitionID interface{}, state interface{}, ts interface{}) *IMetaTable_ChangePartitionState_Call {
return &IMetaTable_ChangePartitionState_Call{Call: _e.mock.On("ChangePartitionState", ctx, collectionID, partitionID, state, ts)}
}
@ -445,11 +445,11 @@ type IMetaTable_CreateAlias_Call struct {
}
// CreateAlias is a helper method to define mock.On call
// - ctx context.Context
// - dbName string
// - alias string
// - collectionName string
// - ts uint64
// - ctx context.Context
// - dbName string
// - alias string
// - collectionName string
// - ts uint64
func (_e *IMetaTable_Expecter) CreateAlias(ctx interface{}, dbName interface{}, alias interface{}, collectionName interface{}, ts interface{}) *IMetaTable_CreateAlias_Call {
return &IMetaTable_CreateAlias_Call{Call: _e.mock.On("CreateAlias", ctx, dbName, alias, collectionName, ts)}
}
@ -491,9 +491,9 @@ type IMetaTable_CreateDatabase_Call struct {
}
// CreateDatabase is a helper method to define mock.On call
// - ctx context.Context
// - db *model.Database
// - ts uint64
// - ctx context.Context
// - db *model.Database
// - ts uint64
func (_e *IMetaTable_Expecter) CreateDatabase(ctx interface{}, db interface{}, ts interface{}) *IMetaTable_CreateDatabase_Call {
return &IMetaTable_CreateDatabase_Call{Call: _e.mock.On("CreateDatabase", ctx, db, ts)}
}
@ -535,8 +535,8 @@ type IMetaTable_CreateRole_Call struct {
}
// CreateRole is a helper method to define mock.On call
// - tenant string
// - entity *milvuspb.RoleEntity
// - tenant string
// - entity *milvuspb.RoleEntity
func (_e *IMetaTable_Expecter) CreateRole(tenant interface{}, entity interface{}) *IMetaTable_CreateRole_Call {
return &IMetaTable_CreateRole_Call{Call: _e.mock.On("CreateRole", tenant, entity)}
}
@ -578,7 +578,7 @@ type IMetaTable_DeleteCredential_Call struct {
}
// DeleteCredential is a helper method to define mock.On call
// - username string
// - username string
func (_e *IMetaTable_Expecter) DeleteCredential(username interface{}) *IMetaTable_DeleteCredential_Call {
return &IMetaTable_DeleteCredential_Call{Call: _e.mock.On("DeleteCredential", username)}
}
@ -630,10 +630,10 @@ type IMetaTable_DescribeAlias_Call struct {
}
// DescribeAlias is a helper method to define mock.On call
// - ctx context.Context
// - dbName string
// - alias string
// - ts uint64
// - ctx context.Context
// - dbName string
// - alias string
// - ts uint64
func (_e *IMetaTable_Expecter) DescribeAlias(ctx interface{}, dbName interface{}, alias interface{}, ts interface{}) *IMetaTable_DescribeAlias_Call {
return &IMetaTable_DescribeAlias_Call{Call: _e.mock.On("DescribeAlias", ctx, dbName, alias, ts)}
}
@ -675,10 +675,10 @@ type IMetaTable_DropAlias_Call struct {
}
// DropAlias is a helper method to define mock.On call
// - ctx context.Context
// - dbName string
// - alias string
// - ts uint64
// - ctx context.Context
// - dbName string
// - alias string
// - ts uint64
func (_e *IMetaTable_Expecter) DropAlias(ctx interface{}, dbName interface{}, alias interface{}, ts interface{}) *IMetaTable_DropAlias_Call {
return &IMetaTable_DropAlias_Call{Call: _e.mock.On("DropAlias", ctx, dbName, alias, ts)}
}
@ -720,9 +720,9 @@ type IMetaTable_DropDatabase_Call struct {
}
// DropDatabase is a helper method to define mock.On call
// - ctx context.Context
// - dbName string
// - ts uint64
// - ctx context.Context
// - dbName string
// - ts uint64
func (_e *IMetaTable_Expecter) DropDatabase(ctx interface{}, dbName interface{}, ts interface{}) *IMetaTable_DropDatabase_Call {
return &IMetaTable_DropDatabase_Call{Call: _e.mock.On("DropDatabase", ctx, dbName, ts)}
}
@ -764,8 +764,8 @@ type IMetaTable_DropGrant_Call struct {
}
// DropGrant is a helper method to define mock.On call
// - tenant string
// - role *milvuspb.RoleEntity
// - tenant string
// - role *milvuspb.RoleEntity
func (_e *IMetaTable_Expecter) DropGrant(tenant interface{}, role interface{}) *IMetaTable_DropGrant_Call {
return &IMetaTable_DropGrant_Call{Call: _e.mock.On("DropGrant", tenant, role)}
}
@ -807,8 +807,8 @@ type IMetaTable_DropRole_Call struct {
}
// DropRole is a helper method to define mock.On call
// - tenant string
// - roleName string
// - tenant string
// - roleName string
func (_e *IMetaTable_Expecter) DropRole(tenant interface{}, roleName interface{}) *IMetaTable_DropRole_Call {
return &IMetaTable_DropRole_Call{Call: _e.mock.On("DropRole", tenant, roleName)}
}
@ -862,11 +862,11 @@ type IMetaTable_GetCollectionByID_Call struct {
}
// GetCollectionByID is a helper method to define mock.On call
// - ctx context.Context
// - dbName string
// - collectionID int64
// - ts uint64
// - allowUnavailable bool
// - ctx context.Context
// - dbName string
// - collectionID int64
// - ts uint64
// - allowUnavailable bool
func (_e *IMetaTable_Expecter) GetCollectionByID(ctx interface{}, dbName interface{}, collectionID interface{}, ts interface{}, allowUnavailable interface{}) *IMetaTable_GetCollectionByID_Call {
return &IMetaTable_GetCollectionByID_Call{Call: _e.mock.On("GetCollectionByID", ctx, dbName, collectionID, ts, allowUnavailable)}
}
@ -920,8 +920,8 @@ type IMetaTable_GetCollectionByIDWithMaxTs_Call struct {
}
// GetCollectionByIDWithMaxTs is a helper method to define mock.On call
// - ctx context.Context
// - collectionID int64
// - ctx context.Context
// - collectionID int64
func (_e *IMetaTable_Expecter) GetCollectionByIDWithMaxTs(ctx interface{}, collectionID interface{}) *IMetaTable_GetCollectionByIDWithMaxTs_Call {
return &IMetaTable_GetCollectionByIDWithMaxTs_Call{Call: _e.mock.On("GetCollectionByIDWithMaxTs", ctx, collectionID)}
}
@ -975,10 +975,10 @@ type IMetaTable_GetCollectionByName_Call struct {
}
// GetCollectionByName is a helper method to define mock.On call
// - ctx context.Context
// - dbName string
// - collectionName string
// - ts uint64
// - ctx context.Context
// - dbName string
// - collectionName string
// - ts uint64
func (_e *IMetaTable_Expecter) GetCollectionByName(ctx interface{}, dbName interface{}, collectionName interface{}, ts interface{}) *IMetaTable_GetCollectionByName_Call {
return &IMetaTable_GetCollectionByName_Call{Call: _e.mock.On("GetCollectionByName", ctx, dbName, collectionName, ts)}
}
@ -1022,7 +1022,7 @@ type IMetaTable_GetCollectionVirtualChannels_Call struct {
}
// GetCollectionVirtualChannels is a helper method to define mock.On call
// - colID int64
// - colID int64
func (_e *IMetaTable_Expecter) GetCollectionVirtualChannels(colID interface{}) *IMetaTable_GetCollectionVirtualChannels_Call {
return &IMetaTable_GetCollectionVirtualChannels_Call{Call: _e.mock.On("GetCollectionVirtualChannels", colID)}
}
@ -1076,7 +1076,7 @@ type IMetaTable_GetCredential_Call struct {
}
// GetCredential is a helper method to define mock.On call
// - username string
// - username string
func (_e *IMetaTable_Expecter) GetCredential(username interface{}) *IMetaTable_GetCredential_Call {
return &IMetaTable_GetCredential_Call{Call: _e.mock.On("GetCredential", username)}
}
@ -1130,9 +1130,9 @@ type IMetaTable_GetDatabaseByID_Call struct {
}
// GetDatabaseByID is a helper method to define mock.On call
// - ctx context.Context
// - dbID int64
// - ts uint64
// - ctx context.Context
// - dbID int64
// - ts uint64
func (_e *IMetaTable_Expecter) GetDatabaseByID(ctx interface{}, dbID interface{}, ts interface{}) *IMetaTable_GetDatabaseByID_Call {
return &IMetaTable_GetDatabaseByID_Call{Call: _e.mock.On("GetDatabaseByID", ctx, dbID, ts)}
}
@ -1186,9 +1186,9 @@ type IMetaTable_GetDatabaseByName_Call struct {
}
// GetDatabaseByName is a helper method to define mock.On call
// - ctx context.Context
// - dbName string
// - ts uint64
// - ctx context.Context
// - dbName string
// - ts uint64
func (_e *IMetaTable_Expecter) GetDatabaseByName(ctx interface{}, dbName interface{}, ts interface{}) *IMetaTable_GetDatabaseByName_Call {
return &IMetaTable_GetDatabaseByName_Call{Call: _e.mock.On("GetDatabaseByName", ctx, dbName, ts)}
}
@ -1232,7 +1232,7 @@ type IMetaTable_GetVChannelsByPchannel_Call struct {
}
// GetVChannelsByPchannel is a helper method to define mock.On call
// - pchannel string
// - pchannel string
func (_e *IMetaTable_Expecter) GetVChannelsByPchannel(pchannel interface{}) *IMetaTable_GetVChannelsByPchannel_Call {
return &IMetaTable_GetVChannelsByPchannel_Call{Call: _e.mock.On("GetVChannelsByPchannel", pchannel)}
}
@ -1274,8 +1274,8 @@ type IMetaTable_IsAlias_Call struct {
}
// IsAlias is a helper method to define mock.On call
// - db string
// - name string
// - db string
// - name string
func (_e *IMetaTable_Expecter) IsAlias(db interface{}, name interface{}) *IMetaTable_IsAlias_Call {
return &IMetaTable_IsAlias_Call{Call: _e.mock.On("IsAlias", db, name)}
}
@ -1329,10 +1329,10 @@ type IMetaTable_ListAliases_Call struct {
}
// ListAliases is a helper method to define mock.On call
// - ctx context.Context
// - dbName string
// - collectionName string
// - ts uint64
// - ctx context.Context
// - dbName string
// - collectionName string
// - ts uint64
func (_e *IMetaTable_Expecter) ListAliases(ctx interface{}, dbName interface{}, collectionName interface{}, ts interface{}) *IMetaTable_ListAliases_Call {
return &IMetaTable_ListAliases_Call{Call: _e.mock.On("ListAliases", ctx, dbName, collectionName, ts)}
}
@ -1376,7 +1376,7 @@ type IMetaTable_ListAliasesByID_Call struct {
}
// ListAliasesByID is a helper method to define mock.On call
// - collID int64
// - collID int64
func (_e *IMetaTable_Expecter) ListAliasesByID(collID interface{}) *IMetaTable_ListAliasesByID_Call {
return &IMetaTable_ListAliasesByID_Call{Call: _e.mock.On("ListAliasesByID", collID)}
}
@ -1420,7 +1420,7 @@ type IMetaTable_ListAllAvailCollections_Call struct {
}
// ListAllAvailCollections is a helper method to define mock.On call
// - ctx context.Context
// - ctx context.Context
func (_e *IMetaTable_Expecter) ListAllAvailCollections(ctx interface{}) *IMetaTable_ListAllAvailCollections_Call {
return &IMetaTable_ListAllAvailCollections_Call{Call: _e.mock.On("ListAllAvailCollections", ctx)}
}
@ -1517,10 +1517,10 @@ type IMetaTable_ListCollections_Call struct {
}
// ListCollections is a helper method to define mock.On call
// - ctx context.Context
// - dbName string
// - ts uint64
// - onlyAvail bool
// - ctx context.Context
// - dbName string
// - ts uint64
// - onlyAvail bool
func (_e *IMetaTable_Expecter) ListCollections(ctx interface{}, dbName interface{}, ts interface{}, onlyAvail interface{}) *IMetaTable_ListCollections_Call {
return &IMetaTable_ListCollections_Call{Call: _e.mock.On("ListCollections", ctx, dbName, ts, onlyAvail)}
}
@ -1627,8 +1627,8 @@ type IMetaTable_ListDatabases_Call struct {
}
// ListDatabases is a helper method to define mock.On call
// - ctx context.Context
// - ts uint64
// - ctx context.Context
// - ts uint64
func (_e *IMetaTable_Expecter) ListDatabases(ctx interface{}, ts interface{}) *IMetaTable_ListDatabases_Call {
return &IMetaTable_ListDatabases_Call{Call: _e.mock.On("ListDatabases", ctx, ts)}
}
@ -1682,7 +1682,7 @@ type IMetaTable_ListPolicy_Call struct {
}
// ListPolicy is a helper method to define mock.On call
// - tenant string
// - tenant string
func (_e *IMetaTable_Expecter) ListPolicy(tenant interface{}) *IMetaTable_ListPolicy_Call {
return &IMetaTable_ListPolicy_Call{Call: _e.mock.On("ListPolicy", tenant)}
}
@ -1736,7 +1736,7 @@ type IMetaTable_ListUserRole_Call struct {
}
// ListUserRole is a helper method to define mock.On call
// - tenant string
// - tenant string
func (_e *IMetaTable_Expecter) ListUserRole(tenant interface{}) *IMetaTable_ListUserRole_Call {
return &IMetaTable_ListUserRole_Call{Call: _e.mock.On("ListUserRole", tenant)}
}
@ -1778,9 +1778,9 @@ type IMetaTable_OperatePrivilege_Call struct {
}
// OperatePrivilege is a helper method to define mock.On call
// - tenant string
// - entity *milvuspb.GrantEntity
// - operateType milvuspb.OperatePrivilegeType
// - tenant string
// - entity *milvuspb.GrantEntity
// - operateType milvuspb.OperatePrivilegeType
func (_e *IMetaTable_Expecter) OperatePrivilege(tenant interface{}, entity interface{}, operateType interface{}) *IMetaTable_OperatePrivilege_Call {
return &IMetaTable_OperatePrivilege_Call{Call: _e.mock.On("OperatePrivilege", tenant, entity, operateType)}
}
@ -1822,10 +1822,10 @@ type IMetaTable_OperateUserRole_Call struct {
}
// OperateUserRole is a helper method to define mock.On call
// - tenant string
// - userEntity *milvuspb.UserEntity
// - roleEntity *milvuspb.RoleEntity
// - operateType milvuspb.OperateUserRoleType
// - tenant string
// - userEntity *milvuspb.UserEntity
// - roleEntity *milvuspb.RoleEntity
// - operateType milvuspb.OperateUserRoleType
func (_e *IMetaTable_Expecter) OperateUserRole(tenant interface{}, userEntity interface{}, roleEntity interface{}, operateType interface{}) *IMetaTable_OperateUserRole_Call {
return &IMetaTable_OperateUserRole_Call{Call: _e.mock.On("OperateUserRole", tenant, userEntity, roleEntity, operateType)}
}
@ -1867,9 +1867,9 @@ type IMetaTable_RemoveCollection_Call struct {
}
// RemoveCollection is a helper method to define mock.On call
// - ctx context.Context
// - collectionID int64
// - ts uint64
// - ctx context.Context
// - collectionID int64
// - ts uint64
func (_e *IMetaTable_Expecter) RemoveCollection(ctx interface{}, collectionID interface{}, ts interface{}) *IMetaTable_RemoveCollection_Call {
return &IMetaTable_RemoveCollection_Call{Call: _e.mock.On("RemoveCollection", ctx, collectionID, ts)}
}
@ -1911,11 +1911,11 @@ type IMetaTable_RemovePartition_Call struct {
}
// RemovePartition is a helper method to define mock.On call
// - ctx context.Context
// - dbID int64
// - collectionID int64
// - partitionID int64
// - ts uint64
// - ctx context.Context
// - dbID int64
// - collectionID int64
// - partitionID int64
// - ts uint64
func (_e *IMetaTable_Expecter) RemovePartition(ctx interface{}, dbID interface{}, collectionID interface{}, partitionID interface{}, ts interface{}) *IMetaTable_RemovePartition_Call {
return &IMetaTable_RemovePartition_Call{Call: _e.mock.On("RemovePartition", ctx, dbID, collectionID, partitionID, ts)}
}
@ -1957,12 +1957,12 @@ type IMetaTable_RenameCollection_Call struct {
}
// RenameCollection is a helper method to define mock.On call
// - ctx context.Context
// - dbName string
// - oldName string
// - newDBName string
// - newName string
// - ts uint64
// - ctx context.Context
// - dbName string
// - oldName string
// - newDBName string
// - newName string
// - ts uint64
func (_e *IMetaTable_Expecter) RenameCollection(ctx interface{}, dbName interface{}, oldName interface{}, newDBName interface{}, newName interface{}, ts interface{}) *IMetaTable_RenameCollection_Call {
return &IMetaTable_RenameCollection_Call{Call: _e.mock.On("RenameCollection", ctx, dbName, oldName, newDBName, newName, ts)}
}
@ -2016,8 +2016,8 @@ type IMetaTable_SelectGrant_Call struct {
}
// SelectGrant is a helper method to define mock.On call
// - tenant string
// - entity *milvuspb.GrantEntity
// - tenant string
// - entity *milvuspb.GrantEntity
func (_e *IMetaTable_Expecter) SelectGrant(tenant interface{}, entity interface{}) *IMetaTable_SelectGrant_Call {
return &IMetaTable_SelectGrant_Call{Call: _e.mock.On("SelectGrant", tenant, entity)}
}
@ -2071,9 +2071,9 @@ type IMetaTable_SelectRole_Call struct {
}
// SelectRole is a helper method to define mock.On call
// - tenant string
// - entity *milvuspb.RoleEntity
// - includeUserInfo bool
// - tenant string
// - entity *milvuspb.RoleEntity
// - includeUserInfo bool
func (_e *IMetaTable_Expecter) SelectRole(tenant interface{}, entity interface{}, includeUserInfo interface{}) *IMetaTable_SelectRole_Call {
return &IMetaTable_SelectRole_Call{Call: _e.mock.On("SelectRole", tenant, entity, includeUserInfo)}
}
@ -2127,9 +2127,9 @@ type IMetaTable_SelectUser_Call struct {
}
// SelectUser is a helper method to define mock.On call
// - tenant string
// - entity *milvuspb.UserEntity
// - includeRoleInfo bool
// - tenant string
// - entity *milvuspb.UserEntity
// - includeRoleInfo bool
func (_e *IMetaTable_Expecter) SelectUser(tenant interface{}, entity interface{}, includeRoleInfo interface{}) *IMetaTable_SelectUser_Call {
return &IMetaTable_SelectUser_Call{Call: _e.mock.On("SelectUser", tenant, entity, includeRoleInfo)}
}

19
internal/streamingnode/client/handler/producer/producer_test.go
View File

@ -59,16 +59,13 @@ func TestProducer(t *testing.T) {
assert.NotNil(t, producer)
ch := make(chan struct{})
go func() {
msgID, err := producer.Produce(ctx, message.NewMutableMessageBuilder().
WithMessageType(message.MessageTypeUnknown).
WithPayload([]byte{}).
BuildMutable())
msg := message.CreateTestEmptyInsertMesage(1, nil)
msgID, err := producer.Produce(ctx, msg)
assert.Error(t, err)
assert.Nil(t, msgID)
msgID, err = producer.Produce(ctx, message.NewMutableMessageBuilder().
WithMessageType(message.MessageTypeUnknown).
WithPayload([]byte{}).
BuildMutable())
msg = message.CreateTestEmptyInsertMesage(1, nil)
msgID, err = producer.Produce(ctx, msg)
assert.NoError(t, err)
assert.NotNil(t, msgID)
close(ch)
@ -101,10 +98,8 @@ func TestProducer(t *testing.T) {
ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
defer cancel()
_, err = producer.Produce(ctx, message.NewMutableMessageBuilder().
WithMessageType(message.MessageTypeUnknown).
WithPayload([]byte{}).
BuildMutable())
msg := message.CreateTestEmptyInsertMesage(1, nil)
_, err = producer.Produce(ctx, msg)
assert.ErrorIs(t, err, context.DeadlineExceeded)
assert.True(t, producer.IsAvailable())
producer.Close()

6
internal/streamingnode/server/service/handler/producer/produce_server.go
View File

@ -143,11 +143,7 @@ func (p *ProduceServer) recvLoop() (err error) {
// handleProduce handles the produce message request.
func (p *ProduceServer) handleProduce(req *streamingpb.ProduceMessageRequest) {
p.logger.Debug("recv produce message from client", zap.Int64("requestID", req.RequestId))
msg := message.NewMutableMessageBuilder().
WithPayload(req.GetMessage().GetPayload()).
WithProperties(req.GetMessage().GetProperties()).
BuildMutable()
msg := message.NewMutableMessage(req.GetMessage().GetPayload(), req.GetMessage().GetProperties())
if err := p.validateMessage(msg); err != nil {
p.logger.Warn("produce message validation failed", zap.Int64("requestID", req.RequestId), zap.Error(err))
p.sendProduceResult(req.RequestId, nil, err)

131
internal/streamingnode/server/wal/adaptor/message_handler.go Normal file
View File

@ -0,0 +1,131 @@
package adaptor
import (
"context"
"go.uber.org/zap"
"github.com/milvus-io/milvus/internal/streamingnode/server/wal"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/mq/msgstream"
"github.com/milvus-io/milvus/pkg/streaming/util/message"
"github.com/milvus-io/milvus/pkg/streaming/util/message/adaptor"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
type defaultMessageHandler chan message.ImmutableMessage
func (h defaultMessageHandler) Handle(ctx context.Context, upstream <-chan message.ImmutableMessage, msg message.ImmutableMessage) (incoming message.ImmutableMessage, ok bool, err error) {
var sendingCh chan message.ImmutableMessage
if msg != nil {
sendingCh = h
}
select {
case <-ctx.Done():
return nil, false, ctx.Err()
case msg, ok := <-upstream:
if !ok {
return nil, false, wal.ErrUpstreamClosed
}
return msg, false, nil
case sendingCh <- msg:
return nil, true, nil
}
}
func (d defaultMessageHandler) Close() {
close(d)
}
// NewMsgPackAdaptorHandler create a new message pack adaptor handler.
func NewMsgPackAdaptorHandler() *MsgPackAdaptorHandler {
return &MsgPackAdaptorHandler{
logger: log.With(),
channel: make(chan *msgstream.MsgPack),
pendings: make([]message.ImmutableMessage, 0),
pendingMsgPack: typeutil.NewMultipartQueue[*msgstream.MsgPack](),
}
}
// MsgPackAdaptorHandler is the handler for message pack.
type MsgPackAdaptorHandler struct {
logger *log.MLogger
channel chan *msgstream.MsgPack
pendings []message.ImmutableMessage // pendings hold the vOld message which has same time tick.
pendingMsgPack *typeutil.MultipartQueue[*msgstream.MsgPack] // pendingMsgPack hold unsent msgPack.
}
// Chan is the channel for message.
func (m *MsgPackAdaptorHandler) Chan() <-chan *msgstream.MsgPack {
return m.channel
}
// Handle is the callback for handling message.
func (m *MsgPackAdaptorHandler) Handle(ctx context.Context, upstream <-chan message.ImmutableMessage, msg message.ImmutableMessage) (incoming message.ImmutableMessage, ok bool, err error) {
// not handle new message if there are pending msgPack.
if msg != nil && m.pendingMsgPack.Len() == 0 {
m.generateMsgPack(msg)
ok = true
}
for {
var sendCh chan<- *msgstream.MsgPack
if m.pendingMsgPack.Len() != 0 {
sendCh = m.channel
}
select {
case <-ctx.Done():
return nil, ok, ctx.Err()
case msg, notClose := <-upstream:
if !notClose {
return nil, ok, wal.ErrUpstreamClosed
}
return msg, ok, nil
case sendCh <- m.pendingMsgPack.Next():
m.pendingMsgPack.UnsafeAdvance()
if m.pendingMsgPack.Len() > 0 {
continue
}
return nil, ok, nil
}
}
}
// generateMsgPack generate msgPack from message.
func (m *MsgPackAdaptorHandler) generateMsgPack(msg message.ImmutableMessage) {
switch msg.Version() {
case message.VersionOld:
if len(m.pendings) != 0 {
if msg.TimeTick() > m.pendings[0].TimeTick() {
m.addMsgPackIntoPending(m.pendings...)
m.pendings = nil
}
}
m.pendings = append(m.pendings, msg)
case message.VersionV1:
if len(m.pendings) != 0 { // all previous message should be vOld.
m.addMsgPackIntoPending(m.pendings...)
m.pendings = nil
}
m.addMsgPackIntoPending(msg)
default:
panic("unsupported message version")
}
}
// addMsgPackIntoPending add message into pending msgPack.
func (m *MsgPackAdaptorHandler) addMsgPackIntoPending(msgs ...message.ImmutableMessage) {
newPack, err := adaptor.NewMsgPackFromMessage(msgs...)
if err != nil {
m.logger.Warn("failed to convert message to msgpack", zap.Error(err))
}
if newPack != nil {
m.pendingMsgPack.AddOne(newPack)
}
}
// Close close the handler.
func (m *MsgPackAdaptorHandler) Close() {
close(m.channel)
}

76
internal/streamingnode/server/wal/adaptor/message_handler_test.go Normal file
View File

@ -0,0 +1,76 @@
package adaptor
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus/pkg/mocks/streaming/util/mock_message"
"github.com/milvus-io/milvus/pkg/streaming/util/message"
"github.com/milvus-io/milvus/pkg/streaming/walimpls/impls/rmq"
)
func TestMsgPackAdaptorHandler(t *testing.T) {
messageID := rmq.NewRmqID(1)
tt := uint64(100)
msg := message.CreateTestInsertMessage(
t,
1,
1000,
tt,
messageID,
)
immutableMsg := msg.IntoImmutableMessage(messageID)
upstream := make(chan message.ImmutableMessage, 1)
ctx := context.Background()
h := NewMsgPackAdaptorHandler()
done := make(chan struct{})
go func() {
for range h.Chan() {
}
close(done)
}()
upstream <- immutableMsg
newMsg, ok, err := h.Handle(ctx, upstream, nil)
assert.Equal(t, newMsg, immutableMsg)
assert.False(t, ok)
assert.NoError(t, err)
newMsg, ok, err = h.Handle(ctx, upstream, newMsg)
assert.NoError(t, err)
assert.Nil(t, newMsg)
assert.True(t, ok)
h.Close()
<-done
}
func TestDefaultHandler(t *testing.T) {
h := make(defaultMessageHandler, 1)
done := make(chan struct{})
go func() {
for range h {
}
close(done)
}()
upstream := make(chan message.ImmutableMessage, 1)
msg := mock_message.NewMockImmutableMessage(t)
upstream <- msg
newMsg, ok, err := h.Handle(context.Background(), upstream, nil)
assert.NotNil(t, newMsg)
assert.NoError(t, err)
assert.False(t, ok)
assert.Equal(t, newMsg, msg)
newMsg, ok, err = h.Handle(context.Background(), upstream, newMsg)
assert.NoError(t, err)
assert.Nil(t, newMsg)
assert.True(t, ok)
h.Close()
<-done
}

49
internal/streamingnode/server/wal/adaptor/scanner_adaptor.go
View File

@ -10,6 +10,7 @@ import (
"github.com/milvus-io/milvus/pkg/streaming/util/types"
"github.com/milvus-io/milvus/pkg/streaming/walimpls"
"github.com/milvus-io/milvus/pkg/streaming/walimpls/helper"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
var _ wal.Scanner = (*scannerAdaptorImpl)(nil)
@ -21,13 +22,15 @@ func newScannerAdaptor(
readOption wal.ReadOption,
cleanup func(),
) wal.Scanner {
if readOption.MesasgeHandler == nil {
readOption.MesasgeHandler = defaultMessageHandler(make(chan message.ImmutableMessage))
}
s := &scannerAdaptorImpl{
logger: log.With(zap.String("name", name), zap.String("channel", l.Channel().Name)),
innerWAL: l,
readOption: readOption,
sendingCh: make(chan message.ImmutableMessage, 1),
reorderBuffer: utility.NewReOrderBuffer(),
pendingQueue: utility.NewImmutableMessageQueue(),
pendingQueue: typeutil.NewMultipartQueue[message.ImmutableMessage](),
cleanup: cleanup,
ScannerHelper: helper.NewScannerHelper(name),
}
@ -41,9 +44,8 @@ type scannerAdaptorImpl struct {
logger *log.MLogger
innerWAL walimpls.WALImpls
readOption wal.ReadOption
sendingCh chan message.ImmutableMessage
reorderBuffer *utility.ReOrderByTimeTickBuffer // only support time tick reorder now.
pendingQueue *utility.ImmutableMessageQueue //
reorderBuffer *utility.ReOrderByTimeTickBuffer // only support time tick reorder now.
pendingQueue *typeutil.MultipartQueue[message.ImmutableMessage] //
cleanup func()
}
@ -52,9 +54,9 @@ func (s *scannerAdaptorImpl) Channel() types.PChannelInfo {
return s.innerWAL.Channel()
}
// Chan returns the channel of message.
// Chan returns the message channel of the scanner.
func (s *scannerAdaptorImpl) Chan() <-chan message.ImmutableMessage {
return s.sendingCh
return s.readOption.MesasgeHandler.(defaultMessageHandler)
}
// Close the scanner, release the underlying resources.
@ -68,7 +70,7 @@ func (s *scannerAdaptorImpl) Close() error {
}
func (s *scannerAdaptorImpl) executeConsume() {
defer close(s.sendingCh)
defer s.readOption.MesasgeHandler.Close()
innerScanner, err := s.innerWAL.Read(s.Context(), walimpls.ReadOption{
Name: s.Name(),
@ -83,36 +85,29 @@ func (s *scannerAdaptorImpl) executeConsume() {
for {
// generate the event channel and do the event loop.
// TODO: Consume from local cache.
upstream, sending := s.getEventCh(innerScanner)
select {
case <-s.Context().Done():
upstream := s.getUpstream(innerScanner)
msg, ok, err := s.readOption.MesasgeHandler.Handle(s.Context(), upstream, s.pendingQueue.Next())
if err != nil {
s.Finish(err)
return
case msg, ok := <-upstream:
if !ok {
s.Finish(innerScanner.Error())
return
}
s.handleUpstream(msg)
case sending <- s.pendingQueue.Next():
}
if ok {
s.pendingQueue.UnsafeAdvance()
}
if msg != nil {
s.handleUpstream(msg)
}
}
}
func (s *scannerAdaptorImpl) getEventCh(scanner walimpls.ScannerImpls) (<-chan message.ImmutableMessage, chan<- message.ImmutableMessage) {
if s.pendingQueue.Len() == 0 {
// If pending queue is empty,
// no more message can be sent,
// we always need to recv message from upstream to avoid starve.
return scanner.Chan(), nil
}
func (s *scannerAdaptorImpl) getUpstream(scanner walimpls.ScannerImpls) <-chan message.ImmutableMessage {
// TODO: configurable pending buffer count.
// If the pending queue is full, we need to wait until it's consumed to avoid scanner overloading.
if s.pendingQueue.Len() > 16 {
return nil, s.sendingCh
return nil
}
return scanner.Chan(), s.sendingCh
return scanner.Chan()
}
func (s *scannerAdaptorImpl) handleUpstream(msg message.ImmutableMessage) {

43
internal/streamingnode/server/wal/adaptor/wal_test.go
View File

@ -10,11 +10,9 @@ import (
"testing"
"time"
"github.com/golang/protobuf/proto"
"github.com/remeh/sizedwaitgroup"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus/internal/streamingnode/server/resource"
"github.com/milvus-io/milvus/internal/streamingnode/server/resource/idalloc"
"github.com/milvus-io/milvus/internal/streamingnode/server/wal"
@ -154,26 +152,10 @@ func (f *testOneWALFramework) testAppend(ctx context.Context, w wal.WAL) ([]mess
time.Sleep(time.Duration(5+rand.Int31n(10)) * time.Millisecond)
// ...rocksmq has a dirty implement of properties,
// without commonpb.MsgHeader, it can not work.
header := commonpb.MsgHeader{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: int64(i),
},
}
payload, err := proto.Marshal(&header)
if err != nil {
panic(err)
}
properties := map[string]string{
msg := message.CreateTestEmptyInsertMesage(int64(i), map[string]string{
"id": fmt.Sprintf("%d", i),
"const": "t",
}
typ := message.MessageTypeUnknown
msg := message.NewMutableMessageBuilder().
WithMessageType(typ).
WithPayload(payload).
WithProperties(properties).
BuildMutable()
})
id, err := w.Append(ctx, msg)
assert.NoError(f.t, err)
assert.NotNil(f.t, id)
@ -181,27 +163,12 @@ func (f *testOneWALFramework) testAppend(ctx context.Context, w wal.WAL) ([]mess
}(i)
}
swg.Wait()
// send a final hint message
header := commonpb.MsgHeader{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: int64(f.messageCount - 1),
},
}
payload, err := proto.Marshal(&header)
if err != nil {
panic(err)
}
properties := map[string]string{
msg := message.CreateTestEmptyInsertMesage(int64(f.messageCount-1), map[string]string{
"id": fmt.Sprintf("%d", f.messageCount-1),
"const": "t",
"term": strconv.FormatInt(int64(f.term), 10),
}
msg := message.NewMutableMessageBuilder().
WithPayload(payload).
WithProperties(properties).
WithMessageType(message.MessageTypeUnknown).
BuildMutable()
})
id, err := w.Append(ctx, msg)
assert.NoError(f.t, err)
messages[f.messageCount-1] = msg.IntoImmutableMessage(id)

36
internal/streamingnode/server/wal/interceptors/timetick/timetick_message.go
View File

@ -1,48 +1,28 @@
package timetick
import (
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
"github.com/milvus-io/milvus/pkg/mq/msgstream"
"github.com/milvus-io/milvus/pkg/streaming/util/message"
"github.com/milvus-io/milvus/pkg/util/commonpbutil"
)
func newTimeTickMsg(ts uint64, sourceID int64) (message.MutableMessage, error) {
// TODO: time tick should be put on properties, for compatibility, we put it on message body now.
msgstreamMsg := &msgstream.TimeTickMsg{
BaseMsg: msgstream.BaseMsg{
BeginTimestamp: ts,
EndTimestamp: ts,
HashValues: []uint32{0},
},
TimeTickMsg: msgpb.TimeTickMsg{
// Common message's time tick is set on interceptor.
// TimeTickMsg's time tick should be set here.
msg, err := message.NewTimeTickMessageBuilderV1().
WithMessageHeader(&message.TimeTickMessageHeader{}).
WithPayload(&msgpb.TimeTickMsg{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType_TimeTick),
commonpbutil.WithMsgID(0),
commonpbutil.WithTimeStamp(ts),
commonpbutil.WithSourceID(sourceID),
),
},
}
bytes, err := msgstreamMsg.Marshal(msgstreamMsg)
}).BuildMutable()
if err != nil {
return nil, errors.Wrap(err, "marshal time tick message failed")
return nil, err
}
payload, ok := bytes.([]byte)
if !ok {
return nil, errors.New("marshal time tick message as []byte failed")
}
// Common message's time tick is set on interceptor.
// TimeTickMsg's time tick should be set here.
msg := message.NewMutableMessageBuilder().
WithMessageType(message.MessageTypeTimeTick).
WithPayload(payload).
BuildMutable().
WithTimeTick(ts)
return msg, nil
return msg.WithTimeTick(ts), nil
}

33
internal/streamingnode/server/wal/scanner.go
View File

@ -1,6 +1,10 @@
package wal
import (
"context"
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus/pkg/streaming/util/message"
"github.com/milvus-io/milvus/pkg/streaming/util/options"
"github.com/milvus-io/milvus/pkg/streaming/util/types"
@ -8,15 +12,22 @@ import (
type MessageFilter = func(message.ImmutableMessage) bool
var ErrUpstreamClosed = errors.New("upstream closed")
// ReadOption is the option for reading records from the wal.
type ReadOption struct {
DeliverPolicy options.DeliverPolicy
MessageFilter MessageFilter
DeliverPolicy options.DeliverPolicy
MessageFilter MessageFilter
MesasgeHandler MessageHandler // message handler for message processing.
// If the message handler is nil (no redundant operation need to apply),
// the default message handler will be used, and the receiver will be returned from Chan.
// Otherwise, Chan will panic.
// vaild every message will be passed to this handler before being delivered to the consumer.
}
// Scanner is the interface for reading records from the wal.
type Scanner interface {
// Chan returns the channel of message.
// Chan returns the channel of message if Option.MessageHandler is nil.
Chan() <-chan message.ImmutableMessage
// Channel returns the channel assignment info of the wal.
@ -33,3 +44,19 @@ type Scanner interface {
// Return the error same with `Error`
Close() error
}
// MessageHandler is used to handle message read from log.
// TODO: should be removed in future after msgstream is removed.
type MessageHandler interface {
// Handle is the callback for handling message.
// The message will be passed to the handler for processing.
// Handle operation can be blocked, but should listen to the context.Done() and upstream.
// If the context is canceled, the handler should return immediately with ctx.Err.
// If the upstream is closed, the handler should return immediately with ErrUpstreamClosed.
// If the upstream recv a message, the handler should return the incoming message.
// If the handler handle the message successfully, it should return the ok=true.
Handle(ctx context.Context, upstream <-chan message.ImmutableMessage, msg message.ImmutableMessage) (incoming message.ImmutableMessage, ok bool, err error)
// Close is called after all messages are handled or handling is interrupted.
Close()
}

51
internal/streamingnode/server/wal/utility/immutable_message_queue.go
View File

@ -1,51 +0,0 @@
package utility
import "github.com/milvus-io/milvus/pkg/streaming/util/message"
// NewImmutableMessageQueue create a new immutable message queue.
func NewImmutableMessageQueue() *ImmutableMessageQueue {
return &ImmutableMessageQueue{
pendings: make([][]message.ImmutableMessage, 0),
cnt: 0,
}
}
// ImmutableMessageQueue is a queue of messages.
type ImmutableMessageQueue struct {
pendings [][]message.ImmutableMessage
cnt int
}
// Len return the queue size.
func (pq *ImmutableMessageQueue) Len() int {
return pq.cnt
}
// Add add a slice of message as pending one
func (pq *ImmutableMessageQueue) Add(msgs []message.ImmutableMessage) {
if len(msgs) == 0 {
return
}
pq.pendings = append(pq.pendings, msgs)
pq.cnt += len(msgs)
}
// Next return the next message in pending queue.
func (pq *ImmutableMessageQueue) Next() message.ImmutableMessage {
if len(pq.pendings) != 0 && len(pq.pendings[0]) != 0 {
return pq.pendings[0][0]
}
return nil
}
// UnsafeAdvance do a advance without check.
// !!! Should only be called `Next` do not return nil.
func (pq *ImmutableMessageQueue) UnsafeAdvance() {
if len(pq.pendings[0]) == 1 {
pq.pendings = pq.pendings[1:]
pq.cnt--
return
}
pq.pendings[0] = pq.pendings[0][1:]
pq.cnt--
}

25
internal/streamingnode/server/wal/utility/immutable_message_queue_test.go
View File

@ -1,25 +0,0 @@
package utility
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus/pkg/mocks/streaming/util/mock_message"
"github.com/milvus-io/milvus/pkg/streaming/util/message"
)
func TestImmutableMessageQueue(t *testing.T) {
q := NewImmutableMessageQueue()
for i := 0; i < 100; i++ {
q.Add([]message.ImmutableMessage{
mock_message.NewMockImmutableMessage(t),
})
assert.Equal(t, i+1, q.Len())
}
for i := 100; i > 0; i-- {
assert.NotNil(t, q.Next())
q.UnsafeAdvance()
assert.Equal(t, i-1, q.Len())
}
}

3
internal/util/flowgraph/message_test.go
View File

@ -45,6 +45,9 @@ func (bm *MockMsg) SetID(id msgstream.UniqueID) {
// do nothing
}
func (bm *MockMsg) SetTs(ts uint64) {
}
func (bm *MockMsg) BeginTs() Timestamp {
return 0
}

5
pkg/Makefile
View File

@ -17,4 +17,7 @@ generate-mockery: getdeps
$(INSTALL_PATH)/mockery --name=Factory --dir=$(PWD)/mq/msgstream --output=$(PWD)/mq/msgstream --filename=mock_msgstream_factory.go --with-expecter --structname=MockFactory --outpkg=msgstream --inpackage
$(INSTALL_PATH)/mockery --name=Client --dir=$(PWD)/mq/msgdispatcher --output=$(PWD)/mq/msgsdispatcher --filename=mock_client.go --with-expecter --structname=MockClient --outpkg=msgdispatcher --inpackage
$(INSTALL_PATH)/mockery --name=Logger --dir=$(PWD)/eventlog --output=$(PWD)/eventlog --filename=mock_logger.go --with-expecter --structname=MockLogger --outpkg=eventlog --inpackage
$(INSTALL_PATH)/mockery --name=MessageID --dir=$(PWD)/mq/msgstream/mqwrapper --output=$(PWD)/mq/msgstream/mqwrapper --filename=mock_id.go --with-expecter --structname=MockMessageID --outpkg=mqwrapper --inpackage
$(INSTALL_PATH)/mockery --name=MessageID --dir=$(PWD)/mq/msgstream/mqwrapper --output=$(PWD)/mq/msgstream/mqwrapper --filename=mock_id.go --with-expecter --structname=MockMessageID --outpkg=mqwrapper --inpackage
generate-proto:
$(ROOTPATH)/cmake_build/bin/protoc --proto_path=$(PWD)/streaming/util/message/messagepb --go_out=plugins=grpc,paths=source_relative:./streaming/util/message/messagepb $(PWD)/streaming/util/message/messagepb/message.proto

7
pkg/mq/msgstream/mq_msgstream.go
View File

@ -384,6 +384,11 @@ func (ms *mqMsgStream) Broadcast(msgPack *MsgPack) (map[string][]MessageID, erro
}
func (ms *mqMsgStream) getTsMsgFromConsumerMsg(msg common.Message) (TsMsg, error) {
return GetTsMsgFromConsumerMsg(ms.unmarshal, msg)
}
// GetTsMsgFromConsumerMsg get TsMsg from consumer message
func GetTsMsgFromConsumerMsg(unmarshalDispatcher UnmarshalDispatcher, msg common.Message) (TsMsg, error) {
header := commonpb.MsgHeader{}
if msg.Payload() == nil {
return nil, fmt.Errorf("failed to unmarshal message header, payload is empty")
@ -395,7 +400,7 @@ func (ms *mqMsgStream) getTsMsgFromConsumerMsg(msg common.Message) (TsMsg, error
if header.Base == nil {
return nil, fmt.Errorf("failed to unmarshal message, header is uncomplete")
}
tsMsg, err := ms.unmarshal.Unmarshal(msg.Payload(), header.Base.MsgType)
tsMsg, err := unmarshalDispatcher.Unmarshal(msg.Payload(), header.Base.MsgType)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal tsMsg, err %s", err.Error())
}

11
pkg/mq/msgstream/mqwrapper/pulsar/pulsar_id.go
View File

@ -24,6 +24,13 @@ import (
"github.com/milvus-io/milvus/pkg/mq/common"
)
// NewPulsarID creates a new pulsarID
func NewPulsarID(id pulsar.MessageID) *pulsarID {
return &pulsarID{
messageID: id,
}
}
type pulsarID struct {
messageID pulsar.MessageID
}
@ -31,6 +38,10 @@ type pulsarID struct {
// Check if pulsarID implements and MessageID interface
var _ common.MessageID = &pulsarID{}
func (pid *pulsarID) PulsarID() pulsar.MessageID {
return pid.messageID
}
func (pid *pulsarID) Serialize() []byte {
return pid.messageID.Serialize()
}

6
pkg/mq/msgstream/msg.go
View File

@ -53,6 +53,7 @@ type TsMsg interface {
Unmarshal(MarshalType) (TsMsg, error)
Position() *MsgPosition
SetPosition(*MsgPosition)
SetTs(ts uint64)
Size() int
}
@ -111,6 +112,11 @@ func (bm *BaseMsg) SetPosition(position *MsgPosition) {
bm.MsgPosition = position
}
func (bm *BaseMsg) SetTs(ts uint64) {
bm.BeginTimestamp = ts
bm.EndTimestamp = ts
}
func convertToByteArray(input interface{}) ([]byte, error) {
switch output := input.(type) {
case []byte:

5
pkg/streaming/OWNERS Normal file
View File

@ -0,0 +1,5 @@
reviewers:
- chyezh
approvers:
- maintainers

124
pkg/streaming/util/message/adaptor/message.go Normal file
View File

@ -0,0 +1,124 @@
package adaptor
import (
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
"github.com/milvus-io/milvus/pkg/mq/msgstream"
"github.com/milvus-io/milvus/pkg/streaming/util/message"
)
var unmashalerDispatcher = (&msgstream.ProtoUDFactory{}).NewUnmarshalDispatcher()
// FromMessageToMsgPack converts message to msgpack.
// Same TimeTick must be sent with same msgpack.
// !!! Msgs must be keep same time tick.
// TODO: remove this function after remove the msgstream implementation.
func NewMsgPackFromMessage(msgs ...message.ImmutableMessage) (*msgstream.MsgPack, error) {
if len(msgs) == 0 {
return nil, nil
}
allTsMsgs := make([]msgstream.TsMsg, 0, len(msgs))
var finalErr error
for _, msg := range msgs {
var tsMsg msgstream.TsMsg
var err error
switch msg.Version() {
case message.VersionOld:
tsMsg, err = fromMessageToTsMsgVOld(msg)
case message.VersionV1:
tsMsg, err = fromMessageToTsMsgV1(msg)
default:
panic("unsupported message version")
}
if err != nil {
finalErr = errors.CombineErrors(finalErr, errors.Wrapf(err, "Failed to convert message to msgpack, %v", msg.MessageID()))
continue
}
allTsMsgs = append(allTsMsgs, tsMsg)
}
if len(allTsMsgs) == 0 {
return nil, finalErr
}
// msgs is sorted by time tick.
// Postition use the last confirmed message id.
// 1. So use the first tsMsgs's Position can read all messages which timetick is greater or equal than the first tsMsgs's BeginTs.
// In other words, from the StartPositions, you can read the full msgPack.
// 2. Use the last tsMsgs's Position as the EndPosition, you can read all messages following the msgPack.
return &msgstream.MsgPack{
BeginTs: allTsMsgs[0].BeginTs(),
EndTs: allTsMsgs[len(allTsMsgs)-1].EndTs(),
Msgs: allTsMsgs,
StartPositions: []*msgstream.MsgPosition{allTsMsgs[0].Position()},
EndPositions: []*msgstream.MsgPosition{allTsMsgs[len(allTsMsgs)-1].Position()},
}, finalErr
}
func fromMessageToTsMsgVOld(msg message.ImmutableMessage) (msgstream.TsMsg, error) {
panic("Not implemented")
}
// fromMessageToTsMsgV1 converts message to ts message.
func fromMessageToTsMsgV1(msg message.ImmutableMessage) (msgstream.TsMsg, error) {
tsMsg, err := unmashalerDispatcher.Unmarshal(msg.Payload(), commonpb.MsgType(msg.MessageType()))
if err != nil {
return nil, errors.Wrap(err, "Failed to unmarshal message")
}
tsMsg.SetTs(msg.TimeTick())
tsMsg.SetPosition(&msgpb.MsgPosition{
ChannelName: msg.VChannel(),
// from the last confirmed message id, you can read all messages which timetick is greater or equal than current message id.
MsgID: MustGetMQWrapperIDFromMessage(msg.LastConfirmedMessageID()).Serialize(),
MsgGroup: "", // Not important any more.
Timestamp: msg.TimeTick(),
})
return recoverMessageFromHeader(tsMsg, msg)
}
// recoverMessageFromHeader recovers message from header.
func recoverMessageFromHeader(tsMsg msgstream.TsMsg, msg message.ImmutableMessage) (msgstream.TsMsg, error) {
switch msg.MessageType() {
case message.MessageTypeInsert:
insertMessage, err := message.AsImmutableInsertMessage(msg)
if err != nil {
return nil, errors.Wrap(err, "Failed to convert message to insert message")
}
// insertMsg has multiple partition and segment assignment is done by insert message header.
// so recover insert message from header before send it.
return recoverInsertMsgFromHeader(tsMsg.(*msgstream.InsertMsg), insertMessage.MessageHeader(), msg.TimeTick())
default:
return tsMsg, nil
}
}
// recoverInsertMsgFromHeader recovers insert message from header.
func recoverInsertMsgFromHeader(insertMsg *msgstream.InsertMsg, header *message.InsertMessageHeader, timetick uint64) (msgstream.TsMsg, error) {
if insertMsg.GetCollectionID() != header.GetCollectionId() {
panic("unreachable code, collection id is not equal")
}
// header promise a batch insert on vchannel in future, so header has multiple partition.
var assignment *message.PartitionSegmentAssignment
for _, p := range header.Partitions {
if p.GetPartitionId() == insertMsg.GetPartitionID() {
assignment = p
break
}
}
if assignment.GetSegmentAssignment().GetSegmentId() == 0 {
panic("unreachable code, partition id is not exist")
}
insertMsg.SegmentID = assignment.GetSegmentAssignment().GetSegmentId()
// timetick should has been assign at streaming node.
// so overwrite the timetick on insertRequest.
timestamps := make([]uint64, insertMsg.GetNumRows())
for i := 0; i < len(timestamps); i++ {
timestamps[i] = timetick
}
insertMsg.Timestamps = timestamps
return insertMsg, nil
}

34
pkg/streaming/util/message/adaptor/message_id.go Normal file
View File

@ -0,0 +1,34 @@
package adaptor
import (
"github.com/apache/pulsar-client-go/pulsar"
"github.com/milvus-io/milvus/pkg/mq/common"
"github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/server"
mqpulsar "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper/pulsar"
"github.com/milvus-io/milvus/pkg/streaming/util/message"
msgpulsar "github.com/milvus-io/milvus/pkg/streaming/walimpls/impls/pulsar"
"github.com/milvus-io/milvus/pkg/streaming/walimpls/impls/rmq"
)
// MustGetMQWrapperIDFromMessage converts message.MessageID to common.MessageID
// TODO: should be removed in future after common.MessageID is removed
func MustGetMQWrapperIDFromMessage(messageID message.MessageID) common.MessageID {
if id, ok := messageID.(interface{ PulsarID() pulsar.MessageID }); ok {
return mqpulsar.NewPulsarID(id.PulsarID())
} else if id, ok := messageID.(interface{ RmqID() int64 }); ok {
return &server.RmqID{MessageID: id.RmqID()}
}
panic("unsupported now")
}
// MustGetMessageIDFromMQWrapperID converts common.MessageID to message.MessageID
// TODO: should be removed in future after common.MessageID is removed
func MustGetMessageIDFromMQWrapperID(commonMessageID common.MessageID) message.MessageID {
if id, ok := commonMessageID.(interface{ PulsarID() pulsar.MessageID }); ok {
return msgpulsar.NewPulsarID(id.PulsarID())
} else if id, ok := commonMessageID.(*server.RmqID); ok {
return rmq.NewRmqID(id.MessageID)
}
return nil
}

20
pkg/streaming/util/message/adaptor/message_id_test.go Normal file
View File

@ -0,0 +1,20 @@
package adaptor
import (
"testing"
"github.com/apache/pulsar-client-go/pulsar"
"github.com/stretchr/testify/assert"
msgpulsar "github.com/milvus-io/milvus/pkg/streaming/walimpls/impls/pulsar"
"github.com/milvus-io/milvus/pkg/streaming/walimpls/impls/rmq"
)
func TestIDCoversion(t *testing.T) {
id := MustGetMessageIDFromMQWrapperID(MustGetMQWrapperIDFromMessage(rmq.NewRmqID(1)))
assert.True(t, id.EQ(rmq.NewRmqID(1)))
msgID := pulsar.EarliestMessageID()
id = MustGetMessageIDFromMQWrapperID(MustGetMQWrapperIDFromMessage(msgpulsar.NewPulsarID(msgID)))
assert.True(t, id.EQ(msgpulsar.NewPulsarID(msgID)))
}

72
pkg/streaming/util/message/adaptor/message_test.go Normal file
View File

@ -0,0 +1,72 @@
package adaptor
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus/pkg/mq/msgstream"
"github.com/milvus-io/milvus/pkg/streaming/util/message"
"github.com/milvus-io/milvus/pkg/streaming/walimpls/impls/rmq"
)
func TestNewMsgPackFromInsertMessage(t *testing.T) {
id := rmq.NewRmqID(1)
fieldCount := map[int64]int{
3: 1000,
4: 2000,
5: 3000,
6: 5000,
}
tt := uint64(time.Now().UnixNano())
immutableMessages := make([]message.ImmutableMessage, 0, len(fieldCount))
for segmentID, rowNum := range fieldCount {
insertMsg := message.CreateTestInsertMessage(t, segmentID, rowNum, tt, id)
immutableMessage := insertMsg.IntoImmutableMessage(id)
immutableMessages = append(immutableMessages, immutableMessage)
}
pack, err := NewMsgPackFromMessage(immutableMessages...)
assert.NoError(t, err)
assert.NotNil(t, pack)
assert.Equal(t, tt, pack.BeginTs)
assert.Equal(t, tt, pack.EndTs)
assert.Len(t, pack.Msgs, len(fieldCount))
for _, msg := range pack.Msgs {
insertMsg := msg.(*msgstream.InsertMsg)
rowNum, ok := fieldCount[insertMsg.GetSegmentID()]
assert.True(t, ok)
assert.Len(t, insertMsg.Timestamps, rowNum)
assert.Len(t, insertMsg.RowIDs, rowNum)
assert.Len(t, insertMsg.FieldsData, 2)
for _, fieldData := range insertMsg.FieldsData {
if data := fieldData.GetScalars().GetBoolData(); data != nil {
assert.Len(t, data.Data, rowNum)
} else if data := fieldData.GetScalars().GetIntData(); data != nil {
assert.Len(t, data.Data, rowNum)
}
}
for _, ts := range insertMsg.Timestamps {
assert.Equal(t, ts, tt)
}
}
}
func TestNewMsgPackFromCreateCollectionMessage(t *testing.T) {
id := rmq.NewRmqID(1)
tt := uint64(time.Now().UnixNano())
msg := message.CreateTestCreateCollectionMessage(t, 1, tt, id)
immutableMessage := msg.IntoImmutableMessage(id)
pack, err := NewMsgPackFromMessage(immutableMessage)
assert.NoError(t, err)
assert.NotNil(t, pack)
assert.Equal(t, tt, pack.BeginTs)
assert.Equal(t, tt, pack.EndTs)
}

104
pkg/streaming/util/message/builder.go
View File

@ -1,5 +1,24 @@
package message
import (
"fmt"
"reflect"
"github.com/cockroachdb/errors"
"github.com/golang/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
)
// NewMutableMessage creates a new mutable message.
// Only used at server side for streamingnode internal service, don't use it at client side.
func NewMutableMessage(payload []byte, properties map[string]string) MutableMessage {
return &messageImpl{
payload: payload,
properties: properties,
}
}
// NewImmutableMessage creates a new immutable message.
func NewImmutableMesasge(
id MessageID,
@ -15,43 +34,69 @@ func NewImmutableMesasge(
}
}
// NewMutableMessageBuilder creates a new builder.
// Should only used at client side.
func NewMutableMessageBuilder() *MutableMesasgeBuilder {
return &MutableMesasgeBuilder{
payload: nil,
properties: make(propertiesImpl),
// List all type-safe mutable message builders here.
var (
NewTimeTickMessageBuilderV1 = createNewMessageBuilderV1[*TimeTickMessageHeader, *msgpb.TimeTickMsg]()
NewInsertMessageBuilderV1 = createNewMessageBuilderV1[*InsertMessageHeader, *msgpb.InsertRequest]()
NewDeleteMessageBuilderV1 = createNewMessageBuilderV1[*DeleteMessageHeader, *msgpb.DeleteRequest]()
NewCreateCollectionMessageBuilderV1 = createNewMessageBuilderV1[*CreateCollectionMessageHeader, *msgpb.CreateCollectionRequest]()
NewDropCollectionMessageBuilderV1 = createNewMessageBuilderV1[*DropCollectionMessageHeader, *msgpb.DropCollectionRequest]()
NewCreatePartitionMessageBuilderV1 = createNewMessageBuilderV1[*CreatePartitionMessageHeader, *msgpb.CreatePartitionRequest]()
NewDropPartitionMessageBuilderV1 = createNewMessageBuilderV1[*DropPartitionMessageHeader, *msgpb.DropPartitionRequest]()
)
// createNewMessageBuilderV1 creates a new message builder with v1 marker.
func createNewMessageBuilderV1[H proto.Message, P proto.Message]() func() *mutableMesasgeBuilder[H, P] {
return func() *mutableMesasgeBuilder[H, P] {
return newMutableMessageBuilder[H, P](VersionV1)
}
}
// MutableMesasgeBuilder is the builder for message.
type MutableMesasgeBuilder struct {
payload []byte
// newMutableMessageBuilder creates a new builder.
// Should only used at client side.
func newMutableMessageBuilder[H proto.Message, P proto.Message](v Version) *mutableMesasgeBuilder[H, P] {
var h H
messageType := mustGetMessageTypeFromMessageHeader(h)
properties := make(propertiesImpl)
properties.Set(messageTypeKey, messageType.marshal())
properties.Set(messageVersion, v.String())
return &mutableMesasgeBuilder[H, P]{
properties: properties,
}
}
// mutableMesasgeBuilder is the builder for message.
type mutableMesasgeBuilder[H proto.Message, P proto.Message] struct {
header H
payload P
properties propertiesImpl
}
func (b *MutableMesasgeBuilder) WithMessageType(t MessageType) *MutableMesasgeBuilder {
b.properties.Set(messageTypeKey, t.marshal())
// WithMessageHeader creates a new builder with determined message type.
func (b *mutableMesasgeBuilder[H, P]) WithMessageHeader(h H) *mutableMesasgeBuilder[H, P] {
b.header = h
return b
}
// WithPayload creates a new builder with message payload.
// The MessageType is required to indicate which message type payload is.
func (b *MutableMesasgeBuilder) WithPayload(payload []byte) *MutableMesasgeBuilder {
b.payload = payload
func (b *mutableMesasgeBuilder[H, P]) WithPayload(p P) *mutableMesasgeBuilder[H, P] {
b.payload = p
return b
}
// WithProperty creates a new builder with message property.
// A key started with '_' is reserved for streaming system, should never used at user of client.
func (b *MutableMesasgeBuilder) WithProperty(key string, val string) *MutableMesasgeBuilder {
func (b *mutableMesasgeBuilder[H, P]) WithProperty(key string, val string) *mutableMesasgeBuilder[H, P] {
if b.properties.Exist(key) {
panic(fmt.Sprintf("message builder already set property field, key = %s", key))
}
b.properties.Set(key, val)
return b
}
// WithProperties creates a new builder with message properties.
// A key started with '_' is reserved for streaming system, should never used at user of client.
func (b *MutableMesasgeBuilder) WithProperties(kvs map[string]string) *MutableMesasgeBuilder {
func (b *mutableMesasgeBuilder[H, P]) WithProperties(kvs map[string]string) *mutableMesasgeBuilder[H, P] {
for key, val := range kvs {
b.properties.Set(key, val)
}
@ -61,17 +106,28 @@ func (b *MutableMesasgeBuilder) WithProperties(kvs map[string]string) *MutableMe
// BuildMutable builds a mutable message.
// Panic if not set payload and message type.
// should only used at client side.
func (b *MutableMesasgeBuilder) BuildMutable() MutableMessage {
if b.payload == nil {
func (b *mutableMesasgeBuilder[H, P]) BuildMutable() (MutableMessage, error) {
// payload and header must be a pointer
if reflect.ValueOf(b.header).IsNil() {
panic("message builder not ready for header field")
}
if reflect.ValueOf(b.payload).IsNil() {
panic("message builder not ready for payload field")
}
if !b.properties.Exist(messageTypeKey) {
panic("message builder not ready for message type field")
// setup header.
sp, err := EncodeProto(b.header)
if err != nil {
return nil, errors.Wrap(err, "failed to encode header")
}
b.properties.Set(messageSpecialiedHeader, sp)
payload, err := proto.Marshal(b.payload)
if err != nil {
return nil, errors.Wrap(err, "failed to marshal payload")
}
// Set message version.
b.properties.Set(messageVersion, VersionV1.String())
return &messageImpl{
payload: b.payload,
payload: payload,
properties: b.properties,
}
}, nil
}

26
pkg/streaming/util/message/encoder.go
View File

@ -1,6 +1,12 @@
package message
import "strconv"
import (
"encoding/base64"
"strconv"
"github.com/golang/protobuf/proto"
"github.com/pkg/errors"
)
const base = 36
@ -23,3 +29,21 @@ func DecodeUint64(value string) (uint64, error) {
func DecodeInt64(value string) (int64, error) {
return strconv.ParseInt(value, base, 64)
}
// EncodeProto encodes proto message to json string.
func EncodeProto(m proto.Message) (string, error) {
result, err := proto.Marshal(m)
if err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(result), nil
}
// DecodeProto
func DecodeProto(data string, m proto.Message) error {
val, err := base64.StdEncoding.DecodeString(data)
if err != nil {
return errors.Wrap(err, "failed to decode base64")
}
return proto.Unmarshal(val, m)
}

27
pkg/streaming/util/message/encoder_test.go Normal file
View File

@ -0,0 +1,27 @@
package message
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestEncoder(t *testing.T) {
result, err := DecodeInt64(EncodeInt64(1))
assert.NoError(t, err)
assert.Equal(t, int64(1), result)
result2, err := DecodeUint64(EncodeUint64(1))
assert.NoError(t, err)
assert.Equal(t, uint64(1), result2)
result3, err := EncodeProto(&InsertMessageHeader{
CollectionId: 1,
})
assert.NoError(t, err)
var result4 InsertMessageHeader
err = DecodeProto(result3, &result4)
assert.NoError(t, err)
assert.Equal(t, result4.CollectionId, int64(1))
}

29
pkg/streaming/util/message/message.go
View File

@ -1,5 +1,7 @@
package message
import "github.com/golang/protobuf/proto"
var (
_ BasicMessage = (*messageImpl)(nil)
_ MutableMessage = (*messageImpl)(nil)
@ -76,3 +78,30 @@ type ImmutableMessage interface {
// MessageID returns the message id of current message.
MessageID() MessageID
}
// specializedMutableMessage is the specialized mutable message interface.
type specializedMutableMessage[H proto.Message] interface {
BasicMessage
// VChannel returns the vchannel of the message.
VChannel() string
// TimeTick returns the time tick of the message.
TimeTick() uint64
// MessageHeader returns the message header.
// Modifications to the returned header will be reflected in the message.
MessageHeader() H
// OverwriteMessageHeader overwrites the message header.
OverwriteMessageHeader(header H)
}
// specializedImmutableMessage is the specialized immutable message interface.
type specializedImmutableMessage[H proto.Message] interface {
ImmutableMessage
// MessageHeader returns the message header.
// Modifications to the returned header will be reflected in the message.
MessageHeader() H
}

21
pkg/streaming/util/message/message_builder_test.go
View File

@ -1,30 +1,35 @@
package message_test
import (
"bytes"
"fmt"
"testing"
"github.com/golang/protobuf/proto"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
"github.com/milvus-io/milvus/pkg/mocks/streaming/util/mock_message"
"github.com/milvus-io/milvus/pkg/streaming/util/message"
)
func TestMessage(t *testing.T) {
b := message.NewMutableMessageBuilder()
mutableMessage := b.
WithMessageType(message.MessageTypeTimeTick).
WithPayload([]byte("payload")).
b := message.NewTimeTickMessageBuilderV1()
mutableMessage, err := b.WithMessageHeader(&message.TimeTickMessageHeader{}).
WithProperties(map[string]string{"key": "value"}).
BuildMutable()
WithPayload(&msgpb.TimeTickMsg{}).BuildMutable()
assert.NoError(t, err)
assert.Equal(t, "payload", string(mutableMessage.Payload()))
payload, err := proto.Marshal(&message.TimeTickMessageHeader{})
assert.NoError(t, err)
assert.True(t, bytes.Equal(payload, mutableMessage.Payload()))
assert.True(t, mutableMessage.Properties().Exist("key"))
v, ok := mutableMessage.Properties().Get("key")
assert.Equal(t, "value", v)
assert.True(t, ok)
assert.Equal(t, message.MessageTypeTimeTick, mutableMessage.MessageType())
assert.Equal(t, 24, mutableMessage.EstimateSize())
assert.Equal(t, 20, mutableMessage.EstimateSize())
mutableMessage.WithTimeTick(123)
v, ok = mutableMessage.Properties().Get("_tt")
assert.True(t, ok)
@ -96,6 +101,6 @@ func TestMessage(t *testing.T) {
})
assert.Panics(t, func() {
message.NewMutableMessageBuilder().BuildMutable()
message.NewTimeTickMessageBuilderV1().BuildMutable()
})
}

38
pkg/streaming/util/message/message_impl.go
View File

@ -69,6 +69,28 @@ func (m *messageImpl) IntoImmutableMessage(id MessageID) ImmutableMessage {
}
}
// TimeTick returns the time tick of current message.
func (m *messageImpl) TimeTick() uint64 {
value, ok := m.properties.Get(messageTimeTick)
if !ok {
panic(fmt.Sprintf("there's a bug in the message codes, timetick lost in properties of message"))
}
tt, err := DecodeUint64(value)
if err != nil {
panic(fmt.Sprintf("there's a bug in the message codes, dirty timetick %s in properties of message", value))
}
return tt
}
// VChannel returns the vchannel of current message.
func (m *messageImpl) VChannel() string {
value, ok := m.properties.Get(messageVChannel)
if !ok {
panic(fmt.Sprintf("there's a bug in the message codes, vchannel lost in properties of message"))
}
return value
}
type immutableMessageImpl struct {
messageImpl
id MessageID
@ -92,6 +114,14 @@ func (m *immutableMessageImpl) TimeTick() uint64 {
return tt
}
func (m *immutableMessageImpl) VChannel() string {
value, ok := m.properties.Get(messageVChannel)
if !ok {
panic(fmt.Sprintf("there's a bug in the message codes, vchannel lost in properties of message, id: %+v", m.id))
}
return value
}
func (m *immutableMessageImpl) LastConfirmedMessageID() MessageID {
value, ok := m.properties.Get(messageLastConfirmed)
if !ok {
@ -108,11 +138,3 @@ func (m *immutableMessageImpl) LastConfirmedMessageID() MessageID {
func (m *immutableMessageImpl) MessageID() MessageID {
return m.id
}
func (m *immutableMessageImpl) VChannel() string {
value, ok := m.properties.Get(messageVChannel)
if !ok {
panic(fmt.Sprintf("there's a bug in the message codes, vchannel lost in properties of message, id: %+v", m.id))
}
return value
}

84
pkg/streaming/util/message/messagepb/message.proto Normal file
View File

@ -0,0 +1,84 @@
syntax = "proto3";
package milvus.proto.message;
option go_package = "github.com/milvus-io/milvus/pkg/streaming/util/message/messagepb";
///
/// Message Payload Definitions
/// Some message payload is defined at msg.proto at milvus-proto for
/// compatibility.
/// 1. InsertRequest
/// 2. DeleteRequest
/// 3. TimeTickRequest
/// 4. CreateCollectionRequest
/// 5. DropCollectionRequest
/// 6. CreatePartitionRequest
/// 7. DropPartitionRequest
///
// FlushMessagePayload is the payload of flush message.
message FlushMessagePayload {
int64 collection_id =
1; // indicate which the collection that segment belong to.
repeated int64 segment_id = 2; // indicate which segment to flush.
}
///
/// Message Header Definitions
/// Used to fast handling at streaming node write ahead.
/// The header should be simple and light enough to be parsed.
/// Do not put too much information in the header if unnecessary.
///
// TimeTickMessageHeader just nothing.
message TimeTickMessageHeader {}
// InsertMessageHeader is the header of insert message.
message InsertMessageHeader {
int64 collection_id = 1;
repeated PartitionSegmentAssignment partitions = 2;
}
// PartitionSegmentAssignment is the segment assignment of a partition.
message PartitionSegmentAssignment {
int64 partition_id = 1;
uint64 rows = 2;
uint64 binary_size = 3;
SegmentAssignment segment_assignment = 4;
}
// SegmentAssignment is the assignment of a segment.
message SegmentAssignment {
int64 segment_id = 1;
}
// DeleteMessageHeader
message DeleteMessageHeader {
int64 collection_id = 1;
}
// FlushMessageHeader just nothing.
message FlushMessageHeader {}
// CreateCollectionMessageHeader is the header of create collection message.
message CreateCollectionMessageHeader {
int64 collection_id = 1;
}
// DropCollectionMessageHeader is the header of drop collection message.
message DropCollectionMessageHeader {
int64 collection_id = 1;
}
// CreatePartitionMessageHeader is the header of create partition message.
message CreatePartitionMessageHeader {
int64 collection_id = 1;
int64 partition_id = 2;
}
// DropPartitionMessageHeader is the header of drop partition message.
message DropPartitionMessageHeader {
int64 collection_id = 1;
int64 partition_id = 2;
}

11
pkg/streaming/util/message/properties.go
View File

@ -2,11 +2,12 @@ package message
const (
// preserved properties
messageVersion = "_v" // message version for compatibility.
messageTypeKey = "_t" // message type key.
messageTimeTick = "_tt" // message time tick.
messageLastConfirmed = "_lc" // message last confirmed message id.
messageVChannel = "_vc" // message virtual channel.
messageVersion = "_v" // message version for compatibility.
messageTypeKey = "_t" // message type key.
messageTimeTick = "_tt" // message time tick.
messageLastConfirmed = "_lc" // message last confirmed message id.
messageVChannel = "_vc" // message virtual channel.
messageSpecialiedHeader = "_sh" // specialized message header.
)
var (

186
pkg/streaming/util/message/specialized_message.go Normal file
View File

@ -0,0 +1,186 @@
package message
import (
"fmt"
"reflect"
"github.com/golang/protobuf/proto"
"github.com/pkg/errors"
"github.com/milvus-io/milvus/pkg/streaming/util/message/messagepb"
)
type (
SegmentAssignment = messagepb.SegmentAssignment
PartitionSegmentAssignment = messagepb.PartitionSegmentAssignment
TimeTickMessageHeader = messagepb.TimeTickMessageHeader
InsertMessageHeader = messagepb.InsertMessageHeader
DeleteMessageHeader = messagepb.DeleteMessageHeader
CreateCollectionMessageHeader = messagepb.CreateCollectionMessageHeader
DropCollectionMessageHeader = messagepb.DropCollectionMessageHeader
CreatePartitionMessageHeader = messagepb.CreatePartitionMessageHeader
DropPartitionMessageHeader = messagepb.DropPartitionMessageHeader
)
// messageTypeMap maps the proto message type to the message type.
var messageTypeMap = map[reflect.Type]MessageType{
reflect.TypeOf(&TimeTickMessageHeader{}): MessageTypeTimeTick,
reflect.TypeOf(&InsertMessageHeader{}): MessageTypeInsert,
reflect.TypeOf(&DeleteMessageHeader{}): MessageTypeDelete,
reflect.TypeOf(&CreateCollectionMessageHeader{}): MessageTypeCreateCollection,
reflect.TypeOf(&DropCollectionMessageHeader{}): MessageTypeDropCollection,
reflect.TypeOf(&CreatePartitionMessageHeader{}): MessageTypeCreatePartition,
reflect.TypeOf(&DropPartitionMessageHeader{}): MessageTypeDropPartition,
}
// List all specialized message types.
type (
MutableTimeTickMessage = specializedMutableMessage[*TimeTickMessageHeader]
MutableInsertMessage = specializedMutableMessage[*InsertMessageHeader]
MutableDeleteMessage = specializedMutableMessage[*DeleteMessageHeader]
MutableCreateCollection = specializedMutableMessage[*CreateCollectionMessageHeader]
MutableDropCollection = specializedMutableMessage[*DropCollectionMessageHeader]
MutableCreatePartition = specializedMutableMessage[*CreatePartitionMessageHeader]
MutableDropPartition = specializedMutableMessage[*DropPartitionMessageHeader]
ImmutableTimeTickMessage = specializedImmutableMessage[*TimeTickMessageHeader]
ImmutableInsertMessage = specializedImmutableMessage[*InsertMessageHeader]
ImmutableDeleteMessage = specializedImmutableMessage[*DeleteMessageHeader]
ImmutableCreateCollection = specializedImmutableMessage[*CreateCollectionMessageHeader]
ImmutableDropCollection = specializedImmutableMessage[*DropCollectionMessageHeader]
ImmutableCreatePartition = specializedImmutableMessage[*CreatePartitionMessageHeader]
ImmutableDropPartition = specializedImmutableMessage[*DropPartitionMessageHeader]
)
// List all as functions for specialized messages.
var (
AsMutableTimeTickMessage = asSpecializedMutableMessage[*TimeTickMessageHeader]
AsMutableInsertMessage = asSpecializedMutableMessage[*InsertMessageHeader]
AsMutableDeleteMessage = asSpecializedMutableMessage[*DeleteMessageHeader]
AsMutableCreateCollection = asSpecializedMutableMessage[*CreateCollectionMessageHeader]
AsMutableDropCollection = asSpecializedMutableMessage[*DropCollectionMessageHeader]
AsMutableCreatePartition = asSpecializedMutableMessage[*CreatePartitionMessageHeader]
AsMutableDropPartition = asSpecializedMutableMessage[*DropPartitionMessageHeader]
AsImmutableTimeTickMessage = asSpecializedImmutableMessage[*TimeTickMessageHeader]
AsImmutableInsertMessage = asSpecializedImmutableMessage[*InsertMessageHeader]
AsImmutableDeleteMessage = asSpecializedImmutableMessage[*DeleteMessageHeader]
AsImmutableCreateCollection = asSpecializedImmutableMessage[*CreateCollectionMessageHeader]
AsImmutableDropCollection = asSpecializedImmutableMessage[*DropCollectionMessageHeader]
AsImmutableCreatePartition = asSpecializedImmutableMessage[*CreatePartitionMessageHeader]
AsImmutableDropPartition = asSpecializedImmutableMessage[*DropPartitionMessageHeader]
)
// asSpecializedMutableMessage converts a MutableMessage to a specialized MutableMessage.
// Return nil, nil if the message is not the target specialized message.
// Return nil, error if the message is the target specialized message but failed to decode the specialized header.
// Return specializedMutableMessage, nil if the message is the target specialized message and successfully decoded the specialized header.
func asSpecializedMutableMessage[H proto.Message](msg MutableMessage) (specializedMutableMessage[H], error) {
underlying := msg.(*messageImpl)
var header H
msgType := mustGetMessageTypeFromMessageHeader(header)
if underlying.MessageType() != msgType {
// The message type do not match the specialized header.
return nil, nil
}
// Get the specialized header from the message.
val, ok := underlying.properties.Get(messageSpecialiedHeader)
if !ok {
return nil, errors.Errorf("lost specialized header, %s", msgType.String())
}
// Decode the specialized header.
// Must be pointer type.
t := reflect.TypeOf(header)
t.Elem()
header = reflect.New(t.Elem()).Interface().(H)
// must be a pointer to a proto message
if err := DecodeProto(val, header); err != nil {
return nil, errors.Wrap(err, "failed to decode specialized header")
}
return &specializedMutableMessageImpl[H]{
header: header,
messageImpl: underlying,
}, nil
}
// asSpecializedImmutableMessage converts a ImmutableMessage to a specialized ImmutableMessage.
// Return nil, nil if the message is not the target specialized message.
// Return nil, error if the message is the target specialized message but failed to decode the specialized header.
// Return asSpecializedImmutableMessage, nil if the message is the target specialized message and successfully decoded the specialized header.
func asSpecializedImmutableMessage[H proto.Message](msg ImmutableMessage) (specializedImmutableMessage[H], error) {
underlying := msg.(*immutableMessageImpl)
var header H
msgType := mustGetMessageTypeFromMessageHeader(header)
if underlying.MessageType() != msgType {
// The message type do not match the specialized header.
return nil, nil
}
// Get the specialized header from the message.
val, ok := underlying.properties.Get(messageSpecialiedHeader)
if !ok {
return nil, errors.Errorf("lost specialized header, %s", msgType.String())
}
// Decode the specialized header.
// Must be pointer type.
t := reflect.TypeOf(header)
t.Elem()
header = reflect.New(t.Elem()).Interface().(H)
// must be a pointer to a proto message
if err := DecodeProto(val, header); err != nil {
return nil, errors.Wrap(err, "failed to decode specialized header")
}
return &specializedImmutableMessageImpl[H]{
header: header,
immutableMessageImpl: underlying,
}, nil
}
// mustGetMessageTypeFromMessageHeader returns the message type of the given message header.
func mustGetMessageTypeFromMessageHeader(msg proto.Message) MessageType {
t := reflect.TypeOf(msg)
mt, ok := messageTypeMap[t]
if !ok {
panic(fmt.Sprintf("unsupported message type of proto header: %s", t.Name()))
}
return mt
}
// specializedMutableMessageImpl is the specialized mutable message implementation.
type specializedMutableMessageImpl[H proto.Message] struct {
header H
*messageImpl
}
// MessageHeader returns the message header.
func (m *specializedMutableMessageImpl[H]) MessageHeader() H {
return m.header
}
// OverwriteMessageHeader overwrites the message header.
func (m *specializedMutableMessageImpl[H]) OverwriteMessageHeader(header H) {
m.header = header
newHeader, err := EncodeProto(m.header)
if err != nil {
panic(fmt.Sprintf("failed to encode insert header, there's a bug, %+v, %s", m.header, err.Error()))
}
m.messageImpl.properties.Set(messageSpecialiedHeader, newHeader)
}
// specializedImmutableMessageImpl is the specialized immmutable message implementation.
type specializedImmutableMessageImpl[H proto.Message] struct {
header H
*immutableMessageImpl
}
// MessageHeader returns the message header.
func (m *specializedImmutableMessageImpl[H]) MessageHeader() H {
return m.header
}

54
pkg/streaming/util/message/specialized_message_test.go Normal file
View File

@ -0,0 +1,54 @@
package message_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
"github.com/milvus-io/milvus/pkg/mocks/streaming/util/mock_message"
"github.com/milvus-io/milvus/pkg/streaming/util/message"
)
func TestAsSpecializedMessage(t *testing.T) {
m, err := message.NewInsertMessageBuilderV1().
WithMessageHeader(&message.InsertMessageHeader{
CollectionId: 1,
Partitions: []*message.PartitionSegmentAssignment{
{
PartitionId: 1,
Rows: 100,
BinarySize: 1000,
},
},
}).
WithPayload(&msgpb.InsertRequest{}).BuildMutable()
assert.NoError(t, err)
insertMsg, err := message.AsMutableInsertMessage(m)
assert.NoError(t, err)
assert.NotNil(t, insertMsg)
assert.Equal(t, int64(1), insertMsg.MessageHeader().CollectionId)
h := insertMsg.MessageHeader()
h.Partitions[0].SegmentAssignment = &message.SegmentAssignment{
SegmentId: 1,
}
insertMsg.OverwriteMessageHeader(h)
createColMsg, err := message.AsMutableCreateCollection(m)
assert.NoError(t, err)
assert.Nil(t, createColMsg)
m2 := m.IntoImmutableMessage(mock_message.NewMockMessageID(t))
insertMsg2, err := message.AsImmutableInsertMessage(m2)
assert.NoError(t, err)
assert.NotNil(t, insertMsg2)
assert.Equal(t, int64(1), insertMsg2.MessageHeader().CollectionId)
assert.Equal(t, insertMsg2.MessageHeader().Partitions[0].SegmentAssignment.SegmentId, int64(1))
createColMsg2, err := message.AsMutableCreateCollection(m)
assert.NoError(t, err)
assert.Nil(t, createColMsg2)
}

148
pkg/streaming/util/message/test_case.go Normal file
View File

@ -0,0 +1,148 @@
//go:build test
// +build test
package message
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
)
func CreateTestInsertMessage(t *testing.T, segmentID int64, totalRows int, timetick uint64, messageID MessageID) MutableMessage {
timestamps := make([]uint64, 0, totalRows)
for i := 0; i < totalRows; i++ {
timestamps = append(timestamps, uint64(0))
}
rowIDs := make([]int64, 0, totalRows)
for i := 0; i < totalRows; i++ {
rowIDs = append(rowIDs, int64(i))
}
intFieldArray := make([]int32, 0, totalRows)
for i := 0; i < totalRows; i++ {
intFieldArray = append(intFieldArray, int32(i))
}
boolFieldArray := make([]bool, 0, totalRows)
for i := 0; i < totalRows; i++ {
boolFieldArray = append(boolFieldArray, i%2 == 0)
}
fieldsData := []*schemapb.FieldData{
{
Type: schemapb.DataType_Int64,
FieldName: "f1",
FieldId: 1,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: intFieldArray,
},
},
},
},
},
{
Type: schemapb.DataType_Bool,
FieldName: "f2",
FieldId: 2,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_BoolData{
BoolData: &schemapb.BoolArray{
Data: boolFieldArray,
},
},
},
},
},
}
msg, err := NewInsertMessageBuilderV1().
WithMessageHeader(&InsertMessageHeader{
CollectionId: 1,
Partitions: []*PartitionSegmentAssignment{
{
PartitionId: 2,
Rows: uint64(totalRows),
BinarySize: 10000,
SegmentAssignment: &SegmentAssignment{SegmentId: segmentID},
},
},
}).
WithPayload(&msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
Timestamp: 100,
SourceID: 1,
},
ShardName: "v1",
DbName: "test_name",
CollectionName: "test_name",
PartitionName: "test_name",
DbID: 1,
CollectionID: 1,
PartitionID: 2,
SegmentID: 0,
Version: msgpb.InsertDataVersion_ColumnBased,
FieldsData: fieldsData,
RowIDs: rowIDs,
Timestamps: timestamps,
NumRows: uint64(totalRows),
}).BuildMutable()
if err != nil {
panic(err)
}
msg.WithVChannel("v1")
msg.WithTimeTick(timetick)
msg.WithLastConfirmed(messageID)
return msg
}
func CreateTestCreateCollectionMessage(t *testing.T, collectionID int64, timetick uint64, messageID MessageID) MutableMessage {
header := &CreateCollectionMessageHeader{
CollectionId: collectionID,
}
payload := &msgpb.CreateCollectionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_CreateCollection,
MsgID: collectionID,
Timestamp: 100,
},
DbName: "db",
CollectionName: "collection",
PartitionName: "partition",
DbID: 1,
CollectionID: collectionID,
}
msg, err := NewCreateCollectionMessageBuilderV1().
WithMessageHeader(header).
WithPayload(payload).
BuildMutable()
assert.NoError(t, err)
msg.WithVChannel("v1")
msg.WithTimeTick(timetick)
msg.WithLastConfirmed(messageID)
return msg
}
// CreateTestEmptyInsertMesage creates an empty insert message for testing
func CreateTestEmptyInsertMesage(msgID int64, extraProperties map[string]string) MutableMessage {
msg, err := NewInsertMessageBuilderV1().
WithMessageHeader(&InsertMessageHeader{}).
WithPayload(&msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: msgID,
},
}).
WithProperties(extraProperties).
BuildMutable()
if err != nil {
panic(err)
}
return msg
}

21
pkg/streaming/walimpls/impls/pulsar/message_id.go
View File

@ -1,7 +1,7 @@
package pulsar
import (
"encoding/hex"
"encoding/base64"
"github.com/apache/pulsar-client-go/pulsar"
"github.com/cockroachdb/errors"
@ -11,6 +11,12 @@ import (
var _ message.MessageID = pulsarID{}
// NewPulsarID creates a new pulsarID
// TODO: remove in future.
func NewPulsarID(id pulsar.MessageID) message.MessageID {
return pulsarID{id}
}
func UnmarshalMessageID(data string) (message.MessageID, error) {
id, err := unmarshalMessageID(data)
if err != nil {
@ -20,9 +26,9 @@ func UnmarshalMessageID(data string) (message.MessageID, error) {
}
func unmarshalMessageID(data string) (pulsarID, error) {
val, err := hex.DecodeString(data)
val, err := base64.StdEncoding.DecodeString(data)
if err != nil {
return pulsarID{nil}, errors.Wrapf(message.ErrInvalidMessageID, "decode pulsar fail when decode hex with err: %s, id: %s", err.Error(), data)
return pulsarID{nil}, errors.Wrapf(message.ErrInvalidMessageID, "decode pulsar fail when decode base64 with err: %s, id: %s", err.Error(), data)
}
msgID, err := pulsar.DeserializeMessageID(val)
if err != nil {
@ -35,6 +41,13 @@ type pulsarID struct {
pulsar.MessageID
}
// PulsarID returns the pulsar message id.
// Don't delete this function until conversion logic removed.
// TODO: remove in future.
func (id pulsarID) PulsarID() pulsar.MessageID {
return id.MessageID
}
func (id pulsarID) WALName() string {
return walName
}
@ -69,5 +82,5 @@ func (id pulsarID) EQ(other message.MessageID) bool {
}
func (id pulsarID) Marshal() string {
return hex.EncodeToString(id.Serialize())
return base64.StdEncoding.EncodeToString(id.Serialize())
}

13
pkg/streaming/walimpls/impls/rmq/message_id.go
View File

@ -8,6 +8,12 @@ import (
var _ message.MessageID = rmqID(0)
// NewRmqID creates a new rmqID.
// TODO: remove in future.
func NewRmqID(id int64) message.MessageID {
return rmqID(id)
}
// UnmarshalMessageID unmarshal the message id.
func UnmarshalMessageID(data string) (message.MessageID, error) {
id, err := unmarshalMessageID(data)
@ -29,6 +35,13 @@ func unmarshalMessageID(data string) (rmqID, error) {
// rmqID is the message id for rmq.
type rmqID int64
// RmqID returns the message id for conversion
// Don't delete this function until conversion logic removed.
// TODO: remove in future.
func (id rmqID) RmqID() int64 {
return int64(id)
}
// WALName returns the name of message id related wal.
func (id rmqID) WALName() string {
return walName

44
pkg/streaming/walimpls/test_framework.go
View File

@ -14,11 +14,11 @@ import (
"testing"
"time"
"github.com/golang/protobuf/proto"
"github.com/remeh/sizedwaitgroup"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
"github.com/milvus-io/milvus/pkg/streaming/util/message"
"github.com/milvus-io/milvus/pkg/streaming/util/options"
"github.com/milvus-io/milvus/pkg/streaming/util/types"
@ -230,26 +230,11 @@ func (f *testOneWALImplsFramework) testAppend(ctx context.Context, w WALImpls) (
defer swg.Done()
// ...rocksmq has a dirty implement of properties,
// without commonpb.MsgHeader, it can not work.
header := commonpb.MsgHeader{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: int64(i),
},
}
payload, err := proto.Marshal(&header)
if err != nil {
panic(err)
}
properties := map[string]string{
"id": fmt.Sprintf("%d", i),
"const": "t",
}
typ := message.MessageTypeUnknown
msg := message.NewMutableMessageBuilder().
WithMessageType(typ).
WithPayload(payload).
WithProperties(properties).
BuildMutable()
msg := message.CreateTestEmptyInsertMesage(int64(i), properties)
id, err := w.Append(ctx, msg)
assert.NoError(f.t, err)
assert.NotNil(f.t, id)
@ -257,27 +242,20 @@ func (f *testOneWALImplsFramework) testAppend(ctx context.Context, w WALImpls) (
}(i)
}
swg.Wait()
// send a final hint message
header := commonpb.MsgHeader{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: int64(f.messageCount - 1),
},
}
payload, err := proto.Marshal(&header)
if err != nil {
panic(err)
}
properties := map[string]string{
"id": fmt.Sprintf("%d", f.messageCount-1),
"const": "t",
"term": strconv.FormatInt(int64(f.term), 10),
}
msg := message.NewMutableMessageBuilder().
WithPayload(payload).
WithProperties(properties).
WithMessageType(message.MessageTypeTimeTick).
BuildMutable()
msg, err := message.NewTimeTickMessageBuilderV1().WithMessageHeader(&message.TimeTickMessageHeader{}).WithPayload(&msgpb.TimeTickMsg{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_TimeTick,
MsgID: int64(f.messageCount - 1),
},
}).WithProperties(properties).BuildMutable()
assert.NoError(f.t, err)
id, err := w.Append(ctx, msg)
assert.NoError(f.t, err)
ids[f.messageCount-1] = msg.IntoImmutableMessage(id)

55
pkg/util/typeutil/multipart_queue.go Normal file
View File

@ -0,0 +1,55 @@
package typeutil
// NewMultipartQueue create a new multi-part queue.
func NewMultipartQueue[T any]() *MultipartQueue[T] {
return &MultipartQueue[T]{
pendings: make([][]T, 0),
cnt: 0,
}
}
// MultipartQueue is a multi-part queue.
type MultipartQueue[T any] struct {
pendings [][]T
cnt int
}
// Len return the queue size.
func (pq *MultipartQueue[T]) Len() int {
return pq.cnt
}
// AddOne add a message as pending one
func (pq *MultipartQueue[T]) AddOne(msg T) {
pq.Add([]T{msg})
}
// Add add a slice of message as pending one
func (pq *MultipartQueue[T]) Add(msgs []T) {
if len(msgs) == 0 {
return
}
pq.pendings = append(pq.pendings, msgs)
pq.cnt += len(msgs)
}
// Next return the next message in pending queue.
func (pq *MultipartQueue[T]) Next() T {
if len(pq.pendings) != 0 && len(pq.pendings[0]) != 0 {
return pq.pendings[0][0]
}
var val T
return val
}
// UnsafeAdvance do a advance without check.
// !!! Should only be called `Next` do not return nil.
func (pq *MultipartQueue[T]) UnsafeAdvance() {
if len(pq.pendings[0]) == 1 {
pq.pendings = pq.pendings[1:]
pq.cnt--
return
}
pq.pendings[0] = pq.pendings[0][1:]
pq.cnt--
}

20
pkg/util/typeutil/multipart_queue_test.go Normal file
View File

@ -0,0 +1,20 @@
package typeutil
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestMultipartQueue(t *testing.T) {
q := NewMultipartQueue[int]()
for i := 0; i < 100; i++ {
q.AddOne(i)
assert.Equal(t, i+1, q.Len())
}
for i := 100; i > 0; i-- {
assert.NotNil(t, q.Next())
q.UnsafeAdvance()
assert.Equal(t, i-1, q.Len())
}
}

7
scripts/generate_proto.sh
View File

@ -89,3 +89,10 @@ ${protoc_opt} --cpp_out=$CPP_SRC_DIR/src/pb cgo_msg.proto|| { echo 'generate cgo
${protoc_opt} --cpp_out=$CPP_SRC_DIR/src/pb plan.proto|| { echo 'generate plan.proto failed'; exit 1; }
popd
pushd $ROOT_DIR/pkg/streaming/util/message/messagepb
# streaming node message protobuf
${PROTOC_BIN} --proto_path=. --go_out=plugins=grpc,paths=source_relative:. message.proto
popd