milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove AddTenant from meta table (#16717) 

Signed-off-by: yun.zhang <yun.zhang@zilliz.com>
pull/16696/head
jaime 2022-05-05 09:23:51 +08:00 committed by GitHub
parent 0b5a593d69
commit c25b337c36
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 63 additions and 222 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
docs/developer_guides/chap06_root_coordinator.md
View File

@ -554,17 +554,6 @@ In most cases, a data definition task need to
###### 6.6.1 Meta ###### 6.6.1 Meta
- Tenant Meta
```protobuf
message TenantMeta {
uint64 id = 1;
uint64 num_query_nodes = 2;
repeated string insert_channel_names = 3;
string query_channel_name = 4;
}
```
- Proxy Meta - Proxy Meta
```protobuf ```protobuf
@ -621,7 +610,6 @@ message SegmentIndexInfo {
###### 6.6.2 KV pairs in EtcdKV ###### 6.6.2 KV pairs in EtcdKV
```go ```go
"tenant/$tenantId" string -> tenantMetaBlob string
"proxy/$proxyId" string -> proxyMetaBlob string "proxy/$proxyId" string -> proxyMetaBlob string
"collection/$collectionId" string -> collectionInfoBlob string "collection/$collectionId" string -> collectionInfoBlob string
"partition/$collectionId/$partitionId" string -> partitionInfoBlob string "partition/$collectionId/$partitionId" string -> partitionInfoBlob string
@ -631,7 +619,7 @@ message SegmentIndexInfo {
Note that _tenantId_, _proxyId_, _collectionId_, _partitionId_, _indexId_, _segmentId_ are unique strings converted from int64. Note that _tenantId_, _proxyId_, _collectionId_, _partitionId_, _indexId_, _segmentId_ are unique strings converted from int64.
_tenantMetaBlob_, _proxyMetaBlob_, _collectionInfoBlob_, _partitionInfoBlob_, _IndexInfoBlob_, _segmentIndexInfoBlog_ are serialized protos. _proxyMetaBlob_, _collectionInfoBlob_, _partitionInfoBlob_, _IndexInfoBlob_, _segmentIndexInfoBlog_ are serialized protos.
###### 6.6.3 Meta Table ###### 6.6.3 Meta Table
@ -639,7 +627,6 @@ _tenantMetaBlob_, _proxyMetaBlob_, _collectionInfoBlob_, _partitionInfoBlob_, _I
type metaTable struct { type metaTable struct {
txn kv.TxnKV // client of a reliable txnkv service, i.e. etcd client txn kv.TxnKV // client of a reliable txnkv service, i.e. etcd client
snapshot kv.SnapShotKV // client of a reliable snapshotkv service, i.e. etcd client snapshot kv.SnapShotKV // client of a reliable snapshotkv service, i.e. etcd client
tenantID2Meta map[typeutil.UniqueID]pb.TenantMeta // tenant id to tenant meta
proxyID2Meta map[typeutil.UniqueID]pb.ProxyMeta // proxy id to proxy meta proxyID2Meta map[typeutil.UniqueID]pb.ProxyMeta // proxy id to proxy meta
collID2Meta map[typeutil.UniqueID]pb.CollectionInfo // collection_id -> meta collID2Meta map[typeutil.UniqueID]pb.CollectionInfo // collection_id -> meta
collName2ID map[string]typeutil.UniqueID // collection name to collection id collName2ID map[string]typeutil.UniqueID // collection name to collection id
@ -648,14 +635,12 @@ type metaTable struct {
segID2IndexMeta map[typeutil.UniqueID]map[typeutil.UniqueID]pb.SegmentIndexInfo // collection_id/index_id/partition_id/segment_id -> meta segID2IndexMeta map[typeutil.UniqueID]map[typeutil.UniqueID]pb.SegmentIndexInfo // collection_id/index_id/partition_id/segment_id -> meta
indexID2Meta map[typeutil.UniqueID]pb.IndexInfo // collection_id/index_id -> meta indexID2Meta map[typeutil.UniqueID]pb.IndexInfo // collection_id/index_id -> meta
tenantLock sync.RWMutex
proxyLock sync.RWMutex proxyLock sync.RWMutex
ddLock sync.RWMutex ddLock sync.RWMutex
} }
func NewMetaTable(kv kv.SnapShotKV) (*metaTable, error) func NewMetaTable(kv kv.SnapShotKV) (*metaTable, error)
func (mt *metaTable) AddTenant(te *pb.TenantMeta) (typeutil.Timestamp, error)
func (mt *metaTable) AddProxy(po *pb.ProxyMeta) (typeutil.Timestamp, error) func (mt *metaTable) AddProxy(po *pb.ProxyMeta) (typeutil.Timestamp, error)
func (mt *metaTable) AddCollection(coll *pb.CollectionInfo, part *pb.PartitionInfo, idx []*pb.IndexInfo, ddOpStr func(ts typeutil.Timestamp) (string, error)) (typeutil.Timestamp, error) func (mt *metaTable) AddCollection(coll *pb.CollectionInfo, part *pb.PartitionInfo, idx []*pb.IndexInfo, ddOpStr func(ts typeutil.Timestamp) (string, error)) (typeutil.Timestamp, error)
func (mt *metaTable) DeleteCollection(collID typeutil.UniqueID, ddOpStr func(ts typeutil.Timestamp) (string, error)) (typeutil.Timestamp, error) func (mt *metaTable) DeleteCollection(collID typeutil.UniqueID, ddOpStr func(ts typeutil.Timestamp) (string, error)) (typeutil.Timestamp, error)

7
internal/proto/etcd_meta.proto
View File

@ -5,13 +5,6 @@ option go_package="github.com/milvus-io/milvus/internal/proto/etcdpb";
import "common.proto"; import "common.proto";
import "schema.proto"; import "schema.proto";
message TenantMeta {
int64 ID = 1;
int64 num_query_nodes = 2;
repeated string insert_channelIDs = 3;
string query_channelID = 4;
}
message ProxyMeta { message ProxyMeta {
int64 ID = 1; int64 ID = 1;
common.Address address = 2; common.Address address = 2;

172
internal/proto/etcdpb/etcd_meta.pb.go
View File

@ -22,69 +22,6 @@ var _ = math.Inf
// proto package needs to be updated. // proto package needs to be updated.
const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
type TenantMeta struct {
ID int64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"`
NumQueryNodes int64 `protobuf:"varint,2,opt,name=num_query_nodes,json=numQueryNodes,proto3" json:"num_query_nodes,omitempty"`
InsertChannelIDs []string `protobuf:"bytes,3,rep,name=insert_channelIDs,json=insertChannelIDs,proto3" json:"insert_channelIDs,omitempty"`
QueryChannelID string `protobuf:"bytes,4,opt,name=query_channelID,json=queryChannelID,proto3" json:"query_channelID,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *TenantMeta) Reset() { *m = TenantMeta{} }
func (m *TenantMeta) String() string { return proto.CompactTextString(m) }
func (*TenantMeta) ProtoMessage() {}
func (*TenantMeta) Descriptor() ([]byte, []int) {
return fileDescriptor_975d306d62b73e88, []int{0}
}
func (m *TenantMeta) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_TenantMeta.Unmarshal(m, b)
}
func (m *TenantMeta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_TenantMeta.Marshal(b, m, deterministic)
}
func (m *TenantMeta) XXX_Merge(src proto.Message) {
xxx_messageInfo_TenantMeta.Merge(m, src)
}
func (m *TenantMeta) XXX_Size() int {
return xxx_messageInfo_TenantMeta.Size(m)
}
func (m *TenantMeta) XXX_DiscardUnknown() {
xxx_messageInfo_TenantMeta.DiscardUnknown(m)
}
var xxx_messageInfo_TenantMeta proto.InternalMessageInfo
func (m *TenantMeta) GetID() int64 {
if m != nil {
return m.ID
}
return 0
}
func (m *TenantMeta) GetNumQueryNodes() int64 {
if m != nil {
return m.NumQueryNodes
}
return 0
}
func (m *TenantMeta) GetInsertChannelIDs() []string {
if m != nil {
return m.InsertChannelIDs
}
return nil
}
func (m *TenantMeta) GetQueryChannelID() string {
if m != nil {
return m.QueryChannelID
}
return ""
}
type ProxyMeta struct { type ProxyMeta struct {
ID int64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` ID int64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"`
Address *commonpb.Address `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"` Address *commonpb.Address `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"`
@ -98,7 +35,7 @@ func (m *ProxyMeta) Reset() { *m = ProxyMeta{} }
func (m *ProxyMeta) String() string { return proto.CompactTextString(m) } func (m *ProxyMeta) String() string { return proto.CompactTextString(m) }
func (*ProxyMeta) ProtoMessage() {} func (*ProxyMeta) ProtoMessage() {}
func (*ProxyMeta) Descriptor() ([]byte, []int) { func (*ProxyMeta) Descriptor() ([]byte, []int) {
return fileDescriptor_975d306d62b73e88, []int{1} return fileDescriptor_975d306d62b73e88, []int{0}
} }
func (m *ProxyMeta) XXX_Unmarshal(b []byte) error { func (m *ProxyMeta) XXX_Unmarshal(b []byte) error {
@ -153,7 +90,7 @@ func (m *IndexInfo) Reset() { *m = IndexInfo{} }
func (m *IndexInfo) String() string { return proto.CompactTextString(m) } func (m *IndexInfo) String() string { return proto.CompactTextString(m) }
func (*IndexInfo) ProtoMessage() {} func (*IndexInfo) ProtoMessage() {}
func (*IndexInfo) Descriptor() ([]byte, []int) { func (*IndexInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_975d306d62b73e88, []int{2} return fileDescriptor_975d306d62b73e88, []int{1}
} }
func (m *IndexInfo) XXX_Unmarshal(b []byte) error { func (m *IndexInfo) XXX_Unmarshal(b []byte) error {
@ -207,7 +144,7 @@ func (m *FieldIndexInfo) Reset() { *m = FieldIndexInfo{} }
func (m *FieldIndexInfo) String() string { return proto.CompactTextString(m) } func (m *FieldIndexInfo) String() string { return proto.CompactTextString(m) }
func (*FieldIndexInfo) ProtoMessage() {} func (*FieldIndexInfo) ProtoMessage() {}
func (*FieldIndexInfo) Descriptor() ([]byte, []int) { func (*FieldIndexInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_975d306d62b73e88, []int{3} return fileDescriptor_975d306d62b73e88, []int{2}
} }
func (m *FieldIndexInfo) XXX_Unmarshal(b []byte) error { func (m *FieldIndexInfo) XXX_Unmarshal(b []byte) error {
@ -264,7 +201,7 @@ func (m *CollectionInfo) Reset() { *m = CollectionInfo{} }
func (m *CollectionInfo) String() string { return proto.CompactTextString(m) } func (m *CollectionInfo) String() string { return proto.CompactTextString(m) }
func (*CollectionInfo) ProtoMessage() {} func (*CollectionInfo) ProtoMessage() {}
func (*CollectionInfo) Descriptor() ([]byte, []int) { func (*CollectionInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_975d306d62b73e88, []int{4} return fileDescriptor_975d306d62b73e88, []int{3}
} }
func (m *CollectionInfo) XXX_Unmarshal(b []byte) error { func (m *CollectionInfo) XXX_Unmarshal(b []byte) error {
@ -386,7 +323,7 @@ func (m *SegmentIndexInfo) Reset() { *m = SegmentIndexInfo{} }
func (m *SegmentIndexInfo) String() string { return proto.CompactTextString(m) } func (m *SegmentIndexInfo) String() string { return proto.CompactTextString(m) }
func (*SegmentIndexInfo) ProtoMessage() {} func (*SegmentIndexInfo) ProtoMessage() {}
func (*SegmentIndexInfo) Descriptor() ([]byte, []int) { func (*SegmentIndexInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_975d306d62b73e88, []int{5} return fileDescriptor_975d306d62b73e88, []int{4}
} }
func (m *SegmentIndexInfo) XXX_Unmarshal(b []byte) error { func (m *SegmentIndexInfo) XXX_Unmarshal(b []byte) error {
@ -472,7 +409,7 @@ func (m *CollectionMeta) Reset() { *m = CollectionMeta{} }
func (m *CollectionMeta) String() string { return proto.CompactTextString(m) } func (m *CollectionMeta) String() string { return proto.CompactTextString(m) }
func (*CollectionMeta) ProtoMessage() {} func (*CollectionMeta) ProtoMessage() {}
func (*CollectionMeta) Descriptor() ([]byte, []int) { func (*CollectionMeta) Descriptor() ([]byte, []int) {
return fileDescriptor_975d306d62b73e88, []int{6} return fileDescriptor_975d306d62b73e88, []int{5}
} }
func (m *CollectionMeta) XXX_Unmarshal(b []byte) error { func (m *CollectionMeta) XXX_Unmarshal(b []byte) error {
@ -536,7 +473,6 @@ func (m *CollectionMeta) GetPartitionIDs() []int64 {
} }
func init() { func init() {
proto.RegisterType((*TenantMeta)(nil), "milvus.proto.etcd.TenantMeta")
proto.RegisterType((*ProxyMeta)(nil), "milvus.proto.etcd.ProxyMeta") proto.RegisterType((*ProxyMeta)(nil), "milvus.proto.etcd.ProxyMeta")
proto.RegisterType((*IndexInfo)(nil), "milvus.proto.etcd.IndexInfo") proto.RegisterType((*IndexInfo)(nil), "milvus.proto.etcd.IndexInfo")
proto.RegisterType((*FieldIndexInfo)(nil), "milvus.proto.etcd.FieldIndexInfo") proto.RegisterType((*FieldIndexInfo)(nil), "milvus.proto.etcd.FieldIndexInfo")
@ -548,54 +484,50 @@ func init() {
func init() { proto.RegisterFile("etcd_meta.proto", fileDescriptor_975d306d62b73e88) } func init() { proto.RegisterFile("etcd_meta.proto", fileDescriptor_975d306d62b73e88) }
var fileDescriptor_975d306d62b73e88 = []byte{ var fileDescriptor_975d306d62b73e88 = []byte{
// 777 bytes of a gzipped FileDescriptorProto // 714 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x54, 0x4d, 0x6f, 0xeb, 0x44, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x54, 0xcd, 0x6a, 0xdb, 0x4c,
0x14, 0x95, 0x9b, 0x34, 0x79, 0xbe, 0x71, 0xd3, 0x76, 0xf8, 0xd0, 0xa8, 0x2a, 0xe0, 0x67, 0xe9, 0x14, 0x45, 0x91, 0x63, 0x47, 0xd7, 0x8a, 0x93, 0xcc, 0xf7, 0xb5, 0x0c, 0x21, 0x6d, 0x15, 0x41,
0x3d, 0x2c, 0x21, 0x5a, 0xd1, 0x87, 0xd8, 0x21, 0x01, 0xb5, 0x9e, 0x14, 0x01, 0x55, 0x99, 0x57, 0x8a, 0xa0, 0xd4, 0xa6, 0x49, 0xe9, 0xae, 0xd0, 0xd6, 0x22, 0x60, 0xda, 0x06, 0x33, 0x09, 0x5d,
0xb1, 0x60, 0x63, 0x4d, 0xec, 0xdb, 0x66, 0x24, 0xcf, 0x38, 0x78, 0xc6, 0xd5, 0xcb, 0x8e, 0x35, 0x74, 0x23, 0xc6, 0xd2, 0xd8, 0x1e, 0xd0, 0x8f, 0xd1, 0x8c, 0x42, 0xbc, 0xeb, 0xba, 0x2f, 0xda,
0x3f, 0x81, 0x3f, 0xc8, 0x82, 0xff, 0x80, 0x90, 0x67, 0x6c, 0xe7, 0xa3, 0xe9, 0x92, 0x5d, 0xee, 0x45, 0xdf, 0xa1, 0x14, 0xcd, 0x48, 0xf2, 0x4f, 0x92, 0x65, 0x77, 0xbe, 0x67, 0xee, 0xb9, 0xba,
0xb9, 0x1f, 0xbe, 0xf7, 0xcc, 0x39, 0x81, 0x63, 0x34, 0x59, 0x9e, 0x4a, 0x34, 0xfc, 0x62, 0x59, 0x73, 0xe6, 0x1c, 0xc3, 0x01, 0x93, 0x61, 0x14, 0x24, 0x4c, 0xd2, 0xfe, 0x22, 0xcf, 0x64, 0x86,
0x95, 0xa6, 0x24, 0xa7, 0x52, 0x14, 0x8f, 0xb5, 0x76, 0xd1, 0x45, 0x93, 0x3d, 0x0b, 0xb2, 0x52, 0x8e, 0x12, 0x1e, 0xdf, 0x16, 0x42, 0x57, 0xfd, 0xf2, 0xf4, 0xd8, 0x0e, 0xb3, 0x24, 0xc9, 0x52,
0xca, 0x52, 0x39, 0xe8, 0x2c, 0xd0, 0xd9, 0x02, 0x65, 0x5b, 0x1e, 0xfd, 0xe5, 0x01, 0xdc, 0xa1, 0x0d, 0x1d, 0xdb, 0x22, 0x9c, 0xb3, 0xa4, 0x6a, 0x77, 0x7f, 0x18, 0x60, 0x8d, 0xf3, 0xec, 0x6e,
0xe2, 0xca, 0xfc, 0x8c, 0x86, 0x93, 0x29, 0x1c, 0xcc, 0x12, 0xea, 0x85, 0x5e, 0x3c, 0x60, 0x07, 0xf9, 0x95, 0x49, 0x8a, 0x7a, 0xb0, 0x33, 0xf2, 0xb1, 0xe1, 0x18, 0x9e, 0x49, 0x76, 0x46, 0x3e,
0xb3, 0x84, 0xbc, 0x86, 0x63, 0x55, 0xcb, 0xf4, 0xf7, 0x1a, 0xab, 0x55, 0xaa, 0xca, 0x1c, 0x35, 0x7a, 0x07, 0x1d, 0x1a, 0x45, 0x39, 0x13, 0x02, 0xef, 0x38, 0x86, 0xd7, 0x3d, 0x3f, 0xe9, 0x6f,
0x3d, 0xb0, 0xc9, 0x23, 0x55, 0xcb, 0x5f, 0x1a, 0xf4, 0xa6, 0x01, 0xc9, 0x17, 0x70, 0x2a, 0x94, 0x8c, 0xaf, 0x06, 0x7f, 0xd4, 0x3d, 0xa4, 0x6e, 0x46, 0xaf, 0xe0, 0x28, 0x67, 0xa2, 0x88, 0x65,
0xc6, 0xca, 0xa4, 0xd9, 0x82, 0x2b, 0x85, 0xc5, 0x2c, 0xd1, 0x74, 0x10, 0x0e, 0x62, 0x9f, 0x9d, 0x10, 0xce, 0x69, 0x9a, 0xb2, 0x78, 0xe4, 0x0b, 0x6c, 0x3a, 0xa6, 0x67, 0x91, 0x43, 0x7d, 0x30,
0xb8, 0xc4, 0x75, 0x8f, 0x93, 0xcf, 0xe1, 0xd8, 0x0d, 0xec, 0x6b, 0xe9, 0x30, 0xf4, 0x62, 0x9f, 0x6c, 0x70, 0xf7, 0xa7, 0x01, 0xd6, 0x28, 0x8d, 0xd8, 0xdd, 0x28, 0x9d, 0x66, 0xe8, 0x19, 0x00,
0x4d, 0x2d, 0xdc, 0x57, 0x46, 0x7f, 0x78, 0xe0, 0xdf, 0x56, 0xe5, 0xfb, 0xd5, 0xde, 0xdd, 0xbe, 0x2f, 0x8b, 0x20, 0xa5, 0x09, 0x53, 0xab, 0x58, 0xc4, 0x52, 0xc8, 0x15, 0x4d, 0x18, 0xc2, 0xd0,
0x81, 0x31, 0xcf, 0xf3, 0x0a, 0xb5, 0xdb, 0x69, 0x72, 0x75, 0x7e, 0xb1, 0x75, 0x7b, 0x7b, 0xf5, 0x51, 0xc5, 0xc8, 0x57, 0x1b, 0x99, 0xa4, 0x2e, 0x91, 0x0f, 0xb6, 0x26, 0x2e, 0x68, 0x4e, 0x13,
0xf7, 0xae, 0x86, 0x75, 0xc5, 0xcd, 0xae, 0x15, 0xea, 0xba, 0xd8, 0xb7, 0xab, 0x4b, 0xac, 0x77, 0xfd, 0xb9, 0xee, 0xf9, 0xe9, 0x83, 0x0b, 0x7f, 0x66, 0xcb, 0x6f, 0x34, 0x2e, 0xd8, 0x98, 0xf2,
0x8d, 0xfe, 0xf4, 0xc0, 0x9f, 0xa9, 0x1c, 0xdf, 0xcf, 0xd4, 0x7d, 0x49, 0x3e, 0x01, 0x10, 0x4d, 0x9c, 0x74, 0x15, 0x6d, 0xac, 0x58, 0xae, 0x0f, 0xbd, 0x4b, 0xce, 0xe2, 0x68, 0xb5, 0x10, 0x86,
0x90, 0x2a, 0x2e, 0xd1, 0xae, 0xe2, 0x33, 0xdf, 0x22, 0x37, 0x5c, 0x22, 0xa1, 0x30, 0xb6, 0xc1, 0xce, 0x94, 0xc7, 0x2c, 0x6a, 0x84, 0xa9, 0xcb, 0xc7, 0x77, 0x71, 0xff, 0xb4, 0xa0, 0x37, 0xcc,
0x2c, 0x69, 0x59, 0xea, 0x42, 0x92, 0x40, 0xe0, 0x1a, 0x97, 0xbc, 0xe2, 0xd2, 0x7d, 0x6e, 0x72, 0xe2, 0x98, 0x85, 0x92, 0x67, 0xa9, 0x1a, 0xb3, 0x2d, 0xed, 0x7b, 0x68, 0xeb, 0x87, 0xa8, 0x94,
0xf5, 0x72, 0xef, 0xc2, 0x3f, 0xe2, 0xea, 0x57, 0x5e, 0xd4, 0x78, 0xcb, 0x45, 0xc5, 0x26, 0xb6, 0x3d, 0xdb, 0x5c, 0xb4, 0x7a, 0xa4, 0xd5, 0x90, 0x6b, 0x05, 0x90, 0x8a, 0x84, 0x5e, 0x40, 0x37,
0xed, 0xd6, 0x76, 0x45, 0x09, 0x4c, 0xdf, 0x0a, 0x2c, 0xf2, 0xf5, 0x42, 0x14, 0xc6, 0xf7, 0xa2, 0xcc, 0x19, 0x95, 0x2c, 0x90, 0x3c, 0x61, 0xd8, 0x74, 0x0c, 0xaf, 0x45, 0x40, 0x43, 0x37, 0x3c,
0xc0, 0xbc, 0x27, 0xa6, 0x0b, 0x9f, 0xdf, 0x25, 0xfa, 0x77, 0x08, 0xd3, 0xeb, 0xb2, 0x28, 0x30, 0x61, 0xc8, 0x05, 0x7b, 0x41, 0x73, 0xc9, 0xd5, 0x02, 0xbe, 0xc0, 0x2d, 0xc7, 0xf4, 0x4c, 0xb2,
0x33, 0xa2, 0x54, 0x76, 0xcc, 0x2e, 0xb5, 0xdf, 0xc2, 0xc8, 0xa9, 0xa4, 0x65, 0xf6, 0xd5, 0xf6, 0x81, 0xa1, 0x97, 0xd0, 0x6b, 0xea, 0x52, 0x5d, 0x81, 0x77, 0xd5, 0x1b, 0x6d, 0xa1, 0xe8, 0x12,
0xa2, 0xad, 0x82, 0xd6, 0x43, 0xde, 0x59, 0x80, 0xb5, 0x4d, 0xe4, 0x33, 0x98, 0x64, 0x15, 0x72, 0xf6, 0xa7, 0xa5, 0x28, 0x81, 0xba, 0x1f, 0x13, 0xb8, 0xfd, 0x90, 0xb6, 0xa5, 0xd7, 0xfa, 0x9b,
0x83, 0xa9, 0x11, 0x12, 0xe9, 0x20, 0xf4, 0xe2, 0x21, 0x03, 0x07, 0xdd, 0x09, 0x89, 0x24, 0x82, 0xe2, 0x11, 0x7b, 0xda, 0xd4, 0x4c, 0xa0, 0x73, 0x78, 0x72, 0xcb, 0x73, 0x59, 0xd0, 0xb8, 0xf6,
0x60, 0xc9, 0x2b, 0x23, 0xec, 0x02, 0x89, 0xa6, 0xc3, 0x70, 0x10, 0x0f, 0xd8, 0x16, 0x46, 0x5e, 0x85, 0x7a, 0x65, 0x81, 0x3b, 0xea, 0xb3, 0xff, 0x55, 0x87, 0x95, 0x37, 0xf4, 0xb7, 0xdf, 0xc2,
0xc3, 0xb4, 0x8f, 0x1b, 0x76, 0x35, 0x3d, 0xb4, 0x6f, 0xb4, 0x83, 0x92, 0xb7, 0x70, 0x74, 0xdf, 0xd3, 0xc5, 0x7c, 0x29, 0x78, 0x78, 0x8f, 0xb4, 0xa7, 0x48, 0xff, 0xd7, 0xa7, 0x1b, 0xac, 0x0f,
0x90, 0x92, 0xda, 0xfb, 0x50, 0xd3, 0xd1, 0x3e, 0x6e, 0x1b, 0x23, 0x5c, 0x6c, 0x93, 0xc7, 0x82, 0x70, 0xd2, 0xdc, 0x21, 0xd0, 0xaa, 0x44, 0x4a, 0x29, 0x21, 0x69, 0xb2, 0x10, 0xd8, 0x72, 0x4c,
0xfb, 0x3e, 0x46, 0x4d, 0xae, 0xe0, 0xa3, 0x47, 0x51, 0x99, 0x9a, 0x17, 0x9d, 0x2e, 0xec, 0x2b, 0xaf, 0x45, 0x8e, 0x9b, 0x9e, 0xa1, 0x6e, 0xb9, 0x69, 0x3a, 0x4a, 0x1f, 0x8a, 0x39, 0xcd, 0x23,
0x6b, 0x3a, 0xb6, 0x9f, 0xfd, 0xa0, 0x4d, 0xb6, 0xda, 0x70, 0xdf, 0xfe, 0x1a, 0x3e, 0x5e, 0x2e, 0x11, 0xa4, 0x45, 0x82, 0xc1, 0x31, 0xbc, 0x5d, 0x62, 0x69, 0xe4, 0xaa, 0x48, 0xd0, 0x08, 0x0e,
0x56, 0x5a, 0x64, 0x4f, 0x9a, 0x5e, 0xd8, 0xa6, 0x0f, 0xbb, 0xec, 0x56, 0xd7, 0x77, 0x70, 0xde, 0x84, 0xa4, 0xb9, 0x0c, 0x16, 0x99, 0x50, 0x13, 0x04, 0xee, 0x2a, 0x51, 0x9c, 0xc7, 0x0c, 0xe7,
0xdf, 0x90, 0x3a, 0x56, 0x72, 0xcb, 0x94, 0x36, 0x5c, 0x2e, 0x35, 0xf5, 0xc3, 0x41, 0x3c, 0x64, 0x53, 0x49, 0x95, 0xdf, 0x7a, 0x8a, 0x38, 0xae, 0x79, 0x88, 0xc0, 0x51, 0x98, 0xa5, 0x82, 0x0b,
0x67, 0x7d, 0xcd, 0xb5, 0x2b, 0xb9, 0xeb, 0x2b, 0x1a, 0x1d, 0xea, 0x05, 0xaf, 0x72, 0x9d, 0xaa, 0xc9, 0xd2, 0x70, 0x19, 0xc4, 0xec, 0x96, 0xc5, 0xd8, 0x76, 0x0c, 0xaf, 0xb7, 0x6d, 0x8a, 0x6a,
0x5a, 0x52, 0x08, 0xbd, 0xf8, 0x90, 0xf9, 0x0e, 0xb9, 0xa9, 0x25, 0x99, 0xc1, 0xb1, 0x36, 0xbc, 0xd8, 0x70, 0xd5, 0xfd, 0xa5, 0x6c, 0x26, 0x87, 0xe1, 0x16, 0xe2, 0xfe, 0x32, 0xe0, 0xf0, 0x9a,
0x32, 0xe9, 0xb2, 0xd4, 0x76, 0x82, 0xa6, 0x13, 0x4b, 0x4a, 0xf8, 0x9c, 0xe0, 0x12, 0x6e, 0xb8, 0xcd, 0x12, 0x96, 0xca, 0x95, 0x93, 0x5d, 0xb0, 0xc3, 0x95, 0x29, 0x6b, 0x33, 0x6e, 0x60, 0xc8,
0xd5, 0xdb, 0xd4, 0x36, 0xde, 0x76, 0x7d, 0x84, 0xc1, 0x69, 0x56, 0x2a, 0x2d, 0xb4, 0x41, 0x95, 0x81, 0xee, 0x9a, 0x45, 0x2a, 0x5f, 0xaf, 0x43, 0xe8, 0x04, 0x2c, 0x51, 0x4d, 0xf6, 0x95, 0xef,
0xad, 0xd2, 0x02, 0x1f, 0xb1, 0xa0, 0x41, 0xe8, 0xc5, 0xd3, 0x5d, 0x51, 0xb4, 0xc3, 0xae, 0xd7, 0x4c, 0xb2, 0x02, 0x74, 0x5a, 0xca, 0x27, 0xf7, 0x71, 0xab, 0x4e, 0x8b, 0x2a, 0xd7, 0xd3, 0xb2,
0xd5, 0x3f, 0x35, 0xc5, 0xec, 0x24, 0xdb, 0x41, 0xa2, 0xbf, 0x3d, 0x38, 0x79, 0x87, 0x0f, 0x12, 0xbb, 0x99, 0x5c, 0x0c, 0x9d, 0x49, 0xc1, 0x15, 0xa7, 0xad, 0x4f, 0xaa, 0x12, 0x9d, 0x82, 0xcd,
0x95, 0x59, 0x2b, 0x39, 0x82, 0x20, 0x5b, 0x8b, 0xb2, 0x13, 0xe3, 0x16, 0x46, 0x42, 0x98, 0x6c, 0x52, 0x3a, 0x89, 0x99, 0x76, 0x1e, 0xee, 0x38, 0x86, 0xb7, 0x47, 0xba, 0x1a, 0x53, 0x17, 0x73,
0x48, 0xa4, 0xd5, 0xf5, 0x26, 0x44, 0xce, 0xc1, 0xd7, 0xed, 0xe4, 0xc4, 0xea, 0x6e, 0xc0, 0xd6, 0x7f, 0x1b, 0xeb, 0x51, 0x7b, 0xf0, 0x5f, 0xec, 0x5f, 0x47, 0xed, 0x39, 0x40, 0x23, 0x40, 0x1d,
0x80, 0x73, 0x4b, 0xf3, 0xe4, 0xee, 0x0f, 0xc7, 0xba, 0xc5, 0x86, 0x9b, 0x6e, 0x39, 0xdc, 0x76, 0xb4, 0x35, 0x04, 0x9d, 0xad, 0xc5, 0x2c, 0x90, 0x74, 0x56, 0xc7, 0x6c, 0xbf, 0x41, 0x6f, 0xe8,
0x2e, 0x85, 0xf1, 0xbc, 0x16, 0xb6, 0x67, 0xe4, 0x32, 0x6d, 0x48, 0x5e, 0x42, 0x80, 0x8a, 0xcf, 0x4c, 0xdc, 0x4b, 0x6c, 0xfb, 0x7e, 0x62, 0x3f, 0x5d, 0x7c, 0x7f, 0x33, 0xe3, 0x72, 0x5e, 0x4c,
0x0b, 0x74, 0xca, 0xa3, 0xe3, 0xd0, 0x8b, 0x5f, 0xb0, 0x89, 0xc3, 0xec, 0x61, 0xd1, 0x3f, 0xde, 0x4a, 0x2f, 0x0c, 0xf4, 0x35, 0x5e, 0xf3, 0xac, 0xfa, 0x35, 0xe0, 0xa9, 0x64, 0x79, 0x4a, 0xe3,
0xa6, 0xd5, 0xf6, 0xfe, 0x8b, 0xfd, 0xdf, 0x56, 0xfb, 0x14, 0xa0, 0x27, 0xa0, 0x33, 0xda, 0x06, 0x81, 0xba, 0xd9, 0xa0, 0x4c, 0xe4, 0x62, 0x32, 0x69, 0xab, 0xea, 0xe2, 0x6f, 0x00, 0x00, 0x00,
0x42, 0x5e, 0x6d, 0xd8, 0x2c, 0x35, 0xfc, 0xa1, 0xb3, 0xd9, 0x51, 0x8f, 0xde, 0xf1, 0x07, 0xfd, 0xff, 0xff, 0xf7, 0x5a, 0x33, 0x47, 0x2c, 0x06, 0x00, 0x00,
0xc4, 0xb1, 0xa3, 0xa7, 0x8e, 0xfd, 0xe1, 0xcd, 0x6f, 0x5f, 0x3d, 0x08, 0xb3, 0xa8, 0xe7, 0x8d,
0x16, 0x2e, 0xdd, 0x19, 0x5f, 0x8a, 0xb2, 0xfd, 0x75, 0x29, 0x94, 0xc1, 0x4a, 0xf1, 0xe2, 0xd2,
0x5e, 0x76, 0xd9, 0x38, 0x72, 0x39, 0x9f, 0x8f, 0x6c, 0xf4, 0xe6, 0xbf, 0x00, 0x00, 0x00, 0xff,
0xff, 0x9f, 0xd3, 0xc4, 0x4c, 0xc9, 0x06, 0x00, 0x00,
} }

60
internal/rootcoord/meta_table.go
View File

@ -41,9 +41,6 @@ const (
// ComponentPrefix prefix for rootcoord component // ComponentPrefix prefix for rootcoord component
ComponentPrefix = "root-coord" ComponentPrefix = "root-coord"
// TenantMetaPrefix prefix for tenant meta
TenantMetaPrefix = ComponentPrefix + "/tenant"
// ProxyMetaPrefix prefix for proxy meta // ProxyMetaPrefix prefix for proxy meta
ProxyMetaPrefix = ComponentPrefix + "/proxy" ProxyMetaPrefix = ComponentPrefix + "/proxy"
@ -97,7 +94,6 @@ const (
type MetaTable struct { type MetaTable struct {
txn kv.TxnKV // client of a reliable txnkv service, i.e. etcd client txn kv.TxnKV // client of a reliable txnkv service, i.e. etcd client
snapshot kv.SnapShotKV // client of a reliable snapshotkv service, i.e. etcd client snapshot kv.SnapShotKV // client of a reliable snapshotkv service, i.e. etcd client
tenantID2Meta map[typeutil.UniqueID]pb.TenantMeta // tenant id to tenant meta
proxyID2Meta map[typeutil.UniqueID]pb.ProxyMeta // proxy id to proxy meta proxyID2Meta map[typeutil.UniqueID]pb.ProxyMeta // proxy id to proxy meta
collID2Meta map[typeutil.UniqueID]pb.CollectionInfo // collection_id -> meta collID2Meta map[typeutil.UniqueID]pb.CollectionInfo // collection_id -> meta
collName2ID map[string]typeutil.UniqueID // collection name to collection id collName2ID map[string]typeutil.UniqueID // collection name to collection id
@ -106,22 +102,20 @@ type MetaTable struct {
segID2IndexMeta map[typeutil.UniqueID]map[typeutil.UniqueID]pb.SegmentIndexInfo // collection_id/index_id/partition_id/segment_id -> meta segID2IndexMeta map[typeutil.UniqueID]map[typeutil.UniqueID]pb.SegmentIndexInfo // collection_id/index_id/partition_id/segment_id -> meta
indexID2Meta map[typeutil.UniqueID]pb.IndexInfo // collection_id/index_id -> meta indexID2Meta map[typeutil.UniqueID]pb.IndexInfo // collection_id/index_id -> meta
tenantLock sync.RWMutex proxyLock sync.RWMutex
proxyLock sync.RWMutex ddLock sync.RWMutex
ddLock sync.RWMutex credLock sync.RWMutex
credLock sync.RWMutex
} }
// NewMetaTable creates meta table for rootcoord, which stores all in-memory information // NewMetaTable creates meta table for rootcoord, which stores all in-memory information
// for collection, partition, segment, index etc. // for collection, partition, segment, index etc.
func NewMetaTable(txn kv.TxnKV, snap kv.SnapShotKV) (*MetaTable, error) { func NewMetaTable(txn kv.TxnKV, snap kv.SnapShotKV) (*MetaTable, error) {
mt := &MetaTable{ mt := &MetaTable{
txn: txn, txn: txn,
snapshot: snap, snapshot: snap,
tenantLock: sync.RWMutex{}, proxyLock: sync.RWMutex{},
proxyLock: sync.RWMutex{}, ddLock: sync.RWMutex{},
ddLock: sync.RWMutex{}, credLock: sync.RWMutex{},
credLock: sync.RWMutex{},
} }
err := mt.reloadFromKV() err := mt.reloadFromKV()
if err != nil { if err != nil {
@ -131,7 +125,6 @@ func NewMetaTable(txn kv.TxnKV, snap kv.SnapShotKV) (*MetaTable, error) {
} }
func (mt *MetaTable) reloadFromKV() error { func (mt *MetaTable) reloadFromKV() error {
mt.tenantID2Meta = make(map[typeutil.UniqueID]pb.TenantMeta)
mt.proxyID2Meta = make(map[typeutil.UniqueID]pb.ProxyMeta) mt.proxyID2Meta = make(map[typeutil.UniqueID]pb.ProxyMeta)
mt.collID2Meta = make(map[typeutil.UniqueID]pb.CollectionInfo) mt.collID2Meta = make(map[typeutil.UniqueID]pb.CollectionInfo)
mt.collName2ID = make(map[string]typeutil.UniqueID) mt.collName2ID = make(map[string]typeutil.UniqueID)
@ -140,21 +133,7 @@ func (mt *MetaTable) reloadFromKV() error {
mt.segID2IndexMeta = make(map[typeutil.UniqueID]map[typeutil.UniqueID]pb.SegmentIndexInfo) mt.segID2IndexMeta = make(map[typeutil.UniqueID]map[typeutil.UniqueID]pb.SegmentIndexInfo)
mt.indexID2Meta = make(map[typeutil.UniqueID]pb.IndexInfo) mt.indexID2Meta = make(map[typeutil.UniqueID]pb.IndexInfo)
_, values, err := mt.snapshot.LoadWithPrefix(TenantMetaPrefix, 0) _, values, err := mt.txn.LoadWithPrefix(ProxyMetaPrefix)
if err != nil {
return err
}
for _, value := range values {
tenantMeta := pb.TenantMeta{}
err := proto.Unmarshal([]byte(value), &tenantMeta)
if err != nil {
return fmt.Errorf("rootcoord Unmarshal pb.TenantMeta err:%w", err)
}
mt.tenantID2Meta[tenantMeta.ID] = tenantMeta
}
_, values, err = mt.txn.LoadWithPrefix(ProxyMetaPrefix)
if err != nil { if err != nil {
return err return err
} }
@ -257,27 +236,6 @@ func (mt *MetaTable) reloadFromKV() error {
return nil return nil
} }
// AddTenant add tenant
func (mt *MetaTable) AddTenant(te *pb.TenantMeta, ts typeutil.Timestamp) error {
mt.tenantLock.Lock()
defer mt.tenantLock.Unlock()
k := fmt.Sprintf("%s/%d", TenantMetaPrefix, te.ID)
v, err := proto.Marshal(te)
if err != nil {
log.Error("Failed to marshal TenantMeta in AddTenant", zap.Error(err))
return err
}
err = mt.snapshot.Save(k, string(v), ts)
if err != nil {
log.Error("Failed to save TenantMeta in AddTenant", zap.Error(err))
return err
}
mt.tenantID2Meta[te.ID] = *te
return nil
}
// AddProxy add proxy // AddProxy add proxy
func (mt *MetaTable) AddProxy(po *pb.ProxyMeta) error { func (mt *MetaTable) AddProxy(po *pb.ProxyMeta) error {
mt.proxyLock.Lock() mt.proxyLock.Lock()

29
internal/rootcoord/meta_table_test.go
View File

@ -118,23 +118,12 @@ func Test_MockKV(t *testing.T) {
assert.NotNil(t, err) assert.NotNil(t, err)
assert.EqualError(t, err, "load prefix error") assert.EqualError(t, err, "load prefix error")
// tenant
prefix[TenantMetaPrefix] = []string{"tenant-prefix"}
_, err = NewMetaTable(kt, k1)
assert.NotNil(t, err)
value, err := proto.Marshal(&pb.TenantMeta{})
assert.Nil(t, err)
prefix[TenantMetaPrefix] = []string{string(value)}
_, err = NewMetaTable(kt, k1)
assert.NotNil(t, err)
// proxy // proxy
prefix[ProxyMetaPrefix] = []string{"porxy-meta"} prefix[ProxyMetaPrefix] = []string{"porxy-meta"}
_, err = NewMetaTable(kt, k1) _, err = NewMetaTable(kt, k1)
assert.NotNil(t, err) assert.NotNil(t, err)
value, err = proto.Marshal(&pb.ProxyMeta{}) value, err := proto.Marshal(&pb.ProxyMeta{})
assert.Nil(t, err) assert.Nil(t, err)
prefix[ProxyMetaPrefix] = []string{string(value)} prefix[ProxyMetaPrefix] = []string{string(value)}
_, err = NewMetaTable(kt, k1) _, err = NewMetaTable(kt, k1)
@ -180,21 +169,10 @@ func Test_MockKV(t *testing.T) {
assert.EqualError(t, err, "load prefix error") assert.EqualError(t, err, "load prefix error")
prefix[CollectionAliasMetaPrefix] = []string{"alias-meta"} prefix[CollectionAliasMetaPrefix] = []string{"alias-meta"}
k1.save = func(key string, value string, ts typeutil.Timestamp) error {
return fmt.Errorf("save tenant error")
}
assert.Panics(t, func() { m1.AddTenant(&pb.TenantMeta{}, 0) })
//err = m1.AddTenant(&pb.TenantMeta{}, 0)
//assert.NotNil(t, err)
//assert.EqualError(t, err, "save tenant error")
k1.save = func(key string, value string, ts typeutil.Timestamp) error { k1.save = func(key string, value string, ts typeutil.Timestamp) error {
return fmt.Errorf("save proxy error") return fmt.Errorf("save proxy error")
} }
assert.Panics(t, func() { m1.AddProxy(&pb.ProxyMeta{}) }) assert.Panics(t, func() { m1.AddProxy(&pb.ProxyMeta{}) })
//err = m1.AddProxy(&pb.ProxyMeta{}, 0)
//assert.NotNil(t, err)
//assert.EqualError(t, err, "save proxy error")
} }
func TestMetaTable(t *testing.T) { func TestMetaTable(t *testing.T) {
@ -493,11 +471,6 @@ func TestMetaTable(t *testing.T) {
wg.Add(1) wg.Add(1)
t.Run("reload meta", func(t *testing.T) { t.Run("reload meta", func(t *testing.T) {
defer wg.Done() defer wg.Done()
te := pb.TenantMeta{
ID: 100,
}
err := mt.AddTenant(&te, 0)
assert.Nil(t, err)
po := pb.ProxyMeta{ po := pb.ProxyMeta{
ID: 101, ID: 101,
} }