milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update proto:add partition id 

Signed-off-by: neza2017 <yefu.chen@zilliz.com>
pull/4973/head^2
neza2017 2021-01-15 11:53:22 +08:00 committed by yefu.chen
parent 9038521aa3
commit 56ab2bda2d
76 changed files with 653 additions and 1328 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/indexbuilder/indexbuilder.go
View File

@ -7,14 +7,14 @@ import (
"os/signal"
"syscall"
"github.com/zilliztech/milvus-distributed/internal/indexnode"
"github.com/zilliztech/milvus-distributed/internal/indexbuilder"
"go.uber.org/zap"
)
func main() {
indexnode.Init()
indexbuilder.Init()
ctx, cancel := context.WithCancel(context.Background())
svr, err := indexnode.CreateBuilder(ctx)
svr, err := indexbuilder.CreateBuilder(ctx)
if err != nil {
log.Print("create server failed", zap.Error(err))
}

8
cmd/proxy/proxy.go
View File

@ -8,15 +8,15 @@ import (
"os/signal"
"syscall"
"github.com/zilliztech/milvus-distributed/internal/proxynode"
"github.com/zilliztech/milvus-distributed/internal/proxy"
"go.uber.org/zap"
)
func main() {
proxynode.Init()
fmt.Println("ProxyID is", proxynode.Params.ProxyID())
proxy.Init()
fmt.Println("ProxyID is", proxy.Params.ProxyID())
ctx, cancel := context.WithCancel(context.Background())
svr, err := proxynode.CreateProxy(ctx)
svr, err := proxy.CreateProxy(ctx)
if err != nil {
log.Print("create server failed", zap.Error(err))
}

14
cmd/singlenode/main.go
View File

@ -14,9 +14,9 @@ import (
"go.uber.org/zap"
"github.com/zilliztech/milvus-distributed/internal/indexnode"
"github.com/zilliztech/milvus-distributed/internal/indexbuilder"
"github.com/zilliztech/milvus-distributed/internal/master"
"github.com/zilliztech/milvus-distributed/internal/proxynode"
"github.com/zilliztech/milvus-distributed/internal/proxy"
"github.com/zilliztech/milvus-distributed/internal/querynode"
"github.com/zilliztech/milvus-distributed/internal/writenode"
)
@ -62,10 +62,10 @@ func InitMaster(cpuprofile *string, wg *sync.WaitGroup) {
func InitProxy(wg *sync.WaitGroup) {
defer wg.Done()
proxynode.Init()
fmt.Println("ProxyID is", proxynode.Params.ProxyID())
proxy.Init()
fmt.Println("ProxyID is", proxy.Params.ProxyID())
ctx, cancel := context.WithCancel(context.Background())
svr, err := proxynode.CreateProxy(ctx)
svr, err := proxy.CreateProxy(ctx)
if err != nil {
log.Print("create server failed", zap.Error(err))
}
@ -138,9 +138,9 @@ func InitQueryNode(wg *sync.WaitGroup) {
func InitIndexBuilder(wg *sync.WaitGroup) {
defer wg.Done()
indexnode.Init()
indexbuilder.Init()
ctx, cancel := context.WithCancel(context.Background())
svr, err := indexnode.CreateBuilder(ctx)
svr, err := indexbuilder.CreateBuilder(ctx)
if err != nil {
log.Print("create server failed", zap.Error(err))
}

4
configs/advanced/proxy_node.yaml → configs/advanced/proxy.yaml
View File

@ -9,7 +9,7 @@
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
proxyNode:
proxy:
timeTickInterval: 200 # ms
msgStream:
@ -29,4 +29,4 @@ proxyNode:
maxNameLength: 255
maxFieldNum: 64
maxDimension: 32768
maxDimension: 32768

2
configs/milvus.yaml
View File

@ -44,7 +44,7 @@ master:
address: localhost
port: 53100
proxyNode:
proxy:
address: localhost
port: 19530

2
docs/developer_guides/chap03_index_service.md
View File

@ -26,7 +26,7 @@ type IndexService interface {
```go
type RegisterNodeRequest struct {
MsgBase
RequestBase
Address string
Port int64
}

12
docs/developer_guides/chap04_message_stream.md
View File

@ -1,7 +1,9 @@
## 8. Message Stream
## 8. Message Stream Service
#### 8.1 Overview
#### 8.2 Message Stream Service API
@ -65,14 +67,6 @@ type DescribeChannelResponse struct {
#### A.3 Message Stream
* Overview
<img src="./figs/msg_stream_input_output.jpeg" width=700>
* Interface
``` go
type MsgType uint32
const {

26
docs/developer_guides/chap05_proxy.md
View File

@ -19,14 +19,14 @@ type ProxyService interface {
* *MsgBase*
* *RequestBase*
```go
type MsgBase struct {
type RequestBase struct {
MsgType MsgType
MsgID UniqueID
ReqID UniqueID
Timestamp Timestamp
SourceID UniqueID
RequestorID UniqueID
}
```
@ -43,7 +43,7 @@ type RegisterLinkResponse struct {
```go
type RegisterNodeRequest struct {
MsgBase
RequestBase
Address string
Port int64
}
@ -57,7 +57,7 @@ type RegisterNodeResponse struct {
```go
type InvalidateCollMetaCacheRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
}
@ -117,7 +117,7 @@ See *Master API* for detailed definitions.
```go
type LoadCollectionRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
}
@ -127,7 +127,7 @@ type LoadCollectionRequest struct {
```go
type ReleaseCollectionRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
}
@ -161,7 +161,7 @@ See *Master API* for detailed definitions.
```go
type LoadPartitonRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
PartitionNames []string
@ -172,7 +172,7 @@ type LoadPartitonRequest struct {
```go
type ReleasePartitionRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
PartitionNames []string
@ -199,7 +199,7 @@ See *Master API* for detailed definitions.
```go
type InsertRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
PartitionName string
@ -217,7 +217,7 @@ type InsertResponse struct {
```go
type SearchRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
PartitionNames []string
@ -230,7 +230,7 @@ type SearchRequest struct {
```go
type FlushRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
}

124
docs/developer_guides/chap06_master.md
View File

@ -35,14 +35,14 @@ type Master interface {
* *MsgBase*
* *RequestBase*
```go
type MsgBase struct {
type RequestBase struct {
MsgType MsgType
MsgID UniqueID
ReqID UniqueID
Timestamp Timestamp
SourceID UniqueID
RequestorID UniqueID
}
```
@ -50,7 +50,7 @@ type MsgBase struct {
```go
type CreateCollectionRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
Schema []bytes
@ -61,7 +61,7 @@ type CreateCollectionRequest struct {
```go
type DropCollectionRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
}
@ -71,7 +71,7 @@ type DropCollectionRequest struct {
```go
type HasCollectionRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
}
@ -81,7 +81,7 @@ type HasCollectionRequest struct {
```go
type DescribeCollectionRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
}
@ -95,7 +95,7 @@ type DescribeCollectionResponse struct {
```go
type CollectionStatsRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
}
@ -109,7 +109,7 @@ type CollectionStatsResponse struct {
```go
type ShowCollectionRequest struct {
MsgBase
RequestBase
DbName string
}
@ -122,7 +122,7 @@ type ShowCollectionResponse struct {
```go
type CreatePartitionRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
PartitionName string
@ -133,7 +133,7 @@ type CreatePartitionRequest struct {
```go
type DropPartitionRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
PartitionName string
@ -144,7 +144,7 @@ type DropPartitionRequest struct {
```go
type HasPartitionRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
PartitionName string
@ -155,7 +155,7 @@ type HasPartitionRequest struct {
```go
type PartitionStatsRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
PartitionName string
@ -170,7 +170,7 @@ type PartitionStatsResponse struct {
```go
type ShowPartitionRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
}
@ -184,7 +184,7 @@ type ShowPartitionResponse struct {
```go
type CreateIndexRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
FieldName string
@ -196,7 +196,7 @@ type CreateIndexRequest struct {
```go
type DescribeIndexRequest struct {
MsgBase
RequestBase
DbName string
CollectionName string
FieldName string
@ -216,7 +216,7 @@ type DescribeIndexResponse struct {
```go
type TsoRequest struct {
MsgBase
RequestBase
Count uint32
}
@ -230,7 +230,7 @@ type TsoResponse struct {
```go
type IDRequest struct {
MsgBase
RequestBase
Count uint32
}
@ -242,75 +242,25 @@ type IDResponse struct {
#### 10.2 Dd (Data definitions) Channel
#### 10.1 Interfaces (RPC)
* *CreateCollection*
```go
type CreateCollectionRequest struct {
RequestBase
DbName string
CollectionName string
DbID UniqueID
CollectionID UniqueID
Schema []bytes
}
```
* *DropCollection*
```go
type DropCollectionRequest struct {
RequestBase
DbName string
CollectionName string
DbID UniqueID
CollectionID UniqueID
}
```
* *CreatePartition*
```go
type CreatePartitionRequest struct {
RequestBase
DbName string
CollectionName string
PartitionName string
DbID UniqueID
CollectionID UniqueID
PartitionID UniqueID
}
```
* *DropPartition*
```go
type DropPartitionRequest struct {
RequestBase
DbName string
CollectionName string
PartitionName string
DbID UniqueID
CollectionID UniqueID
PartitionID UniqueID
}
```
* *CreateIndex*
```go
type CreateIndexRequest struct {
RequestBase
DbName string
CollectionName string
FieldName string
DbID UniqueID
CollectionID UniqueID
FieldID int64
Params [] KeyValuePair
}
```
| RPC | description |
| :----------------- | ------------------------------------------------------------ |
| CreateCollection | create a collection base on schema statement |
| DropCollection | drop a collection |
| HasCollection | whether or not a collection exists |
| DescribeCollection | show a collection's schema and its descriptive statistics |
| ShowCollections | list all collections |
| CreatePartition | create a partition |
| DropPartition | drop a partition |
| HasPartition | whether or not a partition exists |
| DescribePartition | show a partition's name and its descriptive statistics |
| ShowPartitions | list a collection's all partitions |
| AllocTimestamp | allocate a batch of consecutive timestamps |
| AllocID | allocate a batch of consecutive IDs |
| AssignSegmentID | assign segment id to insert rows (master determines which segment these rows belong to) |
| GetSysConfigs | get system configurations |
| | |

48
docs/developer_guides/chap07_query_service.md
View File

@ -31,14 +31,14 @@ type QueryService interface {
* *MsgBase*
* *RequestBase*
```go
type MsgBase struct {
type RequestBase struct {
MsgType MsgType
MsgID UniqueID
ReqID UniqueID
Timestamp Timestamp
SourceID UniqueID
RequestorID UniqueID
}
```
@ -46,7 +46,7 @@ type MsgBase struct {
```go
type RegisterNodeRequest struct {
MsgBase
RequestBase
Address string
Port int64
}
@ -60,7 +60,7 @@ type RegisterNodeResponse struct {
```go
type ShowCollectionRequest struct {
MsgBase
RequestBase
DbID UniqueID
}
@ -73,7 +73,7 @@ type ShowCollectionResponse struct {
```go
type LoadCollectionRequest struct {
MsgBase
RequestBase
DbID UniqueID
CollectionID UniqueID
}
@ -83,7 +83,7 @@ type LoadCollectionRequest struct {
```go
type ReleaseCollectionRequest struct {
MsgBase
RequestBase
DbID UniqueID
CollectionID UniqueID
}
@ -93,7 +93,7 @@ type ReleaseCollectionRequest struct {
```go
type ShowPartitionRequest struct {
MsgBase
RequestBase
DbID UniqueID
CollectionID UniqueID
}
@ -119,7 +119,7 @@ const (
)
type PartitionStatesRequest struct {
MsgBase
RequestBase
DbID UniqueID
CollectionID UniqueID
PartitionIDs []UniqueID
@ -139,7 +139,7 @@ type PartitionStatesResponse struct {
```go
type LoadPartitonRequest struct {
MsgBase
RequestBase
DbID UniqueID
CollectionID UniqueID
PartitionIDs []UniqueID
@ -150,7 +150,7 @@ type LoadPartitonRequest struct {
```go
type ReleasePartitionRequest struct {
MsgBase
RequestBase
DbID UniqueID
CollectionID UniqueID
PartitionIDs []UniqueID
@ -168,24 +168,6 @@ type CreateQueryChannelResponse struct {
#### 8.2 Query Channel
```go
type SearchRequest struct {
RequestBase
DbName string
CollectionName string
PartitionNames []string
DbID UniqueID
CollectionID UniqueID
PartitionIDs []UniqueID
Dsl string
PlaceholderGroup []byte
}
```
#### 8.2 Query Node Interface
```go
@ -210,7 +192,7 @@ type QueryNode interface {
```go
type AddQueryChannelRequest struct {
MsgBase
RequestBase
RequestChannelName string
ResultChannelName string
}
@ -237,7 +219,7 @@ type WatchDmChannelRequest struct {
```go
type LoadSegmentRequest struct {
MsgBase
RequestBase
DbID UniqueID
CollectionID UniqueID
PartitionID UniqueID
@ -250,7 +232,7 @@ type LoadSegmentRequest struct {
```go
type ReleaseSegmentRequest struct {
MsgBase
RequestBase
DbID UniqueID
CollectionID UniqueID
PartitionID UniqueID

42
docs/developer_guides/chap09_data_service.md
View File

@ -28,14 +28,14 @@ type DataService interface {
* *MsgBase*
* *RequestBase*
```go
type MsgBase struct {
type RequestBase struct {
MsgType MsgType
MsgID UniqueID
ReqID UniqueID
Timestamp Timestamp
SourceID UniqueID
RequestorID UniqueID
}
```
@ -43,7 +43,7 @@ type MsgBase struct {
```go
type RegisterNodeRequest struct {
MsgBase
RequestBase
Address string
Port int64
}
@ -64,7 +64,7 @@ type SegIDRequest struct {
}
type AssignSegIDRequest struct {
MsgBase
RequestBase
PerChannelRequest []SegIDRequest
}
@ -88,7 +88,7 @@ type AssignSegIDResponse struct {
```go
type FlushRequest struct {
MsgBase
RequestBase
DbID UniqueID
CollectionID UniqueID
}
@ -100,7 +100,7 @@ type FlushRequest struct {
```go
type ShowSegmentRequest struct {
MsgBase
RequestBase
CollectionID UniqueID
PartitionID UniqueID
}
@ -123,7 +123,7 @@ enum SegmentState {
}
type SegmentStatesRequest struct {
MsgBase
RequestBase
SegmentID UniqueID
}
@ -140,7 +140,7 @@ type SegmentStatesResponse struct {
```go
type InsertBinlogPathRequest struct {
MsgBase
RequestBase
SegmentID UniqueID
}
@ -155,27 +155,9 @@ type InsertBinlogPathsResponse struct {
```go
type InsertChannelRequest struct {
MsgBase
DbID UniqueID
CollectionID UniqueID
}
```
#### 8.2 Insert Channel
```go
type InsertRequest struct {
RequestBase
DbName string
CollectionName string
PartitionName string
DbID UniqueID
CollectionID UniqueID
PartitionID UniqueID
RowData []Blob
HashKeys []uint32
}
```
@ -202,7 +184,7 @@ type DataNode interface {
```go
type WatchDmChannelRequest struct {
MsgBase
RequestBase
InsertChannelNames []string
}
```
@ -211,7 +193,7 @@ type WatchDmChannelRequest struct {
```go
type FlushSegRequest struct {
MsgBase
RequestBase
DbID UniqueID
CollectionID UniqueID
SegmentID []UniqueID

BIN
docs/developer_guides/figs/msg_stream_input_output.jpeg
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 1.5 MiB

10
internal/core/src/common/Span.h
View File

@ -13,8 +13,6 @@
#include <type_traits>
#include "common/Types.h"
#include <cassert>
#include "VectorTrait.h"
namespace milvus {
// type erasure to work around virtual restriction
class SpanBase {
@ -88,8 +86,14 @@ class Span<T, typename std::enable_if_t<std::is_fundamental_v<T>>> {
const int64_t row_count_;
};
namespace segcore {
class VectorTrait;
class FloatVector;
class BinaryVector;
} // namespace segcore
template <typename VectorType>
class Span<VectorType, typename std::enable_if_t<std::is_base_of_v<VectorTrait, VectorType>>> {
class Span<VectorType, typename std::enable_if_t<std::is_base_of_v<segcore::VectorTrait, VectorType>>> {
public:
using embedded_type = typename VectorType::embedded_type;

11
internal/core/src/common/Types.h
View File

@ -11,7 +11,6 @@
#pragma once
#include "utils/Types.h"
#include "faiss/utils/BitsetView.h"
#include <faiss/MetricType.h>
#include <string>
#include <boost/align/aligned_allocator.hpp>
@ -76,14 +75,4 @@ using FieldId = fluent::NamedType<int64_t, struct FieldIdTag, fluent::Comparable
using FieldName = fluent::NamedType<std::string, struct FieldNameTag, fluent::Comparable, fluent::Hashable>;
using FieldOffset = fluent::NamedType<int64_t, struct FieldOffsetTag, fluent::Comparable, fluent::Hashable>;
using BitsetView = faiss::BitsetView;
inline BitsetView
BitsetSubView(const BitsetView& view, int64_t offset, int64_t size) {
if (view.empty()) {
return BitsetView();
}
assert(offset % 8 == 0);
return BitsetView(view.data() + offset / 8, size);
}
} // namespace milvus

64
internal/core/src/common/VectorTrait.h
View File

@ -1,64 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#pragma once
#include "Types.h"
namespace milvus {
class VectorTrait {};
class FloatVector : public VectorTrait {
public:
using embedded_type = float;
static constexpr auto metric_type = DataType::VECTOR_FLOAT;
};
class BinaryVector : public VectorTrait {
public:
using embedded_type = uint8_t;
static constexpr auto metric_type = DataType::VECTOR_BINARY;
};
template <typename VectorType>
inline constexpr int64_t
get_element_sizeof(int64_t dim) {
static_assert(std::is_base_of_v<VectorType, VectorTrait>);
if constexpr (std::is_same_v<VectorType, FloatVector>) {
return dim * sizeof(float);
} else {
return dim / 8;
}
}
template <typename T>
constexpr bool IsVector = std::is_base_of_v<VectorTrait, T>;
template <typename T>
constexpr bool IsScalar = std::is_fundamental_v<T>;
template <typename T, typename Enabled = void>
struct EmbeddedTypeImpl;
template <typename T>
struct EmbeddedTypeImpl<T, std::enable_if_t<IsScalar<T>>> {
using type = T;
};
template <typename T>
struct EmbeddedTypeImpl<T, std::enable_if_t<IsVector<T>>> {
using type = std::conditional_t<std::is_same_v<T, FloatVector>, float, uint8_t>;
};
template <typename T>
using EmbeddedType = typename EmbeddedTypeImpl<T>::type;
} // namespace milvus

115
internal/core/src/pb/etcd_meta.pb.cc
View File

@ -169,6 +169,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_etcd_5fmeta_2eproto::offsets[]
PROTOBUF_FIELD_OFFSET(::milvus::proto::etcd::CollectionMeta, create_time_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::etcd::CollectionMeta, segmentids_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::etcd::CollectionMeta, partition_tags_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::etcd::CollectionMeta, partitionids_),
~0u, // no _has_bits_
PROTOBUF_FIELD_OFFSET(::milvus::proto::etcd::FieldBinlogFiles, _internal_metadata_),
~0u, // no _extensions_
@ -207,9 +208,9 @@ static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOB
{ 0, -1, sizeof(::milvus::proto::etcd::TenantMeta)},
{ 9, -1, sizeof(::milvus::proto::etcd::ProxyMeta)},
{ 17, -1, sizeof(::milvus::proto::etcd::CollectionMeta)},
{ 27, -1, sizeof(::milvus::proto::etcd::FieldBinlogFiles)},
{ 34, -1, sizeof(::milvus::proto::etcd::SegmentMeta)},
{ 49, -1, sizeof(::milvus::proto::etcd::FieldIndexMeta)},
{ 28, -1, sizeof(::milvus::proto::etcd::FieldBinlogFiles)},
{ 35, -1, sizeof(::milvus::proto::etcd::SegmentMeta)},
{ 50, -1, sizeof(::milvus::proto::etcd::FieldIndexMeta)},
};
static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = {
@ -229,25 +230,26 @@ const char descriptor_table_protodef_etcd_5fmeta_2eproto[] PROTOBUF_SECTION_VARI
"\003(\t\022\027\n\017query_channelID\030\004 \001(\t\"a\n\tProxyMet"
"a\022\n\n\002ID\030\001 \001(\003\022-\n\007address\030\002 \001(\0132\034.milvus."
"proto.common.Address\022\031\n\021result_channelID"
"s\030\003 \003(\t\"\224\001\n\016CollectionMeta\022\n\n\002ID\030\001 \001(\003\0225"
"s\030\003 \003(\t\"\252\001\n\016CollectionMeta\022\n\n\002ID\030\001 \001(\003\0225"
"\n\006schema\030\002 \001(\0132%.milvus.proto.schema.Col"
"lectionSchema\022\023\n\013create_time\030\003 \001(\004\022\022\n\nse"
"gmentIDs\030\004 \003(\003\022\026\n\016partition_tags\030\005 \003(\t\"9"
"\n\020FieldBinlogFiles\022\017\n\007fieldID\030\001 \001(\003\022\024\n\014b"
"inlog_files\030\002 \003(\t\"\204\002\n\013SegmentMeta\022\021\n\tseg"
"mentID\030\001 \001(\003\022\024\n\014collectionID\030\002 \001(\003\022\025\n\rpa"
"rtition_tag\030\003 \001(\t\022\025\n\rchannel_start\030\004 \001(\005"
"\022\023\n\013channel_end\030\005 \001(\005\022\021\n\topen_time\030\006 \001(\004"
"\022\022\n\nclose_time\030\007 \001(\004\022\020\n\010num_rows\030\010 \001(\003\022\020"
"\n\010mem_size\030\t \001(\003\022>\n\021binlog_file_paths\030\n "
"\003(\0132#.milvus.proto.etcd.FieldBinlogFiles"
"\"\313\001\n\016FieldIndexMeta\022\021\n\tsegmentID\030\001 \001(\003\022\017"
"\n\007fieldID\030\002 \001(\003\022\017\n\007indexID\030\003 \001(\003\0227\n\014inde"
"x_params\030\004 \003(\0132!.milvus.proto.common.Key"
"ValuePair\0221\n\006status\030\005 \001(\0162!.milvus.proto"
".service.IndexStatus\022\030\n\020index_file_paths"
"\030\006 \003(\tB@Z>github.com/zilliztech/milvus-d"
"istributed/internal/proto/etcdpbb\006proto3"
"gmentIDs\030\004 \003(\003\022\026\n\016partition_tags\030\005 \003(\t\022\024"
"\n\014partitionIDs\030\006 \003(\003\"9\n\020FieldBinlogFiles"
"\022\017\n\007fieldID\030\001 \001(\003\022\024\n\014binlog_files\030\002 \003(\t\""
"\204\002\n\013SegmentMeta\022\021\n\tsegmentID\030\001 \001(\003\022\024\n\014co"
"llectionID\030\002 \001(\003\022\025\n\rpartition_tag\030\003 \001(\t\022"
"\025\n\rchannel_start\030\004 \001(\005\022\023\n\013channel_end\030\005 "
"\001(\005\022\021\n\topen_time\030\006 \001(\004\022\022\n\nclose_time\030\007 \001"
"(\004\022\020\n\010num_rows\030\010 \001(\003\022\020\n\010mem_size\030\t \001(\003\022>"
"\n\021binlog_file_paths\030\n \003(\0132#.milvus.proto"
".etcd.FieldBinlogFiles\"\313\001\n\016FieldIndexMet"
"a\022\021\n\tsegmentID\030\001 \001(\003\022\017\n\007fieldID\030\002 \001(\003\022\017\n"
"\007indexID\030\003 \001(\003\0227\n\014index_params\030\004 \003(\0132!.m"
"ilvus.proto.common.KeyValuePair\0221\n\006statu"
"s\030\005 \001(\0162!.milvus.proto.service.IndexStat"
"us\022\030\n\020index_file_paths\030\006 \003(\tB@Z>github.c"
"om/zilliztech/milvus-distributed/interna"
"l/proto/etcdpbb\006proto3"
;
static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_etcd_5fmeta_2eproto_deps[3] = {
&::descriptor_table_common_2eproto,
@ -265,7 +267,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_etc
static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_etcd_5fmeta_2eproto_once;
static bool descriptor_table_etcd_5fmeta_2eproto_initialized = false;
const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_etcd_5fmeta_2eproto = {
&descriptor_table_etcd_5fmeta_2eproto_initialized, descriptor_table_protodef_etcd_5fmeta_2eproto, "etcd_meta.proto", 1040,
&descriptor_table_etcd_5fmeta_2eproto_initialized, descriptor_table_protodef_etcd_5fmeta_2eproto, "etcd_meta.proto", 1062,
&descriptor_table_etcd_5fmeta_2eproto_once, descriptor_table_etcd_5fmeta_2eproto_sccs, descriptor_table_etcd_5fmeta_2eproto_deps, 6, 3,
schemas, file_default_instances, TableStruct_etcd_5fmeta_2eproto::offsets,
file_level_metadata_etcd_5fmeta_2eproto, 6, file_level_enum_descriptors_etcd_5fmeta_2eproto, file_level_service_descriptors_etcd_5fmeta_2eproto,
@ -1104,7 +1106,8 @@ CollectionMeta::CollectionMeta(const CollectionMeta& from)
: ::PROTOBUF_NAMESPACE_ID::Message(),
_internal_metadata_(nullptr),
segmentids_(from.segmentids_),
partition_tags_(from.partition_tags_) {
partition_tags_(from.partition_tags_),
partitionids_(from.partitionids_) {
_internal_metadata_.MergeFrom(from._internal_metadata_);
if (from.has_schema()) {
schema_ = new ::milvus::proto::schema::CollectionSchema(*from.schema_);
@ -1150,6 +1153,7 @@ void CollectionMeta::Clear() {
segmentids_.Clear();
partition_tags_.Clear();
partitionids_.Clear();
if (GetArenaNoVirtual() == nullptr && schema_ != nullptr) {
delete schema_;
}
@ -1211,6 +1215,16 @@ const char* CollectionMeta::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE
} while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 42);
} else goto handle_unusual;
continue;
// repeated int64 partitionIDs = 6;
case 6:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 50)) {
ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedInt64Parser(mutable_partitionids(), ptr, ctx);
CHK_(ptr);
} else if (static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 48) {
add_partitionids(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr));
CHK_(ptr);
} else goto handle_unusual;
continue;
default: {
handle_unusual:
if ((tag & 7) == 4 || tag == 0) {
@ -1310,6 +1324,22 @@ bool CollectionMeta::MergePartialFromCodedStream(
break;
}
// repeated int64 partitionIDs = 6;
case 6: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (50 & 0xFF)) {
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPackedPrimitive<
::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>(
input, this->mutable_partitionids())));
} else if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (48 & 0xFF)) {
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadRepeatedPrimitiveNoInline<
::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>(
1, 50u, input, this->mutable_partitionids())));
} else {
goto handle_unusual;
}
break;
}
default: {
handle_unusual:
if (tag == 0) {
@ -1374,6 +1404,17 @@ void CollectionMeta::SerializeWithCachedSizes(
5, this->partition_tags(i), output);
}
// repeated int64 partitionIDs = 6;
if (this->partitionids_size() > 0) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteTag(6, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED, output);
output->WriteVarint32(_partitionids_cached_byte_size_.load(
std::memory_order_relaxed));
}
for (int i = 0, n = this->partitionids_size(); i < n; i++) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64NoTag(
this->partitionids(i), output);
}
if (_internal_metadata_.have_unknown_fields()) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields(
_internal_metadata_.unknown_fields(), output);
@ -1427,6 +1468,19 @@ void CollectionMeta::SerializeWithCachedSizes(
WriteStringToArray(5, this->partition_tags(i), target);
}
// repeated int64 partitionIDs = 6;
if (this->partitionids_size() > 0) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteTagToArray(
6,
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED,
target);
target = ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream::WriteVarint32ToArray(
_partitionids_cached_byte_size_.load(std::memory_order_relaxed),
target);
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
WriteInt64NoTagToArray(this->partitionids_, target);
}
if (_internal_metadata_.have_unknown_fields()) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray(
_internal_metadata_.unknown_fields(), target);
@ -1471,6 +1525,21 @@ size_t CollectionMeta::ByteSizeLong() const {
this->partition_tags(i));
}
// repeated int64 partitionIDs = 6;
{
size_t data_size = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
Int64Size(this->partitionids_);
if (data_size > 0) {
total_size += 1 +
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
static_cast<::PROTOBUF_NAMESPACE_ID::int32>(data_size));
}
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(data_size);
_partitionids_cached_byte_size_.store(cached_size,
std::memory_order_relaxed);
total_size += data_size;
}
// .milvus.proto.schema.CollectionSchema schema = 2;
if (this->has_schema()) {
total_size += 1 +
@ -1521,6 +1590,7 @@ void CollectionMeta::MergeFrom(const CollectionMeta& from) {
segmentids_.MergeFrom(from.segmentids_);
partition_tags_.MergeFrom(from.partition_tags_);
partitionids_.MergeFrom(from.partitionids_);
if (from.has_schema()) {
mutable_schema()->::milvus::proto::schema::CollectionSchema::MergeFrom(from.schema());
}
@ -1555,6 +1625,7 @@ void CollectionMeta::InternalSwap(CollectionMeta* other) {
_internal_metadata_.Swap(&other->_internal_metadata_);
segmentids_.InternalSwap(&other->segmentids_);
partition_tags_.InternalSwap(CastToBase(&other->partition_tags_));
partitionids_.InternalSwap(&other->partitionids_);
swap(schema_, other->schema_);
swap(id_, other->id_);
swap(create_time_, other->create_time_);

44
internal/core/src/pb/etcd_meta.pb.h
View File

@ -540,6 +540,7 @@ class CollectionMeta :
enum : int {
kSegmentIDsFieldNumber = 4,
kPartitionTagsFieldNumber = 5,
kPartitionIDsFieldNumber = 6,
kSchemaFieldNumber = 2,
kIDFieldNumber = 1,
kCreateTimeFieldNumber = 3,
@ -572,6 +573,17 @@ class CollectionMeta :
const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>& partition_tags() const;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>* mutable_partition_tags();
// repeated int64 partitionIDs = 6;
int partitionids_size() const;
void clear_partitionids();
::PROTOBUF_NAMESPACE_ID::int64 partitionids(int index) const;
void set_partitionids(int index, ::PROTOBUF_NAMESPACE_ID::int64 value);
void add_partitionids(::PROTOBUF_NAMESPACE_ID::int64 value);
const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >&
partitionids() const;
::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >*
mutable_partitionids();
// .milvus.proto.schema.CollectionSchema schema = 2;
bool has_schema() const;
void clear_schema();
@ -598,6 +610,8 @@ class CollectionMeta :
::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 > segmentids_;
mutable std::atomic<int> _segmentids_cached_byte_size_;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string> partition_tags_;
::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 > partitionids_;
mutable std::atomic<int> _partitionids_cached_byte_size_;
::milvus::proto::schema::CollectionSchema* schema_;
::PROTOBUF_NAMESPACE_ID::int64 id_;
::PROTOBUF_NAMESPACE_ID::uint64 create_time_;
@ -1599,6 +1613,36 @@ CollectionMeta::mutable_partition_tags() {
return &partition_tags_;
}
// repeated int64 partitionIDs = 6;
inline int CollectionMeta::partitionids_size() const {
return partitionids_.size();
}
inline void CollectionMeta::clear_partitionids() {
partitionids_.Clear();
}
inline ::PROTOBUF_NAMESPACE_ID::int64 CollectionMeta::partitionids(int index) const {
// @@protoc_insertion_point(field_get:milvus.proto.etcd.CollectionMeta.partitionIDs)
return partitionids_.Get(index);
}
inline void CollectionMeta::set_partitionids(int index, ::PROTOBUF_NAMESPACE_ID::int64 value) {
partitionids_.Set(index, value);
// @@protoc_insertion_point(field_set:milvus.proto.etcd.CollectionMeta.partitionIDs)
}
inline void CollectionMeta::add_partitionids(::PROTOBUF_NAMESPACE_ID::int64 value) {
partitionids_.Add(value);
// @@protoc_insertion_point(field_add:milvus.proto.etcd.CollectionMeta.partitionIDs)
}
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >&
CollectionMeta::partitionids() const {
// @@protoc_insertion_point(field_list:milvus.proto.etcd.CollectionMeta.partitionIDs)
return partitionids_;
}
inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >*
CollectionMeta::mutable_partitionids() {
// @@protoc_insertion_point(field_mutable_list:milvus.proto.etcd.CollectionMeta.partitionIDs)
return &partitionids_;
}
// -------------------------------------------------------------------
// FieldBinlogFiles

25
internal/core/src/query/Search.cpp
View File

@ -40,7 +40,7 @@ FloatSearch(const segcore::SegmentGrowingImpl& segment,
const float* query_data,
int64_t num_queries,
Timestamp timestamp,
const BitsetView& bitset,
std::optional<const BitmapSimple*> bitmaps_opt,
QueryResult& results) {
auto& schema = segment.get_schema();
auto& indexing_record = segment.get_indexing_record();
@ -79,21 +79,20 @@ FloatSearch(const segcore::SegmentGrowingImpl& segment,
// TODO: use sub_qr
for (int chunk_id = 0; chunk_id < max_indexed_id; ++chunk_id) {
auto chunk_size = indexing_entry.get_chunk_size();
auto bitset = create_bitmap_view(bitmaps_opt, chunk_id);
auto indexing = indexing_entry.get_vec_indexing(chunk_id);
auto sub_view = BitsetSubView(bitset, chunk_id * chunk_size, chunk_size);
auto sub_qr = SearchOnIndex(query_dataset, *indexing, search_conf, sub_view);
auto sub_qr = SearchOnIndex(query_dataset, *indexing, search_conf, bitset);
// convert chunk uid to segment uid
for (auto& x : sub_qr.mutable_labels()) {
if (x != -1) {
x += chunk_id * chunk_size;
x += chunk_id * indexing_entry.get_chunk_size();
}
}
final_qr.merge(sub_qr);
}
using segcore::FloatVector;
auto vec_ptr = record.get_entity<FloatVector>(vecfield_offset);
// step 4: brute force search where small indexing is unavailable
@ -102,14 +101,15 @@ FloatSearch(const segcore::SegmentGrowingImpl& segment,
auto max_chunk = upper_div(ins_barrier, vec_chunk_size);
for (int chunk_id = max_indexed_id; chunk_id < max_chunk; ++chunk_id) {
auto bitmap_view = create_bitmap_view(bitmaps_opt, chunk_id);
auto& chunk = vec_ptr->get_chunk(chunk_id);
auto element_begin = chunk_id * vec_chunk_size;
auto element_end = std::min(ins_barrier, (chunk_id + 1) * vec_chunk_size);
auto chunk_size = element_end - element_begin;
auto sub_view = BitsetSubView(bitset, element_begin, chunk_size);
auto sub_qr = FloatSearchBruteForce(query_dataset, chunk.data(), chunk_size, sub_view);
auto sub_qr = FloatSearchBruteForce(query_dataset, chunk.data(), chunk_size, bitmap_view);
// convert chunk uid to segment uid
for (auto& x : sub_qr.mutable_labels()) {
@ -134,7 +134,7 @@ BinarySearch(const segcore::SegmentGrowingImpl& segment,
const uint8_t* query_data,
int64_t num_queries,
Timestamp timestamp,
const faiss::BitsetView& bitset,
std::optional<const BitmapSimple*> bitmaps_opt,
QueryResult& results) {
auto& schema = segment.get_schema();
auto& indexing_record = segment.get_indexing_record();
@ -161,8 +161,11 @@ BinarySearch(const segcore::SegmentGrowingImpl& segment,
auto total_count = topK * num_queries;
// step 3: small indexing search
// TODO: this is too intrusive
// TODO: use QuerySubResult instead
query::dataset::BinaryQueryDataset query_dataset{metric_type, num_queries, topK, dim, query_data};
using segcore::BinaryVector;
auto vec_ptr = record.get_entity<BinaryVector>(vecfield_offset);
auto max_indexed_id = 0;
@ -177,8 +180,8 @@ BinarySearch(const segcore::SegmentGrowingImpl& segment,
auto element_end = std::min(ins_barrier, (chunk_id + 1) * vec_chunk_size);
auto nsize = element_end - element_begin;
auto sub_view = BitsetSubView(bitset, element_begin, nsize);
auto sub_result = BinarySearchBruteForce(query_dataset, chunk.data(), nsize, sub_view);
auto bitmap_view = create_bitmap_view(bitmaps_opt, chunk_id);
auto sub_result = BinarySearchBruteForce(query_dataset, chunk.data(), nsize, bitmap_view);
// convert chunk uid to segment uid
for (auto& x : sub_result.mutable_labels()) {

4
internal/core/src/query/Search.h
View File

@ -28,7 +28,7 @@ FloatSearch(const segcore::SegmentGrowingImpl& segment,
const float* query_data,
int64_t num_queries,
Timestamp timestamp,
const faiss::BitsetView& bitset,
std::optional<const BitmapSimple*> bitmap_opt,
QueryResult& results);
Status
@ -37,6 +37,6 @@ BinarySearch(const segcore::SegmentGrowingImpl& segment,
const uint8_t* query_data,
int64_t num_queries,
Timestamp timestamp,
const faiss::BitsetView& bitset,
std::optional<const BitmapSimple*> bitmaps_opt,
QueryResult& results);
} // namespace milvus::query

12
internal/core/src/query/SearchOnSealed.cpp
View File

@ -20,9 +20,8 @@
namespace milvus::query {
// negate bitset, and merge them into one
aligned_vector<uint8_t>
AssembleNegBitmap(const BitmapSimple& bitmap_simple) {
AssembleBitmap(const BitmapSimple& bitmap_simple) {
int64_t N = 0;
for (auto& bitmap : bitmap_simple) {
@ -53,7 +52,7 @@ SearchOnSealed(const Schema& schema,
const void* query_data,
int64_t num_queries,
Timestamp timestamp,
const faiss::BitsetView& bitset,
std::optional<const BitmapSimple*> bitmaps_opt,
QueryResult& result) {
auto topK = query_info.topK_;
@ -74,7 +73,12 @@ SearchOnSealed(const Schema& schema,
auto conf = query_info.search_params_;
conf[milvus::knowhere::meta::TOPK] = query_info.topK_;
conf[milvus::knowhere::Metric::TYPE] = MetricTypeToName(indexing_entry->metric_type_);
return indexing_entry->indexing_->Query(ds, conf, bitset);
if (bitmaps_opt.has_value()) {
auto bitmap = AssembleBitmap(*bitmaps_opt.value());
return indexing_entry->indexing_->Query(ds, conf, faiss::BitsetView(bitmap.data(), num_queries));
} else {
return indexing_entry->indexing_->Query(ds, conf, nullptr);
}
}();
auto ids = final->Get<idx_t*>(knowhere::meta::IDS);

6
internal/core/src/query/SearchOnSealed.h
View File

@ -16,10 +16,6 @@
#include "query/Search.h"
namespace milvus::query {
aligned_vector<uint8_t>
AssembleNegBitmap(const BitmapSimple& bitmap_simple);
void
SearchOnSealed(const Schema& schema,
const segcore::SealedIndexingRecord& record,
@ -27,7 +23,7 @@ SearchOnSealed(const Schema& schema,
const void* query_data,
int64_t num_queries,
Timestamp timestamp,
const faiss::BitsetView& view,
std::optional<const BitmapSimple*> bitmaps_opt,
QueryResult& result);
} // namespace milvus::query

29
internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp
View File

@ -67,20 +67,21 @@ ExecPlanNodeVisitor::visit(FloatVectorANNS& node) {
auto src_data = ph.get_blob<float>();
auto num_queries = ph.num_of_queries_;
aligned_vector<uint8_t> bitset_holder;
BitsetView view;
ExecExprVisitor::RetType bitmap_holder;
std::optional<const ExecExprVisitor::RetType*> bitset_pack;
if (node.predicate_.has_value()) {
ExecExprVisitor::RetType expr_ret = ExecExprVisitor(*segment).call_child(*node.predicate_.value());
bitset_holder = AssembleNegBitmap(expr_ret);
view = BitsetView(bitset_holder.data(), bitset_holder.size() * 8);
bitmap_holder = ExecExprVisitor(*segment).call_child(*node.predicate_.value());
bitset_pack = &bitmap_holder;
}
auto& sealed_indexing = segment->get_sealed_indexing_record();
if (sealed_indexing.is_ready(node.query_info_.field_offset_)) {
SearchOnSealed(segment->get_schema(), sealed_indexing, node.query_info_, src_data, num_queries, timestamp_,
view, ret);
bitset_pack, ret);
} else {
FloatSearch(*segment, node.query_info_, src_data, num_queries, timestamp_, view, ret);
FloatSearch(*segment, node.query_info_, src_data, num_queries, timestamp_, bitset_pack, ret);
}
ret_ = ret;
@ -97,20 +98,20 @@ ExecPlanNodeVisitor::visit(BinaryVectorANNS& node) {
auto src_data = ph.get_blob<uint8_t>();
auto num_queries = ph.num_of_queries_;
aligned_vector<uint8_t> bitset_holder;
BitsetView view;
ExecExprVisitor::RetType bitmap_holder;
std::optional<const ExecExprVisitor::RetType*> bitset_pack;
if (node.predicate_.has_value()) {
ExecExprVisitor::RetType expr_ret = ExecExprVisitor(*segment).call_child(*node.predicate_.value());
bitset_holder = AssembleNegBitmap(expr_ret);
view = BitsetView(bitset_holder.data(), bitset_holder.size() * 8);
bitmap_holder = ExecExprVisitor(*segment).call_child(*node.predicate_.value());
bitset_pack = &bitmap_holder;
}
auto& sealed_indexing = segment->get_sealed_indexing_record();
if (sealed_indexing.is_ready(node.query_info_.field_offset_)) {
SearchOnSealed(segment->get_schema(), sealed_indexing, node.query_info_, src_data, num_queries, timestamp_,
view, ret);
bitset_pack, ret);
} else {
BinarySearch(*segment, node.query_info_, src_data, num_queries, timestamp_, view, ret);
BinarySearch(*segment, node.query_info_, src_data, num_queries, timestamp_, bitset_pack, ret);
}
ret_ = ret;
}

41
internal/core/src/segcore/ConcurrentVector.h
View File

@ -23,7 +23,6 @@
#include "utils/tools.h"
#include <boost/container/vector.hpp>
#include "common/Types.h"
#include "common/Span.h"
namespace milvus::segcore {
@ -83,9 +82,6 @@ class VectorBase {
virtual void
set_data_raw(ssize_t element_offset, void* source, ssize_t element_count) = 0;
virtual SpanBase
get_span_base(int64_t chunk_id) const = 0;
int64_t
get_chunk_size() const {
return chunk_size_;
@ -108,9 +104,6 @@ class ConcurrentVectorImpl : public VectorBase {
ConcurrentVectorImpl&
operator=(const ConcurrentVectorImpl&) = delete;
using TraitType =
std::conditional_t<is_scalar, Type, std::conditional_t<std::is_same_v<Type, float>, FloatVector, BinaryVector>>;
public:
explicit ConcurrentVectorImpl(ssize_t dim, int64_t chunk_size) : VectorBase(chunk_size), Dim(is_scalar ? 1 : dim) {
Assert(is_scalar ? dim == 1 : dim != 1);
@ -122,25 +115,6 @@ class ConcurrentVectorImpl : public VectorBase {
chunks_.emplace_to_at_least(chunk_count, Dim * chunk_size_);
}
Span<TraitType>
get_span(int64_t chunk_id) const {
auto& chunk = get_chunk(chunk_id);
if constexpr (is_scalar) {
return Span<TraitType>(chunk.data(), chunk_size_);
} else if constexpr (std::is_same_v<Type, int64_t> || std::is_same_v<Type, int>) {
// only for testing
PanicInfo("unimplemented");
} else {
static_assert(std::is_same_v<typename TraitType::embedded_type, Type>);
return Span<TraitType>(chunk.data(), chunk_size_, Dim);
}
}
SpanBase
get_span_base(int64_t chunk_id) const override {
return get_span(chunk_id);
}
void
set_data_raw(ssize_t element_offset, void* source, ssize_t element_count) override {
set_data(element_offset, static_cast<const Type*>(source), element_count);
@ -232,12 +206,25 @@ class ConcurrentVectorImpl : public VectorBase {
template <typename Type>
class ConcurrentVector : public ConcurrentVectorImpl<Type, true> {
public:
static_assert(std::is_fundamental_v<Type>);
explicit ConcurrentVector(int64_t chunk_size)
: ConcurrentVectorImpl<Type, true>::ConcurrentVectorImpl(1, chunk_size) {
}
};
class VectorTrait {};
class FloatVector : public VectorTrait {
public:
using embedded_type = float;
static constexpr auto metric_type = DataType::VECTOR_FLOAT;
};
class BinaryVector : public VectorTrait {
public:
using embedded_type = uint8_t;
static constexpr auto metric_type = DataType::VECTOR_BINARY;
};
template <>
class ConcurrentVector<FloatVector> : public ConcurrentVectorImpl<float, false> {
public:

2
internal/core/src/segcore/IndexingEntry.cpp
View File

@ -76,7 +76,7 @@ IndexingRecord::UpdateResourceAck(int64_t chunk_ack, const InsertRecord& record)
// std::thread([this, old_ack, chunk_ack, &record] {
for (auto& [field_offset, entry] : entries_) {
auto vec_base = record.get_base_entity(field_offset);
entry->BuildIndexRange(old_ack, chunk_ack, vec_base);
entry->BuildIndexRange(old_ack, chunk_ack, vec_base.get());
}
finished_ack_.AddSegment(old_ack, chunk_ack);
// }).detach();

12
internal/core/src/segcore/InsertRecord.h
View File

@ -28,7 +28,7 @@ struct InsertRecord {
auto
get_base_entity(FieldOffset field_offset) const {
auto ptr = entity_vec_[field_offset.get()].get();
auto ptr = entity_vec_[field_offset.get()];
return ptr;
}
@ -36,7 +36,7 @@ struct InsertRecord {
auto
get_entity(FieldOffset field_offset) const {
auto base_ptr = get_base_entity(field_offset);
auto ptr = dynamic_cast<const ConcurrentVector<Type>*>(base_ptr);
auto ptr = std::dynamic_pointer_cast<const ConcurrentVector<Type>>(base_ptr);
Assert(ptr);
return ptr;
}
@ -45,7 +45,7 @@ struct InsertRecord {
auto
get_entity(FieldOffset field_offset) {
auto base_ptr = get_base_entity(field_offset);
auto ptr = dynamic_cast<ConcurrentVector<Type>*>(base_ptr);
auto ptr = std::dynamic_pointer_cast<ConcurrentVector<Type>>(base_ptr);
Assert(ptr);
return ptr;
}
@ -54,17 +54,17 @@ struct InsertRecord {
void
insert_entity(int64_t chunk_size) {
static_assert(std::is_fundamental_v<Type>);
entity_vec_.emplace_back(std::make_unique<ConcurrentVector<Type>>(chunk_size));
entity_vec_.emplace_back(std::make_shared<ConcurrentVector<Type>>(chunk_size));
}
template <typename VectorType>
void
insert_entity(int64_t dim, int64_t chunk_size) {
static_assert(std::is_base_of_v<VectorTrait, VectorType>);
entity_vec_.emplace_back(std::make_unique<ConcurrentVector<VectorType>>(dim, chunk_size));
entity_vec_.emplace_back(std::make_shared<ConcurrentVector<VectorType>>(dim, chunk_size));
}
private:
std::vector<std::unique_ptr<VectorBase>> entity_vec_;
std::vector<std::shared_ptr<VectorBase>> entity_vec_;
};
} // namespace milvus::segcore

12
internal/core/src/segcore/SegmentGrowingImpl.cpp
View File

@ -299,16 +299,4 @@ SegmentGrowingImpl::LoadIndexing(const LoadIndexInfo& info) {
return Status::OK();
}
SpanBase
SegmentGrowingImpl::chunk_data_impl(FieldOffset field_offset, int64_t chunk_id) const {
auto vec = get_insert_record().get_base_entity(field_offset);
return vec->get_span_base(chunk_id);
}
int64_t
SegmentGrowingImpl::get_safe_num_chunk() const {
auto size = get_insert_record().ack_responder_.GetAck();
return upper_div(size, chunk_size_);
}
} // namespace milvus::segcore

8
internal/core/src/segcore/SegmentGrowingImpl.h
View File

@ -112,7 +112,9 @@ class SegmentGrowingImpl : public SegmentGrowing {
}
int64_t
get_safe_num_chunk() const override;
get_num_chunk() const override {
PanicInfo("unimplemented");
}
Status
LoadIndexing(const LoadIndexInfo& info) override;
@ -137,7 +139,9 @@ class SegmentGrowingImpl : public SegmentGrowing {
protected:
SpanBase
chunk_data_impl(FieldOffset field_offset, int64_t chunk_id) const override;
chunk_data_impl(FieldOffset field_offset, int64_t chunk_id) const override {
PanicInfo("unimplemented");
}
private:
int64_t chunk_size_;

6
internal/core/src/segcore/SegmentInterface.h
View File

@ -44,16 +44,16 @@ class SegmentInternalInterface : public SegmentInterface {
get_schema() const = 0;
virtual int64_t
get_safe_num_chunk() const = 0;
get_num_chunk() const = 0;
template <typename T>
Span<T>
chunk_data(FieldOffset field_offset, int64_t chunk_id) const {
return static_cast<Span<T>>(chunk_data_impl(field_offset, chunk_id));
auto span = chunk_data_impl(field_offset, chunk_id);
return static_cast<Span<T>>(span);
}
protected:
// blob and row_count
virtual SpanBase
chunk_data_impl(FieldOffset field_offset, int64_t chunk_id) const = 0;
};

1
internal/core/unittest/CMakeLists.txt
View File

@ -17,7 +17,6 @@ set(MILVUS_TEST_FILES
test_sealed.cpp
test_reduce.cpp
test_interface.cpp
test_span.cpp
)
add_executable(all_tests
${MILVUS_TEST_FILES}

57
internal/core/unittest/test_span.cpp
View File

@ -1,57 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include <gtest/gtest.h>
#include "utils/tools.h"
#include "test_utils/DataGen.h"
#include "segcore/SegmentGrowing.h"
TEST(Span, Naive) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
int64_t N = 1000 * 1000;
constexpr int64_t chunk_size = 32 * 1024;
auto schema = std::make_shared<Schema>();
schema->AddDebugField("binaryvec", DataType::VECTOR_BINARY, 512, MetricType::METRIC_Jaccard);
schema->AddDebugField("age", DataType::FLOAT);
schema->AddDebugField("floatvec", DataType::VECTOR_FLOAT, 32, MetricType::METRIC_L2);
auto dataset = DataGen(schema, N);
auto segment = CreateGrowingSegment(schema, chunk_size);
segment->PreInsert(N);
segment->Insert(0, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_);
auto vec_ptr = dataset.get_col<uint8_t>(0);
auto age_ptr = dataset.get_col<float>(1);
auto float_ptr = dataset.get_col<float>(2);
SegmentInternalInterface& interface = *segment;
auto num_chunk = interface.get_safe_num_chunk();
ASSERT_EQ(num_chunk, upper_div(N, chunk_size));
auto row_count = interface.get_row_count();
ASSERT_EQ(N, row_count);
for (auto chunk_id = 0; chunk_id < num_chunk; ++chunk_id) {
auto vec_span = interface.chunk_data<BinaryVector>(FieldOffset(0), chunk_id);
auto age_span = interface.chunk_data<float>(FieldOffset(1), chunk_id);
auto float_span = interface.chunk_data<FloatVector>(FieldOffset(2), chunk_id);
auto begin = chunk_id * chunk_size;
auto end = std::min((chunk_id + 1) * chunk_size, N);
auto chunk_size = end - begin;
for (int i = 0; i < chunk_size * 512 / 8; ++i) {
ASSERT_EQ(vec_span.data()[i], vec_ptr[i + begin * 512 / 8]);
}
for (int i = 0; i < chunk_size; ++i) {
ASSERT_EQ(age_span.data()[i], age_ptr[i + begin]);
}
for (int i = 0; i < chunk_size; ++i) {
ASSERT_EQ(float_span.data()[i], float_ptr[i + begin * 32]);
}
}
}

42
internal/distributed/indexnode/client.go
View File

@ -1,42 +0,0 @@
package grpcindexnode
import (
"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
)
type Client struct {
grpcClient indexpb.IndexNodeClient
}
func (c Client) Init() {
panic("implement me")
}
func (c Client) Start() {
panic("implement me")
}
func (c Client) Stop() {
panic("implement me")
}
func (c Client) GetServiceStates() (internalpb2.ServiceStates, error) {
panic("implement me")
}
func (c Client) GetTimeTickChannel() (string, error) {
panic("implement me")
}
func (c Client) GetStatisticsChannel() (string, error) {
panic("implement me")
}
func (c Client) BuildIndex(req indexpb.BuildIndexRequest) (indexpb.BuildIndexResponse, error) {
panic("implement me")
}
func NewClient() *Client {
return &Client{}
}

19
internal/distributed/indexnode/service.go
View File

@ -1,19 +0,0 @@
package grpcindexnode
import (
"github.com/zilliztech/milvus-distributed/internal/indexnode"
"google.golang.org/grpc"
)
type Server struct {
node indexnode.Interface
grpcServer *grpc.Server
}
func NewGrpcServer() *Server {
ret := &Server{
node: &indexnode.IndexNode{},
}
return ret
}

54
internal/distributed/indexservice/client.go
View File

@ -1,54 +0,0 @@
package grpcindexservice
import (
"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
)
type Client struct {
grpcClient indexpb.IndexServiceClient
}
func (g Client) Init() {
panic("implement me")
}
func (g Client) Start() {
panic("implement me")
}
func (g Client) Stop() {
panic("implement me")
}
func (g Client) GetServiceStates() (internalpb2.ServiceStates, error) {
panic("implement me")
}
func (g Client) GetTimeTickChannel() (string, error) {
panic("implement me")
}
func (g Client) GetStatisticsChannel() (string, error) {
panic("implement me")
}
func (g Client) RegisterNode(req indexpb.RegisterNodeRequest) (indexpb.RegisterNodeResponse, error) {
panic("implement me")
}
func (g Client) BuildIndex(req indexpb.BuildIndexRequest) (indexpb.BuildIndexResponse, error) {
panic("implement me")
}
func (g Client) GetIndexStates(req indexpb.IndexStatesRequest) (indexpb.IndexStatesResponse, error) {
panic("implement me")
}
func (g Client) GetIndexFilePaths(req indexpb.IndexFilePathRequest) (indexpb.IndexFilePathsResponse, error) {
panic("implement me")
}
func NewClient() *Client {
return &Client{}
}

62
internal/distributed/indexservice/service.go
View File

@ -1,62 +0,0 @@
package grpcindexservice
import (
"github.com/zilliztech/milvus-distributed/internal/indexservice"
"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
"google.golang.org/grpc"
)
type Server struct {
server indexservice.Interface
grpcServer *grpc.Server
}
func (g Server) Init() {
panic("implement me")
}
func (g Server) Start() {
panic("implement me")
}
func (g Server) Stop() {
panic("implement me")
}
func (g Server) GetServiceStates() (internalpb2.ServiceStates, error) {
panic("implement me")
}
func (g Server) GetTimeTickChannel() (string, error) {
panic("implement me")
}
func (g Server) GetStatisticsChannel() (string, error) {
panic("implement me")
}
func (g Server) RegisterNode(req indexpb.RegisterNodeRequest) (indexpb.RegisterNodeResponse, error) {
panic("implement me")
}
func (g Server) BuildIndex(req indexpb.BuildIndexRequest) (indexpb.BuildIndexResponse, error) {
panic("implement me")
}
func (g Server) GetIndexStates(req indexpb.IndexStatesRequest) (indexpb.IndexStatesResponse, error) {
panic("implement me")
}
func (g Server) GetIndexFilePaths(req indexpb.IndexFilePathRequest) (indexpb.IndexFilePathsResponse, error) {
panic("implement me")
}
//varindex
func NewServer() *Server {
return &Server{
server: &indexservice.IndexService{},
}
}

5
internal/indexnode/client/client.go → internal/indexbuilder/client/client.go
View File

@ -1,14 +1,13 @@
package indexnodeclient
package indexbuilderclient
import (
"context"
"encoding/json"
"fmt"
"github.com/zilliztech/milvus-distributed/internal/errors"
"log"
"time"
"github.com/zilliztech/milvus-distributed/internal/errors"
"google.golang.org/grpc"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"

2
internal/indexnode/grpc_service.go → internal/indexbuilder/grpc_service.go
View File

@ -1,4 +1,4 @@
package indexnode
package indexbuilder
import (
"context"

2
internal/indexnode/index.go → internal/indexbuilder/index.go
View File

@ -1,4 +1,4 @@
package indexnode
package indexbuilder
/*

125
internal/indexnode/index_test.go → internal/indexbuilder/index_test.go
View File

@ -1,32 +1,31 @@
package indexnode
package indexbuilder
import (
"fmt"
"github.com/stretchr/testify/assert"
"math/rand"
"strconv"
"testing"
"github.com/stretchr/testify/assert"
)
const (
// index type
IndexFaissIDMap = "FLAT"
IndexFaissIVFFlat = "IVF_FLAT"
IndexFaissIVFPQ = "IVF_PQ"
IndexFaissIVFSQ8 = "IVF_SQ8"
IndexFaissIVFSQ8H = "IVF_SQ8_HYBRID"
IndexFaissBinIDMap = "BIN_FLAT"
IndexFaissBinIVFFlat = "BIN_IVF_FLAT"
IndexNsg = "NSG"
INDEX_FAISS_IDMAP = "FLAT"
INDEX_FAISS_IVFFLAT = "IVF_FLAT"
INDEX_FAISS_IVFPQ = "IVF_PQ"
INDEX_FAISS_IVFSQ8 = "IVF_SQ8"
INDEX_FAISS_IVFSQ8H = "IVF_SQ8_HYBRID"
INDEX_FAISS_BIN_IDMAP = "BIN_FLAT"
INDEX_FAISS_BIN_IVFFLAT = "BIN_IVF_FLAT"
INDEX_NSG = "NSG"
IndexHNSW = "HNSW"
IndexRHNSWFlat = "RHNSW_FLAT"
IndexRHNSWPQ = "RHNSW_PQ"
IndexRHNSWSQ = "RHNSW_SQ"
IndexANNOY = "ANNOY"
IndexNGTPANNG = "NGT_PANNG"
IndexNGTONNG = "NGT_ONNG"
INDEX_HNSW = "HNSW"
INDEX_RHNSWFlat = "RHNSW_FLAT"
INDEX_RHNSWPQ = "RHNSW_PQ"
INDEX_RHNSWSQ = "RHNSW_SQ"
INDEX_ANNOY = "ANNOY"
INDEX_NGTPANNG = "NGT_PANNG"
INDEX_NGTONNG = "NGT_ONNG"
// metric type
L2 = "L2"
@ -56,41 +55,41 @@ type testCase struct {
func generateFloatVectorTestCases() []testCase {
return []testCase{
{IndexFaissIDMap, L2, false},
{IndexFaissIDMap, IP, false},
{IndexFaissIVFFlat, L2, false},
{IndexFaissIVFFlat, IP, false},
{IndexFaissIVFPQ, L2, false},
{IndexFaissIVFPQ, IP, false},
{IndexFaissIVFSQ8, L2, false},
{IndexFaissIVFSQ8, IP, false},
//{IndexFaissIVFSQ8H, L2, false}, // TODO: enable gpu
//{IndexFaissIVFSQ8H, IP, false},
{IndexNsg, L2, false},
{IndexNsg, IP, false},
//{IndexHNSW, L2, false}, // TODO: fix json parse exception
//{IndexHNSW, IP, false},
//{IndexRHNSWFlat, L2, false},
//{IndexRHNSWFlat, IP, false},
//{IndexRHNSWPQ, L2, false},
//{IndexRHNSWPQ, IP, false},
//{IndexRHNSWSQ, L2, false},
//{IndexRHNSWSQ, IP, false},
{IndexANNOY, L2, false},
{IndexANNOY, IP, false},
{IndexNGTPANNG, L2, false},
{IndexNGTPANNG, IP, false},
{IndexNGTONNG, L2, false},
{IndexNGTONNG, IP, false},
{INDEX_FAISS_IDMAP, L2, false},
{INDEX_FAISS_IDMAP, IP, false},
{INDEX_FAISS_IVFFLAT, L2, false},
{INDEX_FAISS_IVFFLAT, IP, false},
{INDEX_FAISS_IVFPQ, L2, false},
{INDEX_FAISS_IVFPQ, IP, false},
{INDEX_FAISS_IVFSQ8, L2, false},
{INDEX_FAISS_IVFSQ8, IP, false},
//{INDEX_FAISS_IVFSQ8H, L2, false}, // TODO: enable gpu
//{INDEX_FAISS_IVFSQ8H, IP, false},
{INDEX_NSG, L2, false},
{INDEX_NSG, IP, false},
//{INDEX_HNSW, L2, false}, // TODO: fix json parse exception
//{INDEX_HNSW, IP, false},
//{INDEX_RHNSWFlat, L2, false},
//{INDEX_RHNSWFlat, IP, false},
//{INDEX_RHNSWPQ, L2, false},
//{INDEX_RHNSWPQ, IP, false},
//{INDEX_RHNSWSQ, L2, false},
//{INDEX_RHNSWSQ, IP, false},
{INDEX_ANNOY, L2, false},
{INDEX_ANNOY, IP, false},
{INDEX_NGTPANNG, L2, false},
{INDEX_NGTPANNG, IP, false},
{INDEX_NGTONNG, L2, false},
{INDEX_NGTONNG, IP, false},
}
}
func generateBinaryVectorTestCases() []testCase {
return []testCase{
{IndexFaissBinIVFFlat, Jaccard, true},
{IndexFaissBinIVFFlat, hamming, true},
{IndexFaissBinIDMap, Jaccard, true},
{IndexFaissBinIDMap, hamming, true},
{INDEX_FAISS_BIN_IVFFLAT, Jaccard, true},
{INDEX_FAISS_BIN_IVFFLAT, hamming, true},
{INDEX_FAISS_BIN_IDMAP, Jaccard, true},
{INDEX_FAISS_BIN_IDMAP, hamming, true},
}
}
@ -103,26 +102,26 @@ func generateParams(indexType, metricType string) (map[string]string, map[string
indexParams := make(map[string]string)
indexParams["index_type"] = indexType
indexParams["metric_type"] = metricType
if indexType == IndexFaissIDMap { // float vector
if indexType == INDEX_FAISS_IDMAP { // float vector
indexParams["dim"] = strconv.Itoa(dim)
indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize)
} else if indexType == IndexFaissIVFFlat {
} else if indexType == INDEX_FAISS_IVFFLAT {
indexParams["dim"] = strconv.Itoa(dim)
indexParams["nlist"] = strconv.Itoa(nlist)
} else if indexType == IndexFaissIVFPQ {
} else if indexType == INDEX_FAISS_IVFPQ {
indexParams["dim"] = strconv.Itoa(dim)
indexParams["nlist"] = strconv.Itoa(nlist)
indexParams["m"] = strconv.Itoa(m)
indexParams["nbits"] = strconv.Itoa(nbits)
indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize)
} else if indexType == IndexFaissIVFSQ8 {
} else if indexType == INDEX_FAISS_IVFSQ8 {
indexParams["dim"] = strconv.Itoa(dim)
indexParams["nlist"] = strconv.Itoa(nlist)
indexParams["nbits"] = strconv.Itoa(nbits)
indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize)
} else if indexType == IndexFaissIVFSQ8H {
} else if indexType == INDEX_FAISS_IVFSQ8H {
// TODO: enable gpu
} else if indexType == IndexNsg {
} else if indexType == INDEX_NSG {
indexParams["dim"] = strconv.Itoa(dim)
indexParams["nlist"] = strconv.Itoa(163)
indexParams["nprobe"] = strconv.Itoa(nprobe)
@ -130,36 +129,36 @@ func generateParams(indexType, metricType string) (map[string]string, map[string
indexParams["search_length"] = strconv.Itoa(40)
indexParams["out_degree"] = strconv.Itoa(30)
indexParams["candidate_pool_size"] = strconv.Itoa(100)
} else if indexType == IndexHNSW {
} else if indexType == INDEX_HNSW {
indexParams["dim"] = strconv.Itoa(dim)
indexParams["m"] = strconv.Itoa(16)
indexParams["efConstruction"] = strconv.Itoa(efConstruction)
indexParams["ef"] = strconv.Itoa(ef)
} else if indexType == IndexRHNSWFlat {
} else if indexType == INDEX_RHNSWFlat {
indexParams["dim"] = strconv.Itoa(dim)
indexParams["m"] = strconv.Itoa(16)
indexParams["efConstruction"] = strconv.Itoa(efConstruction)
indexParams["ef"] = strconv.Itoa(ef)
indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize)
} else if indexType == IndexRHNSWPQ {
} else if indexType == INDEX_RHNSWPQ {
indexParams["dim"] = strconv.Itoa(dim)
indexParams["m"] = strconv.Itoa(16)
indexParams["efConstruction"] = strconv.Itoa(efConstruction)
indexParams["ef"] = strconv.Itoa(ef)
indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize)
indexParams["PQM"] = strconv.Itoa(8)
} else if indexType == IndexRHNSWSQ {
} else if indexType == INDEX_RHNSWSQ {
indexParams["dim"] = strconv.Itoa(dim)
indexParams["m"] = strconv.Itoa(16)
indexParams["efConstruction"] = strconv.Itoa(efConstruction)
indexParams["ef"] = strconv.Itoa(ef)
indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize)
} else if indexType == IndexANNOY {
} else if indexType == INDEX_ANNOY {
indexParams["dim"] = strconv.Itoa(dim)
indexParams["n_trees"] = strconv.Itoa(4)
indexParams["search_k"] = strconv.Itoa(100)
indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize)
} else if indexType == IndexNGTPANNG {
} else if indexType == INDEX_NGTPANNG {
indexParams["dim"] = strconv.Itoa(dim)
indexParams["edge_size"] = strconv.Itoa(edgeSize)
indexParams["epsilon"] = fmt.Sprint(epsilon)
@ -167,7 +166,7 @@ func generateParams(indexType, metricType string) (map[string]string, map[string
indexParams["forcedly_pruned_edge_size"] = strconv.Itoa(60)
indexParams["selectively_pruned_edge_size"] = strconv.Itoa(30)
indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize)
} else if indexType == IndexNGTONNG {
} else if indexType == INDEX_NGTONNG {
indexParams["dim"] = strconv.Itoa(dim)
indexParams["edge_size"] = strconv.Itoa(edgeSize)
indexParams["epsilon"] = fmt.Sprint(epsilon)
@ -175,13 +174,13 @@ func generateParams(indexType, metricType string) (map[string]string, map[string
indexParams["outgoing_edge_size"] = strconv.Itoa(5)
indexParams["incoming_edge_size"] = strconv.Itoa(40)
indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize)
} else if indexType == IndexFaissBinIVFFlat { // binary vector
} else if indexType == INDEX_FAISS_BIN_IVFFLAT { // binary vector
indexParams["dim"] = strconv.Itoa(dim)
indexParams["nlist"] = strconv.Itoa(nlist)
indexParams["m"] = strconv.Itoa(m)
indexParams["nbits"] = strconv.Itoa(nbits)
indexParams["SLICE_SIZE"] = strconv.Itoa(sliceSize)
} else if indexType == IndexFaissBinIDMap {
} else if indexType == INDEX_FAISS_BIN_IDMAP {
indexParams["dim"] = strconv.Itoa(dim)
} else {
panic("")

44
internal/indexnode/indexnode.go → internal/indexbuilder/indexbuilder.go
View File

@ -1,4 +1,4 @@
package indexnode
package indexbuilder
import (
"context"
@ -9,11 +9,6 @@ import (
"sync"
"time"
"github.com/zilliztech/milvus-distributed/internal/indexservice"
"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio"
"go.etcd.io/etcd/clientv3"
@ -29,43 +24,6 @@ import (
type UniqueID = typeutil.UniqueID
type Timestamp = typeutil.Timestamp
type IndexNode struct {
Builder
serviceClient indexservice.Interface // method factory
}
func (i *IndexNode) Init() {
panic("implement me")
}
func (i *IndexNode) Start() {
panic("implement me")
}
func (i *IndexNode) Stop() {
panic("implement me")
}
func (i *IndexNode) GetServiceStates() (internalpb2.ServiceStates, error) {
panic("implement me")
}
func (i *IndexNode) GetTimeTickChannel() (string, error) {
panic("implement me")
}
func (i *IndexNode) GetStatisticsChannel() (string, error) {
panic("implement me")
}
func (i *IndexNode) BuildIndex(req indexpb.BuildIndexRequest) (indexpb.BuildIndexResponse, error) {
panic("implement me")
}
func CreateIndexNode(ctx context.Context) (Interface, error) {
return &IndexNode{}, nil
}
type Builder struct {
loopCtx context.Context
loopCancel func()

16
internal/indexnode/indexnode_test.go → internal/indexbuilder/indexbuilder_test.go
View File

@ -1,4 +1,4 @@
package indexnode
package indexbuilder
import (
"context"
@ -14,7 +14,7 @@ import (
"go.uber.org/zap"
"github.com/stretchr/testify/assert"
indexnodeclient "github.com/zilliztech/milvus-distributed/internal/indexnode/client"
indexbuilderclient "github.com/zilliztech/milvus-distributed/internal/indexbuilder/client"
"github.com/zilliztech/milvus-distributed/internal/master"
"github.com/zilliztech/milvus-distributed/internal/proto/indexbuilderpb"
)
@ -22,7 +22,7 @@ import (
var ctx context.Context
var cancel func()
var buildClient *indexnodeclient.Client
var buildClient *indexbuilderclient.Client
var builderServer *Builder
@ -88,7 +88,7 @@ func setup() {
startBuilder(ctx)
addr := Params.Address
var err error
buildClient, err = indexnodeclient.NewBuildIndexClient(ctx, addr)
buildClient, err = indexbuilderclient.NewBuildIndexClient(ctx, addr)
if err != nil {
panic("Create buildClient Failed!")
}
@ -117,15 +117,17 @@ func TestBuilder_GRPC(t *testing.T) {
indexID, err := buildClient.BuildIndexWithoutID(columnDataPaths, typeParams, indexParams)
assert.Nil(t, err)
time.Sleep(time.Second * 3)
select {
case <-time.After(time.Second * 3):
}
description, err := buildClient.DescribeIndex(indexID)
assert.Nil(t, err)
assert.Equal(t, indexbuilderpb.IndexStatus_INPROGRESS, description.Status)
assert.Equal(t, indexbuilderpb.IndexStatus_FINISHED, description.Status)
assert.Equal(t, indexID, description.ID)
indexDataPaths, err := buildClient.GetIndexFilePaths(indexID)
assert.Nil(t, err)
assert.Nil(t, indexDataPaths)
assert.NotNil(t, indexDataPaths)
}

2
internal/indexnode/meta_table.go → internal/indexbuilder/meta_table.go
View File

@ -1,4 +1,4 @@
package indexnode
package indexbuilder
import (
"fmt"

14
internal/indexnode/paramtable.go → internal/indexbuilder/paramtable.go
View File

@ -1,4 +1,4 @@
package indexnode
package indexbuilder
import (
"net"
@ -19,6 +19,7 @@ type ParamTable struct {
MetaRootPath string
MinIOAddress string
MinIOPort int
MinIOAccessKeyID string
MinIOSecretAccessKey string
MinIOUseSSL bool
@ -106,6 +107,17 @@ func (pt *ParamTable) initMinIOAddress() {
pt.MinIOAddress = ret
}
func (pt *ParamTable) initMinIOPort() {
ret, err := pt.Load("_MinIOPort")
if err != nil {
panic(err)
}
pt.MinIOPort, err = strconv.Atoi(ret)
if err != nil {
panic(err)
}
}
func (pt *ParamTable) initMinIOAccessKeyID() {
ret, err := pt.Load("minio.accessKeyID")
if err != nil {

14
internal/indexnode/paramtable_test.go → internal/indexbuilder/paramtable_test.go
View File

@ -1,7 +1,6 @@
package indexnode
package indexbuilder
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
@ -13,7 +12,7 @@ func TestParamTable_Init(t *testing.T) {
func TestParamTable_Address(t *testing.T) {
address := Params.Address
fmt.Println(address)
assert.Equal(t, address, "localhost:31000")
}
func TestParamTable_Port(t *testing.T) {
@ -27,8 +26,13 @@ func TestParamTable_MetaRootPath(t *testing.T) {
}
func TestParamTable_MinIOAddress(t *testing.T) {
address := Params.MinIOAddress
fmt.Println(address)
address := Params.MinIOAccessKeyID
assert.Equal(t, address, "localhost")
}
func TestParamTable_MinIOPort(t *testing.T) {
port := Params.MinIOPort
assert.Equal(t, port, 9000)
}
func TestParamTable_MinIOAccessKeyID(t *testing.T) {

5
internal/indexnode/query_result.go → internal/indexbuilder/query_result.go
View File

@ -1,4 +1,4 @@
package indexnode
package indexbuilder
/*
@ -13,10 +13,9 @@ package indexnode
*/
import "C"
import (
"github.com/zilliztech/milvus-distributed/internal/errors"
"strconv"
"unsafe"
"github.com/zilliztech/milvus-distributed/internal/errors"
)
type QueryResult interface {

2
internal/indexnode/retry.go → internal/indexbuilder/retry.go
View File

@ -1,4 +1,4 @@
package indexnode
package indexbuilder
import (
"log"

7
internal/indexnode/task.go → internal/indexbuilder/task.go
View File

@ -1,4 +1,4 @@
package indexnode
package indexbuilder
import (
"context"
@ -219,10 +219,7 @@ func (it *IndexBuildTask) Execute() error {
storageBlobs := getStorageBlobs(blobs)
var insertCodec storage.InsertCodec
partitionID, segmentID, insertData, err2 := insertCodec.Deserialize(storageBlobs)
if err2 != nil {
return err2
}
partitionID, segmentID, insertData, err := insertCodec.Deserialize(storageBlobs)
if len(insertData.Data) != 1 {
return errors.New("we expect only one field in deserialized insert data")
}

2
internal/indexnode/task_scheduler.go → internal/indexbuilder/task_scheduler.go
View File

@ -1,4 +1,4 @@
package indexnode
package indexbuilder
import (
"container/list"

13
internal/indexnode/interface.go
View File

@ -1,13 +0,0 @@
package indexnode
import (
"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
)
type ServiceBase = typeutil.Service
type Interface interface {
ServiceBase
BuildIndex(req indexpb.BuildIndexRequest) (indexpb.BuildIndexResponse, error)
}

58
internal/indexservice/indexservice.go
View File

@ -1,58 +0,0 @@
package indexservice
import (
"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
)
type IndexService struct {
// implement Service
//nodeClients [] .Interface
// factory method
}
func (i IndexService) Init() {
panic("implement me")
}
func (i IndexService) Start() {
panic("implement me")
}
func (i IndexService) Stop() {
panic("implement me")
}
func (i IndexService) GetServiceStates() (internalpb2.ServiceStates, error) {
panic("implement me")
}
func (i IndexService) GetTimeTickChannel() (string, error) {
panic("implement me")
}
func (i IndexService) GetStatisticsChannel() (string, error) {
panic("implement me")
}
func (i IndexService) RegisterNode(req indexpb.RegisterNodeRequest) (indexpb.RegisterNodeResponse, error) {
panic("implement me")
}
func (i IndexService) BuildIndex(req indexpb.BuildIndexRequest) (indexpb.BuildIndexResponse, error) {
panic("implement me")
}
func (i IndexService) GetIndexStates(req indexpb.IndexStatesRequest) (indexpb.IndexStatesResponse, error) {
panic("implement me")
}
func (i IndexService) GetIndexFilePaths(req indexpb.IndexFilePathRequest) (indexpb.IndexFilePathsResponse, error) {
panic("implement me")
}
func NewIndexServiceImpl() Interface {
return &IndexService{}
}

16
internal/indexservice/interface.go
View File

@ -1,16 +0,0 @@
package indexservice
import (
"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
)
type ServiceBase = typeutil.Service
type Interface interface {
ServiceBase
RegisterNode(req indexpb.RegisterNodeRequest) (indexpb.RegisterNodeResponse, error)
BuildIndex(req indexpb.BuildIndexRequest) (indexpb.BuildIndexResponse, error)
GetIndexStates(req indexpb.IndexStatesRequest) (indexpb.IndexStatesResponse, error)
GetIndexFilePaths(req indexpb.IndexFilePathRequest) (indexpb.IndexFilePathsResponse, error)
}

2
internal/master/client.go
View File

@ -4,7 +4,7 @@ import (
"sync"
"time"
buildindexclient "github.com/zilliztech/milvus-distributed/internal/indexnode/client"
buildindexclient "github.com/zilliztech/milvus-distributed/internal/indexbuilder/client"
"github.com/zilliztech/milvus-distributed/internal/proto/indexbuilderpb"
writerclient "github.com/zilliztech/milvus-distributed/internal/writenode/client"
)

4
internal/master/master.go
View File

@ -12,7 +12,7 @@ import (
"github.com/zilliztech/milvus-distributed/internal/querynode/client"
indexnodeclient "github.com/zilliztech/milvus-distributed/internal/indexnode/client"
indexbuilderclient "github.com/zilliztech/milvus-distributed/internal/indexbuilder/client"
writerclient "github.com/zilliztech/milvus-distributed/internal/writenode/client"
@ -185,7 +185,7 @@ func CreateServer(ctx context.Context) (*Master, error) {
if err != nil {
return nil, err
}
buildIndexClient, err := indexnodeclient.NewBuildIndexClient(ctx, Params.IndexBuilderAddress)
buildIndexClient, err := indexbuilderclient.NewBuildIndexClient(ctx, Params.IndexBuilderAddress)
if err != nil {
return nil, err
}

1
internal/proto/etcd_meta.proto
View File

@ -27,6 +27,7 @@ message CollectionMeta {
uint64 create_time=3;
repeated int64 segmentIDs=4;
repeated string partition_tags=5;
repeated int64 partitionIDs=6;
}
message FieldBinlogFiles {

101
internal/proto/etcdpb/etcd_meta.pb.go
View File

@ -147,6 +147,7 @@ type CollectionMeta struct {
CreateTime uint64 `protobuf:"varint,3,opt,name=create_time,json=createTime,proto3" json:"create_time,omitempty"`
SegmentIDs []int64 `protobuf:"varint,4,rep,packed,name=segmentIDs,proto3" json:"segmentIDs,omitempty"`
PartitionTags []string `protobuf:"bytes,5,rep,name=partition_tags,json=partitionTags,proto3" json:"partition_tags,omitempty"`
PartitionIDs []int64 `protobuf:"varint,6,rep,packed,name=partitionIDs,proto3" json:"partitionIDs,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -212,6 +213,13 @@ func (m *CollectionMeta) GetPartitionTags() []string {
return nil
}
func (m *CollectionMeta) GetPartitionIDs() []int64 {
if m != nil {
return m.PartitionIDs
}
return nil
}
type FieldBinlogFiles struct {
FieldID int64 `protobuf:"varint,1,opt,name=fieldID,proto3" json:"fieldID,omitempty"`
BinlogFiles []string `protobuf:"bytes,2,rep,name=binlog_files,json=binlogFiles,proto3" json:"binlog_files,omitempty"`
@ -461,50 +469,51 @@ func init() {
func init() { proto.RegisterFile("etcd_meta.proto", fileDescriptor_975d306d62b73e88) }
var fileDescriptor_975d306d62b73e88 = []byte{
// 716 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x5d, 0x6f, 0xe2, 0x46,
0x14, 0x15, 0x38, 0x81, 0xf8, 0x42, 0x20, 0x99, 0xbe, 0xb8, 0x69, 0xda, 0x12, 0xa2, 0xb4, 0x48,
0x55, 0x41, 0x4a, 0xa5, 0x4a, 0x7d, 0x68, 0xd5, 0x26, 0x34, 0x12, 0xaa, 0xda, 0x50, 0x13, 0xed,
0xc3, 0xbe, 0x58, 0x83, 0x7d, 0x03, 0x23, 0x79, 0xc6, 0xec, 0xcc, 0x38, 0x1f, 0x3c, 0xed, 0x1f,
0xd8, 0x3f, 0xb0, 0x7f, 0x68, 0x7f, 0xd5, 0x4a, 0xab, 0x99, 0x31, 0x06, 0xb2, 0x3c, 0xec, 0xe3,
0x9c, 0x73, 0xee, 0xf5, 0xb9, 0xe7, 0xce, 0x18, 0xda, 0xa8, 0xe3, 0x24, 0xe2, 0xa8, 0x69, 0x7f,
0x21, 0x33, 0x9d, 0x91, 0x63, 0xce, 0xd2, 0x87, 0x5c, 0xb9, 0x53, 0xdf, 0xb0, 0x27, 0x5f, 0x31,
0x91, 0xe0, 0x53, 0x34, 0xcd, 0x59, 0x9a, 0xa0, 0x74, 0xcc, 0x49, 0x33, 0xce, 0x38, 0xcf, 0xc4,
0xea, 0xa4, 0xe2, 0x39, 0xf2, 0xa2, 0x47, 0xf7, 0x7d, 0x05, 0xe0, 0x0e, 0x05, 0x15, 0xfa, 0x5f,
0xd4, 0x94, 0xb4, 0xa0, 0x3a, 0x1a, 0x06, 0x95, 0x4e, 0xa5, 0xe7, 0x85, 0xd5, 0xd1, 0x90, 0xfc,
0x00, 0x6d, 0x91, 0xf3, 0xe8, 0x4d, 0x8e, 0xf2, 0x39, 0x12, 0x59, 0x82, 0x2a, 0xa8, 0x5a, 0xf2,
0x50, 0xe4, 0xfc, 0x7f, 0x83, 0xfe, 0x67, 0x40, 0xf2, 0x13, 0x1c, 0x33, 0xa1, 0x50, 0xea, 0x28,
0x9e, 0x53, 0x21, 0x30, 0x1d, 0x0d, 0x55, 0xe0, 0x75, 0xbc, 0x9e, 0x1f, 0x1e, 0x39, 0xe2, 0xba,
0xc4, 0xc9, 0x8f, 0xd0, 0x76, 0x0d, 0x4b, 0x6d, 0xb0, 0xd7, 0xa9, 0xf4, 0xfc, 0xb0, 0x65, 0xe1,
0x52, 0xd9, 0x7d, 0x5b, 0x01, 0x7f, 0x2c, 0xb3, 0xa7, 0xe7, 0x9d, 0xde, 0x7e, 0x85, 0x3a, 0x4d,
0x12, 0x89, 0xca, 0x79, 0x6a, 0x5c, 0x9e, 0xf6, 0xb7, 0x02, 0x29, 0xa6, 0xfe, 0xcb, 0x69, 0xc2,
0x95, 0xd8, 0x78, 0x95, 0xa8, 0xf2, 0x74, 0x97, 0x57, 0x47, 0xac, 0xbd, 0x76, 0x3f, 0x54, 0xa0,
0x75, 0x9d, 0xa5, 0x29, 0xc6, 0x9a, 0x65, 0x62, 0xa7, 0x8f, 0xdf, 0xa1, 0xe6, 0x22, 0x2d, 0x6c,
0x5c, 0x6c, 0xdb, 0x28, 0xe2, 0x5e, 0x37, 0x99, 0x58, 0x20, 0x2c, 0x8a, 0xc8, 0xf7, 0xd0, 0x88,
0x25, 0x52, 0x8d, 0x91, 0x66, 0x1c, 0x03, 0xaf, 0x53, 0xe9, 0xed, 0x85, 0xe0, 0xa0, 0x3b, 0xc6,
0x91, 0x7c, 0x07, 0xa0, 0x70, 0xc6, 0x51, 0x68, 0x63, 0x74, 0xaf, 0xe3, 0xf5, 0xbc, 0x70, 0x03,
0x21, 0x17, 0xd0, 0x5a, 0x50, 0xa9, 0x99, 0xe9, 0x1d, 0x69, 0x3a, 0x53, 0xc1, 0xbe, 0x1d, 0xe6,
0xb0, 0x44, 0xef, 0xe8, 0x4c, 0x75, 0x6f, 0xe1, 0xe8, 0x86, 0x61, 0x9a, 0x5c, 0x31, 0x91, 0x66,
0xb3, 0x1b, 0x96, 0xa2, 0x22, 0x01, 0xd4, 0xef, 0x0d, 0x56, 0xce, 0xb3, 0x3a, 0x92, 0x33, 0x68,
0x4e, 0xad, 0x30, 0xba, 0x37, 0xca, 0xa0, 0x6a, 0x5b, 0x36, 0xa6, 0xeb, 0xe2, 0xee, 0xc7, 0x2a,
0x34, 0x26, 0xce, 0x86, 0xcd, 0xe5, 0x14, 0xfc, 0xd2, 0x55, 0xd1, 0x6e, 0x0d, 0x90, 0x2e, 0x34,
0xe3, 0x32, 0x82, 0xd1, 0xb0, 0xb8, 0x46, 0x5b, 0x18, 0x39, 0x87, 0xc3, 0xad, 0x49, 0x6c, 0x18,
0x7e, 0xd8, 0xdc, 0x1c, 0xc4, 0x88, 0x8a, 0xbd, 0x45, 0x4a, 0x53, 0xa9, 0xed, 0xdd, 0xd9, 0x0f,
0x9b, 0x05, 0x38, 0x31, 0x98, 0x0d, 0xb5, 0x10, 0xa1, 0x48, 0x82, 0x7d, 0x2b, 0x81, 0x02, 0xfa,
0x5b, 0x24, 0xe4, 0x1b, 0xf0, 0xb3, 0x05, 0x0a, 0x97, 0x79, 0xcd, 0x66, 0x7e, 0x60, 0x00, 0x9b,
0xf8, 0xb7, 0x00, 0x71, 0x9a, 0xa9, 0x62, 0x23, 0x75, 0xcb, 0xfa, 0x16, 0xb1, 0xf4, 0xd7, 0x70,
0x60, 0x1e, 0x85, 0xcc, 0x1e, 0x55, 0x70, 0xe0, 0x62, 0x13, 0x39, 0x0f, 0xb3, 0x47, 0x65, 0x28,
0x8e, 0x3c, 0x52, 0x6c, 0x89, 0x81, 0xef, 0x28, 0x8e, 0x7c, 0xc2, 0x96, 0x48, 0x6e, 0xe1, 0x78,
0x23, 0xd1, 0x68, 0x41, 0xf5, 0x5c, 0x05, 0xd0, 0xf1, 0x7a, 0x8d, 0xcb, 0xf3, 0xfe, 0x67, 0x2f,
0xb9, 0xff, 0x72, 0x57, 0x61, 0x7b, 0x9d, 0xfd, 0xd8, 0xd4, 0x76, 0xdf, 0x55, 0xa1, 0x65, 0x55,
0x23, 0xf3, 0xe6, 0xbf, 0x60, 0x05, 0x1b, 0xdb, 0xae, 0x6e, 0x6f, 0x3b, 0x80, 0xba, 0xfd, 0x71,
0x8c, 0x86, 0x36, 0x72, 0x2f, 0x5c, 0x1d, 0xc9, 0x10, 0x9a, 0xee, 0x97, 0xb2, 0xa0, 0x92, 0x72,
0x77, 0xfd, 0x1a, 0x97, 0x67, 0x3b, 0x5f, 0xda, 0x3f, 0xf8, 0xfc, 0x8a, 0xa6, 0x39, 0x8e, 0x29,
0x93, 0x61, 0xc3, 0x96, 0x8d, 0x6d, 0x15, 0xf9, 0x0d, 0x6a, 0x4a, 0x53, 0x9d, 0x2b, 0xbb, 0x89,
0xd6, 0xcb, 0x7a, 0x85, 0xf2, 0x81, 0xc5, 0xd8, 0xb7, 0x83, 0x4c, 0xac, 0x30, 0x2c, 0x0a, 0x48,
0x0f, 0x8e, 0x9c, 0x81, 0x8d, 0xd4, 0x6a, 0xf6, 0x32, 0xb6, 0x2c, 0x5e, 0xe6, 0x71, 0xf5, 0xe7,
0xeb, 0x3f, 0x66, 0x4c, 0xcf, 0xf3, 0xa9, 0xf1, 0x33, 0x58, 0xb2, 0x34, 0x65, 0x4b, 0x8d, 0xf1,
0x7c, 0xe0, 0xbe, 0xf5, 0x73, 0xc2, 0x94, 0x96, 0x6c, 0x9a, 0x6b, 0x4c, 0x06, 0x4c, 0x68, 0x94,
0x82, 0xa6, 0x03, 0x6b, 0x60, 0x60, 0x12, 0x5f, 0x4c, 0xa7, 0x35, 0x7b, 0xfa, 0xe5, 0x53, 0x00,
0x00, 0x00, 0xff, 0xff, 0x0a, 0x10, 0xb5, 0x6d, 0x6a, 0x05, 0x00, 0x00,
// 726 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4d, 0x6f, 0xea, 0x46,
0x14, 0x15, 0x38, 0x81, 0xf8, 0x42, 0x20, 0x99, 0x6e, 0xdc, 0x34, 0x6d, 0x09, 0x51, 0x5a, 0xa4,
0xaa, 0x20, 0xa5, 0x52, 0xa5, 0x2e, 0x5a, 0xb5, 0x09, 0x8d, 0x84, 0xaa, 0x36, 0xd4, 0xa0, 0x2e,
0xba, 0xb1, 0x06, 0xfb, 0x06, 0x46, 0xf2, 0x8c, 0xe9, 0xcc, 0x38, 0x1f, 0xac, 0xfa, 0x07, 0xde,
0x1f, 0x78, 0x7f, 0xf1, 0xfd, 0x86, 0x27, 0x3d, 0xcd, 0x8c, 0x31, 0x90, 0xc7, 0xe2, 0x2d, 0xe7,
0xdc, 0x73, 0xaf, 0xcf, 0x3d, 0x67, 0x3c, 0xd0, 0x46, 0x1d, 0x27, 0x11, 0x47, 0x4d, 0xfb, 0x4b,
0x99, 0xe9, 0x8c, 0x9c, 0x72, 0x96, 0x3e, 0xe6, 0xca, 0x9d, 0xfa, 0xa6, 0x7a, 0xf6, 0x19, 0x13,
0x09, 0x3e, 0x47, 0xb3, 0x9c, 0xa5, 0x09, 0x4a, 0x57, 0x39, 0x6b, 0xc6, 0x19, 0xe7, 0x99, 0x58,
0x9f, 0x54, 0xbc, 0x40, 0x5e, 0xcc, 0xe8, 0xbe, 0xad, 0x00, 0x4c, 0x51, 0x50, 0xa1, 0xff, 0x44,
0x4d, 0x49, 0x0b, 0xaa, 0xa3, 0x61, 0x50, 0xe9, 0x54, 0x7a, 0x5e, 0x58, 0x1d, 0x0d, 0xc9, 0x37,
0xd0, 0x16, 0x39, 0x8f, 0xfe, 0xcb, 0x51, 0xbe, 0x44, 0x22, 0x4b, 0x50, 0x05, 0x55, 0x5b, 0x3c,
0x16, 0x39, 0xff, 0xdb, 0xa0, 0x7f, 0x19, 0x90, 0x7c, 0x07, 0xa7, 0x4c, 0x28, 0x94, 0x3a, 0x8a,
0x17, 0x54, 0x08, 0x4c, 0x47, 0x43, 0x15, 0x78, 0x1d, 0xaf, 0xe7, 0x87, 0x27, 0xae, 0x70, 0x5b,
0xe2, 0xe4, 0x5b, 0x68, 0xbb, 0x81, 0x25, 0x37, 0x38, 0xe8, 0x54, 0x7a, 0x7e, 0xd8, 0xb2, 0x70,
0xc9, 0xec, 0xfe, 0x5f, 0x01, 0x7f, 0x2c, 0xb3, 0xe7, 0x97, 0xbd, 0xda, 0x7e, 0x84, 0x3a, 0x4d,
0x12, 0x89, 0xca, 0x69, 0x6a, 0x5c, 0x9f, 0xf7, 0x77, 0x0c, 0x29, 0xb6, 0xfe, 0xcd, 0x71, 0xc2,
0x35, 0xd9, 0x68, 0x95, 0xa8, 0xf2, 0x74, 0x9f, 0x56, 0x57, 0xd8, 0x68, 0xed, 0xbe, 0xab, 0x40,
0xeb, 0x36, 0x4b, 0x53, 0x8c, 0x35, 0xcb, 0xc4, 0x5e, 0x1d, 0x3f, 0x43, 0xcd, 0x59, 0x5a, 0xc8,
0xb8, 0xda, 0x95, 0x51, 0xd8, 0xbd, 0x19, 0x32, 0xb1, 0x40, 0x58, 0x34, 0x91, 0xaf, 0xa1, 0x11,
0x4b, 0xa4, 0x1a, 0x23, 0xcd, 0x38, 0x06, 0x5e, 0xa7, 0xd2, 0x3b, 0x08, 0xc1, 0x41, 0x53, 0xc6,
0x91, 0x7c, 0x05, 0xa0, 0x70, 0xce, 0x51, 0x68, 0x23, 0xf4, 0xa0, 0xe3, 0xf5, 0xbc, 0x70, 0x0b,
0x21, 0x57, 0xd0, 0x5a, 0x52, 0xa9, 0x99, 0x99, 0x1d, 0x69, 0x3a, 0x57, 0xc1, 0xa1, 0x5d, 0xe6,
0xb8, 0x44, 0xa7, 0x74, 0xae, 0x48, 0x17, 0x9a, 0x25, 0x60, 0x06, 0xd5, 0xec, 0xa0, 0x1d, 0xac,
0x7b, 0x0f, 0x27, 0x77, 0x0c, 0xd3, 0xe4, 0x86, 0x89, 0x34, 0x9b, 0xdf, 0xb1, 0x14, 0x15, 0x09,
0xa0, 0xfe, 0x60, 0xb0, 0x72, 0xe7, 0xf5, 0x91, 0x5c, 0x40, 0x73, 0x66, 0x89, 0xd1, 0x83, 0x61,
0x06, 0x55, 0xfb, 0xd9, 0xc6, 0x6c, 0xd3, 0xdc, 0x7d, 0x5f, 0x85, 0xc6, 0xc4, 0x49, 0xb5, 0xde,
0x9d, 0x83, 0x5f, 0x2a, 0x2f, 0xc6, 0x6d, 0x00, 0x23, 0x31, 0x2e, 0x6d, 0x1a, 0x0d, 0x8b, 0xab,
0xb6, 0x83, 0x91, 0x4b, 0x38, 0xde, 0xd9, 0xd6, 0x1a, 0xe6, 0x6f, 0xed, 0x31, 0xa5, 0x73, 0x43,
0x2a, 0xb2, 0x8d, 0x94, 0xa6, 0x52, 0xdb, 0xfb, 0x75, 0x18, 0x36, 0x0b, 0x70, 0x62, 0x30, 0x6b,
0x7c, 0x41, 0x42, 0x91, 0x04, 0x87, 0x96, 0x02, 0x05, 0xf4, 0xbb, 0x48, 0xc8, 0x17, 0xe0, 0x67,
0x4b, 0x14, 0x2e, 0x97, 0x9a, 0xcd, 0xe5, 0xc8, 0x00, 0x36, 0x95, 0x2f, 0x01, 0xe2, 0x34, 0x53,
0x45, 0x6a, 0x75, 0x5b, 0xf5, 0x2d, 0x62, 0xcb, 0x9f, 0xc3, 0x91, 0xf9, 0x71, 0x64, 0xf6, 0xa4,
0x82, 0x23, 0x67, 0x9b, 0xc8, 0x79, 0x98, 0x3d, 0x29, 0x53, 0xe2, 0xc8, 0x23, 0xc5, 0x56, 0x18,
0xf8, 0xae, 0xc4, 0x91, 0x4f, 0xd8, 0x0a, 0xc9, 0x3d, 0x9c, 0x6e, 0x39, 0x1a, 0x2d, 0xa9, 0x5e,
0xa8, 0x00, 0x3a, 0x5e, 0xaf, 0x71, 0x7d, 0xd9, 0xff, 0xe8, 0x6f, 0xef, 0xbf, 0xce, 0x2a, 0x6c,
0x6f, 0xbc, 0x1f, 0x9b, 0xde, 0xee, 0x9b, 0x2a, 0xb4, 0x2c, 0x6b, 0x64, 0xde, 0x85, 0x4f, 0x88,
0x60, 0x2b, 0xed, 0xea, 0x6e, 0xda, 0x01, 0xd4, 0xed, 0xe3, 0x32, 0x1a, 0x5a, 0xcb, 0xbd, 0x70,
0x7d, 0x24, 0x43, 0x68, 0xba, 0x67, 0x67, 0x49, 0x25, 0xe5, 0xee, 0x8a, 0x36, 0xae, 0x2f, 0xf6,
0xfe, 0x8d, 0x7f, 0xe0, 0xcb, 0x3f, 0x34, 0xcd, 0x71, 0x4c, 0x99, 0x0c, 0x1b, 0xb6, 0x6d, 0x6c,
0xbb, 0xc8, 0x4f, 0x50, 0x53, 0x9a, 0xea, 0x5c, 0xd9, 0x24, 0x5a, 0xaf, 0xfb, 0x15, 0xca, 0x47,
0x16, 0x63, 0xdf, 0x2e, 0x32, 0xb1, 0xc4, 0xb0, 0x68, 0x20, 0x3d, 0x38, 0x71, 0x02, 0xb6, 0x5c,
0xab, 0xd9, 0xcb, 0xd8, 0xb2, 0x78, 0xe9, 0xc7, 0xcd, 0xaf, 0xff, 0xfe, 0x32, 0x67, 0x7a, 0x91,
0xcf, 0x8c, 0x9e, 0xc1, 0x8a, 0xa5, 0x29, 0x5b, 0x69, 0x8c, 0x17, 0x03, 0xf7, 0xad, 0xef, 0x13,
0xa6, 0xb4, 0x64, 0xb3, 0x5c, 0x63, 0x32, 0x60, 0x42, 0xa3, 0x14, 0x34, 0x1d, 0x58, 0x01, 0x03,
0xe3, 0xf8, 0x72, 0x36, 0xab, 0xd9, 0xd3, 0x0f, 0x1f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x12, 0x8d,
0xcf, 0x40, 0x8e, 0x05, 0x00, 0x00,
}

27
internal/proto/index_service.proto
View File

@ -47,18 +47,6 @@ message BuildIndexResponse {
int64 indexID = 2;
}
message BuildIndexCmd {
int64 indexID = 1;
BuildIndexRequest req = 2;
}
message BuildIndexNotification {
common.Status status = 1;
int64 indexID = 2;
repeated string index_file_paths = 3;
}
message IndexFilePathRequest {
int64 indexID = 1;
}
@ -92,18 +80,5 @@ service IndexService {
rpc BuildIndex(BuildIndexRequest) returns (BuildIndexResponse){}
rpc GetIndexStates(IndexStatesRequest) returns (IndexStatesResponse) {}
rpc GetIndexFilePaths(IndexFilePathRequest) returns (IndexFilePathsResponse){}
rpc NotifyBuildIndex(BuildIndexNotification) returns (common.Status) {}
}
service IndexNode {
/**
* @brief This method is used to create collection
*
* @param CollectionSchema, use to provide collection information to be created.
*
* @return Status
*/
rpc BuildIndex(BuildIndexCmd) returns (common.Status){}
}
}

326
internal/proto/indexpb/index_service.pb.go
View File

@ -342,108 +342,6 @@ func (m *BuildIndexResponse) GetIndexID() int64 {
return 0
}
type BuildIndexCmd struct {
IndexID int64 `protobuf:"varint,1,opt,name=indexID,proto3" json:"indexID,omitempty"`
Req *BuildIndexRequest `protobuf:"bytes,2,opt,name=req,proto3" json:"req,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *BuildIndexCmd) Reset() { *m = BuildIndexCmd{} }
func (m *BuildIndexCmd) String() string { return proto.CompactTextString(m) }
func (*BuildIndexCmd) ProtoMessage() {}
func (*BuildIndexCmd) Descriptor() ([]byte, []int) {
return fileDescriptor_a5d2036b4df73e0a, []int{6}
}
func (m *BuildIndexCmd) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_BuildIndexCmd.Unmarshal(m, b)
}
func (m *BuildIndexCmd) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_BuildIndexCmd.Marshal(b, m, deterministic)
}
func (m *BuildIndexCmd) XXX_Merge(src proto.Message) {
xxx_messageInfo_BuildIndexCmd.Merge(m, src)
}
func (m *BuildIndexCmd) XXX_Size() int {
return xxx_messageInfo_BuildIndexCmd.Size(m)
}
func (m *BuildIndexCmd) XXX_DiscardUnknown() {
xxx_messageInfo_BuildIndexCmd.DiscardUnknown(m)
}
var xxx_messageInfo_BuildIndexCmd proto.InternalMessageInfo
func (m *BuildIndexCmd) GetIndexID() int64 {
if m != nil {
return m.IndexID
}
return 0
}
func (m *BuildIndexCmd) GetReq() *BuildIndexRequest {
if m != nil {
return m.Req
}
return nil
}
type BuildIndexNotification struct {
Status *commonpb.Status `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"`
IndexID int64 `protobuf:"varint,2,opt,name=indexID,proto3" json:"indexID,omitempty"`
IndexFilePaths []string `protobuf:"bytes,3,rep,name=index_file_paths,json=indexFilePaths,proto3" json:"index_file_paths,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *BuildIndexNotification) Reset() { *m = BuildIndexNotification{} }
func (m *BuildIndexNotification) String() string { return proto.CompactTextString(m) }
func (*BuildIndexNotification) ProtoMessage() {}
func (*BuildIndexNotification) Descriptor() ([]byte, []int) {
return fileDescriptor_a5d2036b4df73e0a, []int{7}
}
func (m *BuildIndexNotification) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_BuildIndexNotification.Unmarshal(m, b)
}
func (m *BuildIndexNotification) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_BuildIndexNotification.Marshal(b, m, deterministic)
}
func (m *BuildIndexNotification) XXX_Merge(src proto.Message) {
xxx_messageInfo_BuildIndexNotification.Merge(m, src)
}
func (m *BuildIndexNotification) XXX_Size() int {
return xxx_messageInfo_BuildIndexNotification.Size(m)
}
func (m *BuildIndexNotification) XXX_DiscardUnknown() {
xxx_messageInfo_BuildIndexNotification.DiscardUnknown(m)
}
var xxx_messageInfo_BuildIndexNotification proto.InternalMessageInfo
func (m *BuildIndexNotification) GetStatus() *commonpb.Status {
if m != nil {
return m.Status
}
return nil
}
func (m *BuildIndexNotification) GetIndexID() int64 {
if m != nil {
return m.IndexID
}
return 0
}
func (m *BuildIndexNotification) GetIndexFilePaths() []string {
if m != nil {
return m.IndexFilePaths
}
return nil
}
type IndexFilePathRequest struct {
IndexID int64 `protobuf:"varint,1,opt,name=indexID,proto3" json:"indexID,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
@ -455,7 +353,7 @@ func (m *IndexFilePathRequest) Reset() { *m = IndexFilePathRequest{} }
func (m *IndexFilePathRequest) String() string { return proto.CompactTextString(m) }
func (*IndexFilePathRequest) ProtoMessage() {}
func (*IndexFilePathRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_a5d2036b4df73e0a, []int{8}
return fileDescriptor_a5d2036b4df73e0a, []int{6}
}
func (m *IndexFilePathRequest) XXX_Unmarshal(b []byte) error {
@ -496,7 +394,7 @@ func (m *IndexFilePathsResponse) Reset() { *m = IndexFilePathsResponse{}
func (m *IndexFilePathsResponse) String() string { return proto.CompactTextString(m) }
func (*IndexFilePathsResponse) ProtoMessage() {}
func (*IndexFilePathsResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_a5d2036b4df73e0a, []int{9}
return fileDescriptor_a5d2036b4df73e0a, []int{7}
}
func (m *IndexFilePathsResponse) XXX_Unmarshal(b []byte) error {
@ -555,7 +453,7 @@ func (m *IndexMeta) Reset() { *m = IndexMeta{} }
func (m *IndexMeta) String() string { return proto.CompactTextString(m) }
func (*IndexMeta) ProtoMessage() {}
func (*IndexMeta) Descriptor() ([]byte, []int) {
return fileDescriptor_a5d2036b4df73e0a, []int{10}
return fileDescriptor_a5d2036b4df73e0a, []int{8}
}
func (m *IndexMeta) XXX_Unmarshal(b []byte) error {
@ -633,8 +531,6 @@ func init() {
proto.RegisterType((*IndexStatesResponse)(nil), "milvus.proto.index.IndexStatesResponse")
proto.RegisterType((*BuildIndexRequest)(nil), "milvus.proto.index.BuildIndexRequest")
proto.RegisterType((*BuildIndexResponse)(nil), "milvus.proto.index.BuildIndexResponse")
proto.RegisterType((*BuildIndexCmd)(nil), "milvus.proto.index.BuildIndexCmd")
proto.RegisterType((*BuildIndexNotification)(nil), "milvus.proto.index.BuildIndexNotification")
proto.RegisterType((*IndexFilePathRequest)(nil), "milvus.proto.index.IndexFilePathRequest")
proto.RegisterType((*IndexFilePathsResponse)(nil), "milvus.proto.index.IndexFilePathsResponse")
proto.RegisterType((*IndexMeta)(nil), "milvus.proto.index.IndexMeta")
@ -643,56 +539,52 @@ func init() {
func init() { proto.RegisterFile("index_service.proto", fileDescriptor_a5d2036b4df73e0a) }
var fileDescriptor_a5d2036b4df73e0a = []byte{
// 783 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0xdd, 0x4e, 0xeb, 0x46,
0x10, 0xc6, 0x71, 0x4e, 0x38, 0x19, 0x72, 0xa2, 0xb0, 0x39, 0xaa, 0x50, 0x5a, 0x2a, 0x70, 0x55,
0x88, 0x90, 0x9a, 0x54, 0xa1, 0x6a, 0x2f, 0x2b, 0x42, 0x02, 0xb5, 0x5a, 0x4c, 0xe4, 0x94, 0x5e,
0x50, 0x55, 0x91, 0x63, 0x0f, 0x64, 0x55, 0xff, 0x04, 0xef, 0x1a, 0x01, 0x77, 0x55, 0x9f, 0xa0,
0xb7, 0x7d, 0x8c, 0x5e, 0xf7, 0xe1, 0x2a, 0xaf, 0xd7, 0x89, 0x0d, 0x21, 0x81, 0xb6, 0xe7, 0xce,
0x9e, 0xfd, 0x66, 0xe6, 0x9b, 0x6f, 0x66, 0x76, 0xa1, 0x4e, 0x7d, 0x07, 0xef, 0x46, 0x0c, 0xc3,
0x5b, 0x6a, 0x63, 0x6b, 0x1a, 0x06, 0x3c, 0x20, 0xc4, 0xa3, 0xee, 0x6d, 0xc4, 0x92, 0xbf, 0x96,
0x40, 0x34, 0x2a, 0x76, 0xe0, 0x79, 0x81, 0x9f, 0xd8, 0x1a, 0x55, 0xea, 0x73, 0x0c, 0x7d, 0xcb,
0x4d, 0xfe, 0xb5, 0xdf, 0x14, 0xa8, 0x9b, 0x78, 0x4d, 0x19, 0xc7, 0xd0, 0x08, 0x1c, 0x34, 0xf1,
0x26, 0x42, 0xc6, 0x49, 0x07, 0x8a, 0x63, 0x8b, 0xe1, 0x96, 0xb2, 0xa3, 0x34, 0x37, 0x3a, 0x9f,
0xb6, 0x1e, 0x05, 0x96, 0x31, 0xce, 0xd8, 0x75, 0xd7, 0x62, 0x68, 0x0a, 0x2c, 0xf9, 0x1a, 0xd6,
0x2d, 0xc7, 0x09, 0x91, 0xb1, 0xad, 0x82, 0x70, 0xfb, 0x24, 0xef, 0x26, 0x89, 0x1c, 0x25, 0x18,
0x33, 0x05, 0x6b, 0x97, 0xf0, 0x3e, 0x4f, 0x81, 0x4d, 0x03, 0x9f, 0x21, 0xe9, 0xc2, 0x06, 0xf5,
0x29, 0x1f, 0x4d, 0xad, 0xd0, 0xf2, 0x98, 0xa4, 0xb2, 0xfb, 0x0c, 0x15, 0xdd, 0xa7, 0x7c, 0x20,
0x80, 0x26, 0xd0, 0xd9, 0xb7, 0xd6, 0x02, 0xa2, 0xc7, 0x32, 0x0c, 0xb9, 0xc5, 0x91, 0xa5, 0xd5,
0x6d, 0xc1, 0xba, 0x10, 0x47, 0xef, 0x89, 0xa8, 0xaa, 0x99, 0xfe, 0x6a, 0x7f, 0x2a, 0x50, 0xcf,
0x39, 0x48, 0x2e, 0x87, 0x50, 0x62, 0xdc, 0xe2, 0x51, 0x4a, 0xe3, 0xe3, 0x85, 0xa5, 0x0d, 0x05,
0xc4, 0x94, 0x50, 0xf2, 0x15, 0xbc, 0x89, 0xbf, 0x50, 0xc8, 0x51, 0x7d, 0xaa, 0xa2, 0x83, 0x77,
0xad, 0x79, 0x32, 0x33, 0x01, 0x67, 0xc9, 0xa9, 0x79, 0x72, 0x7f, 0x2b, 0xb0, 0xd9, 0x8d, 0xa8,
0xeb, 0x08, 0xa7, 0xb4, 0x98, 0x6d, 0x00, 0xc7, 0xe2, 0xd6, 0x68, 0x6a, 0xf1, 0x49, 0xac, 0xbc,
0xda, 0x2c, 0x9b, 0xe5, 0xd8, 0x32, 0x88, 0x0d, 0xb1, 0x8a, 0xfc, 0x7e, 0x8a, 0xa9, 0x8a, 0xea,
0x8e, 0xfa, 0x54, 0x45, 0x49, 0xff, 0x7b, 0xbc, 0xff, 0xc9, 0x72, 0x23, 0x1c, 0x58, 0x34, 0x34,
0x21, 0xf6, 0x4a, 0x54, 0x24, 0x3d, 0xa8, 0x24, 0xe3, 0x26, 0x83, 0x14, 0x5f, 0x1a, 0x64, 0x43,
0xb8, 0xc9, 0x5e, 0xd8, 0x40, 0xb2, 0xec, 0xff, 0x8b, 0xb2, 0x19, 0x8d, 0x0a, 0x79, 0x8d, 0xc6,
0xf0, 0x6e, 0x9e, 0xe4, 0xd8, 0x73, 0x9e, 0xef, 0x35, 0xf9, 0x06, 0xd4, 0x10, 0x6f, 0xe4, 0xac,
0x7e, 0xbe, 0xa8, 0x39, 0x4f, 0xc4, 0x36, 0x63, 0x0f, 0xed, 0x0f, 0x05, 0x3e, 0x9a, 0x1f, 0x19,
0x01, 0xa7, 0x57, 0xd4, 0xb6, 0x38, 0x0d, 0xfc, 0xff, 0xb9, 0x1a, 0xd2, 0x84, 0x5a, 0x22, 0xfc,
0x15, 0x75, 0x51, 0x76, 0x58, 0x15, 0x1d, 0xae, 0x0a, 0xfb, 0x09, 0x75, 0x51, 0xb4, 0x59, 0xfb,
0x12, 0xde, 0xeb, 0x59, 0xcb, 0xea, 0x51, 0x8f, 0xab, 0xc8, 0xb9, 0xb0, 0x0f, 0xd4, 0x93, 0x57,
0x54, 0xf1, 0x57, 0x01, 0xca, 0x82, 0xd3, 0x19, 0x72, 0x6b, 0xbe, 0x3f, 0xca, 0xbf, 0xdc, 0x9f,
0x47, 0x3c, 0xb6, 0x01, 0xd0, 0xbf, 0x89, 0x70, 0xc4, 0xa9, 0x87, 0x72, 0xb9, 0xca, 0xc2, 0xf2,
0x23, 0xf5, 0x90, 0x7c, 0x06, 0xef, 0x98, 0x3d, 0x41, 0x27, 0x72, 0x25, 0xa2, 0x28, 0x10, 0x95,
0xd4, 0x28, 0x40, 0x2d, 0xa8, 0x8f, 0xe3, 0xd6, 0x8f, 0xec, 0xc0, 0x9b, 0xba, 0xc8, 0x25, 0xf4,
0x8d, 0x80, 0x6e, 0x8a, 0xa3, 0x63, 0x79, 0x22, 0xf0, 0x72, 0xc8, 0x4a, 0xaf, 0x1d, 0xb2, 0x85,
0xa2, 0xad, 0x2f, 0x12, 0xed, 0xc0, 0x00, 0x98, 0xab, 0x40, 0xde, 0x42, 0xd1, 0x38, 0x37, 0xfa,
0xb5, 0x35, 0x52, 0x81, 0xb7, 0x17, 0x86, 0x3e, 0x1c, 0x5e, 0xf4, 0x7b, 0x35, 0x85, 0x54, 0x01,
0x74, 0x63, 0x60, 0x9e, 0x9f, 0x9a, 0xfd, 0xe1, 0xb0, 0x56, 0x88, 0x4f, 0x4f, 0x74, 0x43, 0x1f,
0x7e, 0xd7, 0xef, 0xd5, 0x54, 0x02, 0x50, 0x3a, 0x39, 0xd2, 0x7f, 0xe8, 0xf7, 0x6a, 0xc5, 0xce,
0xef, 0x45, 0xa8, 0x24, 0x01, 0x93, 0xc7, 0x85, 0xd8, 0x50, 0xc9, 0x5e, 0xd0, 0x64, 0x7f, 0x51,
0x19, 0x0b, 0x5e, 0x91, 0x46, 0x73, 0x35, 0x30, 0x99, 0x38, 0x6d, 0x8d, 0xfc, 0x02, 0x30, 0x57,
0x82, 0xbc, 0x4c, 0xa9, 0xc6, 0xde, 0x2a, 0xd8, 0x2c, 0xbc, 0x0d, 0xd5, 0x53, 0xe4, 0x99, 0xab,
0x9d, 0xec, 0x2d, 0x1f, 0xa7, 0xf4, 0xb1, 0x68, 0xec, 0xaf, 0xc4, 0xcd, 0x92, 0xfc, 0x0a, 0x9b,
0x69, 0x92, 0x59, 0x7b, 0x48, 0xf3, 0x59, 0xff, 0x47, 0xbb, 0xda, 0x38, 0x58, 0x89, 0x64, 0x39,
0xc1, 0x6a, 0xe2, 0xea, 0xb9, 0xcf, 0xc8, 0x76, 0xb0, 0x5c, 0x8f, 0xec, 0x55, 0xd5, 0x58, 0xb6,
0xd4, 0xda, 0x5a, 0xe7, 0x67, 0xb9, 0x89, 0xa2, 0xe3, 0x46, 0xae, 0x39, 0xbb, 0xcb, 0xb3, 0x1c,
0x7b, 0xce, 0x8a, 0xe0, 0xdd, 0xa3, 0xcb, 0x6f, 0xaf, 0x29, 0x9f, 0x44, 0xe3, 0xf8, 0xa4, 0xfd,
0x40, 0x5d, 0x97, 0x3e, 0x70, 0xb4, 0x27, 0xed, 0xc4, 0xeb, 0x0b, 0x87, 0x32, 0x1e, 0xd2, 0x71,
0xc4, 0xd1, 0x69, 0xa7, 0x4f, 0x7c, 0x5b, 0x84, 0x6a, 0x8b, 0x6c, 0xd3, 0xf1, 0xb8, 0x24, 0x7e,
0x0f, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0x12, 0x91, 0xac, 0xbb, 0x09, 0x09, 0x00, 0x00,
// 709 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0xdf, 0x4f, 0x13, 0x41,
0x10, 0xe6, 0x7a, 0xa5, 0xd0, 0xa1, 0x36, 0x65, 0x4b, 0x4c, 0x53, 0xc5, 0x60, 0x8d, 0xd0, 0x90,
0xd8, 0x1a, 0x30, 0xfa, 0x68, 0xa8, 0x2d, 0x78, 0x51, 0x4a, 0x73, 0x15, 0x1f, 0x48, 0x4c, 0xb3,
0xbd, 0x1b, 0xe9, 0xc6, 0xfb, 0xc5, 0xed, 0x1e, 0x01, 0xde, 0xfc, 0x13, 0x7c, 0xf5, 0xd9, 0xbf,
0xc0, 0x67, 0xff, 0x38, 0x73, 0x7b, 0x77, 0x6d, 0x0f, 0x8a, 0x25, 0x1a, 0xdf, 0xba, 0xb3, 0xdf,
0x7c, 0xfb, 0xcd, 0x37, 0x73, 0x53, 0x28, 0x33, 0xc7, 0xc4, 0x8b, 0x01, 0x47, 0xff, 0x9c, 0x19,
0xd8, 0xf0, 0x7c, 0x57, 0xb8, 0x84, 0xd8, 0xcc, 0x3a, 0x0f, 0x78, 0x74, 0x6a, 0x48, 0x44, 0xb5,
0x60, 0xb8, 0xb6, 0xed, 0x3a, 0x51, 0xac, 0x5a, 0x64, 0x8e, 0x40, 0xdf, 0xa1, 0x56, 0x74, 0xae,
0x7d, 0x55, 0xa0, 0xac, 0xe3, 0x29, 0xe3, 0x02, 0xfd, 0xae, 0x6b, 0xa2, 0x8e, 0x67, 0x01, 0x72,
0x41, 0x76, 0x20, 0x3b, 0xa4, 0x1c, 0x2b, 0xca, 0x86, 0x52, 0x5f, 0xd9, 0x79, 0xd4, 0xb8, 0x46,
0x1c, 0x73, 0x1c, 0xf2, 0xd3, 0x16, 0xe5, 0xa8, 0x4b, 0x2c, 0x79, 0x09, 0x4b, 0xd4, 0x34, 0x7d,
0xe4, 0xbc, 0x92, 0x91, 0x69, 0x0f, 0xd3, 0x69, 0xb1, 0x90, 0xbd, 0x08, 0xa3, 0x27, 0xe0, 0xda,
0x09, 0xac, 0xa5, 0x25, 0x70, 0xcf, 0x75, 0x38, 0x92, 0x16, 0xac, 0x30, 0x87, 0x89, 0x81, 0x47,
0x7d, 0x6a, 0xf3, 0x58, 0xca, 0xe3, 0x5b, 0xa4, 0x68, 0x0e, 0x13, 0x3d, 0x09, 0xd4, 0x81, 0x8d,
0x7f, 0xd7, 0x1a, 0x40, 0xb4, 0xd0, 0x86, 0xbe, 0xa0, 0x02, 0x79, 0x52, 0x5d, 0x05, 0x96, 0xa4,
0x39, 0x5a, 0x5b, 0xb2, 0xaa, 0x7a, 0x72, 0xac, 0x7d, 0x57, 0xa0, 0x9c, 0x4a, 0x88, 0xb5, 0xec,
0x42, 0x8e, 0x0b, 0x2a, 0x82, 0x44, 0xc6, 0x83, 0x99, 0xa5, 0xf5, 0x25, 0x44, 0x8f, 0xa1, 0xe4,
0x05, 0x2c, 0x86, 0xbf, 0x50, 0xda, 0x51, 0xbc, 0xe9, 0xa2, 0x89, 0x17, 0x8d, 0xc9, 0x63, 0x7a,
0x04, 0x9e, 0x16, 0xa7, 0xa6, 0xc5, 0xfd, 0x52, 0x60, 0xb5, 0x15, 0x30, 0xcb, 0x94, 0x49, 0x49,
0x31, 0xeb, 0x00, 0x26, 0x15, 0x74, 0xe0, 0x51, 0x31, 0x0a, 0x9d, 0x57, 0xeb, 0x79, 0x3d, 0x1f,
0x46, 0x7a, 0x61, 0x20, 0x74, 0x51, 0x5c, 0x7a, 0x98, 0xb8, 0xa8, 0x6e, 0xa8, 0x37, 0x5d, 0x8c,
0xe5, 0xbf, 0xc3, 0xcb, 0x8f, 0xd4, 0x0a, 0xb0, 0x47, 0x99, 0xaf, 0x43, 0x98, 0x15, 0xb9, 0x48,
0xda, 0x50, 0x88, 0xc6, 0x2d, 0x26, 0xc9, 0xde, 0x95, 0x64, 0x45, 0xa6, 0xc5, 0xbd, 0x30, 0x80,
0x4c, 0xab, 0xff, 0x17, 0x67, 0xa7, 0x3c, 0xca, 0xa4, 0x3d, 0x7a, 0x0e, 0x6b, 0x92, 0x7f, 0x9f,
0x59, 0x18, 0x1a, 0x30, 0xbf, 0xe5, 0xdf, 0x14, 0xb8, 0x9f, 0x4a, 0xe1, 0xff, 0x49, 0x1b, 0xa9,
0x43, 0x29, 0xb2, 0xf1, 0x33, 0xb3, 0x30, 0xee, 0x97, 0x2a, 0xfb, 0x55, 0x64, 0x29, 0x01, 0xb5,
0x9f, 0x19, 0xc8, 0x4b, 0x4d, 0x87, 0x28, 0xe8, 0x64, 0x8e, 0x94, 0xbf, 0x9c, 0xa3, 0x6b, 0x3a,
0xd6, 0x01, 0xd0, 0x39, 0x0b, 0x70, 0x20, 0x98, 0x8d, 0xf1, 0x90, 0xe5, 0x65, 0xe4, 0x03, 0xb3,
0x91, 0x3c, 0x81, 0x7b, 0xdc, 0x18, 0xa1, 0x19, 0x58, 0x31, 0x22, 0x2b, 0x11, 0x85, 0x24, 0x28,
0x41, 0x0d, 0x28, 0x0f, 0xc3, 0x66, 0x0e, 0x0c, 0xd7, 0xf6, 0x2c, 0x14, 0x31, 0x74, 0x51, 0x42,
0x57, 0xe5, 0xd5, 0x9b, 0xf8, 0x46, 0xe2, 0x5f, 0x81, 0xea, 0xe3, 0x59, 0x25, 0x27, 0x7d, 0x7c,
0x3a, 0xab, 0x82, 0x1b, 0x93, 0xad, 0x87, 0x19, 0x33, 0x4d, 0x5b, 0x9a, 0x65, 0xda, 0x76, 0x17,
0x60, 0xe2, 0x02, 0x59, 0x86, 0x6c, 0xf7, 0xa8, 0xdb, 0x29, 0x2d, 0x90, 0x02, 0x2c, 0x1f, 0x77,
0xb5, 0x7e, 0xff, 0xb8, 0xd3, 0x2e, 0x29, 0xa4, 0x08, 0xa0, 0x75, 0x7b, 0xfa, 0xd1, 0x81, 0xde,
0xe9, 0xf7, 0x4b, 0x99, 0xf0, 0x76, 0x5f, 0xeb, 0x6a, 0xfd, 0xb7, 0x9d, 0x76, 0x49, 0x25, 0x00,
0xb9, 0xfd, 0x3d, 0xed, 0x7d, 0xa7, 0x5d, 0xca, 0xee, 0xfc, 0x50, 0xa1, 0x10, 0x11, 0x46, 0x4b,
0x96, 0x18, 0x50, 0x98, 0x5e, 0x54, 0x64, 0x6b, 0x56, 0x19, 0x33, 0xb6, 0x69, 0xb5, 0x3e, 0x1f,
0x18, 0x4d, 0x5c, 0x6d, 0x81, 0x7c, 0x02, 0x98, 0x38, 0x41, 0xee, 0xe6, 0x54, 0x75, 0x73, 0x1e,
0x6c, 0x4c, 0x6f, 0x40, 0xf1, 0x00, 0xc5, 0xd4, 0x8a, 0x23, 0x9b, 0x7f, 0x1e, 0xa7, 0x64, 0x69,
0x56, 0xb7, 0xe6, 0xe2, 0xc6, 0x8f, 0x7c, 0x81, 0xd5, 0xe4, 0x91, 0x71, 0x7b, 0x48, 0xfd, 0xd6,
0xfc, 0x6b, 0xdf, 0x6a, 0x75, 0x7b, 0x2e, 0x72, 0xea, 0xb1, 0xd6, 0xde, 0xc9, 0xeb, 0x53, 0x26,
0x46, 0xc1, 0x30, 0xfc, 0x1e, 0x9b, 0x57, 0xcc, 0xb2, 0xd8, 0x95, 0x40, 0x63, 0xd4, 0x8c, 0x48,
0x9e, 0x99, 0x8c, 0x0b, 0x9f, 0x0d, 0x03, 0x81, 0x66, 0x33, 0xf9, 0xbb, 0x68, 0x4a, 0xe6, 0xa6,
0x64, 0xf6, 0x86, 0xc3, 0x9c, 0x3c, 0xee, 0xfe, 0x0e, 0x00, 0x00, 0xff, 0xff, 0x65, 0x3f, 0xa4,
0x88, 0x55, 0x07, 0x00, 0x00,
}
// Reference imports to suppress errors if they are not otherwise used.
@ -717,7 +609,6 @@ type IndexServiceClient interface {
BuildIndex(ctx context.Context, in *BuildIndexRequest, opts ...grpc.CallOption) (*BuildIndexResponse, error)
GetIndexStates(ctx context.Context, in *IndexStatesRequest, opts ...grpc.CallOption) (*IndexStatesResponse, error)
GetIndexFilePaths(ctx context.Context, in *IndexFilePathRequest, opts ...grpc.CallOption) (*IndexFilePathsResponse, error)
NotifyBuildIndex(ctx context.Context, in *BuildIndexNotification, opts ...grpc.CallOption) (*commonpb.Status, error)
}
type indexServiceClient struct {
@ -764,15 +655,6 @@ func (c *indexServiceClient) GetIndexFilePaths(ctx context.Context, in *IndexFil
return out, nil
}
func (c *indexServiceClient) NotifyBuildIndex(ctx context.Context, in *BuildIndexNotification, opts ...grpc.CallOption) (*commonpb.Status, error) {
out := new(commonpb.Status)
err := c.cc.Invoke(ctx, "/milvus.proto.index.IndexService/NotifyBuildIndex", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// IndexServiceServer is the server API for IndexService service.
type IndexServiceServer interface {
//*
@ -785,7 +667,6 @@ type IndexServiceServer interface {
BuildIndex(context.Context, *BuildIndexRequest) (*BuildIndexResponse, error)
GetIndexStates(context.Context, *IndexStatesRequest) (*IndexStatesResponse, error)
GetIndexFilePaths(context.Context, *IndexFilePathRequest) (*IndexFilePathsResponse, error)
NotifyBuildIndex(context.Context, *BuildIndexNotification) (*commonpb.Status, error)
}
// UnimplementedIndexServiceServer can be embedded to have forward compatible implementations.
@ -804,9 +685,6 @@ func (*UnimplementedIndexServiceServer) GetIndexStates(ctx context.Context, req
func (*UnimplementedIndexServiceServer) GetIndexFilePaths(ctx context.Context, req *IndexFilePathRequest) (*IndexFilePathsResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetIndexFilePaths not implemented")
}
func (*UnimplementedIndexServiceServer) NotifyBuildIndex(ctx context.Context, req *BuildIndexNotification) (*commonpb.Status, error) {
return nil, status.Errorf(codes.Unimplemented, "method NotifyBuildIndex not implemented")
}
func RegisterIndexServiceServer(s *grpc.Server, srv IndexServiceServer) {
s.RegisterService(&_IndexService_serviceDesc, srv)
@ -884,24 +762,6 @@ func _IndexService_GetIndexFilePaths_Handler(srv interface{}, ctx context.Contex
return interceptor(ctx, in, info, handler)
}
func _IndexService_NotifyBuildIndex_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(BuildIndexNotification)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(IndexServiceServer).NotifyBuildIndex(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/milvus.proto.index.IndexService/NotifyBuildIndex",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(IndexServiceServer).NotifyBuildIndex(ctx, req.(*BuildIndexNotification))
}
return interceptor(ctx, in, info, handler)
}
var _IndexService_serviceDesc = grpc.ServiceDesc{
ServiceName: "milvus.proto.index.IndexService",
HandlerType: (*IndexServiceServer)(nil),
@ -922,94 +782,6 @@ var _IndexService_serviceDesc = grpc.ServiceDesc{
MethodName: "GetIndexFilePaths",
Handler: _IndexService_GetIndexFilePaths_Handler,
},
{
MethodName: "NotifyBuildIndex",
Handler: _IndexService_NotifyBuildIndex_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "index_service.proto",
}
// IndexNodeClient is the client API for IndexNode service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
type IndexNodeClient interface {
//*
// @brief This method is used to create collection
//
// @param CollectionSchema, use to provide collection information to be created.
//
// @return Status
BuildIndex(ctx context.Context, in *BuildIndexCmd, opts ...grpc.CallOption) (*commonpb.Status, error)
}
type indexNodeClient struct {
cc *grpc.ClientConn
}
func NewIndexNodeClient(cc *grpc.ClientConn) IndexNodeClient {
return &indexNodeClient{cc}
}
func (c *indexNodeClient) BuildIndex(ctx context.Context, in *BuildIndexCmd, opts ...grpc.CallOption) (*commonpb.Status, error) {
out := new(commonpb.Status)
err := c.cc.Invoke(ctx, "/milvus.proto.index.IndexNode/BuildIndex", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// IndexNodeServer is the server API for IndexNode service.
type IndexNodeServer interface {
//*
// @brief This method is used to create collection
//
// @param CollectionSchema, use to provide collection information to be created.
//
// @return Status
BuildIndex(context.Context, *BuildIndexCmd) (*commonpb.Status, error)
}
// UnimplementedIndexNodeServer can be embedded to have forward compatible implementations.
type UnimplementedIndexNodeServer struct {
}
func (*UnimplementedIndexNodeServer) BuildIndex(ctx context.Context, req *BuildIndexCmd) (*commonpb.Status, error) {
return nil, status.Errorf(codes.Unimplemented, "method BuildIndex not implemented")
}
func RegisterIndexNodeServer(s *grpc.Server, srv IndexNodeServer) {
s.RegisterService(&_IndexNode_serviceDesc, srv)
}
func _IndexNode_BuildIndex_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(BuildIndexCmd)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(IndexNodeServer).BuildIndex(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/milvus.proto.index.IndexNode/BuildIndex",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(IndexNodeServer).BuildIndex(ctx, req.(*BuildIndexCmd))
}
return interceptor(ctx, in, info, handler)
}
var _IndexNode_serviceDesc = grpc.ServiceDesc{
ServiceName: "milvus.proto.index.IndexNode",
HandlerType: (*IndexNodeServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "BuildIndex",
Handler: _IndexNode_BuildIndex_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "index_service.proto",

4
internal/proto/internal.proto
View File

@ -4,7 +4,7 @@ option go_package = "github.com/zilliztech/milvus-distributed/internal/proto/int
import "common.proto";
enum MsgType2 {
enum MsgType {
kNone = 0;
/* Definition Requests: collection */
kCreateCollection = 100;
@ -78,7 +78,7 @@ message StringList {
}
message MsgBase {
MsgType2 msg_type = 1;
MsgType msg_type = 1;
int64 msgID = 2;
uint64 timestamp = 3;
int64 sourceID = 4;

166
internal/proto/internalpb2/internal.pb.go
View File

@ -21,41 +21,41 @@ var _ = math.Inf
// proto package needs to be updated.
const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
type MsgType2 int32
type MsgType int32
const (
MsgType2_kNone MsgType2 = 0
MsgType_kNone MsgType = 0
// Definition Requests: collection
MsgType2_kCreateCollection MsgType2 = 100
MsgType2_kDropCollection MsgType2 = 101
MsgType2_kHasCollection MsgType2 = 102
MsgType2_kDescribeCollection MsgType2 = 103
MsgType2_kShowCollections MsgType2 = 104
MsgType2_kGetSysConfigs MsgType2 = 105
MsgType_kCreateCollection MsgType = 100
MsgType_kDropCollection MsgType = 101
MsgType_kHasCollection MsgType = 102
MsgType_kDescribeCollection MsgType = 103
MsgType_kShowCollections MsgType = 104
MsgType_kGetSysConfigs MsgType = 105
// Definition Requests: partition
MsgType2_kCreatePartition MsgType2 = 200
MsgType2_kDropPartition MsgType2 = 201
MsgType2_kHasPartition MsgType2 = 202
MsgType2_kDescribePartition MsgType2 = 203
MsgType2_kShowPartitions MsgType2 = 204
MsgType_kCreatePartition MsgType = 200
MsgType_kDropPartition MsgType = 201
MsgType_kHasPartition MsgType = 202
MsgType_kDescribePartition MsgType = 203
MsgType_kShowPartitions MsgType = 204
// Definition Requests: Index
MsgType2_kCreateIndex MsgType2 = 300
MsgType2_kDescribeIndex MsgType2 = 301
MsgType2_kDescribeIndexProgress MsgType2 = 302
MsgType_kCreateIndex MsgType = 300
MsgType_kDescribeIndex MsgType = 301
MsgType_kDescribeIndexProgress MsgType = 302
// Manipulation Requests
MsgType2_kInsert MsgType2 = 400
MsgType2_kDelete MsgType2 = 401
MsgType2_kFlush MsgType2 = 402
MsgType_kInsert MsgType = 400
MsgType_kDelete MsgType = 401
MsgType_kFlush MsgType = 402
// Query
MsgType2_kSearch MsgType2 = 500
MsgType2_kSearchResult MsgType2 = 501
MsgType_kSearch MsgType = 500
MsgType_kSearchResult MsgType = 501
// System Control
MsgType2_kTimeTick MsgType2 = 1200
MsgType2_kQueryNodeStats MsgType2 = 1201
MsgType2_kLoadIndex MsgType2 = 1202
MsgType_kTimeTick MsgType = 1200
MsgType_kQueryNodeStats MsgType = 1201
MsgType_kLoadIndex MsgType = 1202
)
var MsgType2_name = map[int32]string{
var MsgType_name = map[int32]string{
0: "kNone",
100: "kCreateCollection",
101: "kDropCollection",
@ -81,7 +81,7 @@ var MsgType2_name = map[int32]string{
1202: "kLoadIndex",
}
var MsgType2_value = map[string]int32{
var MsgType_value = map[string]int32{
"kNone": 0,
"kCreateCollection": 100,
"kDropCollection": 101,
@ -107,11 +107,11 @@ var MsgType2_value = map[string]int32{
"kLoadIndex": 1202,
}
func (x MsgType2) String() string {
return proto.EnumName(MsgType2_name, int32(x))
func (x MsgType) String() string {
return proto.EnumName(MsgType_name, int32(x))
}
func (MsgType2) EnumDescriptor() ([]byte, []int) {
func (MsgType) EnumDescriptor() ([]byte, []int) {
return fileDescriptor_41f4a519b878ee3b, []int{0}
}
@ -395,7 +395,7 @@ func (m *StringList) GetValues() []string {
}
type MsgBase struct {
MsgType MsgType2 `protobuf:"varint,1,opt,name=msg_type,json=msgType,proto3,enum=milvus.proto.internal.MsgType2" json:"msg_type,omitempty"`
MsgType MsgType `protobuf:"varint,1,opt,name=msg_type,json=msgType,proto3,enum=milvus.proto.internal.MsgType" json:"msg_type,omitempty"`
MsgID int64 `protobuf:"varint,2,opt,name=msgID,proto3" json:"msgID,omitempty"`
Timestamp uint64 `protobuf:"varint,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
SourceID int64 `protobuf:"varint,4,opt,name=sourceID,proto3" json:"sourceID,omitempty"`
@ -429,11 +429,11 @@ func (m *MsgBase) XXX_DiscardUnknown() {
var xxx_messageInfo_MsgBase proto.InternalMessageInfo
func (m *MsgBase) GetMsgType() MsgType2 {
func (m *MsgBase) GetMsgType() MsgType {
if m != nil {
return m.MsgType
}
return MsgType2_kNone
return MsgType_kNone
}
func (m *MsgBase) GetMsgID() int64 {
@ -497,7 +497,7 @@ func (m *TimeTickMsg) GetBase() *MsgBase {
}
func init() {
proto.RegisterEnum("milvus.proto.internal.MsgType2", MsgType2_name, MsgType2_value)
proto.RegisterEnum("milvus.proto.internal.MsgType", MsgType_name, MsgType_value)
proto.RegisterEnum("milvus.proto.internal.StateCode", StateCode_name, StateCode_value)
proto.RegisterType((*NodeStates)(nil), "milvus.proto.internal.NodeStates")
proto.RegisterType((*ServiceStates)(nil), "milvus.proto.internal.ServiceStates")
@ -511,55 +511,55 @@ func init() {
func init() { proto.RegisterFile("internal.proto", fileDescriptor_41f4a519b878ee3b) }
var fileDescriptor_41f4a519b878ee3b = []byte{
// 791 bytes of a gzipped FileDescriptorProto
// 788 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x54, 0xcd, 0x6e, 0x23, 0x45,
0x10, 0xde, 0xf1, 0x78, 0x63, 0x4f, 0xd9, 0xeb, 0xf4, 0x56, 0x92, 0x5d, 0x6b, 0x59, 0x81, 0xb1,
0x38, 0x44, 0x2b, 0x91, 0x48, 0x46, 0x42, 0x88, 0x0b, 0x38, 0xf1, 0xb2, 0x19, 0xe1, 0x98, 0x30,
0xb6, 0x56, 0x62, 0x2f, 0xd6, 0xd8, 0x53, 0x19, 0x37, 0xf3, 0xd3, 0x56, 0x77, 0x3b, 0xac, 0xf7,
0x29, 0x00, 0x89, 0xb7, 0x00, 0x04, 0x5c, 0x79, 0x01, 0xfe, 0xc4, 0x91, 0x27, 0x81, 0x3b, 0x9a,
0x1e, 0x8f, 0x9d, 0x48, 0xce, 0x01, 0x6e, 0x5d, 0x5f, 0x57, 0x7d, 0xfd, 0x7d, 0x55, 0x35, 0x03,
0x0d, 0x9e, 0x6a, 0x92, 0xa9, 0x1f, 0x1f, 0xcd, 0xa5, 0xd0, 0x02, 0x0f, 0x12, 0x1e, 0x5f, 0x2d,
0x54, 0x1e, 0x1d, 0x15, 0x97, 0x8f, 0xea, 0x53, 0x91, 0x24, 0x22, 0xcd, 0xe1, 0xf6, 0xcf, 0x16,
0xc0, 0x40, 0x04, 0x34, 0xd4, 0xbe, 0x26, 0x85, 0x0f, 0x60, 0x27, 0x15, 0x01, 0xb9, 0xbd, 0xa6,
0xd5, 0xb2, 0x0e, 0x6d, 0x6f, 0x15, 0x21, 0x42, 0x59, 0x8a, 0x98, 0x9a, 0xa5, 0x96, 0x75, 0xe8,
0x78, 0xe6, 0x8c, 0x1f, 0x00, 0xa8, 0xac, 0x6a, 0x3c, 0x15, 0x01, 0x35, 0xed, 0x96, 0x75, 0xd8,
0xe8, 0xb4, 0x8e, 0xb6, 0x3e, 0x7a, 0x64, 0xe8, 0x4f, 0x45, 0x40, 0x9e, 0xa3, 0x8a, 0x23, 0x7e,
0x08, 0x40, 0x2f, 0xb5, 0xf4, 0xc7, 0x3c, 0xbd, 0x14, 0xcd, 0x72, 0xcb, 0x3e, 0xac, 0x75, 0xde,
0xbc, 0x49, 0xb0, 0xd2, 0xfa, 0x31, 0x2d, 0x9f, 0xfb, 0xf1, 0x82, 0x2e, 0x7c, 0x2e, 0x3d, 0xc7,
0x14, 0xb9, 0xe9, 0xa5, 0x68, 0xff, 0x65, 0xc1, 0xbd, 0x21, 0xc9, 0x2b, 0x3e, 0x2d, 0x0c, 0xdc,
0x14, 0x65, 0xfd, 0x77, 0x51, 0x27, 0x50, 0xcb, 0x3c, 0x8f, 0x0d, 0xa2, 0x9a, 0xa5, 0x6d, 0xaa,
0xd6, 0x0c, 0x9b, 0xce, 0x79, 0x90, 0x6e, 0xba, 0x78, 0xd3, 0x98, 0xfd, 0x3f, 0x8c, 0x3d, 0x87,
0x6a, 0xc6, 0x9d, 0x9d, 0xf1, 0x5d, 0xa8, 0xf8, 0x41, 0x20, 0x49, 0x29, 0xe3, 0xa7, 0xd6, 0x79,
0xbc, 0x95, 0xaa, 0x9b, 0xe7, 0x78, 0x45, 0xf2, 0xb6, 0x99, 0xb5, 0x3f, 0x07, 0x70, 0x53, 0xae,
0x2f, 0x7c, 0xe9, 0x27, 0xb7, 0x4f, 0xbb, 0x07, 0x75, 0xa5, 0x7d, 0xa9, 0xc7, 0x73, 0x93, 0xb7,
0xbd, 0x09, 0xdb, 0x1c, 0xd4, 0x4c, 0x59, 0xce, 0xde, 0x7e, 0x0b, 0x60, 0xa8, 0x25, 0x4f, 0xc3,
0x3e, 0x57, 0x3a, 0x7b, 0xeb, 0x2a, 0xcb, 0xcb, 0xd9, 0x1c, 0x6f, 0x15, 0xb5, 0xbf, 0xb1, 0xa0,
0x72, 0xae, 0xc2, 0x13, 0x5f, 0x11, 0xbe, 0x0f, 0xd5, 0x44, 0x85, 0x63, 0xbd, 0x9c, 0x17, 0xa3,
0x7b, 0xe3, 0x96, 0xc6, 0x9f, 0xab, 0x70, 0xb4, 0x9c, 0x53, 0xc7, 0xab, 0x24, 0xf9, 0x09, 0xf7,
0xe1, 0x6e, 0xa2, 0x42, 0xb7, 0x67, 0xec, 0xda, 0x5e, 0x1e, 0xe0, 0x63, 0x70, 0x34, 0x4f, 0x48,
0x69, 0x3f, 0x99, 0x9b, 0x15, 0x2d, 0x7b, 0x1b, 0x00, 0x1f, 0x41, 0x55, 0x89, 0x85, 0x9c, 0x66,
0x1d, 0x28, 0x9b, 0xb2, 0x75, 0xdc, 0xee, 0x42, 0x6d, 0xc4, 0x13, 0x1a, 0xf1, 0x69, 0x74, 0xae,
0x42, 0xec, 0x40, 0x79, 0xe2, 0x2b, 0x5a, 0x4d, 0xe0, 0xf5, 0xdb, 0x65, 0x65, 0x46, 0x3c, 0x93,
0xfb, 0xe4, 0x4f, 0x1b, 0xaa, 0x85, 0x50, 0x74, 0xe0, 0x6e, 0x34, 0x10, 0x29, 0xb1, 0x3b, 0x78,
0x00, 0xf7, 0xa3, 0x53, 0x49, 0x66, 0xe3, 0xe2, 0x98, 0xa6, 0x9a, 0x8b, 0x94, 0x05, 0xb8, 0x07,
0xbb, 0x51, 0x4f, 0x8a, 0xf9, 0x35, 0x90, 0x10, 0xa1, 0x11, 0x9d, 0xf9, 0xea, 0x1a, 0x76, 0x89,
0x0f, 0x61, 0x2f, 0xea, 0x91, 0x9a, 0x4a, 0x3e, 0xb9, 0xce, 0x10, 0xe2, 0x3e, 0xb0, 0x68, 0x38,
0x13, 0x5f, 0x6c, 0x40, 0xc5, 0x66, 0x86, 0xe2, 0x19, 0xe9, 0xe1, 0x52, 0x9d, 0x8a, 0xf4, 0x92,
0x87, 0x8a, 0x71, 0x3c, 0x00, 0xb6, 0x92, 0x70, 0xe1, 0x4b, 0xcd, 0x4d, 0xfd, 0x2f, 0x16, 0xee,
0x41, 0xc3, 0x48, 0xd8, 0x80, 0xbf, 0x5a, 0x88, 0x70, 0x2f, 0x93, 0xb0, 0xc1, 0x7e, 0xb3, 0xf0,
0x21, 0xe0, 0x5a, 0xc2, 0xe6, 0xe2, 0x77, 0x0b, 0xf7, 0x61, 0xd7, 0x48, 0x58, 0x83, 0x8a, 0xfd,
0x61, 0xe1, 0x7d, 0xa8, 0xaf, 0x9e, 0x73, 0xd3, 0x80, 0x5e, 0xb2, 0x6f, 0x4b, 0xf9, 0x53, 0x2b,
0x86, 0x1c, 0xfc, 0xae, 0x84, 0xaf, 0xc1, 0x83, 0x9b, 0xe0, 0x85, 0x14, 0x61, 0xb6, 0xcc, 0xec,
0xfb, 0x12, 0xd6, 0xa1, 0x12, 0xb9, 0xa9, 0x22, 0xa9, 0xd9, 0x97, 0xb6, 0x89, 0x7a, 0x14, 0x93,
0x26, 0xf6, 0x95, 0x8d, 0x35, 0xd8, 0x89, 0x3e, 0x8a, 0x17, 0x6a, 0xc6, 0xbe, 0xce, 0xaf, 0x86,
0xe4, 0xcb, 0xe9, 0x8c, 0xfd, 0x6d, 0x1b, 0xf9, 0x79, 0xe4, 0x91, 0x5a, 0xc4, 0x9a, 0xfd, 0x63,
0x63, 0x03, 0x9c, 0xa8, 0x98, 0x2e, 0xfb, 0xc1, 0x31, 0xaa, 0x3f, 0x5d, 0x90, 0x5c, 0x16, 0x1f,
0xb4, 0x62, 0x3f, 0x3a, 0xb8, 0x0b, 0x10, 0xf5, 0x85, 0x1f, 0xe4, 0xf2, 0x7e, 0x72, 0x9e, 0xbc,
0x07, 0xce, 0xfa, 0x9f, 0x81, 0x0c, 0xea, 0xee, 0xc0, 0x1d, 0xb9, 0xdd, 0xbe, 0xfb, 0xc2, 0x1d,
0x3c, 0x63, 0x77, 0xb0, 0x06, 0x95, 0xb3, 0xa7, 0xdd, 0xfe, 0xe8, 0xec, 0x33, 0x66, 0x61, 0x1d,
0xaa, 0xdd, 0x93, 0xc1, 0x27, 0xde, 0x79, 0xb7, 0xcf, 0x4a, 0x27, 0x4f, 0x5f, 0x9c, 0x86, 0x5c,
0xcf, 0x16, 0x93, 0xec, 0xb3, 0x39, 0x7e, 0xc5, 0xe3, 0x98, 0xbf, 0xd2, 0x34, 0x9d, 0x1d, 0xe7,
0x7b, 0xf4, 0x76, 0xc0, 0x95, 0x96, 0x7c, 0xb2, 0xd0, 0x14, 0x1c, 0x17, 0xdb, 0x74, 0x6c, 0x96,
0x6b, 0x1d, 0xce, 0x27, 0x9d, 0xc9, 0x8e, 0x81, 0xde, 0xf9, 0x37, 0x00, 0x00, 0xff, 0xff, 0x38,
0xa8, 0x72, 0x51, 0xeb, 0x05, 0x00, 0x00,
0x10, 0xde, 0xf1, 0x78, 0xe3, 0x4c, 0xd9, 0xeb, 0xf4, 0x56, 0x92, 0x5d, 0x6b, 0x59, 0x21, 0x63,
0x71, 0x88, 0x56, 0x22, 0x91, 0x8c, 0x84, 0xe0, 0x04, 0x4e, 0xbc, 0x6c, 0x46, 0x38, 0x26, 0x8c,
0xad, 0x95, 0xd8, 0x8b, 0x35, 0x9e, 0xa9, 0x8c, 0x9b, 0xf9, 0x69, 0xab, 0xbb, 0x1d, 0xd6, 0xfb,
0x14, 0x80, 0x78, 0x0c, 0x40, 0xc0, 0x95, 0x17, 0xe0, 0xf7, 0xca, 0x93, 0xc0, 0x1d, 0x4d, 0x8f,
0x7f, 0x62, 0xc9, 0x41, 0x82, 0x5b, 0xd7, 0xd7, 0x55, 0x5f, 0x7f, 0x5f, 0x55, 0xcd, 0x40, 0x9d,
0x67, 0x9a, 0x64, 0xe6, 0x27, 0xc7, 0x53, 0x29, 0xb4, 0xc0, 0xc3, 0x94, 0x27, 0xd7, 0x33, 0x55,
0x44, 0xc7, 0xcb, 0xcb, 0x47, 0xb5, 0x40, 0xa4, 0xa9, 0xc8, 0x0a, 0xb8, 0xf5, 0x93, 0x05, 0xd0,
0x17, 0x21, 0x0d, 0xb4, 0xaf, 0x49, 0xe1, 0x03, 0xd8, 0xc9, 0x44, 0x48, 0x6e, 0xb7, 0x61, 0x35,
0xad, 0x23, 0xdb, 0x5b, 0x44, 0x88, 0x50, 0x96, 0x22, 0xa1, 0x46, 0xa9, 0x69, 0x1d, 0x39, 0x9e,
0x39, 0xe3, 0xfb, 0x00, 0x2a, 0xaf, 0x1a, 0x05, 0x22, 0xa4, 0x86, 0xdd, 0xb4, 0x8e, 0xea, 0xed,
0xe6, 0xf1, 0xd6, 0x47, 0x8f, 0x0d, 0xfd, 0x99, 0x08, 0xc9, 0x73, 0xd4, 0xf2, 0x88, 0x1f, 0x00,
0xd0, 0x4b, 0x2d, 0xfd, 0x11, 0xcf, 0xae, 0x44, 0xa3, 0xdc, 0xb4, 0x8f, 0xaa, 0xed, 0x37, 0x36,
0x09, 0x16, 0x5a, 0x3f, 0xa2, 0xf9, 0x73, 0x3f, 0x99, 0xd1, 0xa5, 0xcf, 0xa5, 0xe7, 0x98, 0x22,
0x37, 0xbb, 0x12, 0xad, 0x3f, 0x2d, 0xb8, 0x37, 0x20, 0x79, 0xcd, 0x83, 0xa5, 0x81, 0x4d, 0x51,
0xd6, 0x7f, 0x17, 0x75, 0x0a, 0xd5, 0xdc, 0xf3, 0xc8, 0x20, 0xaa, 0x51, 0xda, 0xa6, 0x6a, 0xc5,
0xb0, 0xee, 0x9c, 0x07, 0xd9, 0xba, 0x8b, 0x9b, 0xc6, 0xec, 0xff, 0x61, 0xec, 0x39, 0xec, 0xe6,
0xdc, 0xf9, 0x19, 0xdf, 0x81, 0x8a, 0x1f, 0x86, 0x92, 0x94, 0x32, 0x7e, 0xaa, 0xed, 0xc7, 0x5b,
0xa9, 0x3a, 0x45, 0x8e, 0xb7, 0x4c, 0xde, 0x36, 0xb3, 0xd6, 0x67, 0x00, 0x6e, 0xc6, 0xf5, 0xa5,
0x2f, 0xfd, 0xf4, 0xf6, 0x69, 0x77, 0xa1, 0xa6, 0xb4, 0x2f, 0xf5, 0x68, 0x6a, 0xf2, 0xb6, 0x37,
0x61, 0x9b, 0x83, 0xaa, 0x29, 0x2b, 0xd8, 0x5b, 0x6f, 0x02, 0x0c, 0xb4, 0xe4, 0x59, 0xd4, 0xe3,
0x4a, 0xe7, 0x6f, 0x5d, 0xe7, 0x79, 0x05, 0x9b, 0xe3, 0x2d, 0xa2, 0xd6, 0xd7, 0x16, 0x54, 0x2e,
0x54, 0x74, 0xea, 0x2b, 0xc2, 0xf7, 0x60, 0x37, 0x55, 0xd1, 0x48, 0xcf, 0xa7, 0xcb, 0xd1, 0xbd,
0x7e, 0x4b, 0xe3, 0x2f, 0x54, 0x34, 0x9c, 0x4f, 0xc9, 0xab, 0xa4, 0xc5, 0x01, 0x0f, 0xe0, 0x6e,
0xaa, 0x22, 0xb7, 0x6b, 0xdc, 0xda, 0x5e, 0x11, 0xe0, 0x63, 0x70, 0x34, 0x4f, 0x49, 0x69, 0x3f,
0x9d, 0x9a, 0x0d, 0x2d, 0x7b, 0x6b, 0x00, 0x1f, 0xc1, 0xae, 0x12, 0x33, 0x19, 0xe4, 0x0d, 0x28,
0x9b, 0xb2, 0x55, 0xdc, 0xea, 0x40, 0x75, 0xc8, 0x53, 0x1a, 0xf2, 0x20, 0xbe, 0x50, 0x11, 0xb6,
0xa1, 0x3c, 0xf6, 0x15, 0x2d, 0x06, 0xf0, 0x2f, 0xaa, 0x72, 0x1f, 0x9e, 0xc9, 0x7d, 0xf2, 0x87,
0x6d, 0x9c, 0x19, 0x79, 0x0e, 0xdc, 0x8d, 0xfb, 0x22, 0x23, 0x76, 0x07, 0x0f, 0xe1, 0x7e, 0x7c,
0x26, 0xc9, 0xec, 0x5b, 0x92, 0x50, 0xa0, 0xb9, 0xc8, 0x58, 0x88, 0xfb, 0xb0, 0x17, 0x77, 0xa5,
0x98, 0xde, 0x00, 0x09, 0x11, 0xea, 0xf1, 0xb9, 0xaf, 0x6e, 0x60, 0x57, 0xf8, 0x10, 0xf6, 0xe3,
0x2e, 0xa9, 0x40, 0xf2, 0xf1, 0x4d, 0x86, 0x08, 0x0f, 0x80, 0xc5, 0x83, 0x89, 0xf8, 0x7c, 0x0d,
0x2a, 0x36, 0x31, 0x14, 0xcf, 0x48, 0x0f, 0xe6, 0xea, 0x4c, 0x64, 0x57, 0x3c, 0x52, 0x8c, 0xe3,
0x21, 0xb0, 0x85, 0x84, 0x4b, 0x5f, 0x6a, 0x6e, 0xea, 0x7f, 0xb6, 0x70, 0x1f, 0xea, 0x46, 0xc2,
0x1a, 0xfc, 0xc5, 0x42, 0x84, 0x7b, 0xb9, 0x84, 0x35, 0xf6, 0xab, 0x85, 0x0f, 0x01, 0x57, 0x12,
0xd6, 0x17, 0xbf, 0x59, 0x78, 0x00, 0x7b, 0x46, 0xc2, 0x0a, 0x54, 0xec, 0x77, 0x0b, 0xef, 0x43,
0x6d, 0xf1, 0x9c, 0x9b, 0x85, 0xf4, 0x92, 0x7d, 0x53, 0x2a, 0x9e, 0x5a, 0x30, 0x14, 0xe0, 0xb7,
0x25, 0x7c, 0x0d, 0x1e, 0x6c, 0x82, 0x97, 0x52, 0x44, 0xf9, 0x2a, 0xb3, 0xef, 0x4a, 0x58, 0x83,
0x4a, 0xec, 0x66, 0x8a, 0xa4, 0x66, 0x5f, 0xd8, 0x26, 0xea, 0x52, 0x42, 0x9a, 0xd8, 0x97, 0x36,
0x56, 0x61, 0x27, 0xfe, 0x30, 0x99, 0xa9, 0x09, 0xfb, 0xaa, 0xb8, 0x1a, 0x90, 0x2f, 0x83, 0x09,
0xfb, 0xcb, 0x36, 0xf2, 0x8b, 0xc8, 0x23, 0x35, 0x4b, 0x34, 0xfb, 0xdb, 0xc6, 0x3a, 0x38, 0xf1,
0x72, 0xb8, 0xec, 0x7b, 0xc7, 0xa8, 0xfe, 0x64, 0x46, 0x72, 0xbe, 0xfc, 0x9c, 0x15, 0xfb, 0xc1,
0xc1, 0x3d, 0x80, 0xb8, 0x27, 0xfc, 0xb0, 0x90, 0xf7, 0xa3, 0xf3, 0xe4, 0x5d, 0x70, 0x56, 0x7f,
0x0c, 0x64, 0x50, 0x73, 0xfb, 0xee, 0xd0, 0xed, 0xf4, 0xdc, 0x17, 0x6e, 0xff, 0x19, 0xbb, 0x83,
0x55, 0xa8, 0x9c, 0x3f, 0xed, 0xf4, 0x86, 0xe7, 0x9f, 0x32, 0x0b, 0x6b, 0xb0, 0xdb, 0x39, 0xed,
0x7f, 0xec, 0x5d, 0x74, 0x7a, 0xac, 0x74, 0xfa, 0xf4, 0xc5, 0x59, 0xc4, 0xf5, 0x64, 0x36, 0xce,
0x3f, 0x9a, 0x93, 0x57, 0x3c, 0x49, 0xf8, 0x2b, 0x4d, 0xc1, 0xe4, 0xa4, 0x58, 0xa3, 0xb7, 0x42,
0xae, 0xb4, 0xe4, 0xe3, 0x99, 0xa6, 0xf0, 0x64, 0xb9, 0x4c, 0x27, 0x66, 0xb7, 0x56, 0xe1, 0x74,
0xdc, 0x1e, 0xef, 0x18, 0xe8, 0xed, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xa1, 0x75, 0x95, 0xa0,
0xe9, 0x05, 0x00, 0x00,
}

2
internal/proxynode/condition.go → internal/proxy/condition.go
View File

@ -1,4 +1,4 @@
package proxynode
package proxy
import (
"context"

2
internal/proxynode/grpc_service.go → internal/proxy/grpc_service.go
View File

@ -1,4 +1,4 @@
package proxynode
package proxy
import (
"context"

2
internal/proxynode/meta_cache.go → internal/proxy/meta_cache.go
View File

@ -1,4 +1,4 @@
package proxynode
package proxy
import (
"context"

32
internal/proxynode/paramtable.go → internal/proxy/paramtable.go
View File

@ -1,4 +1,4 @@
package proxynode
package proxy
import (
"log"
@ -20,7 +20,7 @@ var Params ParamTable
func (pt *ParamTable) Init() {
pt.BaseTable.Init()
err := pt.LoadYaml("advanced/proxy_node.yaml")
err := pt.LoadYaml("advanced/proxy.yaml")
if err != nil {
panic(err)
}
@ -38,11 +38,11 @@ func (pt *ParamTable) Init() {
}
func (pt *ParamTable) NetworkPort() int {
return pt.ParseInt("proxyNode.port")
return pt.ParseInt("proxy.port")
}
func (pt *ParamTable) NetworkAddress() string {
addr, err := pt.Load("proxyNode.address")
addr, err := pt.Load("proxy.address")
if err != nil {
panic(err)
}
@ -50,11 +50,11 @@ func (pt *ParamTable) NetworkAddress() string {
hostName, _ := net.LookupHost(addr)
if len(hostName) <= 0 {
if ip := net.ParseIP(addr); ip == nil {
panic("invalid ip proxyNode.address")
panic("invalid ip proxy.address")
}
}
port, err := pt.Load("proxyNode.port")
port, err := pt.Load("proxy.port")
if err != nil {
panic(err)
}
@ -100,7 +100,7 @@ func (pt *ParamTable) queryNodeIDList() []UniqueID {
for _, i := range queryNodeIDs {
v, err := strconv.Atoi(i)
if err != nil {
log.Panicf("load proxynode id list error, %s", err.Error())
log.Panicf("load proxy id list error, %s", err.Error())
}
ret = append(ret, UniqueID(v))
}
@ -120,7 +120,7 @@ func (pt *ParamTable) ProxyID() UniqueID {
}
func (pt *ParamTable) TimeTickInterval() time.Duration {
internalStr, err := pt.Load("proxyNode.timeTickInterval")
internalStr, err := pt.Load("proxy.timeTickInterval")
if err != nil {
panic(err)
}
@ -279,27 +279,27 @@ func (pt *ParamTable) DataDefinitionChannelNames() []string {
}
func (pt *ParamTable) MsgStreamInsertBufSize() int64 {
return pt.ParseInt64("proxyNode.msgStream.insert.bufSize")
return pt.ParseInt64("proxy.msgStream.insert.bufSize")
}
func (pt *ParamTable) MsgStreamSearchBufSize() int64 {
return pt.ParseInt64("proxyNode.msgStream.search.bufSize")
return pt.ParseInt64("proxy.msgStream.search.bufSize")
}
func (pt *ParamTable) MsgStreamSearchResultBufSize() int64 {
return pt.ParseInt64("proxyNode.msgStream.searchResult.recvBufSize")
return pt.ParseInt64("proxy.msgStream.searchResult.recvBufSize")
}
func (pt *ParamTable) MsgStreamSearchResultPulsarBufSize() int64 {
return pt.ParseInt64("proxyNode.msgStream.searchResult.pulsarBufSize")
return pt.ParseInt64("proxy.msgStream.searchResult.pulsarBufSize")
}
func (pt *ParamTable) MsgStreamTimeTickBufSize() int64 {
return pt.ParseInt64("proxyNode.msgStream.timeTick.bufSize")
return pt.ParseInt64("proxy.msgStream.timeTick.bufSize")
}
func (pt *ParamTable) MaxNameLength() int64 {
str, err := pt.Load("proxyNode.maxNameLength")
str, err := pt.Load("proxy.maxNameLength")
if err != nil {
panic(err)
}
@ -311,7 +311,7 @@ func (pt *ParamTable) MaxNameLength() int64 {
}
func (pt *ParamTable) MaxFieldNum() int64 {
str, err := pt.Load("proxyNode.maxFieldNum")
str, err := pt.Load("proxy.maxFieldNum")
if err != nil {
panic(err)
}
@ -323,7 +323,7 @@ func (pt *ParamTable) MaxFieldNum() int64 {
}
func (pt *ParamTable) MaxDimension() int64 {
str, err := pt.Load("proxyNode.maxDimension")
str, err := pt.Load("proxy.maxDimension")
if err != nil {
panic(err)
}

2
internal/proxynode/paramtable_test.go → internal/proxy/paramtable_test.go
View File

@ -1,4 +1,4 @@
package proxynode
package proxy
import (
"fmt"

6
internal/proxynode/proxy.go → internal/proxy/proxy.go
View File

@ -1,4 +1,4 @@
package proxynode
package proxy
import (
"context"
@ -67,7 +67,7 @@ func CreateProxy(ctx context.Context) (*Proxy, error) {
}
cfg := &config.Configuration{
ServiceName: "proxynode",
ServiceName: "proxy",
Sampler: &config.SamplerConfig{
Type: "const",
Param: 1,
@ -238,5 +238,5 @@ func (p *Proxy) Close() {
for _, cb := range p.closeCallbacks {
cb()
}
log.Print("proxynode closed.")
log.Print("proxy closed.")
}

8
internal/proxynode/proxy_test.go → internal/proxy/proxy_test.go
View File

@ -1,4 +1,4 @@
package proxynode
package proxy
import (
"context"
@ -88,12 +88,12 @@ func startProxy(ctx context.Context) {
svr, err := CreateProxy(ctx)
proxyServer = svr
if err != nil {
log.Print("create proxynode failed", zap.Error(err))
log.Print("create proxy failed", zap.Error(err))
}
// TODO: change to wait until master is ready
if err := svr.Start(); err != nil {
log.Fatal("run proxynode failed", zap.Error(err))
log.Fatal("run proxy failed", zap.Error(err))
}
}
@ -111,7 +111,7 @@ func setup() {
conn, err := grpc.DialContext(ctx, proxyAddr, grpc.WithInsecure(), grpc.WithBlock())
if err != nil {
log.Fatalf("Connect to proxynode failed, error= %v", err)
log.Fatalf("Connect to proxy failed, error= %v", err)
}
proxyConn = conn
proxyClient = servicepb.NewMilvusServiceClient(proxyConn)

2
internal/proxynode/repack_func.go → internal/proxy/repack_func.go
View File

@ -1,4 +1,4 @@
package proxynode
package proxy
import (
"log"

2
internal/proxynode/task.go → internal/proxy/task.go
View File

@ -1,4 +1,4 @@
package proxynode
package proxy
import (
"context"

4
internal/proxynode/task_scheduler.go → internal/proxy/task_scheduler.go
View File

@ -1,4 +1,4 @@
package proxynode
package proxy
import (
"container/list"
@ -410,7 +410,7 @@ func (sched *TaskScheduler) queryResultLoop() {
}
}
case <-sched.ctx.Done():
log.Print("proxynode server is closed ...")
log.Print("proxy server is closed ...")
return
}
}

6
internal/proxynode/timetick.go → internal/proxy/timetick.go
View File

@ -1,4 +1,4 @@
package proxynode
package proxy
import (
"context"
@ -81,9 +81,9 @@ func (tt *timeTick) tick() error {
msgPack.Msgs = append(msgPack.Msgs, timeTickMsg)
err := tt.tickMsgStream.Produce(&msgPack)
if err != nil {
log.Printf("proxynode send time tick error: %v", err)
log.Printf("proxy send time tick error: %v", err)
} else {
//log.Printf("proxynode send time tick message")
//log.Printf("proxy send time tick message")
}
tt.tickLock.Lock()
defer tt.tickLock.Unlock()

2
internal/proxynode/timetick_test.go → internal/proxy/timetick_test.go
View File

@ -1,4 +1,4 @@
package proxynode
package proxy
import (
"context"

2
internal/proxynode/util.go → internal/proxy/util.go
View File

@ -1,4 +1,4 @@
package proxynode
package proxy
import (
"log"

2
internal/proxynode/validate_util.go → internal/proxy/validate_util.go
View File

@ -1,4 +1,4 @@
package proxynode
package proxy
import (
"strconv"

2
internal/proxynode/validate_util_test.go → internal/proxy/validate_util_test.go
View File

@ -1,4 +1,4 @@
package proxynode
package proxy
import (
"testing"

6
internal/querynode/load_index_service_test.go
View File

@ -13,7 +13,7 @@ import (
"github.com/golang/protobuf/proto"
"github.com/stretchr/testify/assert"
"github.com/zilliztech/milvus-distributed/internal/indexnode"
"github.com/zilliztech/milvus-distributed/internal/indexbuilder"
minioKV "github.com/zilliztech/milvus-distributed/internal/kv/minio"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
@ -243,7 +243,7 @@ func TestLoadIndexService_FloatVector(t *testing.T) {
indexRowData = append(indexRowData, float32(n*i))
}
}
index, err := indexnode.NewCIndex(typeParams, indexParams)
index, err := indexbuilder.NewCIndex(typeParams, indexParams)
assert.Nil(t, err)
err = index.BuildFloatVecIndexWithoutIds(indexRowData)
assert.Equal(t, err, nil)
@ -551,7 +551,7 @@ func TestLoadIndexService_BinaryVector(t *testing.T) {
// generator index
typeParams := make(map[string]string)
typeParams["dim"] = "128"
index, err := indexnode.NewCIndex(typeParams, indexParams)
index, err := indexbuilder.NewCIndex(typeParams, indexParams)
assert.Nil(t, err)
err = index.BuildBinaryVecIndexWithoutIds(indexRowData)
assert.Equal(t, err, nil)

14
internal/util/typeutil/service.go
View File

@ -1,14 +0,0 @@
package typeutil
import (
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
)
type Service interface {
Init()
Start()
Stop()
GetServiceStates() (internalpb2.ServiceStates, error)
GetTimeTickChannel() (string, error)
GetStatisticsChannel() (string, error)
}

3
scripts/run_go_unittest.sh
View File

@ -15,9 +15,8 @@ MILVUS_DIR="${ROOT_DIR}/internal/"
echo $MILVUS_DIR
go test -race -cover "${MILVUS_DIR}/kv/..." -failfast
go test -race -cover "${MILVUS_DIR}/proxynode/..." -failfast
go test -race -cover "${MILVUS_DIR}/proxy/..." -failfast
go test -race -cover "${MILVUS_DIR}/writenode/..." -failfast
go test -race -cover "${MILVUS_DIR}/master/..." -failfast
go test -race -cover "${MILVUS_DIR}/indexnode/..." -failfast
go test -race -cover "${MILVUS_DIR}/msgstream/..." "${MILVUS_DIR}/querynode/..." "${MILVUS_DIR}/storage" "${MILVUS_DIR}/util/..." -failfast
#go test -race -cover "${MILVUS_DIR}/kv/..." "${MILVUS_DIR}/msgstream/..." "${MILVUS_DIR}/master/..." "${MILVUS_DIR}/querynode/..." -failfast