From 1159880036a01b1b2750889592ad2826b31a470a Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Fri, 15 Nov 2019 17:00:53 +0800 Subject: [PATCH 01/21] ignore proto python --- core/.gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/.gitignore b/core/.gitignore index 74e41dba6b..8db8df41db 100644 --- a/core/.gitignore +++ b/core/.gitignore @@ -9,3 +9,5 @@ output.info output_new.info server.info *.pyc +src/grpc/python_gen.h +src/grpc/python/ From 15315158e790d2be24ba45696bd27b27705aff7f Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Sat, 16 Nov 2019 09:34:17 +0800 Subject: [PATCH 02/21] modify sdk --- .../sdk/examples/partition/src/ClientTest.cpp | 65 ++++++++++--------- core/src/sdk/examples/simple/main.cpp | 2 +- .../sdk/examples/simple/src/ClientTest.cpp | 3 +- core/src/sdk/grpc/ClientProxy.cpp | 30 ++++++--- core/src/sdk/grpc/ClientProxy.h | 2 +- 5 files changed, 58 insertions(+), 44 deletions(-) diff --git a/core/src/sdk/examples/partition/src/ClientTest.cpp b/core/src/sdk/examples/partition/src/ClientTest.cpp index 775e1f6d60..b67b1fe907 100644 --- a/core/src/sdk/examples/partition/src/ClientTest.cpp +++ b/core/src/sdk/examples/partition/src/ClientTest.cpp @@ -31,45 +31,45 @@ namespace { -const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); + const char *TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); -constexpr int64_t TABLE_DIMENSION = 512; -constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; -constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; -constexpr int64_t BATCH_ROW_COUNT = 10000; -constexpr int64_t NQ = 5; -constexpr int64_t TOP_K = 10; -constexpr int64_t NPROBE = 32; -constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different -constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; -constexpr int32_t N_LIST = 15000; -constexpr int32_t PARTITION_COUNT = 5; -constexpr int32_t TARGET_PARTITION = 3; + constexpr int64_t TABLE_DIMENSION = 512; + constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; + constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; + constexpr int64_t BATCH_ROW_COUNT = 10000; + constexpr int64_t NQ = 5; + constexpr int64_t TOP_K = 10; + constexpr int64_t NPROBE = 32; + constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different + constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; + constexpr int32_t N_LIST = 15000; + constexpr int32_t PARTITION_COUNT = 5; + constexpr int32_t TARGET_PARTITION = 3; -milvus::TableSchema -BuildTableSchema() { - milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; - return tb_schema; -} + milvus::TableSchema + BuildTableSchema() { + milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; + return tb_schema; + } -milvus::PartitionParam -BuildPartitionParam(int32_t index) { - std::string tag = std::to_string(index); - std::string partition_name = std::string(TABLE_NAME) + "_" + tag; - milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; - return partition_param; -} + milvus::PartitionParam + BuildPartitionParam(int32_t index) { + std::string tag = std::to_string(index); + std::string partition_name = std::string(TABLE_NAME) + "_" + tag; + milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; + return partition_param; + } -milvus::IndexParam -BuildIndexParam() { - milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; - return index_param; -} + milvus::IndexParam + BuildIndexParam() { + milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; + return index_param; + } } // namespace void -ClientTest::Test(const std::string& address, const std::string& port) { +ClientTest::Test(const std::string &address, const std::string &port) { std::shared_ptr conn = milvus::Connection::Create(); milvus::Status stat; @@ -78,7 +78,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } - +#ifdef yhz { // create table milvus::TableSchema tb_schema = BuildTableSchema(); stat = conn->CreateTable(tb_schema); @@ -202,4 +202,5 @@ ClientTest::Test(const std::string& address, const std::string& port) { } milvus::Connection::Destroy(conn); +#endif } diff --git a/core/src/sdk/examples/simple/main.cpp b/core/src/sdk/examples/simple/main.cpp index c08741606c..d9b6194329 100644 --- a/core/src/sdk/examples/simple/main.cpp +++ b/core/src/sdk/examples/simple/main.cpp @@ -36,7 +36,7 @@ main(int argc, char* argv[]) { {nullptr, 0, nullptr, 0}}; int option_index = 0; - std::string address = "127.0.0.1", port = "19530"; + std::string address = "192.168.1.89", port = "19530"; app_name = argv[0]; int value; diff --git a/core/src/sdk/examples/simple/src/ClientTest.cpp b/core/src/sdk/examples/simple/src/ClientTest.cpp index dfa5e2219e..da43f2f2aa 100644 --- a/core/src/sdk/examples/simple/src/ClientTest.cpp +++ b/core/src/sdk/examples/simple/src/ClientTest.cpp @@ -68,7 +68,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } - +#ifdef yhz { // server version std::string version = conn->ServerVersion(); std::cout << "Server version: " << version << std::endl; @@ -206,4 +206,5 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::string status = conn->ServerStatus(); std::cout << "Server status after disconnect: " << status << std::endl; } +#endif } diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 4ec94cfa98..5e22904a08 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -43,16 +43,28 @@ Status ClientProxy::Connect(const ConnectParam& param) { std::string uri = param.ip_address + ":" + param.port; - channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); - if (channel_ != nullptr) { - connected_ = true; - client_ptr_ = std::make_shared(channel_); - return Status::OK(); - } +// channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); - std::string reason = "connect failed!"; - connected_ = false; - return Status(StatusCode::NotConnected, reason); +// channel_ = std::make_shared(grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr)); +// channel_ = std::shared_ptr(grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr)); + auto uri_str = uri.c_str(); + grpc_channel * channel = grpc_insecure_channel_create(uri_str, nullptr, nullptr); +// grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr); + auto state = grpc_channel_check_connectivity_state(channel, true); + if (state == GRPC_CHANNEL_READY) { + std::cout << "Connect " << uri << " successfully"; + } else { + std::cout << "Connect " << uri << " failed."; + } +// if (channel_ != nullptr) { +// connected_ = true; +// client_ptr_ = std::make_shared(channel_); +// return Status::OK(); +// } + +// std::string reason = "connect failed!"; +// connected_ = false; +// return Status(StatusCode::NotConnected, reason); } Status diff --git a/core/src/sdk/grpc/ClientProxy.h b/core/src/sdk/grpc/ClientProxy.h index e332266acf..572b782769 100644 --- a/core/src/sdk/grpc/ClientProxy.h +++ b/core/src/sdk/grpc/ClientProxy.h @@ -105,7 +105,7 @@ class ClientProxy : public Connection { DropPartition(const PartitionParam& partition_param) override; private: - std::shared_ptr<::grpc::Channel> channel_; + std::shared_ptr channel_; private: std::shared_ptr client_ptr_; From a5f4484f9d760385a3f972c84eb4377c670fd9ec Mon Sep 17 00:00:00 2001 From: "G.Y Feng" <40229765+GuanyunFeng@users.noreply.github.com> Date: Mon, 18 Nov 2019 18:31:12 +0800 Subject: [PATCH 03/21] Update README_CN.md Add a link to C++ SDK in README_CN #383 --- README_CN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_CN.md b/README_CN.md index 5dad64af9b..8f4d4b05aa 100644 --- a/README_CN.md +++ b/README_CN.md @@ -20,7 +20,7 @@ Milvus 是一款开源的、针对海量特征向量的相似性搜索引擎。基于异构众核计算框架设计，成本更低，性能更好。在有限的计算资源下，十亿向量搜索仅毫秒响应。 -Milvus 提供稳定的 Python、Java 以及 C++ 的 API 接口。 +Milvus 提供稳定的 [Python](https://github.com/milvus-io/pymilvus)、[Java](https://github.com/milvus-io/milvus-sdk-java) 以及 [C++](https://github.com/milvus-io/milvus/tree/master/core/src/sdk) 的 API 接口。 通过 [版本发布说明](https://milvus.io/docs/zh-CN/release/v0.5.0/) 获取最新发行版本的 Milvus。 From 039f23e08205831017022d2dcab560d5f279431c Mon Sep 17 00:00:00 2001 From: "G.Y Feng" <40229765+GuanyunFeng@users.noreply.github.com> Date: Mon, 18 Nov 2019 18:32:32 +0800 Subject: [PATCH 04/21] Update README.md Add a link to C++ SDK in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 311d0d6843..4a1151dbf8 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Milvus is the world's fastest similarity search engine for massive-scale feature For more detailed introduction of Milvus and its architecture, see [Milvus overview](https://www.milvus.io/docs/en/aboutmilvus/overview/). -Milvus provides stable [Python](https://github.com/milvus-io/pymilvus), [Java](https://github.com/milvus-io/milvus-sdk-java) and C++ APIs. +Milvus provides stable [Python](https://github.com/milvus-io/pymilvus), [Java](https://github.com/milvus-io/milvus-sdk-java) and [C++](https://github.com/milvus-io/milvus/tree/master/core/src/sdk) APIs. Keep up-to-date with newest releases and latest updates by reading Milvus [release notes](https://www.milvus.io/docs/en/release/v0.5.0/). From 7e67c76e2c5c9a5791de18e9d5ed131f03ef1681 Mon Sep 17 00:00:00 2001 From: "G.Y Feng" <40229765+GuanyunFeng@users.noreply.github.com> Date: Mon, 18 Nov 2019 18:34:31 +0800 Subject: [PATCH 05/21] Update ubuntu_build_deps.sh solve issue #387, "failed to create symbolic link '/usr/lib/x86_64-linux-gnu/libmysqlclient_r.so': File exists" --- core/ubuntu_build_deps.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/ubuntu_build_deps.sh b/core/ubuntu_build_deps.sh index 911046aa1f..6f3fabc084 100755 --- a/core/ubuntu_build_deps.sh +++ b/core/ubuntu_build_deps.sh @@ -9,4 +9,6 @@ sudo apt-get -y update && sudo apt-get -y install intel-mkl-gnu-2019.5-281 intel sudo apt-get install -y gfortran libmysqlclient-dev mysql-client libcurl4-openssl-dev libboost-system-dev \ libboost-filesystem-dev libboost-serialization-dev libboost-regex-dev -sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so +if [ ! -f "/usr/lib/x86_64-linux-gnu/libmysqlclient_r.so" ]; then + sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so +fi From 99deaf5c503b20b6af135549bc94a2b28c7d9da1 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Tue, 19 Nov 2019 17:37:13 +0800 Subject: [PATCH 06/21] modify shards for v0.5.3 --- shards/mishards/connections.py | 10 +++ shards/mishards/service_handler.py | 107 +++++++++++++++++++++-------- 2 files changed, 89 insertions(+), 28 deletions(-) diff --git a/shards/mishards/connections.py b/shards/mishards/connections.py index 618690a099..50e214ec9a 100644 --- a/shards/mishards/connections.py +++ b/shards/mishards/connections.py @@ -2,6 +2,7 @@ import logging import threading from functools import wraps from milvus import Milvus +from milvus.client.hooks import BaseaSearchHook from mishards import (settings, exceptions) from utils import singleton @@ -9,6 +10,12 @@ from utils import singleton logger = logging.getLogger(__name__) +class Searchook(BaseaSearchHook): + + def on_response(self, *args, **kwargs): + return True + + class Connection: def __init__(self, name, uri, max_retry=1, error_handlers=None, **kwargs): self.name = name @@ -18,6 +25,9 @@ class Connection: self.conn = Milvus() self.error_handlers = [] if not error_handlers else error_handlers self.on_retry_func = kwargs.get('on_retry_func', None) + + # define search hook + self.conn._set_hook(search_in_file=Searchook()) # self._connect() def __str__(self): diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index 2f19152ae6..620f6213de 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -29,39 +29,88 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): self.router = router self.max_workers = max_workers + def _reduce(self, source_ids, ids, source_diss, diss, k, reverse): + if source_diss[k - 1] <= diss[0]: + return source_ids, source_diss + if diss[k - 1] <= source_diss[0]: + return ids, diss + + diss_t = enumerate(source_diss.extend(diss)) + diss_m_rst = sorted(diss_t, key=lambda x: x[1])[:k] + diss_m_out = [id_ for _, id_ in diss_m_rst] + + id_t = source_ids.extend(ids) + id_m_out = [id_t[i] for i, _ in diss_m_rst] + + return id_m_out, diss_m_out + def _do_merge(self, files_n_topk_results, topk, reverse=False, **kwargs): status = status_pb2.Status(error_code=status_pb2.SUCCESS, reason="Success") if not files_n_topk_results: return status, [] - request_results = defaultdict(list) + # request_results = defaultdict(list) + # row_num = files_n_topk_results[0].row_num + merge_id_results = [] + merge_dis_results = [] calc_time = time.time() for files_collection in files_n_topk_results: if isinstance(files_collection, tuple): status, _ = files_collection return status, [] - for request_pos, each_request_results in enumerate( - files_collection.topk_query_result): - request_results[request_pos].extend( - each_request_results.query_result_arrays) - request_results[request_pos] = sorted( - request_results[request_pos], - key=lambda x: x.distance, - reverse=reverse)[:topk] + + row_num = files_collection.row_num + ids = files_collection.ids + diss = files_collection.distances # distance collections + batch_len = len(ids) // row_num + + for row_index in range(row_num): + id_batch = ids[row_index * batch_len: (row_index + 1) * batch_len] + dis_batch = diss[row_index * batch_len: (row_index + 1) * batch_len] + + if len(merge_id_results) < row_index: + raise ValueError("merge error") + elif len(merge_id_results) == row_index: + # TODO: may bug here + merge_id_results.append(id_batch) + merge_dis_results.append(dis_batch) + else: + merge_id_results[row_index].extend(ids[row_index * batch_len, (row_index + 1) * batch_len]) + merge_dis_results[row_index].extend(diss[row_index * batch_len, (row_index + 1) * batch_len]) + # _reduce(_ids, _diss, k, reverse) + merge_id_results[row_index], merge_dis_results[row_index] = \ + self._reduce(merge_id_results[row_index], id_batch, + merge_dis_results[row_index], dis_batch, + batch_len, + reverse) + + # for request_pos, each_request_results in enumerate( + # files_collection.topk_query_result): + # request_results[request_pos].extend( + # each_request_results.query_result_arrays) + # request_results[request_pos] = sorted( + # request_results[request_pos], + # key=lambda x: x.distance, + # reverse=reverse)[:topk] calc_time = time.time() - calc_time logger.info('Merge takes {}'.format(calc_time)) - results = sorted(request_results.items()) - topk_query_result = [] + # results = sorted(request_results.items()) + id_mrege_list = [] + dis_mrege_list = [] - for result in results: - query_result = TopKQueryResult(query_result_arrays=result[1]) - topk_query_result.append(query_result) + for id_results, dis_results in zip(merge_id_results, merge_dis_results): + id_mrege_list.extend(id_results) + dis_mrege_list.extend(dis_results) - return status, topk_query_result + # for result in results: + # query_result = TopKQueryResult(query_result_arrays=result[1]) + # topk_query_result.append(query_result) + + return status, id_mrege_list, dis_mrege_list def _do_query(self, context, @@ -109,8 +158,8 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): file_ids=query_params['file_ids'], query_records=vectors, top_k=topk, - nprobe=nprobe, - lazy_=True) + nprobe=nprobe + ) end = time.time() logger.info('search_vectors_in_files takes: {}'.format(end - start)) @@ -241,7 +290,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): logger.info('Search {}: topk={} nprobe={}'.format( table_name, topk, nprobe)) - metadata = {'resp_class': milvus_pb2.TopKQueryResultList} + metadata = {'resp_class': milvus_pb2.TopKQueryResult} if nprobe > self.MAX_NPROBE or nprobe <= 0: raise exceptions.InvalidArgumentError( @@ -275,22 +324,24 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): query_range_array.append( Range(query_range.start_value, query_range.end_value)) - status, results = self._do_query(context, - table_name, - table_meta, - query_record_array, - topk, - nprobe, - query_range_array, - metadata=metadata) + status, id_results, dis_results = self._do_query(context, + table_name, + table_meta, + query_record_array, + topk, + nprobe, + query_range_array, + metadata=metadata) now = time.time() logger.info('SearchVector takes: {}'.format(now - start)) - topk_result_list = milvus_pb2.TopKQueryResultList( + topk_result_list = milvus_pb2.TopKQueryResult( status=status_pb2.Status(error_code=status.error_code, reason=status.reason), - topk_query_result=results) + row_num=len(query_record_array), + ids=id_results, + distances=dis_results) return topk_result_list @mark_grpc_method From 67605968b8977ddb9e29c7f0a2ea8d49a3e4f703 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Tue, 19 Nov 2019 20:36:08 +0800 Subject: [PATCH 07/21] finish results reduce in mishards --- shards/mishards/service_handler.py | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index 620f6213de..640ae61ba8 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -34,13 +34,14 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): return source_ids, source_diss if diss[k - 1] <= source_diss[0]: return ids, diss - - diss_t = enumerate(source_diss.extend(diss)) + + source_diss.extend(diss) + diss_t = enumerate(source_diss) diss_m_rst = sorted(diss_t, key=lambda x: x[1])[:k] diss_m_out = [id_ for _, id_ in diss_m_rst] - id_t = source_ids.extend(ids) - id_m_out = [id_t[i] for i, _ in diss_m_rst] + source_ids.extend(ids) + id_m_out = [source_ids[i] for i, _ in diss_m_rst] return id_m_out, diss_m_out @@ -50,8 +51,6 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): if not files_n_topk_results: return status, [] - # request_results = defaultdict(list) - # row_num = files_n_topk_results[0].row_num merge_id_results = [] merge_dis_results = [] @@ -64,6 +63,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): row_num = files_collection.row_num ids = files_collection.ids diss = files_collection.distances # distance collections + # TODO: batch_len is equal to topk batch_len = len(ids) // row_num for row_index in range(row_num): @@ -77,28 +77,16 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): merge_id_results.append(id_batch) merge_dis_results.append(dis_batch) else: - merge_id_results[row_index].extend(ids[row_index * batch_len, (row_index + 1) * batch_len]) - merge_dis_results[row_index].extend(diss[row_index * batch_len, (row_index + 1) * batch_len]) - # _reduce(_ids, _diss, k, reverse) merge_id_results[row_index], merge_dis_results[row_index] = \ self._reduce(merge_id_results[row_index], id_batch, merge_dis_results[row_index], dis_batch, batch_len, reverse) - # for request_pos, each_request_results in enumerate( - # files_collection.topk_query_result): - # request_results[request_pos].extend( - # each_request_results.query_result_arrays) - # request_results[request_pos] = sorted( - # request_results[request_pos], - # key=lambda x: x.distance, - # reverse=reverse)[:topk] calc_time = time.time() - calc_time logger.info('Merge takes {}'.format(calc_time)) - # results = sorted(request_results.items()) id_mrege_list = [] dis_mrege_list = [] @@ -106,10 +94,6 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): id_mrege_list.extend(id_results) dis_mrege_list.extend(dis_results) - # for result in results: - # query_result = TopKQueryResult(query_result_arrays=result[1]) - # topk_query_result.append(query_result) - return status, id_mrege_list, dis_mrege_list def _do_query(self, From 37df2b6fc42e1647f8dd760eacdfeb08e60210a4 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 20 Nov 2019 10:22:42 +0800 Subject: [PATCH 08/21] #346 rename gpu_resource_config.enable_gpu to gpu_resource_config.enable --- core/conf/server_cpu_config.template | 2 +- core/conf/server_gpu_config.template | 2 +- core/src/server/Config.cpp | 57 ++++++++++++++-------------- core/src/server/Config.h | 10 ++--- core/unittest/server/test_config.cpp | 6 +-- 5 files changed, 38 insertions(+), 39 deletions(-) diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 1a384f19a1..41889f5cef 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -36,7 +36,7 @@ engine_config: gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only gpu_resource_config: - enable_gpu: false # whether to enable GPU resources + enable: false # whether to enable GPU resources cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer search_resources: # define the GPU devices used for search computation, must be in format gpux - gpu0 diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 9790c93491..531c633da7 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -36,7 +36,7 @@ engine_config: gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only gpu_resource_config: - enable_gpu: true # whether to enable GPU resources + enable: true # whether to enable GPU resources cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer search_resources: # define the GPU devices used for search computation, must be in format gpux - gpu0 diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index e3a48b6ae4..2ff91bfe3a 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -190,8 +190,8 @@ Config::ValidateConfig() { /* gpu resource config */ #ifdef MILVUS_GPU_VERSION - bool resource_enable_gpu; - s = GetGpuResourceConfigEnableGpu(resource_enable_gpu); + bool gpu_resource_enable; + s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } @@ -330,7 +330,7 @@ Config::ResetDefaultConfig() { /* gpu resource config */ #ifdef MILVUS_GPU_VERSION - s = SetGpuResourceConfigEnableGpu(CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT); + s = SetGpuResourceConfigEnable(CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); if (!s.ok()) { return s; } @@ -641,10 +641,10 @@ Config::CheckEngineConfigGpuSearchThreshold(const std::string& value) { } Status -Config::CheckGpuResourceConfigEnableGpu(const std::string& value) { +Config::CheckGpuResourceConfigEnable(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { - std::string msg = "Invalid gpu resource config: " + value + - ". Possible reason: gpu_resource_config.enable_gpu is not a boolean."; + std::string msg = + "Invalid gpu resource config: " + value + ". Possible reason: gpu_resource_config.enable is not a boolean."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -992,10 +992,9 @@ Config::GetEngineConfigGpuSearchThreshold(int32_t& value) { } Status -Config::GetGpuResourceConfigEnableGpu(bool& value) { - std::string str = - GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE_GPU, CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT); - Status s = CheckGpuResourceConfigEnableGpu(str); +Config::GetGpuResourceConfigEnable(bool& value) { + std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); + Status s = CheckGpuResourceConfigEnable(str); if (!s.ok()) { return s; } @@ -1006,13 +1005,13 @@ Config::GetGpuResourceConfigEnableGpu(bool& value) { Status Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { - bool enable_gpu = false; - Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - if (!enable_gpu) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_CAPACITY, @@ -1027,13 +1026,13 @@ Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { Status Config::GetGpuResourceConfigCacheThreshold(float& value) { - bool enable_gpu = false; - Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - if (!enable_gpu) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, @@ -1048,13 +1047,13 @@ Config::GetGpuResourceConfigCacheThreshold(float& value) { Status Config::GetGpuResourceConfigSearchResources(std::vector& value) { - bool enable_gpu = false; - Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - if (!enable_gpu) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = GetConfigSequenceStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, @@ -1073,13 +1072,13 @@ Config::GetGpuResourceConfigSearchResources(std::vector& value) { Status Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { - bool enable_gpu = false; - Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - if (!enable_gpu) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = @@ -1295,12 +1294,12 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) { /* gpu resource config */ Status -Config::SetGpuResourceConfigEnableGpu(const std::string& value) { - Status s = CheckGpuResourceConfigEnableGpu(value); +Config::SetGpuResourceConfigEnable(const std::string& value) { + Status s = CheckGpuResourceConfigEnable(value); if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE_GPU, value); + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, value); return Status::OK(); } diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 0442ae0626..7c141be57e 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -85,9 +85,9 @@ static const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT = "1000"; /* gpu resource config */ static const char* CONFIG_GPU_RESOURCE = "gpu_resource_config"; -static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU = "enable_gpu"; +static const char* CONFIG_GPU_RESOURCE_ENABLE = "enable"; #ifdef MILVUS_GPU_VERSION -static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT = "true"; +static const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "true"; #else static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT = "false"; #endif @@ -175,7 +175,7 @@ class Config { /* gpu resource config */ Status - CheckGpuResourceConfigEnableGpu(const std::string& value); + CheckGpuResourceConfigEnable(const std::string& value); Status CheckGpuResourceConfigCacheCapacity(const std::string& value); Status @@ -244,7 +244,7 @@ class Config { /* gpu resource config */ Status - GetGpuResourceConfigEnableGpu(bool& value); + GetGpuResourceConfigEnable(bool& value); Status GetGpuResourceConfigCacheCapacity(int64_t& value); Status @@ -305,7 +305,7 @@ class Config { /* gpu resource config */ Status - SetGpuResourceConfigEnableGpu(const std::string& value); + SetGpuResourceConfigEnable(const std::string& value); Status SetGpuResourceConfigCacheCapacity(const std::string& value); Status diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 6ba7f19160..e0fde62916 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -246,9 +246,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { /* gpu resource config */ bool resource_enable_gpu = true; - s = config.SetGpuResourceConfigEnableGpu(std::to_string(resource_enable_gpu)); + s = config.SetGpuResourceConfigEnable(std::to_string(resource_enable_gpu)); ASSERT_TRUE(s.ok()); - s = config.GetGpuResourceConfigEnableGpu(bool_val); + s = config.GetGpuResourceConfigEnable(bool_val); ASSERT_TRUE(s.ok()); ASSERT_TRUE(bool_val == resource_enable_gpu); @@ -394,7 +394,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { ASSERT_FALSE(s.ok()); /* gpu resource config */ - s = config.SetGpuResourceConfigEnableGpu("ok"); + s = config.SetGpuResourceConfigEnable("ok"); ASSERT_FALSE(s.ok()); #ifdef MILVUS_GPU_VERSION From f9ece14f0aa57b9465e5886f84f631b2762b18b2 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Fri, 15 Nov 2019 17:00:53 +0800 Subject: [PATCH 09/21] ignore proto python --- core/.gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/.gitignore b/core/.gitignore index 74e41dba6b..8db8df41db 100644 --- a/core/.gitignore +++ b/core/.gitignore @@ -9,3 +9,5 @@ output.info output_new.info server.info *.pyc +src/grpc/python_gen.h +src/grpc/python/ From 660953afa2d55ec7ba3d31f02a73e7d750803d95 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Sat, 16 Nov 2019 09:34:17 +0800 Subject: [PATCH 10/21] modify sdk --- .../sdk/examples/partition/src/ClientTest.cpp | 65 ++++++++++--------- core/src/sdk/examples/simple/main.cpp | 2 +- .../sdk/examples/simple/src/ClientTest.cpp | 3 +- core/src/sdk/grpc/ClientProxy.cpp | 30 ++++++--- core/src/sdk/grpc/ClientProxy.h | 2 +- 5 files changed, 58 insertions(+), 44 deletions(-) diff --git a/core/src/sdk/examples/partition/src/ClientTest.cpp b/core/src/sdk/examples/partition/src/ClientTest.cpp index 775e1f6d60..b67b1fe907 100644 --- a/core/src/sdk/examples/partition/src/ClientTest.cpp +++ b/core/src/sdk/examples/partition/src/ClientTest.cpp @@ -31,45 +31,45 @@ namespace { -const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); + const char *TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); -constexpr int64_t TABLE_DIMENSION = 512; -constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; -constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; -constexpr int64_t BATCH_ROW_COUNT = 10000; -constexpr int64_t NQ = 5; -constexpr int64_t TOP_K = 10; -constexpr int64_t NPROBE = 32; -constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different -constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; -constexpr int32_t N_LIST = 15000; -constexpr int32_t PARTITION_COUNT = 5; -constexpr int32_t TARGET_PARTITION = 3; + constexpr int64_t TABLE_DIMENSION = 512; + constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; + constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; + constexpr int64_t BATCH_ROW_COUNT = 10000; + constexpr int64_t NQ = 5; + constexpr int64_t TOP_K = 10; + constexpr int64_t NPROBE = 32; + constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different + constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; + constexpr int32_t N_LIST = 15000; + constexpr int32_t PARTITION_COUNT = 5; + constexpr int32_t TARGET_PARTITION = 3; -milvus::TableSchema -BuildTableSchema() { - milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; - return tb_schema; -} + milvus::TableSchema + BuildTableSchema() { + milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; + return tb_schema; + } -milvus::PartitionParam -BuildPartitionParam(int32_t index) { - std::string tag = std::to_string(index); - std::string partition_name = std::string(TABLE_NAME) + "_" + tag; - milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; - return partition_param; -} + milvus::PartitionParam + BuildPartitionParam(int32_t index) { + std::string tag = std::to_string(index); + std::string partition_name = std::string(TABLE_NAME) + "_" + tag; + milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; + return partition_param; + } -milvus::IndexParam -BuildIndexParam() { - milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; - return index_param; -} + milvus::IndexParam + BuildIndexParam() { + milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; + return index_param; + } } // namespace void -ClientTest::Test(const std::string& address, const std::string& port) { +ClientTest::Test(const std::string &address, const std::string &port) { std::shared_ptr conn = milvus::Connection::Create(); milvus::Status stat; @@ -78,7 +78,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } - +#ifdef yhz { // create table milvus::TableSchema tb_schema = BuildTableSchema(); stat = conn->CreateTable(tb_schema); @@ -202,4 +202,5 @@ ClientTest::Test(const std::string& address, const std::string& port) { } milvus::Connection::Destroy(conn); +#endif } diff --git a/core/src/sdk/examples/simple/main.cpp b/core/src/sdk/examples/simple/main.cpp index c08741606c..d9b6194329 100644 --- a/core/src/sdk/examples/simple/main.cpp +++ b/core/src/sdk/examples/simple/main.cpp @@ -36,7 +36,7 @@ main(int argc, char* argv[]) { {nullptr, 0, nullptr, 0}}; int option_index = 0; - std::string address = "127.0.0.1", port = "19530"; + std::string address = "192.168.1.89", port = "19530"; app_name = argv[0]; int value; diff --git a/core/src/sdk/examples/simple/src/ClientTest.cpp b/core/src/sdk/examples/simple/src/ClientTest.cpp index dfa5e2219e..da43f2f2aa 100644 --- a/core/src/sdk/examples/simple/src/ClientTest.cpp +++ b/core/src/sdk/examples/simple/src/ClientTest.cpp @@ -68,7 +68,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } - +#ifdef yhz { // server version std::string version = conn->ServerVersion(); std::cout << "Server version: " << version << std::endl; @@ -206,4 +206,5 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::string status = conn->ServerStatus(); std::cout << "Server status after disconnect: " << status << std::endl; } +#endif } diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 4a9c319b4d..1805ae81fe 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -43,16 +43,28 @@ Status ClientProxy::Connect(const ConnectParam& param) { std::string uri = param.ip_address + ":" + param.port; - channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); - if (channel_ != nullptr) { - connected_ = true; - client_ptr_ = std::make_shared(channel_); - return Status::OK(); - } +// channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); - std::string reason = "connect failed!"; - connected_ = false; - return Status(StatusCode::NotConnected, reason); +// channel_ = std::make_shared(grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr)); +// channel_ = std::shared_ptr(grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr)); + auto uri_str = uri.c_str(); + grpc_channel * channel = grpc_insecure_channel_create(uri_str, nullptr, nullptr); +// grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr); + auto state = grpc_channel_check_connectivity_state(channel, true); + if (state == GRPC_CHANNEL_READY) { + std::cout << "Connect " << uri << " successfully"; + } else { + std::cout << "Connect " << uri << " failed."; + } +// if (channel_ != nullptr) { +// connected_ = true; +// client_ptr_ = std::make_shared(channel_); +// return Status::OK(); +// } + +// std::string reason = "connect failed!"; +// connected_ = false; +// return Status(StatusCode::NotConnected, reason); } Status diff --git a/core/src/sdk/grpc/ClientProxy.h b/core/src/sdk/grpc/ClientProxy.h index e332266acf..572b782769 100644 --- a/core/src/sdk/grpc/ClientProxy.h +++ b/core/src/sdk/grpc/ClientProxy.h @@ -105,7 +105,7 @@ class ClientProxy : public Connection { DropPartition(const PartitionParam& partition_param) override; private: - std::shared_ptr<::grpc::Channel> channel_; + std::shared_ptr channel_; private: std::shared_ptr client_ptr_; From 83d9bf69662f4bbbbe51959be08f8858b1f3485d Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Tue, 19 Nov 2019 17:37:13 +0800 Subject: [PATCH 11/21] modify shards for v0.5.3 --- shards/mishards/connections.py | 10 +++ shards/mishards/service_handler.py | 107 +++++++++++++++++++++-------- 2 files changed, 89 insertions(+), 28 deletions(-) diff --git a/shards/mishards/connections.py b/shards/mishards/connections.py index 618690a099..50e214ec9a 100644 --- a/shards/mishards/connections.py +++ b/shards/mishards/connections.py @@ -2,6 +2,7 @@ import logging import threading from functools import wraps from milvus import Milvus +from milvus.client.hooks import BaseaSearchHook from mishards import (settings, exceptions) from utils import singleton @@ -9,6 +10,12 @@ from utils import singleton logger = logging.getLogger(__name__) +class Searchook(BaseaSearchHook): + + def on_response(self, *args, **kwargs): + return True + + class Connection: def __init__(self, name, uri, max_retry=1, error_handlers=None, **kwargs): self.name = name @@ -18,6 +25,9 @@ class Connection: self.conn = Milvus() self.error_handlers = [] if not error_handlers else error_handlers self.on_retry_func = kwargs.get('on_retry_func', None) + + # define search hook + self.conn._set_hook(search_in_file=Searchook()) # self._connect() def __str__(self): diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index 2f19152ae6..620f6213de 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -29,39 +29,88 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): self.router = router self.max_workers = max_workers + def _reduce(self, source_ids, ids, source_diss, diss, k, reverse): + if source_diss[k - 1] <= diss[0]: + return source_ids, source_diss + if diss[k - 1] <= source_diss[0]: + return ids, diss + + diss_t = enumerate(source_diss.extend(diss)) + diss_m_rst = sorted(diss_t, key=lambda x: x[1])[:k] + diss_m_out = [id_ for _, id_ in diss_m_rst] + + id_t = source_ids.extend(ids) + id_m_out = [id_t[i] for i, _ in diss_m_rst] + + return id_m_out, diss_m_out + def _do_merge(self, files_n_topk_results, topk, reverse=False, **kwargs): status = status_pb2.Status(error_code=status_pb2.SUCCESS, reason="Success") if not files_n_topk_results: return status, [] - request_results = defaultdict(list) + # request_results = defaultdict(list) + # row_num = files_n_topk_results[0].row_num + merge_id_results = [] + merge_dis_results = [] calc_time = time.time() for files_collection in files_n_topk_results: if isinstance(files_collection, tuple): status, _ = files_collection return status, [] - for request_pos, each_request_results in enumerate( - files_collection.topk_query_result): - request_results[request_pos].extend( - each_request_results.query_result_arrays) - request_results[request_pos] = sorted( - request_results[request_pos], - key=lambda x: x.distance, - reverse=reverse)[:topk] + + row_num = files_collection.row_num + ids = files_collection.ids + diss = files_collection.distances # distance collections + batch_len = len(ids) // row_num + + for row_index in range(row_num): + id_batch = ids[row_index * batch_len: (row_index + 1) * batch_len] + dis_batch = diss[row_index * batch_len: (row_index + 1) * batch_len] + + if len(merge_id_results) < row_index: + raise ValueError("merge error") + elif len(merge_id_results) == row_index: + # TODO: may bug here + merge_id_results.append(id_batch) + merge_dis_results.append(dis_batch) + else: + merge_id_results[row_index].extend(ids[row_index * batch_len, (row_index + 1) * batch_len]) + merge_dis_results[row_index].extend(diss[row_index * batch_len, (row_index + 1) * batch_len]) + # _reduce(_ids, _diss, k, reverse) + merge_id_results[row_index], merge_dis_results[row_index] = \ + self._reduce(merge_id_results[row_index], id_batch, + merge_dis_results[row_index], dis_batch, + batch_len, + reverse) + + # for request_pos, each_request_results in enumerate( + # files_collection.topk_query_result): + # request_results[request_pos].extend( + # each_request_results.query_result_arrays) + # request_results[request_pos] = sorted( + # request_results[request_pos], + # key=lambda x: x.distance, + # reverse=reverse)[:topk] calc_time = time.time() - calc_time logger.info('Merge takes {}'.format(calc_time)) - results = sorted(request_results.items()) - topk_query_result = [] + # results = sorted(request_results.items()) + id_mrege_list = [] + dis_mrege_list = [] - for result in results: - query_result = TopKQueryResult(query_result_arrays=result[1]) - topk_query_result.append(query_result) + for id_results, dis_results in zip(merge_id_results, merge_dis_results): + id_mrege_list.extend(id_results) + dis_mrege_list.extend(dis_results) - return status, topk_query_result + # for result in results: + # query_result = TopKQueryResult(query_result_arrays=result[1]) + # topk_query_result.append(query_result) + + return status, id_mrege_list, dis_mrege_list def _do_query(self, context, @@ -109,8 +158,8 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): file_ids=query_params['file_ids'], query_records=vectors, top_k=topk, - nprobe=nprobe, - lazy_=True) + nprobe=nprobe + ) end = time.time() logger.info('search_vectors_in_files takes: {}'.format(end - start)) @@ -241,7 +290,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): logger.info('Search {}: topk={} nprobe={}'.format( table_name, topk, nprobe)) - metadata = {'resp_class': milvus_pb2.TopKQueryResultList} + metadata = {'resp_class': milvus_pb2.TopKQueryResult} if nprobe > self.MAX_NPROBE or nprobe <= 0: raise exceptions.InvalidArgumentError( @@ -275,22 +324,24 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): query_range_array.append( Range(query_range.start_value, query_range.end_value)) - status, results = self._do_query(context, - table_name, - table_meta, - query_record_array, - topk, - nprobe, - query_range_array, - metadata=metadata) + status, id_results, dis_results = self._do_query(context, + table_name, + table_meta, + query_record_array, + topk, + nprobe, + query_range_array, + metadata=metadata) now = time.time() logger.info('SearchVector takes: {}'.format(now - start)) - topk_result_list = milvus_pb2.TopKQueryResultList( + topk_result_list = milvus_pb2.TopKQueryResult( status=status_pb2.Status(error_code=status.error_code, reason=status.reason), - topk_query_result=results) + row_num=len(query_record_array), + ids=id_results, + distances=dis_results) return topk_result_list @mark_grpc_method From 2f8be3d0586625d65f08622933d85fa8eabd6553 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Tue, 19 Nov 2019 20:36:08 +0800 Subject: [PATCH 12/21] finish results reduce in mishards --- shards/mishards/service_handler.py | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index 620f6213de..640ae61ba8 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -34,13 +34,14 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): return source_ids, source_diss if diss[k - 1] <= source_diss[0]: return ids, diss - - diss_t = enumerate(source_diss.extend(diss)) + + source_diss.extend(diss) + diss_t = enumerate(source_diss) diss_m_rst = sorted(diss_t, key=lambda x: x[1])[:k] diss_m_out = [id_ for _, id_ in diss_m_rst] - id_t = source_ids.extend(ids) - id_m_out = [id_t[i] for i, _ in diss_m_rst] + source_ids.extend(ids) + id_m_out = [source_ids[i] for i, _ in diss_m_rst] return id_m_out, diss_m_out @@ -50,8 +51,6 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): if not files_n_topk_results: return status, [] - # request_results = defaultdict(list) - # row_num = files_n_topk_results[0].row_num merge_id_results = [] merge_dis_results = [] @@ -64,6 +63,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): row_num = files_collection.row_num ids = files_collection.ids diss = files_collection.distances # distance collections + # TODO: batch_len is equal to topk batch_len = len(ids) // row_num for row_index in range(row_num): @@ -77,28 +77,16 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): merge_id_results.append(id_batch) merge_dis_results.append(dis_batch) else: - merge_id_results[row_index].extend(ids[row_index * batch_len, (row_index + 1) * batch_len]) - merge_dis_results[row_index].extend(diss[row_index * batch_len, (row_index + 1) * batch_len]) - # _reduce(_ids, _diss, k, reverse) merge_id_results[row_index], merge_dis_results[row_index] = \ self._reduce(merge_id_results[row_index], id_batch, merge_dis_results[row_index], dis_batch, batch_len, reverse) - # for request_pos, each_request_results in enumerate( - # files_collection.topk_query_result): - # request_results[request_pos].extend( - # each_request_results.query_result_arrays) - # request_results[request_pos] = sorted( - # request_results[request_pos], - # key=lambda x: x.distance, - # reverse=reverse)[:topk] calc_time = time.time() - calc_time logger.info('Merge takes {}'.format(calc_time)) - # results = sorted(request_results.items()) id_mrege_list = [] dis_mrege_list = [] @@ -106,10 +94,6 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): id_mrege_list.extend(id_results) dis_mrege_list.extend(dis_results) - # for result in results: - # query_result = TopKQueryResult(query_result_arrays=result[1]) - # topk_query_result.append(query_result) - return status, id_mrege_list, dis_mrege_list def _do_query(self, From b552b77a7eb6aaec2afc4ce67f452a00adea982d Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Wed, 20 Nov 2019 10:53:28 +0800 Subject: [PATCH 13/21] [skip ci] add comments in shards --- shards/mishards/service_handler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index 640ae61ba8..fc0ee0fa2b 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -63,7 +63,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): row_num = files_collection.row_num ids = files_collection.ids diss = files_collection.distances # distance collections - # TODO: batch_len is equal to topk + # TODO: batch_len is equal to topk, may need to compare with topk batch_len = len(ids) // row_num for row_index in range(row_num): @@ -83,7 +83,6 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): batch_len, reverse) - calc_time = time.time() - calc_time logger.info('Merge takes {}'.format(calc_time)) From 631cf7143976f6e08bc4afb7064b46b773af8010 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 20 Nov 2019 10:57:01 +0800 Subject: [PATCH 14/21] #346 fix CPU version build error --- core/src/server/Config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 7c141be57e..a6e78671b3 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -89,7 +89,7 @@ static const char* CONFIG_GPU_RESOURCE_ENABLE = "enable"; #ifdef MILVUS_GPU_VERSION static const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "true"; #else -static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT = "false"; +static const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "false"; #endif static const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY = "cache_capacity"; static const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT = "4"; From a5a41212f8489e3e69b4249dfcc1c1b42a13d513 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Wed, 20 Nov 2019 11:06:24 +0800 Subject: [PATCH 15/21] remove test change' --- .../sdk/examples/partition/src/ClientTest.cpp | 3 +- core/src/sdk/examples/simple/main.cpp | 2 +- .../sdk/examples/simple/src/ClientTest.cpp | 3 +- core/src/sdk/grpc/ClientProxy.cpp | 28 ++++++------------- core/src/sdk/grpc/ClientProxy.h | 2 +- 5 files changed, 12 insertions(+), 26 deletions(-) diff --git a/core/src/sdk/examples/partition/src/ClientTest.cpp b/core/src/sdk/examples/partition/src/ClientTest.cpp index b67b1fe907..79ed7646b9 100644 --- a/core/src/sdk/examples/partition/src/ClientTest.cpp +++ b/core/src/sdk/examples/partition/src/ClientTest.cpp @@ -78,7 +78,7 @@ ClientTest::Test(const std::string &address, const std::string &port) { stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } -#ifdef yhz + { // create table milvus::TableSchema tb_schema = BuildTableSchema(); stat = conn->CreateTable(tb_schema); @@ -202,5 +202,4 @@ ClientTest::Test(const std::string &address, const std::string &port) { } milvus::Connection::Destroy(conn); -#endif } diff --git a/core/src/sdk/examples/simple/main.cpp b/core/src/sdk/examples/simple/main.cpp index d9b6194329..c08741606c 100644 --- a/core/src/sdk/examples/simple/main.cpp +++ b/core/src/sdk/examples/simple/main.cpp @@ -36,7 +36,7 @@ main(int argc, char* argv[]) { {nullptr, 0, nullptr, 0}}; int option_index = 0; - std::string address = "192.168.1.89", port = "19530"; + std::string address = "127.0.0.1", port = "19530"; app_name = argv[0]; int value; diff --git a/core/src/sdk/examples/simple/src/ClientTest.cpp b/core/src/sdk/examples/simple/src/ClientTest.cpp index da43f2f2aa..dfa5e2219e 100644 --- a/core/src/sdk/examples/simple/src/ClientTest.cpp +++ b/core/src/sdk/examples/simple/src/ClientTest.cpp @@ -68,7 +68,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } -#ifdef yhz + { // server version std::string version = conn->ServerVersion(); std::cout << "Server version: " << version << std::endl; @@ -206,5 +206,4 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::string status = conn->ServerStatus(); std::cout << "Server status after disconnect: " << status << std::endl; } -#endif } diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 1805ae81fe..4a9c319b4d 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -43,28 +43,16 @@ Status ClientProxy::Connect(const ConnectParam& param) { std::string uri = param.ip_address + ":" + param.port; -// channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); - -// channel_ = std::make_shared(grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr)); -// channel_ = std::shared_ptr(grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr)); - auto uri_str = uri.c_str(); - grpc_channel * channel = grpc_insecure_channel_create(uri_str, nullptr, nullptr); -// grpc_insecure_channel_create(uri.c_str(), nullptr, nullptr); - auto state = grpc_channel_check_connectivity_state(channel, true); - if (state == GRPC_CHANNEL_READY) { - std::cout << "Connect " << uri << " successfully"; - } else { - std::cout << "Connect " << uri << " failed."; + channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); + if (channel_ != nullptr) { + connected_ = true; + client_ptr_ = std::make_shared(channel_); + return Status::OK(); } -// if (channel_ != nullptr) { -// connected_ = true; -// client_ptr_ = std::make_shared(channel_); -// return Status::OK(); -// } -// std::string reason = "connect failed!"; -// connected_ = false; -// return Status(StatusCode::NotConnected, reason); + std::string reason = "connect failed!"; + connected_ = false; + return Status(StatusCode::NotConnected, reason); } Status diff --git a/core/src/sdk/grpc/ClientProxy.h b/core/src/sdk/grpc/ClientProxy.h index 572b782769..e332266acf 100644 --- a/core/src/sdk/grpc/ClientProxy.h +++ b/core/src/sdk/grpc/ClientProxy.h @@ -105,7 +105,7 @@ class ClientProxy : public Connection { DropPartition(const PartitionParam& partition_param) override; private: - std::shared_ptr channel_; + std::shared_ptr<::grpc::Channel> channel_; private: std::shared_ptr client_ptr_; From ed5219b72804d86fc910495418411aa981348a49 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Wed, 20 Nov 2019 11:23:48 +0800 Subject: [PATCH 16/21] [skip ci] foramt pass --- .../sdk/examples/partition/src/ClientTest.cpp | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/core/src/sdk/examples/partition/src/ClientTest.cpp b/core/src/sdk/examples/partition/src/ClientTest.cpp index 79ed7646b9..775e1f6d60 100644 --- a/core/src/sdk/examples/partition/src/ClientTest.cpp +++ b/core/src/sdk/examples/partition/src/ClientTest.cpp @@ -31,45 +31,45 @@ namespace { - const char *TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); +const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); - constexpr int64_t TABLE_DIMENSION = 512; - constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; - constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; - constexpr int64_t BATCH_ROW_COUNT = 10000; - constexpr int64_t NQ = 5; - constexpr int64_t TOP_K = 10; - constexpr int64_t NPROBE = 32; - constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different - constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; - constexpr int32_t N_LIST = 15000; - constexpr int32_t PARTITION_COUNT = 5; - constexpr int32_t TARGET_PARTITION = 3; +constexpr int64_t TABLE_DIMENSION = 512; +constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; +constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; +constexpr int64_t BATCH_ROW_COUNT = 10000; +constexpr int64_t NQ = 5; +constexpr int64_t TOP_K = 10; +constexpr int64_t NPROBE = 32; +constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different +constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; +constexpr int32_t N_LIST = 15000; +constexpr int32_t PARTITION_COUNT = 5; +constexpr int32_t TARGET_PARTITION = 3; - milvus::TableSchema - BuildTableSchema() { - milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; - return tb_schema; - } +milvus::TableSchema +BuildTableSchema() { + milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; + return tb_schema; +} - milvus::PartitionParam - BuildPartitionParam(int32_t index) { - std::string tag = std::to_string(index); - std::string partition_name = std::string(TABLE_NAME) + "_" + tag; - milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; - return partition_param; - } +milvus::PartitionParam +BuildPartitionParam(int32_t index) { + std::string tag = std::to_string(index); + std::string partition_name = std::string(TABLE_NAME) + "_" + tag; + milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; + return partition_param; +} - milvus::IndexParam - BuildIndexParam() { - milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; - return index_param; - } +milvus::IndexParam +BuildIndexParam() { + milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; + return index_param; +} } // namespace void -ClientTest::Test(const std::string &address, const std::string &port) { +ClientTest::Test(const std::string& address, const std::string& port) { std::shared_ptr conn = milvus::Connection::Create(); milvus::Status stat; From 359197eaafad45e8b0d32df1cc1726b7ac2f8bd3 Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Wed, 20 Nov 2019 11:35:36 +0800 Subject: [PATCH 17/21] update requirements.txt --- shards/requirements.txt | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/shards/requirements.txt b/shards/requirements.txt index 14bdde2a06..d17c13ddc9 100644 --- a/shards/requirements.txt +++ b/shards/requirements.txt @@ -1,21 +1,41 @@ +astroid==2.3.3 +atomicwrites==1.3.0 +attrs==19.3.0 +cachetools==3.1.1 +certifi==2019.9.11 +chardet==3.0.4 environs==4.2.0 factory-boy==2.12.0 Faker==1.0.7 fire==0.1.3 google-auth==1.6.3 grpcio==1.22.0 +grpcio-opentracing==1.1.4 grpcio-tools==1.22.0 +idna==2.8 +importlib-metadata==0.23 +isort==4.3.21 +jaeger-client==4.1.0 kubernetes==10.0.1 +lazy-object-proxy==1.4.3 MarkupSafe==1.1.1 marshmallow==2.19.5 -pymysql==0.9.3 +mccabe==0.6.1 +mock==2.0.0 +more-itertools==7.2.0 +oauthlib==3.1.0 +opentracing==2.2.0 +packaging==19.2 +pbr==5.4.3 +pluggy==0.13.0 +pluginbase==1.0.0 protobuf==3.9.1 py==1.8.0 pyasn1==0.4.7 pyasn1-modules==0.2.6 pylint==2.3.1 -pymilvus-test==0.2.28 -#pymilvus==0.2.0 +pymilvus==0.2.5 +PyMySQL==0.9.3 pyparsing==2.4.0 pytest==4.6.3 pytest-level==0.1.1 @@ -25,13 +45,19 @@ pytest-timeout==1.3.3 python-dateutil==2.8.0 python-dotenv==0.10.3 pytz==2019.1 +PyYAML==5.1.2 requests==2.22.0 requests-oauthlib==1.2.0 rsa==4.0 six==1.12.0 SQLAlchemy==1.3.5 +text-unidecode==1.2 +threadloop==1.0.2 +thrift==0.13.0 +tornado==5.1.1 +typed-ast==1.4.0 urllib3==1.25.3 -jaeger-client>=3.4.0 -grpcio-opentracing>=1.0 -mock==2.0.0 -pluginbase==1.0.0 +wcwidth==0.1.7 +websocket-client==0.56.0 +wrapt==1.11.2 +zipp==0.6.0 From 692b2110315fc093ab908d44059875417ec5f4ec Mon Sep 17 00:00:00 2001 From: yhz <413554850@qq.com> Date: Wed, 20 Nov 2019 11:51:23 +0800 Subject: [PATCH 18/21] update requirements only pymilvus --- shards/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shards/requirements.txt b/shards/requirements.txt index 14bdde2a06..4ea03e0b17 100644 --- a/shards/requirements.txt +++ b/shards/requirements.txt @@ -14,7 +14,7 @@ py==1.8.0 pyasn1==0.4.7 pyasn1-modules==0.2.6 pylint==2.3.1 -pymilvus-test==0.2.28 +pymilvus==0.2.5 #pymilvus==0.2.0 pyparsing==2.4.0 pytest==4.6.3 From 58e591ae7c1d24b929f7a542fead68fdae803e3c Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 20 Nov 2019 14:19:31 +0800 Subject: [PATCH 19/21] Update Jenkinsfile --- ci/jenkins/Jenkinsfile | 40 ++++++++++++++++++++-------------------- ci/scripts/build.sh | 10 ++++++++-- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 8d3953b112..bf7a70d11a 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -50,7 +50,7 @@ pipeline { } stages { - stage("Run Build") { + stage("Run GPU Version Build") { agent { kubernetes { label "${BINRARY_VERSION}-build" @@ -60,7 +60,7 @@ pipeline { } stages { - stage('Build') { + stage('GPU Version Build') { steps { container('milvus-build-env') { script { @@ -69,7 +69,7 @@ pipeline { } } } - stage('Code Coverage') { + stage('GPU Version Code Coverage') { steps { container('milvus-build-env') { script { @@ -78,7 +78,7 @@ pipeline { } } } - stage('Upload Package') { + stage('Upload GPU Version Package') { steps { container('milvus-build-env') { script { @@ -90,7 +90,7 @@ pipeline { } } - stage("Publish docker images") { + stage("Publish GPU Version docker images") { agent { kubernetes { label "${BINRARY_VERSION}-publish" @@ -100,7 +100,7 @@ pipeline { } stages { - stage('Publish') { + stage('Publish GPU Version') { steps { container('publish-images'){ script { @@ -112,7 +112,7 @@ pipeline { } } - stage("Deploy to Development") { + stage("Deploy GPU Version to Development") { agent { kubernetes { label "${BINRARY_VERSION}-dev-test" @@ -122,7 +122,7 @@ pipeline { } stages { - stage("Deploy to Dev") { + stage("Deploy GPU Version to Dev") { steps { container('milvus-test-env') { script { @@ -132,7 +132,7 @@ pipeline { } } - stage("Dev Test") { + stage("GPU Version Dev Test") { steps { container('milvus-test-env') { script { @@ -147,7 +147,7 @@ pipeline { } } - stage ("Cleanup Dev") { + stage ("Cleanup GPU Version Dev") { steps { container('milvus-test-env') { script { @@ -180,7 +180,7 @@ pipeline { } stages { - stage("Run Build") { + stage("Run CPU Version Build") { agent { kubernetes { label "${BINRARY_VERSION}-build" @@ -190,7 +190,7 @@ pipeline { } stages { - stage('Build') { + stage('Build CPU Version') { steps { container('milvus-build-env') { script { @@ -199,7 +199,7 @@ pipeline { } } } - stage('Code Coverage') { + stage('CPU Version Code Coverage') { steps { container('milvus-build-env') { script { @@ -208,7 +208,7 @@ pipeline { } } } - stage('Upload Package') { + stage('Upload CPU Version Package') { steps { container('milvus-build-env') { script { @@ -220,7 +220,7 @@ pipeline { } } - stage("Publish docker images") { + stage("Publish CPU Version docker images") { agent { kubernetes { label "${BINRARY_VERSION}-publish" @@ -230,7 +230,7 @@ pipeline { } stages { - stage('Publish') { + stage('Publish CPU Version') { steps { container('publish-images'){ script { @@ -242,7 +242,7 @@ pipeline { } } - stage("Deploy to Development") { + stage("Deploy CPU Version to Development") { agent { kubernetes { label "${BINRARY_VERSION}-dev-test" @@ -252,7 +252,7 @@ pipeline { } stages { - stage("Deploy to Dev") { + stage("Deploy CPU Version to Dev") { steps { container('milvus-test-env') { script { @@ -262,7 +262,7 @@ pipeline { } } - stage("Dev Test") { + stage("CPU Version Dev Test") { steps { container('milvus-test-env') { script { @@ -277,7 +277,7 @@ pipeline { } } - stage ("Cleanup Dev") { + stage ("Cleanup CPU Version Dev") { steps { container('milvus-test-env') { script { diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 27962ccb54..eb5205ad26 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -16,6 +16,7 @@ BUILD_TYPE="Debug" BUILD_UNITTEST="OFF" INSTALL_PREFIX="/opt/milvus" FAISS_ROOT="" +CUSTOMIZATION="OFF" # default use origin faiss BUILD_COVERAGE="OFF" USE_JFROG_CACHE="OFF" RUN_CPPLINT="OFF" @@ -23,7 +24,7 @@ GPU_VERSION="OFF" WITH_MKL="OFF" CUDA_COMPILER=/usr/local/cuda/bin/nvcc -while getopts "o:t:b:f:gulcjmh" arg +while getopts "o:t:b:f:gxulcjmh" arg do case $arg in o) @@ -41,6 +42,9 @@ do g) GPU_VERSION="ON"; ;; + x) + CUSTOMIZATION="ON"; + ;; u) echo "Build and run unittest cases" ; BUILD_UNITTEST="ON"; @@ -66,6 +70,7 @@ parameter: -b: core code build directory -f: faiss root path -g: gpu version +-x: milvus customization (default: OFF) -u: building unit test options(default: OFF) -l: run cpplint, clang-format and clang-tidy(default: OFF) -c: code coverage(default: OFF) @@ -74,7 +79,7 @@ parameter: -h: help usage: -./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-u] [-l] [-c] [-j] [-m] [-h] +./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-g] [-x] [-u] [-l] [-c] [-j] [-m] [-h] " exit 0 ;; @@ -96,6 +101,7 @@ CMAKE_CMD="cmake \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DMILVUS_GPU_VERSION=${GPU_VERSION} \ +-DCUSTOMIZATION=${CUSTOMIZATION} \ -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ From c89bb6699ebcfdc1239635b7775bf3a827e0970b Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 20 Nov 2019 11:24:54 +0800 Subject: [PATCH 20/21] #306 Use int64 for all config integer --- core/src/db/Options.h | 2 +- core/src/db/engine/ExecutionEngineImpl.cpp | 4 +- core/src/db/engine/ExecutionEngineImpl.h | 4 +- core/src/scheduler/SchedInst.cpp | 4 +- .../scheduler/optimizer/BuildIndexPass.cpp | 2 +- core/src/scheduler/optimizer/FaissFlatPass.h | 4 +- .../scheduler/optimizer/FaissIVFFlatPass.h | 4 +- .../scheduler/optimizer/FaissIVFSQ8HPass.cpp | 2 +- .../scheduler/optimizer/FaissIVFSQ8HPass.h | 4 +- .../src/scheduler/optimizer/FaissIVFSQ8Pass.h | 4 +- core/src/server/Config.cpp | 70 +++++++++---------- core/src/server/Config.h | 16 ++--- core/src/server/DBWrapper.cpp | 8 +-- core/src/utils/CommonUtil.cpp | 2 +- core/src/utils/CommonUtil.h | 2 +- core/src/wrapper/KnowhereResource.cpp | 4 +- core/unittest/server/test_config.cpp | 45 ++++++------ core/unittest/server/test_util.cpp | 2 +- 18 files changed, 91 insertions(+), 92 deletions(-) diff --git a/core/src/db/Options.h b/core/src/db/Options.h index ebecb4de5a..91459a8d5f 100644 --- a/core/src/db/Options.h +++ b/core/src/db/Options.h @@ -33,7 +33,7 @@ static const char* ARCHIVE_CONF_DISK = "disk"; static const char* ARCHIVE_CONF_DAYS = "days"; struct ArchiveConf { - using CriteriaT = std::map; + using CriteriaT = std::map; explicit ArchiveConf(const std::string& type, const std::string& criterias = std::string()); diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 4f2f542254..973d8b5912 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -146,7 +146,7 @@ ExecutionEngineImpl::HybridLoad() const { const std::string key = location_ + ".quantizer"; server::Config& config = server::Config::GetInstance(); - std::vector gpus; + std::vector gpus; Status s = config.GetGpuResourceConfigSearchResources(gpus); if (!s.ok()) { ENGINE_LOG_ERROR << s.message(); @@ -586,7 +586,7 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { Status ExecutionEngineImpl::Init() { server::Config& config = server::Config::GetInstance(); - std::vector gpu_ids; + std::vector gpu_ids; Status s = config.GetGpuResourceConfigBuildIndexResources(gpu_ids); for (auto id : gpu_ids) { if (gpu_num_ == id) { diff --git a/core/src/db/engine/ExecutionEngineImpl.h b/core/src/db/engine/ExecutionEngineImpl.h index 7eb304426e..da0e7cfb64 100644 --- a/core/src/db/engine/ExecutionEngineImpl.h +++ b/core/src/db/engine/ExecutionEngineImpl.h @@ -122,8 +122,8 @@ class ExecutionEngineImpl : public ExecutionEngine { int64_t dim_; std::string location_; - int32_t nlist_ = 0; - int32_t gpu_num_ = 0; + int64_t nlist_ = 0; + int64_t gpu_num_ = 0; }; } // namespace engine diff --git a/core/src/scheduler/SchedInst.cpp b/core/src/scheduler/SchedInst.cpp index 867052ef5d..69d293f986 100644 --- a/core/src/scheduler/SchedInst.cpp +++ b/core/src/scheduler/SchedInst.cpp @@ -55,9 +55,9 @@ load_simple_config() { // get resources #ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); - std::vector gpu_ids; + std::vector gpu_ids; config.GetGpuResourceConfigSearchResources(gpu_ids); - std::vector build_gpu_ids; + std::vector build_gpu_ids; config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids); auto pcie = Connection("pcie", 12000); diff --git a/core/src/scheduler/optimizer/BuildIndexPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp index cd1b7550d8..d535b9675f 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.cpp +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -26,7 +26,7 @@ namespace scheduler { void BuildIndexPass::Init() { server::Config& config = server::Config::GetInstance(); - std::vector build_resources; + std::vector build_resources; Status s = config.GetGpuResourceConfigBuildIndexResources(build_resources); if (!s.ok()) { throw; diff --git a/core/src/scheduler/optimizer/FaissFlatPass.h b/core/src/scheduler/optimizer/FaissFlatPass.h index 60adf62a07..f219bebdf3 100644 --- a/core/src/scheduler/optimizer/FaissFlatPass.h +++ b/core/src/scheduler/optimizer/FaissFlatPass.h @@ -45,9 +45,9 @@ class FaissFlatPass : public Pass { Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; - std::vector gpus; + std::vector gpus; }; using FaissFlatPassPtr = std::shared_ptr; diff --git a/core/src/scheduler/optimizer/FaissIVFFlatPass.h b/core/src/scheduler/optimizer/FaissIVFFlatPass.h index 9264db32a8..2d15539014 100644 --- a/core/src/scheduler/optimizer/FaissIVFFlatPass.h +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.h @@ -45,9 +45,9 @@ class FaissIVFFlatPass : public Pass { Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; - std::vector gpus; + std::vector gpus; }; using FaissIVFFlatPassPtr = std::shared_ptr; diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp index b63a0f7490..a99e861e03 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp @@ -32,7 +32,7 @@ FaissIVFSQ8HPass::Init() { server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { - threshold_ = std::numeric_limits::max(); + threshold_ = std::numeric_limits::max(); } s = config.GetGpuResourceConfigSearchResources(gpus); } diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h index fa830ff08a..0d2892809f 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h @@ -45,9 +45,9 @@ class FaissIVFSQ8HPass : public Pass { Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; - std::vector gpus; + std::vector gpus; }; using FaissIVFSQ8HPassPtr = std::shared_ptr; diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h index c5853283f7..e92ea2fe4e 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h @@ -45,9 +45,9 @@ class FaissIVFSQ8Pass : public Pass { Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; - std::vector gpus; + std::vector gpus; }; using FaissIVFSQ8PassPtr = std::shared_ptr; diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 2ff91bfe3a..f3efcff0cc 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -113,19 +113,19 @@ Config::ValidateConfig() { return s; } - int32_t db_archive_disk_threshold; + int64_t db_archive_disk_threshold; s = GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold); if (!s.ok()) { return s; } - int32_t db_archive_days_threshold; + int64_t db_archive_days_threshold; s = GetDBConfigArchiveDaysThreshold(db_archive_days_threshold); if (!s.ok()) { return s; } - int32_t db_insert_buffer_size; + int64_t db_insert_buffer_size; s = GetDBConfigInsertBufferSize(db_insert_buffer_size); if (!s.ok()) { return s; @@ -170,19 +170,19 @@ Config::ValidateConfig() { } /* engine config */ - int32_t engine_use_blas_threshold; + int64_t engine_use_blas_threshold; s = GetEngineConfigUseBlasThreshold(engine_use_blas_threshold); if (!s.ok()) { return s; } - int32_t engine_omp_thread_num; + int64_t engine_omp_thread_num; s = GetEngineConfigOmpThreadNum(engine_omp_thread_num); if (!s.ok()) { return s; } - int32_t engine_gpu_search_threshold; + int64_t engine_gpu_search_threshold; s = GetEngineConfigGpuSearchThreshold(engine_gpu_search_threshold); if (!s.ok()) { return s; @@ -208,13 +208,13 @@ Config::ValidateConfig() { return s; } - std::vector search_resources; + std::vector search_resources; s = GetGpuResourceConfigSearchResources(search_resources); if (!s.ok()) { return s; } - std::vector index_build_resources; + std::vector index_build_resources; s = GetGpuResourceConfigBuildIndexResources(index_build_resources); if (!s.ok()) { return s; @@ -485,7 +485,7 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) { ". Possible reason: db_config.insert_buffer_size is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - int64_t buffer_size = std::stoi(value) * GB; + int64_t buffer_size = std::stoll(value) * GB; if (buffer_size <= 0) { std::string msg = "Invalid insert buffer size: " + value + ". Possible reason: db_config.insert_buffer_size is not a positive integer."; @@ -540,7 +540,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - int64_t cpu_cache_capacity = std::stoi(value) * GB; + int64_t cpu_cache_capacity = std::stoll(value) * GB; if (cpu_cache_capacity <= 0) { std::string msg = "Invalid cpu cache capacity: " + value + ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; @@ -557,7 +557,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { std::cerr << "WARNING: cpu cache capacity value is too big" << std::endl; } - int32_t buffer_value; + int64_t buffer_value; Status s = GetDBConfigInsertBufferSize(buffer_value); if (!s.ok()) { return s; @@ -619,10 +619,10 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) { return Status(SERVER_INVALID_ARGUMENT, msg); } - int32_t omp_thread = std::stoi(value); - uint32_t sys_thread_cnt = 8; + int64_t omp_thread = std::stoll(value); + int64_t sys_thread_cnt = 8; CommonUtil::GetSystemAvailableThreads(sys_thread_cnt); - if (omp_thread > static_cast(sys_thread_cnt)) { + if (omp_thread > sys_thread_cnt) { std::string msg = "Invalid omp thread num: " + value + ". Possible reason: engine_config.omp_thread_num exceeds system cpu cores."; return Status(SERVER_INVALID_ARGUMENT, msg); @@ -657,14 +657,14 @@ Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) { ". Possible reason: gpu_resource_config.cache_capacity is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - uint64_t gpu_cache_capacity = std::stoi(value) * GB; - std::vector gpu_ids; + int64_t gpu_cache_capacity = std::stoll(value) * GB; + std::vector gpu_ids; Status s = GetGpuResourceConfigBuildIndexResources(gpu_ids); if (!s.ok()) { return s; } - for (int32_t gpu_id : gpu_ids) { + for (int64_t gpu_id : gpu_ids) { size_t gpu_memory; if (!ValidationUtil::GetGpuMemory(gpu_id, gpu_memory).ok()) { std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_id); @@ -855,37 +855,37 @@ Config::GetDBConfigBackendUrl(std::string& value) { } Status -Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { +Config::GetDBConfigArchiveDiskThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); Status s = CheckDBConfigArchiveDiskThreshold(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { +Config::GetDBConfigArchiveDaysThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); Status s = CheckDBConfigArchiveDaysThreshold(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetDBConfigInsertBufferSize(int32_t& value) { +Config::GetDBConfigInsertBufferSize(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); Status s = CheckDBConfigInsertBufferSize(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -927,7 +927,7 @@ Config::GetCacheConfigCpuCacheCapacity(int64_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -957,37 +957,37 @@ Config::GetCacheConfigCacheInsertData(bool& value) { } Status -Config::GetEngineConfigUseBlasThreshold(int32_t& value) { +Config::GetEngineConfigUseBlasThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); Status s = CheckEngineConfigUseBlasThreshold(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetEngineConfigOmpThreadNum(int32_t& value) { +Config::GetEngineConfigOmpThreadNum(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); Status s = CheckEngineConfigOmpThreadNum(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetEngineConfigGpuSearchThreshold(int32_t& value) { +Config::GetEngineConfigGpuSearchThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT); Status s = CheckEngineConfigGpuSearchThreshold(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -1020,7 +1020,7 @@ Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -1046,7 +1046,7 @@ Config::GetGpuResourceConfigCacheThreshold(float& value) { } Status -Config::GetGpuResourceConfigSearchResources(std::vector& value) { +Config::GetGpuResourceConfigSearchResources(std::vector& value) { bool gpu_resource_enable = false; Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { @@ -1065,13 +1065,13 @@ Config::GetGpuResourceConfigSearchResources(std::vector& value) { return s; } for (std::string& res : res_vec) { - value.push_back(std::stoi(res.substr(3))); + value.push_back(std::stoll(res.substr(3))); } return Status::OK(); } Status -Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { +Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { bool gpu_resource_enable = false; Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { @@ -1091,7 +1091,7 @@ Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { return s; } for (std::string& res : res_vec) { - value.push_back(std::stoi(res.substr(3))); + value.push_back(std::stoll(res.substr(3))); } return Status::OK(); } diff --git a/core/src/server/Config.h b/core/src/server/Config.h index a6e78671b3..0907080a6f 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -210,11 +210,11 @@ class Config { Status GetDBConfigBackendUrl(std::string& value); Status - GetDBConfigArchiveDiskThreshold(int32_t& value); + GetDBConfigArchiveDiskThreshold(int64_t& value); Status - GetDBConfigArchiveDaysThreshold(int32_t& value); + GetDBConfigArchiveDaysThreshold(int64_t& value); Status - GetDBConfigInsertBufferSize(int32_t& value); + GetDBConfigInsertBufferSize(int64_t& value); Status GetDBConfigPreloadTable(std::string& value); @@ -236,11 +236,11 @@ class Config { /* engine config */ Status - GetEngineConfigUseBlasThreshold(int32_t& value); + GetEngineConfigUseBlasThreshold(int64_t& value); Status - GetEngineConfigOmpThreadNum(int32_t& value); + GetEngineConfigOmpThreadNum(int64_t& value); Status - GetEngineConfigGpuSearchThreshold(int32_t& value); + GetEngineConfigGpuSearchThreshold(int64_t& value); /* gpu resource config */ Status @@ -250,9 +250,9 @@ class Config { Status GetGpuResourceConfigCacheThreshold(float& value); Status - GetGpuResourceConfigSearchResources(std::vector& value); + GetGpuResourceConfigSearchResources(std::vector& value); Status - GetGpuResourceConfigBuildIndexResources(std::vector& value); + GetGpuResourceConfigBuildIndexResources(std::vector& value); public: /* server config */ diff --git a/core/src/server/DBWrapper.cpp b/core/src/server/DBWrapper.cpp index 1ff914dc69..f7a43d671c 100644 --- a/core/src/server/DBWrapper.cpp +++ b/core/src/server/DBWrapper.cpp @@ -89,7 +89,7 @@ DBWrapper::StartService() { } // engine config - int32_t omp_thread; + int64_t omp_thread; s = config.GetEngineConfigOmpThreadNum(omp_thread); if (!s.ok()) { std::cerr << s.ToString() << std::endl; @@ -100,7 +100,7 @@ DBWrapper::StartService() { omp_set_num_threads(omp_thread); SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; } else { - uint32_t sys_thread_cnt = 8; + int64_t sys_thread_cnt = 8; if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { omp_thread = static_cast(ceil(sys_thread_cnt * 0.5)); omp_set_num_threads(omp_thread); @@ -108,7 +108,7 @@ DBWrapper::StartService() { } // init faiss global variable - int32_t use_blas_threshold; + int64_t use_blas_threshold; s = config.GetEngineConfigUseBlasThreshold(use_blas_threshold); if (!s.ok()) { std::cerr << s.ToString() << std::endl; @@ -119,7 +119,7 @@ DBWrapper::StartService() { // set archive config engine::ArchiveConf::CriteriaT criterial; - int32_t disk, days; + int64_t disk, days; s = config.GetDBConfigArchiveDiskThreshold(disk); if (!s.ok()) { std::cerr << s.ToString() << std::endl; diff --git a/core/src/utils/CommonUtil.cpp b/core/src/utils/CommonUtil.cpp index fbf3112aeb..26e43619fb 100644 --- a/core/src/utils/CommonUtil.cpp +++ b/core/src/utils/CommonUtil.cpp @@ -54,7 +54,7 @@ CommonUtil::GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem) { } bool -CommonUtil::GetSystemAvailableThreads(uint32_t& thread_count) { +CommonUtil::GetSystemAvailableThreads(int64_t& thread_count) { // threadCnt = std::thread::hardware_concurrency(); thread_count = sysconf(_SC_NPROCESSORS_CONF); thread_count *= THREAD_MULTIPLY_CPU; diff --git a/core/src/utils/CommonUtil.h b/core/src/utils/CommonUtil.h index 939bdd6d31..121196986a 100644 --- a/core/src/utils/CommonUtil.h +++ b/core/src/utils/CommonUtil.h @@ -30,7 +30,7 @@ class CommonUtil { static bool GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem); static bool - GetSystemAvailableThreads(uint32_t& thread_count); + GetSystemAvailableThreads(int64_t& thread_count); static bool IsFileExist(const std::string& path); diff --git a/core/src/wrapper/KnowhereResource.cpp b/core/src/wrapper/KnowhereResource.cpp index bf6675cafa..5a2296b16e 100644 --- a/core/src/wrapper/KnowhereResource.cpp +++ b/core/src/wrapper/KnowhereResource.cpp @@ -48,7 +48,7 @@ KnowhereResource::Initialize() { // get build index gpu resource server::Config& config = server::Config::GetInstance(); - std::vector build_index_gpus; + std::vector build_index_gpus; s = config.GetGpuResourceConfigBuildIndexResources(build_index_gpus); if (!s.ok()) return s; @@ -58,7 +58,7 @@ KnowhereResource::Initialize() { } // get search gpu resource - std::vector search_gpus; + std::vector search_gpus; s = config.GetGpuResourceConfigSearchResources(search_gpus); if (!s.ok()) return s; diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index e0fde62916..664a08d631 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -104,7 +104,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { milvus::server::Config& config = milvus::server::Config::GetInstance(); milvus::Status s; std::string str_val; - int32_t int32_val; int64_t int64_val; float float_val; bool bool_val; @@ -160,26 +159,26 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(s.ok()); ASSERT_TRUE(str_val == db_backend_url); - int32_t db_archive_disk_threshold = 100; + int64_t db_archive_disk_threshold = 100; s = config.SetDBConfigArchiveDiskThreshold(std::to_string(db_archive_disk_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetDBConfigArchiveDiskThreshold(int32_val); + s = config.GetDBConfigArchiveDiskThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == db_archive_disk_threshold); + ASSERT_TRUE(int64_val == db_archive_disk_threshold); - int32_t db_archive_days_threshold = 365; + int64_t db_archive_days_threshold = 365; s = config.SetDBConfigArchiveDaysThreshold(std::to_string(db_archive_days_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetDBConfigArchiveDaysThreshold(int32_val); + s = config.GetDBConfigArchiveDaysThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == db_archive_days_threshold); + ASSERT_TRUE(int64_val == db_archive_days_threshold); - int32_t db_insert_buffer_size = 2; + int64_t db_insert_buffer_size = 2; s = config.SetDBConfigInsertBufferSize(std::to_string(db_insert_buffer_size)); ASSERT_TRUE(s.ok()); - s = config.GetDBConfigInsertBufferSize(int32_val); + s = config.GetDBConfigInsertBufferSize(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == db_insert_buffer_size); + ASSERT_TRUE(int64_val == db_insert_buffer_size); /* metric config */ bool metric_enable_monitor = false; @@ -223,26 +222,26 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(bool_val == cache_insert_data); /* engine config */ - int32_t engine_use_blas_threshold = 50; + int64_t engine_use_blas_threshold = 50; s = config.SetEngineConfigUseBlasThreshold(std::to_string(engine_use_blas_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigUseBlasThreshold(int32_val); + s = config.GetEngineConfigUseBlasThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == engine_use_blas_threshold); + ASSERT_TRUE(int64_val == engine_use_blas_threshold); - int32_t engine_omp_thread_num = 8; + int64_t engine_omp_thread_num = 8; s = config.SetEngineConfigOmpThreadNum(std::to_string(engine_omp_thread_num)); ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigOmpThreadNum(int32_val); + s = config.GetEngineConfigOmpThreadNum(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == engine_omp_thread_num); + ASSERT_TRUE(int64_val == engine_omp_thread_num); - int32_t engine_gpu_search_threshold = 800; + int64_t engine_gpu_search_threshold = 800; s = config.SetEngineConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigGpuSearchThreshold(int32_val); + s = config.GetEngineConfigGpuSearchThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == engine_gpu_search_threshold); + ASSERT_TRUE(int64_val == engine_gpu_search_threshold); /* gpu resource config */ bool resource_enable_gpu = true; @@ -267,7 +266,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(float_val == gpu_cache_threshold); std::vector search_resources = {"gpu0"}; - std::vector search_res_vec; + std::vector search_res_vec; std::string search_res_str; milvus::server::StringHelpFunctions::MergeStringWithDelimeter( search_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, search_res_str); @@ -276,11 +275,11 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { s = config.GetGpuResourceConfigSearchResources(search_res_vec); ASSERT_TRUE(s.ok()); for (size_t i = 0; i < search_resources.size(); i++) { - ASSERT_TRUE(std::stoi(search_resources[i].substr(3)) == search_res_vec[i]); + ASSERT_TRUE(std::stoll(search_resources[i].substr(3)) == search_res_vec[i]); } std::vector build_index_resources = {"gpu0"}; - std::vector build_index_res_vec; + std::vector build_index_res_vec; std::string build_index_res_str; milvus::server::StringHelpFunctions::MergeStringWithDelimeter( build_index_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, build_index_res_str); @@ -289,7 +288,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { s = config.GetGpuResourceConfigBuildIndexResources(build_index_res_vec); ASSERT_TRUE(s.ok()); for (size_t i = 0; i < build_index_resources.size(); i++) { - ASSERT_TRUE(std::stoi(build_index_resources[i].substr(3)) == build_index_res_vec[i]); + ASSERT_TRUE(std::stoll(build_index_resources[i].substr(3)) == build_index_res_vec[i]); } #endif } diff --git a/core/unittest/server/test_util.cpp b/core/unittest/server/test_util.cpp index 68400f2454..e5884cac65 100644 --- a/core/unittest/server/test_util.cpp +++ b/core/unittest/server/test_util.cpp @@ -60,7 +60,7 @@ TEST(UtilTest, COMMON_TEST) { ASSERT_GT(total_mem, 0); ASSERT_GT(free_mem, 0); - uint32_t thread_cnt = 0; + int64_t thread_cnt = 0; milvus::server::CommonUtil::GetSystemAvailableThreads(thread_cnt); ASSERT_GT(thread_cnt, 0); From 82e6b2738b9b72af25b1495ff50a697cff2c5b1b Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 20 Nov 2019 14:35:10 +0800 Subject: [PATCH 21/21] #306 update changelog --- CHANGELOG.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42d5b8978f..7d88bc270e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,15 +22,16 @@ Please mark all change in change log and use the ticket from JIRA. - \#346 - Support build index with multiple gpu ## Improvement -- \#275 - Rename C++ SDK IndexType -- \#284 - Change C++ SDK to shared library +- \#255 - Add ivfsq8 test report detailed version - \#260 - C++ SDK README - \#266 - Rpc request source code refactor -- \#314 - add Find FAISS in CMake +- \#275 - Rename C++ SDK IndexType +- \#284 - Change C++ SDK to shared library +- \#306 - Use int64 for all config integer - \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue +- \#314 - add Find FAISS in CMake - \#322 - Add option to enable / disable prometheus - \#358 - Add more information in build.sh and install.md -- \#255 - Add ivfsq8 test report detailed version - \#404 - Add virtual method Init() in Pass abstract class - \#409 - Add a Fallback pass in optimizer