diff --git a/CHANGELOG.md b/CHANGELOG.md index 222ac5a8be..486e88c852 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,21 +10,35 @@ Please mark all change in change log and use the ticket from JIRA. - \#316 - Some files not merged after vectors added - \#327 - Search does not use GPU when index type is FLAT - \#340 - Test cases run failed on 0.6.0 +- \#353 - Rename config.h.in to version.h.in +- \#374 - sdk_simple return empty result +- \#377 - Create partition success if tag name only contains spaces +- \#397 - sdk_simple return incorrect result +- \#399 - Create partition should be failed if partition tag existed +- \#412 - Message returned is confused when partition created with null partition name +- \#416 - Drop the same partition success repeatally ## Feature - \#12 - Pure CPU version for Milvus - \#77 - Support table partition -- \#226 - Experimental shards middleware for Milvus - \#127 - Support new Index type IVFPQ +- \#226 - Experimental shards middleware for Milvus +- \#227 - Support new index types SPTAG-KDT and SPTAG-BKT +- \#346 - Support build index with multiple gpu ## Improvement +- \#255 - Add ivfsq8 test report detailed version +- \#260 - C++ SDK README +- \#266 - Rpc request source code refactor - \#275 - Rename C++ SDK IndexType - \#284 - Change C++ SDK to shared library -- \#260 - C++ SDK README -- \#314 - add Find FAISS in CMake +- \#306 - Use int64 for all config integer - \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue +- \#314 - add Find FAISS in CMake - \#322 - Add option to enable / disable prometheus - \#358 - Add more information in build.sh and install.md +- \#404 - Add virtual method Init() in Pass abstract class +- \#409 - Add a Fallback pass in optimizer ## Task diff --git a/README.md b/README.md index 311d0d6843..4a1151dbf8 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Milvus is the world's fastest similarity search engine for massive-scale feature For more detailed introduction of Milvus and its architecture, see [Milvus overview](https://www.milvus.io/docs/en/aboutmilvus/overview/). -Milvus provides stable [Python](https://github.com/milvus-io/pymilvus), [Java](https://github.com/milvus-io/milvus-sdk-java) and C++ APIs. +Milvus provides stable [Python](https://github.com/milvus-io/pymilvus), [Java](https://github.com/milvus-io/milvus-sdk-java) and [C++](https://github.com/milvus-io/milvus/tree/master/core/src/sdk) APIs. Keep up-to-date with newest releases and latest updates by reading Milvus [release notes](https://www.milvus.io/docs/en/release/v0.5.0/). diff --git a/README_CN.md b/README_CN.md index 5dad64af9b..8f4d4b05aa 100644 --- a/README_CN.md +++ b/README_CN.md @@ -20,7 +20,7 @@ Milvus 是一款开源的、针对海量特征向量的相似性搜索引擎。基于异构众核计算框架设计，成本更低，性能更好。在有限的计算资源下，十亿向量搜索仅毫秒响应。 -Milvus 提供稳定的 Python、Java 以及 C++ 的 API 接口。 +Milvus 提供稳定的 [Python](https://github.com/milvus-io/pymilvus)、[Java](https://github.com/milvus-io/milvus-sdk-java) 以及 [C++](https://github.com/milvus-io/milvus/tree/master/core/src/sdk) 的 API 接口。 通过 [版本发布说明](https://milvus.io/docs/zh-CN/release/v0.5.0/) 获取最新发行版本的 Milvus。 diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index f0562ada64..bf7a70d11a 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -2,7 +2,7 @@ String cron_timezone = "TZ=Asia/Shanghai" String cron_string = BRANCH_NAME == "master" ? "H 0 * * * " : "" -cron_string = BRANCH_NAME == "0.5.1" ? "H 1 * * * " : cron_string +cron_string = BRANCH_NAME == "0.6.0" ? "H 1 * * * " : cron_string pipeline { agent none @@ -50,7 +50,7 @@ pipeline { } stages { - stage("Run Build") { + stage("Run GPU Version Build") { agent { kubernetes { label "${BINRARY_VERSION}-build" @@ -60,7 +60,7 @@ pipeline { } stages { - stage('Build') { + stage('GPU Version Build') { steps { container('milvus-build-env') { script { @@ -69,7 +69,7 @@ pipeline { } } } - stage('Code Coverage') { + stage('GPU Version Code Coverage') { steps { container('milvus-build-env') { script { @@ -78,7 +78,7 @@ pipeline { } } } - stage('Upload Package') { + stage('Upload GPU Version Package') { steps { container('milvus-build-env') { script { @@ -90,7 +90,7 @@ pipeline { } } - stage("Publish docker images") { + stage("Publish GPU Version docker images") { agent { kubernetes { label "${BINRARY_VERSION}-publish" @@ -100,7 +100,7 @@ pipeline { } stages { - stage('Publish') { + stage('Publish GPU Version') { steps { container('publish-images'){ script { @@ -112,7 +112,7 @@ pipeline { } } - stage("Deploy to Development") { + stage("Deploy GPU Version to Development") { agent { kubernetes { label "${BINRARY_VERSION}-dev-test" @@ -122,7 +122,7 @@ pipeline { } stages { - stage("Deploy to Dev") { + stage("Deploy GPU Version to Dev") { steps { container('milvus-test-env') { script { @@ -132,7 +132,7 @@ pipeline { } } - stage("Dev Test") { + stage("GPU Version Dev Test") { steps { container('milvus-test-env') { script { @@ -147,7 +147,7 @@ pipeline { } } - stage ("Cleanup Dev") { + stage ("Cleanup GPU Version Dev") { steps { container('milvus-test-env') { script { @@ -180,7 +180,7 @@ pipeline { } stages { - stage("Run Build") { + stage("Run CPU Version Build") { agent { kubernetes { label "${BINRARY_VERSION}-build" @@ -190,7 +190,7 @@ pipeline { } stages { - stage('Build') { + stage('Build CPU Version') { steps { container('milvus-build-env') { script { @@ -199,7 +199,7 @@ pipeline { } } } - stage('Code Coverage') { + stage('CPU Version Code Coverage') { steps { container('milvus-build-env') { script { @@ -208,7 +208,7 @@ pipeline { } } } - stage('Upload Package') { + stage('Upload CPU Version Package') { steps { container('milvus-build-env') { script { @@ -220,7 +220,7 @@ pipeline { } } - stage("Publish docker images") { + stage("Publish CPU Version docker images") { agent { kubernetes { label "${BINRARY_VERSION}-publish" @@ -230,7 +230,7 @@ pipeline { } stages { - stage('Publish') { + stage('Publish CPU Version') { steps { container('publish-images'){ script { @@ -242,7 +242,7 @@ pipeline { } } - stage("Deploy to Development") { + stage("Deploy CPU Version to Development") { agent { kubernetes { label "${BINRARY_VERSION}-dev-test" @@ -252,7 +252,7 @@ pipeline { } stages { - stage("Deploy to Dev") { + stage("Deploy CPU Version to Dev") { steps { container('milvus-test-env') { script { @@ -262,7 +262,7 @@ pipeline { } } - stage("Dev Test") { + stage("CPU Version Dev Test") { steps { container('milvus-test-env') { script { @@ -277,7 +277,7 @@ pipeline { } } - stage ("Cleanup Dev") { + stage ("Cleanup CPU Version Dev") { steps { container('milvus-test-env') { script { diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index f1daaf22ec..7b479ff44a 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -3,11 +3,7 @@ sh 'helm repo update' dir ('milvus-helm') { checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) dir ("milvus") { - if ("${env.BINRARY_VERSION}" == "gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - } else { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/filebeat/values.yaml --namespace milvus ." - } + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/sqlite_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index d357badfd3..e27bc6c0a5 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -13,11 +13,7 @@ timeout(time: 90, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus") { - if ("${env.BINRARY_VERSION}" == "gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - } else { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - } + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } dir ("tests/milvus_python_test") { diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index c1de5907b0..08812ae01f 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -14,11 +14,7 @@ timeout(time: 60, unit: 'MINUTES') { // } // dir ("milvus-helm") { // dir ("milvus") { - // if ("${env.BINRARY_VERSION}" == "gpu") { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - // } else { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - // } + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." // } // } // dir ("tests/milvus_python_test") { diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 27962ccb54..eb5205ad26 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -16,6 +16,7 @@ BUILD_TYPE="Debug" BUILD_UNITTEST="OFF" INSTALL_PREFIX="/opt/milvus" FAISS_ROOT="" +CUSTOMIZATION="OFF" # default use origin faiss BUILD_COVERAGE="OFF" USE_JFROG_CACHE="OFF" RUN_CPPLINT="OFF" @@ -23,7 +24,7 @@ GPU_VERSION="OFF" WITH_MKL="OFF" CUDA_COMPILER=/usr/local/cuda/bin/nvcc -while getopts "o:t:b:f:gulcjmh" arg +while getopts "o:t:b:f:gxulcjmh" arg do case $arg in o) @@ -41,6 +42,9 @@ do g) GPU_VERSION="ON"; ;; + x) + CUSTOMIZATION="ON"; + ;; u) echo "Build and run unittest cases" ; BUILD_UNITTEST="ON"; @@ -66,6 +70,7 @@ parameter: -b: core code build directory -f: faiss root path -g: gpu version +-x: milvus customization (default: OFF) -u: building unit test options(default: OFF) -l: run cpplint, clang-format and clang-tidy(default: OFF) -c: code coverage(default: OFF) @@ -74,7 +79,7 @@ parameter: -h: help usage: -./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-u] [-l] [-c] [-j] [-m] [-h] +./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-g] [-x] [-u] [-l] [-c] [-j] [-m] [-h] " exit 0 ;; @@ -96,6 +101,7 @@ CMAKE_CMD="cmake \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DMILVUS_GPU_VERSION=${GPU_VERSION} \ +-DCUSTOMIZATION=${CUSTOMIZATION} \ -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ diff --git a/core/.gitignore b/core/.gitignore index 74e41dba6b..8db8df41db 100644 --- a/core/.gitignore +++ b/core/.gitignore @@ -9,3 +9,5 @@ output.info output_new.info server.info *.pyc +src/grpc/python_gen.h +src/grpc/python/ diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index f24935f788..671330c192 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -70,7 +70,7 @@ if (MILVUS_VERSION_MAJOR STREQUAL "" endif() message(STATUS "Build version = ${MILVUS_VERSION}") -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/config.h @ONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/version.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/version.h @ONLY) message(STATUS "Milvus version: " "${MILVUS_VERSION_MAJOR}.${MILVUS_VERSION_MINOR}.${MILVUS_VERSION_PATCH} " diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 6c95126390..41889f5cef 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -27,9 +27,7 @@ metric_config: port: 8080 # port prometheus uses to fetch metrics, must in range [1025, 65534] cache_config: - - cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer - cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] + cpu_cache_capacity: 16 # GB, size of CPU memory used for cache, must be a positive integer cache_insert_data: false # whether to load inserted data into cache, must be a boolean engine_config: @@ -37,7 +35,10 @@ engine_config: # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only -resource_config: - search_resources: # define the device used for search computation - - cpu - index_build_device: cpu # CPU used for building index +gpu_resource_config: + enable: false # whether to enable GPU resources + cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer + search_resources: # define the GPU devices used for search computation, must be in format gpux + - gpu0 + build_index_resources: # define the GPU devices used for index building, must be in format gpux + - gpu0 diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 154db5d134..531c633da7 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -27,10 +27,7 @@ metric_config: port: 8080 # port prometheus uses to fetch metrics, must in range [1025, 65534] cache_config: - cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer - cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] - gpu_cache_capacity: 4 # GB, GPU memory used for cache, must be a positive integer - gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] + cpu_cache_capacity: 16 # GB, size of CPU memory used for cache, must be a positive integer cache_insert_data: false # whether to load inserted data into cache, must be a boolean engine_config: @@ -38,8 +35,10 @@ engine_config: # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only -resource_config: - search_resources: # define the devices used for search computation, must be in format: cpu or gpux - - cpu +gpu_resource_config: + enable: true # whether to enable GPU resources + cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer + search_resources: # define the GPU devices used for search computation, must be in format gpux + - gpu0 + build_index_resources: # define the GPU devices used for index building, must be in format gpux - gpu0 - index_build_device: gpu0 # CPU / GPU used for building index, must be in format: cpu or gpux diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index aa79c084f1..14a75bb539 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -75,7 +75,13 @@ set(thirdparty_files ) aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl/request grpc_request_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_impl_files) +set(grpc_server_files + ${grpc_request_files} + ${grpc_impl_files} + ) + aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files) aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_files) diff --git a/core/src/cache/GpuCacheMgr.cpp b/core/src/cache/GpuCacheMgr.cpp index d862bc0393..72229527fa 100644 --- a/core/src/cache/GpuCacheMgr.cpp +++ b/core/src/cache/GpuCacheMgr.cpp @@ -37,7 +37,7 @@ GpuCacheMgr::GpuCacheMgr() { Status s; int64_t gpu_cache_cap; - s = config.GetCacheConfigGpuCacheCapacity(gpu_cache_cap); + s = config.GetGpuResourceConfigCacheCapacity(gpu_cache_cap); if (!s.ok()) { SERVER_LOG_ERROR << s.message(); } @@ -45,7 +45,7 @@ GpuCacheMgr::GpuCacheMgr() { cache_ = std::make_shared>(cap, 1UL << 32); float gpu_mem_threshold; - s = config.GetCacheConfigGpuCacheThreshold(gpu_mem_threshold); + s = config.GetGpuResourceConfigCacheThreshold(gpu_mem_threshold); if (!s.ok()) { SERVER_LOG_ERROR << s.message(); } diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 2559b3a46b..dd230ce0d1 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -84,12 +84,12 @@ DBImpl::Start() { return Status::OK(); } - ENGINE_LOG_TRACE << "DB service start"; + // ENGINE_LOG_TRACE << "DB service start"; shutting_down_.store(false, std::memory_order_release); // for distribute version, some nodes are read only if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) { - ENGINE_LOG_TRACE << "StartTimerTasks"; + // ENGINE_LOG_TRACE << "StartTimerTasks"; bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this); } @@ -114,7 +114,7 @@ DBImpl::Stop() { meta_ptr_->CleanUp(); } - ENGINE_LOG_TRACE << "DB service stop"; + // ENGINE_LOG_TRACE << "DB service stop"; return Status::OK(); } @@ -279,6 +279,11 @@ DBImpl::DropPartitionByTag(const std::string& table_id, const std::string& parti std::string partition_name; auto status = meta_ptr_->GetPartitionName(table_id, partition_tag, partition_name); + if (!status.ok()) { + ENGINE_LOG_ERROR << status.message(); + return status; + } + return DropPartition(partition_name); } @@ -553,7 +558,7 @@ DBImpl::StartMetricTask() { return; } - ENGINE_LOG_TRACE << "Start metric task"; + // ENGINE_LOG_TRACE << "Start metric task"; server::Metrics::GetInstance().KeepingAliveCounterIncrement(METRIC_ACTION_INTERVAL); int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); @@ -579,7 +584,7 @@ DBImpl::StartMetricTask() { server::Metrics::GetInstance().GPUTemperature(); server::Metrics::GetInstance().CPUTemperature(); - ENGINE_LOG_TRACE << "Metric task finished"; + // ENGINE_LOG_TRACE << "Metric task finished"; } Status @@ -751,7 +756,7 @@ DBImpl::BackgroundMergeFiles(const std::string& table_id) { void DBImpl::BackgroundCompaction(std::set table_ids) { - ENGINE_LOG_TRACE << "Background compaction thread start"; + // ENGINE_LOG_TRACE << " Background compaction thread start"; Status status; for (auto& table_id : table_ids) { @@ -774,7 +779,7 @@ DBImpl::BackgroundCompaction(std::set table_ids) { } meta_ptr_->CleanUpFilesWithTTL(ttl); - ENGINE_LOG_TRACE << "Background compaction thread exit"; + // ENGINE_LOG_TRACE << " Background compaction thread exit"; } void @@ -807,7 +812,7 @@ DBImpl::StartBuildIndexTask(bool force) { void DBImpl::BackgroundBuildIndex() { - ENGINE_LOG_TRACE << "Background build index thread start"; + // ENGINE_LOG_TRACE << "Background build index thread start"; std::unique_lock lock(build_index_mutex_); meta::TableFilesSchema to_index_files; @@ -830,7 +835,7 @@ DBImpl::BackgroundBuildIndex() { } } - ENGINE_LOG_TRACE << "Background build index thread exit"; + // ENGINE_LOG_TRACE << "Background build index thread exit"; } Status @@ -853,8 +858,12 @@ DBImpl::GetPartitionsByTags(const std::string& table_id, const std::vectorShowPartitions(table_id, partiton_array); for (auto& tag : partition_tags) { + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); for (auto& schema : partiton_array) { - if (server::StringHelpFunctions::IsRegexMatch(schema.partition_tag_, tag)) { + if (server::StringHelpFunctions::IsRegexMatch(schema.partition_tag_, valid_tag)) { partition_name_array.insert(schema.table_id_); } } diff --git a/core/src/db/Options.h b/core/src/db/Options.h index ebecb4de5a..91459a8d5f 100644 --- a/core/src/db/Options.h +++ b/core/src/db/Options.h @@ -33,7 +33,7 @@ static const char* ARCHIVE_CONF_DISK = "disk"; static const char* ARCHIVE_CONF_DAYS = "days"; struct ArchiveConf { - using CriteriaT = std::map; + using CriteriaT = std::map; explicit ArchiveConf(const std::string& type, const std::string& criterias = std::string()); diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index e2db7179be..ca307b90fc 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -152,7 +152,14 @@ ExecutionEngineImpl::HybridLoad() const { } const std::string key = location_ + ".quantizer"; - std::vector gpus = scheduler::get_gpu_pool(); + + server::Config& config = server::Config::GetInstance(); + std::vector gpus; + Status s = config.GetGpuResourceConfigSearchResources(gpus); + if (!s.ok()) { + ENGINE_LOG_ERROR << s.message(); + return; + } // cache hit { @@ -363,6 +370,7 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) { Status ExecutionEngineImpl::CopyToIndexFileToGpu(uint64_t device_id) { + gpu_num_ = device_id; auto to_index_data = std::make_shared(PhysicalSize()); cache::DataObjPtr obj = std::static_pointer_cast(to_index_data); milvus::cache::GpuCacheMgr::GetInstance(device_id)->InsertItem(location_, obj); @@ -586,12 +594,16 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { Status ExecutionEngineImpl::Init() { server::Config& config = server::Config::GetInstance(); - Status s = config.GetResourceConfigIndexBuildDevice(gpu_num_); - if (!s.ok()) { - return s; + std::vector gpu_ids; + Status s = config.GetGpuResourceConfigBuildIndexResources(gpu_ids); + for (auto id : gpu_ids) { + if (gpu_num_ == id) { + return Status::OK(); + } } - return Status::OK(); + std::string msg = "Invalid gpu_num"; + return Status(SERVER_INVALID_ARGUMENT, msg); } } // namespace engine diff --git a/core/src/db/engine/ExecutionEngineImpl.h b/core/src/db/engine/ExecutionEngineImpl.h index 7eb304426e..da0e7cfb64 100644 --- a/core/src/db/engine/ExecutionEngineImpl.h +++ b/core/src/db/engine/ExecutionEngineImpl.h @@ -122,8 +122,8 @@ class ExecutionEngineImpl : public ExecutionEngine { int64_t dim_; std::string location_; - int32_t nlist_ = 0; - int32_t gpu_num_ = 0; + int64_t nlist_ = 0; + int64_t gpu_num_ = 0; }; } // namespace engine diff --git a/core/src/db/meta/MetaTypes.h b/core/src/db/meta/MetaTypes.h index 28f35e76fc..d98b74be7d 100644 --- a/core/src/db/meta/MetaTypes.h +++ b/core/src/db/meta/MetaTypes.h @@ -19,7 +19,7 @@ #include "db/Constants.h" #include "db/engine/ExecutionEngine.h" -#include "src/config.h" +#include "src/version.h" #include #include diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index bf83447806..4406b87f7e 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -22,6 +22,7 @@ #include "metrics/Metrics.h" #include "utils/Exception.h" #include "utils/Log.h" +#include "utils/StringHelpFunctions.h" #include #include @@ -1162,17 +1163,23 @@ MySQLMetaImpl::CreatePartition(const std::string& table_id, const std::string& p // not allow create partition under partition if (!table_schema.owner_table_.empty()) { - return Status(DB_ERROR, "Nested partition is not allow"); + return Status(DB_ERROR, "Nested partition is not allowed"); + } + + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + + // not allow duplicated partition + std::string exist_partition; + GetPartitionName(table_id, valid_tag, exist_partition); + if (!exist_partition.empty()) { + return Status(DB_ERROR, "Duplicate partition is not allowed"); } if (partition_name == "") { - // not allow duplicated partition - std::string exist_partition; - GetPartitionName(table_id, tag, exist_partition); - if (!exist_partition.empty()) { - return Status(DB_ERROR, "Duplicated partition is not allow"); - } - + // generate unique partition name NextTableId(table_schema.table_id_); } else { table_schema.table_id_ = partition_name; @@ -1182,9 +1189,14 @@ MySQLMetaImpl::CreatePartition(const std::string& table_id, const std::string& p table_schema.flag_ = 0; table_schema.created_on_ = utils::GetMicroSecTimeStamp(); table_schema.owner_table_ = table_id; - table_schema.partition_tag_ = tag; + table_schema.partition_tag_ = valid_tag; - return CreateTable(table_schema); + status = CreateTable(table_schema); + if (status.code() == DB_ALREADY_EXIST) { + return Status(DB_ALREADY_EXIST, "Partition already exists"); + } + + return status; } Status @@ -1231,6 +1243,12 @@ MySQLMetaImpl::GetPartitionName(const std::string& table_id, const std::string& try { server::MetricCollector metric; mysqlpp::StoreQueryResult res; + + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + { mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); @@ -1240,7 +1258,7 @@ MySQLMetaImpl::GetPartitionName(const std::string& table_id, const std::string& mysqlpp::Query allPartitionsQuery = connectionPtr->query(); allPartitionsQuery << "SELECT table_id FROM " << META_TABLES << " WHERE owner_table = " << mysqlpp::quote - << table_id << " AND partition_tag = " << mysqlpp::quote << tag << " AND state <> " + << table_id << " AND partition_tag = " << mysqlpp::quote << valid_tag << " AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allPartitionsQuery.str(); @@ -1252,7 +1270,7 @@ MySQLMetaImpl::GetPartitionName(const std::string& table_id, const std::string& const mysqlpp::Row& resRow = res[0]; resRow["table_id"].to_string(partition_name); } else { - return Status(DB_NOT_FOUND, "Partition " + tag + " of table " + table_id + " not found"); + return Status(DB_NOT_FOUND, "Partition " + valid_tag + " of table " + table_id + " not found"); } } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN GET PARTITION NAME", e.what()); diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 22e953fe9d..12128c074d 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -22,6 +22,7 @@ #include "metrics/Metrics.h" #include "utils/Exception.h" #include "utils/Log.h" +#include "utils/StringHelpFunctions.h" #include #include @@ -757,17 +758,23 @@ SqliteMetaImpl::CreatePartition(const std::string& table_id, const std::string& // not allow create partition under partition if(!table_schema.owner_table_.empty()) { - return Status(DB_ERROR, "Nested partition is not allow"); + return Status(DB_ERROR, "Nested partition is not allowed"); + } + + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + + // not allow duplicated partition + std::string exist_partition; + GetPartitionName(table_id, valid_tag, exist_partition); + if(!exist_partition.empty()) { + return Status(DB_ERROR, "Duplicate partition is not allowed"); } if (partition_name == "") { - // not allow duplicated partition - std::string exist_partition; - GetPartitionName(table_id, tag, exist_partition); - if(!exist_partition.empty()) { - return Status(DB_ERROR, "Duplicated partition is not allow"); - } - + // generate unique partition name NextTableId(table_schema.table_id_); } else { table_schema.table_id_ = partition_name; @@ -777,9 +784,14 @@ SqliteMetaImpl::CreatePartition(const std::string& table_id, const std::string& table_schema.flag_ = 0; table_schema.created_on_ = utils::GetMicroSecTimeStamp(); table_schema.owner_table_ = table_id; - table_schema.partition_tag_ = tag; + table_schema.partition_tag_ = valid_tag; - return CreateTable(table_schema); + status = CreateTable(table_schema); + if (status.code() == DB_ALREADY_EXIST) { + return Status(DB_ALREADY_EXIST, "Partition already exists"); + } + + return status; } Status @@ -814,13 +826,18 @@ SqliteMetaImpl::GetPartitionName(const std::string& table_id, const std::string& try { server::MetricCollector metric; + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + auto name = ConnectorPtr->select(columns(&TableSchema::table_id_), where(c(&TableSchema::owner_table_) == table_id - and c(&TableSchema::partition_tag_) == tag)); + and c(&TableSchema::partition_tag_) == valid_tag)); if (name.size() > 0) { partition_name = std::get<0>(name[0]); } else { - return Status(DB_NOT_FOUND, "Table " + table_id + "'s partition " + tag + " not found"); + return Status(DB_NOT_FOUND, "Table " + table_id + "'s partition " + valid_tag + " not found"); } } catch (std::exception &e) { return HandleException("Encounter exception when get partition name", e.what()); diff --git a/core/src/index/CMakeLists.txt b/core/src/index/CMakeLists.txt index 4b5c1b1de3..53453d53aa 100644 --- a/core/src/index/CMakeLists.txt +++ b/core/src/index/CMakeLists.txt @@ -88,14 +88,14 @@ endif () include(ThirdPartyPackagesCore) if (CMAKE_BUILD_TYPE STREQUAL "Release") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt") if (KNOWHERE_GPU_VERSION) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3") endif () else () - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -g -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt") if (KNOWHERE_GPU_VERSION) - set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3 -g") endif () endif () diff --git a/core/src/index/knowhere/knowhere/common/Config.h b/core/src/index/knowhere/knowhere/common/Config.h index 6191ecb771..48c2de8b1b 100644 --- a/core/src/index/knowhere/knowhere/common/Config.h +++ b/core/src/index/knowhere/knowhere/common/Config.h @@ -18,6 +18,8 @@ #pragma once #include +#include +#include "Log.h" namespace knowhere { @@ -50,6 +52,18 @@ struct Cfg { CheckValid() { return true; } + + void + Dump() { + KNOWHERE_LOG_DEBUG << DumpImpl().str(); + } + + virtual std::stringstream + DumpImpl() { + std::stringstream ss; + ss << "dim: " << d << ", metric: " << int(metric_type) << ", gpuid: " << gpu_id << ", k: " << k; + return ss; + } }; using Config = std::shared_ptr; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp index c64baebb93..17a93fdcc7 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp @@ -19,330 +19,330 @@ #include #include +#include #include #include -#include #undef mkdir -#include "knowhere/index/vector_index/IndexSPTAG.h" -#include "knowhere/index/vector_index/helpers/Definitions.h" #include "knowhere/adapter/SptagAdapter.h" #include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexSPTAG.h" +#include "knowhere/index/vector_index/helpers/Definitions.h" #include "knowhere/index/vector_index/helpers/SPTAGParameterMgr.h" namespace knowhere { - CPUSPTAGRNG::CPUSPTAGRNG(const std::string& IndexType) { - if (IndexType == "KDT") { - index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::KDT, SPTAG::VectorValueType::Float); - index_ptr_->SetParameter("DistCalcMethod", "L2"); - index_type_ = SPTAG::IndexAlgoType::KDT; - } else { - index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::BKT, SPTAG::VectorValueType::Float); - index_ptr_->SetParameter("DistCalcMethod", "L2"); - index_type_ = SPTAG::IndexAlgoType::BKT; - } +CPUSPTAGRNG::CPUSPTAGRNG(const std::string& IndexType) { + if (IndexType == "KDT") { + index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::KDT, SPTAG::VectorValueType::Float); + index_ptr_->SetParameter("DistCalcMethod", "L2"); + index_type_ = SPTAG::IndexAlgoType::KDT; + } else { + index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::BKT, SPTAG::VectorValueType::Float); + index_ptr_->SetParameter("DistCalcMethod", "L2"); + index_type_ = SPTAG::IndexAlgoType::BKT; + } +} + +BinarySet +CPUSPTAGRNG::Serialize() { + std::string index_config; + std::vector index_blobs; + + std::shared_ptr> buffersize = index_ptr_->CalculateBufferSize(); + std::vector res(buffersize->size() + 1); + for (uint64_t i = 1; i < res.size(); i++) { + res[i] = new char[buffersize->at(i - 1)]; + auto ptr = &res[i][0]; + index_blobs.emplace_back(SPTAG::ByteArray((std::uint8_t*)ptr, buffersize->at(i - 1), false)); } - BinarySet - CPUSPTAGRNG::Serialize() { - std::string index_config; - std::vector index_blobs; + index_ptr_->SaveIndex(index_config, index_blobs); - std::shared_ptr> buffersize = index_ptr_->CalculateBufferSize(); - std::vector res(buffersize->size() + 1); - for (uint64_t i = 1; i < res.size(); i++) { - res[i] = new char[buffersize->at(i - 1)]; - auto ptr = &res[i][0]; - index_blobs.emplace_back(SPTAG::ByteArray((std::uint8_t*)ptr, buffersize->at(i - 1), false)); - } + size_t length = index_config.length(); + char* cstr = new char[length]; + snprintf(cstr, length, "%s", index_config.c_str()); - index_ptr_->SaveIndex(index_config, index_blobs); + BinarySet binary_set; + auto sample = std::make_shared(); + sample.reset(static_cast(index_blobs[0].Data())); + auto tree = std::make_shared(); + tree.reset(static_cast(index_blobs[1].Data())); + auto graph = std::make_shared(); + graph.reset(static_cast(index_blobs[2].Data())); + auto deleteid = std::make_shared(); + deleteid.reset(static_cast(index_blobs[3].Data())); + auto metadata1 = std::make_shared(); + metadata1.reset(static_cast(index_blobs[4].Data())); + auto metadata2 = std::make_shared(); + metadata2.reset(static_cast(index_blobs[5].Data())); + auto config = std::make_shared(); + config.reset(static_cast((void*)cstr)); - size_t length = index_config.length(); - char* cstr = new char[length]; - snprintf(cstr, length, "%s", index_config.c_str()); + binary_set.Append("samples", sample, index_blobs[0].Length()); + binary_set.Append("tree", tree, index_blobs[1].Length()); + binary_set.Append("deleteid", deleteid, index_blobs[3].Length()); + binary_set.Append("metadata1", metadata1, index_blobs[4].Length()); + binary_set.Append("metadata2", metadata2, index_blobs[5].Length()); + binary_set.Append("config", config, length); + binary_set.Append("graph", graph, index_blobs[2].Length()); - BinarySet binary_set; - auto sample = std::make_shared(); - sample.reset(static_cast(index_blobs[0].Data())); - auto tree = std::make_shared(); - tree.reset(static_cast(index_blobs[1].Data())); - auto graph = std::make_shared(); - graph.reset(static_cast(index_blobs[2].Data())); - auto deleteid = std::make_shared(); - deleteid.reset(static_cast(index_blobs[3].Data())); - auto metadata1 = std::make_shared(); - metadata1.reset(static_cast(index_blobs[4].Data())); - auto metadata2 = std::make_shared(); - metadata2.reset(static_cast(index_blobs[5].Data())); - auto config = std::make_shared(); - config.reset(static_cast((void*)cstr)); + // MemoryIOWriter writer; + // size_t len = 0; + // for (int i = 0; i < 6; ++i) { + // len = index_blobs[i].Length(); + // assert(len != 0); + // writer(&len, sizeof(size_t), 1); + // writer(index_blobs[i].Data(), len, 1); + // len = 0; + // } + // writer(&length, sizeof(size_t), 1); + // writer(cstr, length, 1); + // auto data = std::make_shared(); + // data.reset(writer.data_); + // BinarySet binary_set; + // binary_set.Append("sptag", data, writer.total); - binary_set.Append("samples", sample, index_blobs[0].Length()); - binary_set.Append("tree", tree, index_blobs[1].Length()); - binary_set.Append("deleteid", deleteid, index_blobs[3].Length()); - binary_set.Append("metadata1", metadata1, index_blobs[4].Length()); - binary_set.Append("metadata2", metadata2, index_blobs[5].Length()); - binary_set.Append("config", config, length); - binary_set.Append("graph", graph, index_blobs[2].Length()); + // MemoryIOWriter writer; + // size_t len = 0; + // for (int i = 0; i < 6; ++i) { + // if (i == 2) continue; + // len = index_blobs[i].Length(); + // assert(len != 0); + // writer(&len, sizeof(size_t), 1); + // writer(index_blobs[i].Data(), len, 1); + // len = 0; + // } + // writer(&length, sizeof(size_t), 1); + // writer(cstr, length, 1); + // auto data = std::make_shared(); + // data.reset(writer.data_); + // BinarySet binary_set; + // binary_set.Append("sptag", data, writer.total); + // auto graph = std::make_shared(); + // graph.reset(static_cast(index_blobs[2].Data())); + // binary_set.Append("graph", graph, index_blobs[2].Length()); -// MemoryIOWriter writer; -// size_t len = 0; -// for (int i = 0; i < 6; ++i) { -// len = index_blobs[i].Length(); -// assert(len != 0); -// writer(&len, sizeof(size_t), 1); -// writer(index_blobs[i].Data(), len, 1); -// len = 0; -// } -// writer(&length, sizeof(size_t), 1); -// writer(cstr, length, 1); -// auto data = std::make_shared(); -// data.reset(writer.data_); -// BinarySet binary_set; -// binary_set.Append("sptag", data, writer.total); + return binary_set; +} -// MemoryIOWriter writer; -// size_t len = 0; -// for (int i = 0; i < 6; ++i) { -// if (i == 2) continue; -// len = index_blobs[i].Length(); -// assert(len != 0); -// writer(&len, sizeof(size_t), 1); -// writer(index_blobs[i].Data(), len, 1); -// len = 0; -// } -// writer(&length, sizeof(size_t), 1); -// writer(cstr, length, 1); -// auto data = std::make_shared(); -// data.reset(writer.data_); -// BinarySet binary_set; -// binary_set.Append("sptag", data, writer.total); -// auto graph = std::make_shared(); -// graph.reset(static_cast(index_blobs[2].Data())); -// binary_set.Append("graph", graph, index_blobs[2].Length()); +void +CPUSPTAGRNG::Load(const BinarySet& binary_set) { + std::string index_config; + std::vector index_blobs; - return binary_set; - } + auto samples = binary_set.GetByName("samples"); + index_blobs.push_back(SPTAG::ByteArray(samples->data.get(), samples->size, false)); - void - CPUSPTAGRNG::Load(const BinarySet& binary_set) { - std::string index_config; - std::vector index_blobs; + auto tree = binary_set.GetByName("tree"); + index_blobs.push_back(SPTAG::ByteArray(tree->data.get(), tree->size, false)); - auto samples = binary_set.GetByName("samples"); - index_blobs.push_back(SPTAG::ByteArray(samples->data.get(), samples->size, false)); + auto graph = binary_set.GetByName("graph"); + index_blobs.push_back(SPTAG::ByteArray(graph->data.get(), graph->size, false)); - auto tree = binary_set.GetByName("tree"); - index_blobs.push_back(SPTAG::ByteArray(tree->data.get(), tree->size, false)); + auto deleteid = binary_set.GetByName("deleteid"); + index_blobs.push_back(SPTAG::ByteArray(deleteid->data.get(), deleteid->size, false)); - auto graph = binary_set.GetByName("graph"); - index_blobs.push_back(SPTAG::ByteArray(graph->data.get(), graph->size, false)); + auto metadata1 = binary_set.GetByName("metadata1"); + index_blobs.push_back(SPTAG::ByteArray(metadata1->data.get(), metadata1->size, false)); - auto deleteid = binary_set.GetByName("deleteid"); - index_blobs.push_back(SPTAG::ByteArray(deleteid->data.get(), deleteid->size, false)); + auto metadata2 = binary_set.GetByName("metadata2"); + index_blobs.push_back(SPTAG::ByteArray(metadata2->data.get(), metadata2->size, false)); - auto metadata1 = binary_set.GetByName("metadata1"); - index_blobs.push_back(SPTAG::ByteArray(metadata1->data.get(), metadata1->size, false)); + auto config = binary_set.GetByName("config"); + index_config = reinterpret_cast(config->data.get()); - auto metadata2 = binary_set.GetByName("metadata2"); - index_blobs.push_back(SPTAG::ByteArray(metadata2->data.get(), metadata2->size, false)); + // std::vector index_blobs; + // auto data = binary_set.GetByName("sptag"); + // MemoryIOReader reader; + // reader.total = data->size; + // reader.data_ = data->data.get(); + // size_t len = 0; + // for (int i = 0; i < 6; ++i) { + // reader(&len, sizeof(size_t), 1); + // assert(len != 0); + // auto binary = new uint8_t[len]; + // reader(binary, len, 1); + // index_blobs.emplace_back(SPTAG::ByteArray(binary, len, true)); + // len = 0; + // } + // reader(&len, sizeof(size_t), 1); + // assert(len != 0); + // auto config = new char[len]; + // reader(config, len, 1); + // std::string index_config = config; + // delete[] config; - auto config = binary_set.GetByName("config"); - index_config = reinterpret_cast(config->data.get()); - -// std::vector index_blobs; -// auto data = binary_set.GetByName("sptag"); -// MemoryIOReader reader; -// reader.total = data->size; -// reader.data_ = data->data.get(); -// size_t len = 0; -// for (int i = 0; i < 6; ++i) { -// reader(&len, sizeof(size_t), 1); -// assert(len != 0); -// auto binary = new uint8_t[len]; -// reader(binary, len, 1); -// index_blobs.emplace_back(SPTAG::ByteArray(binary, len, true)); -// len = 0; -// } -// reader(&len, sizeof(size_t), 1); -// assert(len != 0); -// auto config = new char[len]; -// reader(config, len, 1); -// std::string index_config = config; -// delete[] config; - -// std::vector index_blobs; -// auto data = binary_set.GetByName("sptag"); -// MemoryIOReader reader; -// reader.total = data->size; -// reader.data_ = data->data.get(); -// size_t len = 0; -// for (int i = 0; i < 6; ++i) { -// if (i == 2) { -// auto graph = binary_set.GetByName("graph"); -// index_blobs.emplace_back(SPTAG::ByteArray(graph->data.get(), graph->size, false)); -// continue; -// } -// reader(&len, sizeof(size_t), 1); -// assert(len != 0); -// auto binary = new uint8_t[len]; -// reader(binary, len, 1); -// index_blobs.emplace_back(SPTAG::ByteArray(binary, len, true)); -// len = 0; -// } -// reader(&len, sizeof(size_t), 1); -// assert(len != 0); -// auto config = new char[len]; -// reader(config, len, 1); -// std::string index_config = config; -// delete[] config; - index_ptr_->LoadIndex(index_config, index_blobs); - } + // std::vector index_blobs; + // auto data = binary_set.GetByName("sptag"); + // MemoryIOReader reader; + // reader.total = data->size; + // reader.data_ = data->data.get(); + // size_t len = 0; + // for (int i = 0; i < 6; ++i) { + // if (i == 2) { + // auto graph = binary_set.GetByName("graph"); + // index_blobs.emplace_back(SPTAG::ByteArray(graph->data.get(), graph->size, false)); + // continue; + // } + // reader(&len, sizeof(size_t), 1); + // assert(len != 0); + // auto binary = new uint8_t[len]; + // reader(binary, len, 1); + // index_blobs.emplace_back(SPTAG::ByteArray(binary, len, true)); + // len = 0; + // } + // reader(&len, sizeof(size_t), 1); + // assert(len != 0); + // auto config = new char[len]; + // reader(config, len, 1); + // std::string index_config = config; + // delete[] config; + index_ptr_->LoadIndex(index_config, index_blobs); +} // PreprocessorPtr // CPUKDTRNG::BuildPreprocessor(const DatasetPtr &dataset, const Config &config) { // return std::make_shared(); //} - IndexModelPtr - CPUSPTAGRNG::Train(const DatasetPtr& origin, const Config& train_config) { - SetParameters(train_config); - DatasetPtr dataset = origin->Clone(); +IndexModelPtr +CPUSPTAGRNG::Train(const DatasetPtr& origin, const Config& train_config) { + SetParameters(train_config); + DatasetPtr dataset = origin->Clone(); - // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine - // && preprocessor_) { - // preprocessor_->Preprocess(dataset); - //} + // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine + // && preprocessor_) { + // preprocessor_->Preprocess(dataset); + //} - auto vectorset = ConvertToVectorSet(dataset); - auto metaset = ConvertToMetadataSet(dataset); - index_ptr_->BuildIndex(vectorset, metaset); + auto vectorset = ConvertToVectorSet(dataset); + auto metaset = ConvertToMetadataSet(dataset); + index_ptr_->BuildIndex(vectorset, metaset); - // TODO: return IndexModelPtr - return nullptr; - } + // TODO: return IndexModelPtr + return nullptr; +} - void - CPUSPTAGRNG::Add(const DatasetPtr& origin, const Config& add_config) { - // SetParameters(add_config); - // DatasetPtr dataset = origin->Clone(); - // - // // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine - // // && preprocessor_) { - // // preprocessor_->Preprocess(dataset); - // //} - // - // auto vectorset = ConvertToVectorSet(dataset); - // auto metaset = ConvertToMetadataSet(dataset); - // index_ptr_->AddIndex(vectorset, metaset); - } +void +CPUSPTAGRNG::Add(const DatasetPtr& origin, const Config& add_config) { + // SetParameters(add_config); + // DatasetPtr dataset = origin->Clone(); + // + // // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine + // // && preprocessor_) { + // // preprocessor_->Preprocess(dataset); + // //} + // + // auto vectorset = ConvertToVectorSet(dataset); + // auto metaset = ConvertToMetadataSet(dataset); + // index_ptr_->AddIndex(vectorset, metaset); +} - void - CPUSPTAGRNG::SetParameters(const Config& config) { +void +CPUSPTAGRNG::SetParameters(const Config& config) { #define Assign(param_name, str_name) \ conf->param_name == INVALID_VALUE ? index_ptr_->SetParameter(str_name, std::to_string(build_cfg->param_name)) \ : index_ptr_->SetParameter(str_name, std::to_string(conf->param_name)) - if (index_type_ == SPTAG::IndexAlgoType::KDT) { - auto conf = std::dynamic_pointer_cast(config); - auto build_cfg = SPTAGParameterMgr::GetInstance().GetKDTParameters(); + if (index_type_ == SPTAG::IndexAlgoType::KDT) { + auto conf = std::dynamic_pointer_cast(config); + auto build_cfg = SPTAGParameterMgr::GetInstance().GetKDTParameters(); - Assign(kdtnumber, "KDTNumber"); - Assign(numtopdimensionkdtsplit, "NumTopDimensionKDTSplit"); - Assign(samples, "Samples"); - Assign(tptnumber, "TPTNumber"); - Assign(tptleafsize, "TPTLeafSize"); - Assign(numtopdimensiontptsplit, "NumTopDimensionTPTSplit"); - Assign(neighborhoodsize, "NeighborhoodSize"); - Assign(graphneighborhoodscale, "GraphNeighborhoodScale"); - Assign(graphcefscale, "GraphCEFScale"); - Assign(refineiterations, "RefineIterations"); - Assign(cef, "CEF"); - Assign(maxcheckforrefinegraph, "MaxCheckForRefineGraph"); - Assign(numofthreads, "NumberOfThreads"); - Assign(maxcheck, "MaxCheck"); - Assign(thresholdofnumberofcontinuousnobetterpropagation, "ThresholdOfNumberOfContinuousNoBetterPropagation"); - Assign(numberofinitialdynamicpivots, "NumberOfInitialDynamicPivots"); - Assign(numberofotherdynamicpivots, "NumberOfOtherDynamicPivots"); - } else { - auto conf = std::dynamic_pointer_cast(config); - auto build_cfg = SPTAGParameterMgr::GetInstance().GetBKTParameters(); + Assign(kdtnumber, "KDTNumber"); + Assign(numtopdimensionkdtsplit, "NumTopDimensionKDTSplit"); + Assign(samples, "Samples"); + Assign(tptnumber, "TPTNumber"); + Assign(tptleafsize, "TPTLeafSize"); + Assign(numtopdimensiontptsplit, "NumTopDimensionTPTSplit"); + Assign(neighborhoodsize, "NeighborhoodSize"); + Assign(graphneighborhoodscale, "GraphNeighborhoodScale"); + Assign(graphcefscale, "GraphCEFScale"); + Assign(refineiterations, "RefineIterations"); + Assign(cef, "CEF"); + Assign(maxcheckforrefinegraph, "MaxCheckForRefineGraph"); + Assign(numofthreads, "NumberOfThreads"); + Assign(maxcheck, "MaxCheck"); + Assign(thresholdofnumberofcontinuousnobetterpropagation, "ThresholdOfNumberOfContinuousNoBetterPropagation"); + Assign(numberofinitialdynamicpivots, "NumberOfInitialDynamicPivots"); + Assign(numberofotherdynamicpivots, "NumberOfOtherDynamicPivots"); + } else { + auto conf = std::dynamic_pointer_cast(config); + auto build_cfg = SPTAGParameterMgr::GetInstance().GetBKTParameters(); - Assign(bktnumber, "BKTNumber"); - Assign(bktkmeansk, "BKTKMeansK"); - Assign(bktleafsize, "BKTLeafSize"); - Assign(samples, "Samples"); - Assign(tptnumber, "TPTNumber"); - Assign(tptleafsize, "TPTLeafSize"); - Assign(numtopdimensiontptsplit, "NumTopDimensionTPTSplit"); - Assign(neighborhoodsize, "NeighborhoodSize"); - Assign(graphneighborhoodscale, "GraphNeighborhoodScale"); - Assign(graphcefscale, "GraphCEFScale"); - Assign(refineiterations, "RefineIterations"); - Assign(cef, "CEF"); - Assign(maxcheckforrefinegraph, "MaxCheckForRefineGraph"); - Assign(numofthreads, "NumberOfThreads"); - Assign(maxcheck, "MaxCheck"); - Assign(thresholdofnumberofcontinuousnobetterpropagation, "ThresholdOfNumberOfContinuousNoBetterPropagation"); - Assign(numberofinitialdynamicpivots, "NumberOfInitialDynamicPivots"); - Assign(numberofotherdynamicpivots, "NumberOfOtherDynamicPivots"); - } + Assign(bktnumber, "BKTNumber"); + Assign(bktkmeansk, "BKTKMeansK"); + Assign(bktleafsize, "BKTLeafSize"); + Assign(samples, "Samples"); + Assign(tptnumber, "TPTNumber"); + Assign(tptleafsize, "TPTLeafSize"); + Assign(numtopdimensiontptsplit, "NumTopDimensionTPTSplit"); + Assign(neighborhoodsize, "NeighborhoodSize"); + Assign(graphneighborhoodscale, "GraphNeighborhoodScale"); + Assign(graphcefscale, "GraphCEFScale"); + Assign(refineiterations, "RefineIterations"); + Assign(cef, "CEF"); + Assign(maxcheckforrefinegraph, "MaxCheckForRefineGraph"); + Assign(numofthreads, "NumberOfThreads"); + Assign(maxcheck, "MaxCheck"); + Assign(thresholdofnumberofcontinuousnobetterpropagation, "ThresholdOfNumberOfContinuousNoBetterPropagation"); + Assign(numberofinitialdynamicpivots, "NumberOfInitialDynamicPivots"); + Assign(numberofotherdynamicpivots, "NumberOfOtherDynamicPivots"); } +} - DatasetPtr - CPUSPTAGRNG::Search(const DatasetPtr& dataset, const Config& config) { - SetParameters(config); - auto tensor = dataset->tensor()[0]; - auto p = (float*)tensor->raw_mutable_data(); - for (auto i = 0; i < 10; ++i) { - for (auto j = 0; j < 10; ++j) { - std::cout << p[i * 10 + j] << " "; - } - std::cout << std::endl; +DatasetPtr +CPUSPTAGRNG::Search(const DatasetPtr& dataset, const Config& config) { + SetParameters(config); + auto tensor = dataset->tensor()[0]; + auto p = (float*)tensor->raw_mutable_data(); + for (auto i = 0; i < 10; ++i) { + for (auto j = 0; j < 10; ++j) { + std::cout << p[i * 10 + j] << " "; } - std::vector query_results = ConvertToQueryResult(dataset, config); + std::cout << std::endl; + } + std::vector query_results = ConvertToQueryResult(dataset, config); #pragma omp parallel for - for (auto i = 0; i < query_results.size(); ++i) { - auto target = (float*)query_results[i].GetTarget(); - std::cout << target[0] << ", " << target[1] << ", " << target[2] << std::endl; - index_ptr_->SearchIndex(query_results[i]); - } - - return ConvertToDataset(query_results); + for (auto i = 0; i < query_results.size(); ++i) { + auto target = (float*)query_results[i].GetTarget(); + std::cout << target[0] << ", " << target[1] << ", " << target[2] << std::endl; + index_ptr_->SearchIndex(query_results[i]); } - int64_t - CPUSPTAGRNG::Count() { - return index_ptr_->GetNumSamples(); - } + return ConvertToDataset(query_results); +} - int64_t - CPUSPTAGRNG::Dimension() { - return index_ptr_->GetFeatureDim(); - } +int64_t +CPUSPTAGRNG::Count() { + return index_ptr_->GetNumSamples(); +} - VectorIndexPtr - CPUSPTAGRNG::Clone() { - KNOWHERE_THROW_MSG("not support"); - } +int64_t +CPUSPTAGRNG::Dimension() { + return index_ptr_->GetFeatureDim(); +} - void - CPUSPTAGRNG::Seal() { - return; // do nothing - } +VectorIndexPtr +CPUSPTAGRNG::Clone() { + KNOWHERE_THROW_MSG("not support"); +} - BinarySet - CPUSPTAGRNGIndexModel::Serialize() { - // KNOWHERE_THROW_MSG("not support"); // not support - } +void +CPUSPTAGRNG::Seal() { + return; // do nothing +} - void - CPUSPTAGRNGIndexModel::Load(const BinarySet& binary) { - // KNOWHERE_THROW_MSG("not support"); // not support - } +BinarySet +CPUSPTAGRNGIndexModel::Serialize() { + // KNOWHERE_THROW_MSG("not support"); // not support +} + +void +CPUSPTAGRNGIndexModel::Load(const BinarySet& binary) { + // KNOWHERE_THROW_MSG("not support"); // not support +} } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h index c78807a3ee..01380ce943 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h @@ -28,66 +28,66 @@ namespace knowhere { - class CPUSPTAGRNG : public VectorIndex { - public: - explicit CPUSPTAGRNG(const std::string& IndexType); +class CPUSPTAGRNG : public VectorIndex { + public: + explicit CPUSPTAGRNG(const std::string& IndexType); - public: - BinarySet - Serialize() override; + public: + BinarySet + Serialize() override; - VectorIndexPtr - Clone() override; + VectorIndexPtr + Clone() override; - void - Load(const BinarySet& index_array) override; + void + Load(const BinarySet& index_array) override; - public: - // PreprocessorPtr - // BuildPreprocessor(const DatasetPtr &dataset, const Config &config) override; + public: + // PreprocessorPtr + // BuildPreprocessor(const DatasetPtr &dataset, const Config &config) override; - int64_t - Count() override; + int64_t + Count() override; - int64_t - Dimension() override; + int64_t + Dimension() override; - IndexModelPtr - Train(const DatasetPtr& dataset, const Config& config) override; + IndexModelPtr + Train(const DatasetPtr& dataset, const Config& config) override; - void - Add(const DatasetPtr& dataset, const Config& config) override; + void + Add(const DatasetPtr& dataset, const Config& config) override; - DatasetPtr - Search(const DatasetPtr& dataset, const Config& config) override; + DatasetPtr + Search(const DatasetPtr& dataset, const Config& config) override; - void - Seal() override; + void + Seal() override; - private: - void - SetParameters(const Config& config); + private: + void + SetParameters(const Config& config); - private: - PreprocessorPtr preprocessor_; - std::shared_ptr index_ptr_; - SPTAG::IndexAlgoType index_type_; - }; + private: + PreprocessorPtr preprocessor_; + std::shared_ptr index_ptr_; + SPTAG::IndexAlgoType index_type_; +}; - using CPUSPTAGRNGPtr = std::shared_ptr; +using CPUSPTAGRNGPtr = std::shared_ptr; - class CPUSPTAGRNGIndexModel : public IndexModel { - public: - BinarySet - Serialize() override; +class CPUSPTAGRNGIndexModel : public IndexModel { + public: + BinarySet + Serialize() override; - void - Load(const BinarySet& binary) override; + void + Load(const BinarySet& binary) override; - private: - std::shared_ptr index_; - }; + private: + std::shared_ptr index_; +}; - using CPUSPTAGRNGIndexModelPtr = std::shared_ptr; +using CPUSPTAGRNGIndexModelPtr = std::shared_ptr; } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp index 20f3388174..6e9ee55658 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp @@ -34,4 +34,26 @@ GetMetricType(METRICTYPE& type) { KNOWHERE_THROW_MSG("Metric type is invalid"); } +std::stringstream +IVFCfg::DumpImpl() { + auto ss = Cfg::DumpImpl(); + ss << ", nlist: " << nlist << ", nprobe: " << nprobe; + return ss; +} + +std::stringstream +IVFSQCfg::DumpImpl() { + auto ss = IVFCfg::DumpImpl(); + ss << ", nbits: " << nbits; + return ss; +} + +std::stringstream +NSGCfg::DumpImpl() { + auto ss = IVFCfg::DumpImpl(); + ss << ", knng: " << knng << ", search_length: " << search_length << ", out_degree: " << out_degree + << ", candidate: " << candidate_pool_size; + return ss; +} + } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h index 9bb3cac303..e30088ecdf 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h @@ -79,6 +79,9 @@ struct IVFCfg : public Cfg { IVFCfg() = default; + std::stringstream + DumpImpl() override; + bool CheckValid() override { return true; @@ -95,6 +98,9 @@ struct IVFSQCfg : public IVFCfg { : IVFCfg(dim, k, gpu_id, nlist, nprobe, type), nbits(nbits) { } + std::stringstream + DumpImpl() override; + IVFSQCfg() = default; bool @@ -145,6 +151,9 @@ struct NSGCfg : public IVFCfg { NSGCfg() = default; + std::stringstream + DumpImpl() override; + bool CheckValid() override { return true; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp index f6e1409abd..836f204c77 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp @@ -21,55 +21,55 @@ namespace knowhere { - const KDTConfig& - SPTAGParameterMgr::GetKDTParameters() { - return kdt_config_; - } +const KDTConfig& +SPTAGParameterMgr::GetKDTParameters() { + return kdt_config_; +} - const BKTConfig& - SPTAGParameterMgr::GetBKTParameters() { - return bkt_config_; - } +const BKTConfig& +SPTAGParameterMgr::GetBKTParameters() { + return bkt_config_; +} - SPTAGParameterMgr::SPTAGParameterMgr() { - kdt_config_ = std::make_shared(); - kdt_config_->kdtnumber = 1; - kdt_config_->numtopdimensionkdtsplit = 5; - kdt_config_->samples = 100; - kdt_config_->tptnumber = 1; - kdt_config_->tptleafsize = 2000; - kdt_config_->numtopdimensiontptsplit = 5; - kdt_config_->neighborhoodsize = 32; - kdt_config_->graphneighborhoodscale = 2; - kdt_config_->graphcefscale = 2; - kdt_config_->refineiterations = 0; - kdt_config_->cef = 1000; - kdt_config_->maxcheckforrefinegraph = 10000; - kdt_config_->numofthreads = 1; - kdt_config_->maxcheck = 8192; - kdt_config_->thresholdofnumberofcontinuousnobetterpropagation = 3; - kdt_config_->numberofinitialdynamicpivots = 50; - kdt_config_->numberofotherdynamicpivots = 4; +SPTAGParameterMgr::SPTAGParameterMgr() { + kdt_config_ = std::make_shared(); + kdt_config_->kdtnumber = 1; + kdt_config_->numtopdimensionkdtsplit = 5; + kdt_config_->samples = 100; + kdt_config_->tptnumber = 1; + kdt_config_->tptleafsize = 2000; + kdt_config_->numtopdimensiontptsplit = 5; + kdt_config_->neighborhoodsize = 32; + kdt_config_->graphneighborhoodscale = 2; + kdt_config_->graphcefscale = 2; + kdt_config_->refineiterations = 0; + kdt_config_->cef = 1000; + kdt_config_->maxcheckforrefinegraph = 10000; + kdt_config_->numofthreads = 1; + kdt_config_->maxcheck = 8192; + kdt_config_->thresholdofnumberofcontinuousnobetterpropagation = 3; + kdt_config_->numberofinitialdynamicpivots = 50; + kdt_config_->numberofotherdynamicpivots = 4; - bkt_config_ = std::make_shared(); - bkt_config_->bktnumber = 1; - bkt_config_->bktkmeansk = 32; - bkt_config_->bktleafsize = 8; - bkt_config_->samples = 100; - bkt_config_->tptnumber = 1; - bkt_config_->tptleafsize = 2000; - bkt_config_->numtopdimensiontptsplit = 5; - bkt_config_->neighborhoodsize = 32; - bkt_config_->graphneighborhoodscale = 2; - bkt_config_->graphcefscale = 2; - bkt_config_->refineiterations = 0; - bkt_config_->cef = 1000; - bkt_config_->maxcheckforrefinegraph = 10000; - bkt_config_->numofthreads = 1; - bkt_config_->maxcheck = 8192; - bkt_config_->thresholdofnumberofcontinuousnobetterpropagation = 3; - bkt_config_->numberofinitialdynamicpivots = 50; - bkt_config_->numberofotherdynamicpivots = 4; - } + bkt_config_ = std::make_shared(); + bkt_config_->bktnumber = 1; + bkt_config_->bktkmeansk = 32; + bkt_config_->bktleafsize = 8; + bkt_config_->samples = 100; + bkt_config_->tptnumber = 1; + bkt_config_->tptleafsize = 2000; + bkt_config_->numtopdimensiontptsplit = 5; + bkt_config_->neighborhoodsize = 32; + bkt_config_->graphneighborhoodscale = 2; + bkt_config_->graphcefscale = 2; + bkt_config_->refineiterations = 0; + bkt_config_->cef = 1000; + bkt_config_->maxcheckforrefinegraph = 10000; + bkt_config_->numofthreads = 1; + bkt_config_->maxcheck = 8192; + bkt_config_->thresholdofnumberofcontinuousnobetterpropagation = 3; + bkt_config_->numberofinitialdynamicpivots = 50; + bkt_config_->numberofotherdynamicpivots = 4; +} -} // namespace knowhere \ No newline at end of file +} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h index d215d9e275..6a6f7c48d1 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h @@ -27,35 +27,35 @@ namespace knowhere { - using KDTConfig = std::shared_ptr; - using BKTConfig = std::shared_ptr; +using KDTConfig = std::shared_ptr; +using BKTConfig = std::shared_ptr; - class SPTAGParameterMgr { - public: - const KDTConfig& - GetKDTParameters(); +class SPTAGParameterMgr { + public: + const KDTConfig& + GetKDTParameters(); - const BKTConfig& - GetBKTParameters(); + const BKTConfig& + GetBKTParameters(); - public: - static SPTAGParameterMgr& - GetInstance() { - static SPTAGParameterMgr instance; - return instance; - } + public: + static SPTAGParameterMgr& + GetInstance() { + static SPTAGParameterMgr instance; + return instance; + } - SPTAGParameterMgr(const SPTAGParameterMgr&) = delete; + SPTAGParameterMgr(const SPTAGParameterMgr&) = delete; - SPTAGParameterMgr& - operator=(const SPTAGParameterMgr&) = delete; + SPTAGParameterMgr& + operator=(const SPTAGParameterMgr&) = delete; - private: - SPTAGParameterMgr(); + private: + SPTAGParameterMgr(); - private: - KDTConfig kdt_config_; - BKTConfig bkt_config_; - }; + private: + KDTConfig kdt_config_; + BKTConfig bkt_config_; +}; } // namespace knowhere diff --git a/core/src/index/unittest/test_sptag.cpp b/core/src/index/unittest/test_sptag.cpp index a0898a619b..d472dab0d7 100644 --- a/core/src/index/unittest/test_sptag.cpp +++ b/core/src/index/unittest/test_sptag.cpp @@ -76,8 +76,8 @@ TEST_P(SPTAGTest, sptag_basic) { AssertAnns(result, nq, k); { - //auto ids = result->array()[0]; - //auto dists = result->array()[1]; + // auto ids = result->array()[0]; + // auto dists = result->array()[1]; auto ids = result->ids(); auto dists = result->dist(); @@ -85,8 +85,8 @@ TEST_P(SPTAGTest, sptag_basic) { std::stringstream ss_dist; for (auto i = 0; i < nq; i++) { for (auto j = 0; j < k; ++j) { - //ss_id << *ids->data()->GetValues(1, i * k + j) << " "; - //ss_dist << *dists->data()->GetValues(1, i * k + j) << " "; + // ss_id << *ids->data()->GetValues(1, i * k + j) << " "; + // ss_dist << *dists->data()->GetValues(1, i * k + j) << " "; ss_id << *((int64_t*)(ids) + i * k + j) << " "; ss_dist << *((float*)(dists) + i * k + j) << " "; } diff --git a/core/src/index/unittest/utils.cpp b/core/src/index/unittest/utils.cpp index 110d8fbf92..11dad4a8b9 100644 --- a/core/src/index/unittest/utils.cpp +++ b/core/src/index/unittest/utils.cpp @@ -153,7 +153,7 @@ void AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) { auto ids = result->ids(); for (auto i = 0; i < nq; i++) { - EXPECT_EQ(i, *((int64_t*)(ids) + i * k)); + EXPECT_EQ(i, *((int64_t*)(ids) + i * k)); // EXPECT_EQ(i, *(ids->data()->GetValues(1, i * k))); } } @@ -167,8 +167,8 @@ PrintResult(const knowhere::DatasetPtr& result, const int& nq, const int& k) { std::stringstream ss_dist; for (auto i = 0; i < nq; i++) { for (auto j = 0; j < k; ++j) { - //ss_id << *(ids->data()->GetValues(1, i * k + j)) << " "; - //ss_dist << *(dists->data()->GetValues(1, i * k + j)) << " "; + // ss_id << *(ids->data()->GetValues(1, i * k + j)) << " "; + // ss_dist << *(dists->data()->GetValues(1, i * k + j)) << " "; ss_id << *((int64_t*)(ids) + i * k + j) << " "; ss_dist << *((float*)(dists) + i * k + j) << " "; } diff --git a/core/src/main.cpp b/core/src/main.cpp index 401736c34f..b39ba87997 100644 --- a/core/src/main.cpp +++ b/core/src/main.cpp @@ -25,7 +25,7 @@ #include "easyloggingpp/easylogging++.h" #include "metrics/Metrics.h" #include "server/Server.h" -#include "src/config.h" +#include "src/version.h" #include "utils/CommonUtil.h" #include "utils/SignalUtil.h" diff --git a/core/src/scheduler/JobMgr.cpp b/core/src/scheduler/JobMgr.cpp index 794f6a0f37..76c07fe459 100644 --- a/core/src/scheduler/JobMgr.cpp +++ b/core/src/scheduler/JobMgr.cpp @@ -85,7 +85,7 @@ JobMgr::worker_function() { } for (auto& task : tasks) { - calculate_path(task); + calculate_path(res_mgr_, task); } // disk resources NEVER be empty. @@ -103,8 +103,8 @@ JobMgr::build_task(const JobPtr& job) { } void -JobMgr::calculate_path(const TaskPtr& task) { - if (task->type_ != TaskType::SearchTask) { +JobMgr::calculate_path(const ResourceMgrPtr& res_mgr, const TaskPtr& task) { + if (task->type_ != TaskType::SearchTask && task->type_ != TaskType::BuildIndexTask) { return; } @@ -114,9 +114,9 @@ JobMgr::calculate_path(const TaskPtr& task) { std::vector path; auto spec_label = std::static_pointer_cast(task->label()); - auto src = res_mgr_->GetDiskResources()[0]; + auto src = res_mgr->GetDiskResources()[0]; auto dest = spec_label->resource(); - ShortestPath(src.lock(), dest.lock(), res_mgr_, path); + ShortestPath(src.lock(), dest.lock(), res_mgr, path); task->path() = Path(path, path.size() - 1); } diff --git a/core/src/scheduler/JobMgr.h b/core/src/scheduler/JobMgr.h index fbd6c0ee45..af072614b5 100644 --- a/core/src/scheduler/JobMgr.h +++ b/core/src/scheduler/JobMgr.h @@ -59,8 +59,9 @@ class JobMgr : public interface::dumpable { static std::vector build_task(const JobPtr& job); - void - calculate_path(const TaskPtr& task); + public: + static void + calculate_path(const ResourceMgrPtr& res_mgr, const TaskPtr& task); private: bool running_ = false; diff --git a/core/src/scheduler/SchedInst.cpp b/core/src/scheduler/SchedInst.cpp index 61e0c09759..69d293f986 100644 --- a/core/src/scheduler/SchedInst.cpp +++ b/core/src/scheduler/SchedInst.cpp @@ -45,18 +45,6 @@ std::mutex BuildMgrInst::mutex_; void load_simple_config() { - server::Config& config = server::Config::GetInstance(); - std::string mode; - config.GetResourceConfigMode(mode); - std::vector pool; - config.GetResourceConfigSearchResources(pool); - - // get resources - auto gpu_ids = get_gpu_pool(); - - int32_t build_gpu_id; - config.GetResourceConfigIndexBuildDevice(build_gpu_id); - // create and connect ResMgrInst::GetInstance()->Add(ResourceFactory::Create("disk", "DISK", 0, true, false)); @@ -64,26 +52,46 @@ load_simple_config() { ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0, true, true)); ResMgrInst::GetInstance()->Connect("disk", "cpu", io); + // get resources +#ifdef MILVUS_GPU_VERSION + server::Config& config = server::Config::GetInstance(); + std::vector gpu_ids; + config.GetGpuResourceConfigSearchResources(gpu_ids); + std::vector build_gpu_ids; + config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids); auto pcie = Connection("pcie", 12000); - bool find_build_gpu_id = false; - for (auto& gpu_id : gpu_ids) { - ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); - ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie); - if (build_gpu_id == gpu_id) { - find_build_gpu_id = true; + + std::vector not_find_build_ids; + for (auto& build_id : build_gpu_ids) { + bool find_gpu_id = false; + for (auto& gpu_id : gpu_ids) { + if (gpu_id == build_id) { + find_gpu_id = true; + break; + } + } + if (not find_gpu_id) { + not_find_build_ids.emplace_back(build_id); } } - if (not find_build_gpu_id) { - ResMgrInst::GetInstance()->Add( - ResourceFactory::Create(std::to_string(build_gpu_id), "GPU", build_gpu_id, true, true)); - ResMgrInst::GetInstance()->Connect("cpu", std::to_string(build_gpu_id), pcie); + for (auto& gpu_id : gpu_ids) { + ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); + ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie); } + + for (auto& not_find_id : not_find_build_ids) { + ResMgrInst::GetInstance()->Add( + ResourceFactory::Create(std::to_string(not_find_id), "GPU", not_find_id, true, true)); + ResMgrInst::GetInstance()->Connect("cpu", std::to_string(not_find_id), pcie); + } +#endif } void StartSchedulerService() { load_simple_config(); + OptimizerInst::GetInstance()->Init(); ResMgrInst::GetInstance()->Start(); SchedInst::GetInstance()->Start(); JobMgrInst::GetInstance()->Start(); diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index a3048069f9..dc2d5ade35 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -21,10 +21,13 @@ #include "JobMgr.h" #include "ResourceMgr.h" #include "Scheduler.h" -#include "optimizer/HybridPass.h" -#include "optimizer/LargeSQ8HPass.h" -#include "optimizer/OnlyCPUPass.h" -#include "optimizer/OnlyGPUPass.h" +#include "Utils.h" +#include "optimizer/BuildIndexPass.h" +#include "optimizer/FaissFlatPass.h" +#include "optimizer/FaissIVFFlatPass.h" +#include "optimizer/FaissIVFSQ8HPass.h" +#include "optimizer/FaissIVFSQ8Pass.h" +#include "optimizer/FallbackPass.h" #include "optimizer/Optimizer.h" #include "server/Config.h" @@ -97,21 +100,15 @@ class OptimizerInst { if (instance == nullptr) { std::lock_guard lock(mutex_); if (instance == nullptr) { - server::Config& config = server::Config::GetInstance(); - std::vector search_resources; - bool has_cpu = false; - config.GetResourceConfigSearchResources(search_resources); - for (auto& resource : search_resources) { - if (resource == "cpu") { - has_cpu = true; - } - } - std::vector pass_list; - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared(has_cpu)); +#ifdef MILVUS_GPU_VERSION + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); +#endif + pass_list.push_back(std::make_shared()); instance = std::make_shared(pass_list); } } diff --git a/core/src/scheduler/Scheduler.cpp b/core/src/scheduler/Scheduler.cpp index 8d2d4406f8..68d7457aa9 100644 --- a/core/src/scheduler/Scheduler.cpp +++ b/core/src/scheduler/Scheduler.cpp @@ -108,10 +108,6 @@ Scheduler::OnLoadCompleted(const EventPtr& event) { auto task_table_type = load_completed_event->task_table_item_->task->label()->Type(); switch (task_table_type) { - case TaskLabelType::DEFAULT: { - Action::DefaultLabelTaskScheduler(res_mgr_, resource, load_completed_event); - break; - } case TaskLabelType::SPECIFIED_RESOURCE: { Action::SpecifiedResourceLabelTaskScheduler(res_mgr_, resource, load_completed_event); break; diff --git a/core/src/scheduler/TaskCreator.cpp b/core/src/scheduler/TaskCreator.cpp index 40cfa9aac6..30b76cc5bb 100644 --- a/core/src/scheduler/TaskCreator.cpp +++ b/core/src/scheduler/TaskCreator.cpp @@ -18,7 +18,6 @@ #include "scheduler/TaskCreator.h" #include "SchedInst.h" #include "tasklabel/BroadcastLabel.h" -#include "tasklabel/DefaultLabel.h" #include "tasklabel/SpecResLabel.h" namespace milvus { @@ -47,8 +46,7 @@ std::vector TaskCreator::Create(const SearchJobPtr& job) { std::vector tasks; for (auto& index_file : job->index_files()) { - auto label = std::make_shared(); - auto task = std::make_shared(index_file.second, label); + auto task = std::make_shared(index_file.second, nullptr); task->job_ = job; tasks.emplace_back(task); } @@ -70,12 +68,8 @@ TaskCreator::Create(const DeleteJobPtr& job) { std::vector TaskCreator::Create(const BuildIndexJobPtr& job) { std::vector tasks; - // TODO(yukun): remove "disk" hardcode here - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("disk"); - for (auto& to_index_file : job->to_index_files()) { - auto label = std::make_shared(std::weak_ptr(res_ptr)); - auto task = std::make_shared(to_index_file.second, label); + auto task = std::make_shared(to_index_file.second, nullptr); task->job_ = job; tasks.emplace_back(task); } diff --git a/core/src/scheduler/Utils.cpp b/core/src/scheduler/Utils.cpp index 2fd573e47a..527b9adf03 100644 --- a/core/src/scheduler/Utils.cpp +++ b/core/src/scheduler/Utils.cpp @@ -16,8 +16,6 @@ // under the License. #include "scheduler/Utils.h" -#include "server/Config.h" -#include "utils/Log.h" #ifdef MILVUS_GPU_VERSION #include @@ -46,42 +44,5 @@ get_num_gpu() { return n_devices; } -std::vector -get_gpu_pool() { - std::vector gpu_pool; - - server::Config& config = server::Config::GetInstance(); - std::vector pool; - Status s = config.GetResourceConfigSearchResources(pool); - if (!s.ok()) { - SERVER_LOG_ERROR << s.message(); - } - - std::set gpu_ids; - - for (auto& resource : pool) { - if (resource == "cpu") { - continue; - } else { - if (resource.length() < 4 || resource.substr(0, 3) != "gpu") { - // error - exit(-1); - } - auto gpu_id = std::stoi(resource.substr(3)); - if (gpu_id >= scheduler::get_num_gpu()) { - // error - exit(-1); - } - gpu_ids.insert(gpu_id); - } - } - - for (auto& gpu_id : gpu_ids) { - gpu_pool.push_back(gpu_id); - } - - return gpu_pool; -} - } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/Utils.h b/core/src/scheduler/Utils.h index 24876eeb96..bf88cf0345 100644 --- a/core/src/scheduler/Utils.h +++ b/core/src/scheduler/Utils.h @@ -27,8 +27,5 @@ get_current_timestamp(); uint64_t get_num_gpu(); -std::vector -get_gpu_pool(); - } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/action/Action.h b/core/src/scheduler/action/Action.h index f5f828cbf6..7391d287a8 100644 --- a/core/src/scheduler/action/Action.h +++ b/core/src/scheduler/action/Action.h @@ -36,10 +36,6 @@ class Action { static void PushTaskToResource(TaskTableItemPtr task_item, const ResourcePtr& dest); - static void - DefaultLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, - std::shared_ptr event); - static void SpecifiedResourceLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, std::shared_ptr event); diff --git a/core/src/scheduler/action/PushTaskToNeighbour.cpp b/core/src/scheduler/action/PushTaskToNeighbour.cpp index b8a4a1164b..f49f1d871f 100644 --- a/core/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/core/src/scheduler/action/PushTaskToNeighbour.cpp @@ -101,110 +101,46 @@ Action::PushTaskToResource(TaskTableItemPtr task_item, const ResourcePtr& dest) dest->task_table().Put(task_item->task, task_item); } -void -Action::DefaultLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, - std::shared_ptr event) { - if (not resource->HasExecutor() && event->task_table_item_->Move()) { - auto task_item = event->task_table_item_; - auto task = event->task_table_item_->task; - auto search_task = std::static_pointer_cast(task); - bool moved = false; - - // to support test task, REFACTOR - if (resource->type() == ResourceType::CPU) { - if (auto index_engine = search_task->index_engine_) { - auto location = index_engine->GetLocation(); - - for (auto i = 0; i < res_mgr->GetNumGpuResource(); ++i) { - auto index = milvus::cache::GpuCacheMgr::GetInstance(i)->GetIndex(location); - if (index != nullptr) { - moved = true; - auto dest_resource = res_mgr->GetResource(ResourceType::GPU, i); - PushTaskToResource(event->task_table_item_, dest_resource); - break; - } - } - } - } - - if (not moved) { - PushTaskToNeighbourRandomly(task_item, resource); - } - } -} - void Action::SpecifiedResourceLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, std::shared_ptr event) { auto task_item = event->task_table_item_; auto task = event->task_table_item_->task; - if (resource->type() == ResourceType::DISK) { - // step 1: calculate shortest path per resource, from disk to compute resource - auto compute_resources = res_mgr->GetComputeResources(); - std::vector> paths; - std::vector transport_costs; - for (auto& res : compute_resources) { - std::vector path; - uint64_t transport_cost = ShortestPath(resource, res, res_mgr, path); - transport_costs.push_back(transport_cost); - paths.emplace_back(path); - } - // if (task->job_.lock()->type() == JobType::SEARCH) { - // auto label = task->label(); - // auto spec_label = std::static_pointer_cast(label); - // if (spec_label->resource().lock()->type() == ResourceType::CPU) { - // std::vector spec_path; - // spec_path.push_back(spec_label->resource().lock()->name()); - // spec_path.push_back(resource->name()); - // task->path() = Path(spec_path, spec_path.size() - 1); - // } else { - // // step 2: select min cost, cost(resource) = avg_cost * task_to_do + transport_cost - // uint64_t min_cost = std::numeric_limits::max(); - // uint64_t min_cost_idx = 0; - // for (uint64_t i = 0; i < compute_resources.size(); ++i) { - // if (compute_resources[i]->TotalTasks() == 0) { - // min_cost_idx = i; - // break; - // } - // uint64_t cost = compute_resources[i]->TaskAvgCost() * - // compute_resources[i]->NumOfTaskToExec() + - // transport_costs[i]; - // if (min_cost > cost) { - // min_cost = cost; - // min_cost_idx = i; - // } - // } - // - // // step 3: set path in task - // Path task_path(paths[min_cost_idx], paths[min_cost_idx].size() - 1); - // task->path() = task_path; - // } - // - // } else - if (task->job_.lock()->type() == JobType::BUILD) { - // step2: Read device id in config - // get build index gpu resource - server::Config& config = server::Config::GetInstance(); - int32_t build_index_gpu; - Status stat = config.GetResourceConfigIndexBuildDevice(build_index_gpu); - - bool find_gpu_res = false; - if (res_mgr->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) { - for (uint64_t i = 0; i < compute_resources.size(); ++i) { - if (compute_resources[i]->name() == - res_mgr->GetResource(ResourceType::GPU, build_index_gpu)->name()) { - find_gpu_res = true; - Path task_path(paths[i], paths[i].size() - 1); - task->path() = task_path; - break; - } - } - } - if (not find_gpu_res) { - task->path() = Path(paths[0], paths[0].size() - 1); - } - } - } + // if (resource->type() == ResourceType::DISK) { + // // step 1: calculate shortest path per resource, from disk to compute resource + // auto compute_resources = res_mgr->GetComputeResources(); + // std::vector> paths; + // std::vector transport_costs; + // for (auto& res : compute_resources) { + // std::vector path; + // uint64_t transport_cost = ShortestPath(resource, res, res_mgr, path); + // transport_costs.push_back(transport_cost); + // paths.emplace_back(path); + // } + // if (task->job_.lock()->type() == JobType::BUILD) { + // // step2: Read device id in config + // // get build index gpu resource + // server::Config& config = server::Config::GetInstance(); + // int32_t build_index_gpu; + // Status stat = config.GetResourceConfigIndexBuildDevice(build_index_gpu); + // + // bool find_gpu_res = false; + // if (res_mgr->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) { + // for (uint64_t i = 0; i < compute_resources.size(); ++i) { + // if (compute_resources[i]->name() == + // res_mgr->GetResource(ResourceType::GPU, build_index_gpu)->name()) { + // find_gpu_res = true; + // Path task_path(paths[i], paths[i].size() - 1); + // task->path() = task_path; + // break; + // } + // } + // } + // if (not find_gpu_res) { + // task->path() = Path(paths[0], paths[0].size() - 1); + // } + // } + // } if (resource->name() == task->path().Last()) { resource->WakeupExecutor(); diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp similarity index 65% rename from core/src/scheduler/optimizer/OnlyCPUPass.cpp rename to core/src/scheduler/optimizer/BuildIndexPass.cpp index 238a91a82c..d535b9675f 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.cpp +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -15,32 +15,38 @@ // specific language governing permissions and limitations // under the License. -#include "scheduler/optimizer/OnlyCPUPass.h" +#include "scheduler/optimizer/BuildIndexPass.h" #include "scheduler/SchedInst.h" #include "scheduler/Utils.h" -#include "scheduler/task/SearchTask.h" #include "scheduler/tasklabel/SpecResLabel.h" namespace milvus { namespace scheduler { -bool -OnlyCPUPass::Run(const TaskPtr& task) { - if (task->Type() != TaskType::SearchTask) - return false; - auto search_task = std::static_pointer_cast(task); - if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8 && - search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFFLAT) { - return false; +void +BuildIndexPass::Init() { + server::Config& config = server::Config::GetInstance(); + std::vector build_resources; + Status s = config.GetGpuResourceConfigBuildIndexResources(build_resources); + if (!s.ok()) { + throw; } +} - auto gpu_id = get_gpu_pool(); - if (not gpu_id.empty()) +bool +BuildIndexPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::BuildIndexTask) return false; - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + if (build_gpu_ids_.empty()) + return false; + + ResourcePtr res_ptr; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, build_gpu_ids_[specified_gpu_id_]); auto label = std::make_shared(std::weak_ptr(res_ptr)); task->label() = label; + + specified_gpu_id_ = (specified_gpu_id_ + 1) % build_gpu_ids_.size(); return true; } diff --git a/core/src/scheduler/optimizer/OnlyGPUPass.h b/core/src/scheduler/optimizer/BuildIndexPass.h similarity index 86% rename from core/src/scheduler/optimizer/OnlyGPUPass.h rename to core/src/scheduler/optimizer/BuildIndexPass.h index 10d909d30e..4f7117fc4e 100644 --- a/core/src/scheduler/optimizer/OnlyGPUPass.h +++ b/core/src/scheduler/optimizer/BuildIndexPass.h @@ -32,20 +32,23 @@ namespace milvus { namespace scheduler { -class OnlyGPUPass : public Pass { +class BuildIndexPass : public Pass { public: - explicit OnlyGPUPass(bool has_cpu); + BuildIndexPass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; private: uint64_t specified_gpu_id_ = 0; - bool has_cpu_ = false; + std::vector build_gpu_ids_; }; -using OnlyGPUPassPtr = std::shared_ptr; +using BuildIndexPassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp b/core/src/scheduler/optimizer/FaissFlatPass.cpp similarity index 59% rename from core/src/scheduler/optimizer/LargeSQ8HPass.cpp rename to core/src/scheduler/optimizer/FaissFlatPass.cpp index b9784e3c0a..61ca1b9ec9 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/FaissFlatPass.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "scheduler/optimizer/LargeSQ8HPass.h" +#include "scheduler/optimizer/FaissFlatPass.h" #include "cache/GpuCacheMgr.h" #include "scheduler/SchedInst.h" #include "scheduler/Utils.h" @@ -27,57 +27,41 @@ namespace milvus { namespace scheduler { -LargeSQ8HPass::LargeSQ8HPass() { +void +FaissFlatPass::Init() { server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { threshold_ = std::numeric_limits::max(); } + s = config.GetGpuResourceConfigSearchResources(gpus); + if (!s.ok()) { + throw; + } } bool -LargeSQ8HPass::Run(const TaskPtr& task) { +FaissFlatPass::Run(const TaskPtr& task) { if (task->Type() != TaskType::SearchTask) { return false; } auto search_task = std::static_pointer_cast(task); - if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8H) { + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IDMAP) { return false; } auto search_job = std::static_pointer_cast(search_task->job_.lock()); - - // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu - + ResourcePtr res_ptr; if (search_job->nq() < threshold_) { - return false; + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); } - - std::vector gpus = scheduler::get_gpu_pool(); - // std::vector all_free_mem; - // for (auto& gpu : gpus) { - // auto cache = cache::GpuCacheMgr::GetInstance(gpu); - // auto free_mem = cache->CacheCapacity() - cache->CacheUsage(); - // all_free_mem.push_back(free_mem); - // } - // - // auto max_e = std::max_element(all_free_mem.begin(), all_free_mem.end()); - // auto best_index = std::distance(all_free_mem.begin(), max_e); - // auto best_device_id = gpus[best_index]; - auto best_device_id = count_ % gpus.size(); - count_++; - - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); - if (not res_ptr) { - SERVER_LOG_ERROR << "GpuResource " << best_device_id << " invalid."; - // TODO: throw critical error and exit - return false; - } - - auto label = std::make_shared(std::weak_ptr(res_ptr)); + auto label = std::make_shared(res_ptr); task->label() = label; - return true; } diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.h b/core/src/scheduler/optimizer/FaissFlatPass.h similarity index 83% rename from core/src/scheduler/optimizer/LargeSQ8HPass.h rename to core/src/scheduler/optimizer/FaissFlatPass.h index 9d135d413a..f219bebdf3 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.h +++ b/core/src/scheduler/optimizer/FaissFlatPass.h @@ -33,20 +33,24 @@ namespace milvus { namespace scheduler { -class LargeSQ8HPass : public Pass { +class FaissFlatPass : public Pass { public: - LargeSQ8HPass(); + FaissFlatPass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; + std::vector gpus; }; -using LargeSQ8HPassPtr = std::shared_ptr; +using FaissFlatPassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp new file mode 100644 index 0000000000..1f1efb374b --- /dev/null +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/FaissIVFFlatPass.h" +#include "cache/GpuCacheMgr.h" +#include "scheduler/SchedInst.h" +#include "scheduler/Utils.h" +#include "scheduler/task/SearchTask.h" +#include "scheduler/tasklabel/SpecResLabel.h" +#include "server/Config.h" +#include "utils/Log.h" + +namespace milvus { +namespace scheduler { + +void +FaissIVFFlatPass::Init() { + server::Config& config = server::Config::GetInstance(); + Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + if (!s.ok()) { + threshold_ = std::numeric_limits::max(); + } + s = config.GetGpuResourceConfigSearchResources(gpus); + if (!s.ok()) { + throw; + } +} + +bool +FaissIVFFlatPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) { + return false; + } + + auto search_task = std::static_pointer_cast(task); + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFFLAT) { + return false; + } + + auto search_job = std::static_pointer_cast(search_task->job_.lock()); + ResourcePtr res_ptr; + if (search_job->nq() < threshold_) { + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + } + auto label = std::make_shared(res_ptr); + task->label() = label; + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.h b/core/src/scheduler/optimizer/FaissIVFFlatPass.h similarity index 79% rename from core/src/scheduler/optimizer/OnlyCPUPass.h rename to core/src/scheduler/optimizer/FaissIVFFlatPass.h index 76b42e3766..2d15539014 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.h +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.h @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -32,16 +33,24 @@ namespace milvus { namespace scheduler { -class OnlyCPUPass : public Pass { +class FaissIVFFlatPass : public Pass { public: - OnlyCPUPass() = default; + FaissIVFFlatPass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; + + private: + int64_t threshold_ = std::numeric_limits::max(); + int64_t count_ = 0; + std::vector gpus; }; -using OnlyCPUPassPtr = std::shared_ptr; +using FaissIVFFlatPassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/OnlyGPUPass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp similarity index 55% rename from core/src/scheduler/optimizer/OnlyGPUPass.cpp rename to core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp index 2a72f9757e..a99e861e03 100644 --- a/core/src/scheduler/optimizer/OnlyGPUPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp @@ -15,39 +15,50 @@ // specific language governing permissions and limitations // under the License. -#include "scheduler/optimizer/OnlyGPUPass.h" +#include "scheduler/optimizer/FaissIVFSQ8HPass.h" +#include "cache/GpuCacheMgr.h" #include "scheduler/SchedInst.h" #include "scheduler/Utils.h" #include "scheduler/task/SearchTask.h" #include "scheduler/tasklabel/SpecResLabel.h" +#include "server/Config.h" +#include "utils/Log.h" namespace milvus { namespace scheduler { -OnlyGPUPass::OnlyGPUPass(bool has_cpu) : has_cpu_(has_cpu) { +void +FaissIVFSQ8HPass::Init() { + server::Config& config = server::Config::GetInstance(); + Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + if (!s.ok()) { + threshold_ = std::numeric_limits::max(); + } + s = config.GetGpuResourceConfigSearchResources(gpus); } bool -OnlyGPUPass::Run(const TaskPtr& task) { - if (task->Type() != TaskType::SearchTask || has_cpu_) - return false; - - auto search_task = std::static_pointer_cast(task); - if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8 && - search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFFLAT && - search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IDMAP) { +FaissIVFSQ8HPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) { return false; } - auto gpu_id = get_gpu_pool(); - if (gpu_id.empty()) + auto search_task = std::static_pointer_cast(task); + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8H) { return false; + } - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpu_id[specified_gpu_id_]); - auto label = std::make_shared(std::weak_ptr(res_ptr)); + auto search_job = std::static_pointer_cast(search_task->job_.lock()); + ResourcePtr res_ptr; + if (search_job->nq() < threshold_) { + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + } + auto label = std::make_shared(res_ptr); task->label() = label; - - specified_gpu_id_ = specified_gpu_id_++ % gpu_id.size(); return true; } diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h new file mode 100644 index 0000000000..0d2892809f --- /dev/null +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Pass.h" + +namespace milvus { +namespace scheduler { + +class FaissIVFSQ8HPass : public Pass { + public: + FaissIVFSQ8HPass() = default; + + public: + void + Init() override; + + bool + Run(const TaskPtr& task) override; + + private: + int64_t threshold_ = std::numeric_limits::max(); + int64_t count_ = 0; + std::vector gpus; +}; + +using FaissIVFSQ8HPassPtr = std::shared_ptr; + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp new file mode 100644 index 0000000000..30dd306b3b --- /dev/null +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/FaissIVFSQ8Pass.h" +#include "cache/GpuCacheMgr.h" +#include "scheduler/SchedInst.h" +#include "scheduler/Utils.h" +#include "scheduler/task/SearchTask.h" +#include "scheduler/tasklabel/SpecResLabel.h" +#include "server/Config.h" +#include "utils/Log.h" + +namespace milvus { +namespace scheduler { + +void +FaissIVFSQ8Pass::Init() { + server::Config& config = server::Config::GetInstance(); + Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + if (!s.ok()) { + threshold_ = std::numeric_limits::max(); + } + s = config.GetGpuResourceConfigSearchResources(gpus); + if (!s.ok()) { + throw; + } +} + +bool +FaissIVFSQ8Pass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) { + return false; + } + + auto search_task = std::static_pointer_cast(task); + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8) { + return false; + } + + auto search_job = std::static_pointer_cast(search_task->job_.lock()); + ResourcePtr res_ptr; + if (search_job->nq() < threshold_) { + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + } + auto label = std::make_shared(res_ptr); + task->label() = label; + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/HybridPass.h b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h similarity index 79% rename from core/src/scheduler/optimizer/HybridPass.h rename to core/src/scheduler/optimizer/FaissIVFSQ8Pass.h index 0d02a8bda9..e92ea2fe4e 100644 --- a/core/src/scheduler/optimizer/HybridPass.h +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -32,16 +33,24 @@ namespace milvus { namespace scheduler { -class HybridPass : public Pass { +class FaissIVFSQ8Pass : public Pass { public: - HybridPass() = default; + FaissIVFSQ8Pass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; + + private: + int64_t threshold_ = std::numeric_limits::max(); + int64_t count_ = 0; + std::vector gpus; }; -using HybridPassPtr = std::shared_ptr; +using FaissIVFSQ8PassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/FallbackPass.cpp b/core/src/scheduler/optimizer/FallbackPass.cpp new file mode 100644 index 0000000000..2e275ede4b --- /dev/null +++ b/core/src/scheduler/optimizer/FallbackPass.cpp @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/FallbackPass.h" +#include "scheduler/SchedInst.h" +#include "scheduler/tasklabel/SpecResLabel.h" + +namespace milvus { +namespace scheduler { + +void +FallbackPass::Init() { +} + +bool +FallbackPass::Run(const TaskPtr& task) { + auto task_type = task->Type(); + if (task_type != TaskType::SearchTask && task_type != TaskType::BuildIndexTask) { + return false; + } + // NEVER be empty + auto cpu = ResMgrInst::GetInstance()->GetCpuResources()[0]; + auto label = std::make_shared(cpu); + task->label() = label; + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/tasklabel/DefaultLabel.h b/core/src/scheduler/optimizer/FallbackPass.h similarity index 82% rename from core/src/scheduler/tasklabel/DefaultLabel.h rename to core/src/scheduler/optimizer/FallbackPass.h index c215743575..728740d53a 100644 --- a/core/src/scheduler/tasklabel/DefaultLabel.h +++ b/core/src/scheduler/optimizer/FallbackPass.h @@ -14,23 +14,27 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - #pragma once -#include "TaskLabel.h" - +#include #include +#include "Pass.h" + namespace milvus { namespace scheduler { -class DefaultLabel : public TaskLabel { +class FallbackPass : public Pass { public: - DefaultLabel() : TaskLabel(TaskLabelType::DEFAULT) { - } -}; + FallbackPass() = default; -using DefaultLabelPtr = std::shared_ptr; + public: + void + Init() override; + + bool + Run(const TaskPtr& task) override; +}; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/HybridPass.cpp b/core/src/scheduler/optimizer/HybridPass.cpp deleted file mode 100644 index d63fc2e819..0000000000 --- a/core/src/scheduler/optimizer/HybridPass.cpp +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "scheduler/optimizer/HybridPass.h" -#include "scheduler/SchedInst.h" -#include "scheduler/task/SearchTask.h" -#include "scheduler/tasklabel/SpecResLabel.h" - -namespace milvus { -namespace scheduler { - -bool -HybridPass::Run(const TaskPtr& task) { - // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu - if (task->Type() != TaskType::SearchTask) - return false; - auto search_task = std::static_pointer_cast(task); - if (search_task->file_->engine_type_ == (int)engine::EngineType::FAISS_IVFSQ8H) { - // TODO: remove "cpu" hardcode - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); - auto label = std::make_shared(std::weak_ptr(res_ptr)); - task->label() = label; - return true; - } - return false; -} - -} // namespace scheduler -} // namespace milvus diff --git a/core/src/scheduler/optimizer/Optimizer.cpp b/core/src/scheduler/optimizer/Optimizer.cpp index 46f24ea712..c5fa311a27 100644 --- a/core/src/scheduler/optimizer/Optimizer.cpp +++ b/core/src/scheduler/optimizer/Optimizer.cpp @@ -20,12 +20,12 @@ namespace milvus { namespace scheduler { -// void -// Optimizer::Init() { -// for (auto& pass : pass_list_) { -// pass->Init(); -// } -// } +void +Optimizer::Init() { + for (auto& pass : pass_list_) { + pass->Init(); + } +} bool Optimizer::Run(const TaskPtr& task) { diff --git a/core/src/scheduler/optimizer/Optimizer.h b/core/src/scheduler/optimizer/Optimizer.h index bfabbf7de3..68b519e115 100644 --- a/core/src/scheduler/optimizer/Optimizer.h +++ b/core/src/scheduler/optimizer/Optimizer.h @@ -38,8 +38,8 @@ class Optimizer { explicit Optimizer(std::vector pass_list) : pass_list_(std::move(pass_list)) { } - // void - // Init(); + void + Init(); bool Run(const TaskPtr& task); diff --git a/core/src/scheduler/optimizer/Pass.h b/core/src/scheduler/optimizer/Pass.h index 016b05e457..36a36a1df5 100644 --- a/core/src/scheduler/optimizer/Pass.h +++ b/core/src/scheduler/optimizer/Pass.h @@ -34,9 +34,8 @@ namespace scheduler { class Pass { public: - // virtual void - // Init() { - // } + virtual void + Init() = 0; virtual bool Run(const TaskPtr& task) = 0; diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index d8602c141e..f561fa947d 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -146,8 +146,7 @@ XBuildIndexTask::Execute() { status = meta_ptr->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; - std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough" - << std::endl; + ENGINE_LOG_ERROR << "Failed to build index, index file is too large or gpu memory is not enough"; build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = Status(DB_ERROR, msg); @@ -179,8 +178,8 @@ XBuildIndexTask::Execute() { status = meta_ptr->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; - std::cout << "ERROR: failed to persist index file: " << table_file.location_ - << ", possible out of disk space" << std::endl; + ENGINE_LOG_ERROR << "Failed to persist index file: " << table_file.location_ + << ", possible out of disk space"; build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = Status(DB_ERROR, msg); diff --git a/core/src/scheduler/tasklabel/TaskLabel.h b/core/src/scheduler/tasklabel/TaskLabel.h index d35ce409ff..33e6eb6e57 100644 --- a/core/src/scheduler/tasklabel/TaskLabel.h +++ b/core/src/scheduler/tasklabel/TaskLabel.h @@ -23,7 +23,6 @@ namespace milvus { namespace scheduler { enum class TaskLabelType { - DEFAULT, // means can be executed in any resource SPECIFIED_RESOURCE, // means must executing in special resource BROADCAST, // means all enable-executor resource must execute task }; diff --git a/core/src/sdk/examples/utils/Utils.cpp b/core/src/sdk/examples/utils/Utils.cpp index d9dd950710..da5e854e9b 100644 --- a/core/src/sdk/examples/utils/Utils.cpp +++ b/core/src/sdk/examples/utils/Utils.cpp @@ -167,7 +167,7 @@ Utils::PrintSearchResult(const std::vector index++; std::cout << "No." << index << " vector " << search_id << " top " << topk << " search result:" << std::endl; for (size_t j = 0; j < topk; j++) { - size_t idx = i * nq + j; + size_t idx = i * topk + j; std::cout << "\t" << topk_query_result.ids[idx] << "\t" << topk_query_result.distances[idx] << std::endl; } } diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 4ec94cfa98..4a9c319b4d 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -17,7 +17,7 @@ #include "sdk/grpc/ClientProxy.h" #include "grpc/gen-milvus/milvus.grpc.pb.h" -#include "src/config.h" +#include "src/version.h" #include #include @@ -204,9 +204,8 @@ ClientProxy::Insert(const std::string& table_name, const std::string& partition_ if (!id_array.empty()) { /* set user's ids */ auto row_ids = insert_param.mutable_row_id_array(); - row_ids->Reserve(static_cast(id_array.size())); + row_ids->Resize(static_cast(id_array.size()), -1); memcpy(row_ids->mutable_data(), id_array.data(), id_array.size() * sizeof(int64_t)); - client_ptr_->Insert(vector_ids, insert_param, status); } else { client_ptr_->Insert(vector_ids, insert_param, status); diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index f130e73a85..f3efcff0cc 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -113,19 +113,19 @@ Config::ValidateConfig() { return s; } - int32_t db_archive_disk_threshold; + int64_t db_archive_disk_threshold; s = GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold); if (!s.ok()) { return s; } - int32_t db_archive_days_threshold; + int64_t db_archive_days_threshold; s = GetDBConfigArchiveDaysThreshold(db_archive_days_threshold); if (!s.ok()) { return s; } - int32_t db_insert_buffer_size; + int64_t db_insert_buffer_size; s = GetDBConfigInsertBufferSize(db_insert_buffer_size); if (!s.ok()) { return s; @@ -163,20 +163,6 @@ Config::ValidateConfig() { return s; } -#ifdef MILVUS_GPU_VERSION - int64_t cache_gpu_cache_capacity; - s = GetCacheConfigGpuCacheCapacity(cache_gpu_cache_capacity); - if (!s.ok()) { - return s; - } - - float cache_gpu_cache_threshold; - s = GetCacheConfigGpuCacheThreshold(cache_gpu_cache_threshold); - if (!s.ok()) { - return s; - } -#endif - bool cache_insert_data; s = GetCacheConfigCacheInsertData(cache_insert_data); if (!s.ok()) { @@ -184,43 +170,57 @@ Config::ValidateConfig() { } /* engine config */ - int32_t engine_use_blas_threshold; + int64_t engine_use_blas_threshold; s = GetEngineConfigUseBlasThreshold(engine_use_blas_threshold); if (!s.ok()) { return s; } - int32_t engine_omp_thread_num; + int64_t engine_omp_thread_num; s = GetEngineConfigOmpThreadNum(engine_omp_thread_num); if (!s.ok()) { return s; } - int32_t engine_gpu_search_threshold; + int64_t engine_gpu_search_threshold; s = GetEngineConfigGpuSearchThreshold(engine_gpu_search_threshold); if (!s.ok()) { return s; } - /* resource config */ - std::string resource_mode; - s = GetResourceConfigMode(resource_mode); + /* gpu resource config */ +#ifdef MILVUS_GPU_VERSION + bool gpu_resource_enable; + s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - std::vector search_resources; - s = GetResourceConfigSearchResources(search_resources); + int64_t resource_cache_capacity; + s = GetGpuResourceConfigCacheCapacity(resource_cache_capacity); if (!s.ok()) { return s; } - int32_t resource_index_build_device; - s = GetResourceConfigIndexBuildDevice(resource_index_build_device); + float resource_cache_threshold; + s = GetGpuResourceConfigCacheThreshold(resource_cache_threshold); if (!s.ok()) { return s; } + std::vector search_resources; + s = GetGpuResourceConfigSearchResources(search_resources); + if (!s.ok()) { + return s; + } + + std::vector index_build_resources; + s = GetGpuResourceConfigBuildIndexResources(index_build_resources); + if (!s.ok()) { + return s; + } +#endif + return Status::OK(); } @@ -307,18 +307,6 @@ Config::ResetDefaultConfig() { return s; } -#ifdef MILVUS_GPU_VERSION - s = SetCacheConfigGpuCacheCapacity(CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetCacheConfigGpuCacheThreshold(CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); - if (!s.ok()) { - return s; - } -#endif - s = SetCacheConfigCacheInsertData(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); if (!s.ok()) { return s; @@ -340,22 +328,34 @@ Config::ResetDefaultConfig() { return s; } - /* resource config */ - s = SetResourceConfigMode(CONFIG_RESOURCE_MODE_DEFAULT); + /* gpu resource config */ +#ifdef MILVUS_GPU_VERSION + s = SetGpuResourceConfigEnable(CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); if (!s.ok()) { return s; } - s = SetResourceConfigSearchResources(CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT); + s = SetGpuResourceConfigCacheCapacity(CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT); if (!s.ok()) { return s; } - s = SetResourceConfigIndexBuildDevice(CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); + s = SetGpuResourceConfigCacheThreshold(CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT); if (!s.ok()) { return s; } + s = SetGpuResourceConfigSearchResources(CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT); + if (!s.ok()) { + return s; + } + + s = SetGpuResourceConfigBuildIndexResources(CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT); + if (!s.ok()) { + return s; + } +#endif + return Status::OK(); } @@ -377,7 +377,7 @@ Config::PrintAll() { PrintConfigSection(CONFIG_CACHE); PrintConfigSection(CONFIG_METRIC); PrintConfigSection(CONFIG_ENGINE); - PrintConfigSection(CONFIG_RESOURCE); + PrintConfigSection(CONFIG_GPU_RESOURCE); } //////////////////////////////////////////////////////////////////////////////// @@ -485,7 +485,7 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) { ". Possible reason: db_config.insert_buffer_size is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - int64_t buffer_size = std::stoi(value) * GB; + int64_t buffer_size = std::stoll(value) * GB; if (buffer_size <= 0) { std::string msg = "Invalid insert buffer size: " + value + ". Possible reason: db_config.insert_buffer_size is not a positive integer."; @@ -540,7 +540,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - int64_t cpu_cache_capacity = std::stoi(value) * GB; + int64_t cpu_cache_capacity = std::stoll(value) * GB; if (cpu_cache_capacity <= 0) { std::string msg = "Invalid cpu cache capacity: " + value + ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; @@ -557,7 +557,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { std::cerr << "WARNING: cpu cache capacity value is too big" << std::endl; } - int32_t buffer_value; + int64_t buffer_value; Status s = GetDBConfigInsertBufferSize(buffer_value); if (!s.ok()) { return s; @@ -591,52 +591,6 @@ Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) { return Status::OK(); } -Status -Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { - if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid gpu cache capacity: " + value + - ". Possible reason: cache_config.gpu_cache_capacity is not a positive integer."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else { - uint64_t gpu_cache_capacity = std::stoi(value) * GB; - int device_id; - Status s = GetResourceConfigIndexBuildDevice(device_id); - if (!s.ok()) { - return s; - } - - size_t gpu_memory; - if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) { - std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(device_id); - return Status(SERVER_UNEXPECTED_ERROR, msg); - } else if (gpu_cache_capacity >= gpu_memory) { - std::string msg = "Invalid gpu cache capacity: " + value + - ". Possible reason: cache_config.gpu_cache_capacity exceeds GPU memory."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { - std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; - } - } - return Status::OK(); -} - -Status -Config::CheckCacheConfigGpuCacheThreshold(const std::string& value) { - if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { - std::string msg = "Invalid gpu cache threshold: " + value + - ". Possible reason: cache_config.gpu_cache_threshold is not in range (0.0, 1.0]."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else { - float gpu_cache_threshold = std::stof(value); - if (gpu_cache_threshold <= 0.0 || gpu_cache_threshold >= 1.0) { - std::string msg = "Invalid gpu cache threshold: " + value + - ". Possible reason: cache_config.gpu_cache_threshold is not in range (0.0, 1.0]."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - } - return Status::OK(); -} - Status Config::CheckCacheConfigCacheInsertData(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { @@ -665,10 +619,10 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) { return Status(SERVER_INVALID_ARGUMENT, msg); } - int32_t omp_thread = std::stoi(value); - uint32_t sys_thread_cnt = 8; + int64_t omp_thread = std::stoll(value); + int64_t sys_thread_cnt = 8; CommonUtil::GetSystemAvailableThreads(sys_thread_cnt); - if (omp_thread > static_cast(sys_thread_cnt)) { + if (omp_thread > sys_thread_cnt) { std::string msg = "Invalid omp thread num: " + value + ". Possible reason: engine_config.omp_thread_num exceeds system cpu cores."; return Status(SERVER_INVALID_ARGUMENT, msg); @@ -687,56 +641,99 @@ Config::CheckEngineConfigGpuSearchThreshold(const std::string& value) { } Status -Config::CheckResourceConfigMode(const std::string& value) { - if (value != "simple") { - std::string msg = "Invalid resource mode: " + value + ". Possible reason: resource_config.mode is invalid."; +Config::CheckGpuResourceConfigEnable(const std::string& value) { + if (!ValidationUtil::ValidateStringIsBool(value).ok()) { + std::string msg = + "Invalid gpu resource config: " + value + ". Possible reason: gpu_resource_config.enable is not a boolean."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } Status -CheckResource(const std::string& value) { +Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + std::string msg = "Invalid gpu cache capacity: " + value + + ". Possible reason: gpu_resource_config.cache_capacity is not a positive integer."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else { + int64_t gpu_cache_capacity = std::stoll(value) * GB; + std::vector gpu_ids; + Status s = GetGpuResourceConfigBuildIndexResources(gpu_ids); + if (!s.ok()) { + return s; + } + + for (int64_t gpu_id : gpu_ids) { + size_t gpu_memory; + if (!ValidationUtil::GetGpuMemory(gpu_id, gpu_memory).ok()) { + std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_id); + return Status(SERVER_UNEXPECTED_ERROR, msg); + } else if (gpu_cache_capacity >= gpu_memory) { + std::string msg = "Invalid gpu cache capacity: " + value + + ". Possible reason: gpu_resource_config.cache_capacity exceeds GPU memory."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { + std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; + } + } + } + return Status::OK(); +} + +Status +Config::CheckGpuResourceConfigCacheThreshold(const std::string& value) { + if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { + std::string msg = "Invalid gpu cache threshold: " + value + + ". Possible reason: gpu_resource_config.cache_threshold is not in range (0.0, 1.0]."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else { + float gpu_cache_threshold = std::stof(value); + if (gpu_cache_threshold <= 0.0 || gpu_cache_threshold >= 1.0) { + std::string msg = "Invalid gpu cache threshold: " + value + + ". Possible reason: gpu_resource_config.cache_threshold is not in range (0.0, 1.0]."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + } + return Status::OK(); +} + +Status +CheckGpuResource(const std::string& value) { std::string s = value; std::transform(s.begin(), s.end(), s.begin(), ::tolower); -#ifdef MILVUS_CPU_VERSION - if (s != "cpu") { - return Status(SERVER_INVALID_ARGUMENT, "Invalid CPU resource: " + s); - } -#else - const std::regex pat("cpu|gpu(\\d+)"); + const std::regex pat("gpu(\\d+)"); std::smatch m; if (!std::regex_match(s, m, pat)) { - std::string msg = "Invalid search resource: " + value + - ". Possible reason: resource_config.search_resources is not in the format of cpux or gpux"; + std::string msg = "Invalid gpu resource: " + value + + ". Possible reason: gpu_resource_config is not in the format of cpux or gpux"; return Status(SERVER_INVALID_ARGUMENT, msg); } if (s.compare(0, 3, "gpu") == 0) { int32_t gpu_index = std::stoi(s.substr(3)); if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { - std::string msg = "Invalid search resource: " + value + - ". Possible reason: resource_config.search_resources does not match your hardware."; + std::string msg = "Invalid gpu resource: " + value + + ". Possible reason: gpu_resource_config does not match with the hardware."; return Status(SERVER_INVALID_ARGUMENT, msg); } } -#endif return Status::OK(); } Status -Config::CheckResourceConfigSearchResources(const std::vector& value) { +Config::CheckGpuResourceConfigSearchResources(const std::vector& value) { if (value.empty()) { std::string msg = - "Invalid search resource. " - "Possible reason: resource_config.search_resources is empty."; + "Invalid gpu search resource. " + "Possible reason: gpu_resource_config.search_resources is empty."; return Status(SERVER_INVALID_ARGUMENT, msg); } for (auto& resource : value) { - auto status = CheckResource(resource); + auto status = CheckGpuResource(resource); if (!status.ok()) { return Status(SERVER_INVALID_ARGUMENT, status.message()); } @@ -745,11 +742,21 @@ Config::CheckResourceConfigSearchResources(const std::vector& value } Status -Config::CheckResourceConfigIndexBuildDevice(const std::string& value) { - auto status = CheckResource(value); - if (!status.ok()) { - return Status(SERVER_INVALID_ARGUMENT, status.message()); +Config::CheckGpuResourceConfigBuildIndexResources(const std::vector& value) { + if (value.empty()) { + std::string msg = + "Invalid gpu build index resource. " + "Possible reason: gpu_resource_config.build_index_resources is empty."; + return Status(SERVER_INVALID_ARGUMENT, msg); } + + for (auto& resource : value) { + auto status = CheckGpuResource(resource); + if (!status.ok()) { + return Status(SERVER_INVALID_ARGUMENT, status.message()); + } + } + return Status::OK(); } @@ -848,40 +855,37 @@ Config::GetDBConfigBackendUrl(std::string& value) { } Status -Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { +Config::GetDBConfigArchiveDiskThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); Status s = CheckDBConfigArchiveDiskThreshold(str); if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { +Config::GetDBConfigArchiveDaysThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); Status s = CheckDBConfigArchiveDaysThreshold(str); if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetDBConfigInsertBufferSize(int32_t& value) { +Config::GetDBConfigInsertBufferSize(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); Status s = CheckDBConfigInsertBufferSize(str); if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -898,7 +902,6 @@ Config::GetMetricConfigEnableMonitor(bool& value) { if (!s.ok()) { return s; } - std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); return Status::OK(); @@ -924,8 +927,7 @@ Config::GetCacheConfigCpuCacheCapacity(int64_t& value) { if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -937,33 +939,6 @@ Config::GetCacheConfigCpuCacheThreshold(float& value) { if (!s.ok()) { return s; } - - value = std::stof(str); - return Status::OK(); -} - -Status -Config::GetCacheConfigGpuCacheCapacity(int64_t& value) { - std::string str = - GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); - Status s = CheckCacheConfigGpuCacheCapacity(str); - if (!s.ok()) { - return s; - } - - value = std::stoi(str); - return Status::OK(); -} - -Status -Config::GetCacheConfigGpuCacheThreshold(float& value) { - std::string str = - GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); - Status s = CheckCacheConfigGpuCacheThreshold(str); - if (!s.ok()) { - return s; - } - value = std::stof(str); return Status::OK(); } @@ -976,80 +951,148 @@ Config::GetCacheConfigCacheInsertData(bool& value) { if (!s.ok()) { return s; } - std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); return Status::OK(); } Status -Config::GetEngineConfigUseBlasThreshold(int32_t& value) { +Config::GetEngineConfigUseBlasThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); Status s = CheckEngineConfigUseBlasThreshold(str); if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetEngineConfigOmpThreadNum(int32_t& value) { +Config::GetEngineConfigOmpThreadNum(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); Status s = CheckEngineConfigOmpThreadNum(str); if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetEngineConfigGpuSearchThreshold(int32_t& value) { +Config::GetEngineConfigGpuSearchThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT); Status s = CheckEngineConfigGpuSearchThreshold(str); if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetResourceConfigMode(std::string& value) { - value = GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); - return CheckResourceConfigMode(value); -} - -Status -Config::GetResourceConfigSearchResources(std::vector& value) { - std::string str = - GetConfigSequenceStr(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, - CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT); - server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, value); - return CheckResourceConfigSearchResources(value); -} - -Status -Config::GetResourceConfigIndexBuildDevice(int32_t& value) { - std::string str = - GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); - Status s = CheckResourceConfigIndexBuildDevice(str); +Config::GetGpuResourceConfigEnable(bool& value) { + std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); + Status s = CheckGpuResourceConfigEnable(str); if (!s.ok()) { return s; } + std::transform(str.begin(), str.end(), str.begin(), ::tolower); + value = (str == "true" || str == "on" || str == "yes" || str == "1"); + return Status::OK(); +} - if (str == "cpu") { - value = CPU_DEVICE_ID; - } else { - value = std::stoi(str.substr(3)); +Status +Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); + if (!s.ok()) { + return s; } + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_CAPACITY, + CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT); + s = CheckGpuResourceConfigCacheCapacity(str); + if (!s.ok()) { + return s; + } + value = std::stoll(str); + return Status::OK(); +} +Status +Config::GetGpuResourceConfigCacheThreshold(float& value) { + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); + if (!s.ok()) { + return s; + } + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, + CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT); + s = CheckGpuResourceConfigCacheThreshold(str); + if (!s.ok()) { + return s; + } + value = std::stof(str); + return Status::OK(); +} + +Status +Config::GetGpuResourceConfigSearchResources(std::vector& value) { + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); + if (!s.ok()) { + return s; + } + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = GetConfigSequenceStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, + CONFIG_GPU_RESOURCE_DELIMITER, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT); + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + s = CheckGpuResourceConfigSearchResources(res_vec); + if (!s.ok()) { + return s; + } + for (std::string& res : res_vec) { + value.push_back(std::stoll(res.substr(3))); + } + return Status::OK(); +} + +Status +Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); + if (!s.ok()) { + return s; + } + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = + GetConfigSequenceStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, + CONFIG_GPU_RESOURCE_DELIMITER, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT); + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + s = CheckGpuResourceConfigBuildIndexResources(res_vec); + if (!s.ok()) { + return s; + } + for (std::string& res : res_vec) { + value.push_back(std::stoll(res.substr(3))); + } return Status::OK(); } @@ -1061,7 +1104,6 @@ Config::SetServerConfigAddress(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); return Status::OK(); } @@ -1072,7 +1114,6 @@ Config::SetServerConfigPort(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); return Status::OK(); } @@ -1083,7 +1124,6 @@ Config::SetServerConfigDeployMode(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); return Status::OK(); } @@ -1094,7 +1134,6 @@ Config::SetServerConfigTimeZone(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); return Status::OK(); } @@ -1106,7 +1145,6 @@ Config::SetDBConfigPrimaryPath(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); return Status::OK(); } @@ -1117,7 +1155,6 @@ Config::SetDBConfigSecondaryPath(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); return Status::OK(); } @@ -1128,7 +1165,6 @@ Config::SetDBConfigBackendUrl(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); return Status::OK(); } @@ -1139,7 +1175,6 @@ Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); return Status::OK(); } @@ -1150,7 +1185,6 @@ Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); return Status::OK(); } @@ -1161,7 +1195,6 @@ Config::SetDBConfigInsertBufferSize(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); return Status::OK(); } @@ -1173,7 +1206,6 @@ Config::SetMetricConfigEnableMonitor(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value); return Status::OK(); } @@ -1184,7 +1216,6 @@ Config::SetMetricConfigCollector(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); return Status::OK(); } @@ -1195,7 +1226,6 @@ Config::SetMetricConfigPrometheusPort(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); return Status::OK(); } @@ -1207,7 +1237,6 @@ Config::SetCacheConfigCpuCacheCapacity(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); return Status::OK(); } @@ -1218,40 +1247,16 @@ Config::SetCacheConfigCpuCacheThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); return Status::OK(); } -Status -Config::SetCacheConfigGpuCacheCapacity(const std::string& value) { - Status s = CheckCacheConfigGpuCacheCapacity(value); - if (!s.ok()) { - return s; - } - - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value); - return Status::OK(); -} - -Status -Config::SetCacheConfigGpuCacheThreshold(const std::string& value) { - Status s = CheckCacheConfigGpuCacheThreshold(value); - if (!s.ok()) { - return s; - } - - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value); - return Status::OK(); -} - Status Config::SetCacheConfigCacheInsertData(const std::string& value) { Status s = CheckCacheConfigCacheInsertData(value); if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); return Status::OK(); } @@ -1263,7 +1268,6 @@ Config::SetEngineConfigUseBlasThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); return Status::OK(); } @@ -1274,7 +1278,6 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); return Status::OK(); } @@ -1285,47 +1288,64 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value); return Status::OK(); } -/* resource config */ +/* gpu resource config */ Status -Config::SetResourceConfigMode(const std::string& value) { - Status s = CheckResourceConfigMode(value); +Config::SetGpuResourceConfigEnable(const std::string& value) { + Status s = CheckGpuResourceConfigEnable(value); if (!s.ok()) { return s; } - - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, value); return Status::OK(); } Status -Config::SetResourceConfigSearchResources(const std::string& value) { +Config::SetGpuResourceConfigCacheCapacity(const std::string& value) { + Status s = CheckGpuResourceConfigCacheCapacity(value); + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_CAPACITY, value); + return Status::OK(); +} + +Status +Config::SetGpuResourceConfigCacheThreshold(const std::string& value) { + Status s = CheckGpuResourceConfigCacheThreshold(value); + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, value); + return Status::OK(); +} + +Status +Config::SetGpuResourceConfigSearchResources(const std::string& value) { std::vector res_vec; - server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_vec); - - Status s = CheckResourceConfigSearchResources(res_vec); + server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + Status s = CheckGpuResourceConfigSearchResources(res_vec); if (!s.ok()) { return s; } - - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, value); + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, value); return Status::OK(); } Status -Config::SetResourceConfigIndexBuildDevice(const std::string& value) { - Status s = CheckResourceConfigIndexBuildDevice(value); +Config::SetGpuResourceConfigBuildIndexResources(const std::string& value) { + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + Status s = CheckGpuResourceConfigBuildIndexResources(res_vec); if (!s.ok()) { return s; } - - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value); + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, value); return Status::OK(); -} +} // namespace server } // namespace server } // namespace milvus diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 3ab0cd8053..0907080a6f 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -59,12 +59,8 @@ static const char* CONFIG_DB_PRELOAD_TABLE = "preload_table"; static const char* CONFIG_CACHE = "cache_config"; static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY = "cpu_cache_capacity"; static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT = "16"; -static const char* CONFIG_CACHE_GPU_CACHE_CAPACITY = "gpu_cache_capacity"; -static const char* CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT = "4"; -static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_mem_threshold"; +static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_cache_threshold"; static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT = "0.85"; -static const char* CONFIG_CACHE_GPU_CACHE_THRESHOLD = "gpu_mem_threshold"; -static const char* CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT = "0.85"; static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; @@ -87,26 +83,23 @@ static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; static const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD = "gpu_search_threshold"; static const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT = "1000"; -/* resource config */ -static const char* CONFIG_RESOURCE = "resource_config"; -static const char* CONFIG_RESOURCE_MODE = "mode"; -static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources"; -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER = ","; - -#ifdef MILVUS_CPU_VERSION -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu"; +/* gpu resource config */ +static const char* CONFIG_GPU_RESOURCE = "gpu_resource_config"; +static const char* CONFIG_GPU_RESOURCE_ENABLE = "enable"; +#ifdef MILVUS_GPU_VERSION +static const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "true"; #else -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu,gpu0"; +static const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "false"; #endif - -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device"; -#ifdef MILVUS_CPU_VERSION -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "cpu"; -#else -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0"; -#endif -const int32_t CPU_DEVICE_ID = -1; +static const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY = "cache_capacity"; +static const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT = "4"; +static const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD = "cache_threshold"; +static const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT = "0.85"; +static const char* CONFIG_GPU_RESOURCE_DELIMITER = ","; +static const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES = "search_resources"; +static const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT = "gpu0"; +static const char* CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES = "build_index_resources"; +static const char* CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT = "gpu0"; class Config { public: @@ -170,10 +163,6 @@ class Config { Status CheckCacheConfigCpuCacheThreshold(const std::string& value); Status - CheckCacheConfigGpuCacheCapacity(const std::string& value); - Status - CheckCacheConfigGpuCacheThreshold(const std::string& value); - Status CheckCacheConfigCacheInsertData(const std::string& value); /* engine config */ @@ -184,13 +173,17 @@ class Config { Status CheckEngineConfigGpuSearchThreshold(const std::string& value); - /* resource config */ + /* gpu resource config */ Status - CheckResourceConfigMode(const std::string& value); + CheckGpuResourceConfigEnable(const std::string& value); Status - CheckResourceConfigSearchResources(const std::vector& value); + CheckGpuResourceConfigCacheCapacity(const std::string& value); Status - CheckResourceConfigIndexBuildDevice(const std::string& value); + CheckGpuResourceConfigCacheThreshold(const std::string& value); + Status + CheckGpuResourceConfigSearchResources(const std::vector& value); + Status + CheckGpuResourceConfigBuildIndexResources(const std::vector& value); std::string GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = ""); @@ -217,11 +210,11 @@ class Config { Status GetDBConfigBackendUrl(std::string& value); Status - GetDBConfigArchiveDiskThreshold(int32_t& value); + GetDBConfigArchiveDiskThreshold(int64_t& value); Status - GetDBConfigArchiveDaysThreshold(int32_t& value); + GetDBConfigArchiveDaysThreshold(int64_t& value); Status - GetDBConfigInsertBufferSize(int32_t& value); + GetDBConfigInsertBufferSize(int64_t& value); Status GetDBConfigPreloadTable(std::string& value); @@ -239,27 +232,27 @@ class Config { Status GetCacheConfigCpuCacheThreshold(float& value); Status - GetCacheConfigGpuCacheCapacity(int64_t& value); - Status - GetCacheConfigGpuCacheThreshold(float& value); - Status GetCacheConfigCacheInsertData(bool& value); /* engine config */ Status - GetEngineConfigUseBlasThreshold(int32_t& value); + GetEngineConfigUseBlasThreshold(int64_t& value); Status - GetEngineConfigOmpThreadNum(int32_t& value); + GetEngineConfigOmpThreadNum(int64_t& value); Status - GetEngineConfigGpuSearchThreshold(int32_t& value); + GetEngineConfigGpuSearchThreshold(int64_t& value); - /* resource config */ + /* gpu resource config */ Status - GetResourceConfigMode(std::string& value); + GetGpuResourceConfigEnable(bool& value); Status - GetResourceConfigSearchResources(std::vector& value); + GetGpuResourceConfigCacheCapacity(int64_t& value); Status - GetResourceConfigIndexBuildDevice(int32_t& value); + GetGpuResourceConfigCacheThreshold(float& value); + Status + GetGpuResourceConfigSearchResources(std::vector& value); + Status + GetGpuResourceConfigBuildIndexResources(std::vector& value); public: /* server config */ @@ -300,10 +293,6 @@ class Config { Status SetCacheConfigCpuCacheThreshold(const std::string& value); Status - SetCacheConfigGpuCacheCapacity(const std::string& value); - Status - SetCacheConfigGpuCacheThreshold(const std::string& value); - Status SetCacheConfigCacheInsertData(const std::string& value); /* engine config */ @@ -314,13 +303,17 @@ class Config { Status SetEngineConfigGpuSearchThreshold(const std::string& value); - /* resource config */ + /* gpu resource config */ Status - SetResourceConfigMode(const std::string& value); + SetGpuResourceConfigEnable(const std::string& value); Status - SetResourceConfigSearchResources(const std::string& value); + SetGpuResourceConfigCacheCapacity(const std::string& value); Status - SetResourceConfigIndexBuildDevice(const std::string& value); + SetGpuResourceConfigCacheThreshold(const std::string& value); + Status + SetGpuResourceConfigSearchResources(const std::string& value); + Status + SetGpuResourceConfigBuildIndexResources(const std::string& value); private: std::unordered_map> config_map_; diff --git a/core/src/server/DBWrapper.cpp b/core/src/server/DBWrapper.cpp index 1ff914dc69..f7a43d671c 100644 --- a/core/src/server/DBWrapper.cpp +++ b/core/src/server/DBWrapper.cpp @@ -89,7 +89,7 @@ DBWrapper::StartService() { } // engine config - int32_t omp_thread; + int64_t omp_thread; s = config.GetEngineConfigOmpThreadNum(omp_thread); if (!s.ok()) { std::cerr << s.ToString() << std::endl; @@ -100,7 +100,7 @@ DBWrapper::StartService() { omp_set_num_threads(omp_thread); SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; } else { - uint32_t sys_thread_cnt = 8; + int64_t sys_thread_cnt = 8; if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { omp_thread = static_cast(ceil(sys_thread_cnt * 0.5)); omp_set_num_threads(omp_thread); @@ -108,7 +108,7 @@ DBWrapper::StartService() { } // init faiss global variable - int32_t use_blas_threshold; + int64_t use_blas_threshold; s = config.GetEngineConfigUseBlasThreshold(use_blas_threshold); if (!s.ok()) { std::cerr << s.ToString() << std::endl; @@ -119,7 +119,7 @@ DBWrapper::StartService() { // set archive config engine::ArchiveConf::CriteriaT criterial; - int32_t disk, days; + int64_t disk, days; s = config.GetDBConfigArchiveDiskThreshold(disk); if (!s.ok()) { std::cerr << s.ToString() << std::endl; diff --git a/core/src/server/Server.cpp b/core/src/server/Server.cpp index eb2b077829..5676504722 100644 --- a/core/src/server/Server.cpp +++ b/core/src/server/Server.cpp @@ -25,7 +25,7 @@ #include "server/DBWrapper.h" #include "server/Server.h" #include "server/grpc_impl/GrpcServer.h" -#include "src/config.h" +#include "src/version.h" #include "utils/Log.h" #include "utils/LogUtil.h" #include "utils/SignalUtil.h" diff --git a/core/src/server/grpc_impl/GrpcRequestHandler.cpp b/core/src/server/grpc_impl/GrpcRequestHandler.cpp index a6680745ad..014a0e2180 100644 --- a/core/src/server/grpc_impl/GrpcRequestHandler.cpp +++ b/core/src/server/grpc_impl/GrpcRequestHandler.cpp @@ -16,7 +16,24 @@ // under the License. #include "server/grpc_impl/GrpcRequestHandler.h" -#include "server/grpc_impl/GrpcRequestTask.h" +#include "server/grpc_impl/GrpcRequestScheduler.h" +#include "server/grpc_impl/request/CmdRequest.h" +#include "server/grpc_impl/request/CountTableRequest.h" +#include "server/grpc_impl/request/CreateIndexRequest.h" +#include "server/grpc_impl/request/CreatePartitionRequest.h" +#include "server/grpc_impl/request/CreateTableRequest.h" +#include "server/grpc_impl/request/DeleteByDateRequest.h" +#include "server/grpc_impl/request/DescribeIndexRequest.h" +#include "server/grpc_impl/request/DescribeTableRequest.h" +#include "server/grpc_impl/request/DropIndexRequest.h" +#include "server/grpc_impl/request/DropPartitionRequest.h" +#include "server/grpc_impl/request/DropTableRequest.h" +#include "server/grpc_impl/request/HasTableRequest.h" +#include "server/grpc_impl/request/InsertRequest.h" +#include "server/grpc_impl/request/PreloadTableRequest.h" +#include "server/grpc_impl/request/SearchRequest.h" +#include "server/grpc_impl/request/ShowPartitionsRequest.h" +#include "server/grpc_impl/request/ShowTablesRequest.h" #include "utils/TimeRecorder.h" #include @@ -28,8 +45,8 @@ namespace grpc { ::grpc::Status GrpcRequestHandler::CreateTable(::grpc::ServerContext* context, const ::milvus::grpc::TableSchema* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = CreateTableTask::Create(request); - GrpcRequestScheduler::ExecTask(task_ptr, response); + BaseRequestPtr request_ptr = CreateTableRequest::Create(request); + GrpcRequestScheduler::ExecRequest(request_ptr, response); return ::grpc::Status::OK; } @@ -37,9 +54,9 @@ GrpcRequestHandler::CreateTable(::grpc::ServerContext* context, const ::milvus:: GrpcRequestHandler::HasTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::BoolReply* response) { bool has_table = false; - BaseTaskPtr task_ptr = HasTableTask::Create(request->table_name(), has_table); + BaseRequestPtr request_ptr = HasTableRequest::Create(request->table_name(), has_table); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_bool_reply(has_table); response->mutable_status()->set_reason(grpc_status.reason()); response->mutable_status()->set_error_code(grpc_status.error_code()); @@ -49,25 +66,25 @@ GrpcRequestHandler::HasTable(::grpc::ServerContext* context, const ::milvus::grp ::grpc::Status GrpcRequestHandler::DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = DropTableTask::Create(request->table_name()); - GrpcRequestScheduler::ExecTask(task_ptr, response); + BaseRequestPtr request_ptr = DropTableRequest::Create(request->table_name()); + GrpcRequestScheduler::ExecRequest(request_ptr, response); return ::grpc::Status::OK; } ::grpc::Status GrpcRequestHandler::CreateIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = CreateIndexTask::Create(request); - GrpcRequestScheduler::ExecTask(task_ptr, response); + BaseRequestPtr request_ptr = CreateIndexRequest::Create(request); + GrpcRequestScheduler::ExecRequest(request_ptr, response); return ::grpc::Status::OK; } ::grpc::Status GrpcRequestHandler::Insert(::grpc::ServerContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response) { - BaseTaskPtr task_ptr = InsertTask::Create(request, response); + BaseRequestPtr request_ptr = InsertRequest::Create(request, response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_reason(grpc_status.reason()); response->mutable_status()->set_error_code(grpc_status.error_code()); return ::grpc::Status::OK; @@ -77,9 +94,9 @@ GrpcRequestHandler::Insert(::grpc::ServerContext* context, const ::milvus::grpc: GrpcRequestHandler::Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response) { std::vector file_id_array; - BaseTaskPtr task_ptr = SearchTask::Create(request, file_id_array, response); + BaseRequestPtr request_ptr = SearchRequest::Create(request, file_id_array, response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_error_code(grpc_status.error_code()); response->mutable_status()->set_reason(grpc_status.reason()); return ::grpc::Status::OK; @@ -93,9 +110,10 @@ GrpcRequestHandler::SearchInFiles(::grpc::ServerContext* context, const ::milvus file_id_array.push_back(request->file_id_array(i)); } ::milvus::grpc::SearchInFilesParam* request_mutable = const_cast<::milvus::grpc::SearchInFilesParam*>(request); - BaseTaskPtr task_ptr = SearchTask::Create(request_mutable->mutable_search_param(), file_id_array, response); + BaseRequestPtr request_ptr = + SearchRequest::Create(request_mutable->mutable_search_param(), file_id_array, response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_error_code(grpc_status.error_code()); response->mutable_status()->set_reason(grpc_status.reason()); return ::grpc::Status::OK; @@ -104,9 +122,9 @@ GrpcRequestHandler::SearchInFiles(::grpc::ServerContext* context, const ::milvus ::grpc::Status GrpcRequestHandler::DescribeTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableSchema* response) { - BaseTaskPtr task_ptr = DescribeTableTask::Create(request->table_name(), response); + BaseRequestPtr request_ptr = DescribeTableRequest::Create(request->table_name(), response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_error_code(grpc_status.error_code()); response->mutable_status()->set_reason(grpc_status.reason()); return ::grpc::Status::OK; @@ -116,9 +134,9 @@ GrpcRequestHandler::DescribeTable(::grpc::ServerContext* context, const ::milvus GrpcRequestHandler::CountTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableRowCount* response) { int64_t row_count = 0; - BaseTaskPtr task_ptr = CountTableTask::Create(request->table_name(), row_count); + BaseRequestPtr request_ptr = CountTableRequest::Create(request->table_name(), row_count); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_table_row_count(row_count); response->mutable_status()->set_reason(grpc_status.reason()); response->mutable_status()->set_error_code(grpc_status.error_code()); @@ -128,9 +146,9 @@ GrpcRequestHandler::CountTable(::grpc::ServerContext* context, const ::milvus::g ::grpc::Status GrpcRequestHandler::ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response) { - BaseTaskPtr task_ptr = ShowTablesTask::Create(response); + BaseRequestPtr request_ptr = ShowTablesRequest::Create(response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_error_code(grpc_status.error_code()); response->mutable_status()->set_reason(grpc_status.reason()); return ::grpc::Status::OK; @@ -140,9 +158,9 @@ GrpcRequestHandler::ShowTables(::grpc::ServerContext* context, const ::milvus::g GrpcRequestHandler::Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response) { std::string result; - BaseTaskPtr task_ptr = CmdTask::Create(request->cmd(), result); + BaseRequestPtr request_ptr = CmdRequest::Create(request->cmd(), result); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_string_reply(result); response->mutable_status()->set_reason(grpc_status.reason()); response->mutable_status()->set_error_code(grpc_status.error_code()); @@ -152,9 +170,9 @@ GrpcRequestHandler::Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Co ::grpc::Status GrpcRequestHandler::DeleteByDate(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByDateParam* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = DeleteByDateTask::Create(request); + BaseRequestPtr request_ptr = DeleteByDateRequest::Create(request); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_error_code(grpc_status.error_code()); response->set_reason(grpc_status.reason()); return ::grpc::Status::OK; @@ -163,9 +181,9 @@ GrpcRequestHandler::DeleteByDate(::grpc::ServerContext* context, const ::milvus: ::grpc::Status GrpcRequestHandler::PreloadTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = PreloadTableTask::Create(request->table_name()); + BaseRequestPtr request_ptr = PreloadTableRequest::Create(request->table_name()); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_reason(grpc_status.reason()); response->set_error_code(grpc_status.error_code()); return ::grpc::Status::OK; @@ -174,9 +192,9 @@ GrpcRequestHandler::PreloadTable(::grpc::ServerContext* context, const ::milvus: ::grpc::Status GrpcRequestHandler::DescribeIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::IndexParam* response) { - BaseTaskPtr task_ptr = DescribeIndexTask::Create(request->table_name(), response); + BaseRequestPtr request_ptr = DescribeIndexRequest::Create(request->table_name(), response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_reason(grpc_status.reason()); response->mutable_status()->set_error_code(grpc_status.error_code()); return ::grpc::Status::OK; @@ -185,9 +203,9 @@ GrpcRequestHandler::DescribeIndex(::grpc::ServerContext* context, const ::milvus ::grpc::Status GrpcRequestHandler::DropIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = DropIndexTask::Create(request->table_name()); + BaseRequestPtr request_ptr = DropIndexRequest::Create(request->table_name()); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_reason(grpc_status.reason()); response->set_error_code(grpc_status.error_code()); return ::grpc::Status::OK; @@ -196,17 +214,17 @@ GrpcRequestHandler::DropIndex(::grpc::ServerContext* context, const ::milvus::gr ::grpc::Status GrpcRequestHandler::CreatePartition(::grpc::ServerContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = CreatePartitionTask::Create(request); - GrpcRequestScheduler::ExecTask(task_ptr, response); + BaseRequestPtr request_ptr = CreatePartitionRequest::Create(request); + GrpcRequestScheduler::ExecRequest(request_ptr, response); return ::grpc::Status::OK; } ::grpc::Status GrpcRequestHandler::ShowPartitions(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::PartitionList* response) { - BaseTaskPtr task_ptr = ShowPartitionsTask::Create(request->table_name(), response); + BaseRequestPtr request_ptr = ShowPartitionsRequest::Create(request->table_name(), response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_reason(grpc_status.reason()); response->mutable_status()->set_error_code(grpc_status.error_code()); return ::grpc::Status::OK; @@ -215,9 +233,9 @@ GrpcRequestHandler::ShowPartitions(::grpc::ServerContext* context, const ::milvu ::grpc::Status GrpcRequestHandler::DropPartition(::grpc::ServerContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = DropPartitionTask::Create(request); + BaseRequestPtr request_ptr = DropPartitionRequest::Create(request); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_reason(grpc_status.reason()); response->set_error_code(grpc_status.error_code()); return ::grpc::Status::OK; diff --git a/core/src/server/grpc_impl/GrpcRequestScheduler.cpp b/core/src/server/grpc_impl/GrpcRequestScheduler.cpp index ac35f82947..9755678ddc 100644 --- a/core/src/server/grpc_impl/GrpcRequestScheduler.cpp +++ b/core/src/server/grpc_impl/GrpcRequestScheduler.cpp @@ -70,43 +70,6 @@ ErrorMap(ErrorCode code) { } } // namespace -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -GrpcBaseTask::GrpcBaseTask(const std::string& task_group, bool async) - : task_group_(task_group), async_(async), done_(false) { -} - -GrpcBaseTask::~GrpcBaseTask() { - WaitToFinish(); -} - -Status -GrpcBaseTask::Execute() { - status_ = OnExecute(); - Done(); - return status_; -} - -void -GrpcBaseTask::Done() { - done_ = true; - finish_cond_.notify_all(); -} - -Status -GrpcBaseTask::SetStatus(ErrorCode error_code, const std::string& error_msg) { - status_ = Status(error_code, error_msg); - SERVER_LOG_ERROR << error_msg; - return status_; -} - -Status -GrpcBaseTask::WaitToFinish() { - std::unique_lock lock(finish_mtx_); - finish_cond_.wait(lock, [this] { return done_; }); - - return status_; -} - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// GrpcRequestScheduler::GrpcRequestScheduler() : stopped_(false) { Start(); @@ -117,17 +80,17 @@ GrpcRequestScheduler::~GrpcRequestScheduler() { } void -GrpcRequestScheduler::ExecTask(BaseTaskPtr& task_ptr, ::milvus::grpc::Status* grpc_status) { - if (task_ptr == nullptr) { +GrpcRequestScheduler::ExecRequest(BaseRequestPtr& request_ptr, ::milvus::grpc::Status* grpc_status) { + if (request_ptr == nullptr) { return; } GrpcRequestScheduler& scheduler = GrpcRequestScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + scheduler.ExecuteRequest(request_ptr); - if (!task_ptr->IsAsync()) { - task_ptr->WaitToFinish(); - const Status& status = task_ptr->status(); + if (!request_ptr->IsAsync()) { + request_ptr->WaitToFinish(); + const Status& status = request_ptr->status(); if (!status.ok()) { grpc_status->set_reason(status.message()); grpc_status->set_error_code(ErrorMap(status.code())); @@ -153,7 +116,7 @@ GrpcRequestScheduler::Stop() { SERVER_LOG_INFO << "Scheduler gonna stop..."; { std::lock_guard lock(queue_mtx_); - for (auto iter : task_groups_) { + for (auto iter : request_groups_) { if (iter.second != nullptr) { iter.second->Put(nullptr); } @@ -171,64 +134,64 @@ GrpcRequestScheduler::Stop() { } Status -GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr& task_ptr) { - if (task_ptr == nullptr) { +GrpcRequestScheduler::ExecuteRequest(const BaseRequestPtr& request_ptr) { + if (request_ptr == nullptr) { return Status::OK(); } - auto status = PutTaskToQueue(task_ptr); + auto status = PutToQueue(request_ptr); if (!status.ok()) { - SERVER_LOG_ERROR << "Put task to queue failed with code: " << status.ToString(); + SERVER_LOG_ERROR << "Put request to queue failed with code: " << status.ToString(); return status; } - if (task_ptr->IsAsync()) { + if (request_ptr->IsAsync()) { return Status::OK(); // async execution, caller need to call WaitToFinish at somewhere } - return task_ptr->WaitToFinish(); // sync execution + return request_ptr->WaitToFinish(); // sync execution } void -GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) { - if (task_queue == nullptr) { +GrpcRequestScheduler::TakeToExecute(RequestQueuePtr request_queue) { + if (request_queue == nullptr) { return; } while (true) { - BaseTaskPtr task = task_queue->Take(); - if (task == nullptr) { - SERVER_LOG_ERROR << "Take null from task queue, stop thread"; + BaseRequestPtr request = request_queue->Take(); + if (request == nullptr) { + SERVER_LOG_ERROR << "Take null from request queue, stop thread"; break; // stop the thread } try { - auto status = task->Execute(); + auto status = request->Execute(); if (!status.ok()) { - SERVER_LOG_ERROR << "Task failed with code: " << status.ToString(); + SERVER_LOG_ERROR << "Request failed with code: " << status.ToString(); } } catch (std::exception& ex) { - SERVER_LOG_ERROR << "Task failed to execute: " << ex.what(); + SERVER_LOG_ERROR << "Request failed to execute: " << ex.what(); } } } Status -GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr& task_ptr) { +GrpcRequestScheduler::PutToQueue(const BaseRequestPtr& request_ptr) { std::lock_guard lock(queue_mtx_); - std::string group_name = task_ptr->TaskGroup(); - if (task_groups_.count(group_name) > 0) { - task_groups_[group_name]->Put(task_ptr); + std::string group_name = request_ptr->RequestGroup(); + if (request_groups_.count(group_name) > 0) { + request_groups_[group_name]->Put(request_ptr); } else { - TaskQueuePtr queue = std::make_shared(); - queue->Put(task_ptr); - task_groups_.insert(std::make_pair(group_name, queue)); + RequestQueuePtr queue = std::make_shared(); + queue->Put(request_ptr); + request_groups_.insert(std::make_pair(group_name, queue)); // start a thread - ThreadPtr thread = std::make_shared(&GrpcRequestScheduler::TakeTaskToExecute, this, queue); + ThreadPtr thread = std::make_shared(&GrpcRequestScheduler::TakeToExecute, this, queue); execute_threads_.push_back(thread); - SERVER_LOG_INFO << "Create new thread for task group: " << group_name; + SERVER_LOG_INFO << "Create new thread for request group: " << group_name; } return Status::OK(); diff --git a/core/src/server/grpc_impl/GrpcRequestScheduler.h b/core/src/server/grpc_impl/GrpcRequestScheduler.h index 802d247fb5..73f3a6e2c7 100644 --- a/core/src/server/grpc_impl/GrpcRequestScheduler.h +++ b/core/src/server/grpc_impl/GrpcRequestScheduler.h @@ -19,6 +19,7 @@ #include "grpc/gen-status/status.grpc.pb.h" #include "grpc/gen-status/status.pb.h" +#include "server/grpc_impl/request/GrpcBaseRequest.h" #include "utils/BlockingQueue.h" #include "utils/Status.h" @@ -32,57 +33,8 @@ namespace milvus { namespace server { namespace grpc { -class GrpcBaseTask { - protected: - explicit GrpcBaseTask(const std::string& task_group, bool async = false); - - virtual ~GrpcBaseTask(); - - public: - Status - Execute(); - - void - Done(); - - Status - WaitToFinish(); - - std::string - TaskGroup() const { - return task_group_; - } - - const Status& - status() const { - return status_; - } - - bool - IsAsync() const { - return async_; - } - - protected: - virtual Status - OnExecute() = 0; - - Status - SetStatus(ErrorCode error_code, const std::string& error_msg); - - protected: - mutable std::mutex finish_mtx_; - std::condition_variable finish_cond_; - - std::string task_group_; - bool async_; - bool done_; - Status status_; -}; - -using BaseTaskPtr = std::shared_ptr; -using TaskQueue = BlockingQueue; -using TaskQueuePtr = std::shared_ptr; +using RequestQueue = BlockingQueue; +using RequestQueuePtr = std::shared_ptr; using ThreadPtr = std::shared_ptr; class GrpcRequestScheduler { @@ -100,10 +52,10 @@ class GrpcRequestScheduler { Stop(); Status - ExecuteTask(const BaseTaskPtr& task_ptr); + ExecuteRequest(const BaseRequestPtr& request_ptr); static void - ExecTask(BaseTaskPtr& task_ptr, ::milvus::grpc::Status* grpc_status); + ExecRequest(BaseRequestPtr& request_ptr, ::milvus::grpc::Status* grpc_status); protected: GrpcRequestScheduler(); @@ -111,15 +63,15 @@ class GrpcRequestScheduler { virtual ~GrpcRequestScheduler(); void - TakeTaskToExecute(TaskQueuePtr task_queue); + TakeToExecute(RequestQueuePtr request_queue); Status - PutTaskToQueue(const BaseTaskPtr& task_ptr); + PutToQueue(const BaseRequestPtr& request_ptr); private: mutable std::mutex queue_mtx_; - std::map task_groups_; + std::map request_groups_; std::vector execute_threads_; diff --git a/core/src/server/grpc_impl/GrpcRequestTask.cpp b/core/src/server/grpc_impl/GrpcRequestTask.cpp deleted file mode 100644 index b8a9dbccbf..0000000000 --- a/core/src/server/grpc_impl/GrpcRequestTask.cpp +++ /dev/null @@ -1,1069 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "server/grpc_impl/GrpcRequestTask.h" - -#include -#include -#include -#include -#include -//#include - -#include "GrpcServer.h" -#include "db/Utils.h" -#include "scheduler/SchedInst.h" -#include "server/DBWrapper.h" -#include "server/Server.h" -#include "src/config.h" -#include "utils/CommonUtil.h" -#include "utils/Log.h" -#include "utils/TimeRecorder.h" -#include "utils/ValidationUtil.h" - -namespace milvus { -namespace server { -namespace grpc { - -static const char* DQL_TASK_GROUP = "dql"; -static const char* DDL_DML_TASK_GROUP = "ddl_dml"; -static const char* INFO_TASK_GROUP = "info"; - -constexpr int64_t DAY_SECONDS = 24 * 60 * 60; - -using DB_META = milvus::engine::meta::Meta; -using DB_DATE = milvus::engine::meta::DateT; - -namespace { -engine::EngineType -EngineType(int type) { - static std::map map_type = { - {0, engine::EngineType::INVALID}, - {1, engine::EngineType::FAISS_IDMAP}, - {2, engine::EngineType::FAISS_IVFFLAT}, - {3, engine::EngineType::FAISS_IVFSQ8}, - }; - - if (map_type.find(type) == map_type.end()) { - return engine::EngineType::INVALID; - } - - return map_type[type]; -} - -int -IndexType(engine::EngineType type) { - static std::map map_type = { - {engine::EngineType::INVALID, 0}, - {engine::EngineType::FAISS_IDMAP, 1}, - {engine::EngineType::FAISS_IVFFLAT, 2}, - {engine::EngineType::FAISS_IVFSQ8, 3}, - }; - - if (map_type.find(type) == map_type.end()) { - return 0; - } - - return map_type[type]; -} - -Status -ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range>& range_array, std::vector& dates) { - dates.clear(); - for (auto& range : range_array) { - time_t tt_start, tt_end; - tm tm_start, tm_end; - if (!CommonUtil::TimeStrToTime(range.start_value(), tt_start, tm_start)) { - return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); - } - - if (!CommonUtil::TimeStrToTime(range.end_value(), tt_end, tm_end)) { - return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); - } - - int64_t days = (tt_end - tt_start) / DAY_SECONDS; - if (days <= 0) { - return Status(SERVER_INVALID_TIME_RANGE, - "Invalid time range: The start-date should be smaller than end-date!"); - } - - // range: [start_day, end_day) - for (int64_t i = 0; i < days; i++) { - time_t tt_day = tt_start + DAY_SECONDS * i; - tm tm_day; - CommonUtil::ConvertTime(tt_day, tm_day); - - int64_t date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 + tm_day.tm_mday; // according to db logic - dates.push_back(date); - } - } - - return Status::OK(); -} - -std::string -TableNotExistMsg(const std::string& table_name) { - return "Table " + table_name + - " not exist. Use milvus.has_table to verify whether the table exists. You also can check if the table name " - "exists."; -} - -} // namespace - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CreateTableTask::CreateTableTask(const ::milvus::grpc::TableSchema* schema) - : GrpcBaseTask(DDL_DML_TASK_GROUP), schema_(schema) { -} - -BaseTaskPtr -CreateTableTask::Create(const ::milvus::grpc::TableSchema* schema) { - if (schema == nullptr) { - SERVER_LOG_ERROR << "grpc input is null!"; - return nullptr; - } - return std::shared_ptr(new CreateTableTask(schema)); -} - -Status -CreateTableTask::OnExecute() { - TimeRecorder rc("CreateTableTask"); - - try { - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(schema_->table_name()); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidateTableDimension(schema_->dimension()); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidateTableIndexFileSize(schema_->index_file_size()); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidateTableIndexMetricType(schema_->metric_type()); - if (!status.ok()) { - return status; - } - - // step 2: construct table schema - engine::meta::TableSchema table_info; - table_info.table_id_ = schema_->table_name(); - table_info.dimension_ = static_cast(schema_->dimension()); - table_info.index_file_size_ = schema_->index_file_size(); - table_info.metric_type_ = schema_->metric_type(); - - // step 3: create table - status = DBWrapper::DB()->CreateTable(table_info); - if (!status.ok()) { - // table could exist - if (status.code() == DB_ALREADY_EXIST) { - return Status(SERVER_INVALID_TABLE_NAME, status.message()); - } - return status; - } - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - rc.ElapseFromBegin("totally cost"); - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DescribeTableTask::DescribeTableTask(const std::string& table_name, ::milvus::grpc::TableSchema* schema) - : GrpcBaseTask(INFO_TASK_GROUP), table_name_(table_name), schema_(schema) { -} - -BaseTaskPtr -DescribeTableTask::Create(const std::string& table_name, ::milvus::grpc::TableSchema* schema) { - return std::shared_ptr(new DescribeTableTask(table_name, schema)); -} - -Status -DescribeTableTask::OnExecute() { - TimeRecorder rc("DescribeTableTask"); - - try { - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: get table info - engine::meta::TableSchema table_info; - table_info.table_id_ = table_name_; - status = DBWrapper::DB()->DescribeTable(table_info); - if (!status.ok()) { - return status; - } - - schema_->set_table_name(table_info.table_id_); - schema_->set_dimension(table_info.dimension_); - schema_->set_index_file_size(table_info.index_file_size_); - schema_->set_metric_type(table_info.metric_type_); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - rc.ElapseFromBegin("totally cost"); - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CreateIndexTask::CreateIndexTask(const ::milvus::grpc::IndexParam* index_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), index_param_(index_param) { -} - -BaseTaskPtr -CreateIndexTask::Create(const ::milvus::grpc::IndexParam* index_param) { - if (index_param == nullptr) { - SERVER_LOG_ERROR << "grpc input is null!"; - return nullptr; - } - return std::shared_ptr(new CreateIndexTask(index_param)); -} - -Status -CreateIndexTask::OnExecute() { - try { - TimeRecorder rc("CreateIndexTask"); - - // step 1: check arguments - std::string table_name_ = index_param_->table_name(); - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - bool has_table = false; - status = DBWrapper::DB()->HasTable(table_name_, has_table); - if (!status.ok()) { - return status; - } - - if (!has_table) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); - } - - auto& grpc_index = index_param_->index(); - status = ValidationUtil::ValidateTableIndexType(grpc_index.index_type()); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidateTableIndexNlist(grpc_index.nlist()); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - engine::TableIndex index; - index.engine_type_ = grpc_index.index_type(); - index.nlist_ = grpc_index.nlist(); - status = DBWrapper::DB()->CreateIndex(table_name_, index); - if (!status.ok()) { - return status; - } - - rc.ElapseFromBegin("totally cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -HasTableTask::HasTableTask(const std::string& table_name, bool& has_table) - : GrpcBaseTask(INFO_TASK_GROUP), table_name_(table_name), has_table_(has_table) { -} - -BaseTaskPtr -HasTableTask::Create(const std::string& table_name, bool& has_table) { - return std::shared_ptr(new HasTableTask(table_name, has_table)); -} - -Status -HasTableTask::OnExecute() { - try { - TimeRecorder rc("HasTableTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - status = DBWrapper::DB()->HasTable(table_name_, has_table_); - if (!status.ok()) { - return status; - } - - rc.ElapseFromBegin("totally cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DropTableTask::DropTableTask(const std::string& table_name) - : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name) { -} - -BaseTaskPtr -DropTableTask::Create(const std::string& table_name) { - return std::shared_ptr(new DropTableTask(table_name)); -} - -Status -DropTableTask::OnExecute() { - try { - TimeRecorder rc("DropTableTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - engine::meta::TableSchema table_info; - table_info.table_id_ = table_name_; - status = DBWrapper::DB()->DescribeTable(table_info); - if (!status.ok()) { - if (status.code() == DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); - } else { - return status; - } - } - - rc.ElapseFromBegin("check validation"); - - // step 3: Drop table - std::vector dates; - status = DBWrapper::DB()->DropTable(table_name_, dates); - if (!status.ok()) { - return status; - } - - rc.ElapseFromBegin("total cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -ShowTablesTask::ShowTablesTask(::milvus::grpc::TableNameList* table_name_list) - : GrpcBaseTask(INFO_TASK_GROUP), table_name_list_(table_name_list) { -} - -BaseTaskPtr -ShowTablesTask::Create(::milvus::grpc::TableNameList* table_name_list) { - return std::shared_ptr(new ShowTablesTask(table_name_list)); -} - -Status -ShowTablesTask::OnExecute() { - std::vector schema_array; - auto statuts = DBWrapper::DB()->AllTables(schema_array); - if (!statuts.ok()) { - return statuts; - } - - for (auto& schema : schema_array) { - table_name_list_->add_table_names(schema.table_id_); - } - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -InsertTask::InsertTask(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids) - : GrpcBaseTask(DDL_DML_TASK_GROUP), insert_param_(insert_param), record_ids_(record_ids) { -} - -BaseTaskPtr -InsertTask::Create(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids) { - if (insert_param == nullptr) { - SERVER_LOG_ERROR << "grpc input is null!"; - return nullptr; - } - return std::shared_ptr(new InsertTask(insert_param, record_ids)); -} - -Status -InsertTask::OnExecute() { - try { - TimeRecorder rc("InsertVectorTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(insert_param_->table_name()); - if (!status.ok()) { - return status; - } - if (insert_param_->row_record_array().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, - "The vector array is empty. Make sure you have entered vector records."); - } - - if (!insert_param_->row_id_array().empty()) { - if (insert_param_->row_id_array().size() != insert_param_->row_record_array_size()) { - return Status(SERVER_ILLEGAL_VECTOR_ID, - "The size of vector ID array must be equal to the size of the vector."); - } - } - - // step 2: check table existence - engine::meta::TableSchema table_info; - table_info.table_id_ = insert_param_->table_name(); - status = DBWrapper::DB()->DescribeTable(table_info); - if (!status.ok()) { - if (status.code() == DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(insert_param_->table_name())); - } else { - return status; - } - } - - // step 3: check table flag - // all user provide id, or all internal id - bool user_provide_ids = !insert_param_->row_id_array().empty(); - // user already provided id before, all insert action require user id - if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) != 0 && !user_provide_ids) { - return Status(SERVER_ILLEGAL_VECTOR_ID, - "Table vector IDs are user-defined. Please provide IDs for all vectors of this table."); - } - - // user didn't provided id before, no need to provide user id - if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) != 0 && user_provide_ids) { - return Status( - SERVER_ILLEGAL_VECTOR_ID, - "Table vector IDs are auto-generated. All vectors of this table must use auto-generated IDs."); - } - - rc.RecordSection("check validation"); - -#ifdef MILVUS_ENABLE_PROFILING - std::string fname = - "/tmp/insert_" + std::to_string(this->insert_param_->row_record_array_size()) + ".profiling"; - ProfilerStart(fname.c_str()); -#endif - - // step 4: prepare float data - std::vector vec_f(insert_param_->row_record_array_size() * table_info.dimension_, 0); - - // TODO(yk): change to one dimension array or use multiple-thread to copy the data - for (size_t i = 0; i < insert_param_->row_record_array_size(); i++) { - if (insert_param_->row_record_array(i).vector_data().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, - "The vector dimension must be equal to the table dimension."); - } - uint64_t vec_dim = insert_param_->row_record_array(i).vector_data().size(); - if (vec_dim != table_info.dimension_) { - ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; - std::string error_msg = "The vector dimension must be equal to the table dimension."; - return Status(error_code, error_msg); - } - memcpy(&vec_f[i * table_info.dimension_], insert_param_->row_record_array(i).vector_data().data(), - table_info.dimension_ * sizeof(float)); - } - - rc.ElapseFromBegin("prepare vectors data"); - - // step 5: insert vectors - auto vec_count = static_cast(insert_param_->row_record_array_size()); - std::vector vec_ids(insert_param_->row_id_array_size(), 0); - if (!insert_param_->row_id_array().empty()) { - const int64_t* src_data = insert_param_->row_id_array().data(); - int64_t* target_data = vec_ids.data(); - memcpy(target_data, src_data, static_cast(sizeof(int64_t) * insert_param_->row_id_array_size())); - } - - status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), insert_param_->partition_tag(), vec_count, - vec_f.data(), vec_ids); - rc.ElapseFromBegin("add vectors to engine"); - if (!status.ok()) { - return status; - } - for (int64_t id : vec_ids) { - record_ids_->add_vector_id_array(id); - } - - auto ids_size = record_ids_->vector_id_array_size(); - if (ids_size != vec_count) { - std::string msg = - "Add " + std::to_string(vec_count) + " vectors but only return " + std::to_string(ids_size) + " id"; - return Status(SERVER_ILLEGAL_VECTOR_ID, msg); - } - - // step 6: update table flag - user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID - : table_info.flag_ |= engine::meta::FLAG_MASK_NO_USERID; - status = DBWrapper::DB()->UpdateTableFlag(insert_param_->table_name(), table_info.flag_); - -#ifdef MILVUS_ENABLE_PROFILING - ProfilerStop(); -#endif - - rc.RecordSection("add vectors to engine"); - rc.ElapseFromBegin("total cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -SearchTask::SearchTask(const ::milvus::grpc::SearchParam* search_vector_infos, - const std::vector& file_id_array, ::milvus::grpc::TopKQueryResult* response) - : GrpcBaseTask(DQL_TASK_GROUP), - search_param_(search_vector_infos), - file_id_array_(file_id_array), - topk_result_(response) { -} - -BaseTaskPtr -SearchTask::Create(const ::milvus::grpc::SearchParam* search_vector_infos, - const std::vector& file_id_array, ::milvus::grpc::TopKQueryResult* response) { - if (search_vector_infos == nullptr) { - SERVER_LOG_ERROR << "grpc input is null!"; - return nullptr; - } - return std::shared_ptr(new SearchTask(search_vector_infos, file_id_array, response)); -} - -Status -SearchTask::OnExecute() { - try { - int64_t top_k = search_param_->topk(); - int64_t nprobe = search_param_->nprobe(); - - std::string hdr = "SearchTask(k=" + std::to_string(top_k) + ", nprob=" + std::to_string(nprobe) + ")"; - TimeRecorder rc(hdr); - - // step 1: check table name - std::string table_name_ = search_param_->table_name(); - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - engine::meta::TableSchema table_info; - table_info.table_id_ = table_name_; - status = DBWrapper::DB()->DescribeTable(table_info); - if (!status.ok()) { - if (status.code() == DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); - } else { - return status; - } - } - - // step 3: check search parameter - status = ValidationUtil::ValidateSearchTopk(top_k, table_info); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidateSearchNprobe(nprobe, table_info); - if (!status.ok()) { - return status; - } - - if (search_param_->query_record_array().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, - "The vector array is empty. Make sure you have entered vector records."); - } - - // step 4: check date range, and convert to db dates - std::vector dates; - std::vector<::milvus::grpc::Range> range_array; - for (size_t i = 0; i < search_param_->query_range_array_size(); i++) { - range_array.emplace_back(search_param_->query_range_array(i)); - } - - status = ConvertTimeRangeToDBDates(range_array, dates); - if (!status.ok()) { - return status; - } - - rc.RecordSection("check validation"); - - // step 5: prepare float data - auto record_array_size = search_param_->query_record_array_size(); - std::vector vec_f(record_array_size * table_info.dimension_, 0); - for (size_t i = 0; i < record_array_size; i++) { - if (search_param_->query_record_array(i).vector_data().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, - "The vector dimension must be equal to the table dimension."); - } - uint64_t query_vec_dim = search_param_->query_record_array(i).vector_data().size(); - if (query_vec_dim != table_info.dimension_) { - ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; - std::string error_msg = "The vector dimension must be equal to the table dimension."; - return Status(error_code, error_msg); - } - - memcpy(&vec_f[i * table_info.dimension_], search_param_->query_record_array(i).vector_data().data(), - table_info.dimension_ * sizeof(float)); - } - rc.RecordSection("prepare vector data"); - - // step 6: search vectors - engine::ResultIds result_ids; - engine::ResultDistances result_distances; - auto record_count = (uint64_t)search_param_->query_record_array().size(); - -#ifdef MILVUS_ENABLE_PROFILING - std::string fname = - "/tmp/search_nq_" + std::to_string(this->search_param_->query_record_array_size()) + ".profiling"; - ProfilerStart(fname.c_str()); -#endif - - if (file_id_array_.empty()) { - std::vector partition_tags; - for (size_t i = 0; i < search_param_->partition_tag_array_size(); i++) { - partition_tags.emplace_back(search_param_->partition_tag_array(i)); - } - - status = ValidationUtil::ValidatePartitionTags(partition_tags); - if (!status.ok()) { - return status; - } - - status = DBWrapper::DB()->Query(table_name_, partition_tags, (size_t)top_k, record_count, nprobe, - vec_f.data(), dates, result_ids, result_distances); - } else { - status = DBWrapper::DB()->QueryByFileID(table_name_, file_id_array_, (size_t)top_k, record_count, nprobe, - vec_f.data(), dates, result_ids, result_distances); - } - -#ifdef MILVUS_ENABLE_PROFILING - ProfilerStop(); -#endif - - rc.RecordSection("search vectors from engine"); - if (!status.ok()) { - return status; - } - - if (result_ids.empty()) { - return Status::OK(); // empty table - } - - // step 7: construct result array - topk_result_->set_row_num(record_count); - topk_result_->mutable_ids()->Resize(static_cast(result_ids.size()), 0); - memcpy(topk_result_->mutable_ids()->mutable_data(), result_ids.data(), result_ids.size() * sizeof(int64_t)); - topk_result_->mutable_distances()->Resize(static_cast(result_distances.size()), 0.0); - memcpy(topk_result_->mutable_distances()->mutable_data(), result_distances.data(), - result_distances.size() * sizeof(float)); - - // step 8: print time cost percent - rc.RecordSection("construct result and send"); - rc.ElapseFromBegin("totally cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CountTableTask::CountTableTask(const std::string& table_name, int64_t& row_count) - : GrpcBaseTask(INFO_TASK_GROUP), table_name_(table_name), row_count_(row_count) { -} - -BaseTaskPtr -CountTableTask::Create(const std::string& table_name, int64_t& row_count) { - return std::shared_ptr(new CountTableTask(table_name, row_count)); -} - -Status -CountTableTask::OnExecute() { - try { - TimeRecorder rc("GetTableRowCountTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: get row count - uint64_t row_count = 0; - status = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); - if (!status.ok()) { - if (status.code(), DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); - } else { - return status; - } - } - - row_count_ = static_cast(row_count); - - rc.ElapseFromBegin("total cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CmdTask::CmdTask(const std::string& cmd, std::string& result) - : GrpcBaseTask(INFO_TASK_GROUP), cmd_(cmd), result_(result) { -} - -BaseTaskPtr -CmdTask::Create(const std::string& cmd, std::string& result) { - return std::shared_ptr(new CmdTask(cmd, result)); -} - -Status -CmdTask::OnExecute() { - if (cmd_ == "version") { - result_ = MILVUS_VERSION; - } else if (cmd_ == "tasktable") { - result_ = scheduler::ResMgrInst::GetInstance()->DumpTaskTables(); - } else { - result_ = "OK"; - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DeleteByDateTask::DeleteByDateTask(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), delete_by_range_param_(delete_by_range_param) { -} - -BaseTaskPtr -DeleteByDateTask::Create(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param) { - if (delete_by_range_param == nullptr) { - SERVER_LOG_ERROR << "grpc input is null!"; - return nullptr; - } - - return std::shared_ptr(new DeleteByDateTask(delete_by_range_param)); -} - -Status -DeleteByDateTask::OnExecute() { - try { - TimeRecorder rc("DeleteByRangeTask"); - - // step 1: check arguments - std::string table_name = delete_by_range_param_->table_name(); - auto status = ValidationUtil::ValidateTableName(table_name); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - engine::meta::TableSchema table_info; - table_info.table_id_ = table_name; - status = DBWrapper::DB()->DescribeTable(table_info); - if (!status.ok()) { - if (status.code(), DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name)); - } else { - return status; - } - } - - rc.ElapseFromBegin("check validation"); - - // step 3: check date range, and convert to db dates - std::vector dates; - ErrorCode error_code = SERVER_SUCCESS; - std::string error_msg; - - std::vector<::milvus::grpc::Range> range_array; - range_array.emplace_back(delete_by_range_param_->range()); - status = ConvertTimeRangeToDBDates(range_array, dates); - if (!status.ok()) { - return status; - } - -#ifdef MILVUS_ENABLE_PROFILING - std::string fname = "/tmp/search_nq_" + this->delete_by_range_param_->table_name() + ".profiling"; - ProfilerStart(fname.c_str()); -#endif - status = DBWrapper::DB()->DropTable(table_name, dates); - if (!status.ok()) { - return status; - } - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -PreloadTableTask::PreloadTableTask(const std::string& table_name) - : GrpcBaseTask(DQL_TASK_GROUP), table_name_(table_name) { -} - -BaseTaskPtr -PreloadTableTask::Create(const std::string& table_name) { - return std::shared_ptr(new PreloadTableTask(table_name)); -} - -Status -PreloadTableTask::OnExecute() { - try { - TimeRecorder rc("PreloadTableTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - status = DBWrapper::DB()->PreloadTable(table_name_); - if (!status.ok()) { - return status; - } - - rc.ElapseFromBegin("totally cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DescribeIndexTask::DescribeIndexTask(const std::string& table_name, ::milvus::grpc::IndexParam* index_param) - : GrpcBaseTask(INFO_TASK_GROUP), table_name_(table_name), index_param_(index_param) { -} - -BaseTaskPtr -DescribeIndexTask::Create(const std::string& table_name, ::milvus::grpc::IndexParam* index_param) { - return std::shared_ptr(new DescribeIndexTask(table_name, index_param)); -} - -Status -DescribeIndexTask::OnExecute() { - try { - TimeRecorder rc("DescribeIndexTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - engine::TableIndex index; - status = DBWrapper::DB()->DescribeIndex(table_name_, index); - if (!status.ok()) { - return status; - } - - index_param_->set_table_name(table_name_); - index_param_->mutable_index()->set_index_type(index.engine_type_); - index_param_->mutable_index()->set_nlist(index.nlist_); - - rc.ElapseFromBegin("totally cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DropIndexTask::DropIndexTask(const std::string& table_name) - : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name) { -} - -BaseTaskPtr -DropIndexTask::Create(const std::string& table_name) { - return std::shared_ptr(new DropIndexTask(table_name)); -} - -Status -DropIndexTask::OnExecute() { - try { - TimeRecorder rc("DropIndexTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - bool has_table = false; - status = DBWrapper::DB()->HasTable(table_name_, has_table); - if (!status.ok()) { - return status; - } - - if (!has_table) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); - } - - // step 2: check table existence - status = DBWrapper::DB()->DropIndex(table_name_); - if (!status.ok()) { - return status; - } - - rc.ElapseFromBegin("totally cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CreatePartitionTask::CreatePartitionTask(const ::milvus::grpc::PartitionParam* partition_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), partition_param_(partition_param) { -} - -BaseTaskPtr -CreatePartitionTask::Create(const ::milvus::grpc::PartitionParam* partition_param) { - if (partition_param == nullptr) { - SERVER_LOG_ERROR << "grpc input is null!"; - return nullptr; - } - return std::shared_ptr(new CreatePartitionTask(partition_param)); -} - -Status -CreatePartitionTask::OnExecute() { - TimeRecorder rc("CreatePartitionTask"); - - try { - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(partition_param_->table_name()); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidateTableName(partition_param_->partition_name()); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidatePartitionTags({partition_param_->tag()}); - if (!status.ok()) { - return status; - } - - // step 2: create partition - status = DBWrapper::DB()->CreatePartition(partition_param_->table_name(), partition_param_->partition_name(), - partition_param_->tag()); - if (!status.ok()) { - // partition could exist - if (status.code() == DB_ALREADY_EXIST) { - return Status(SERVER_INVALID_TABLE_NAME, status.message()); - } - return status; - } - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - rc.ElapseFromBegin("totally cost"); - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -ShowPartitionsTask::ShowPartitionsTask(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list) - : GrpcBaseTask(INFO_TASK_GROUP), table_name_(table_name), partition_list_(partition_list) { -} - -BaseTaskPtr -ShowPartitionsTask::Create(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list) { - return std::shared_ptr(new ShowPartitionsTask(table_name, partition_list)); -} - -Status -ShowPartitionsTask::OnExecute() { - std::vector schema_array; - auto statuts = DBWrapper::DB()->ShowPartitions(table_name_, schema_array); - if (!statuts.ok()) { - return statuts; - } - - for (auto& schema : schema_array) { - ::milvus::grpc::PartitionParam* param = partition_list_->add_partition_array(); - param->set_table_name(schema.owner_table_); - param->set_partition_name(schema.table_id_); - param->set_tag(schema.partition_tag_); - } - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DropPartitionTask::DropPartitionTask(const ::milvus::grpc::PartitionParam* partition_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), partition_param_(partition_param) { -} - -BaseTaskPtr -DropPartitionTask::Create(const ::milvus::grpc::PartitionParam* partition_param) { - return std::shared_ptr(new DropPartitionTask(partition_param)); -} - -Status -DropPartitionTask::OnExecute() { - if (!partition_param_->partition_name().empty()) { - auto status = ValidationUtil::ValidateTableName(partition_param_->partition_name()); - if (!status.ok()) { - return status; - } - return DBWrapper::DB()->DropPartition(partition_param_->partition_name()); - } else { - auto status = ValidationUtil::ValidateTableName(partition_param_->table_name()); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidatePartitionTags({partition_param_->tag()}); - if (!status.ok()) { - return status; - } - return DBWrapper::DB()->DropPartitionByTag(partition_param_->table_name(), partition_param_->tag()); - } -} - -} // namespace grpc -} // namespace server -} // namespace milvus diff --git a/core/src/server/grpc_impl/GrpcRequestTask.h b/core/src/server/grpc_impl/GrpcRequestTask.h deleted file mode 100644 index 087c71aaaf..0000000000 --- a/core/src/server/grpc_impl/GrpcRequestTask.h +++ /dev/null @@ -1,321 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "db/Types.h" -#include "server/grpc_impl/GrpcRequestScheduler.h" -#include "utils/Status.h" - -#include "grpc/gen-milvus/milvus.grpc.pb.h" -#include "grpc/gen-status/status.pb.h" - -#include -#include -#include -#include - -namespace milvus { -namespace server { -namespace grpc { - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class CreateTableTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::TableSchema* schema); - - protected: - explicit CreateTableTask(const ::milvus::grpc::TableSchema* schema); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::TableSchema* schema_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class HasTableTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name, bool& has_table); - - protected: - HasTableTask(const std::string& table_name, bool& has_table); - - Status - OnExecute() override; - - private: - std::string table_name_; - bool& has_table_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DescribeTableTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name, ::milvus::grpc::TableSchema* schema); - - protected: - DescribeTableTask(const std::string& table_name, ::milvus::grpc::TableSchema* schema); - - Status - OnExecute() override; - - private: - std::string table_name_; - ::milvus::grpc::TableSchema* schema_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DropTableTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name); - - protected: - explicit DropTableTask(const std::string& table_name); - - Status - OnExecute() override; - - private: - std::string table_name_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class CreateIndexTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::IndexParam* index_param); - - protected: - explicit CreateIndexTask(const ::milvus::grpc::IndexParam* index_param); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::IndexParam* index_param_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class ShowTablesTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(::milvus::grpc::TableNameList* table_name_list); - - protected: - explicit ShowTablesTask(::milvus::grpc::TableNameList* table_name_list); - - Status - OnExecute() override; - - private: - ::milvus::grpc::TableNameList* table_name_list_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class InsertTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids); - - protected: - InsertTask(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::InsertParam* insert_param_; - ::milvus::grpc::VectorIds* record_ids_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class SearchTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::SearchParam* search_param, const std::vector& file_id_array, - ::milvus::grpc::TopKQueryResult* response); - - protected: - SearchTask(const ::milvus::grpc::SearchParam* search_param, const std::vector& file_id_array, - ::milvus::grpc::TopKQueryResult* response); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::SearchParam* search_param_; - std::vector file_id_array_; - ::milvus::grpc::TopKQueryResult* topk_result_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class CountTableTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name, int64_t& row_count); - - protected: - CountTableTask(const std::string& table_name, int64_t& row_count); - - Status - OnExecute() override; - - private: - std::string table_name_; - int64_t& row_count_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class CmdTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& cmd, std::string& result); - - protected: - CmdTask(const std::string& cmd, std::string& result); - - Status - OnExecute() override; - - private: - std::string cmd_; - std::string& result_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DeleteByDateTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param); - - protected: - explicit DeleteByDateTask(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::DeleteByDateParam* delete_by_range_param_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class PreloadTableTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name); - - protected: - explicit PreloadTableTask(const std::string& table_name); - - Status - OnExecute() override; - - private: - std::string table_name_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DescribeIndexTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name, ::milvus::grpc::IndexParam* index_param); - - protected: - DescribeIndexTask(const std::string& table_name, ::milvus::grpc::IndexParam* index_param); - - Status - OnExecute() override; - - private: - std::string table_name_; - ::milvus::grpc::IndexParam* index_param_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DropIndexTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name); - - protected: - explicit DropIndexTask(const std::string& table_name); - - Status - OnExecute() override; - - private: - std::string table_name_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class CreatePartitionTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::PartitionParam* partition_param); - - protected: - explicit CreatePartitionTask(const ::milvus::grpc::PartitionParam* partition_param); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::PartitionParam* partition_param_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class ShowPartitionsTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list); - - protected: - ShowPartitionsTask(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list); - - Status - OnExecute() override; - - private: - std::string table_name_; - ::milvus::grpc::PartitionList* partition_list_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DropPartitionTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::PartitionParam* partition_param); - - protected: - explicit DropPartitionTask(const ::milvus::grpc::PartitionParam* partition_param); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::PartitionParam* partition_param_; -}; - -} // namespace grpc -} // namespace server -} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CmdRequest.cpp b/core/src/server/grpc_impl/request/CmdRequest.cpp new file mode 100644 index 0000000000..b215f94d31 --- /dev/null +++ b/core/src/server/grpc_impl/request/CmdRequest.cpp @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/CmdRequest.h" +#include "scheduler/SchedInst.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +CmdRequest::CmdRequest(const std::string& cmd, std::string& result) + : GrpcBaseRequest(INFO_REQUEST_GROUP), cmd_(cmd), result_(result) { +} + +BaseRequestPtr +CmdRequest::Create(const std::string& cmd, std::string& result) { + return std::shared_ptr(new CmdRequest(cmd, result)); +} + +Status +CmdRequest::OnExecute() { + if (cmd_ == "version") { + result_ = MILVUS_VERSION; + } else if (cmd_ == "tasktable") { + result_ = scheduler::ResMgrInst::GetInstance()->DumpTaskTables(); + } else { + result_ = "OK"; + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CmdRequest.h b/core/src/server/grpc_impl/request/CmdRequest.h new file mode 100644 index 0000000000..960b17d436 --- /dev/null +++ b/core/src/server/grpc_impl/request/CmdRequest.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class CmdRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& cmd, std::string& result); + + protected: + CmdRequest(const std::string& cmd, std::string& result); + + Status + OnExecute() override; + + private: + std::string cmd_; + std::string& result_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CountTableRequest.cpp b/core/src/server/grpc_impl/request/CountTableRequest.cpp new file mode 100644 index 0000000000..8559890ad6 --- /dev/null +++ b/core/src/server/grpc_impl/request/CountTableRequest.cpp @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/CountTableRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +CountTableRequest::CountTableRequest(const std::string& table_name, int64_t& row_count) + : GrpcBaseRequest(INFO_REQUEST_GROUP), table_name_(table_name), row_count_(row_count) { +} + +BaseRequestPtr +CountTableRequest::Create(const std::string& table_name, int64_t& row_count) { + return std::shared_ptr(new CountTableRequest(table_name, row_count)); +} + +Status +CountTableRequest::OnExecute() { + try { + TimeRecorder rc("CountTableRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: get row count + uint64_t row_count = 0; + status = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); + if (!status.ok()) { + if (status.code(), DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); + } else { + return status; + } + } + + row_count_ = static_cast(row_count); + + rc.ElapseFromBegin("total cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CountTableRequest.h b/core/src/server/grpc_impl/request/CountTableRequest.h new file mode 100644 index 0000000000..80f8fcfcf5 --- /dev/null +++ b/core/src/server/grpc_impl/request/CountTableRequest.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class CountTableRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name, int64_t& row_count); + + protected: + CountTableRequest(const std::string& table_name, int64_t& row_count); + + Status + OnExecute() override; + + private: + std::string table_name_; + int64_t& row_count_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CreateIndexRequest.cpp b/core/src/server/grpc_impl/request/CreateIndexRequest.cpp new file mode 100644 index 0000000000..72678aee87 --- /dev/null +++ b/core/src/server/grpc_impl/request/CreateIndexRequest.cpp @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/CreateIndexRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +CreateIndexRequest::CreateIndexRequest(const ::milvus::grpc::IndexParam* index_param) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), index_param_(index_param) { +} + +BaseRequestPtr +CreateIndexRequest::Create(const ::milvus::grpc::IndexParam* index_param) { + if (index_param == nullptr) { + SERVER_LOG_ERROR << "grpc input is null!"; + return nullptr; + } + return std::shared_ptr(new CreateIndexRequest(index_param)); +} + +Status +CreateIndexRequest::OnExecute() { + try { + TimeRecorder rc("CreateIndexRequest"); + + // step 1: check arguments + std::string table_name_ = index_param_->table_name(); + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + bool has_table = false; + status = DBWrapper::DB()->HasTable(table_name_, has_table); + if (!status.ok()) { + return status; + } + + if (!has_table) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); + } + + auto& grpc_index = index_param_->index(); + status = ValidationUtil::ValidateTableIndexType(grpc_index.index_type()); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidateTableIndexNlist(grpc_index.nlist()); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + engine::TableIndex index; + index.engine_type_ = grpc_index.index_type(); + index.nlist_ = grpc_index.nlist(); + status = DBWrapper::DB()->CreateIndex(table_name_, index); + if (!status.ok()) { + return status; + } + + rc.ElapseFromBegin("totally cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CreateIndexRequest.h b/core/src/server/grpc_impl/request/CreateIndexRequest.h new file mode 100644 index 0000000000..9503cf0d06 --- /dev/null +++ b/core/src/server/grpc_impl/request/CreateIndexRequest.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class CreateIndexRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::IndexParam* index_param); + + protected: + explicit CreateIndexRequest(const ::milvus::grpc::IndexParam* index_param); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::IndexParam* index_param_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp b/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp new file mode 100644 index 0000000000..3bd4a86ef6 --- /dev/null +++ b/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/CreatePartitionRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +CreatePartitionRequest::CreatePartitionRequest(const ::milvus::grpc::PartitionParam* partition_param) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), partition_param_(partition_param) { +} + +BaseRequestPtr +CreatePartitionRequest::Create(const ::milvus::grpc::PartitionParam* partition_param) { + if (partition_param == nullptr) { + SERVER_LOG_ERROR << "grpc input is null!"; + return nullptr; + } + return std::shared_ptr(new CreatePartitionRequest(partition_param)); +} + +Status +CreatePartitionRequest::OnExecute() { + TimeRecorder rc("CreatePartitionRequest"); + + try { + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(partition_param_->table_name()); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidatePartitionName(partition_param_->partition_name()); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidatePartitionTags({partition_param_->tag()}); + if (!status.ok()) { + return status; + } + + // step 2: create partition + status = DBWrapper::DB()->CreatePartition(partition_param_->table_name(), partition_param_->partition_name(), + partition_param_->tag()); + if (!status.ok()) { + // partition could exist + if (status.code() == DB_ALREADY_EXIST) { + return Status(SERVER_INVALID_TABLE_NAME, status.message()); + } + return status; + } + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + rc.ElapseFromBegin("totally cost"); + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CreatePartitionRequest.h b/core/src/server/grpc_impl/request/CreatePartitionRequest.h new file mode 100644 index 0000000000..70a284bc16 --- /dev/null +++ b/core/src/server/grpc_impl/request/CreatePartitionRequest.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class CreatePartitionRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::PartitionParam* partition_param); + + protected: + explicit CreatePartitionRequest(const ::milvus::grpc::PartitionParam* partition_param); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::PartitionParam* partition_param_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CreateTableRequest.cpp b/core/src/server/grpc_impl/request/CreateTableRequest.cpp new file mode 100644 index 0000000000..67a3eaa877 --- /dev/null +++ b/core/src/server/grpc_impl/request/CreateTableRequest.cpp @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/CreateTableRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +CreateTableRequest::CreateTableRequest(const ::milvus::grpc::TableSchema* schema) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), schema_(schema) { +} + +BaseRequestPtr +CreateTableRequest::Create(const ::milvus::grpc::TableSchema* schema) { + if (schema == nullptr) { + SERVER_LOG_ERROR << "grpc input is null!"; + return nullptr; + } + return std::shared_ptr(new CreateTableRequest(schema)); +} + +Status +CreateTableRequest::OnExecute() { + TimeRecorder rc("CreateTableRequest"); + + try { + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(schema_->table_name()); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidateTableDimension(schema_->dimension()); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidateTableIndexFileSize(schema_->index_file_size()); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidateTableIndexMetricType(schema_->metric_type()); + if (!status.ok()) { + return status; + } + + // step 2: construct table schema + engine::meta::TableSchema table_info; + table_info.table_id_ = schema_->table_name(); + table_info.dimension_ = static_cast(schema_->dimension()); + table_info.index_file_size_ = schema_->index_file_size(); + table_info.metric_type_ = schema_->metric_type(); + + // step 3: create table + status = DBWrapper::DB()->CreateTable(table_info); + if (!status.ok()) { + // table could exist + if (status.code() == DB_ALREADY_EXIST) { + return Status(SERVER_INVALID_TABLE_NAME, status.message()); + } + return status; + } + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + rc.ElapseFromBegin("totally cost"); + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CreateTableRequest.h b/core/src/server/grpc_impl/request/CreateTableRequest.h new file mode 100644 index 0000000000..3a07aae1b7 --- /dev/null +++ b/core/src/server/grpc_impl/request/CreateTableRequest.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class CreateTableRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::TableSchema* schema); + + protected: + explicit CreateTableRequest(const ::milvus::grpc::TableSchema* schema); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::TableSchema* schema_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DeleteByDateRequest.cpp b/core/src/server/grpc_impl/request/DeleteByDateRequest.cpp new file mode 100644 index 0000000000..4a4d414803 --- /dev/null +++ b/core/src/server/grpc_impl/request/DeleteByDateRequest.cpp @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/DeleteByDateRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +DeleteByDateRequest::DeleteByDateRequest(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), delete_by_range_param_(delete_by_range_param) { +} + +BaseRequestPtr +DeleteByDateRequest::Create(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param) { + if (delete_by_range_param == nullptr) { + SERVER_LOG_ERROR << "grpc input is null!"; + return nullptr; + } + + return std::shared_ptr(new DeleteByDateRequest(delete_by_range_param)); +} + +Status +DeleteByDateRequest::OnExecute() { + try { + TimeRecorder rc("DeleteByDateRequest"); + + // step 1: check arguments + std::string table_name = delete_by_range_param_->table_name(); + auto status = ValidationUtil::ValidateTableName(table_name); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + engine::meta::TableSchema table_info; + table_info.table_id_ = table_name; + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code(), DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name)); + } else { + return status; + } + } + + rc.ElapseFromBegin("check validation"); + + // step 3: check date range, and convert to db dates + std::vector dates; + ErrorCode error_code = SERVER_SUCCESS; + std::string error_msg; + + std::vector<::milvus::grpc::Range> range_array; + range_array.emplace_back(delete_by_range_param_->range()); + status = ConvertTimeRangeToDBDates(range_array, dates); + if (!status.ok()) { + return status; + } + +#ifdef MILVUS_ENABLE_PROFILING + std::string fname = "/tmp/search_nq_" + this->delete_by_range_param_->table_name() + ".profiling"; + ProfilerStart(fname.c_str()); +#endif + status = DBWrapper::DB()->DropTable(table_name, dates); + if (!status.ok()) { + return status; + } + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DeleteByDateRequest.h b/core/src/server/grpc_impl/request/DeleteByDateRequest.h new file mode 100644 index 0000000000..b3e04c2850 --- /dev/null +++ b/core/src/server/grpc_impl/request/DeleteByDateRequest.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class DeleteByDateRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param); + + protected: + explicit DeleteByDateRequest(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::DeleteByDateParam* delete_by_range_param_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp b/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp new file mode 100644 index 0000000000..b3a987c6b0 --- /dev/null +++ b/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/DescribeIndexRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +DescribeIndexRequest::DescribeIndexRequest(const std::string& table_name, ::milvus::grpc::IndexParam* index_param) + : GrpcBaseRequest(INFO_REQUEST_GROUP), table_name_(table_name), index_param_(index_param) { +} + +BaseRequestPtr +DescribeIndexRequest::Create(const std::string& table_name, ::milvus::grpc::IndexParam* index_param) { + return std::shared_ptr(new DescribeIndexRequest(table_name, index_param)); +} + +Status +DescribeIndexRequest::OnExecute() { + try { + TimeRecorder rc("DescribeIndexRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + engine::TableIndex index; + status = DBWrapper::DB()->DescribeIndex(table_name_, index); + if (!status.ok()) { + return status; + } + + index_param_->set_table_name(table_name_); + index_param_->mutable_index()->set_index_type(index.engine_type_); + index_param_->mutable_index()->set_nlist(index.nlist_); + + rc.ElapseFromBegin("totally cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DescribeIndexRequest.h b/core/src/server/grpc_impl/request/DescribeIndexRequest.h new file mode 100644 index 0000000000..fa0d72152f --- /dev/null +++ b/core/src/server/grpc_impl/request/DescribeIndexRequest.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class DescribeIndexRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name, ::milvus::grpc::IndexParam* index_param); + + protected: + DescribeIndexRequest(const std::string& table_name, ::milvus::grpc::IndexParam* index_param); + + Status + OnExecute() override; + + private: + std::string table_name_; + ::milvus::grpc::IndexParam* index_param_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DescribeTableRequest.cpp b/core/src/server/grpc_impl/request/DescribeTableRequest.cpp new file mode 100644 index 0000000000..28a5f327c5 --- /dev/null +++ b/core/src/server/grpc_impl/request/DescribeTableRequest.cpp @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/DescribeTableRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +DescribeTableRequest::DescribeTableRequest(const std::string& table_name, ::milvus::grpc::TableSchema* schema) + : GrpcBaseRequest(INFO_REQUEST_GROUP), table_name_(table_name), schema_(schema) { +} + +BaseRequestPtr +DescribeTableRequest::Create(const std::string& table_name, ::milvus::grpc::TableSchema* schema) { + return std::shared_ptr(new DescribeTableRequest(table_name, schema)); +} + +Status +DescribeTableRequest::OnExecute() { + TimeRecorder rc("DescribeTableRequest"); + + try { + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: get table info + engine::meta::TableSchema table_info; + table_info.table_id_ = table_name_; + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + return status; + } + + schema_->set_table_name(table_info.table_id_); + schema_->set_dimension(table_info.dimension_); + schema_->set_index_file_size(table_info.index_file_size_); + schema_->set_metric_type(table_info.metric_type_); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + rc.ElapseFromBegin("totally cost"); + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DescribeTableRequest.h b/core/src/server/grpc_impl/request/DescribeTableRequest.h new file mode 100644 index 0000000000..0d903fedbd --- /dev/null +++ b/core/src/server/grpc_impl/request/DescribeTableRequest.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class DescribeTableRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name, ::milvus::grpc::TableSchema* schema); + + protected: + DescribeTableRequest(const std::string& table_name, ::milvus::grpc::TableSchema* schema); + + Status + OnExecute() override; + + private: + std::string table_name_; + ::milvus::grpc::TableSchema* schema_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DropIndexRequest.cpp b/core/src/server/grpc_impl/request/DropIndexRequest.cpp new file mode 100644 index 0000000000..ab5c83b0e5 --- /dev/null +++ b/core/src/server/grpc_impl/request/DropIndexRequest.cpp @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/DropIndexRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +DropIndexRequest::DropIndexRequest(const std::string& table_name) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), table_name_(table_name) { +} + +BaseRequestPtr +DropIndexRequest::Create(const std::string& table_name) { + return std::shared_ptr(new DropIndexRequest(table_name)); +} + +Status +DropIndexRequest::OnExecute() { + try { + TimeRecorder rc("DropIndexRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + bool has_table = false; + status = DBWrapper::DB()->HasTable(table_name_, has_table); + if (!status.ok()) { + return status; + } + + if (!has_table) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); + } + + // step 2: check table existence + status = DBWrapper::DB()->DropIndex(table_name_); + if (!status.ok()) { + return status; + } + + rc.ElapseFromBegin("totally cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DropIndexRequest.h b/core/src/server/grpc_impl/request/DropIndexRequest.h new file mode 100644 index 0000000000..c7cd51fcd7 --- /dev/null +++ b/core/src/server/grpc_impl/request/DropIndexRequest.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class DropIndexRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name); + + protected: + explicit DropIndexRequest(const std::string& table_name); + + Status + OnExecute() override; + + private: + std::string table_name_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DropPartitionRequest.cpp b/core/src/server/grpc_impl/request/DropPartitionRequest.cpp new file mode 100644 index 0000000000..0e29b6abe8 --- /dev/null +++ b/core/src/server/grpc_impl/request/DropPartitionRequest.cpp @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/DropPartitionRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +DropPartitionRequest::DropPartitionRequest(const ::milvus::grpc::PartitionParam* partition_param) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), partition_param_(partition_param) { +} + +BaseRequestPtr +DropPartitionRequest::Create(const ::milvus::grpc::PartitionParam* partition_param) { + return std::shared_ptr(new DropPartitionRequest(partition_param)); +} + +Status +DropPartitionRequest::OnExecute() { + std::string table_name = partition_param_->table_name(); + std::string partition_name = partition_param_->partition_name(); + std::string partition_tag = partition_param_->tag(); + if (!partition_name.empty()) { + auto status = ValidationUtil::ValidateTableName(partition_name); + if (!status.ok()) { + return status; + } + + // check partition existence + engine::meta::TableSchema table_info; + table_info.table_id_ = partition_name; + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code() == DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, + "Table " + table_name + "'s partition " + partition_name + " not found"); + } else { + return status; + } + } + + return DBWrapper::DB()->DropPartition(partition_name); + } else { + auto status = ValidationUtil::ValidateTableName(table_name); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidatePartitionTags({partition_tag}); + if (!status.ok()) { + return status; + } + return DBWrapper::DB()->DropPartitionByTag(table_name, partition_tag); + } +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DropPartitionRequest.h b/core/src/server/grpc_impl/request/DropPartitionRequest.h new file mode 100644 index 0000000000..4ac5743555 --- /dev/null +++ b/core/src/server/grpc_impl/request/DropPartitionRequest.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class DropPartitionRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::PartitionParam* partition_param); + + protected: + explicit DropPartitionRequest(const ::milvus::grpc::PartitionParam* partition_param); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::PartitionParam* partition_param_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DropTableRequest.cpp b/core/src/server/grpc_impl/request/DropTableRequest.cpp new file mode 100644 index 0000000000..a678e5a6f6 --- /dev/null +++ b/core/src/server/grpc_impl/request/DropTableRequest.cpp @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/DropTableRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +DropTableRequest::DropTableRequest(const std::string& table_name) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), table_name_(table_name) { +} + +BaseRequestPtr +DropTableRequest::Create(const std::string& table_name) { + return std::shared_ptr(new DropTableRequest(table_name)); +} + +Status +DropTableRequest::OnExecute() { + try { + TimeRecorder rc("DropTableRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + engine::meta::TableSchema table_info; + table_info.table_id_ = table_name_; + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code() == DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); + } else { + return status; + } + } + + rc.ElapseFromBegin("check validation"); + + // step 3: Drop table + std::vector dates; + status = DBWrapper::DB()->DropTable(table_name_, dates); + if (!status.ok()) { + return status; + } + + rc.ElapseFromBegin("total cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DropTableRequest.h b/core/src/server/grpc_impl/request/DropTableRequest.h new file mode 100644 index 0000000000..7dd7eca209 --- /dev/null +++ b/core/src/server/grpc_impl/request/DropTableRequest.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class DropTableRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name); + + protected: + explicit DropTableRequest(const std::string& table_name); + + Status + OnExecute() override; + + private: + std::string table_name_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp b/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp new file mode 100644 index 0000000000..e1eb07af5c --- /dev/null +++ b/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/GrpcBaseRequest.h" +#include "utils/CommonUtil.h" +#include "utils/Log.h" + +namespace milvus { +namespace server { +namespace grpc { + +constexpr int64_t DAY_SECONDS = 24 * 60 * 60; + +Status +ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range>& range_array, std::vector& dates) { + dates.clear(); + for (auto& range : range_array) { + time_t tt_start, tt_end; + tm tm_start, tm_end; + if (!CommonUtil::TimeStrToTime(range.start_value(), tt_start, tm_start)) { + return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); + } + + if (!CommonUtil::TimeStrToTime(range.end_value(), tt_end, tm_end)) { + return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); + } + + int64_t days = (tt_end - tt_start) / DAY_SECONDS; + if (days <= 0) { + return Status(SERVER_INVALID_TIME_RANGE, + "Invalid time range: The start-date should be smaller than end-date!"); + } + + // range: [start_day, end_day) + for (int64_t i = 0; i < days; i++) { + time_t tt_day = tt_start + DAY_SECONDS * i; + tm tm_day; + CommonUtil::ConvertTime(tt_day, tm_day); + + int64_t date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 + tm_day.tm_mday; // according to db logic + dates.push_back(date); + } + } + + return Status::OK(); +} + +GrpcBaseRequest::GrpcBaseRequest(const std::string& request_group, bool async) + : request_group_(request_group), async_(async), done_(false) { +} + +GrpcBaseRequest::~GrpcBaseRequest() { + WaitToFinish(); +} + +Status +GrpcBaseRequest::Execute() { + status_ = OnExecute(); + Done(); + return status_; +} + +void +GrpcBaseRequest::Done() { + done_ = true; + finish_cond_.notify_all(); +} + +Status +GrpcBaseRequest::SetStatus(ErrorCode error_code, const std::string& error_msg) { + status_ = Status(error_code, error_msg); + SERVER_LOG_ERROR << error_msg; + return status_; +} + +std::string +GrpcBaseRequest::TableNotExistMsg(const std::string& table_name) { + return "Table " + table_name + + " not exist. Use milvus.has_table to verify whether the table exists. You also can check if the table name " + "exists."; +} + +Status +GrpcBaseRequest::WaitToFinish() { + std::unique_lock lock(finish_mtx_); + finish_cond_.wait(lock, [this] { return done_; }); + + return status_; +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/GrpcBaseRequest.h b/core/src/server/grpc_impl/request/GrpcBaseRequest.h new file mode 100644 index 0000000000..b64662b2cf --- /dev/null +++ b/core/src/server/grpc_impl/request/GrpcBaseRequest.h @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "db/meta/MetaTypes.h" +#include "grpc/gen-milvus/milvus.grpc.pb.h" +#include "grpc/gen-status/status.grpc.pb.h" +#include "grpc/gen-status/status.pb.h" +#include "utils/Status.h" + +#include +//#include +#include +#include +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +static const char* DQL_REQUEST_GROUP = "dql"; +static const char* DDL_DML_REQUEST_GROUP = "ddl_dml"; +static const char* INFO_REQUEST_GROUP = "info"; + +using DB_DATE = milvus::engine::meta::DateT; + +Status +ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range>& range_array, std::vector& dates); + +class GrpcBaseRequest { + protected: + explicit GrpcBaseRequest(const std::string& request_group, bool async = false); + + virtual ~GrpcBaseRequest(); + + public: + Status + Execute(); + + void + Done(); + + Status + WaitToFinish(); + + std::string + RequestGroup() const { + return request_group_; + } + + const Status& + status() const { + return status_; + } + + bool + IsAsync() const { + return async_; + } + + protected: + virtual Status + OnExecute() = 0; + + Status + SetStatus(ErrorCode error_code, const std::string& error_msg); + + std::string + TableNotExistMsg(const std::string& table_name); + + protected: + mutable std::mutex finish_mtx_; + std::condition_variable finish_cond_; + + std::string request_group_; + bool async_; + bool done_; + Status status_; +}; + +using BaseRequestPtr = std::shared_ptr; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/HasTableRequest.cpp b/core/src/server/grpc_impl/request/HasTableRequest.cpp new file mode 100644 index 0000000000..2909c93056 --- /dev/null +++ b/core/src/server/grpc_impl/request/HasTableRequest.cpp @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/HasTableRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +HasTableRequest::HasTableRequest(const std::string& table_name, bool& has_table) + : GrpcBaseRequest(INFO_REQUEST_GROUP), table_name_(table_name), has_table_(has_table) { +} + +BaseRequestPtr +HasTableRequest::Create(const std::string& table_name, bool& has_table) { + return std::shared_ptr(new HasTableRequest(table_name, has_table)); +} + +Status +HasTableRequest::OnExecute() { + try { + TimeRecorder rc("HasTableRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + status = DBWrapper::DB()->HasTable(table_name_, has_table_); + if (!status.ok()) { + return status; + } + + rc.ElapseFromBegin("totally cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/HasTableRequest.h b/core/src/server/grpc_impl/request/HasTableRequest.h new file mode 100644 index 0000000000..85fa2b345b --- /dev/null +++ b/core/src/server/grpc_impl/request/HasTableRequest.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class HasTableRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name, bool& has_table); + + protected: + HasTableRequest(const std::string& table_name, bool& has_table); + + Status + OnExecute() override; + + private: + std::string table_name_; + bool& has_table_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/InsertRequest.cpp b/core/src/server/grpc_impl/request/InsertRequest.cpp new file mode 100644 index 0000000000..f436db074e --- /dev/null +++ b/core/src/server/grpc_impl/request/InsertRequest.cpp @@ -0,0 +1,170 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/InsertRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +InsertRequest::InsertRequest(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), insert_param_(insert_param), record_ids_(record_ids) { +} + +BaseRequestPtr +InsertRequest::Create(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids) { + if (insert_param == nullptr) { + SERVER_LOG_ERROR << "grpc input is null!"; + return nullptr; + } + return std::shared_ptr(new InsertRequest(insert_param, record_ids)); +} + +Status +InsertRequest::OnExecute() { + try { + TimeRecorder rc("InsertRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(insert_param_->table_name()); + if (!status.ok()) { + return status; + } + if (insert_param_->row_record_array().empty()) { + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector array is empty. Make sure you have entered vector records."); + } + + if (!insert_param_->row_id_array().empty()) { + if (insert_param_->row_id_array().size() != insert_param_->row_record_array_size()) { + return Status(SERVER_ILLEGAL_VECTOR_ID, + "The size of vector ID array must be equal to the size of the vector."); + } + } + + // step 2: check table existence + engine::meta::TableSchema table_info; + table_info.table_id_ = insert_param_->table_name(); + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code() == DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(insert_param_->table_name())); + } else { + return status; + } + } + + // step 3: check table flag + // all user provide id, or all internal id + bool user_provide_ids = !insert_param_->row_id_array().empty(); + // user already provided id before, all insert action require user id + if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) != 0 && !user_provide_ids) { + return Status(SERVER_ILLEGAL_VECTOR_ID, + "Table vector IDs are user-defined. Please provide IDs for all vectors of this table."); + } + + // user didn't provided id before, no need to provide user id + if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) != 0 && user_provide_ids) { + return Status( + SERVER_ILLEGAL_VECTOR_ID, + "Table vector IDs are auto-generated. All vectors of this table must use auto-generated IDs."); + } + + rc.RecordSection("check validation"); + +#ifdef MILVUS_ENABLE_PROFILING + std::string fname = + "/tmp/insert_" + std::to_string(this->insert_param_->row_record_array_size()) + ".profiling"; + ProfilerStart(fname.c_str()); +#endif + + // step 4: prepare float data + std::vector vec_f(insert_param_->row_record_array_size() * table_info.dimension_, 0); + + // TODO(yk): change to one dimension array or use multiple-thread to copy the data + for (size_t i = 0; i < insert_param_->row_record_array_size(); i++) { + if (insert_param_->row_record_array(i).vector_data().empty()) { + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector dimension must be equal to the table dimension."); + } + uint64_t vec_dim = insert_param_->row_record_array(i).vector_data().size(); + if (vec_dim != table_info.dimension_) { + ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; + std::string error_msg = "The vector dimension must be equal to the table dimension."; + return Status(error_code, error_msg); + } + memcpy(&vec_f[i * table_info.dimension_], insert_param_->row_record_array(i).vector_data().data(), + table_info.dimension_ * sizeof(float)); + } + + rc.ElapseFromBegin("prepare vectors data"); + + // step 5: insert vectors + auto vec_count = static_cast(insert_param_->row_record_array_size()); + std::vector vec_ids(insert_param_->row_id_array_size(), 0); + if (!insert_param_->row_id_array().empty()) { + const int64_t* src_data = insert_param_->row_id_array().data(); + int64_t* target_data = vec_ids.data(); + memcpy(target_data, src_data, static_cast(sizeof(int64_t) * insert_param_->row_id_array_size())); + } + + status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), insert_param_->partition_tag(), vec_count, + vec_f.data(), vec_ids); + rc.ElapseFromBegin("add vectors to engine"); + if (!status.ok()) { + return status; + } + for (int64_t id : vec_ids) { + record_ids_->add_vector_id_array(id); + } + + auto ids_size = record_ids_->vector_id_array_size(); + if (ids_size != vec_count) { + std::string msg = + "Add " + std::to_string(vec_count) + " vectors but only return " + std::to_string(ids_size) + " id"; + return Status(SERVER_ILLEGAL_VECTOR_ID, msg); + } + + // step 6: update table flag + user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID + : table_info.flag_ |= engine::meta::FLAG_MASK_NO_USERID; + status = DBWrapper::DB()->UpdateTableFlag(insert_param_->table_name(), table_info.flag_); + +#ifdef MILVUS_ENABLE_PROFILING + ProfilerStop(); +#endif + + rc.RecordSection("add vectors to engine"); + rc.ElapseFromBegin("total cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/InsertRequest.h b/core/src/server/grpc_impl/request/InsertRequest.h new file mode 100644 index 0000000000..1582615d2e --- /dev/null +++ b/core/src/server/grpc_impl/request/InsertRequest.h @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class InsertRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids); + + protected: + InsertRequest(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::InsertParam* insert_param_; + ::milvus::grpc::VectorIds* record_ids_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/PreloadTableRequest.cpp b/core/src/server/grpc_impl/request/PreloadTableRequest.cpp new file mode 100644 index 0000000000..e26aa8a877 --- /dev/null +++ b/core/src/server/grpc_impl/request/PreloadTableRequest.cpp @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/PreloadTableRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +PreloadTableRequest::PreloadTableRequest(const std::string& table_name) + : GrpcBaseRequest(DQL_REQUEST_GROUP), table_name_(table_name) { +} + +BaseRequestPtr +PreloadTableRequest::Create(const std::string& table_name) { + return std::shared_ptr(new PreloadTableRequest(table_name)); +} + +Status +PreloadTableRequest::OnExecute() { + try { + TimeRecorder rc("PreloadTableRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + status = DBWrapper::DB()->PreloadTable(table_name_); + if (!status.ok()) { + return status; + } + + rc.ElapseFromBegin("totally cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/PreloadTableRequest.h b/core/src/server/grpc_impl/request/PreloadTableRequest.h new file mode 100644 index 0000000000..208c7477ad --- /dev/null +++ b/core/src/server/grpc_impl/request/PreloadTableRequest.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class PreloadTableRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name); + + protected: + explicit PreloadTableRequest(const std::string& table_name); + + Status + OnExecute() override; + + private: + std::string table_name_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/SearchRequest.cpp b/core/src/server/grpc_impl/request/SearchRequest.cpp new file mode 100644 index 0000000000..28f4ff723e --- /dev/null +++ b/core/src/server/grpc_impl/request/SearchRequest.cpp @@ -0,0 +1,188 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/SearchRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +SearchRequest::SearchRequest(const ::milvus::grpc::SearchParam* search_vector_infos, + const std::vector& file_id_array, ::milvus::grpc::TopKQueryResult* response) + : GrpcBaseRequest(DQL_REQUEST_GROUP), + search_param_(search_vector_infos), + file_id_array_(file_id_array), + topk_result_(response) { +} + +BaseRequestPtr +SearchRequest::Create(const ::milvus::grpc::SearchParam* search_vector_infos, + const std::vector& file_id_array, ::milvus::grpc::TopKQueryResult* response) { + if (search_vector_infos == nullptr) { + SERVER_LOG_ERROR << "grpc input is null!"; + return nullptr; + } + return std::shared_ptr(new SearchRequest(search_vector_infos, file_id_array, response)); +} + +Status +SearchRequest::OnExecute() { + try { + int64_t top_k = search_param_->topk(); + int64_t nprobe = search_param_->nprobe(); + + std::string hdr = "SearchRequest(k=" + std::to_string(top_k) + ", nprob=" + std::to_string(nprobe) + ")"; + TimeRecorder rc(hdr); + + // step 1: check table name + std::string table_name_ = search_param_->table_name(); + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + engine::meta::TableSchema table_info; + table_info.table_id_ = table_name_; + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code() == DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); + } else { + return status; + } + } + + // step 3: check search parameter + status = ValidationUtil::ValidateSearchTopk(top_k, table_info); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidateSearchNprobe(nprobe, table_info); + if (!status.ok()) { + return status; + } + + if (search_param_->query_record_array().empty()) { + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector array is empty. Make sure you have entered vector records."); + } + + // step 4: check date range, and convert to db dates + std::vector dates; + std::vector<::milvus::grpc::Range> range_array; + for (size_t i = 0; i < search_param_->query_range_array_size(); i++) { + range_array.emplace_back(search_param_->query_range_array(i)); + } + + status = ConvertTimeRangeToDBDates(range_array, dates); + if (!status.ok()) { + return status; + } + + rc.RecordSection("check validation"); + + // step 5: prepare float data + auto record_array_size = search_param_->query_record_array_size(); + std::vector vec_f(record_array_size * table_info.dimension_, 0); + for (size_t i = 0; i < record_array_size; i++) { + if (search_param_->query_record_array(i).vector_data().empty()) { + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector dimension must be equal to the table dimension."); + } + uint64_t query_vec_dim = search_param_->query_record_array(i).vector_data().size(); + if (query_vec_dim != table_info.dimension_) { + ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; + std::string error_msg = "The vector dimension must be equal to the table dimension."; + return Status(error_code, error_msg); + } + + memcpy(&vec_f[i * table_info.dimension_], search_param_->query_record_array(i).vector_data().data(), + table_info.dimension_ * sizeof(float)); + } + rc.RecordSection("prepare vector data"); + + // step 6: search vectors + engine::ResultIds result_ids; + engine::ResultDistances result_distances; + auto record_count = (uint64_t)search_param_->query_record_array().size(); + +#ifdef MILVUS_ENABLE_PROFILING + std::string fname = + "/tmp/search_nq_" + std::to_string(this->search_param_->query_record_array_size()) + ".profiling"; + ProfilerStart(fname.c_str()); +#endif + + if (file_id_array_.empty()) { + std::vector partition_tags; + for (size_t i = 0; i < search_param_->partition_tag_array_size(); i++) { + partition_tags.emplace_back(search_param_->partition_tag_array(i)); + } + + status = ValidationUtil::ValidatePartitionTags(partition_tags); + if (!status.ok()) { + return status; + } + + status = DBWrapper::DB()->Query(table_name_, partition_tags, (size_t)top_k, record_count, nprobe, + vec_f.data(), dates, result_ids, result_distances); + } else { + status = DBWrapper::DB()->QueryByFileID(table_name_, file_id_array_, (size_t)top_k, record_count, nprobe, + vec_f.data(), dates, result_ids, result_distances); + } + +#ifdef MILVUS_ENABLE_PROFILING + ProfilerStop(); +#endif + + rc.RecordSection("search vectors from engine"); + if (!status.ok()) { + return status; + } + + if (result_ids.empty()) { + return Status::OK(); // empty table + } + + // step 7: construct result array + topk_result_->set_row_num(record_count); + topk_result_->mutable_ids()->Resize(static_cast(result_ids.size()), -1); + memcpy(topk_result_->mutable_ids()->mutable_data(), result_ids.data(), result_ids.size() * sizeof(int64_t)); + topk_result_->mutable_distances()->Resize(static_cast(result_distances.size()), 0.0); + memcpy(topk_result_->mutable_distances()->mutable_data(), result_distances.data(), + result_distances.size() * sizeof(float)); + + // step 8: print time cost percent + rc.RecordSection("construct result and send"); + rc.ElapseFromBegin("totally cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/SearchRequest.h b/core/src/server/grpc_impl/request/SearchRequest.h new file mode 100644 index 0000000000..667c7e62ee --- /dev/null +++ b/core/src/server/grpc_impl/request/SearchRequest.h @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +class SearchRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::SearchParam* search_param, const std::vector& file_id_array, + ::milvus::grpc::TopKQueryResult* response); + + protected: + SearchRequest(const ::milvus::grpc::SearchParam* search_param, const std::vector& file_id_array, + ::milvus::grpc::TopKQueryResult* response); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::SearchParam* search_param_; + std::vector file_id_array_; + ::milvus::grpc::TopKQueryResult* topk_result_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp b/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp new file mode 100644 index 0000000000..32fa0672c5 --- /dev/null +++ b/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/ShowPartitionsRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +ShowPartitionsRequest::ShowPartitionsRequest(const std::string& table_name, + ::milvus::grpc::PartitionList* partition_list) + : GrpcBaseRequest(INFO_REQUEST_GROUP), table_name_(table_name), partition_list_(partition_list) { +} + +BaseRequestPtr +ShowPartitionsRequest::Create(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list) { + return std::shared_ptr(new ShowPartitionsRequest(table_name, partition_list)); +} + +Status +ShowPartitionsRequest::OnExecute() { + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + std::vector schema_array; + auto statuts = DBWrapper::DB()->ShowPartitions(table_name_, schema_array); + if (!statuts.ok()) { + return statuts; + } + + for (auto& schema : schema_array) { + ::milvus::grpc::PartitionParam* param = partition_list_->add_partition_array(); + param->set_table_name(schema.owner_table_); + param->set_partition_name(schema.table_id_); + param->set_tag(schema.partition_tag_); + } + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/ShowPartitionsRequest.h b/core/src/server/grpc_impl/request/ShowPartitionsRequest.h new file mode 100644 index 0000000000..dd9768b9cb --- /dev/null +++ b/core/src/server/grpc_impl/request/ShowPartitionsRequest.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class ShowPartitionsRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list); + + protected: + ShowPartitionsRequest(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list); + + Status + OnExecute() override; + + private: + std::string table_name_; + ::milvus::grpc::PartitionList* partition_list_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/ShowTablesRequest.cpp b/core/src/server/grpc_impl/request/ShowTablesRequest.cpp new file mode 100644 index 0000000000..90dbad981f --- /dev/null +++ b/core/src/server/grpc_impl/request/ShowTablesRequest.cpp @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/ShowTablesRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" + +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +ShowTablesRequest::ShowTablesRequest(::milvus::grpc::TableNameList* table_name_list) + : GrpcBaseRequest(INFO_REQUEST_GROUP), table_name_list_(table_name_list) { +} + +BaseRequestPtr +ShowTablesRequest::Create(::milvus::grpc::TableNameList* table_name_list) { + return std::shared_ptr(new ShowTablesRequest(table_name_list)); +} + +Status +ShowTablesRequest::OnExecute() { + std::vector schema_array; + auto statuts = DBWrapper::DB()->AllTables(schema_array); + if (!statuts.ok()) { + return statuts; + } + + for (auto& schema : schema_array) { + table_name_list_->add_table_names(schema.table_id_); + } + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/ShowTablesRequest.h b/core/src/server/grpc_impl/request/ShowTablesRequest.h new file mode 100644 index 0000000000..4818f8bb9d --- /dev/null +++ b/core/src/server/grpc_impl/request/ShowTablesRequest.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class ShowTablesRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(::milvus::grpc::TableNameList* table_name_list); + + protected: + explicit ShowTablesRequest(::milvus::grpc::TableNameList* table_name_list); + + Status + OnExecute() override; + + private: + ::milvus::grpc::TableNameList* table_name_list_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/utils/CommonUtil.cpp b/core/src/utils/CommonUtil.cpp index fbf3112aeb..26e43619fb 100644 --- a/core/src/utils/CommonUtil.cpp +++ b/core/src/utils/CommonUtil.cpp @@ -54,7 +54,7 @@ CommonUtil::GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem) { } bool -CommonUtil::GetSystemAvailableThreads(uint32_t& thread_count) { +CommonUtil::GetSystemAvailableThreads(int64_t& thread_count) { // threadCnt = std::thread::hardware_concurrency(); thread_count = sysconf(_SC_NPROCESSORS_CONF); thread_count *= THREAD_MULTIPLY_CPU; diff --git a/core/src/utils/CommonUtil.h b/core/src/utils/CommonUtil.h index 939bdd6d31..121196986a 100644 --- a/core/src/utils/CommonUtil.h +++ b/core/src/utils/CommonUtil.h @@ -30,7 +30,7 @@ class CommonUtil { static bool GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem); static bool - GetSystemAvailableThreads(uint32_t& thread_count); + GetSystemAvailableThreads(int64_t& thread_count); static bool IsFileExist(const std::string& path); diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index ec696ff3e0..12b2372fc5 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -18,6 +18,7 @@ #include "utils/ValidationUtil.h" #include "Log.h" #include "db/engine/ExecutionEngine.h" +#include "utils/StringHelpFunctions.h" #include #ifdef MILVUS_GPU_VERSION @@ -168,11 +169,26 @@ ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSc return Status::OK(); } +Status +ValidationUtil::ValidatePartitionName(const std::string& partition_name) { + if (partition_name.empty()) { + std::string msg = "Partition name should not be empty."; + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_TABLE_NAME, msg); + } + + return ValidateTableName(partition_name); +} + Status ValidationUtil::ValidatePartitionTags(const std::vector& partition_tags) { - for (auto& tag : partition_tags) { - if (tag.empty()) { - std::string msg = "Invalid partition tag: " + tag + ". " + "Partition tag should not be empty."; + for (const std::string& tag : partition_tags) { + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + StringHelpFunctions::TrimStringBlank(valid_tag); + if (valid_tag.empty()) { + std::string msg = "Invalid partition tag: " + valid_tag + ". " + "Partition tag should not be empty."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_NPROBE, msg); } @@ -182,7 +198,7 @@ ValidationUtil::ValidatePartitionTags(const std::vector& partition_ } Status -ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { +ValidationUtil::ValidateGpuIndex(int32_t gpu_index) { #ifdef MILVUS_GPU_VERSION int num_devices = 0; auto cuda_err = cudaGetDeviceCount(&num_devices); @@ -203,7 +219,7 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { } Status -ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) { +ValidationUtil::GetGpuMemory(int32_t gpu_index, size_t& memory) { #ifdef MILVUS_GPU_VERSION cudaDeviceProp deviceProp; diff --git a/core/src/utils/ValidationUtil.h b/core/src/utils/ValidationUtil.h index 01801e295a..ab32c35c40 100644 --- a/core/src/utils/ValidationUtil.h +++ b/core/src/utils/ValidationUtil.h @@ -55,14 +55,17 @@ class ValidationUtil { static Status ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema); + static Status + ValidatePartitionName(const std::string& partition_name); + static Status ValidatePartitionTags(const std::vector& partition_tags); static Status - ValidateGpuIndex(uint32_t gpu_index); + ValidateGpuIndex(int32_t gpu_index); static Status - GetGpuMemory(uint32_t gpu_index, size_t& memory); + GetGpuMemory(int32_t gpu_index, size_t& memory); static Status ValidateIpAddress(const std::string& ip_address); diff --git a/core/src/config.h.in b/core/src/version.h.in similarity index 100% rename from core/src/config.h.in rename to core/src/version.h.in diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index 4114eb2af4..aa4b3c12b8 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -47,7 +47,8 @@ ConfAdapter::Match(const TempMetaConf& metaconf) { auto conf = std::make_shared(); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; + conf->k = metaconf.k; MatchBase(conf); return conf; } @@ -65,7 +66,7 @@ IVFConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; MatchBase(conf); return conf; } @@ -114,7 +115,7 @@ IVFSQConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; conf->nbits = 8; MatchBase(conf); return conf; @@ -126,7 +127,7 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; conf->nbits = 8; if (!(conf->d % 4)) @@ -175,21 +176,17 @@ NSGConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; + conf->k = metaconf.k; - double factor = metaconf.size / TYPICAL_COUNT; auto scale_factor = round(metaconf.dim / 128.0); scale_factor = scale_factor >= 4 ? 4 : scale_factor; - conf->nprobe = conf->nlist > 10000 ? conf->nlist * 0.02 : conf->nlist * 0.1; - conf->knng = (100 + 100 * scale_factor) * factor; - conf->search_length = (40 + 5 * scale_factor) * factor; - conf->out_degree = (50 + 5 * scale_factor) * factor; - conf->candidate_pool_size = (200 + 100 * scale_factor) * factor; + conf->nprobe = int64_t(conf->nlist * 0.01); + conf->knng = 40 + 10 * scale_factor; // the size of knng + conf->search_length = 40 + 5 * scale_factor; + conf->out_degree = 50 + 5 * scale_factor; + conf->candidate_pool_size = 200 + 100 * scale_factor; MatchBase(conf); - - // WRAPPER_LOG_DEBUG << "nlist: " << conf->nlist - // << ", gpu_id: " << conf->gpu_id << ", d: " << conf->d - // << ", nprobe: " << conf->nprobe << ", knng: " << conf->knng; return conf; } diff --git a/core/src/wrapper/ConfAdapter.h b/core/src/wrapper/ConfAdapter.h index 600dd3a517..46fc27eb3b 100644 --- a/core/src/wrapper/ConfAdapter.h +++ b/core/src/wrapper/ConfAdapter.h @@ -46,9 +46,6 @@ class ConfAdapter { virtual knowhere::Config MatchSearch(const TempMetaConf& metaconf, const IndexType& type); - // virtual void - // Dump(){} - protected: static void MatchBase(knowhere::Config conf); diff --git a/core/src/wrapper/KnowhereResource.cpp b/core/src/wrapper/KnowhereResource.cpp index 8ed19232e9..5a2296b16e 100644 --- a/core/src/wrapper/KnowhereResource.cpp +++ b/core/src/wrapper/KnowhereResource.cpp @@ -19,6 +19,8 @@ #ifdef MILVUS_GPU_VERSION #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #endif + +#include "scheduler/Utils.h" #include "server/Config.h" #include @@ -35,7 +37,6 @@ constexpr int64_t M_BYTE = 1024 * 1024; Status KnowhereResource::Initialize() { #ifdef MILVUS_GPU_VERSION - struct GpuResourceSetting { int64_t pinned_memory = 300 * M_BYTE; int64_t temp_memory = 300 * M_BYTE; @@ -47,27 +48,22 @@ KnowhereResource::Initialize() { // get build index gpu resource server::Config& config = server::Config::GetInstance(); - - int32_t build_index_gpu; - s = config.GetResourceConfigIndexBuildDevice(build_index_gpu); + std::vector build_index_gpus; + s = config.GetGpuResourceConfigBuildIndexResources(build_index_gpus); if (!s.ok()) return s; - gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting())); + for (auto gpu_id : build_index_gpus) { + gpu_resources.insert(std::make_pair(gpu_id, GpuResourceSetting())); + } // get search gpu resource - std::vector pool; - s = config.GetResourceConfigSearchResources(pool); + std::vector search_gpus; + s = config.GetGpuResourceConfigSearchResources(search_gpus); if (!s.ok()) return s; - std::set gpu_ids; - for (auto& resource : pool) { - if (resource.length() < 4 || resource.substr(0, 3) != "gpu") { - // invalid - continue; - } - auto gpu_id = std::stoi(resource.substr(3)); + for (auto& gpu_id : search_gpus) { gpu_resources.insert(std::make_pair(gpu_id, GpuResourceSetting())); } diff --git a/core/src/wrapper/VecImpl.cpp b/core/src/wrapper/VecImpl.cpp index e7967cbf59..6d681f7600 100644 --- a/core/src/wrapper/VecImpl.cpp +++ b/core/src/wrapper/VecImpl.cpp @@ -21,17 +21,19 @@ #include "knowhere/index/vector_index/IndexIDMAP.h" #include "utils/Log.h" #include "wrapper/WrapperException.h" +#include "wrapper/gpu/GPUVecImpl.h" #ifdef MILVUS_GPU_VERSION -#include -#include +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexIVFSQHybrid.h" +#include "knowhere/index/vector_index/helpers/Cloner.h" #endif /* * no parameter check in this layer. - * only responible for index combination + * only responsible for index combination */ namespace milvus { @@ -246,5 +248,130 @@ BFIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const return Status::OK(); } +// TODO(linxj): add lock here. +Status +IVFMixIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt, + const float* xt) { + try { + dim = cfg->d; + auto dataset = GenDatasetWithIds(nb, dim, xb, ids); + + auto preprocessor = index_->BuildPreprocessor(dataset, cfg); + index_->set_preprocessor(preprocessor); + auto model = index_->Train(dataset, cfg); + index_->set_index_model(model); + index_->Add(dataset, cfg); + + if (auto device_index = std::dynamic_pointer_cast(index_)) { + auto host_index = device_index->CopyGpuToCpu(Config()); + index_ = host_index; + type = ConvertToCpuIndexType(type); + } else { + WRAPPER_LOG_ERROR << "Build IVFMIXIndex Failed"; + return Status(KNOWHERE_ERROR, "Build IVFMIXIndex Failed"); + } + } catch (knowhere::KnowhereException& e) { + WRAPPER_LOG_ERROR << e.what(); + return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); + } catch (std::exception& e) { + WRAPPER_LOG_ERROR << e.what(); + return Status(KNOWHERE_ERROR, e.what()); + } + return Status::OK(); +} + +Status +IVFMixIndex::Load(const knowhere::BinarySet& index_binary) { + index_->Load(index_binary); + dim = Dimension(); + return Status::OK(); +} + +knowhere::QuantizerPtr +IVFHybridIndex::LoadQuantizer(const Config& conf) { + // TODO(linxj): Hardcode here + if (auto new_idx = std::dynamic_pointer_cast(index_)) { + return new_idx->LoadQuantizer(conf); + } else { + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); + } +} + +Status +IVFHybridIndex::SetQuantizer(const knowhere::QuantizerPtr& q) { + try { + // TODO(linxj): Hardcode here + if (auto new_idx = std::dynamic_pointer_cast(index_)) { + new_idx->SetQuantizer(q); + } else { + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); + return Status(KNOWHERE_ERROR, "not supported"); + } + } catch (knowhere::KnowhereException& e) { + WRAPPER_LOG_ERROR << e.what(); + return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); + } catch (std::exception& e) { + WRAPPER_LOG_ERROR << e.what(); + return Status(KNOWHERE_ERROR, e.what()); + } + return Status::OK(); +} + +Status +IVFHybridIndex::UnsetQuantizer() { + try { + // TODO(linxj): Hardcode here + if (auto new_idx = std::dynamic_pointer_cast(index_)) { + new_idx->UnsetQuantizer(); + } else { + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); + return Status(KNOWHERE_ERROR, "not supported"); + } + } catch (knowhere::KnowhereException& e) { + WRAPPER_LOG_ERROR << e.what(); + return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); + } catch (std::exception& e) { + WRAPPER_LOG_ERROR << e.what(); + return Status(KNOWHERE_ERROR, e.what()); + } + return Status::OK(); +} + +VecIndexPtr +IVFHybridIndex::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { + try { + // TODO(linxj): Hardcode here + if (auto new_idx = std::dynamic_pointer_cast(index_)) { + return std::make_shared(new_idx->LoadData(q, conf), type); + } else { + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); + } + } catch (knowhere::KnowhereException& e) { + WRAPPER_LOG_ERROR << e.what(); + } catch (std::exception& e) { + WRAPPER_LOG_ERROR << e.what(); + } + return nullptr; +} + +std::pair +IVFHybridIndex::CopyToGpuWithQuantizer(const int64_t& device_id, const Config& cfg) { + try { + // TODO(linxj): Hardcode here + if (auto hybrid_idx = std::dynamic_pointer_cast(index_)) { + auto pair = hybrid_idx->CopyCpuToGpuWithQuantizer(device_id, cfg); + auto new_idx = std::make_shared(pair.first, type); + return std::make_pair(new_idx, pair.second); + } else { + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); + } + } catch (knowhere::KnowhereException& e) { + WRAPPER_LOG_ERROR << e.what(); + } catch (std::exception& e) { + WRAPPER_LOG_ERROR << e.what(); + } + return std::make_pair(nullptr, nullptr); +} + } // namespace engine } // namespace milvus diff --git a/core/src/wrapper/VecIndex.cpp b/core/src/wrapper/VecIndex.cpp index 9f68cac0a1..75f75fb983 100644 --- a/core/src/wrapper/VecIndex.cpp +++ b/core/src/wrapper/VecIndex.cpp @@ -22,8 +22,8 @@ #include "knowhere/index/vector_index/IndexIVF.h" #include "knowhere/index/vector_index/IndexIVFPQ.h" #include "knowhere/index/vector_index/IndexIVFSQ.h" -#include "knowhere/index/vector_index/IndexSPTAG.h" #include "knowhere/index/vector_index/IndexNSG.h" +#include "knowhere/index/vector_index/IndexSPTAG.h" #include "utils/Log.h" #ifdef MILVUS_GPU_VERSION diff --git a/core/ubuntu_build_deps.sh b/core/ubuntu_build_deps.sh index 911046aa1f..6f3fabc084 100755 --- a/core/ubuntu_build_deps.sh +++ b/core/ubuntu_build_deps.sh @@ -9,4 +9,6 @@ sudo apt-get -y update && sudo apt-get -y install intel-mkl-gnu-2019.5-281 intel sudo apt-get install -y gfortran libmysqlclient-dev mysql-client libcurl4-openssl-dev libboost-system-dev \ libboost-filesystem-dev libboost-serialization-dev libboost-regex-dev -sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so +if [ ! -f "/usr/lib/x86_64-linux-gnu/libmysqlclient_r.so" ]; then + sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so +fi diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 01e1054f7e..a4d68e7423 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -66,7 +66,13 @@ set(thirdparty_files ) aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl/request grpc_request_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_impl_files) +set(grpc_server_files + ${grpc_request_files} + ${grpc_impl_files} + ) + aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files) aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_files) diff --git a/core/unittest/db/test_db.cpp b/core/unittest/db/test_db.cpp index d8614dd5d1..343e924e8e 100644 --- a/core/unittest/db/test_db.cpp +++ b/core/unittest/db/test_db.cpp @@ -461,6 +461,13 @@ TEST_F(DBTest, PARTITION_TEST) { stat = db_->CreatePartition(table_name, partition_name, partition_tag); ASSERT_TRUE(stat.ok()); + // not allow nested partition + stat = db_->CreatePartition(partition_name, "dumy", "dummy"); + ASSERT_FALSE(stat.ok()); + + // not allow duplicated partition + stat = db_->CreatePartition(table_name, partition_name, partition_tag); + ASSERT_FALSE(stat.ok()); std::vector xb; BuildVectors(INSERT_BATCH, xb); diff --git a/core/unittest/db/test_db_mysql.cpp b/core/unittest/db/test_db_mysql.cpp index f828431838..93eea2fd27 100644 --- a/core/unittest/db/test_db_mysql.cpp +++ b/core/unittest/db/test_db_mysql.cpp @@ -297,6 +297,14 @@ TEST_F(MySqlDBTest, PARTITION_TEST) { stat = db_->CreatePartition(table_name, partition_name, partition_tag); ASSERT_TRUE(stat.ok()); + // not allow nested partition + stat = db_->CreatePartition(partition_name, "dumy", "dummy"); + ASSERT_FALSE(stat.ok()); + + // not allow duplicated partition + stat = db_->CreatePartition(table_name, partition_name, partition_tag); + ASSERT_FALSE(stat.ok()); + std::vector xb; BuildVectors(INSERT_BATCH, xb); diff --git a/core/unittest/db/utils.cpp b/core/unittest/db/utils.cpp index f87371fe6b..afa1d39006 100644 --- a/core/unittest/db/utils.cpp +++ b/core/unittest/db/utils.cpp @@ -155,12 +155,14 @@ DBTest::SetUp() { res_mgr->Clear(); res_mgr->Add(milvus::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false)); res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true)); - res_mgr->Add(milvus::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); auto default_conn = milvus::scheduler::Connection("IO", 500.0); auto PCIE = milvus::scheduler::Connection("IO", 11000.0); res_mgr->Connect("disk", "cpu", default_conn); - res_mgr->Connect("cpu", "gtx1660", PCIE); +#ifdef MILVUS_GPU_VERSION + res_mgr->Add(milvus::scheduler::ResourceFactory::Create("0", "GPU", 0, true, true)); + res_mgr->Connect("cpu", "0", PCIE); +#endif res_mgr->Start(); milvus::scheduler::SchedInst::GetInstance()->Start(); diff --git a/core/unittest/scheduler/CMakeLists.txt b/core/unittest/scheduler/CMakeLists.txt index 0148441c2d..878f0f23a9 100644 --- a/core/unittest/scheduler/CMakeLists.txt +++ b/core/unittest/scheduler/CMakeLists.txt @@ -21,7 +21,6 @@ set(test_files ${CMAKE_CURRENT_SOURCE_DIR}/test_algorithm.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_event.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_node.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/test_normal.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_resource.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_resource_factory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_resource_mgr.cpp diff --git a/core/unittest/scheduler/test_normal.cpp b/core/unittest/scheduler/test_normal.cpp deleted file mode 100644 index 20cd39d0bf..0000000000 --- a/core/unittest/scheduler/test_normal.cpp +++ /dev/null @@ -1,72 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include "scheduler/ResourceFactory.h" -#include "scheduler/ResourceMgr.h" -#include "scheduler/SchedInst.h" -#include "scheduler/Scheduler.h" -#include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" -#include "utils/Log.h" - -namespace { - -namespace ms = milvus::scheduler; - -} // namespace - -TEST(NormalTest, INST_TEST) { - // ResourceMgr only compose resources, provide unified event - auto res_mgr = ms::ResMgrInst::GetInstance(); - - res_mgr->Add(ms::ResourceFactory::Create("disk", "DISK", 0, true, false)); - res_mgr->Add(ms::ResourceFactory::Create("cpu", "CPU", 0, true, true)); - - auto IO = ms::Connection("IO", 500.0); - res_mgr->Connect("disk", "cpu", IO); - - auto scheduler = ms::SchedInst::GetInstance(); - - res_mgr->Start(); - scheduler->Start(); - - const uint64_t NUM_TASK = 2; - std::vector> tasks; - ms::TableFileSchemaPtr dummy = nullptr; - - auto disks = res_mgr->GetDiskResources(); - ASSERT_FALSE(disks.empty()); - if (auto observe = disks[0].lock()) { - for (uint64_t i = 0; i < NUM_TASK; ++i) { - auto label = std::make_shared(); - auto task = std::make_shared(dummy, label); - task->label() = std::make_shared(); - tasks.push_back(task); - observe->task_table().Put(task); - } - } - - for (auto& task : tasks) { - task->Wait(); - ASSERT_EQ(task->load_count_, 1); - ASSERT_EQ(task->exec_count_, 1); - } - - scheduler->Stop(); - res_mgr->Stop(); -} diff --git a/core/unittest/scheduler/test_resource.cpp b/core/unittest/scheduler/test_resource.cpp index 6fe85ec558..101dcd3976 100644 --- a/core/unittest/scheduler/test_resource.cpp +++ b/core/unittest/scheduler/test_resource.cpp @@ -24,7 +24,7 @@ #include "scheduler/resource/TestResource.h" #include "scheduler/task/Task.h" #include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" +#include "scheduler/tasklabel/SpecResLabel.h" namespace milvus { namespace scheduler { @@ -182,8 +182,10 @@ TEST_F(ResourceAdvanceTest, DISK_RESOURCE_TEST) { std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); + auto label = std::make_shared(disk_resource_); auto task = std::make_shared(dummy, label); + std::vector path{disk_resource_->name()}; + task->path() = Path(path, 0); tasks.push_back(task); disk_resource_->task_table().Put(task); } @@ -208,8 +210,10 @@ TEST_F(ResourceAdvanceTest, CPU_RESOURCE_TEST) { std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); + auto label = std::make_shared(cpu_resource_); auto task = std::make_shared(dummy, label); + std::vector path{cpu_resource_->name()}; + task->path() = Path(path, 0); tasks.push_back(task); cpu_resource_->task_table().Put(task); } @@ -234,8 +238,10 @@ TEST_F(ResourceAdvanceTest, GPU_RESOURCE_TEST) { std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); + auto label = std::make_shared(gpu_resource_); auto task = std::make_shared(dummy, label); + std::vector path{gpu_resource_->name()}; + task->path() = Path(path, 0); tasks.push_back(task); gpu_resource_->task_table().Put(task); } @@ -260,8 +266,10 @@ TEST_F(ResourceAdvanceTest, TEST_RESOURCE_TEST) { std::vector> tasks; TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); + auto label = std::make_shared(test_resource_); auto task = std::make_shared(dummy, label); + std::vector path{test_resource_->name()}; + task->path() = Path(path, 0); tasks.push_back(task); test_resource_->task_table().Put(task); } diff --git a/core/unittest/scheduler/test_resource_mgr.cpp b/core/unittest/scheduler/test_resource_mgr.cpp index b9127060bd..d6495971c4 100644 --- a/core/unittest/scheduler/test_resource_mgr.cpp +++ b/core/unittest/scheduler/test_resource_mgr.cpp @@ -22,7 +22,6 @@ #include "scheduler/resource/GpuResource.h" #include "scheduler/resource/TestResource.h" #include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" namespace milvus { namespace scheduler { @@ -187,8 +186,7 @@ TEST_F(ResourceMgrAdvanceTest, REGISTER_SUBSCRIBER) { auto callback = [&](EventPtr event) { flag = true; }; mgr1_->RegisterSubscriber(callback); TableFileSchemaPtr dummy = nullptr; - auto label = std::make_shared(); - disk_res->task_table().Put(std::make_shared(dummy, label)); + disk_res->task_table().Put(std::make_shared(dummy, nullptr)); sleep(1); ASSERT_TRUE(flag); } diff --git a/core/unittest/scheduler/test_scheduler.cpp b/core/unittest/scheduler/test_scheduler.cpp index 22fb9be723..72538113c3 100644 --- a/core/unittest/scheduler/test_scheduler.cpp +++ b/core/unittest/scheduler/test_scheduler.cpp @@ -23,7 +23,6 @@ #include "scheduler/Scheduler.h" #include "scheduler/resource/Resource.h" #include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" #include "scheduler/tasklabel/SpecResLabel.h" #include "utils/Error.h" #include "wrapper/VecIndex.h" @@ -150,46 +149,6 @@ insert_dummy_index_into_gpu_cache(uint64_t device_id) { cache::GpuCacheMgr::GetInstance(device_id)->InsertItem("location", obj); } -TEST_F(SchedulerTest, ON_LOAD_COMPLETED) { - const uint64_t NUM = 10; - std::vector> tasks; - TableFileSchemaPtr dummy = std::make_shared(); - dummy->location_ = "location"; - - insert_dummy_index_into_gpu_cache(1); - - for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); - auto task = std::make_shared(dummy, label); - task->label() = std::make_shared(); - tasks.push_back(task); - cpu_resource_.lock()->task_table().Put(task); - } - - sleep(3); - ASSERT_EQ(res_mgr_->GetResource(ResourceType::GPU, 1)->task_table().size(), NUM); -} - -TEST_F(SchedulerTest, PUSH_TASK_TO_NEIGHBOUR_RANDOMLY_TEST) { - const uint64_t NUM = 10; - std::vector> tasks; - TableFileSchemaPtr dummy1 = std::make_shared(); - dummy1->location_ = "location"; - - tasks.clear(); - - for (uint64_t i = 0; i < NUM; ++i) { - auto label = std::make_shared(); - auto task = std::make_shared(dummy1, label); - task->label() = std::make_shared(); - tasks.push_back(task); - cpu_resource_.lock()->task_table().Put(task); - } - - sleep(3); - // ASSERT_EQ(res_mgr_->GetResource(ResourceType::GPU, 1)->task_table().Size(), NUM); -} - class SchedulerTest2 : public testing::Test { protected: void diff --git a/core/unittest/scheduler/test_tasktable.cpp b/core/unittest/scheduler/test_tasktable.cpp index 28a2e29c98..3fa1beabeb 100644 --- a/core/unittest/scheduler/test_tasktable.cpp +++ b/core/unittest/scheduler/test_tasktable.cpp @@ -18,7 +18,6 @@ #include #include "scheduler/TaskTable.h" #include "scheduler/task/TestTask.h" -#include "scheduler/tasklabel/DefaultLabel.h" /************ TaskTableBaseTest ************/ @@ -162,9 +161,8 @@ class TaskTableBaseTest : public ::testing::Test { SetUp() override { milvus::scheduler::TableFileSchemaPtr dummy = nullptr; invalid_task_ = nullptr; - auto label = std::make_shared(); - task1_ = std::make_shared(dummy, label); - task2_ = std::make_shared(dummy, label); + task1_ = std::make_shared(dummy, nullptr); + task2_ = std::make_shared(dummy, nullptr); } milvus::scheduler::TaskPtr invalid_task_; @@ -320,8 +318,7 @@ class TaskTableAdvanceTest : public ::testing::Test { SetUp() override { milvus::scheduler::TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < 8; ++i) { - auto label = std::make_shared(); - auto task = std::make_shared(dummy, label); + auto task = std::make_shared(dummy, nullptr); table1_.Put(task); } diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 637273732d..664a08d631 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -104,7 +104,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { milvus::server::Config& config = milvus::server::Config::GetInstance(); milvus::Status s; std::string str_val; - int32_t int32_val; int64_t int64_val; float float_val; bool bool_val; @@ -160,26 +159,26 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(s.ok()); ASSERT_TRUE(str_val == db_backend_url); - int32_t db_archive_disk_threshold = 100; + int64_t db_archive_disk_threshold = 100; s = config.SetDBConfigArchiveDiskThreshold(std::to_string(db_archive_disk_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetDBConfigArchiveDiskThreshold(int32_val); + s = config.GetDBConfigArchiveDiskThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == db_archive_disk_threshold); + ASSERT_TRUE(int64_val == db_archive_disk_threshold); - int32_t db_archive_days_threshold = 365; + int64_t db_archive_days_threshold = 365; s = config.SetDBConfigArchiveDaysThreshold(std::to_string(db_archive_days_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetDBConfigArchiveDaysThreshold(int32_val); + s = config.GetDBConfigArchiveDaysThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == db_archive_days_threshold); + ASSERT_TRUE(int64_val == db_archive_days_threshold); - int32_t db_insert_buffer_size = 2; + int64_t db_insert_buffer_size = 2; s = config.SetDBConfigInsertBufferSize(std::to_string(db_insert_buffer_size)); ASSERT_TRUE(s.ok()); - s = config.GetDBConfigInsertBufferSize(int32_val); + s = config.GetDBConfigInsertBufferSize(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == db_insert_buffer_size); + ASSERT_TRUE(int64_val == db_insert_buffer_size); /* metric config */ bool metric_enable_monitor = false; @@ -216,21 +215,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { s = config.GetCacheConfigCpuCacheThreshold(float_val); ASSERT_TRUE(float_val == cache_cpu_cache_threshold); -#ifdef MILVUS_GPU_VERSION - int64_t cache_gpu_cache_capacity = 1; - s = config.SetCacheConfigGpuCacheCapacity(std::to_string(cache_gpu_cache_capacity)); - ASSERT_TRUE(s.ok()); - s = config.GetCacheConfigGpuCacheCapacity(int64_val); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int64_val == cache_gpu_cache_capacity); - - float cache_gpu_cache_threshold = 0.2; - s = config.SetCacheConfigGpuCacheThreshold(std::to_string(cache_gpu_cache_threshold)); - ASSERT_TRUE(s.ok()); - s = config.GetCacheConfigGpuCacheThreshold(float_val); - ASSERT_TRUE(float_val == cache_gpu_cache_threshold); -#endif - bool cache_insert_data = true; s = config.SetCacheConfigCacheInsertData(std::to_string(cache_insert_data)); ASSERT_TRUE(s.ok()); @@ -238,63 +222,75 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(bool_val == cache_insert_data); /* engine config */ - int32_t engine_use_blas_threshold = 50; + int64_t engine_use_blas_threshold = 50; s = config.SetEngineConfigUseBlasThreshold(std::to_string(engine_use_blas_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigUseBlasThreshold(int32_val); + s = config.GetEngineConfigUseBlasThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == engine_use_blas_threshold); + ASSERT_TRUE(int64_val == engine_use_blas_threshold); - int32_t engine_omp_thread_num = 8; + int64_t engine_omp_thread_num = 8; s = config.SetEngineConfigOmpThreadNum(std::to_string(engine_omp_thread_num)); ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigOmpThreadNum(int32_val); + s = config.GetEngineConfigOmpThreadNum(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == engine_omp_thread_num); + ASSERT_TRUE(int64_val == engine_omp_thread_num); - int32_t engine_gpu_search_threshold = 800; + int64_t engine_gpu_search_threshold = 800; s = config.SetEngineConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigGpuSearchThreshold(int32_val); + s = config.GetEngineConfigGpuSearchThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == engine_gpu_search_threshold); + ASSERT_TRUE(int64_val == engine_gpu_search_threshold); - /* resource config */ - std::string resource_mode = "simple"; - s = config.SetResourceConfigMode(resource_mode); + /* gpu resource config */ + bool resource_enable_gpu = true; + s = config.SetGpuResourceConfigEnable(std::to_string(resource_enable_gpu)); ASSERT_TRUE(s.ok()); - s = config.GetResourceConfigMode(str_val); + s = config.GetGpuResourceConfigEnable(bool_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(str_val == resource_mode); + ASSERT_TRUE(bool_val == resource_enable_gpu); -#ifdef MILVUS_CPU_VERSION - std::vector search_resources = {"cpu"}; -#else - std::vector search_resources = {"cpu", "gpu0"}; -#endif - std::vector res_vec; - std::string res_str; +#ifdef MILVUS_GPU_VERSION + int64_t gpu_cache_capacity = 1; + s = config.SetGpuResourceConfigCacheCapacity(std::to_string(gpu_cache_capacity)); + ASSERT_TRUE(s.ok()); + s = config.GetGpuResourceConfigCacheCapacity(int64_val); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(int64_val == gpu_cache_capacity); + + float gpu_cache_threshold = 0.2; + s = config.SetGpuResourceConfigCacheThreshold(std::to_string(gpu_cache_threshold)); + ASSERT_TRUE(s.ok()); + s = config.GetGpuResourceConfigCacheThreshold(float_val); + ASSERT_TRUE(float_val == gpu_cache_threshold); + + std::vector search_resources = {"gpu0"}; + std::vector search_res_vec; + std::string search_res_str; milvus::server::StringHelpFunctions::MergeStringWithDelimeter( - search_resources, milvus::server::CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_str); - s = config.SetResourceConfigSearchResources(res_str); + search_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, search_res_str); + s = config.SetGpuResourceConfigSearchResources(search_res_str); ASSERT_TRUE(s.ok()); - s = config.GetResourceConfigSearchResources(res_vec); + s = config.GetGpuResourceConfigSearchResources(search_res_vec); ASSERT_TRUE(s.ok()); for (size_t i = 0; i < search_resources.size(); i++) { - ASSERT_TRUE(search_resources[i] == res_vec[i]); + ASSERT_TRUE(std::stoll(search_resources[i].substr(3)) == search_res_vec[i]); } -#ifdef MILVUS_CPU_VERSION - int32_t resource_index_build_device = milvus::server::CPU_DEVICE_ID; - s = config.SetResourceConfigIndexBuildDevice("cpu"); -#else - int32_t resource_index_build_device = 0; - s = config.SetResourceConfigIndexBuildDevice("gpu" + std::to_string(resource_index_build_device)); + std::vector build_index_resources = {"gpu0"}; + std::vector build_index_res_vec; + std::string build_index_res_str; + milvus::server::StringHelpFunctions::MergeStringWithDelimeter( + build_index_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, build_index_res_str); + s = config.SetGpuResourceConfigBuildIndexResources(build_index_res_str); + ASSERT_TRUE(s.ok()); + s = config.GetGpuResourceConfigBuildIndexResources(build_index_res_vec); + ASSERT_TRUE(s.ok()); + for (size_t i = 0; i < build_index_resources.size(); i++) { + ASSERT_TRUE(std::stoll(build_index_resources[i].substr(3)) == build_index_res_vec[i]); + } #endif - ASSERT_TRUE(s.ok()); - s = config.GetResourceConfigIndexBuildDevice(int32_val); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == resource_index_build_device); } TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { @@ -381,18 +377,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { s = config.SetCacheConfigCpuCacheThreshold("1.0"); ASSERT_FALSE(s.ok()); -#ifdef MILVUS_GPU_VERSION - s = config.SetCacheConfigGpuCacheCapacity("a"); - ASSERT_FALSE(s.ok()); - s = config.SetCacheConfigGpuCacheCapacity("128"); - ASSERT_FALSE(s.ok()); - - s = config.SetCacheConfigGpuCacheThreshold("a"); - ASSERT_FALSE(s.ok()); - s = config.SetCacheConfigGpuCacheThreshold("1.0"); - ASSERT_FALSE(s.ok()); -#endif - s = config.SetCacheConfigCacheInsertData("N"); ASSERT_FALSE(s.ok()); @@ -408,20 +392,29 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { s = config.SetEngineConfigGpuSearchThreshold("-1"); ASSERT_FALSE(s.ok()); - /* resource config */ - s = config.SetResourceConfigMode("default"); + /* gpu resource config */ + s = config.SetGpuResourceConfigEnable("ok"); ASSERT_FALSE(s.ok()); - s = config.SetResourceConfigSearchResources("gpu10"); +#ifdef MILVUS_GPU_VERSION + s = config.SetGpuResourceConfigCacheCapacity("a"); + ASSERT_FALSE(s.ok()); + s = config.SetGpuResourceConfigCacheCapacity("128"); ASSERT_FALSE(s.ok()); - s = config.SetResourceConfigSearchResources("cpu"); - ASSERT_TRUE(s.ok()); + s = config.SetGpuResourceConfigCacheThreshold("a"); + ASSERT_FALSE(s.ok()); + s = config.SetGpuResourceConfigCacheThreshold("1.0"); + ASSERT_FALSE(s.ok()); - s = config.SetResourceConfigIndexBuildDevice("gup2"); + s = config.SetGpuResourceConfigSearchResources("gpu10"); ASSERT_FALSE(s.ok()); - s = config.SetResourceConfigIndexBuildDevice("gpu16"); + + s = config.SetGpuResourceConfigBuildIndexResources("gup2"); ASSERT_FALSE(s.ok()); + s = config.SetGpuResourceConfigBuildIndexResources("gpu16"); + ASSERT_FALSE(s.ok()); +#endif } TEST_F(ConfigTest, SERVER_CONFIG_TEST) { @@ -438,4 +431,3 @@ TEST_F(ConfigTest, SERVER_CONFIG_TEST) { s = config.ResetDefaultConfig(); ASSERT_TRUE(s.ok()); } - diff --git a/core/unittest/server/test_rpc.cpp b/core/unittest/server/test_rpc.cpp index 76c10769a7..5753c68422 100644 --- a/core/unittest/server/test_rpc.cpp +++ b/core/unittest/server/test_rpc.cpp @@ -22,8 +22,8 @@ #include "server/Server.h" #include "server/grpc_impl/GrpcRequestHandler.h" #include "server/grpc_impl/GrpcRequestScheduler.h" -#include "server/grpc_impl/GrpcRequestTask.h" -#include "src/config.h" +#include "server/grpc_impl/request/GrpcBaseRequest.h" +#include "src/version.h" #include "grpc/gen-milvus/milvus.grpc.pb.h" #include "grpc/gen-status/status.pb.h" @@ -85,7 +85,6 @@ class RpcHandlerTest : public testing::Test { // DBWrapper::GetInstance().GetInstance().StartService(); // DBWrapper::GetInstance().GetInstance().StopService(); - milvus::server::Config::GetInstance().SetResourceConfigMode("single"); milvus::server::DBWrapper::GetInstance().StartService(); // initialize handler, create table @@ -410,7 +409,7 @@ TEST_F(RpcHandlerTest, PARTITION_TEST) { partition_parm.set_partition_name(partition_name); handler->DropPartition(&context, &partition_parm, &response); - ASSERT_EQ(response.error_code(), ::grpc::Status::OK.error_code()); + ASSERT_NE(response.error_code(), ::grpc::Status::OK.error_code()); } TEST_F(RpcHandlerTest, CMD_TEST) { @@ -453,20 +452,20 @@ TEST_F(RpcHandlerTest, DELETE_BY_RANGE_TEST) { ////////////////////////////////////////////////////////////////////// namespace { -class DummyTask : public milvus::server::grpc::GrpcBaseTask { +class DummyRequest : public milvus::server::grpc::GrpcBaseRequest { public: milvus::Status OnExecute() override { return milvus::Status::OK(); } - static milvus::server::grpc::BaseTaskPtr + static milvus::server::grpc::BaseRequestPtr Create(std::string& dummy) { - return std::shared_ptr(new DummyTask(dummy)); + return std::shared_ptr(new DummyRequest(dummy)); } public: - explicit DummyTask(std::string& dummy) : GrpcBaseTask(dummy) { + explicit DummyRequest(std::string& dummy) : GrpcBaseRequest(dummy) { } }; @@ -475,27 +474,27 @@ class RpcSchedulerTest : public testing::Test { void SetUp() override { std::string dummy = "dql"; - task_ptr = std::make_shared(dummy); + request_ptr = std::make_shared(dummy); } - std::shared_ptr task_ptr; + std::shared_ptr request_ptr; }; } // namespace TEST_F(RpcSchedulerTest, BASE_TASK_TEST) { - auto status = task_ptr->Execute(); + auto status = request_ptr->Execute(); ASSERT_TRUE(status.ok()); milvus::server::grpc::GrpcRequestScheduler::GetInstance().Start(); ::milvus::grpc::Status grpc_status; std::string dummy = "dql"; - milvus::server::grpc::BaseTaskPtr base_task_ptr = DummyTask::Create(dummy); - milvus::server::grpc::GrpcRequestScheduler::GetInstance().ExecTask(base_task_ptr, &grpc_status); + milvus::server::grpc::BaseRequestPtr base_task_ptr = DummyRequest::Create(dummy); + milvus::server::grpc::GrpcRequestScheduler::GetInstance().ExecRequest(base_task_ptr, &grpc_status); - milvus::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); - task_ptr = nullptr; - milvus::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); + milvus::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteRequest(request_ptr); + request_ptr = nullptr; + milvus::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteRequest(request_ptr); milvus::server::grpc::GrpcRequestScheduler::GetInstance().Stop(); } diff --git a/core/unittest/server/test_util.cpp b/core/unittest/server/test_util.cpp index 68400f2454..e5884cac65 100644 --- a/core/unittest/server/test_util.cpp +++ b/core/unittest/server/test_util.cpp @@ -60,7 +60,7 @@ TEST(UtilTest, COMMON_TEST) { ASSERT_GT(total_mem, 0); ASSERT_GT(free_mem, 0); - uint32_t thread_cnt = 0; + int64_t thread_cnt = 0; milvus::server::CommonUtil::GetSystemAvailableThreads(thread_cnt); ASSERT_GT(thread_cnt, 0); diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index 413db60e37..4019c0f63c 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -16,13 +16,13 @@ // under the License. #include "easyloggingpp/easylogging++.h" -#include "wrapper/VecIndex.h" #ifdef MILVUS_GPU_VERSION #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #endif #include "knowhere/index/vector_index/helpers/IndexParameter.h" +#include "wrapper/VecIndex.h" #include "wrapper/utils.h" #include @@ -57,10 +57,6 @@ class KnowhereWrapperTest index_ = GetVecIndexFactory(index_type); conf = ParamGenerator::GetInstance().GenBuild(index_type, tempconf); searchconf = ParamGenerator::GetInstance().GenSearchConf(index_type, tempconf); - - // conf->k = k; - // conf->d = dim; - // conf->gpu_id = DEVICEID; } void @@ -80,8 +76,7 @@ class KnowhereWrapperTest INSTANTIATE_TEST_CASE_P( WrapperParam, KnowhereWrapperTest, Values( -//["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"] - + //["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"] #ifdef MILVUS_GPU_VERSION std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_GPU, "Default", DIM, NB, 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_MIX, "Default", 64, 1000, 10, 10), @@ -98,6 +93,7 @@ INSTANTIATE_TEST_CASE_P( TEST_P(KnowhereWrapperTest, BASE_TEST) { EXPECT_EQ(index_->GetType(), index_type); + // conf->Dump(); auto elems = nq * k; std::vector res_ids(elems); @@ -192,3 +188,119 @@ TEST(whatever, test_config) { auto pq_conf = std::make_shared(); pq_conf->Match(conf); } + +// #include "knowhere/index/vector_index/IndexIDMAP.h" +// #include "src/wrapper/VecImpl.h" +// #include "src/index/unittest/utils.h" +// The two case below prove NSG is concern with data distribution +// Further work: 1. Use right basedata and pass it by milvus +// a. batch size is 100000 [Pass] +// b. transfer all at once [Pass] +// 2. Use SIFT1M in test and check time cost [] +// TEST_P(KnowhereWrapperTest, nsgwithidmap) { +// auto idmap = GetVecIndexFactory(milvus::engine::IndexType::FAISS_IDMAP); +// auto ori_xb = xb; +// auto ori_ids = ids; +// std::vector temp_xb; +// std::vector temp_ids; +// nb = 50000; +// for (int i = 0; i < 20; ++i) { +// GenData(dim, nb, nq, xb, xq, ids, k, gt_ids, gt_dis); +// assert(xb.size() == nb*dim); +// //#define IDMAP +// #ifdef IDMAP +// temp_xb.insert(temp_xb.end(), xb.data(), xb.data() + nb*dim); +// temp_ids.insert(temp_ids.end(), ori_ids.data()+nb*i, ori_ids.data() + nb*(i+1)); +// if (i == 0) { +// idmap->BuildAll(nb, temp_xb.data(), temp_ids.data(), conf); +// } else { +// idmap->Add(nb, temp_xb.data(), temp_ids.data()); +// } +// temp_xb.clear(); +// temp_ids.clear(); +// #else +// temp_xb.insert(temp_xb.end(), xb.data(), xb.data() + nb*dim); +// temp_ids.insert(temp_ids.end(), ori_ids.data()+nb*i, ori_ids.data() + nb*(i+1)); +// #endif +// } + +// #ifdef IDMAP +// auto idmap_idx = std::dynamic_pointer_cast(idmap); +// auto x = idmap_idx->Count(); +// index_->BuildAll(idmap_idx->Count(), idmap_idx->GetRawVectors(), idmap_idx->GetRawIds(), conf); +// #else +// assert(temp_xb.size() == 1000000*128); +// index_->BuildAll(1000000, temp_xb.data(), ori_ids.data(), conf); +// #endif +// } + +// TEST_P(KnowhereWrapperTest, nsgwithsidmap) { +// auto idmap = GetVecIndexFactory(milvus::engine::IndexType::FAISS_IDMAP); +// auto ori_xb = xb; +// std::vector temp_xb; +// std::vector temp_ids; +// nb = 50000; +// for (int i = 0; i < 20; ++i) { +// #define IDMAP +// #ifdef IDMAP +// temp_xb.insert(temp_xb.end(), ori_xb.data()+nb*dim*i, ori_xb.data() + nb*dim*(i+1)); +// temp_ids.insert(temp_ids.end(), ids.data()+nb*i, ids.data() + nb*(i+1)); +// if (i == 0) { +// idmap->BuildAll(nb, temp_xb.data(), temp_ids.data(), conf); +// } else { +// idmap->Add(nb, temp_xb.data(), temp_ids.data()); +// } +// temp_xb.clear(); +// temp_ids.clear(); +// #else +// temp_xb.insert(temp_xb.end(), ori_xb.data()+nb*dim*i, ori_xb.data() + nb*dim*(i+1)); +// temp_ids.insert(temp_ids.end(), ids.data()+nb*i, ids.data() + nb*(i+1)); +// #endif +// } + +// #ifdef IDMAP +// auto idmap_idx = std::dynamic_pointer_cast(idmap); +// auto x = idmap_idx->Count(); +// index_->BuildAll(idmap_idx->Count(), idmap_idx->GetRawVectors(), idmap_idx->GetRawIds(), conf); +// #else +// index_->BuildAll(1000000, temp_xb.data(), temp_ids.data(), conf); +// #endif + +// // The code use to store raw base data +// FileIOWriter writer("/tmp/newraw"); +// ori_xb.shrink_to_fit(); +// std::cout << "size" << ori_xb.size(); +// writer(static_cast(ori_xb.data()), ori_xb.size()* sizeof(float)); +// std::cout << "Finish!" << std::endl; +// } + +// void load_data(char* filename, float*& data, unsigned& num, +// unsigned& dim) { // load data with sift10K pattern +// std::ifstream in(filename, std::ios::binary); +// if (!in.is_open()) { +// std::cout << "open file error" << std::endl; +// exit(-1); +// } +// in.read((char*)&dim, 4); +// in.seekg(0, std::ios::end); +// std::ios::pos_type ss = in.tellg(); +// size_t fsize = (size_t)ss; +// num = (unsigned)(fsize / (dim + 1) / 4); +// data = new float[(size_t)num * (size_t)dim]; + +// in.seekg(0, std::ios::beg); +// for (size_t i = 0; i < num; i++) { +// in.seekg(4, std::ios::cur); +// in.read((char*)(data + i * dim), dim * 4); +// } +// in.close(); +// } + +// TEST_P(KnowhereWrapperTest, Sift1M) { +// float* data = nullptr; +// unsigned points_num, dim; +// load_data("/mnt/112d53a6-5592-4360-a33b-7fd789456fce/workspace/Data/sift/sift_base.fvecs", data, points_num, +// dim); std::cout << points_num << " " << dim << std::endl; + +// index_->BuildAll(points_num, data, ids.data(), conf); +// } diff --git a/core/unittest/wrapper/utils.cpp b/core/unittest/wrapper/utils.cpp index 00a1db2c88..96b9e643f5 100644 --- a/core/unittest/wrapper/utils.cpp +++ b/core/unittest/wrapper/utils.cpp @@ -117,6 +117,11 @@ void DataGenBase::GenData(const int& dim, const int& nb, const int& nq, std::vector& xb, std::vector& xq, std::vector& ids, const int& k, std::vector& gt_ids, std::vector& gt_dis) { + xb.clear(); + xq.clear(); + ids.clear(); + gt_ids.clear(); + gt_dis.clear(); xb.resize(nb * dim); xq.resize(nq * dim); ids.resize(nb); diff --git a/core/unittest/wrapper/utils.h b/core/unittest/wrapper/utils.h index 1eaa7ae4ec..05dc92f2d6 100644 --- a/core/unittest/wrapper/utils.h +++ b/core/unittest/wrapper/utils.h @@ -25,7 +25,6 @@ #include #include - #include "wrapper/VecIndex.h" #include "wrapper/utils.h" #include "knowhere/index/vector_index/helpers/IndexParameter.h" @@ -90,17 +89,20 @@ class ParamGenerator { return instance; } - knowhere::Config GenSearchConf(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { + knowhere::Config + GenSearchConf(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type); return adapter->MatchSearch(conf, type); } - knowhere::Config GenBuild(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { + knowhere::Config + GenBuild(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type); return adapter->Match(conf); } - knowhere::Config Gen(const milvus::engine::IndexType& type) { + knowhere::Config + Gen(const milvus::engine::IndexType& type) { switch (type) { case milvus::engine::IndexType::FAISS_IDMAP: { auto tempconf = std::make_shared(); diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000..64409d8240 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,11 @@ +# Docs + +This repository contains test reports on the search performance of different index types on standalone Milvus. + +The tests are run on [SIFT1B dataset](http://corpus-texmex.irisa.fr/), and provide results on the following measures: + +- Query Elapsed Time: Time cost (in seconds) to run a query. + +- Recall: The fraction of the total amount of relevant instances that were actually retrieved. + +Test variables are `nq` and `topk`. diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md new file mode 100644 index 0000000000..7bea764df0 --- /dev/null +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md @@ -0,0 +1,206 @@ +# milvus_ivfsq8_test_report_detailed_version + +## Summary + +This document contains the test reports of IVF_SQ8 index on Milvus single server. + + + +## Test objectives + +The time cost and recall when searching with different parameters. + + + +## Test method + +### Hardware/Software requirements + +Operating System: CentOS Linux release 7.6.1810 (Core) + +CPU: Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz + +GPU0: GeForce GTX 1080 + +GPU1: GeForce GTX 1080 + +Memory: 503GB + +Docker version: 18.09 + +NVIDIA Driver version: 430.34 + +Milvus version: 0.5.3 + +SDK interface: Python 3.6.8 + +pymilvus version: 0.2.5 + + + +### Data model + +The data used in the tests are: + +- Data source: sift1b +- Data type: hdf5 + +For details on this dataset, you can check : http://corpus-texmex.irisa.fr/ . + + + +### Measures + +- Query Elapsed Time: Time cost (in seconds) to run a query. Variables that affect Query Elapsed Time: + + - nq (Number of queried vectors) + + > Note: In the query test of query elapsed time, we will test the following parameters with different values: + > + > nq - grouped by: [1, 5, 10, 200, 400, 600, 800, 1000], + +- Recall: The fraction of the total amount of relevant instances that were actually retrieved . Variables that affect Recall: + + - nq (Number of queried vectors) + - topk (Top k result of a query) + + > Note: In the query test of recall, we will test the following parameters with different values: + > + > nq - grouped by: [10, 200, 400, 600, 800, 1000], + > + > topk - grouped by: [1, 10, 100] + + + +## Test reports + +### Test environment + +Data base: sift1b-1,000,000,000 vectors, 128-dimension + +Table Attributes + +- nlist: 16384 +- metric_type: L2 + +Query configuration + +- nprobe: 32 + +Milvus configuration + +- cpu_cache_capacity: 150 +- gpu_cache_capacity: 6 +- use_blas_threshold: 1100 + +The definitions of Milvus configuration are on https://milvus.io/docs/en/reference/milvus_config/. + +Test method + +Test the query elapsed time and recall with several parameters, and once only change one parameter. + +- Whether to restart Milvus after each query: No + + + +### Performance test + +#### Data query + +**Test result** + +Query Elapsed Time + +topk : 100 + +search_resources: gpu0, gpu1 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 15.57 | +| nq=10 | 15.80 | +| nq=200 | 15.72 | +| nq=400 | 15.94 | +| nq=600 | 16.58 | +| nq=800 | 16.71 | +| nq=1000 | 16.91 | + +When nq is 1000, the query time cost of a 128-dimension vector is around 17ms in GPU Mode. + + + +topk : 100 + +search_resources: cpu, gpu0 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 1.12 | +| nq=10 | 2.89 | +| nq=200 | 8.10 | +| nq=400 | 12.36 | +| nq=600 | 17.81 | +| nq=800 | 23.24 | +| nq=1000 | 27.41 | + +When nq is 1000, the query time cost of a 128-dimension vector is around 27ms in CPU Mode. + + + +**Conclusion** + +The query elapsed time in CPU Mode increases quickly with nq, while in GPU Mode query elapsed time increases much slower. When nq is small, CPU Mode consumes less time than GPU Mode. However, as nq becomes larger, GPU Mode shows its advantage against CPU Mode. + +The query elapsed time in GPU Mode consists of two parts: (1) index CPU-to-GPU copy time; (2) nprobe buckets search time. When nq is smaller than 500, index CPU-to-GPU copy time cannot be amortized efficiently, CPU Mode is a better choice; when nq is larger than 500, choosing GPU Mode is better. + +Compared with CPU, GPU has much more cores and stronger computing capability. When nq is large, it can better reflect GPU's advantages on computing. + + + +### Recall test + +**Test result** + +topk = 1 : recall - recall@1 + +topk = 10 : recall - recall@10 + +topk = 100 : recall - recall@100 + +We use the ground_truth in sift1b dataset to calculate the recall of query results. + + + +Recall of GPU Mode + +search_resources: gpu0, gpu1 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | + + + +Recall of CPU Mode + +search_resources: cpu, gpu0 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | + + + +**Conclusion** + +As nq increases, the recall gradually stabilizes to over 93%. \ No newline at end of file diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md new file mode 100644 index 0000000000..a6e5e75ea4 --- /dev/null +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md @@ -0,0 +1,207 @@ +# milvus_ivfsq8_test_report_detailed_version_cn + +## 概述 + +本文描述了ivfsq8索引在milvus单机部署方式下的测试结果。 + + + +## 测试目标 + +参数不同情况下的查询时间和召回率。 + + + +## 测试方法 + +### 软硬件环境 + +操作系统: CentOS Linux release 7.6.1810 (Core) + +CPU: Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz + +GPU0: GeForce GTX 1080 + +GPU1: GeForce GTX 1080 + +内存: 503GB + +Docker版本: 18.09 + +NVIDIA Driver版本: 430.34 + +Milvus版本: 0.5.3 + +SDK接口: Python 3.6.8 + +pymilvus版本: 0.2.5 + + + +### 数据模型 + +本测试中用到的主要数据: + +- 数据来源: sift1b +- 数据类型: hdf5 + +关于该数据集的详细信息请参考 : http://corpus-texmex.irisa.fr/ 。 + + + +### 测试指标 + +- Query Elapsed Time: 数据库查询所有向量的时间（以秒计）。影响Query Elapsed Time的变量: + + - nq (被查询向量的数量) + + > 备注：在向量查询测试中，我们会测试下面参数不同的取值来观察结果： + > + > 被查询向量的数量nq将按照 [1, 5, 10, 200, 400, 600, 800, 1000]的数量分组。 + +- Recall: 实际返回的正确结果占总数之比 . 影响Recall的变量: + + - nq (被查询向量的数量) + - topk (单条查询中最相似的K个结果) + + > 备注：在向量准确性测试中，我们会测试下面参数不同的取值来观察结果： + > + > 被查询向量的数量nq将按照 [10, 200, 400, 600, 800, 1000]的数量分组， + > + > 单条查询中最相似的K个结果topk将按照[1, 10, 100]的数量分组。 + + + +## 测试报告 + +### 测试环境 + +数据集: sift1b-1,000,000,000向量, 128维 + +表格属性： + +- nlist: 16384 +- metric_type: L2 + +查询设置： + +- nprobe: 32 + +Milvus设置： + +- cpu_cache_capacity: 150 +- gpu_cache_capacity: 6 +- use_blas_threshold: 1100 + +Milvus设置的详细定义可以参考 https://milvus.io/docs/en/reference/milvus_config/ 。 + +测试方法 + +通过一次仅改变一个参数的值，测试查询向量时间和召回率。 + +- 查询后是否重启Milvus：否 + + + +### 性能测试 + +#### 数据查询 + +测试结果 + +Query Elapsed Time + +topk : 100 + +search_resources: gpu0, gpu1 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 15.57 | +| nq=10 | 15.80 | +| nq=200 | 15.72 | +| nq=400 | 15.94 | +| nq=600 | 16.58 | +| nq=800 | 16.71 | +| nq=1000 | 16.91 | + +当nq为1000时，在GPU模式下查询一条128维向量需要耗时约17毫秒。 + + + +topk : 100 + +search_resources: cpu, gpu0 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 1.12 | +| nq=10 | 2.89 | +| nq=200 | 8.10 | +| nq=400 | 12.36 | +| nq=600 | 17.81 | +| nq=800 | 23.24 | +| nq=1000 | 27.41 | + +当nq为1000时，在GPU模式下查询一条128维向量需要耗时约27毫秒。 + + + +**总结** + +在CPU模式下查询耗时随nq的增长快速增大，而在GPU模式下查询耗时的增大则缓慢许多。当nq较小时，CPU模式比GPU模式耗时更少。但当nq足够大时，GPU模式则更具有优势。 + +在GPU模式下的查询耗时由两部分组成：（1）索引从CPU到GPU的拷贝时间；（2）所有分桶的查询时间。当nq小于500时，索引从CPU到GPU 的拷贝时间无法被有效均摊，此时CPU模式时一个更优的选择；当nq大于500时，选择GPU模式更合理。 + +和CPU相比，GPU具有更多的核数和更强的算力。当nq较大时，GPU在计算上的优势能被更好地被体现。 + + + +### 召回率测试 + +**测试结果** + +topk = 1 : recall - recall@1 + +topk = 10 : recall - recall@10 + +topk = 100 : recall - recall@100 + +我们利用sift1b数据集中的ground_truth来计算查询结果的召回率。 + + + +Recall of GPU Mode + +search_resources: gpu0, gpu1 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | + + + +Recall of CPU Mode + +search_resources: cpu, gpu0 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | + + + +**总结** + +随着nq的增大，召回率逐渐稳定至93%以上。 + diff --git a/docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md new file mode 100755 index 0000000000..042d059684 --- /dev/null +++ b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md @@ -0,0 +1,179 @@ +# milvus_ivfsq8h_test_report_detailed_version + +## Summary + +This document contains the test reports of IVF_SQ8H index on Milvus single server. + + + +## Test objectives + +The time cost and recall when searching with different parameters. + + + +## Test method + +### Hardware/Software requirements + +Operating System: CentOS Linux release 7.6.1810 (Core) + +CPU: Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz + +GPU0: GeForce GTX 1080 + +GPU1: GeForce GTX 1080 + +Memory: 503GB + +Docker version: 18.09 + +NVIDIA Driver version: 430.34 + +Milvus version: 0.5.3 + +SDK interface: Python 3.6.8 + +pymilvus version: 0.2.5 + + + +### Data model + +The data used in the tests are: + +- Data source: sift1b +- Data type: hdf5 + +For details on this dataset, please check : http://corpus-texmex.irisa.fr/ . + + + +### Measures + +- Query Elapsed Time: Time cost (in seconds) to run a query. Variables that affect Query Elapsed Time: + + - nq (Number of queried vectors) + +> Note: In the query test of query elapsed time, we will test the following parameters with different values: + > + > nq - grouped by: [1, 5, 10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800]. + > + +- Recall: The fraction of the total amount of relevant instances that were actually retrieved . Variables that affect Recall: + + - nq (Number of queried vectors) + - topk (Top k result of a query) + + > Note: In the query test of recall, we will test the following parameters with different values: + > + > nq - grouped by: [10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800], + > + > topk - grouped by: [1, 10, 100] + + + +## Test reports + +### Test environment + +Data base: sift1b-1,000,000,000 vectors, 128-dimension + +Table Attributes + +- nlist: 16384 +- metric_type: L2 + +Query configuration + +- nprobe: 32 + +Milvus configuration + +- cpu_cache_capacity: 150 +- gpu_cache_capacity: 6 +- use_blas_threshold: 1100 +- gpu_search_threshold: 1200 +- search_resources: cpu, gpu0, gpu1 + +The definitions of Milvus configuration are on https://milvus.io/docs/en/reference/milvus_config/. + +Test method + +Test the query elapsed time and recall with several parameters, and once only change one parameter. + +- Whether to restart Milvus after each query: No + + + +### Performance test + +#### Data query + +**Test result** + +Query Elapsed Time + +topk = 100 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 0.34 | +| nq=5 | 0.72 | +| nq=10 | 0.91 | +| nq=50 | 1.51 | +| nq=100 | 2.49 | +| nq=200 | 4.09 | +| nq=400 | 7.32 | +| nq=600 | 10.63 | +| nq=800 | 13.84 | +| nq=1000 | 16.83 | +| nq=1200 | 18.20 | +| nq=1400 | 20.1 | +| nq=1600 | 20.0 | +| nq=1800 | 19.86 | + +When nq is 1800, the query time cost of a 128-dimension vector is around 11ms. + + + +**Conclusion** + +When nq < 1200, the query elapsed time increases quickly with nq; when nq > 1200, the query elapsed time increases much slower. It is because gpu_search_threshold is set to 1200, when nq < 1200, CPU is chosen to do the query, otherwise GPU is chosen. Compared with CPU, GPU has much more cores and stronger computing capability. When nq is large, it can better reflect GPU's advantages on computing. + +The query elapsed time consists of two parts: (1) index CPU-to-GPU copy time; (2) nprobe buckets search time. When nq is larger enough, index CPU-to-GPU copy time can be amortized efficiently. So Milvus performs well through setting suitable gpu_search_threshold. + + + +### Recall test + +**Test result** + +topk = 1 : recall - recall@1 + +topk = 10 : recall - recall@10 + +topk = 100 : recall - recall@100 + +We use the ground_truth in sift1b dataset to calculate the recall of query results. + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=50 | 0.980 | 0.950 | 0.941 | +| nq=100 | 0.970 | 0.937 | 0.931 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | +| nq=1200 | 0.937 | 0.943 | 0.931 | +| nq=1400 | 0.939 | 0.945 | 0.931 | +| nq=1600 | 0.936 | 0.945 | 0.931 | +| nq=1800 | 0.937 | 0.946 | 0.932 | + + + +**Conclusion** + +As nq increases, the recall gradually stabilizes to over 93%. The usage of CPU or GPU and different topk are not related to recall. \ No newline at end of file diff --git a/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md new file mode 100644 index 0000000000..b50d00f9bd --- /dev/null +++ b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md @@ -0,0 +1,180 @@ +# milvus_ivfsq8h_test_report_detailed_version_cn + +## 概述 + +本文描述了ivfsq8h索引在milvus单机部署方式下的测试结果。 + + + +## 测试目标 + +参数不同情况下的查询时间和召回率。 + + + +## 测试方法 + +### 软硬件环境 + +操作系统：CentOS Linux release 7.6.1810 (Core) + +CPU：Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz + +GPU0：GeForce GTX 1080 + +GPU1：GeForce GTX 1080 + +内存：503GB + +Docker版本：18.09 + +NVIDIA Driver版本：430.34 + +Milvus版本：0.5.3 + +SDK接口：Python 3.6.8 + +pymilvus版本：0.2.5 + + + +### 数据模型 + +本测试中用到的主要数据: + +- 数据来源：sift1b +- 数据类型：hdf5 + +关于该数据集的详细信息请参考：http://corpus-texmex.irisa.fr/ 。 + + + +### 测试指标 + +- Query Elapsed Time: 数据库查询所有向量的时间（以秒计）。影响Query Elapsed Time的变量： + + - nq (被查询向量的数量) + +> 备注：在向量查询测试中，我们会测试下面参数不同的取值来观察结果： + > + > 被查询向量的数量nq将按照 [1, 5, 10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800]的数量分组。 + > + +- Recall: 实际返回的正确结果占总数之比。影响Recall的变量： + + - nq (被查询向量的数量) + - topk (单条查询中最相似的K个结果) + + > 备注：在向量准确性测试中，我们会测试下面参数不同的取值来观察结果： + > + > 被查询向量的数量nq将按照 [10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800]的数量分组， + > + > 单条查询中最相似的K个结果topk将按照[1, 10, 100]的数量分组。 + + + +## 测试报告 + +### 测试环境 + +数据集: sift1b-1,000,000,000向量, 128维 + +表格属性： + +- nlist: 16384 +- metric_type: L2 + +查询设置： + +- nprobe: 32 + +Milvus设置： + +- cpu_cache_capacity: 150 +- gpu_cache_capacity: 6 +- use_blas_threshold: 1100 +- gpu_search_threshold: 1200 +- search_resources: cpu, gpu0, gpu1 + +Milvus设置的详细定义可以参考 https://milvus.io/docs/en/reference/milvus_config/ 。 + +测试方法 + +通过一次仅改变一个参数的值，测试查询向量时间和召回率。 + +- 查询后是否重启Milvus：否 + + + +### 性能测试 + +#### 数据查询 + +测试结果 + +Query Elapsed Time + +topk = 100 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 0.34 | +| nq=5 | 0.72 | +| nq=10 | 0.91 | +| nq=50 | 1.51 | +| nq=100 | 2.49 | +| nq=200 | 4.09 | +| nq=400 | 7.32 | +| nq=600 | 10.63 | +| nq=800 | 13.84 | +| nq=1000 | 16.83 | +| nq=1200 | 18.20 | +| nq=1400 | 20.1 | +| nq=1600 | 20.0 | +| nq=1800 | 19.86 | + +当nq为1800时，查询一条128维向量需要耗时约11毫秒。 + + + +**总结** + +当nq小于1200时，查询耗时随nq的增长快速增大；当nq大于1200时，查询耗时的增大则缓慢许多。这是因为gpu_search_threshold这一参数的值被设为1200，当nq<1200时，选择CPU进行操作，否则选择GPU进行操作。与CPU。 + +在GPU模式下的查询耗时由两部分组成：（1）索引从CPU到GPU的拷贝时间；（2）所有分桶的查询时间。当nq小于500时，索引从CPU到GPU 的拷贝时间无法被有效均摊，此时CPU模式时一个更优的选择；当nq大于500时，选择GPU模式更合理。和CPU相比，GPU具有更多的核数和更强的算力。当nq较大时，GPU在计算上的优势能被更好地被体现。 + + + +### 召回率测试 + +**测试结果** + +topk = 1 : recall - recall@1 + +topk = 10 : recall - recall@10 + +topk = 100 : recall - recall@100 + +我们利用sift1b数据集中的ground_truth来计算查询结果的召回率。 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=50 | 0.980 | 0.950 | 0.941 | +| nq=100 | 0.970 | 0.937 | 0.931 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | +| nq=1200 | 0.937 | 0.943 | 0.931 | +| nq=1400 | 0.939 | 0.945 | 0.931 | +| nq=1600 | 0.936 | 0.945 | 0.931 | +| nq=1800 | 0.937 | 0.946 | 0.932 | + + + +**总结** + +随着nq的增大，召回率逐渐稳定至93%以上。CPU/GPU的使用以及topk的值与召回率的大小无关。 + diff --git a/install.md b/install.md index 6711b41f76..f53586af36 100644 --- a/install.md +++ b/install.md @@ -3,6 +3,9 @@ ## Software requirements - Ubuntu 18.04 or higher + + If your operating system is not Ubuntu 18.04 or higher, we recommend you to pull a [docker image of Ubuntu 18.04](https://docs.docker.com/install/linux/docker-ce/ubuntu/) as your compilation environment. + - CMake 3.12 or higher ##### For GPU version, you will also need: diff --git a/shards/mishards/__init__.py b/shards/mishards/__init__.py index a3c55c4ae3..55594220d3 100644 --- a/shards/mishards/__init__.py +++ b/shards/mishards/__init__.py @@ -11,7 +11,9 @@ grpc_server = Server() def create_app(testing_config=None): config = testing_config if testing_config else settings.DefaultConfig - db.init_db(uri=config.SQLALCHEMY_DATABASE_URI, echo=config.SQL_ECHO) + db.init_db(uri=config.SQLALCHEMY_DATABASE_URI, echo=config.SQL_ECHO, pool_size=config.SQL_POOL_SIZE, + pool_recycle=config.SQL_POOL_RECYCLE, pool_timeout=config.SQL_POOL_TIMEOUT, + pool_pre_ping=config.SQL_POOL_PRE_PING, max_overflow=config.SQL_MAX_OVERFLOW) from mishards.connections import ConnectionMgr connect_mgr = ConnectionMgr() diff --git a/shards/mishards/connections.py b/shards/mishards/connections.py index 618690a099..50e214ec9a 100644 --- a/shards/mishards/connections.py +++ b/shards/mishards/connections.py @@ -2,6 +2,7 @@ import logging import threading from functools import wraps from milvus import Milvus +from milvus.client.hooks import BaseaSearchHook from mishards import (settings, exceptions) from utils import singleton @@ -9,6 +10,12 @@ from utils import singleton logger = logging.getLogger(__name__) +class Searchook(BaseaSearchHook): + + def on_response(self, *args, **kwargs): + return True + + class Connection: def __init__(self, name, uri, max_retry=1, error_handlers=None, **kwargs): self.name = name @@ -18,6 +25,9 @@ class Connection: self.conn = Milvus() self.error_handlers = [] if not error_handlers else error_handlers self.on_retry_func = kwargs.get('on_retry_func', None) + + # define search hook + self.conn._set_hook(search_in_file=Searchook()) # self._connect() def __str__(self): diff --git a/shards/mishards/db_base.py b/shards/mishards/db_base.py index 5f2eee9ba1..e55c330352 100644 --- a/shards/mishards/db_base.py +++ b/shards/mishards/db_base.py @@ -23,15 +23,12 @@ class DB: uri and self.init_db(uri, echo) self.session_factory = scoped_session(sessionmaker(class_=LocalSession, db=self)) - def init_db(self, uri, echo=False): + def init_db(self, uri, echo=False, pool_size=100, pool_recycle=5, pool_timeout=30, pool_pre_ping=True, max_overflow=0): url = make_url(uri) if url.get_backend_name() == 'sqlite': self.engine = create_engine(url) else: - self.engine = create_engine(uri, pool_size=100, pool_recycle=5, pool_timeout=30, - pool_pre_ping=True, - echo=echo, - max_overflow=0) + self.engine = create_engine(uri, pool_size, pool_recycle, pool_timeout, pool_pre_ping, echo, max_overflow) self.uri = uri self.url = url diff --git a/shards/mishards/service_handler.py b/shards/mishards/service_handler.py index 2f19152ae6..fc0ee0fa2b 100644 --- a/shards/mishards/service_handler.py +++ b/shards/mishards/service_handler.py @@ -29,39 +29,71 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): self.router = router self.max_workers = max_workers + def _reduce(self, source_ids, ids, source_diss, diss, k, reverse): + if source_diss[k - 1] <= diss[0]: + return source_ids, source_diss + if diss[k - 1] <= source_diss[0]: + return ids, diss + + source_diss.extend(diss) + diss_t = enumerate(source_diss) + diss_m_rst = sorted(diss_t, key=lambda x: x[1])[:k] + diss_m_out = [id_ for _, id_ in diss_m_rst] + + source_ids.extend(ids) + id_m_out = [source_ids[i] for i, _ in diss_m_rst] + + return id_m_out, diss_m_out + def _do_merge(self, files_n_topk_results, topk, reverse=False, **kwargs): status = status_pb2.Status(error_code=status_pb2.SUCCESS, reason="Success") if not files_n_topk_results: return status, [] - request_results = defaultdict(list) + merge_id_results = [] + merge_dis_results = [] calc_time = time.time() for files_collection in files_n_topk_results: if isinstance(files_collection, tuple): status, _ = files_collection return status, [] - for request_pos, each_request_results in enumerate( - files_collection.topk_query_result): - request_results[request_pos].extend( - each_request_results.query_result_arrays) - request_results[request_pos] = sorted( - request_results[request_pos], - key=lambda x: x.distance, - reverse=reverse)[:topk] + + row_num = files_collection.row_num + ids = files_collection.ids + diss = files_collection.distances # distance collections + # TODO: batch_len is equal to topk, may need to compare with topk + batch_len = len(ids) // row_num + + for row_index in range(row_num): + id_batch = ids[row_index * batch_len: (row_index + 1) * batch_len] + dis_batch = diss[row_index * batch_len: (row_index + 1) * batch_len] + + if len(merge_id_results) < row_index: + raise ValueError("merge error") + elif len(merge_id_results) == row_index: + # TODO: may bug here + merge_id_results.append(id_batch) + merge_dis_results.append(dis_batch) + else: + merge_id_results[row_index], merge_dis_results[row_index] = \ + self._reduce(merge_id_results[row_index], id_batch, + merge_dis_results[row_index], dis_batch, + batch_len, + reverse) calc_time = time.time() - calc_time logger.info('Merge takes {}'.format(calc_time)) - results = sorted(request_results.items()) - topk_query_result = [] + id_mrege_list = [] + dis_mrege_list = [] - for result in results: - query_result = TopKQueryResult(query_result_arrays=result[1]) - topk_query_result.append(query_result) + for id_results, dis_results in zip(merge_id_results, merge_dis_results): + id_mrege_list.extend(id_results) + dis_mrege_list.extend(dis_results) - return status, topk_query_result + return status, id_mrege_list, dis_mrege_list def _do_query(self, context, @@ -109,8 +141,8 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): file_ids=query_params['file_ids'], query_records=vectors, top_k=topk, - nprobe=nprobe, - lazy_=True) + nprobe=nprobe + ) end = time.time() logger.info('search_vectors_in_files takes: {}'.format(end - start)) @@ -241,7 +273,7 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): logger.info('Search {}: topk={} nprobe={}'.format( table_name, topk, nprobe)) - metadata = {'resp_class': milvus_pb2.TopKQueryResultList} + metadata = {'resp_class': milvus_pb2.TopKQueryResult} if nprobe > self.MAX_NPROBE or nprobe <= 0: raise exceptions.InvalidArgumentError( @@ -275,22 +307,24 @@ class ServiceHandler(milvus_pb2_grpc.MilvusServiceServicer): query_range_array.append( Range(query_range.start_value, query_range.end_value)) - status, results = self._do_query(context, - table_name, - table_meta, - query_record_array, - topk, - nprobe, - query_range_array, - metadata=metadata) + status, id_results, dis_results = self._do_query(context, + table_name, + table_meta, + query_record_array, + topk, + nprobe, + query_range_array, + metadata=metadata) now = time.time() logger.info('SearchVector takes: {}'.format(now - start)) - topk_result_list = milvus_pb2.TopKQueryResultList( + topk_result_list = milvus_pb2.TopKQueryResult( status=status_pb2.Status(error_code=status.error_code, reason=status.reason), - topk_query_result=results) + row_num=len(query_record_array), + ids=id_results, + distances=dis_results) return topk_result_list @mark_grpc_method diff --git a/shards/mishards/settings.py b/shards/mishards/settings.py index 8d7361dddc..832f1639ea 100644 --- a/shards/mishards/settings.py +++ b/shards/mishards/settings.py @@ -50,10 +50,16 @@ class TracingConfig: } } + max_overflow=0 class DefaultConfig: SQLALCHEMY_DATABASE_URI = env.str('SQLALCHEMY_DATABASE_URI') SQL_ECHO = env.bool('SQL_ECHO', False) + SQL_POOL_SIZE = env.int('pool_size', 100) + SQL_POOL_RECYCLE = env.int('pool_recycle', 5) + SQL_POOL_TIMEOUT = env.int('pool_timeout', 30) + SQL_POOL_PRE_PING = env.bool('pool_pre_ping', True) + SQL_MAX_OVERFLOW = env.int('max_overflow', 0) TRACER_PLUGIN_PATH = env.str('TRACER_PLUGIN_PATH', '') TRACER_CLASS_NAME = env.str('TRACER_CLASS_NAME', '') ROUTER_PLUGIN_PATH = env.str('ROUTER_PLUGIN_PATH', '') @@ -65,5 +71,10 @@ class DefaultConfig: class TestingConfig(DefaultConfig): SQLALCHEMY_DATABASE_URI = env.str('SQLALCHEMY_DATABASE_TEST_URI', '') SQL_ECHO = env.bool('SQL_TEST_ECHO', False) + SQL_POOL_SIZE = env.int('pool_size', 100) + SQL_POOL_RECYCLE = env.int('pool_recycle', 5) + SQL_POOL_TIMEOUT = env.int('pool_timeout', 30) + SQL_POOL_PRE_PING = env.bool('pool_pre_ping', True) + SQL_MAX_OVERFLOW = env.int('max_overflow', 0) TRACER_CLASS_NAME = env.str('TRACER_CLASS_TEST_NAME', '') ROUTER_CLASS_NAME = env.str('ROUTER_CLASS_TEST_NAME', 'FileBasedHashRingRouter') diff --git a/shards/requirements.txt b/shards/requirements.txt index 14bdde2a06..426ee0b704 100644 --- a/shards/requirements.txt +++ b/shards/requirements.txt @@ -14,8 +14,7 @@ py==1.8.0 pyasn1==0.4.7 pyasn1-modules==0.2.6 pylint==2.3.1 -pymilvus-test==0.2.28 -#pymilvus==0.2.0 +pymilvus==0.2.5 pyparsing==2.4.0 pytest==4.6.3 pytest-level==0.1.1 diff --git a/tests/milvus_python_test/conftest.py b/tests/milvus_python_test/conftest.py index 8bab824606..dc58063a45 100644 --- a/tests/milvus_python_test/conftest.py +++ b/tests/milvus_python_test/conftest.py @@ -42,16 +42,22 @@ def connect(request): port = request.config.getoption("--port") milvus = Milvus() try: - milvus.connect(host=ip, port=port) - except: + status = milvus.connect(host=ip, port=port) + logging.getLogger().info(status) + if not status.OK(): + # try again + logging.getLogger().info("------------------------------------") + logging.getLogger().info("Try to connect again") + logging.getLogger().info("------------------------------------") + res = milvus.connect(host=ip, port=port) + except Exception as e: + logging.getLogger().error(str(e)) pytest.exit("Milvus server can not connected, exit pytest ...") - def fin(): try: milvus.disconnect() except: pass - request.addfinalizer(fin) return milvus @@ -129,4 +135,4 @@ def ip_table(request, connect): request.addfinalizer(teardown) - return table_name \ No newline at end of file + return table_name diff --git a/tests/milvus_python_test/test_connect.py b/tests/milvus_python_test/test_connect.py index 96ce1d3bdf..dd7e80c1f9 100644 --- a/tests/milvus_python_test/test_connect.py +++ b/tests/milvus_python_test/test_connect.py @@ -249,8 +249,8 @@ class TestConnect: ''' milvus = Milvus() uri_value = "tcp://%s:19540" % args["ip"] - milvus.connect(host=args["ip"], port="", uri=uri_value) - assert milvus.connected() + with pytest.raises(Exception) as e: + milvus.connect(host=args["ip"], port="", uri=uri_value) def test_connect_param_priority_uri(self, args): '''