diff --git a/CHANGELOG.md b/CHANGELOG.md index fcd51d1f45..42d5b8978f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,8 @@ Please mark all change in change log and use the ticket from JIRA. - \#322 - Add option to enable / disable prometheus - \#358 - Add more information in build.sh and install.md - \#255 - Add ivfsq8 test report detailed version +- \#404 - Add virtual method Init() in Pass abstract class +- \#409 - Add a Fallback pass in optimizer ## Task diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index f1daaf22ec..7b479ff44a 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -3,11 +3,7 @@ sh 'helm repo update' dir ('milvus-helm') { checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) dir ("milvus") { - if ("${env.BINRARY_VERSION}" == "gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - } else { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/filebeat/values.yaml --namespace milvus ." - } + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/sqlite_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index d357badfd3..e27bc6c0a5 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -13,11 +13,7 @@ timeout(time: 90, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus") { - if ("${env.BINRARY_VERSION}" == "gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - } else { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - } + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } dir ("tests/milvus_python_test") { diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index c1de5907b0..08812ae01f 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -14,11 +14,7 @@ timeout(time: 60, unit: 'MINUTES') { // } // dir ("milvus-helm") { // dir ("milvus") { - // if ("${env.BINRARY_VERSION}" == "gpu") { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - // } else { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - // } + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." // } // } // dir ("tests/milvus_python_test") { diff --git a/core/src/scheduler/SchedInst.cpp b/core/src/scheduler/SchedInst.cpp index 97c51407c5..867052ef5d 100644 --- a/core/src/scheduler/SchedInst.cpp +++ b/core/src/scheduler/SchedInst.cpp @@ -91,6 +91,7 @@ load_simple_config() { void StartSchedulerService() { load_simple_config(); + OptimizerInst::GetInstance()->Init(); ResMgrInst::GetInstance()->Start(); SchedInst::GetInstance()->Start(); JobMgrInst::GetInstance()->Start(); diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index 102dc17b1d..48eaffc0a5 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -23,6 +23,7 @@ #include "Scheduler.h" #include "Utils.h" #include "optimizer/BuildIndexPass.h" +#include "optimizer/FallbackPass.h" #include "optimizer/HybridPass.h" #include "optimizer/LargeSQ8HPass.h" #include "optimizer/OnlyCPUPass.h" @@ -103,12 +104,14 @@ class OptimizerInst { pass_list.push_back(std::make_shared()); #ifdef MILVUS_CPU_VERSION pass_list.push_back(std::make_shared()); +<<<<<<< HEAD #else server::Config& config = server::Config::GetInstance(); std::vector build_resources; config.GetGpuResourceConfigBuildIndexResources(build_resources); pass_list.push_back(std::make_shared(build_resources)); #endif +// pass_list.push_back(std::make_shared()); instance = std::make_shared(pass_list); } } diff --git a/core/src/scheduler/optimizer/BuildIndexPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp index dea4d9d5c7..4e7d5222e2 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.cpp +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -26,6 +26,10 @@ namespace scheduler { BuildIndexPass::BuildIndexPass(std::vector& build_gpu_ids) : build_gpu_ids_(build_gpu_ids) { } +void +BuildIndexPass::Init() { +} + bool BuildIndexPass::Run(const TaskPtr& task) { if (task->Type() != TaskType::BuildIndexTask) diff --git a/core/src/scheduler/optimizer/BuildIndexPass.h b/core/src/scheduler/optimizer/BuildIndexPass.h index 5b04ec1dce..a3eb426527 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.h +++ b/core/src/scheduler/optimizer/BuildIndexPass.h @@ -37,6 +37,9 @@ class BuildIndexPass : public Pass { explicit BuildIndexPass(std::vector& build_gpu_id); public: + void + Init() override; + bool Run(const TaskPtr& task) override; diff --git a/core/src/scheduler/optimizer/FallbackPass.cpp b/core/src/scheduler/optimizer/FallbackPass.cpp new file mode 100644 index 0000000000..2e275ede4b --- /dev/null +++ b/core/src/scheduler/optimizer/FallbackPass.cpp @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/FallbackPass.h" +#include "scheduler/SchedInst.h" +#include "scheduler/tasklabel/SpecResLabel.h" + +namespace milvus { +namespace scheduler { + +void +FallbackPass::Init() { +} + +bool +FallbackPass::Run(const TaskPtr& task) { + auto task_type = task->Type(); + if (task_type != TaskType::SearchTask && task_type != TaskType::BuildIndexTask) { + return false; + } + // NEVER be empty + auto cpu = ResMgrInst::GetInstance()->GetCpuResources()[0]; + auto label = std::make_shared(cpu); + task->label() = label; + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/FallbackPass.h b/core/src/scheduler/optimizer/FallbackPass.h new file mode 100644 index 0000000000..728740d53a --- /dev/null +++ b/core/src/scheduler/optimizer/FallbackPass.h @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include + +#include "Pass.h" + +namespace milvus { +namespace scheduler { + +class FallbackPass : public Pass { + public: + FallbackPass() = default; + + public: + void + Init() override; + + bool + Run(const TaskPtr& task) override; +}; + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/HybridPass.cpp b/core/src/scheduler/optimizer/HybridPass.cpp index d63fc2e819..8bf0d8990d 100644 --- a/core/src/scheduler/optimizer/HybridPass.cpp +++ b/core/src/scheduler/optimizer/HybridPass.cpp @@ -23,6 +23,10 @@ namespace milvus { namespace scheduler { +void +HybridPass::Init() { +} + bool HybridPass::Run(const TaskPtr& task) { // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu diff --git a/core/src/scheduler/optimizer/HybridPass.h b/core/src/scheduler/optimizer/HybridPass.h index 0d02a8bda9..f84a0884f2 100644 --- a/core/src/scheduler/optimizer/HybridPass.h +++ b/core/src/scheduler/optimizer/HybridPass.h @@ -37,6 +37,9 @@ class HybridPass : public Pass { HybridPass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; }; diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp b/core/src/scheduler/optimizer/LargeSQ8HPass.cpp index 316e0f9aa4..dd58feb512 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/LargeSQ8HPass.cpp @@ -27,7 +27,8 @@ namespace milvus { namespace scheduler { -LargeSQ8HPass::LargeSQ8HPass() { +void +LargeSQ8HPass::Init() { server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.h b/core/src/scheduler/optimizer/LargeSQ8HPass.h index bec9003250..c69fb12c64 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.h +++ b/core/src/scheduler/optimizer/LargeSQ8HPass.h @@ -35,9 +35,12 @@ namespace scheduler { class LargeSQ8HPass : public Pass { public: - LargeSQ8HPass(); + LargeSQ8HPass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.cpp b/core/src/scheduler/optimizer/OnlyCPUPass.cpp index fee79ea277..8cd40092fb 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.cpp +++ b/core/src/scheduler/optimizer/OnlyCPUPass.cpp @@ -24,6 +24,10 @@ namespace milvus { namespace scheduler { +void +OnlyCPUPass::Init() { +} + bool OnlyCPUPass::Run(const TaskPtr& task) { if (task->Type() != TaskType::SearchTask) diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.h b/core/src/scheduler/optimizer/OnlyCPUPass.h index 76b42e3766..253775f77e 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.h +++ b/core/src/scheduler/optimizer/OnlyCPUPass.h @@ -37,6 +37,9 @@ class OnlyCPUPass : public Pass { OnlyCPUPass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; }; diff --git a/core/src/scheduler/optimizer/Optimizer.cpp b/core/src/scheduler/optimizer/Optimizer.cpp index 46f24ea712..c5fa311a27 100644 --- a/core/src/scheduler/optimizer/Optimizer.cpp +++ b/core/src/scheduler/optimizer/Optimizer.cpp @@ -20,12 +20,12 @@ namespace milvus { namespace scheduler { -// void -// Optimizer::Init() { -// for (auto& pass : pass_list_) { -// pass->Init(); -// } -// } +void +Optimizer::Init() { + for (auto& pass : pass_list_) { + pass->Init(); + } +} bool Optimizer::Run(const TaskPtr& task) { diff --git a/core/src/scheduler/optimizer/Optimizer.h b/core/src/scheduler/optimizer/Optimizer.h index bfabbf7de3..68b519e115 100644 --- a/core/src/scheduler/optimizer/Optimizer.h +++ b/core/src/scheduler/optimizer/Optimizer.h @@ -38,8 +38,8 @@ class Optimizer { explicit Optimizer(std::vector pass_list) : pass_list_(std::move(pass_list)) { } - // void - // Init(); + void + Init(); bool Run(const TaskPtr& task); diff --git a/core/src/scheduler/optimizer/Pass.h b/core/src/scheduler/optimizer/Pass.h index 016b05e457..36a36a1df5 100644 --- a/core/src/scheduler/optimizer/Pass.h +++ b/core/src/scheduler/optimizer/Pass.h @@ -34,9 +34,8 @@ namespace scheduler { class Pass { public: - // virtual void - // Init() { - // } + virtual void + Init() = 0; virtual bool Run(const TaskPtr& task) = 0; diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md index 4d6840430e..7bea764df0 100644 --- a/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md @@ -28,13 +28,13 @@ Memory: 503GB Docker version: 18.09 -Nvidia Driver version: 430.34 +NVIDIA Driver version: 430.34 Milvus version: 0.5.3 SDK interface: Python 3.6.8 -Pymilvus version: 0.2.5 +pymilvus version: 0.2.5 @@ -66,7 +66,7 @@ For details on this dataset, you can check : http://corpus-texmex.irisa.fr/ . > Note: In the query test of recall, we will test the following parameters with different values: > - > nq - grouped by: [1, 5, 10, 200, 400, 600, 800, 1000], + > nq - grouped by: [10, 200, 400, 600, 800, 1000], > > topk - grouped by: [1, 10, 100] @@ -93,7 +93,7 @@ Milvus configuration - gpu_cache_capacity: 6 - use_blas_threshold: 1100 -You can check the definition of Milvus configuration on https://milvus.io/docs/en/reference/milvus_config/. +The definitions of Milvus configuration are on https://milvus.io/docs/en/reference/milvus_config/. Test method @@ -177,8 +177,6 @@ search_resources: gpu0, gpu1 | nq/topk | topk=1 | topk=10 | topk=100 | | :-----: | :----: | :-----: | :------: | -| nq=1 | 1.000 | 0.800 | 0.790 | -| nq=5 | 0.800 | 0.820 | 0.908 | | nq=10 | 0.900 | 0.910 | 0.939 | | nq=200 | 0.955 | 0.941 | 0.929 | | nq=400 | 0.958 | 0.944 | 0.932 | @@ -194,8 +192,6 @@ search_resources: cpu, gpu0 | nq/topk | topk=1 | topk=10 | topk=100 | | :-----: | :----: | :-----: | :------: | -| nq=1 | 1.000 | 0.800 | 0.790 | -| nq=5 | 0.800 | 0.820 | 0.908 | | nq=10 | 0.900 | 0.910 | 0.939 | | nq=200 | 0.955 | 0.941 | 0.929 | | nq=400 | 0.958 | 0.944 | 0.932 | diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md index 4b5e18f291..a6e5e75ea4 100644 --- a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md +++ b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md @@ -2,7 +2,7 @@ ## 概述 -本文描述了ivfsq8索引在milvus单机部署方式下的测试报告。 +本文描述了ivfsq8索引在milvus单机部署方式下的测试结果。 @@ -28,13 +28,13 @@ GPU1: GeForce GTX 1080 Docker版本: 18.09 -Nvidia Driver版本: 430.34 +NVIDIA Driver版本: 430.34 Milvus版本: 0.5.3 SDK接口: Python 3.6.8 -Pymilvus版本: 0.2.5 +pymilvus版本: 0.2.5 @@ -66,7 +66,7 @@ Pymilvus版本: 0.2.5 > 备注:在向量准确性测试中,我们会测试下面参数不同的取值来观察结果: > - > 被查询向量的数量nq将按照 [1, 5, 10, 200, 400, 600, 800, 1000]的数量分组, + > 被查询向量的数量nq将按照 [10, 200, 400, 600, 800, 1000]的数量分组, > > 单条查询中最相似的K个结果topk将按照[1, 10, 100]的数量分组。 @@ -93,7 +93,7 @@ Milvus设置: - gpu_cache_capacity: 6 - use_blas_threshold: 1100 -你可以在 https://milvus.io/docs/en/reference/milvus_config/上查询Milvus设置的详细定义。 +Milvus设置的详细定义可以参考 https://milvus.io/docs/en/reference/milvus_config/ 。 测试方法 @@ -177,8 +177,6 @@ search_resources: gpu0, gpu1 | nq/topk | topk=1 | topk=10 | topk=100 | | :-----: | :----: | :-----: | :------: | -| nq=1 | 1.000 | 0.800 | 0.790 | -| nq=5 | 0.800 | 0.820 | 0.908 | | nq=10 | 0.900 | 0.910 | 0.939 | | nq=200 | 0.955 | 0.941 | 0.929 | | nq=400 | 0.958 | 0.944 | 0.932 | @@ -194,8 +192,6 @@ search_resources: cpu, gpu0 | nq/topk | topk=1 | topk=10 | topk=100 | | :-----: | :----: | :-----: | :------: | -| nq=1 | 1.000 | 0.800 | 0.790 | -| nq=5 | 0.800 | 0.820 | 0.908 | | nq=10 | 0.900 | 0.910 | 0.939 | | nq=200 | 0.955 | 0.941 | 0.929 | | nq=400 | 0.958 | 0.944 | 0.932 | @@ -207,5 +203,5 @@ search_resources: cpu, gpu0 **总结** -​ 随着nq的增大,召回率逐渐稳定至93%以上。 +随着nq的增大,召回率逐渐稳定至93%以上。 diff --git a/install.md b/install.md index 6711b41f76..f53586af36 100644 --- a/install.md +++ b/install.md @@ -3,6 +3,9 @@ ## Software requirements - Ubuntu 18.04 or higher + + If your operating system is not Ubuntu 18.04 or higher, we recommend you to pull a [docker image of Ubuntu 18.04](https://docs.docker.com/install/linux/docker-ce/ubuntu/) as your compilation environment. + - CMake 3.12 or higher ##### For GPU version, you will also need: