From 35e28f81b5a14029ae7ae8e9b5bb494ff7c9dd30 Mon Sep 17 00:00:00 2001 From: wxyu Date: Sat, 24 Aug 2019 16:36:56 +0800 Subject: [PATCH 1/5] MS-414 Add TaskType in Scheduler::Task Former-commit-id: d470511e0d22832db09607ad017c6dd4a697cf05 --- cpp/CHANGELOG.md | 1 + cpp/src/scheduler/task/DeleteTask.cpp | 2 +- cpp/src/scheduler/task/SearchTask.cpp | 5 +++-- cpp/src/scheduler/task/Task.h | 13 ++++++++++++- cpp/src/scheduler/task/TestTask.cpp | 2 ++ cpp/src/scheduler/task/TestTask.h | 2 +- 6 files changed, 20 insertions(+), 5 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 671d865a02..3b4888dea4 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -50,6 +50,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-408 - Add device_id in resource construct function - MS-409 - Using new scheduler - MS-410 - Add resource config comment +- MS-414 - Add TaskType in Scheduler::Task ## New Feature - MS-343 - Implement ResourceMgr diff --git a/cpp/src/scheduler/task/DeleteTask.cpp b/cpp/src/scheduler/task/DeleteTask.cpp index 9f76de506e..0e9c7dc360 100644 --- a/cpp/src/scheduler/task/DeleteTask.cpp +++ b/cpp/src/scheduler/task/DeleteTask.cpp @@ -12,7 +12,7 @@ namespace milvus { namespace engine { XDeleteTask::XDeleteTask(DeleteContextPtr &delete_context) - : delete_context_ptr_(delete_context) {} + : Task(TaskType::DeleteTask), delete_context_ptr_(delete_context) {} void XDeleteTask::Load(LoadType type, uint8_t device_id) { diff --git a/cpp/src/scheduler/task/SearchTask.cpp b/cpp/src/scheduler/task/SearchTask.cpp index 7d342a3592..3ddc2beb73 100644 --- a/cpp/src/scheduler/task/SearchTask.cpp +++ b/cpp/src/scheduler/task/SearchTask.cpp @@ -81,11 +81,12 @@ CollectFileMetrics(int file_type, size_t file_size) { } } -XSearchTask::XSearchTask(TableFileSchemaPtr file) : file_(file) { +XSearchTask::XSearchTask(TableFileSchemaPtr file) + : Task(TaskType::SearchTask), file_(file) { index_engine_ = EngineFactory::Build(file_->dimension_, file_->location_, (EngineType) file_->engine_type_, - (MetricType)file_->metric_type_, + (MetricType) file_->metric_type_, file_->nlist_); } diff --git a/cpp/src/scheduler/task/Task.h b/cpp/src/scheduler/task/Task.h index 4d78894d31..bc2dae464b 100644 --- a/cpp/src/scheduler/task/Task.h +++ b/cpp/src/scheduler/task/Task.h @@ -21,13 +21,20 @@ enum class LoadType { GPU2CPU, }; +enum class TaskType { + SearchTask, + DeleteTask, + TestTask, +}; + class Task; using TaskPtr = std::shared_ptr; class Task { public: - Task() = default; + explicit + Task(TaskType type) : type_(type) {} virtual void Load(LoadType type, uint8_t device_id) = 0; @@ -39,9 +46,13 @@ public: virtual TaskPtr Clone() = 0; + inline TaskType + Type() const { return type_; } + public: std::vector search_contexts_; ScheduleTaskPtr task_; + TaskType type_; }; diff --git a/cpp/src/scheduler/task/TestTask.cpp b/cpp/src/scheduler/task/TestTask.cpp index 527e1b0fb1..29f2a94048 100644 --- a/cpp/src/scheduler/task/TestTask.cpp +++ b/cpp/src/scheduler/task/TestTask.cpp @@ -11,6 +11,8 @@ namespace zilliz { namespace milvus { namespace engine { +TestTask::TestTask() : Task(TaskType::TestTask) {} + void TestTask::Load(LoadType type, uint8_t device_id) { load_count_++; diff --git a/cpp/src/scheduler/task/TestTask.h b/cpp/src/scheduler/task/TestTask.h index 0eb77504c8..5a466ec339 100644 --- a/cpp/src/scheduler/task/TestTask.h +++ b/cpp/src/scheduler/task/TestTask.h @@ -14,7 +14,7 @@ namespace engine { class TestTask : public Task { public: - TestTask() = default; + TestTask(); public: void From 14b2254060378a5a1e18aca2c45f9377dd18cb01 Mon Sep 17 00:00:00 2001 From: wxyu Date: Sat, 24 Aug 2019 16:39:47 +0800 Subject: [PATCH 2/5] MS-415 Add command tasktable to dump all tasktables Former-commit-id: 7ce1dedf36d020668af2bcaafa66b5232d4c18a5 --- cpp/CHANGELOG.md | 1 + cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index bf711b944e..e81e61d21a 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -52,6 +52,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-413 - Remove thrift dependency - MS-410 - Add resource config comment - MS-414 - Add TaskType in Scheduler::Task +- MS-415 - Add command tasktable to dump all tasktables ## New Feature - MS-343 - Implement ResourceMgr diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index b2cb3ae05b..d75ad8163c 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -13,6 +13,7 @@ #include "version.h" #include "GrpcMilvusServer.h" #include "db/Utils.h" +#include "scheduler/SchedInst.h" #include "src/server/Server.h" @@ -753,7 +754,10 @@ ServerError CmdTask::OnExecute() { if (cmd_ == "version") { result_ = MILVUS_VERSION; - } else { + } else if (cmd_ == "tasktable") { + result_ = engine::ResMgrInst::GetInstance()->DumpTaskTables(); + } + else { result_ = "OK"; } From 2e8f018e40fb33ce841861fa5f9ba759bccbe5ab Mon Sep 17 00:00:00 2001 From: wxyu Date: Sat, 24 Aug 2019 16:42:15 +0800 Subject: [PATCH 3/5] MS-416 ExecutionEngineImpl::GpuCache has not return value cause crash Former-commit-id: 298969de85a0a8cb4a4e48da21d45fde30df64d1 --- cpp/CHANGELOG.md | 1 + cpp/src/db/engine/ExecutionEngineImpl.cpp | 2 ++ 2 files changed, 3 insertions(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index e81e61d21a..ce3957e821 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -7,6 +7,7 @@ Please mark all change in change log and use the ticket from JIRA. ## Bug - MS-411 - Fix metric unittest linking error - MS-412 - Fix gpu cache logical error +- MS-416 - ExecutionEngineImpl::GpuCache has not return value cause crash ## Improvement - MS-327 - Clean code for milvus diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 7489e7849e..7ae48649fb 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -273,6 +273,8 @@ Status ExecutionEngineImpl::Cache() { Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { zilliz::milvus::cache::GpuCacheMgr::GetInstance(gpu_id)->InsertItem(location_, index_); + + return Status::OK(); } // TODO(linxj): remove. From 527026c4a0f25316acfe40fed24fdd091c7cbff7 Mon Sep 17 00:00:00 2001 From: wxyu Date: Sat, 24 Aug 2019 16:44:30 +0800 Subject: [PATCH 4/5] MS-417 YAML sequence load disable cause scheduler startup failed Former-commit-id: d9055c6ed60306ee8cfc09f13694b9186167a4a5 --- cpp/CHANGELOG.md | 1 + cpp/src/config/YamlConfigMgr.cpp | 30 +++++++++++++++--------------- cpp/src/config/YamlConfigMgr.h | 8 ++++---- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index ce3957e821..12906a4b6d 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -8,6 +8,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-411 - Fix metric unittest linking error - MS-412 - Fix gpu cache logical error - MS-416 - ExecutionEngineImpl::GpuCache has not return value cause crash +- MS-417 - YAML sequence load disable cause scheduler startup failed ## Improvement - MS-327 - Clean code for milvus diff --git a/cpp/src/config/YamlConfigMgr.cpp b/cpp/src/config/YamlConfigMgr.cpp index ee935bf32c..9a34ef3e63 100644 --- a/cpp/src/config/YamlConfigMgr.cpp +++ b/cpp/src/config/YamlConfigMgr.cpp @@ -73,19 +73,19 @@ YamlConfigMgr::SetChildConfig(const YAML::Node& node, return false; } -//bool -//YamlConfigMgr::SetSequence(const YAML::Node &node, -// const std::string &child_name, -// ConfigNode &config) { -// if(node[child_name].IsDefined ()) { -// size_t cnt = node[child_name].size(); -// for(size_t i = 0; i < cnt; i++){ -// config.AddSequenceItem(child_name, node[child_name][i].as()); -// } -// return true; -// } -// return false; -//} +bool +YamlConfigMgr::SetSequence(const YAML::Node &node, + const std::string &child_name, + ConfigNode &config) { + if(node[child_name].IsDefined ()) { + size_t cnt = node[child_name].size(); + for(size_t i = 0; i < cnt; i++){ + config.AddSequenceItem(child_name, node[child_name][i].as()); + } + return true; + } + return false; +} void YamlConfigMgr::LoadConfigNode(const YAML::Node& node, ConfigNode& config) { @@ -98,8 +98,8 @@ YamlConfigMgr::LoadConfigNode(const YAML::Node& node, ConfigNode& config) { SetConfigValue(node, key, config); } else if(node[key].IsMap()){ SetChildConfig(node, key, config); -// } else if(node[key].IsSequence()){ -// SetSequence(node, key, config); + } else if(node[key].IsSequence()){ + SetSequence(node, key, config); } } } diff --git a/cpp/src/config/YamlConfigMgr.h b/cpp/src/config/YamlConfigMgr.h index 05b55d9da5..b8828b7a8c 100644 --- a/cpp/src/config/YamlConfigMgr.h +++ b/cpp/src/config/YamlConfigMgr.h @@ -33,10 +33,10 @@ class YamlConfigMgr : public IConfigMgr { const std::string &name, ConfigNode &config); -// bool -// SetSequence(const YAML::Node &node, -// const std::string &child_name, -// ConfigNode &config); + bool + SetSequence(const YAML::Node &node, + const std::string &child_name, + ConfigNode &config); void LoadConfigNode(const YAML::Node& node, ConfigNode& config); From 539dade5b215fa5f339314de90cb34b3b710b12e Mon Sep 17 00:00:00 2001 From: wxyu Date: Sat, 24 Aug 2019 16:57:15 +0800 Subject: [PATCH 5/5] MS-418 Update server_config.template file, set CPU compute only default Former-commit-id: 62ea51804d4c6ba538de6c329d80411774137a75 --- cpp/CHANGELOG.md | 1 + cpp/conf/server_config.template | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 12906a4b6d..1d7a7384d7 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -55,6 +55,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-410 - Add resource config comment - MS-414 - Add TaskType in Scheduler::Task - MS-415 - Add command tasktable to dump all tasktables +- MS-418 - Update server_config.template file, set CPU compute only default ## New Feature - MS-343 - Implement ResourceMgr diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index c7ae51682f..386b61d1f1 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -64,21 +64,21 @@ resource_config: memory: 64 device_id: 0 enable_loader: true - enable_executor: false + enable_executor: true gtx1060: type: GPU memory: 6 device_id: 0 - enable_loader: true - enable_executor: true + enable_loader: false + enable_executor: false gtx1660: type: GPU memory: 6 device_id: 1 - enable_loader: true - enable_executor: true + enable_loader: false + enable_executor: false # connection list, length: 0~N # format: -${resource_name}===${resource_name}