Merge branch 'branch-0.4.0' into 'branch-0.4.0'

MS-417 YAML sequence load disable cause scheduler startup failed

See merge request megasearch/milvus!424

Former-commit-id: 3812e11a43625bb9135a6df975b922c8e4f9efed

cpp/CHANGELOG.md

@@ -7,6 +7,8 @@ Please mark all change in change log and use the ticket from JIRA.
 ## Bug
 - MS-411 - Fix metric unittest linking error
 - MS-412 - Fix gpu cache logical error
+- MS-416 - ExecutionEngineImpl::GpuCache has not return value cause crash
+- MS-417 - YAML sequence load disable cause scheduler startup failed
 
 ## Improvement
 - MS-327 - Clean code for milvus
@@ -51,6 +53,9 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-409 - Using new scheduler
 - MS-413 - Remove thrift dependency
 - MS-410 - Add resource config comment
+- MS-414 - Add TaskType in Scheduler::Task
+- MS-415 - Add command tasktable to dump all tasktables
+- MS-418 - Update server_config.template file, set CPU compute only default
 - MS-419 - Move index_file_size from IndexParam to TableSchema
 
 ## New Feature

cpp/conf/server_config.template

@@ -64,21 +64,21 @@ resource_config:
       memory: 64
       device_id: 0
       enable_loader: true
-      enable_executor: false
+      enable_executor: true
 
     gtx1060:
       type: GPU
       memory: 6
       device_id: 0
-      enable_loader: true
+      enable_loader: false
-      enable_executor: true
+      enable_executor: false
 
     gtx1660:
       type: GPU
       memory: 6
       device_id: 1
-      enable_loader: true
+      enable_loader: false
-      enable_executor: true
+      enable_executor: false
 
   # connection list, length: 0~N
   # format: -${resource_name}===${resource_name}

cpp/src/config/YamlConfigMgr.cpp

@@ -73,19 +73,19 @@ YamlConfigMgr::SetChildConfig(const YAML::Node& node,
     return false;
 }
 
-//bool
+bool
-//YamlConfigMgr::SetSequence(const YAML::Node &node,
+YamlConfigMgr::SetSequence(const YAML::Node &node,
-//                           const std::string &child_name,
+                           const std::string &child_name,
-//                           ConfigNode &config) {
+                           ConfigNode &config) {
-//    if(node[child_name].IsDefined ()) {
+    if(node[child_name].IsDefined ()) {
-//        size_t cnt = node[child_name].size();
+        size_t cnt = node[child_name].size();
-//        for(size_t i = 0; i < cnt; i++){
+        for(size_t i = 0; i < cnt; i++){
-//            config.AddSequenceItem(child_name, node[child_name][i].as<std::string>());
+            config.AddSequenceItem(child_name, node[child_name][i].as<std::string>());
-//        }
+        }
-//        return true;
+        return true;
-//    }
+    }
-//    return false;
+    return false;
-//}
+}
 
 void
 YamlConfigMgr::LoadConfigNode(const YAML::Node& node, ConfigNode& config) {
@@ -98,8 +98,8 @@ YamlConfigMgr::LoadConfigNode(const YAML::Node& node, ConfigNode& config) {
             SetConfigValue(node, key, config);
         } else if(node[key].IsMap()){
             SetChildConfig(node, key, config);
-//        } else if(node[key].IsSequence()){
+        } else if(node[key].IsSequence()){
-//            SetSequence(node, key, config);
+            SetSequence(node, key, config);
         }
     }
 }

cpp/src/config/YamlConfigMgr.h

@@ -33,10 +33,10 @@ class YamlConfigMgr : public IConfigMgr {
                         const std::string &name,
                         ConfigNode &config);
 
-//    bool
+    bool
-//    SetSequence(const YAML::Node &node,
+    SetSequence(const YAML::Node &node,
-//                const std::string &child_name,
+                const std::string &child_name,
-//                ConfigNode &config);
+                ConfigNode &config);
 
     void LoadConfigNode(const YAML::Node& node, ConfigNode& config);
 

cpp/src/db/engine/ExecutionEngineImpl.cpp

@@ -273,6 +273,8 @@ Status ExecutionEngineImpl::Cache() {
 
 Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
     zilliz::milvus::cache::GpuCacheMgr::GetInstance(gpu_id)->InsertItem(location_, index_);
+
+    return Status::OK();
 }
 
 // TODO(linxj): remove.

cpp/src/scheduler/task/DeleteTask.cpp

@@ -12,7 +12,7 @@ namespace milvus {
 namespace engine {
 
 XDeleteTask::XDeleteTask(DeleteContextPtr &delete_context)
-    : delete_context_ptr_(delete_context) {}
+    : Task(TaskType::DeleteTask), delete_context_ptr_(delete_context) {}
 
 void
 XDeleteTask::Load(LoadType type, uint8_t device_id) {

cpp/src/scheduler/task/SearchTask.cpp

@@ -81,11 +81,12 @@ CollectFileMetrics(int file_type, size_t file_size) {
     }
 }
 
-XSearchTask::XSearchTask(TableFileSchemaPtr file) : file_(file) {
+XSearchTask::XSearchTask(TableFileSchemaPtr file)
+    : Task(TaskType::SearchTask), file_(file) {
     index_engine_ = EngineFactory::Build(file_->dimension_,
                                          file_->location_,
                                          (EngineType) file_->engine_type_,
-                                         (MetricType)file_->metric_type_,
+                                         (MetricType) file_->metric_type_,
                                          file_->nlist_);
 }
 

cpp/src/scheduler/task/Task.h

@@ -21,13 +21,20 @@ enum class LoadType {
     GPU2CPU,
 };
 
+enum class TaskType {
+    SearchTask,
+    DeleteTask,
+    TestTask,
+};
+
 class Task;
 
 using TaskPtr = std::shared_ptr<Task>;
 
 class Task {
 public:
-    Task() = default;
+    explicit
+    Task(TaskType type) : type_(type) {}
 
     virtual void
     Load(LoadType type, uint8_t device_id) = 0;
@@ -39,9 +46,13 @@ public:
     virtual TaskPtr
     Clone() = 0;
 
+    inline TaskType
+    Type() const { return type_; }
+
 public:
     std::vector<SearchContextPtr> search_contexts_;
     ScheduleTaskPtr task_;
+    TaskType type_;
 };
 
 

cpp/src/scheduler/task/TestTask.cpp

@@ -11,6 +11,8 @@ namespace zilliz {
 namespace milvus {
 namespace engine {
 
+TestTask::TestTask() : Task(TaskType::TestTask) {}
+
 void
 TestTask::Load(LoadType type, uint8_t device_id) {
     load_count_++;

cpp/src/scheduler/task/TestTask.h

@@ -14,7 +14,7 @@ namespace engine {
 
 class TestTask : public Task {
 public:
-    TestTask() = default;
+    TestTask();
 
 public:
     void

cpp/src/server/grpc_impl/GrpcRequestTask.cpp

@@ -13,6 +13,7 @@
 #include "version.h"
 #include "GrpcMilvusServer.h"
 #include "db/Utils.h"
+#include "scheduler/SchedInst.h"
 
 #include "src/server/Server.h"
 
@@ -753,7 +754,10 @@ ServerError
 CmdTask::OnExecute() {
     if (cmd_ == "version") {
         result_ = MILVUS_VERSION;
-    } else {
+    } else if (cmd_ == "tasktable") {
+        result_ = engine::ResMgrInst::GetInstance()->DumpTaskTables();
+    }
+    else {
         result_ = "OK";
     }