Add crud APIs and segments APIs into http server (#1462)

* add new http api

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* [DOC] modify comments

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* allow get whole tables & partitions

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* add test case

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* update changlog (fix #1461)

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* format

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* fix cpu build issue

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* fix unittest fail on cpu version

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* fix get system config bug

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* fix crash when show tables

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* fix check server restart fail

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* remove some comments

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* add count field into segment ids

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* optimize code fixing codegacy

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* add default response string value

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

CHANGELOG.md

@@ -46,6 +46,7 @@ Please mark all change in change log and use the issue from GitHub
 - \#1204 - Add api to get table data information
 - \#1250 - Support CPU profiling
 - \#1302 - Get all record IDs in a segment by given a segment id
+- \#1461 - Add crud APIs and segments APIs into http module
 
 ## Improvement
 - \#738 - Use Openblas / lapack from apt install

core/src/server/Config.cpp

@@ -405,8 +405,8 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key
 
     if (status.ok()) {
         status = UpdateFileConfigFromMem(parent_key, child_key);
-        if (status.ok() && (parent_key == CONFIG_SERVER || parent_key == CONFIG_DB || parent_key == CONFIG_STORAGE ||
-                            parent_key == CONFIG_METRIC || parent_key == CONFIG_TRACING)) {
+        if (status.ok() &&
+            !(parent_key == CONFIG_CACHE || parent_key == CONFIG_ENGINE || parent_key == CONFIG_GPU_RESOURCE)) {
             restart_required_ = true;
         }
     }

core/src/server/web_impl/Types.h

@@ -63,11 +63,14 @@ enum StatusCode : int {
 
     // HTTP error code
     PATH_PARAM_LOSS = 31,
-    QUERY_PARAM_LOSS = 32,
-    BODY_FIELD_LOSS = 33,
-    ILLEGAL_BODY = 34,
-    BODY_PARSE_FAIL = 35,
-    ILLEGAL_QUERY_PARAM = 36,
+    UNKNOWN_PATH = 32,
+    QUERY_PARAM_LOSS = 33,
+    BODY_FIELD_LOSS = 34,
+    ILLEGAL_BODY = 35,
+    BODY_PARSE_FAIL = 36,
+    ILLEGAL_QUERY_PARAM = 37,
+
+    MAX = ILLEGAL_QUERY_PARAM
 };
 
 static const std::unordered_map<engine::EngineType, std::string> IndexMap = {

core/src/server/web_impl/controller/WebController.hpp

@@ -67,7 +67,6 @@ class WebController : public oatpp::web::server::api::ApiController {
         return createDtoResponse(Status::CODE_200, StatusDto::createShared());
     }
 
-
     ADD_CORS(GetDevices)
 
     ENDPOINT("GET", "/devices", GetDevices) {
@@ -245,15 +244,13 @@ class WebController : public oatpp::web::server::api::ApiController {
 
         WebRequestHandler handler = WebRequestHandler();
 
-        auto response_dto = TableListFieldsDto::createShared();
-        auto offset = query_params.get("offset");
-        auto page_size = query_params.get("page_size");
 
+        String result;
+        auto status_dto = handler.ShowTables(query_params, result);
         std::shared_ptr<OutgoingResponse> response;
-        auto status_dto = handler.ShowTables(offset, page_size, response_dto);
         switch (status_dto->code->getValue()) {
             case StatusCode::SUCCESS:
-                response = createDtoResponse(Status::CODE_200, response_dto);
+                response = createResponse(Status::CODE_200, result);
                 break;
             default:
                 response = createDtoResponse(Status::CODE_400, status_dto);
@@ -281,13 +278,13 @@ class WebController : public oatpp::web::server::api::ApiController {
 
         WebRequestHandler handler = WebRequestHandler();
 
-        auto fields_dto = TableFieldsDto::createShared();
-        auto status_dto = handler.GetTable(table_name, query_params, fields_dto);
+        String response_str;
+        auto status_dto = handler.GetTable(table_name, query_params, response_str);
 
         std::shared_ptr<OutgoingResponse> response;
         switch (status_dto->code->getValue()) {
             case StatusCode::SUCCESS:
-                response = createDtoResponse(Status::CODE_200, fields_dto);
+                response = createResponse(Status::CODE_200, response_str);
                 break;
             case StatusCode::TABLE_NOT_EXISTS:
                 response = createDtoResponse(Status::CODE_404, status_dto);
@@ -472,7 +469,7 @@ class WebController : public oatpp::web::server::api::ApiController {
         auto handler = WebRequestHandler();
 
         std::shared_ptr<OutgoingResponse> response;
-        auto status_dto = handler.ShowPartitions(offset, page_size, table_name, partition_list_dto);
+        auto status_dto = handler.ShowPartitions(table_name, query_params, partition_list_dto);
         switch (status_dto->code->getValue()) {
             case StatusCode::SUCCESS:
                 response = createDtoResponse(Status::CODE_200, partition_list_dto);
@@ -489,24 +486,18 @@ class WebController : public oatpp::web::server::api::ApiController {
         return response;
     }
 
-    ADD_CORS(PartitionOptions)
-
-    ENDPOINT("OPTIONS", "/tables/{table_name}/partitions/{partition_tag}", PartitionOptions) {
-        return createResponse(Status::CODE_204, "No Content");
-    }
-
     ADD_CORS(DropPartition)
 
-    ENDPOINT("DELETE", "/tables/{table_name}/partitions/{partition_tag}", DropPartition,
-             PATH(String, table_name), PATH(String, partition_tag)) {
+    ENDPOINT("DELETE", "/tables/{table_name}/partitions", DropPartition,
+             PATH(String, table_name), BODY_STRING(String, body)) {
         TimeRecorder tr(std::string(WEB_LOG_PREFIX) +
-                        "DELETE \'/tables/" + table_name->std_str() + "/partitions/" + partition_tag->std_str() + "\'");
+                        "DELETE \'/tables/" + table_name->std_str() + "/partitions\'");
         tr.RecordSection("Received request.");
 
         auto handler = WebRequestHandler();
 
         std::shared_ptr<OutgoingResponse> response;
-        auto status_dto = handler.DropPartition(table_name, partition_tag);
+        auto status_dto = handler.DropPartition(table_name, body);
         switch (status_dto->code->getValue()) {
             case StatusCode::SUCCESS:
                 response = createDtoResponse(Status::CODE_204, status_dto);
@@ -524,16 +515,82 @@ class WebController : public oatpp::web::server::api::ApiController {
         return response;
     }
 
+    ADD_CORS(ShowSegments)
+
+    ENDPOINT("GET", "/tables/{table_name}/segments", ShowSegments,
+             PATH(String, table_name), QUERIES(const QueryParams&, query_params)) {
+        auto offset = query_params.get("offset");
+        auto page_size = query_params.get("page_size");
+
+        auto handler = WebRequestHandler();
+        String response;
+        auto status_dto = handler.ShowSegments(table_name, query_params, response);
+
+        switch (status_dto->code->getValue()) {
+            case StatusCode::SUCCESS:{
+                return createResponse(Status::CODE_200, response);
+            }
+            default:{
+                return createDtoResponse(Status::CODE_400, status_dto);
+            }
+        }
+    }
+
+    ADD_CORS(GetSegmentInfo)
+    /**
+     *
+     * GetSegmentVector
+     */
+    ENDPOINT("GET", "/tables/{table_name}/segments/{segment_name}/{info}", GetSegmentInfo,
+             PATH(String, table_name), PATH(String, segment_name), PATH(String, info), QUERIES(const QueryParams&, query_params)) {
+        auto offset = query_params.get("offset");
+        auto page_size = query_params.get("page_size");
+
+        auto handler = WebRequestHandler();
+        String response;
+        auto status_dto = handler.GetSegmentInfo(table_name, segment_name, info, query_params, response);
+
+        switch (status_dto->code->getValue()) {
+            case StatusCode::SUCCESS:{
+                return createResponse(Status::CODE_200, response);
+            }
+            default:{
+                return createDtoResponse(Status::CODE_400, status_dto);
+            }
+        }
+    }
+
     ADD_CORS(VectorsOptions)
 
     ENDPOINT("OPTIONS", "/tables/{table_name}/vectors", VectorsOptions) {
         return createResponse(Status::CODE_204, "No Content");
     }
 
+    ADD_CORS(GetVectors)
+    /**
+     *
+     * GetVectorByID ?id=
+     */
+    ENDPOINT("GET", "/tables/{table_name}/vectors", GetVectors,
+             PATH(String, table_name), QUERIES(const QueryParams&, query_params)) {
+        auto handler = WebRequestHandler();
+        String response;
+        auto status_dto = handler.GetVector(table_name, query_params, response);
+
+        switch (status_dto->code->getValue()) {
+            case StatusCode::SUCCESS:{
+                return createResponse(Status::CODE_200, response);
+            }
+            default:{
+                return createDtoResponse(Status::CODE_400, status_dto);
+            }
+        }
+    }
+
     ADD_CORS(Insert)
 
     ENDPOINT("POST", "/tables/{table_name}/vectors", Insert,
-             PATH(String, table_name), BODY_DTO(InsertRequestDto::ObjectWrapper, body)) {
+             PATH(String, table_name), BODY_STRING(String, body)) {
         TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/tables/" + table_name->std_str() + "/vectors\'");
         tr.RecordSection("Received request.");
 
@@ -559,21 +616,24 @@ class WebController : public oatpp::web::server::api::ApiController {
         return response;
     }
 
-    ADD_CORS(Search)
-
-    ENDPOINT("PUT", "/tables/{table_name}/vectors", Search,
-             PATH(String, table_name), BODY_DTO(SearchRequestDto::ObjectWrapper, body)) {
+    ADD_CORS(VectorsOp)
+    /*************
+     * Search
+     * Delete by ID
+     * */
+    ENDPOINT("PUT", "/tables/{table_name}/vectors", VectorsOp,
+             PATH(String, table_name), BODY_STRING(String, body)) {
         TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "PUT \'/tables/" + table_name->std_str() + "/vectors\'");
         tr.RecordSection("Received request.");
 
-        auto results_dto = TopkResultsDto::createShared();
         WebRequestHandler handler = WebRequestHandler();
 
+        OString result;
         std::shared_ptr<OutgoingResponse> response;
-        auto status_dto = handler.Search(table_name, body, results_dto);
+        auto status_dto = handler.VectorsOp(table_name, body, result);
         switch (status_dto->code->getValue()) {
             case StatusCode::SUCCESS:
-                response = createDtoResponse(Status::CODE_200, results_dto);
+                response = createResponse(Status::CODE_200, result);
                 break;
             case StatusCode::TABLE_NOT_EXISTS:
                 response = createDtoResponse(Status::CODE_404, status_dto);
@@ -590,18 +650,17 @@ class WebController : public oatpp::web::server::api::ApiController {
 
     ADD_CORS(SystemInfo)
 
-    ENDPOINT("GET", "/system/{msg}", SystemInfo, PATH(String, msg), QUERIES(const QueryParams&, query_params)) {
-        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/system/" + msg->std_str() + "\'");
+    ENDPOINT("GET", "/system/{info}", SystemInfo, PATH(String, info), QUERIES(const QueryParams&, query_params)) {
+        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/system/" + info->std_str() + "\'");
         tr.RecordSection("Received request.");
 
-        auto info_dto = CommandDto::createShared();
         WebRequestHandler handler = WebRequestHandler();
-
-        auto status_dto = handler.SystemInfo(msg, info_dto);
+        OString result = "";
+        auto status_dto = handler.SystemInfo(info, query_params, result);
         std::shared_ptr<OutgoingResponse> response;
         switch (status_dto->code->getValue()) {
             case StatusCode::SUCCESS:
-                response = createDtoResponse(Status::CODE_200, info_dto);
+                response = createResponse(Status::CODE_200, result);
                 break;
             default:
                 response = createDtoResponse(Status::CODE_400, status_dto);
@@ -615,6 +674,11 @@ class WebController : public oatpp::web::server::api::ApiController {
 
     ADD_CORS(SystemOp)
 
+    /**
+    * Load
+    * Compact
+    * Flush
+    */
     ENDPOINT("PUT", "/system/{Op}", SystemOp, PATH(String, Op), BODY_STRING(String, body_str)) {
         TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "PUT \'/system/" + Op->std_str() + "\'");
         tr.RecordSection("Received request.");

core/src/server/web_impl/handler/WebRequestHandler.h

@@ -15,12 +15,16 @@
 #include <memory>
 #include <string>
 #include <utility>
+#include <vector>
 
 #include <opentracing/mocktracer/tracer.h>
 #include <oatpp/core/data/mapping/type/Object.hpp>
 #include <oatpp/core/macro/codegen.hpp>
 #include <oatpp/web/server/api/ApiController.hpp>
 
+#include "db/Types.h"
+#include "server/context/Context.h"
+#include "server/delivery/RequestHandler.h"
 #include "server/web_impl/Types.h"
 #include "server/web_impl/dto/CmdDto.hpp"
 #include "server/web_impl/dto/ConfigDto.hpp"
@@ -29,10 +33,7 @@
 #include "server/web_impl/dto/PartitionDto.hpp"
 #include "server/web_impl/dto/TableDto.hpp"
 #include "server/web_impl/dto/VectorDto.hpp"
-
-#include "db/Types.h"
-#include "server/context/Context.h"
-#include "server/delivery/RequestHandler.h"
+#include "thirdparty/nlohmann/json.hpp"
 #include "utils/Status.h"
 
 namespace milvus {
@@ -76,19 +77,74 @@ class WebRequestHandler {
         return context_ptr;
     }
 
+ private:
+    void
+    AddStatusToJson(nlohmann::json& json, int64_t code, const std::string& msg);
+
+    Status
+    ParseSegmentStat(const SegmentStat& seg_stat, nlohmann::json& json);
+
+    Status
+    ParsePartitionStat(const PartitionStat& par_stat, nlohmann::json& json);
+
+    Status
+    IsBinaryTable(const std::string& table_name, bool& bin);
+
+    Status
+    CopyRecordsFromJson(const nlohmann::json& json, engine::VectorsData& vectors, bool bin);
+
  protected:
     Status
-    GetTableInfo(const std::string& table_name, TableFieldsDto::ObjectWrapper& table_fields);
+    GetTableMetaInfo(const std::string& table_name, nlohmann::json& json_out);
+
+    Status
+    GetTableStat(const std::string& table_name, nlohmann::json& json_out);
+
+    Status
+    GetSegmentVectors(const std::string& table_name, const std::string& segment_name, int64_t page_size, int64_t offset,
+                      nlohmann::json& json_out);
+
+    Status
+    GetSegmentIds(const std::string& table_name, const std::string& segment_name, int64_t page_size, int64_t offset,
+                  nlohmann::json& json_out);
 
     Status
     CommandLine(const std::string& cmd, std::string& reply);
 
+    Status
+    Cmd(const std::string& cmd, std::string& result_str);
+
+    Status
+    PreLoadTable(const nlohmann::json& json, std::string& result_str);
+
+    Status
+    Flush(const nlohmann::json& json, std::string& result_str);
+
+    Status
+    Compact(const nlohmann::json& json, std::string& result_str);
+
+    Status
+    GetConfig(std::string& result_str);
+
+    Status
+    SetConfig(const nlohmann::json& json, std::string& result_str);
+
+    Status
+    Search(const std::string& table_name, const nlohmann::json& json, std::string& result_str);
+
+    Status
+    DeleteByIDs(const std::string& table_name, const nlohmann::json& json, std::string& result_str);
+
+    Status
+    GetVectorsByIDs(const std::string& table_name, const std::vector<int64_t>& ids, nlohmann::json& json_out);
+
  public:
     WebRequestHandler() {
         context_ptr_ = GenContextPtr("Web Handler");
         request_handler_ = RequestHandler();
     }
 
+ public:
     StatusDto::ObjectWrapper
     GetDevices(DevicesDto::ObjectWrapper& devices);
 
@@ -106,18 +162,25 @@ class WebRequestHandler {
     SetGpuConfig(const GPUConfigDto::ObjectWrapper& gpu_config_dto);
 #endif
 
+    /**********
+     *
+     * Table
+     */
     StatusDto::ObjectWrapper
     CreateTable(const TableRequestDto::ObjectWrapper& table_schema);
+    StatusDto::ObjectWrapper
+    ShowTables(const OQueryParams& query_params, OString& result);
 
     StatusDto::ObjectWrapper
-    GetTable(const OString& table_name, const OQueryParams& query_params, TableFieldsDto::ObjectWrapper& schema_dto);
-
-    StatusDto::ObjectWrapper
-    ShowTables(const OString& offset, const OString& page_size, TableListFieldsDto::ObjectWrapper& table_list_dto);
+    GetTable(const OString& table_name, const OQueryParams& query_params, OString& result);
 
     StatusDto::ObjectWrapper
     DropTable(const OString& table_name);
 
+    /**********
+     *
+     * Index
+     */
     StatusDto::ObjectWrapper
     CreateIndex(const OString& table_name, const IndexRequestDto::ObjectWrapper& index_param);
 
@@ -127,26 +190,50 @@ class WebRequestHandler {
     StatusDto::ObjectWrapper
     DropIndex(const OString& table_name);
 
+    /***********
+     *
+     * Partition
+     */
     StatusDto::ObjectWrapper
     CreatePartition(const OString& table_name, const PartitionRequestDto::ObjectWrapper& param);
 
     StatusDto::ObjectWrapper
-    ShowPartitions(const OString& offset, const OString& page_size, const OString& table_name,
+    ShowPartitions(const OString& table_name, const OQueryParams& query_params,
                    PartitionListDto::ObjectWrapper& partition_list_dto);
 
     StatusDto::ObjectWrapper
-    DropPartition(const OString& table_name, const OString& tag);
+    DropPartition(const OString& table_name, const OString& body);
+
+    /***********
+     *
+     * Segment
+     */
+    StatusDto::ObjectWrapper
+    ShowSegments(const OString& table_name, const OQueryParams& query_params, OString& response);
 
     StatusDto::ObjectWrapper
-    Insert(const OString& table_name, const InsertRequestDto::ObjectWrapper& param,
-           VectorIdsDto::ObjectWrapper& ids_dto);
+    GetSegmentInfo(const OString& table_name, const OString& segment_name, const OString& info,
+                   const OQueryParams& query_params, OString& result);
+
+    /**
+     *
+     * Vector
+     */
+    StatusDto::ObjectWrapper
+    Insert(const OString& table_name, const OString& body, VectorIdsDto::ObjectWrapper& ids_dto);
 
     StatusDto::ObjectWrapper
-    Search(const OString& table_name, const SearchRequestDto::ObjectWrapper& search_request,
-           TopkResultsDto::ObjectWrapper& results_dto);
+    GetVector(const OString& table_name, const OQueryParams& query_params, OString& response);
 
     StatusDto::ObjectWrapper
-    SystemInfo(const OString& cmd, CommandDto::ObjectWrapper& cmd_dto);
+    VectorsOp(const OString& table_name, const OString& payload, OString& response);
+
+    /**
+     *
+     * System
+     */
+    StatusDto::ObjectWrapper
+    SystemInfo(const OString& cmd, const OQueryParams& query_params, OString& response_str);
 
     StatusDto::ObjectWrapper
     SystemOp(const OString& op, const OString& body_str, OString& response_str);

tests/milvus_python_test/test_table_info.py

@@ -286,4 +286,4 @@ class TestTableInfoBase:
             index_type = index_param["index_type"]
             match = self.index_string_convert(index_string, index_type)
             assert match
-            assert nb == info.partitions_stat[0].segments_stat[0].count
+            assert nb == info.partitions_stat[0].segments_stat[0].count