Enhancement for MemVector size control

Former-commit-id: b6648148d8d9ba3f894c6c8aace5c1351f4f5126

cpp/conf/server_config.yaml

@@ -10,6 +10,7 @@ db_config:
   db_backend_url: http://127.0.0.1
   db_flush_interval: 5              #unit: second
   idmapper_max_open_file: 128
+  index_trigger_size: 1024          #unit: MB
 
 metric_config:
   is_startup: true # true is on, false is off

cpp/conf/server_config_template.yaml

@@ -10,6 +10,7 @@ db_config:
   db_backend_url: http://127.0.0.1
   db_flush_interval: 5              #unit: second
   idmapper_max_open_file: 128
+  index_trigger_size: 1024          #unit: MB
 
 metric_config:
   is_startup: true # true is on, false is off

cpp/src/db/DBMetaImpl.cpp

@@ -515,7 +515,8 @@ Status DBMetaImpl::FilesToMerge(const std::string &table_id,
                                                      &TableFileSchema::size_,
                                                      &TableFileSchema::date_),
                                              where(c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW and
-                                                 c(&TableFileSchema::table_id_) == table_id));
+                                                 c(&TableFileSchema::table_id_) == table_id),
+                                             order_by(&TableFileSchema::size_).desc());
         auto end_time = METRICS_NOW_TIME;
         auto total_time = METRICS_MICROSECONDS(start_time, end_time);
         server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);

cpp/src/db/Options.h

@@ -40,9 +40,9 @@ struct DBMetaOptions {
 
 struct Options {
     Options();
-    uint16_t  memory_sync_interval = 1;
+    uint16_t  memory_sync_interval = 1;             //unit: second
     uint16_t  merge_trigger_number = 2;
-    size_t  index_trigger_size = 1024*1024*1024;
+    size_t  index_trigger_size = 1024*1024*1024;    //unit: byte
     Env* env;
     DBMetaOptions meta;
 }; // Options

cpp/src/sdk/examples/simple/src/ClientTest.cpp

@@ -20,6 +20,7 @@ namespace {
     static constexpr int64_t TOTAL_ROW_COUNT = 100000;
     static constexpr int64_t TOP_K = 10;
     static constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different
+    static constexpr int64_t ADD_VECTOR_LOOP = 1;
 
 #define BLOCK_SPLITER std::cout << "===========================================" << std::endl;
 
@@ -156,9 +157,9 @@ ClientTest::Test(const std::string& address, const std::string& port) {
 
     {//create table
         TableSchema tb_schema = BuildTableSchema();
-        PrintTableSchema(tb_schema);
         Status stat = conn->CreateTable(tb_schema);
         std::cout << "CreateTable function call status: " << stat.ToString() << std::endl;
+        PrintTableSchema(tb_schema);
     }
 
     {//describe table
@@ -168,9 +169,9 @@ ClientTest::Test(const std::string& address, const std::string& port) {
         PrintTableSchema(tb_schema);
     }
 
-    {//add vectors
+    for(int i = 0; i < ADD_VECTOR_LOOP; i++){//add vectors
         std::vector<RowRecord> record_array;
-        BuildVectors(0, TOTAL_ROW_COUNT, record_array);
+        BuildVectors(i*TOTAL_ROW_COUNT, (i+1)*TOTAL_ROW_COUNT, record_array);
         std::vector<int64_t> record_ids;
         Status stat = conn->AddVector(TABLE_NAME, record_array, record_ids);
         std::cout << "AddVector function call status: " << stat.ToString() << std::endl;

cpp/src/server/RequestTask.cpp

@@ -36,6 +36,10 @@ namespace {
             std::string db_path = config.GetValue(CONFIG_DB_PATH);
             opt.memory_sync_interval = (uint16_t)config.GetInt32Value(CONFIG_DB_FLUSH_INTERVAL, 10);
             opt.meta.path = db_path + "/db";
+            int64_t index_size = config.GetInt64Value(CONFIG_DB_INDEX_TRIGGER_SIZE);
+            if(index_size > 0) {//ensure larger than zero
+                opt.index_trigger_size = (size_t)index_size;
+            }
 
             CommonUtil::CreateDirectory(opt.meta.path);
 

cpp/src/server/ServerConfig.h

@@ -25,6 +25,7 @@ static const std::string CONFIG_DB_URL = "db_backend_url";
 static const std::string CONFIG_DB_PATH = "db_path";
 static const std::string CONFIG_DB_FLUSH_INTERVAL = "db_flush_interval";
 static const std::string CONFIG_DB_IDMAPPER_MAX_FILE = "idmapper_max_open_file";
+static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_trigger_size";
 
 static const std::string CONFIG_LOG = "log_config";