milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'branch-0.4.0' into 'branch-0.4.0' 

fix index nlist bug

See merge request megasearch/milvus!408

Former-commit-id: 6fe8f7a888b681c7199d1730fab93849a8b29370
pull/191/head
peng.xu 2019-08-23 10:44:56 +08:00
parent 6373c486a4 2c69cc3734
commit 0db8b8789c
14 changed files with 149 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
cpp/src/db/DBImpl.cpp
View File

@ -130,7 +130,7 @@ Status DBImpl::PreloadTable(const std::string &table_id) {
for(auto &day_files : files) {
for (auto &file : day_files.second) {
ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_);
ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, (MetricType)file.metric_type_, file.nlist_);
if(engine == nullptr) {
ENGINE_LOG_ERROR << "Invalid engine type";
return Status::Error("Invalid engine type");
@ -411,7 +411,8 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
//step 2: merge files
ExecutionEnginePtr index =
EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType)table_file.engine_type_);
EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType)table_file.engine_type_,
(MetricType)table_file.metric_type_, table_file.nlist_);
meta::TableFilesSchema updated;
long index_size = 0;
@ -613,7 +614,9 @@ Status DBImpl::DropIndex(const std::string& table_id) {
}
Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
ExecutionEnginePtr to_index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_);
ExecutionEnginePtr to_index =
EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_,
(MetricType)file.metric_type_, file.nlist_);
if(to_index == nullptr) {
ENGINE_LOG_ERROR << "Invalid engine type";
return Status::Error("Invalid engine type");

49
cpp/src/db/engine/EngineFactory.cpp
View File

@ -4,7 +4,6 @@
* Proprietary and confidential.
******************************************************************************/
#include "EngineFactory.h"
//#include "FaissExecutionEngine.h"
#include "ExecutionEngineImpl.h"
#include "db/Log.h"
@ -12,61 +11,25 @@ namespace zilliz {
namespace milvus {
namespace engine {
#if 0
ExecutionEnginePtr
EngineFactory::Build(uint16_t dimension,
const std::string &location,
EngineType type) {
EngineType index_type,
MetricType metric_type,
int32_t nlist) {
ExecutionEnginePtr execution_engine_ptr;
switch (type) {
case EngineType::FAISS_IDMAP: {
execution_engine_ptr =
ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, BUILD_INDEX_TYPE_IDMAP, "IDMap,Flat"));
break;
}
case EngineType::FAISS_IVFFLAT_GPU: {
execution_engine_ptr =
ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, BUILD_INDEX_TYPE_IVF, "IDMap,Flat"));
break;
}
case EngineType::FAISS_IVFSQ8: {
execution_engine_ptr =
ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, BUILD_INDEX_TYPE_IVFSQ8, "IDMap,Flat"));
break;
}
default: {
ENGINE_LOG_ERROR << "Unsupported engine type";
return nullptr;
}
}
execution_engine_ptr->Init();
return execution_engine_ptr;
}
#else
ExecutionEnginePtr
EngineFactory::Build(uint16_t dimension,
const std::string &location,
EngineType type) {
if(type == EngineType::INVALID) {
if(index_type == EngineType::INVALID) {
ENGINE_LOG_ERROR << "Unsupported engine type";
return nullptr;
}
ENGINE_LOG_DEBUG << "EngineFactory EngineTypee: " << int(type);
ENGINE_LOG_DEBUG << "EngineFactory EngineTypee: " << (int)index_type;
ExecutionEnginePtr execution_engine_ptr =
std::make_shared<ExecutionEngineImpl>(dimension, location, type);
std::make_shared<ExecutionEngineImpl>(dimension, location, index_type, metric_type, nlist);
execution_engine_ptr->Init();
return execution_engine_ptr;
}
#endif
}
}

4
cpp/src/db/engine/EngineFactory.h
View File

@ -16,7 +16,9 @@ class EngineFactory {
public:
static ExecutionEnginePtr Build(uint16_t dimension,
const std::string& location,
EngineType type);
EngineType index_type,
MetricType metric_type,
int32_t nlist);
};
}

4
cpp/src/db/engine/ExecutionEngine.h
View File

@ -67,6 +67,10 @@ public:
virtual Status GpuCache(uint64_t gpu_id) = 0;
virtual Status Init() = 0;
virtual EngineType IndexEngineType() const = 0;
virtual MetricType IndexMetricType() const = 0;
};
using ExecutionEnginePtr = std::shared_ptr<ExecutionEngine>;

52
cpp/src/db/engine/ExecutionEngineImpl.cpp
View File

@ -6,7 +6,6 @@
#include <stdexcept>
#include "src/cache/GpuCacheMgr.h"
#include "src/server/ServerConfig.h"
#include "src/metrics/Metrics.h"
#include "db/Log.h"
#include "utils/CommonUtil.h"
@ -23,26 +22,23 @@ namespace zilliz {
namespace milvus {
namespace engine {
namespace {
std::string GetMetricType() {
server::ServerConfig &config = server::ServerConfig::GetInstance();
server::ConfigNode engine_config = config.GetConfig(server::CONFIG_ENGINE);
return engine_config.GetValue(server::CONFIG_METRICTYPE, "L2");
}
}
ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension,
const std::string &location,
EngineType type)
: location_(location), dim(dimension), build_type(type) {
current_type = EngineType::FAISS_IDMAP;
EngineType index_type,
MetricType metric_type,
int32_t nlist)
: location_(location),
dim_(dimension),
index_type_(index_type),
metric_type_(metric_type),
nlist_(nlist) {
index_ = CreatetVecIndex(EngineType::FAISS_IDMAP);
if (!index_) throw Exception("Create Empty VecIndex");
Config build_cfg;
build_cfg["dim"] = dimension;
build_cfg["metric_type"] = GetMetricType();
build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2";
AutoGenParams(index_->GetType(), 0, build_cfg);
auto ec = std::static_pointer_cast<BFIndex>(index_)->Build(build_cfg);
if (ec != server::KNOWHERE_SUCCESS) { throw Exception("Build index error"); }
@ -50,9 +46,14 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension,
ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index,
const std::string &location,
EngineType type)
: index_(std::move(index)), location_(location), build_type(type) {
current_type = type;
EngineType index_type,
MetricType metric_type,
int32_t nlist)
: index_(std::move(index)),
location_(location),
index_type_(index_type),
metric_type_(metric_type),
nlist_(nlist) {
}
VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
@ -237,15 +238,15 @@ ExecutionEngineImpl::BuildIndex(const std::string &location) {
ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_;
auto from_index = std::dynamic_pointer_cast<BFIndex>(index_);
auto to_index = CreatetVecIndex(build_type);
auto to_index = CreatetVecIndex(index_type_);
if (!to_index) {
throw Exception("Create Empty VecIndex");
}
Config build_cfg;
build_cfg["dim"] = Dimension();
build_cfg["metric_type"] = GetMetricType();
build_cfg["gpu_id"] = gpu_num;
build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2";
build_cfg["gpu_id"] = gpu_num_;
build_cfg["nlist"] = nlist_;
AutoGenParams(to_index->GetType(), Count(), build_cfg);
@ -255,7 +256,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location) {
build_cfg);
if (ec != server::KNOWHERE_SUCCESS) { throw Exception("Build index error"); }
return std::make_shared<ExecutionEngineImpl>(to_index, location, build_type);
return std::make_shared<ExecutionEngineImpl>(to_index, location, index_type_, metric_type_, nlist_);
}
Status ExecutionEngineImpl::Search(long n,
@ -288,16 +289,7 @@ Status ExecutionEngineImpl::Init() {
using namespace zilliz::milvus::server;
ServerConfig &config = ServerConfig::GetInstance();
ConfigNode server_config = config.GetConfig(CONFIG_SERVER);
gpu_num = server_config.GetInt32Value("gpu_index", 0);
switch (build_type) {
case EngineType::FAISS_IVFSQ8:
case EngineType::FAISS_IVFFLAT: {
ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE);
nlist_ = engine_config.GetInt32Value(CONFIG_NLIST, 16384);
break;
}
}
gpu_num_ = server_config.GetInt32Value("gpu_index", 0);
return Status::OK();
}

22
cpp/src/db/engine/ExecutionEngineImpl.h
View File

@ -22,11 +22,15 @@ public:
ExecutionEngineImpl(uint16_t dimension,
const std::string &location,
EngineType type);
EngineType index_type,
MetricType metric_type,
int32_t nlist);
ExecutionEngineImpl(VecIndexPtr index,
const std::string &location,
EngineType type);
EngineType index_type,
MetricType metric_type,
int32_t nlist);
Status AddWithIds(long n, const float *xdata, const long *xids) override;
@ -63,6 +67,10 @@ public:
Status Init() override;
EngineType IndexEngineType() const override { return index_type_; }
MetricType IndexMetricType() const override { return metric_type_; }
private:
VecIndexPtr CreatetVecIndex(EngineType type);
@ -70,14 +78,14 @@ private:
protected:
VecIndexPtr index_ = nullptr;
EngineType build_type;
EngineType current_type;
EngineType index_type_;
MetricType metric_type_;
int64_t dim;
int64_t dim_;
std::string location_;
size_t nlist_ = 0;
int64_t gpu_num = 0;
int32_t nlist_ = 0;
int64_t gpu_num_ = 0;
};

4
cpp/src/db/insert/MemTableFile.cpp
View File

@ -23,7 +23,9 @@ MemTableFile::MemTableFile(const std::string &table_id,
if (status.ok()) {
execution_engine_ = EngineFactory::Build(table_file_schema_.dimension_,
table_file_schema_.location_,
(EngineType) table_file_schema_.engine_type_);
(EngineType) table_file_schema_.engine_type_,
(MetricType)table_file_schema_.metric_type_,
table_file_schema_.nlist_);
}
}

17
cpp/src/db/meta/MetaTypes.h
View File

@ -17,6 +17,11 @@ namespace milvus {
namespace engine {
namespace meta {
constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP;
constexpr int32_t DEFAULT_NLIST = 16384;
constexpr int32_t DEFAULT_INDEX_FILE_SIZE = 1024*ONE_MB;
constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2;
typedef int DateT;
const DateT EmptyDate = -1;
typedef std::vector<DateT> DatesT;
@ -32,10 +37,10 @@ struct TableSchema {
int32_t state_ = (int)NORMAL;
uint16_t dimension_ = 0;
int64_t created_on_ = 0;
int32_t engine_type_ = (int)EngineType::FAISS_IDMAP;
int32_t nlist_ = 16384;
int32_t index_file_size_ = 1024*ONE_MB;
int32_t metric_type_ = (int)MetricType::L2;
int32_t engine_type_ = DEFAULT_ENGINE_TYPE;
int32_t nlist_ = DEFAULT_NLIST;
int32_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE;
int32_t metric_type_ = DEFAULT_METRIC_TYPE;
}; // TableSchema
struct TableFileSchema {
@ -52,7 +57,6 @@ struct TableFileSchema {
size_t id_ = 0;
std::string table_id_;
int32_t engine_type_ = (int)EngineType::FAISS_IDMAP;
std::string file_id_;
int32_t file_type_ = NEW;
size_t file_size_ = 0;
@ -62,6 +66,9 @@ struct TableFileSchema {
std::string location_;
int64_t updated_time_ = 0;
int64_t created_on_ = 0;
int32_t engine_type_ = DEFAULT_ENGINE_TYPE;
int32_t nlist_ = DEFAULT_NLIST; //not persist to meta
int32_t metric_type_ = DEFAULT_METRIC_TYPE; //not persist to meta
}; // TableFileSchema
typedef std::vector<TableFileSchema> TableFilesSchema;

28
cpp/src/db/meta/MySQLMetaImpl.cpp
View File

@ -747,7 +747,7 @@ Status MySQLMetaImpl::AllTables(std::vector<TableSchema> &table_schema_array) {
}
Query allTablesQuery = connectionPtr->query();
allTablesQuery << "SELECT id, table_id, dimension, engine_type " <<
allTablesQuery << "SELECT id, table_id, dimension, engine_type, nlist, index_file_size, metric_type " <<
"FROM Tables " <<
"WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";";
@ -769,6 +769,12 @@ Status MySQLMetaImpl::AllTables(std::vector<TableSchema> &table_schema_array) {
table_schema.engine_type_ = resRow["engine_type"];
table_schema.nlist_ = resRow["nlist"];
table_schema.index_file_size_ = resRow["index_file_size"];
table_schema.metric_type_ = resRow["metric_type"];
table_schema_array.emplace_back(table_schema);
}
} catch (const BadQuery &er) {
@ -805,6 +811,8 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
file_schema.created_on_ = utils::GetMicroSecTimeStamp();
file_schema.updated_time_ = file_schema.created_on_;
file_schema.engine_type_ = table_schema.engine_type_;
file_schema.nlist_ = table_schema.nlist_;
file_schema.metric_type_ = table_schema.metric_type_;
utils::GetTableFilePath(options_, file_schema);
std::string id = "NULL"; //auto-increment
@ -918,6 +926,8 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) {
groups[table_file.table_id_] = table_schema;
}
table_file.metric_type_ = groups[table_file.table_id_].metric_type_;
table_file.nlist_ = groups[table_file.table_id_].nlist_;
table_file.dimension_ = groups[table_file.table_id_].dimension_;
utils::GetTableFilePath(options_, table_file);
@ -1010,6 +1020,10 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
table_file.engine_type_ = resRow["engine_type"];
table_file.metric_type_ = table_schema.metric_type_;
table_file.nlist_ = table_schema.nlist_;
std::string file_id;
resRow["file_id"].to_string(file_id);
table_file.file_id_ = file_id;
@ -1118,6 +1132,10 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id,
table_file.engine_type_ = resRow["engine_type"];
table_file.metric_type_ = table_schema.metric_type_;
table_file.nlist_ = table_schema.nlist_;
std::string file_id;
resRow["file_id"].to_string(file_id);
table_file.file_id_ = file_id;
@ -1214,6 +1232,10 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id,
table_file.engine_type_ = resRow["engine_type"];
table_file.metric_type_ = table_schema.metric_type_;
table_file.nlist_ = table_schema.nlist_;
table_file.created_on_ = resRow["created_on"];
table_file.dimension_ = table_schema.dimension_;
@ -1293,6 +1315,10 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id,
file_schema.engine_type_ = resRow["engine_type"];
file_schema.metric_type_ = table_schema.metric_type_;
file_schema.nlist_ = table_schema.nlist_;
std::string file_id;
resRow["file_id"].to_string(file_id);
file_schema.file_id_ = file_id;

56
cpp/src/db/meta/SqliteMetaImpl.cpp
View File

@ -218,22 +218,15 @@ Status SqliteMetaImpl::DeleteTable(const std::string& table_id) {
std::lock_guard<std::mutex> meta_lock(meta_mutex_);
//soft delete table
auto tables = ConnectorPtr->select(columns(&TableSchema::id_,
&TableSchema::dimension_,
&TableSchema::engine_type_,
&TableSchema::created_on_),
where(c(&TableSchema::table_id_) == table_id));
for (auto &table : tables) {
TableSchema table_schema;
table_schema.table_id_ = table_id;
table_schema.state_ = (int)TableSchema::TO_DELETE;
table_schema.id_ = std::get<0>(table);
table_schema.dimension_ = std::get<1>(table);
table_schema.engine_type_ = std::get<2>(table);
table_schema.created_on_ = std::get<3>(table);
ConnectorPtr->update_all(
set(
c(&TableSchema::state_) = (int) TableSchema::TO_DELETE
),
where(
c(&TableSchema::table_id_) == table_id and
c(&TableSchema::state_) != (int) TableSchema::TO_DELETE
));
ConnectorPtr->update<TableSchema>(table_schema);
}
} catch (std::exception &e) {
return HandleException("Encounter exception when delete table", e);
}
@ -493,16 +486,24 @@ Status SqliteMetaImpl::AllTables(std::vector<TableSchema>& table_schema_array) {
MetricCollector metric;
auto selected = ConnectorPtr->select(columns(&TableSchema::id_,
&TableSchema::table_id_,
&TableSchema::dimension_,
&TableSchema::engine_type_),
&TableSchema::table_id_,
&TableSchema::dimension_,
&TableSchema::created_on_,
&TableSchema::engine_type_,
&TableSchema::nlist_,
&TableSchema::index_file_size_,
&TableSchema::metric_type_),
where(c(&TableSchema::state_) != (int)TableSchema::TO_DELETE));
for (auto &table : selected) {
TableSchema schema;
schema.id_ = std::get<0>(table);
schema.table_id_ = std::get<1>(table);
schema.dimension_ = std::get<2>(table);
schema.engine_type_ = std::get<3>(table);
schema.created_on_ = std::get<2>(table);
schema.dimension_ = std::get<3>(table);
schema.engine_type_ = std::get<4>(table);
schema.nlist_ = std::get<5>(table);
schema.index_file_size_ = std::get<6>(table);
schema.metric_type_ = std::get<7>(table);
table_schema_array.emplace_back(schema);
}
@ -535,6 +536,8 @@ Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
file_schema.created_on_ = utils::GetMicroSecTimeStamp();
file_schema.updated_time_ = file_schema.created_on_;
file_schema.engine_type_ = table_schema.engine_type_;
file_schema.nlist_ = table_schema.nlist_;
file_schema.metric_type_ = table_schema.metric_type_;
//multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
std::lock_guard<std::mutex> meta_lock(meta_mutex_);
@ -594,6 +597,8 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) {
}
groups[table_file.table_id_] = table_schema;
}
table_file.metric_type_ = groups[table_file.table_id_].metric_type_;
table_file.nlist_ = groups[table_file.table_id_].nlist_;
table_file.dimension_ = groups[table_file.table_id_].dimension_;
files.push_back(table_file);
}
@ -644,6 +649,8 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
table_file.row_count_ = std::get<5>(file);
table_file.date_ = std::get<6>(file);
table_file.engine_type_ = std::get<7>(file);
table_file.metric_type_ = table_schema.metric_type_;
table_file.nlist_ = table_schema.nlist_;
table_file.dimension_ = table_schema.dimension_;
utils::GetTableFilePath(options_, table_file);
auto dateItr = files.find(table_file.date_);
@ -685,6 +692,8 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
table_file.row_count_ = std::get<5>(file);
table_file.date_ = std::get<6>(file);
table_file.engine_type_ = std::get<7>(file);
table_file.metric_type_ = table_schema.metric_type_;
table_file.nlist_ = table_schema.nlist_;
table_file.dimension_ = table_schema.dimension_;
utils::GetTableFilePath(options_, table_file);
auto dateItr = files.find(table_file.date_);
@ -762,6 +771,8 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
table_file.date_ = std::get<6>(file);
table_file.engine_type_ = std::get<7>(file);
table_file.dimension_ = table_schema.dimension_;
table_file.metric_type_ = table_schema.metric_type_;
table_file.nlist_ = table_schema.nlist_;
utils::GetTableFilePath(options_, table_file);
auto dateItr = files.find(table_file.date_);
if (dateItr == files.end()) {
@ -820,6 +831,8 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id,
table_file.date_ = std::get<6>(file);
table_file.created_on_ = std::get<7>(file);
table_file.dimension_ = table_schema.dimension_;
table_file.metric_type_ = table_schema.metric_type_;
table_file.nlist_ = table_schema.nlist_;
utils::GetTableFilePath(options_, table_file);
auto dateItr = files.find(table_file.date_);
if (dateItr == files.end()) {
@ -868,8 +881,11 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id,
file_schema.row_count_ = std::get<4>(file);
file_schema.date_ = std::get<5>(file);
file_schema.engine_type_ = std::get<6>(file);
file_schema.metric_type_ = table_schema.metric_type_;
file_schema.nlist_ = table_schema.nlist_;
file_schema.created_on_ = std::get<7>(file);
file_schema.dimension_ = table_schema.dimension_;
utils::GetTableFilePath(options_, file_schema);
table_files.emplace_back(file_schema);

6
cpp/src/db/scheduler/task/IndexLoadTask.cpp
View File

@ -45,7 +45,9 @@ std::shared_ptr<IScheduleTask> IndexLoadTask::Execute() {
//step 1: load index
ExecutionEnginePtr index_ptr = EngineFactory::Build(file_->dimension_,
file_->location_,
(EngineType)file_->engine_type_);
(EngineType)file_->engine_type_,
(MetricType)file_->metric_type_,
file_->nlist_);
try {
index_ptr->Load();
@ -75,7 +77,7 @@ std::shared_ptr<IScheduleTask> IndexLoadTask::Execute() {
//step 2: return search task for later execution
SearchTaskPtr task_ptr = std::make_shared<SearchTask>();
task_ptr->index_id_ = file_->id_;
task_ptr->index_type_ = file_->file_type_;
task_ptr->file_type_ = file_->file_type_;
task_ptr->index_engine_ = index_ptr;
task_ptr->search_contexts_.swap(search_contexts_);
return std::static_pointer_cast<IScheduleTask>(task_ptr);

14
cpp/src/db/scheduler/task/SearchTask.cpp
View File

@ -76,20 +76,10 @@ void CollectDurationMetrics(int index_type, double total_time) {
}
}
std::string GetMetricType() {
server::ServerConfig &config = server::ServerConfig::GetInstance();
server::ConfigNode& engine_config = config.GetConfig(server::CONFIG_ENGINE);
return engine_config.GetValue(server::CONFIG_METRICTYPE, "L2");
}
}
SearchTask::SearchTask()
: IScheduleTask(ScheduleTaskType::kSearch) {
std::string metric_type = GetMetricType();
if(metric_type != "L2") {
metric_l2 = false;
}
}
std::shared_ptr<IScheduleTask> SearchTask::Execute() {
@ -104,6 +94,8 @@ std::shared_ptr<IScheduleTask> SearchTask::Execute() {
auto start_time = METRICS_NOW_TIME;
bool metric_l2 = (index_engine_->IndexMetricType() == MetricType::L2);
std::vector<long> output_ids;
std::vector<float> output_distence;
for(auto& context : search_contexts_) {
@ -147,7 +139,7 @@ std::shared_ptr<IScheduleTask> SearchTask::Execute() {
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
CollectDurationMetrics(index_type_, total_time);
CollectDurationMetrics(file_type_, total_time);
rc.ElapseFromBegin("totally cost");

3
cpp/src/db/scheduler/task/SearchTask.h
View File

@ -37,10 +37,9 @@ public:
public:
size_t index_id_ = 0;
int index_type_ = 0; //for metrics
int file_type_ = 0; //for metrics
ExecutionEnginePtr index_engine_;
std::vector<SearchContextPtr> search_contexts_;
bool metric_l2 = true;
};
using SearchTaskPtr = std::shared_ptr<SearchTask>;

10
cpp/unittest/db/mem_test.cpp
View File

@ -65,9 +65,13 @@ TEST_F(NewMemManagerTest, VECTOR_SOURCE_TEST) {
engine::VectorSource source(n, vectors.data());
size_t num_vectors_added;
engine::ExecutionEnginePtr execution_engine_ = engine::EngineFactory::Build(table_file_schema.dimension_,
table_file_schema.location_,
(engine::EngineType) table_file_schema.engine_type_);
engine::ExecutionEnginePtr execution_engine_ =
engine::EngineFactory::Build(table_file_schema.dimension_,
table_file_schema.location_,
(engine::EngineType) table_file_schema.engine_type_,
(engine::MetricType)table_file_schema.metric_type_,
table_schema.nlist_);
engine::IDNumbers vector_ids;
status = source.Add(execution_engine_, table_file_schema, 50, num_vectors_added, vector_ids);
ASSERT_TRUE(status.ok());