milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

format code 

Former-commit-id: 648cc3be7f2dc4cca5c1394ff83c2d383c7b70b7
pull/191/head
starlord 2019-10-10 16:03:29 +08:00
parent 6769b6d960
commit defd24b2c6
26 changed files with 850 additions and 927 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cpp/src/core/unittest/kdtree.cpp
View File

@ -22,13 +22,8 @@
//#include "knowhere/index/vector_index/cpu_kdt_rng.h"
//#include "knowhere/index/vector_index/definitions.h"
//
// namespace {
//
// namespace kn = knowhere;
//
//} // namespace
//
// kn::DatasetPtr
// knowhere::DatasetPtr
// generate_dataset(int64_t n, int64_t d, int64_t base) {
// auto elems = n * d;
// auto p_data = (float*)malloc(elems * sizeof(float));
@ -58,7 +53,7 @@
// return dataset;
//}
//
// kn::DatasetPtr
// knowhere::DatasetPtr
// generate_queries(int64_t n, int64_t d, int64_t k, int64_t base) {
// size_t size = sizeof(float) * n * d;
// auto v = (float*)malloc(size);
@ -86,7 +81,7 @@
// std::vector<FieldPtr> fields{field};
// auto schema = std::make_shared<Schema>(fields);
//
// return std::make_shared<kn::Dataset>(data, schema);
// return std::make_shared<knowhere::Dataset>(data, schema);
//}
//
// int

53
cpp/src/core/unittest/test_idmap.cpp
View File

@ -25,33 +25,27 @@
#include "unittest/utils.h"
namespace {
namespace kn = knowhere;
} // namespace
static int device_id = 0;
class IDMAPTest : public DataGen, public ::testing::Test {
protected:
void
SetUp() override {
kn::FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024 * 1024 * 200, 1024 * 1024 * 300, 2);
knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024 * 1024 * 200, 1024 * 1024 * 300, 2);
Init_with_default();
index_ = std::make_shared<kn::IDMAP>();
index_ = std::make_shared<knowhere::IDMAP>();
}
void
TearDown() override {
kn::FaissGpuResourceMgr::GetInstance().Free();
knowhere::FaissGpuResourceMgr::GetInstance().Free();
}
protected:
kn::IDMAPPtr index_ = nullptr;
knowhere::IDMAPPtr index_ = nullptr;
};
void
AssertAnns(const kn::DatasetPtr& result, const int& nq, const int& k) {
AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) {
auto ids = result->array()[0];
for (auto i = 0; i < nq; i++) {
EXPECT_EQ(i, *(ids->data()->GetValues<int64_t>(1, i * k)));
@ -59,7 +53,7 @@ AssertAnns(const kn::DatasetPtr& result, const int& nq, const int& k) {
}
void
PrintResult(const kn::DatasetPtr& result, const int& nq, const int& k) {
PrintResult(const knowhere::DatasetPtr& result, const int& nq, const int& k) {
auto ids = result->array()[0];
auto dists = result->array()[1];
@ -80,10 +74,10 @@ PrintResult(const kn::DatasetPtr& result, const int& nq, const int& k) {
TEST_F(IDMAPTest, idmap_basic) {
ASSERT_TRUE(!xb.empty());
auto conf = std::make_shared<kn::Cfg>();
auto conf = std::make_shared<knowhere::Cfg>();
conf->d = dim;
conf->k = k;
conf->metric_type = kn::METRICTYPE::L2;
conf->metric_type = knowhere::METRICTYPE::L2;
index_->Train(conf);
index_->Add(base_dataset, conf);
@ -97,7 +91,7 @@ TEST_F(IDMAPTest, idmap_basic) {
index_->Seal();
auto binaryset = index_->Serialize();
auto new_index = std::make_shared<kn::IDMAP>();
auto new_index = std::make_shared<knowhere::IDMAP>();
new_index->Load(binaryset);
auto re_result = index_->Search(query_dataset, conf);
AssertAnns(re_result, nq, k);
@ -105,7 +99,7 @@ TEST_F(IDMAPTest, idmap_basic) {
}
TEST_F(IDMAPTest, idmap_serialize) {
auto serialize = [](const std::string& filename, kn::BinaryPtr& bin, uint8_t* ret) {
auto serialize = [](const std::string& filename, knowhere::BinaryPtr& bin, uint8_t* ret) {
FileIOWriter writer(filename);
writer(static_cast<void*>(bin->data.get()), bin->size);
@ -113,15 +107,15 @@ TEST_F(IDMAPTest, idmap_serialize) {
reader(ret, bin->size);
};
auto conf = std::make_shared<kn::Cfg>();
auto conf = std::make_shared<knowhere::Cfg>();
conf->d = dim;
conf->k = k;
conf->metric_type = kn::METRICTYPE::L2;
conf->metric_type = knowhere::METRICTYPE::L2;
{
// serialize index
index_->Train(conf);
index_->Add(base_dataset, kn::Config());
index_->Add(base_dataset, knowhere::Config());
auto re_result = index_->Search(query_dataset, conf);
AssertAnns(re_result, nq, k);
PrintResult(re_result, nq, k);
@ -151,10 +145,10 @@ TEST_F(IDMAPTest, idmap_serialize) {
TEST_F(IDMAPTest, copy_test) {
ASSERT_TRUE(!xb.empty());
auto conf = std::make_shared<kn::Cfg>();
auto conf = std::make_shared<knowhere::Cfg>();
conf->d = dim;
conf->k = k;
conf->metric_type = kn::METRICTYPE::L2;
conf->metric_type = knowhere::METRICTYPE::L2;
index_->Train(conf);
index_->Add(base_dataset, conf);
@ -175,12 +169,12 @@ TEST_F(IDMAPTest, copy_test) {
{
// cpu to gpu
auto clone_index = kn::cloner::CopyCpuToGpu(index_, device_id, conf);
auto clone_index = knowhere::cloner::CopyCpuToGpu(index_, device_id, conf);
auto clone_result = clone_index->Search(query_dataset, conf);
AssertAnns(clone_result, nq, k);
ASSERT_THROW({ std::static_pointer_cast<kn::GPUIDMAP>(clone_index)->GetRawVectors(); },
ASSERT_THROW({ std::static_pointer_cast<knowhere::GPUIDMAP>(clone_index)->GetRawVectors(); },
knowhere::KnowhereException);
ASSERT_THROW({ std::static_pointer_cast<kn::GPUIDMAP>(clone_index)->GetRawIds(); },
ASSERT_THROW({ std::static_pointer_cast<knowhere::GPUIDMAP>(clone_index)->GetRawIds(); },
knowhere::KnowhereException);
auto binary = clone_index->Serialize();
@ -193,15 +187,16 @@ TEST_F(IDMAPTest, copy_test) {
AssertAnns(clone_gpu_res, nq, k);
// gpu to cpu
auto host_index = kn::cloner::CopyGpuToCpu(clone_index, conf);
auto host_index = knowhere::cloner::CopyGpuToCpu(clone_index, conf);
auto host_result = host_index->Search(query_dataset, conf);
AssertAnns(host_result, nq, k);
ASSERT_TRUE(std::static_pointer_cast<kn::IDMAP>(host_index)->GetRawVectors() != nullptr);
ASSERT_TRUE(std::static_pointer_cast<kn::IDMAP>(host_index)->GetRawIds() != nullptr);
ASSERT_TRUE(std::static_pointer_cast<knowhere::IDMAP>(host_index)->GetRawVectors() != nullptr);
ASSERT_TRUE(std::static_pointer_cast<knowhere::IDMAP>(host_index)->GetRawIds() != nullptr);
// gpu to gpu
auto device_index = kn::cloner::CopyCpuToGpu(index_, device_id, conf);
auto new_device_index = std::static_pointer_cast<kn::GPUIDMAP>(device_index)->CopyGpuToGpu(device_id, conf);
auto device_index = knowhere::cloner::CopyCpuToGpu(index_, device_id, conf);
auto new_device_index =
std::static_pointer_cast<knowhere::GPUIDMAP>(device_index)->CopyGpuToGpu(device_id, conf);
auto device_result = new_device_index->Search(query_dataset, conf);
AssertAnns(device_result, nq, k);
}

146
cpp/src/core/unittest/test_ivf.cpp
View File

@ -37,12 +37,6 @@
#include "unittest/utils.h"
namespace {
namespace kn = knowhere;
} // namespace
using ::testing::Combine;
using ::testing::TestWithParam;
using ::testing::Values;
@ -53,22 +47,22 @@ constexpr int64_t NB = 1000000 / 100;
constexpr int64_t NQ = 10;
constexpr int64_t K = 10;
kn::IVFIndexPtr
knowhere::IVFIndexPtr
IndexFactory(const std::string& type) {
if (type == "IVF") {
return std::make_shared<kn::IVF>();
return std::make_shared<knowhere::IVF>();
} else if (type == "IVFPQ") {
return std::make_shared<kn::IVFPQ>();
return std::make_shared<knowhere::IVFPQ>();
} else if (type == "GPUIVF") {
return std::make_shared<kn::GPUIVF>(device_id);
return std::make_shared<knowhere::GPUIVF>(device_id);
} else if (type == "GPUIVFPQ") {
return std::make_shared<kn::GPUIVFPQ>(device_id);
return std::make_shared<knowhere::GPUIVFPQ>(device_id);
} else if (type == "IVFSQ") {
return std::make_shared<kn::IVFSQ>();
return std::make_shared<knowhere::IVFSQ>();
} else if (type == "GPUIVFSQ") {
return std::make_shared<kn::GPUIVFSQ>(device_id);
return std::make_shared<knowhere::GPUIVFSQ>(device_id);
} else if (type == "IVFSQHybrid") {
return std::make_shared<kn::IVFSQHybrid>(device_id);
return std::make_shared<knowhere::IVFSQHybrid>(device_id);
}
}
@ -88,19 +82,19 @@ class ParamGenerator {
return instance;
}
kn::Config
knowhere::Config
Gen(const ParameterType& type) {
if (type == ParameterType::ivf) {
auto tempconf = std::make_shared<kn::IVFCfg>();
auto tempconf = std::make_shared<knowhere::IVFCfg>();
tempconf->d = DIM;
tempconf->gpu_id = device_id;
tempconf->nlist = 100;
tempconf->nprobe = 16;
tempconf->k = K;
tempconf->metric_type = kn::METRICTYPE::L2;
tempconf->metric_type = knowhere::METRICTYPE::L2;
return tempconf;
} else if (type == ParameterType::ivfpq) {
auto tempconf = std::make_shared<kn::IVFPQCfg>();
auto tempconf = std::make_shared<knowhere::IVFPQCfg>();
tempconf->d = DIM;
tempconf->gpu_id = device_id;
tempconf->nlist = 100;
@ -108,17 +102,17 @@ class ParamGenerator {
tempconf->k = K;
tempconf->m = 8;
tempconf->nbits = 8;
tempconf->metric_type = kn::METRICTYPE::L2;
tempconf->metric_type = knowhere::METRICTYPE::L2;
return tempconf;
} else if (type == ParameterType::ivfsq || type == ParameterType::ivfsqhybrid) {
auto tempconf = std::make_shared<kn::IVFSQCfg>();
auto tempconf = std::make_shared<knowhere::IVFSQCfg>();
tempconf->d = DIM;
tempconf->gpu_id = device_id;
tempconf->nlist = 100;
tempconf->nprobe = 16;
tempconf->k = K;
tempconf->nbits = 8;
tempconf->metric_type = kn::METRICTYPE::L2;
tempconf->metric_type = knowhere::METRICTYPE::L2;
return tempconf;
}
}
@ -134,28 +128,28 @@ class IVFTest : public DataGen, public TestWithParam<::std::tuple<std::string, P
Generate(DIM, NB, NQ);
index_ = IndexFactory(index_type);
conf = ParamGenerator::GetInstance().Gen(parameter_type);
kn::FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024 * 1024 * 200, 1024 * 1024 * 600, 2);
knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024 * 1024 * 200, 1024 * 1024 * 600, 2);
}
void
TearDown() override {
kn::FaissGpuResourceMgr::GetInstance().Free();
knowhere::FaissGpuResourceMgr::GetInstance().Free();
}
kn::VectorIndexPtr
knowhere::VectorIndexPtr
ChooseTodo() {
std::vector<std::string> gpu_idx{"GPUIVFSQ"};
auto finder = std::find(gpu_idx.cbegin(), gpu_idx.cend(), index_type);
if (finder != gpu_idx.cend()) {
return kn::cloner::CopyCpuToGpu(index_, device_id, kn::Config());
return knowhere::cloner::CopyCpuToGpu(index_, device_id, knowhere::Config());
}
return index_;
}
protected:
std::string index_type;
kn::Config conf;
kn::IVFIndexPtr index_ = nullptr;
knowhere::Config conf;
knowhere::IVFIndexPtr index_ = nullptr;
};
INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest,
@ -168,7 +162,7 @@ INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest,
std::make_tuple("IVFSQHybrid", ParameterType::ivfsqhybrid)));
void
AssertAnns(const kn::DatasetPtr& result, const int& nq, const int& k) {
AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) {
auto ids = result->array()[0];
for (auto i = 0; i < nq; i++) {
EXPECT_EQ(i, *(ids->data()->GetValues<int64_t>(1, i * k)));
@ -176,7 +170,7 @@ AssertAnns(const kn::DatasetPtr& result, const int& nq, const int& k) {
}
void
PrintResult(const kn::DatasetPtr& result, const int& nq, const int& k) {
PrintResult(const knowhere::DatasetPtr& result, const int& nq, const int& k) {
auto ids = result->array()[0];
auto dists = result->array()[1];
@ -232,12 +226,12 @@ TEST_P(IVFTest, hybrid) {
// AssertAnns(result, nq, conf->k);
{
auto hybrid_1_idx = std::make_shared<kn::IVFSQHybrid>(device_id);
auto hybrid_1_idx = std::make_shared<knowhere::IVFSQHybrid>(device_id);
auto binaryset = index_->Serialize();
hybrid_1_idx->Load(binaryset);
auto quantizer_conf = std::make_shared<kn::QuantizerCfg>();
auto quantizer_conf = std::make_shared<knowhere::QuantizerCfg>();
quantizer_conf->mode = 1;
quantizer_conf->gpu_id = device_id;
auto q = hybrid_1_idx->LoadQuantizer(quantizer_conf);
@ -248,12 +242,12 @@ TEST_P(IVFTest, hybrid) {
}
{
auto hybrid_2_idx = std::make_shared<kn::IVFSQHybrid>(device_id);
auto hybrid_2_idx = std::make_shared<knowhere::IVFSQHybrid>(device_id);
auto binaryset = index_->Serialize();
hybrid_2_idx->Load(binaryset);
auto quantizer_conf = std::make_shared<kn::QuantizerCfg>();
auto quantizer_conf = std::make_shared<knowhere::QuantizerCfg>();
quantizer_conf->mode = 1;
quantizer_conf->gpu_id = device_id;
auto q = hybrid_2_idx->LoadQuantizer(quantizer_conf);
@ -291,7 +285,7 @@ TEST_P(IVFTest, hybrid) {
//}
TEST_P(IVFTest, ivf_serialize) {
auto serialize = [](const std::string& filename, kn::BinaryPtr& bin, uint8_t* ret) {
auto serialize = [](const std::string& filename, knowhere::BinaryPtr& bin, uint8_t* ret) {
FileIOWriter writer(filename);
writer(static_cast<void*>(bin->data.get()), bin->size);
@ -365,7 +359,7 @@ TEST_P(IVFTest, clone_test) {
AssertAnns(result, nq, conf->k);
// PrintResult(result, nq, k);
auto AssertEqual = [&](kn::DatasetPtr p1, kn::DatasetPtr p2) {
auto AssertEqual = [&](knowhere::DatasetPtr p1, knowhere::DatasetPtr p2) {
auto ids_p1 = p1->array()[0];
auto ids_p2 = p2->array()[0];
@ -406,7 +400,7 @@ TEST_P(IVFTest, clone_test) {
auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type);
if (finder != support_idx_vec.cend()) {
EXPECT_NO_THROW({
auto clone_index = kn::cloner::CopyGpuToCpu(index_, kn::Config());
auto clone_index = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config());
auto clone_result = clone_index->Search(query_dataset, conf);
AssertEqual(result, clone_result);
std::cout << "clone G <=> C [" << index_type << "] success" << std::endl;
@ -415,9 +409,9 @@ TEST_P(IVFTest, clone_test) {
EXPECT_THROW(
{
std::cout << "clone G <=> C [" << index_type << "] failed" << std::endl;
auto clone_index = kn::cloner::CopyGpuToCpu(index_, kn::Config());
auto clone_index = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config());
},
kn::KnowhereException);
knowhere::KnowhereException);
}
}
@ -427,7 +421,7 @@ TEST_P(IVFTest, clone_test) {
auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type);
if (finder != support_idx_vec.cend()) {
EXPECT_NO_THROW({
auto clone_index = kn::cloner::CopyCpuToGpu(index_, device_id, kn::Config());
auto clone_index = knowhere::cloner::CopyCpuToGpu(index_, device_id, knowhere::Config());
auto clone_result = clone_index->Search(query_dataset, conf);
AssertEqual(result, clone_result);
std::cout << "clone C <=> G [" << index_type << "] success" << std::endl;
@ -436,9 +430,9 @@ TEST_P(IVFTest, clone_test) {
EXPECT_THROW(
{
std::cout << "clone C <=> G [" << index_type << "] failed" << std::endl;
auto clone_index = kn::cloner::CopyCpuToGpu(index_, device_id, kn::Config());
auto clone_index = knowhere::cloner::CopyCpuToGpu(index_, device_id, knowhere::Config());
},
kn::KnowhereException);
knowhere::KnowhereException);
}
}
}
@ -466,14 +460,14 @@ TEST_P(IVFTest, seal_test) {
auto result = new_idx->Search(query_dataset, conf);
AssertAnns(result, nq, conf->k);
auto cpu_idx = kn::cloner::CopyGpuToCpu(index_, kn::Config());
auto cpu_idx = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config());
kn::TimeRecorder tc("CopyToGpu");
kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config());
knowhere::TimeRecorder tc("CopyToGpu");
knowhere::cloner::CopyCpuToGpu(cpu_idx, device_id, knowhere::Config());
auto without_seal = tc.RecordSection("Without seal");
cpu_idx->Seal();
tc.RecordSection("seal cost");
kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config());
knowhere::cloner::CopyCpuToGpu(cpu_idx, device_id, knowhere::Config());
auto with_seal = tc.RecordSection("With seal");
ASSERT_GE(without_seal, with_seal);
}
@ -483,7 +477,7 @@ class GPURESTEST : public DataGen, public ::testing::Test {
void
SetUp() override {
Generate(128, 1000000, 1000);
kn::FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024 * 1024 * 200, 1024 * 1024 * 300, 2);
knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024 * 1024 * 200, 1024 * 1024 * 300, 2);
k = 100;
elems = nq * k;
@ -495,12 +489,12 @@ class GPURESTEST : public DataGen, public ::testing::Test {
TearDown() override {
delete ids;
delete dis;
kn::FaissGpuResourceMgr::GetInstance().Free();
knowhere::FaissGpuResourceMgr::GetInstance().Free();
}
protected:
std::string index_type;
kn::IVFIndexPtr index_ = nullptr;
knowhere::IVFIndexPtr index_ = nullptr;
int64_t* ids = nullptr;
float* dis = nullptr;
@ -514,16 +508,16 @@ TEST_F(GPURESTEST, gpu_ivf_resource_test) {
assert(!xb.empty());
{
index_ = std::make_shared<kn::GPUIVF>(-1);
ASSERT_EQ(std::dynamic_pointer_cast<kn::GPUIVF>(index_)->GetGpuDevice(), -1);
std::dynamic_pointer_cast<kn::GPUIVF>(index_)->SetGpuDevice(device_id);
ASSERT_EQ(std::dynamic_pointer_cast<kn::GPUIVF>(index_)->GetGpuDevice(), device_id);
index_ = std::make_shared<knowhere::GPUIVF>(-1);
ASSERT_EQ(std::dynamic_pointer_cast<knowhere::GPUIVF>(index_)->GetGpuDevice(), -1);
std::dynamic_pointer_cast<knowhere::GPUIVF>(index_)->SetGpuDevice(device_id);
ASSERT_EQ(std::dynamic_pointer_cast<knowhere::GPUIVF>(index_)->GetGpuDevice(), device_id);
auto conf = std::make_shared<kn::IVFCfg>();
auto conf = std::make_shared<knowhere::IVFCfg>();
conf->nlist = 1638;
conf->d = dim;
conf->gpu_id = device_id;
conf->metric_type = kn::METRICTYPE::L2;
conf->metric_type = knowhere::METRICTYPE::L2;
conf->k = k;
conf->nprobe = 1;
@ -535,7 +529,7 @@ TEST_F(GPURESTEST, gpu_ivf_resource_test) {
EXPECT_EQ(index_->Count(), nb);
EXPECT_EQ(index_->Dimension(), dim);
kn::TimeRecorder tc("knowere GPUIVF");
knowhere::TimeRecorder tc("knowere GPUIVF");
for (int i = 0; i < search_count; ++i) {
index_->Search(query_dataset, conf);
if (i > search_count - 6 || i < 5)
@ -543,7 +537,7 @@ TEST_F(GPURESTEST, gpu_ivf_resource_test) {
}
tc.ElapseFromBegin("search all");
}
kn::FaissGpuResourceMgr::GetInstance().Dump();
knowhere::FaissGpuResourceMgr::GetInstance().Dump();
{
// IVF-Search
@ -554,7 +548,7 @@ TEST_F(GPURESTEST, gpu_ivf_resource_test) {
device_index.train(nb, xb.data());
device_index.add(nb, xb.data());
kn::TimeRecorder tc("ori IVF");
knowhere::TimeRecorder tc("ori IVF");
for (int i = 0; i < search_count; ++i) {
device_index.search(nq, xq.data(), k, dis, ids);
if (i > search_count - 6 || i < 5)
@ -570,11 +564,11 @@ TEST_F(GPURESTEST, gpuivfsq) {
index_type = "GPUIVFSQ";
index_ = IndexFactory(index_type);
auto conf = std::make_shared<kn::IVFSQCfg>();
auto conf = std::make_shared<knowhere::IVFSQCfg>();
conf->nlist = 1638;
conf->d = dim;
conf->gpu_id = device_id;
conf->metric_type = kn::METRICTYPE::L2;
conf->metric_type = knowhere::METRICTYPE::L2;
conf->k = k;
conf->nbits = 8;
conf->nprobe = 1;
@ -587,11 +581,11 @@ TEST_F(GPURESTEST, gpuivfsq) {
// auto result = index_->Search(query_dataset, conf);
// AssertAnns(result, nq, k);
auto cpu_idx = kn::cloner::CopyGpuToCpu(index_, kn::Config());
auto cpu_idx = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config());
cpu_idx->Seal();
kn::TimeRecorder tc("knowhere GPUSQ8");
auto search_idx = kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config());
knowhere::TimeRecorder tc("knowhere GPUSQ8");
auto search_idx = knowhere::cloner::CopyCpuToGpu(cpu_idx, device_id, knowhere::Config());
tc.RecordSection("Copy to gpu");
for (int i = 0; i < search_count; ++i) {
search_idx->Search(query_dataset, conf);
@ -622,7 +616,7 @@ TEST_F(GPURESTEST, gpuivfsq) {
faiss::gpu::GpuClonerOptions option;
option.allInGpu = true;
kn::TimeRecorder tc("ori GPUSQ8");
knowhere::TimeRecorder tc("ori GPUSQ8");
faiss::Index* search_idx = faiss::gpu::index_cpu_to_gpu(&res, device_id, cpu_index, &option);
tc.RecordSection("Copy to gpu");
for (int i = 0; i < search_count; ++i) {
@ -643,11 +637,11 @@ TEST_F(GPURESTEST, copyandsearch) {
index_type = "GPUIVFSQ";
index_ = IndexFactory(index_type);
auto conf = std::make_shared<kn::IVFSQCfg>();
auto conf = std::make_shared<knowhere::IVFSQCfg>();
conf->nlist = 1638;
conf->d = dim;
conf->gpu_id = device_id;
conf->metric_type = kn::METRICTYPE::L2;
conf->metric_type = knowhere::METRICTYPE::L2;
conf->k = k;
conf->nbits = 8;
conf->nprobe = 1;
@ -660,10 +654,10 @@ TEST_F(GPURESTEST, copyandsearch) {
// auto result = index_->Search(query_dataset, conf);
// AssertAnns(result, nq, k);
auto cpu_idx = kn::cloner::CopyGpuToCpu(index_, kn::Config());
auto cpu_idx = knowhere::cloner::CopyGpuToCpu(index_, knowhere::Config());
cpu_idx->Seal();
auto search_idx = kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config());
auto search_idx = knowhere::cloner::CopyCpuToGpu(cpu_idx, device_id, knowhere::Config());
auto search_func = [&] {
// TimeRecorder tc("search&load");
@ -677,15 +671,15 @@ TEST_F(GPURESTEST, copyandsearch) {
auto load_func = [&] {
// TimeRecorder tc("search&load");
for (int i = 0; i < load_count; ++i) {
kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config());
knowhere::cloner::CopyCpuToGpu(cpu_idx, device_id, knowhere::Config());
// if (i > load_count -5 || i < 5)
// tc.RecordSection("Copy to gpu");
}
// tc.ElapseFromBegin("load finish");
};
kn::TimeRecorder tc("basic");
kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config());
knowhere::TimeRecorder tc("basic");
knowhere::cloner::CopyCpuToGpu(cpu_idx, device_id, knowhere::Config());
tc.RecordSection("Copy to gpu once");
search_idx->Search(query_dataset, conf);
tc.RecordSection("search once");
@ -705,11 +699,11 @@ TEST_F(GPURESTEST, TrainAndSearch) {
index_type = "GPUIVFSQ";
index_ = IndexFactory(index_type);
auto conf = std::make_shared<kn::IVFSQCfg>();
auto conf = std::make_shared<knowhere::IVFSQCfg>();
conf->nlist = 1638;
conf->d = dim;
conf->gpu_id = device_id;
conf->metric_type = kn::METRICTYPE::L2;
conf->metric_type = knowhere::METRICTYPE::L2;
conf->k = k;
conf->nbits = 8;
conf->nprobe = 1;
@ -720,9 +714,9 @@ TEST_F(GPURESTEST, TrainAndSearch) {
auto new_index = IndexFactory(index_type);
new_index->set_index_model(model);
new_index->Add(base_dataset, conf);
auto cpu_idx = kn::cloner::CopyGpuToCpu(new_index, kn::Config());
auto cpu_idx = knowhere::cloner::CopyGpuToCpu(new_index, knowhere::Config());
cpu_idx->Seal();
auto search_idx = kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config());
auto search_idx = knowhere::cloner::CopyCpuToGpu(cpu_idx, device_id, knowhere::Config());
constexpr int train_count = 1;
constexpr int search_count = 5000;
@ -734,7 +728,7 @@ TEST_F(GPURESTEST, TrainAndSearch) {
test_idx->Add(base_dataset, conf);
}
};
auto search_stage = [&](kn::VectorIndexPtr& search_idx) {
auto search_stage = [&](knowhere::VectorIndexPtr& search_idx) {
for (int i = 0; i < search_count; ++i) {
auto result = search_idx->Search(query_dataset, conf);
AssertAnns(result, nq, k);
@ -763,7 +757,7 @@ TEST_F(GPURESTEST, TrainAndSearch) {
}
{
// search parallel
auto search_idx_2 = kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config());
auto search_idx_2 = knowhere::cloner::CopyCpuToGpu(cpu_idx, device_id, knowhere::Config());
std::thread search_1(search_stage, std::ref(search_idx));
std::thread search_2(search_stage, std::ref(search_idx_2));
search_1.join();

8
cpp/src/core/unittest/test_json.cpp
View File

@ -17,15 +17,9 @@
#include "knowhere/common/config.h"
namespace {
namespace kn = knowhere;
} // namespace
int
main() {
kn::Config cfg;
knowhere::Config cfg;
cfg["size"] = size_t(199);
auto size = cfg.get_with_default("size", 123);

24
cpp/src/core/unittest/test_kdt.cpp
View File

@ -28,12 +28,6 @@
#include "unittest/utils.h"
namespace {
namespace kn = knowhere;
} // namespace
using ::testing::Combine;
using ::testing::TestWithParam;
using ::testing::Values;
@ -42,9 +36,9 @@ class KDTTest : public DataGen, public ::testing::Test {
protected:
void
SetUp() override {
index_ = std::make_shared<kn::CPUKDTRNG>();
index_ = std::make_shared<knowhere::CPUKDTRNG>();
auto tempconf = std::make_shared<kn::KDTCfg>();
auto tempconf = std::make_shared<knowhere::KDTCfg>();
tempconf->tptnubmber = 1;
tempconf->k = 10;
conf = tempconf;
@ -53,12 +47,12 @@ class KDTTest : public DataGen, public ::testing::Test {
}
protected:
kn::Config conf;
std::shared_ptr<kn::CPUKDTRNG> index_ = nullptr;
knowhere::Config conf;
std::shared_ptr<knowhere::CPUKDTRNG> index_ = nullptr;
};
void
AssertAnns(const kn::DatasetPtr& result, const int& nq, const int& k) {
AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) {
auto ids = result->array()[0];
for (auto i = 0; i < nq; i++) {
EXPECT_EQ(i, *(ids->data()->GetValues<int64_t>(1, i * k)));
@ -66,7 +60,7 @@ AssertAnns(const kn::DatasetPtr& result, const int& nq, const int& k) {
}
void
PrintResult(const kn::DatasetPtr& result, const int& nq, const int& k) {
PrintResult(const knowhere::DatasetPtr& result, const int& nq, const int& k) {
auto ids = result->array()[0];
auto dists = result->array()[1];
@ -125,7 +119,7 @@ TEST_F(KDTTest, kdt_serialize) {
auto model = index_->Train(base_dataset, conf);
// index_->Add(base_dataset, conf);
auto binaryset = index_->Serialize();
auto new_index = std::make_shared<kn::CPUKDTRNG>();
auto new_index = std::make_shared<knowhere::CPUKDTRNG>();
new_index->Load(binaryset);
auto result = new_index->Search(query_dataset, conf);
AssertAnns(result, nq, k);
@ -150,7 +144,7 @@ TEST_F(KDTTest, kdt_serialize) {
++fileno;
}
kn::BinarySet load_data_list;
knowhere::BinarySet load_data_list;
for (int i = 0; i < filename_list.size() && i < meta_list.size(); ++i) {
auto bin_size = meta_list[i].second;
FileIOReader reader(filename_list[i]);
@ -162,7 +156,7 @@ TEST_F(KDTTest, kdt_serialize) {
load_data_list.Append(meta_list[i].first, data, bin_size);
}
auto new_index = std::make_shared<kn::CPUKDTRNG>();
auto new_index = std::make_shared<knowhere::CPUKDTRNG>();
new_index->Load(load_data_list);
auto result = new_index->Search(query_dataset, conf);
AssertAnns(result, nq, k);

30
cpp/src/core/unittest/test_nsg/test_nsg.cpp
View File

@ -26,12 +26,6 @@
#include "unittest/utils.h"
namespace {
namespace kn = knowhere;
} // namespace
using ::testing::Combine;
using ::testing::TestWithParam;
using ::testing::Values;
@ -43,11 +37,11 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test {
void
SetUp() override {
// Init_with_default();
kn::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID, 1024 * 1024 * 200, 1024 * 1024 * 600, 2);
knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID, 1024 * 1024 * 200, 1024 * 1024 * 600, 2);
Generate(256, 1000000, 1);
index_ = std::make_shared<kn::NSG>();
index_ = std::make_shared<knowhere::NSG>();
auto tmp_conf = std::make_shared<kn::NSGCfg>();
auto tmp_conf = std::make_shared<knowhere::NSGCfg>();
tmp_conf->gpu_id = DEVICE_ID;
tmp_conf->knng = 100;
tmp_conf->nprobe = 32;
@ -55,10 +49,10 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test {
tmp_conf->search_length = 60;
tmp_conf->out_degree = 70;
tmp_conf->candidate_pool_size = 500;
tmp_conf->metric_type = kn::METRICTYPE::L2;
tmp_conf->metric_type = knowhere::METRICTYPE::L2;
train_conf = tmp_conf;
auto tmp2_conf = std::make_shared<kn::NSGCfg>();
auto tmp2_conf = std::make_shared<knowhere::NSGCfg>();
tmp2_conf->k = k;
tmp2_conf->search_length = 30;
search_conf = tmp2_conf;
@ -66,17 +60,17 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test {
void
TearDown() override {
kn::FaissGpuResourceMgr::GetInstance().Free();
knowhere::FaissGpuResourceMgr::GetInstance().Free();
}
protected:
std::shared_ptr<kn::NSG> index_;
kn::Config train_conf;
kn::Config search_conf;
std::shared_ptr<knowhere::NSG> index_;
knowhere::Config train_conf;
knowhere::Config search_conf;
};
void
AssertAnns(const kn::DatasetPtr& result, const int& nq, const int& k) {
AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) {
auto ids = result->array()[0];
for (auto i = 0; i < nq; i++) {
EXPECT_EQ(i, *(ids->data()->GetValues<int64_t>(1, i * k)));
@ -91,7 +85,7 @@ TEST_F(NSGInterfaceTest, basic_test) {
AssertAnns(result, nq, k);
auto binaryset = index_->Serialize();
auto new_index = std::make_shared<kn::NSG>();
auto new_index = std::make_shared<knowhere::NSG>();
new_index->Load(binaryset);
auto new_result = new_index->Search(query_dataset, search_conf);
AssertAnns(result, nq, k);
@ -100,7 +94,7 @@ TEST_F(NSGInterfaceTest, basic_test) {
ASSERT_EQ(index_->Dimension(), dim);
ASSERT_THROW({ index_->Clone(); }, knowhere::KnowhereException);
ASSERT_NO_THROW({
index_->Add(base_dataset, kn::Config());
index_->Add(base_dataset, knowhere::Config());
index_->Seal();
});

39
cpp/src/core/unittest/utils.cpp
View File

@ -23,12 +23,6 @@
INITIALIZE_EASYLOGGINGPP
namespace {
namespace kn = knowhere;
} // namespace
void
InitLog() {
el::Configurations defaultConf;
@ -124,31 +118,32 @@ FileIOWriter::operator()(void* ptr, size_t size) {
return size;
}
kn::DatasetPtr
knowhere::DatasetPtr
generate_dataset(int64_t nb, int64_t dim, float* xb, int64_t* ids) {
std::vector<int64_t> shape{nb, dim};
auto tensor = kn::ConstructFloatTensor((uint8_t*)xb, nb * dim * sizeof(float), shape);
std::vector<kn::TensorPtr> tensors{tensor};
std::vector<kn::FieldPtr> tensor_fields{kn::ConstructFloatField("data")};
auto tensor_schema = std::make_shared<kn::Schema>(tensor_fields);
auto tensor = knowhere::ConstructFloatTensor((uint8_t*)xb, nb * dim * sizeof(float), shape);
std::vector<knowhere::TensorPtr> tensors{tensor};
std::vector<knowhere::FieldPtr> tensor_fields{knowhere::ConstructFloatField("data")};
auto tensor_schema = std::make_shared<knowhere::Schema>(tensor_fields);
auto id_array = kn::ConstructInt64Array((uint8_t*)ids, nb * sizeof(int64_t));
std::vector<kn::ArrayPtr> arrays{id_array};
std::vector<kn::FieldPtr> array_fields{kn::ConstructInt64Field("id")};
auto array_schema = std::make_shared<kn::Schema>(tensor_fields);
auto id_array = knowhere::ConstructInt64Array((uint8_t*)ids, nb * sizeof(int64_t));
std::vector<knowhere::ArrayPtr> arrays{id_array};
std::vector<knowhere::FieldPtr> array_fields{knowhere::ConstructInt64Field("id")};
auto array_schema = std::make_shared<knowhere::Schema>(tensor_fields);
auto dataset = std::make_shared<kn::Dataset>(std::move(arrays), array_schema, std::move(tensors), tensor_schema);
auto dataset =
std::make_shared<knowhere::Dataset>(std::move(arrays), array_schema, std::move(tensors), tensor_schema);
return dataset;
}
kn::DatasetPtr
knowhere::DatasetPtr
generate_query_dataset(int64_t nb, int64_t dim, float* xb) {
std::vector<int64_t> shape{nb, dim};
auto tensor = kn::ConstructFloatTensor((uint8_t*)xb, nb * dim * sizeof(float), shape);
std::vector<kn::TensorPtr> tensors{tensor};
std::vector<kn::FieldPtr> tensor_fields{kn::ConstructFloatField("data")};
auto tensor_schema = std::make_shared<kn::Schema>(tensor_fields);
auto tensor = knowhere::ConstructFloatTensor((uint8_t*)xb, nb * dim * sizeof(float), shape);
std::vector<knowhere::TensorPtr> tensors{tensor};
std::vector<knowhere::FieldPtr> tensor_fields{knowhere::ConstructFloatField("data")};
auto tensor_schema = std::make_shared<knowhere::Schema>(tensor_fields);
auto dataset = std::make_shared<kn::Dataset>(std::move(tensors), tensor_schema);
auto dataset = std::make_shared<knowhere::Dataset>(std::move(tensors), tensor_schema);
return dataset;
}

124
cpp/unittest/db/test_db.cpp
View File

@ -31,8 +31,6 @@
namespace {
namespace ms = milvus;
static const char *TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 25000;
@ -40,9 +38,9 @@ static constexpr int64_t INSERT_LOOP = 1000;
static constexpr int64_t SECONDS_EACH_HOUR = 3600;
static constexpr int64_t DAY_SECONDS = 24 * 60 * 60;
ms::engine::meta::TableSchema
milvus::engine::meta::TableSchema
BuildTableSchema() {
ms::engine::meta::TableSchema table_info;
milvus::engine::meta::TableSchema table_info;
table_info.dimension_ = TABLE_DIM;
table_info.table_id_ = TABLE_NAME;
return table_info;
@ -77,7 +75,7 @@ CurrentTmDate(int64_t offset_day = 0) {
void
ConvertTimeRangeToDBDates(const std::string &start_value,
const std::string &end_value,
std::vector<ms::engine::meta::DateT> &dates) {
std::vector<milvus::engine::meta::DateT> &dates) {
dates.clear();
time_t tt_start, tt_end;
@ -111,41 +109,41 @@ ConvertTimeRangeToDBDates(const std::string &start_value,
TEST_F(DBTest, CONFIG_TEST) {
{
ASSERT_ANY_THROW(ms::engine::ArchiveConf conf("wrong"));
ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf("wrong"));
/* EXPECT_DEATH(engine::ArchiveConf conf("wrong"), ""); */
}
{
ms::engine::ArchiveConf conf("delete");
milvus::engine::ArchiveConf conf("delete");
ASSERT_EQ(conf.GetType(), "delete");
auto criterias = conf.GetCriterias();
ASSERT_EQ(criterias.size(), 0);
}
{
ms::engine::ArchiveConf conf("swap");
milvus::engine::ArchiveConf conf("swap");
ASSERT_EQ(conf.GetType(), "swap");
auto criterias = conf.GetCriterias();
ASSERT_EQ(criterias.size(), 0);
}
{
ASSERT_ANY_THROW(ms::engine::ArchiveConf conf1("swap", "disk:"));
ASSERT_ANY_THROW(ms::engine::ArchiveConf conf2("swap", "disk:a"));
ms::engine::ArchiveConf conf("swap", "disk:1024");
ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf1("swap", "disk:"));
ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf2("swap", "disk:a"));
milvus::engine::ArchiveConf conf("swap", "disk:1024");
auto criterias = conf.GetCriterias();
ASSERT_EQ(criterias.size(), 1);
ASSERT_EQ(criterias["disk"], 1024);
}
{
ASSERT_ANY_THROW(ms::engine::ArchiveConf conf1("swap", "days:"));
ASSERT_ANY_THROW(ms::engine::ArchiveConf conf2("swap", "days:a"));
ms::engine::ArchiveConf conf("swap", "days:100");
ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf1("swap", "days:"));
ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf2("swap", "days:a"));
milvus::engine::ArchiveConf conf("swap", "days:100");
auto criterias = conf.GetCriterias();
ASSERT_EQ(criterias.size(), 1);
ASSERT_EQ(criterias["days"], 100);
}
{
ASSERT_ANY_THROW(ms::engine::ArchiveConf conf1("swap", "days:"));
ASSERT_ANY_THROW(ms::engine::ArchiveConf conf2("swap", "days:a"));
ms::engine::ArchiveConf conf("swap", "days:100;disk:200");
ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf1("swap", "days:"));
ASSERT_ANY_THROW(milvus::engine::ArchiveConf conf2("swap", "days:a"));
milvus::engine::ArchiveConf conf("swap", "days:100;disk:200");
auto criterias = conf.GetCriterias();
ASSERT_EQ(criterias.size(), 2);
ASSERT_EQ(criterias["days"], 100);
@ -154,17 +152,17 @@ TEST_F(DBTest, CONFIG_TEST) {
}
TEST_F(DBTest, DB_TEST) {
ms::engine::meta::TableSchema table_info = BuildTableSchema();
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
ms::engine::meta::TableSchema table_info_get;
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
ms::engine::IDNumbers vector_ids;
ms::engine::IDNumbers target_ids;
milvus::engine::IDNumbers vector_ids;
milvus::engine::IDNumbers target_ids;
int64_t nb = 50;
std::vector<float> xb;
@ -175,7 +173,7 @@ TEST_F(DBTest, DB_TEST) {
BuildVectors(qb, qxb);
std::thread search([&]() {
ms::engine::QueryResults results;
milvus::engine::QueryResults results;
int k = 10;
std::this_thread::sleep_for(std::chrono::seconds(2));
@ -191,7 +189,7 @@ TEST_F(DBTest, DB_TEST) {
START_TIMER;
stat = db_->Query(TABLE_NAME, k, qb, 10, qxb.data(), results);
ss << "Search " << j << " With Size " << count / ms::engine::M << " M";
ss << "Search " << j << " With Size " << count / milvus::engine::M << " M";
STOP_TIMER(ss.str());
ASSERT_TRUE(stat.ok());
@ -230,10 +228,10 @@ TEST_F(DBTest, DB_TEST) {
}
TEST_F(DBTest, SEARCH_TEST) {
ms::engine::meta::TableSchema table_info = BuildTableSchema();
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
ms::engine::meta::TableSchema table_info_get;
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
@ -274,20 +272,20 @@ TEST_F(DBTest, SEARCH_TEST) {
ASSERT_TRUE(stat.ok());
}
ms::engine::TableIndex index;
index.engine_type_ = (int) ms::engine::EngineType::FAISS_IDMAP;
milvus::engine::TableIndex index;
index.engine_type_ = (int) milvus::engine::EngineType::FAISS_IDMAP;
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
{
ms::engine::QueryResults results;
milvus::engine::QueryResults results;
stat = db_->Query(TABLE_NAME, k, nq, 10, xq.data(), results);
ASSERT_TRUE(stat.ok());
}
{//search by specify index file
ms::engine::meta::DatesT dates;
milvus::engine::meta::DatesT dates;
std::vector<std::string> file_ids = {"1", "2", "3", "4", "5", "6"};
ms::engine::QueryResults results;
milvus::engine::QueryResults results;
stat = db_->Query(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, results);
ASSERT_TRUE(stat.ok());
}
@ -296,10 +294,10 @@ TEST_F(DBTest, SEARCH_TEST) {
}
TEST_F(DBTest, PRELOADTABLE_TEST) {
ms::engine::meta::TableSchema table_info = BuildTableSchema();
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
ms::engine::meta::TableSchema table_info_get;
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
@ -311,26 +309,26 @@ TEST_F(DBTest, PRELOADTABLE_TEST) {
int loop = 5;
for (auto i = 0; i < loop; ++i) {
ms::engine::IDNumbers vector_ids;
milvus::engine::IDNumbers vector_ids;
db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
ASSERT_EQ(vector_ids.size(), nb);
}
ms::engine::TableIndex index;
index.engine_type_ = (int) ms::engine::EngineType::FAISS_IDMAP;
milvus::engine::TableIndex index;
index.engine_type_ = (int) milvus::engine::EngineType::FAISS_IDMAP;
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
int64_t prev_cache_usage = ms::cache::CpuCacheMgr::GetInstance()->CacheUsage();
int64_t prev_cache_usage = milvus::cache::CpuCacheMgr::GetInstance()->CacheUsage();
stat = db_->PreloadTable(TABLE_NAME);
ASSERT_TRUE(stat.ok());
int64_t cur_cache_usage = ms::cache::CpuCacheMgr::GetInstance()->CacheUsage();
int64_t cur_cache_usage = milvus::cache::CpuCacheMgr::GetInstance()->CacheUsage();
ASSERT_TRUE(prev_cache_usage < cur_cache_usage);
}
TEST_F(DBTest, SHUTDOWN_TEST) {
db_->Stop();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
ASSERT_FALSE(stat.ok());
@ -341,7 +339,7 @@ TEST_F(DBTest, SHUTDOWN_TEST) {
stat = db_->HasTable(table_info.table_id_, has_table);
ASSERT_FALSE(stat.ok());
ms::engine::IDNumbers ids;
milvus::engine::IDNumbers ids;
stat = db_->InsertVectors(table_info.table_id_, 0, nullptr, ids);
ASSERT_FALSE(stat.ok());
@ -352,15 +350,15 @@ TEST_F(DBTest, SHUTDOWN_TEST) {
stat = db_->GetTableRowCount(table_info.table_id_, row_count);
ASSERT_FALSE(stat.ok());
ms::engine::TableIndex index;
milvus::engine::TableIndex index;
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_FALSE(stat.ok());
stat = db_->DescribeIndex(table_info.table_id_, index);
ASSERT_FALSE(stat.ok());
ms::engine::meta::DatesT dates;
ms::engine::QueryResults results;
milvus::engine::meta::DatesT dates;
milvus::engine::QueryResults results;
stat = db_->Query(table_info.table_id_, 1, 1, 1, nullptr, dates, results);
ASSERT_FALSE(stat.ok());
std::vector<std::string> file_ids;
@ -372,24 +370,24 @@ TEST_F(DBTest, SHUTDOWN_TEST) {
}
TEST_F(DBTest, INDEX_TEST) {
ms::engine::meta::TableSchema table_info = BuildTableSchema();
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
int64_t nb = VECTOR_COUNT;
std::vector<float> xb;
BuildVectors(nb, xb);
ms::engine::IDNumbers vector_ids;
milvus::engine::IDNumbers vector_ids;
db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
ASSERT_EQ(vector_ids.size(), nb);
ms::engine::TableIndex index;
index.engine_type_ = (int) ms::engine::EngineType::FAISS_IVFSQ8;
index.metric_type_ = (int) ms::engine::MetricType::IP;
milvus::engine::TableIndex index;
index.engine_type_ = (int) milvus::engine::EngineType::FAISS_IVFSQ8;
index.metric_type_ = (int) milvus::engine::MetricType::IP;
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_TRUE(stat.ok());
ms::engine::TableIndex index_out;
milvus::engine::TableIndex index_out;
stat = db_->DescribeIndex(table_info.table_id_, index_out);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(index.engine_type_, index_out.engine_type_);
@ -401,10 +399,10 @@ TEST_F(DBTest, INDEX_TEST) {
}
TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
ms::engine::meta::TableSchema table_info = BuildTableSchema();
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
std::vector<ms::engine::meta::TableSchema> table_schema_array;
std::vector<milvus::engine::meta::TableSchema> table_schema_array;
stat = db_->AllTables(table_schema_array);
ASSERT_TRUE(stat.ok());
bool bfound = false;
@ -416,7 +414,7 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
}
ASSERT_TRUE(bfound);
ms::engine::meta::TableSchema table_info_get;
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
@ -431,7 +429,7 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
int loop = INSERT_LOOP;
for (auto i = 0; i < loop; ++i) {
ms::engine::IDNumbers vector_ids;
milvus::engine::IDNumbers vector_ids;
db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
std::this_thread::sleep_for(std::chrono::microseconds(1));
}
@ -440,14 +438,14 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
db_->Size(size);
LOG(DEBUG) << "size=" << size;
ASSERT_LE(size, 1 * ms::engine::G);
ASSERT_LE(size, 1 * milvus::engine::G);
}
TEST_F(DBTest2, DELETE_TEST) {
ms::engine::meta::TableSchema table_info = BuildTableSchema();
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
ms::engine::meta::TableSchema table_info_get;
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
@ -463,12 +461,12 @@ TEST_F(DBTest2, DELETE_TEST) {
std::vector<float> xb;
BuildVectors(nb, xb);
ms::engine::IDNumbers vector_ids;
milvus::engine::IDNumbers vector_ids;
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
ms::engine::TableIndex index;
milvus::engine::TableIndex index;
stat = db_->CreateIndex(TABLE_NAME, index);
std::vector<ms::engine::meta::DateT> dates;
std::vector<milvus::engine::meta::DateT> dates;
stat = db_->DeleteTable(TABLE_NAME, dates);
std::this_thread::sleep_for(std::chrono::seconds(2));
ASSERT_TRUE(stat.ok());
@ -478,10 +476,10 @@ TEST_F(DBTest2, DELETE_TEST) {
}
TEST_F(DBTest2, DELETE_BY_RANGE_TEST) {
ms::engine::meta::TableSchema table_info = BuildTableSchema();
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
ms::engine::meta::TableSchema table_info_get;
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
@ -498,15 +496,15 @@ TEST_F(DBTest2, DELETE_BY_RANGE_TEST) {
std::vector<float> xb;
BuildVectors(nb, xb);
ms::engine::IDNumbers vector_ids;
milvus::engine::IDNumbers vector_ids;
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
ms::engine::TableIndex index;
milvus::engine::TableIndex index;
stat = db_->CreateIndex(TABLE_NAME, index);
db_->Size(size);
ASSERT_NE(size, 0UL);
std::vector<ms::engine::meta::DateT> dates;
std::vector<milvus::engine::meta::DateT> dates;
std::string start_value = CurrentTmDate();
std::string end_value = CurrentTmDate(1);
ConvertTimeRangeToDBDates(start_value, end_value, dates);

44
cpp/unittest/db/test_db_mysql.cpp
View File

@ -29,19 +29,17 @@
namespace {
namespace ms = milvus;
static const char *TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 25000;
static constexpr int64_t INSERT_LOOP = 1000;
ms::engine::meta::TableSchema
milvus::engine::meta::TableSchema
BuildTableSchema() {
ms::engine::meta::TableSchema table_info;
milvus::engine::meta::TableSchema table_info;
table_info.dimension_ = TABLE_DIM;
table_info.table_id_ = TABLE_NAME;
table_info.engine_type_ = (int) ms::engine::EngineType::FAISS_IDMAP;
table_info.engine_type_ = (int) milvus::engine::EngineType::FAISS_IDMAP;
return table_info;
}
@ -59,17 +57,17 @@ BuildVectors(int64_t n, std::vector<float> &vectors) {
} // namespace
TEST_F(MySqlDBTest, DB_TEST) {
ms::engine::meta::TableSchema table_info = BuildTableSchema();
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
ms::engine::meta::TableSchema table_info_get;
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
ms::engine::IDNumbers vector_ids;
ms::engine::IDNumbers target_ids;
milvus::engine::IDNumbers vector_ids;
milvus::engine::IDNumbers target_ids;
int64_t nb = 50;
std::vector<float> xb;
@ -83,7 +81,7 @@ TEST_F(MySqlDBTest, DB_TEST) {
ASSERT_EQ(target_ids.size(), qb);
std::thread search([&]() {
ms::engine::QueryResults results;
milvus::engine::QueryResults results;
int k = 10;
std::this_thread::sleep_for(std::chrono::seconds(5));
@ -99,7 +97,7 @@ TEST_F(MySqlDBTest, DB_TEST) {
START_TIMER;
stat = db_->Query(TABLE_NAME, k, qb, 10, qxb.data(), results);
ss << "Search " << j << " With Size " << count / ms::engine::M << " M";
ss << "Search " << j << " With Size " << count / milvus::engine::M << " M";
STOP_TIMER(ss.str());
ASSERT_TRUE(stat.ok());
@ -144,10 +142,10 @@ TEST_F(MySqlDBTest, DB_TEST) {
}
TEST_F(MySqlDBTest, SEARCH_TEST) {
ms::engine::meta::TableSchema table_info = BuildTableSchema();
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
ms::engine::meta::TableSchema table_info_get;
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
@ -190,16 +188,16 @@ TEST_F(MySqlDBTest, SEARCH_TEST) {
sleep(2); // wait until build index finish
ms::engine::QueryResults results;
milvus::engine::QueryResults results;
stat = db_->Query(TABLE_NAME, k, nq, 10, xq.data(), results);
ASSERT_TRUE(stat.ok());
}
TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
ms::engine::meta::TableSchema table_info = BuildTableSchema();
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
std::vector<ms::engine::meta::TableSchema> table_schema_array;
std::vector<milvus::engine::meta::TableSchema> table_schema_array;
stat = db_->AllTables(table_schema_array);
ASSERT_TRUE(stat.ok());
bool bfound = false;
@ -211,14 +209,14 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
}
ASSERT_TRUE(bfound);
ms::engine::meta::TableSchema table_info_get;
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
ms::engine::IDNumbers vector_ids;
ms::engine::IDNumbers target_ids;
milvus::engine::IDNumbers vector_ids;
milvus::engine::IDNumbers target_ids;
uint64_t size;
db_->Size(size);
@ -237,15 +235,15 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
db_->Size(size);
LOG(DEBUG) << "size=" << size;
ASSERT_LE(size, 1 * ms::engine::G);
ASSERT_LE(size, 1 * milvus::engine::G);
}
TEST_F(MySqlDBTest, DELETE_TEST) {
ms::engine::meta::TableSchema table_info = BuildTableSchema();
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
// std::cout << stat.ToString() << std::endl;
ms::engine::meta::TableSchema table_info_get;
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
@ -254,7 +252,7 @@ TEST_F(MySqlDBTest, DELETE_TEST) {
db_->HasTable(TABLE_NAME, has_table);
ASSERT_TRUE(has_table);
ms::engine::IDNumbers vector_ids;
milvus::engine::IDNumbers vector_ids;
uint64_t size;
db_->Size(size);

42
cpp/unittest/db/test_engine.cpp
View File

@ -23,63 +23,57 @@
#include "db/engine/ExecutionEngineImpl.h"
#include "db/utils.h"
namespace {
namespace ms = milvus;
}
TEST_F(EngineTest, FACTORY_TEST) {
{
auto engine_ptr = ms::engine::EngineFactory::Build(
auto engine_ptr = milvus::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
ms::engine::EngineType::INVALID,
ms::engine::MetricType::IP,
milvus::engine::EngineType::INVALID,
milvus::engine::MetricType::IP,
1024);
ASSERT_TRUE(engine_ptr == nullptr);
}
{
auto engine_ptr = ms::engine::EngineFactory::Build(
auto engine_ptr = milvus::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
ms::engine::EngineType::FAISS_IDMAP,
ms::engine::MetricType::IP,
milvus::engine::EngineType::FAISS_IDMAP,
milvus::engine::MetricType::IP,
1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = ms::engine::EngineFactory::Build(
auto engine_ptr = milvus::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
ms::engine::EngineType::FAISS_IVFFLAT,
ms::engine::MetricType::IP,
milvus::engine::EngineType::FAISS_IVFFLAT,
milvus::engine::MetricType::IP,
1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = ms::engine::EngineFactory::Build(
auto engine_ptr = milvus::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
ms::engine::EngineType::FAISS_IVFSQ8,
ms::engine::MetricType::IP,
milvus::engine::EngineType::FAISS_IVFSQ8,
milvus::engine::MetricType::IP,
1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = ms::engine::EngineFactory::Build(
auto engine_ptr = milvus::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
ms::engine::EngineType::NSG_MIX,
ms::engine::MetricType::IP,
milvus::engine::EngineType::NSG_MIX,
milvus::engine::MetricType::IP,
1024);
ASSERT_TRUE(engine_ptr != nullptr);
@ -89,11 +83,11 @@ TEST_F(EngineTest, FACTORY_TEST) {
TEST_F(EngineTest, ENGINE_IMPL_TEST) {
uint16_t dimension = 64;
std::string file_path = "/tmp/milvus_index_1";
auto engine_ptr = ms::engine::EngineFactory::Build(
auto engine_ptr = milvus::engine::EngineFactory::Build(
dimension,
file_path,
ms::engine::EngineType::FAISS_IVFFLAT,
ms::engine::MetricType::IP,
milvus::engine::EngineType::FAISS_IVFFLAT,
milvus::engine::MetricType::IP,
1024);
std::vector<float> data;

101
cpp/unittest/db/test_mem.cpp
View File

@ -37,8 +37,6 @@
namespace {
namespace ms = milvus;
static constexpr int64_t TABLE_DIM = 256;
std::string
@ -50,20 +48,20 @@ GetTableName() {
return table_name;
}
ms::engine::meta::TableSchema
milvus::engine::meta::TableSchema
BuildTableSchema() {
ms::engine::meta::TableSchema table_info;
milvus::engine::meta::TableSchema table_info;
table_info.dimension_ = TABLE_DIM;
table_info.table_id_ = GetTableName();
table_info.engine_type_ = (int) ms::engine::EngineType::FAISS_IDMAP;
table_info.engine_type_ = (int)milvus::engine::EngineType::FAISS_IDMAP;
return table_info;
}
void
BuildVectors(int64_t n, std::vector<float> &vectors) {
BuildVectors(int64_t n, std::vector<float>& vectors) {
vectors.clear();
vectors.resize(n * TABLE_DIM);
float *data = vectors.data();
float* data = vectors.data();
for (int i = 0; i < n; i++) {
for (int j = 0; j < TABLE_DIM; j++)
data[TABLE_DIM * i + j] = drand48();
@ -72,11 +70,11 @@ BuildVectors(int64_t n, std::vector<float> &vectors) {
} // namespace
TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) {
ms::engine::meta::TableSchema table_schema = BuildTableSchema();
milvus::engine::meta::TableSchema table_schema = BuildTableSchema();
auto status = impl_->CreateTable(table_schema);
ASSERT_TRUE(status.ok());
ms::engine::meta::TableFileSchema table_file_schema;
milvus::engine::meta::TableFileSchema table_file_schema;
table_file_schema.table_id_ = GetTableName();
status = impl_->CreateTableFile(table_file_schema);
ASSERT_TRUE(status.ok());
@ -85,17 +83,17 @@ TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) {
std::vector<float> vectors;
BuildVectors(n, vectors);
ms::engine::VectorSource source(n, vectors.data());
milvus::engine::VectorSource source(n, vectors.data());
size_t num_vectors_added;
ms::engine::ExecutionEnginePtr execution_engine_ =
ms::engine::EngineFactory::Build(table_file_schema.dimension_,
table_file_schema.location_,
(ms::engine::EngineType) table_file_schema.engine_type_,
(ms::engine::MetricType) table_file_schema.metric_type_,
table_schema.nlist_);
milvus::engine::ExecutionEnginePtr execution_engine_ =
milvus::engine::EngineFactory::Build(table_file_schema.dimension_,
table_file_schema.location_,
(milvus::engine::EngineType)table_file_schema.engine_type_,
(milvus::engine::MetricType)table_file_schema.metric_type_,
table_schema.nlist_);
ms::engine::IDNumbers vector_ids;
milvus::engine::IDNumbers vector_ids;
status = source.Add(execution_engine_, table_file_schema, 50, num_vectors_added, vector_ids);
ASSERT_TRUE(status.ok());
vector_ids = source.GetVectorIds();
@ -115,19 +113,19 @@ TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) {
TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
auto options = GetOptions();
ms::engine::meta::TableSchema table_schema = BuildTableSchema();
milvus::engine::meta::TableSchema table_schema = BuildTableSchema();
auto status = impl_->CreateTable(table_schema);
ASSERT_TRUE(status.ok());
ms::engine::MemTableFile mem_table_file(GetTableName(), impl_, options);
milvus::engine::MemTableFile mem_table_file(GetTableName(), impl_, options);
int64_t n_100 = 100;
std::vector<float> vectors_100;
BuildVectors(n_100, vectors_100);
ms::engine::VectorSourcePtr source = std::make_shared<ms::engine::VectorSource>(n_100, vectors_100.data());
milvus::engine::VectorSourcePtr source = std::make_shared<milvus::engine::VectorSource>(n_100, vectors_100.data());
ms::engine::IDNumbers vector_ids;
milvus::engine::IDNumbers vector_ids;
status = mem_table_file.Add(source, vector_ids);
ASSERT_TRUE(status.ok());
@ -139,11 +137,12 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
size_t singleVectorMem = sizeof(float) * TABLE_DIM;
ASSERT_EQ(mem_table_file.GetCurrentMem(), n_100 * singleVectorMem);
int64_t n_max = ms::engine::MAX_TABLE_FILE_MEM / singleVectorMem;
int64_t n_max = milvus::engine::MAX_TABLE_FILE_MEM / singleVectorMem;
std::vector<float> vectors_128M;
BuildVectors(n_max, vectors_128M);
ms::engine::VectorSourcePtr source_128M = std::make_shared<ms::engine::VectorSource>(n_max, vectors_128M.data());
milvus::engine::VectorSourcePtr
source_128M = std::make_shared<milvus::engine::VectorSource>(n_max, vectors_128M.data());
vector_ids.clear();
status = mem_table_file.Add(source_128M, vector_ids);
@ -156,7 +155,7 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
TEST_F(MemManagerTest, MEM_TABLE_TEST) {
auto options = GetOptions();
ms::engine::meta::TableSchema table_schema = BuildTableSchema();
milvus::engine::meta::TableSchema table_schema = BuildTableSchema();
auto status = impl_->CreateTable(table_schema);
ASSERT_TRUE(status.ok());
@ -164,27 +163,29 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
std::vector<float> vectors_100;
BuildVectors(n_100, vectors_100);
ms::engine::VectorSourcePtr source_100 = std::make_shared<ms::engine::VectorSource>(n_100, vectors_100.data());
milvus::engine::VectorSourcePtr
source_100 = std::make_shared<milvus::engine::VectorSource>(n_100, vectors_100.data());
ms::engine::MemTable mem_table(GetTableName(), impl_, options);
milvus::engine::MemTable mem_table(GetTableName(), impl_, options);
ms::engine::IDNumbers vector_ids;
milvus::engine::IDNumbers vector_ids;
status = mem_table.Add(source_100, vector_ids);
ASSERT_TRUE(status.ok());
vector_ids = source_100->GetVectorIds();
ASSERT_EQ(vector_ids.size(), 100);
ms::engine::MemTableFilePtr mem_table_file;
milvus::engine::MemTableFilePtr mem_table_file;
mem_table.GetCurrentMemTableFile(mem_table_file);
size_t singleVectorMem = sizeof(float) * TABLE_DIM;
ASSERT_EQ(mem_table_file->GetCurrentMem(), n_100 * singleVectorMem);
int64_t n_max = ms::engine::MAX_TABLE_FILE_MEM / singleVectorMem;
int64_t n_max = milvus::engine::MAX_TABLE_FILE_MEM / singleVectorMem;
std::vector<float> vectors_128M;
BuildVectors(n_max, vectors_128M);
vector_ids.clear();
ms::engine::VectorSourcePtr source_128M = std::make_shared<ms::engine::VectorSource>(n_max, vectors_128M.data());
milvus::engine::VectorSourcePtr
source_128M = std::make_shared<milvus::engine::VectorSource>(n_max, vectors_128M.data());
status = mem_table.Add(source_128M, vector_ids);
ASSERT_TRUE(status.ok());
@ -200,7 +201,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
std::vector<float> vectors_1G;
BuildVectors(n_1G, vectors_1G);
ms::engine::VectorSourcePtr source_1G = std::make_shared<ms::engine::VectorSource>(n_1G, vectors_1G.data());
milvus::engine::VectorSourcePtr source_1G = std::make_shared<milvus::engine::VectorSource>(n_1G, vectors_1G.data());
vector_ids.clear();
status = mem_table.Add(source_1G, vector_ids);
@ -209,7 +210,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
vector_ids = source_1G->GetVectorIds();
ASSERT_EQ(vector_ids.size(), n_1G);
int expectedTableFileCount = 2 + std::ceil((n_1G - n_100) * singleVectorMem / ms::engine::MAX_TABLE_FILE_MEM);
int expectedTableFileCount = 2 + std::ceil((n_1G - n_100) * singleVectorMem / milvus::engine::MAX_TABLE_FILE_MEM);
ASSERT_EQ(mem_table.GetTableFileCount(), expectedTableFileCount);
status = mem_table.Serialize();
@ -217,10 +218,10 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
}
TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
ms::engine::meta::TableSchema table_info = BuildTableSchema();
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
ms::engine::meta::TableSchema table_info_get;
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
@ -230,7 +231,7 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
std::vector<float> xb;
BuildVectors(nb, xb);
ms::engine::IDNumbers vector_ids;
milvus::engine::IDNumbers vector_ids;
for (int64_t i = 0; i < nb; i++) {
vector_ids.push_back(i);
}
@ -256,9 +257,9 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
}
int topk = 10, nprobe = 10;
for (auto &pair : search_vectors) {
auto &search = pair.second;
ms::engine::QueryResults results;
for (auto& pair : search_vectors) {
auto& search = pair.second;
milvus::engine::QueryResults results;
stat = db_->Query(GetTableName(), topk, 1, nprobe, search.data(), results);
ASSERT_EQ(results[0][0].first, pair.first);
ASSERT_LT(results[0][0].second, 1e-4);
@ -266,10 +267,10 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
}
TEST_F(MemManagerTest2, INSERT_TEST) {
ms::engine::meta::TableSchema table_info = BuildTableSchema();
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
ms::engine::meta::TableSchema table_info_get;
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
@ -282,7 +283,7 @@ TEST_F(MemManagerTest2, INSERT_TEST) {
int64_t nb = 40960;
std::vector<float> xb;
BuildVectors(nb, xb);
ms::engine::IDNumbers vector_ids;
milvus::engine::IDNumbers vector_ids;
stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
ASSERT_TRUE(stat.ok());
}
@ -292,17 +293,17 @@ TEST_F(MemManagerTest2, INSERT_TEST) {
}
TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) {
ms::engine::meta::TableSchema table_info = BuildTableSchema();
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
ms::engine::meta::TableSchema table_info_get;
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
ms::engine::IDNumbers vector_ids;
ms::engine::IDNumbers target_ids;
milvus::engine::IDNumbers vector_ids;
milvus::engine::IDNumbers target_ids;
int64_t nb = 40960;
std::vector<float> xb;
@ -313,7 +314,7 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) {
BuildVectors(qb, qxb);
std::thread search([&]() {
ms::engine::QueryResults results;
milvus::engine::QueryResults results;
int k = 10;
std::this_thread::sleep_for(std::chrono::seconds(2));
@ -329,7 +330,7 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) {
START_TIMER;
stat = db_->Query(GetTableName(), k, qb, 10, qxb.data(), results);
ss << "Search " << j << " With Size " << count / ms::engine::M << " M";
ss << "Search " << j << " With Size " << count / milvus::engine::M << " M";
STOP_TIMER(ss.str());
ASSERT_TRUE(stat.ok());
@ -363,16 +364,16 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) {
}
TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
ms::engine::meta::TableSchema table_info = BuildTableSchema();
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
ms::engine::meta::TableSchema table_info_get;
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
ms::engine::IDNumbers vector_ids;
milvus::engine::IDNumbers vector_ids;
int64_t nb = 100000;
std::vector<float> xb;

126
cpp/unittest/db/test_meta.cpp
View File

@ -26,16 +26,10 @@
#include <stdlib.h>
#include <time.h>
namespace {
namespace ms = milvus;
} // namespace
TEST_F(MetaTest, TABLE_TEST) {
auto table_id = "meta_test_table";
ms::engine::meta::TableSchema table;
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl_->CreateTable(table);
ASSERT_TRUE(status.ok());
@ -53,7 +47,7 @@ TEST_F(MetaTest, TABLE_TEST) {
table.table_id_ = table_id;
status = impl_->CreateTable(table);
ASSERT_EQ(status.code(), ms::DB_ALREADY_EXIST);
ASSERT_EQ(status.code(), milvus::DB_ALREADY_EXIST);
table.table_id_ = "";
status = impl_->CreateTable(table);
@ -63,16 +57,16 @@ TEST_F(MetaTest, TABLE_TEST) {
TEST_F(MetaTest, TABLE_FILE_TEST) {
auto table_id = "meta_test_table";
ms::engine::meta::TableSchema table;
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
table.dimension_ = 256;
auto status = impl_->CreateTable(table);
ms::engine::meta::TableFileSchema table_file;
milvus::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
status = impl_->CreateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type_, ms::engine::meta::TableFileSchema::NEW);
ASSERT_EQ(table_file.file_type_, milvus::engine::meta::TableFileSchema::NEW);
uint64_t cnt = 0;
status = impl_->Count(table_id, cnt);
@ -81,30 +75,30 @@ TEST_F(MetaTest, TABLE_FILE_TEST) {
auto file_id = table_file.file_id_;
auto new_file_type = ms::engine::meta::TableFileSchema::INDEX;
auto new_file_type = milvus::engine::meta::TableFileSchema::INDEX;
table_file.file_type_ = new_file_type;
status = impl_->UpdateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type_, new_file_type);
ms::engine::meta::DatesT dates;
dates.push_back(ms::engine::utils::GetDate());
milvus::engine::meta::DatesT dates;
dates.push_back(milvus::engine::utils::GetDate());
status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok());
dates.clear();
for (auto i = 2; i < 10; ++i) {
dates.push_back(ms::engine::utils::GetDateWithDelta(-1 * i));
dates.push_back(milvus::engine::utils::GetDateWithDelta(-1 * i));
}
status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok());
table_file.date_ = ms::engine::utils::GetDateWithDelta(-2);
table_file.date_ = milvus::engine::utils::GetDateWithDelta(-2);
status = impl_->UpdateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.date_, ms::engine::utils::GetDateWithDelta(-2));
ASSERT_FALSE(table_file.file_type_ == ms::engine::meta::TableFileSchema::TO_DELETE);
ASSERT_EQ(table_file.date_, milvus::engine::utils::GetDateWithDelta(-2));
ASSERT_FALSE(table_file.file_type_ == milvus::engine::meta::TableFileSchema::TO_DELETE);
dates.clear();
dates.push_back(table_file.date_);
@ -112,7 +106,7 @@ TEST_F(MetaTest, TABLE_FILE_TEST) {
ASSERT_TRUE(status.ok());
std::vector<size_t> ids = {table_file.id_};
ms::engine::meta::TableFilesSchema files;
milvus::engine::meta::TableFilesSchema files;
status = impl_->GetTableFiles(table_file.table_id_, ids, files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(files.size(), 0UL);
@ -120,34 +114,34 @@ TEST_F(MetaTest, TABLE_FILE_TEST) {
TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
srand(time(0));
ms::engine::DBMetaOptions options;
milvus::engine::DBMetaOptions options;
options.path_ = "/tmp/milvus_test";
unsigned int seed = 1;
int days_num = rand_r(&seed) % 100;
std::stringstream ss;
ss << "days:" << days_num;
options.archive_conf_ = ms::engine::ArchiveConf("delete", ss.str());
options.archive_conf_ = milvus::engine::ArchiveConf("delete", ss.str());
ms::engine::meta::SqliteMetaImpl impl(options);
milvus::engine::meta::SqliteMetaImpl impl(options);
auto table_id = "meta_test_table";
ms::engine::meta::TableSchema table;
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl.CreateTable(table);
ms::engine::meta::TableFilesSchema files;
ms::engine::meta::TableFileSchema table_file;
milvus::engine::meta::TableFilesSchema files;
milvus::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
auto cnt = 100;
int64_t ts = ms::engine::utils::GetMicroSecTimeStamp();
int64_t ts = milvus::engine::utils::GetMicroSecTimeStamp();
std::vector<int> days;
std::vector<size_t> ids;
for (auto i = 0; i < cnt; ++i) {
status = impl.CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW;
int day = rand_r(&seed) % (days_num * 2);
table_file.created_on_ = ts - day * ms::engine::meta::D_SEC * ms::engine::meta::US_PS - 10000;
table_file.created_on_ = ts - day * milvus::engine::meta::D_SEC * milvus::engine::meta::US_PS - 10000;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
days.push_back(day);
@ -157,13 +151,13 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
impl.Archive();
int i = 0;
ms::engine::meta::TableFilesSchema files_get;
milvus::engine::meta::TableFilesSchema files_get;
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for (auto &file : files_get) {
if (days[i] < days_num) {
ASSERT_EQ(file.file_type_, ms::engine::meta::TableFileSchema::NEW);
ASSERT_EQ(file.file_type_, milvus::engine::meta::TableFileSchema::NEW);
}
i++;
}
@ -172,19 +166,19 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
}
TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
ms::engine::DBMetaOptions options;
milvus::engine::DBMetaOptions options;
options.path_ = "/tmp/milvus_test";
options.archive_conf_ = ms::engine::ArchiveConf("delete", "disk:11");
options.archive_conf_ = milvus::engine::ArchiveConf("delete", "disk:11");
ms::engine::meta::SqliteMetaImpl impl(options);
milvus::engine::meta::SqliteMetaImpl impl(options);
auto table_id = "meta_test_group";
ms::engine::meta::TableSchema table;
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl.CreateTable(table);
ms::engine::meta::TableFilesSchema files;
ms::engine::meta::TableFileSchema table_file;
milvus::engine::meta::TableFilesSchema files;
milvus::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
auto cnt = 10;
@ -192,8 +186,8 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
std::vector<size_t> ids;
for (auto i = 0; i < cnt; ++i) {
status = impl.CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
table_file.file_size_ = each_size * ms::engine::G;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW;
table_file.file_size_ = each_size * milvus::engine::G;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
ids.push_back(table_file.id_);
@ -202,13 +196,13 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
impl.Archive();
int i = 0;
ms::engine::meta::TableFilesSchema files_get;
milvus::engine::meta::TableFilesSchema files_get;
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for (auto &file : files_get) {
if (i >= 5) {
ASSERT_EQ(file.file_type_, ms::engine::meta::TableFileSchema::NEW);
ASSERT_EQ(file.file_type_, milvus::engine::meta::TableFileSchema::NEW);
}
++i;
}
@ -219,7 +213,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
TEST_F(MetaTest, TABLE_FILES_TEST) {
auto table_id = "meta_test_group";
ms::engine::meta::TableSchema table;
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl_->CreateTable(table);
@ -231,51 +225,51 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
uint64_t to_index_files_cnt = 6;
uint64_t index_files_cnt = 7;
ms::engine::meta::TableFileSchema table_file;
milvus::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
for (auto i = 0; i < new_merge_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_MERGE;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW_MERGE;
status = impl_->UpdateTableFile(table_file);
}
for (auto i = 0; i < new_index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_INDEX;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW_INDEX;
status = impl_->UpdateTableFile(table_file);
}
for (auto i = 0; i < backup_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::BACKUP;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::BACKUP;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
for (auto i = 0; i < new_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW;
status = impl_->UpdateTableFile(table_file);
}
for (auto i = 0; i < raw_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::RAW;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::RAW;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
for (auto i = 0; i < to_index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::TO_INDEX;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::TO_INDEX;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
for (auto i = 0; i < index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::INDEX;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::INDEX;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
@ -285,28 +279,28 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
ASSERT_TRUE(status.ok());
ASSERT_EQ(total_row_count, raw_files_cnt + to_index_files_cnt + index_files_cnt);
ms::engine::meta::TableFilesSchema files;
milvus::engine::meta::TableFilesSchema files;
status = impl_->FilesToIndex(files);
ASSERT_EQ(files.size(), to_index_files_cnt);
ms::engine::meta::DatePartionedTableFilesSchema dated_files;
milvus::engine::meta::DatePartionedTableFilesSchema dated_files;
status = impl_->FilesToMerge(table.table_id_, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt);
status = impl_->FilesToIndex(files);
ASSERT_EQ(files.size(), to_index_files_cnt);
ms::engine::meta::DatesT dates = {table_file.date_};
milvus::engine::meta::DatesT dates = {table_file.date_};
std::vector<size_t> ids;
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files);
status = impl_->FilesToSearch(table_id, ids, milvus::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files);
status = impl_->FilesToSearch(table_id, ids, milvus::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt + raw_files_cnt + index_files_cnt);
@ -321,13 +315,13 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
ASSERT_FALSE(status.ok());
file_types = {
ms::engine::meta::TableFileSchema::NEW,
ms::engine::meta::TableFileSchema::NEW_MERGE,
ms::engine::meta::TableFileSchema::NEW_INDEX,
ms::engine::meta::TableFileSchema::TO_INDEX,
ms::engine::meta::TableFileSchema::INDEX,
ms::engine::meta::TableFileSchema::RAW,
ms::engine::meta::TableFileSchema::BACKUP,
milvus::engine::meta::TableFileSchema::NEW,
milvus::engine::meta::TableFileSchema::NEW_MERGE,
milvus::engine::meta::TableFileSchema::NEW_INDEX,
milvus::engine::meta::TableFileSchema::TO_INDEX,
milvus::engine::meta::TableFileSchema::INDEX,
milvus::engine::meta::TableFileSchema::RAW,
milvus::engine::meta::TableFileSchema::BACKUP,
};
status = impl_->FilesByType(table.table_id_, file_types, file_ids);
ASSERT_TRUE(status.ok());
@ -340,7 +334,7 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
ASSERT_TRUE(status.ok());
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW;
status = impl_->UpdateTableFile(table_file);
status = impl_->CleanUp();
ASSERT_TRUE(status.ok());
@ -355,11 +349,11 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
TEST_F(MetaTest, INDEX_TEST) {
auto table_id = "index_test";
ms::engine::meta::TableSchema table;
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl_->CreateTable(table);
ms::engine::TableIndex index;
milvus::engine::TableIndex index;
index.metric_type_ = 2;
index.nlist_ = 1234;
index.engine_type_ = 3;
@ -370,12 +364,12 @@ TEST_F(MetaTest, INDEX_TEST) {
status = impl_->UpdateTableFlag(table_id, flag);
ASSERT_TRUE(status.ok());
ms::engine::meta::TableSchema table_info;
milvus::engine::meta::TableSchema table_info;
table_info.table_id_ = table_id;
status = impl_->DescribeTable(table_info);
ASSERT_EQ(table_info.flag_, flag);
ms::engine::TableIndex index_out;
milvus::engine::TableIndex index_out;
status = impl_->DescribeTableIndex(table_id, index_out);
ASSERT_EQ(index_out.metric_type_, index.metric_type_);
ASSERT_EQ(index_out.nlist_, index.nlist_);

133
cpp/unittest/db/test_meta_mysql.cpp
View File

@ -27,17 +27,10 @@
#include <gtest/gtest.h>
#include <mysql++/mysql++.h>
namespace {
namespace ms = milvus;
}
TEST_F(MySqlMetaTest, TABLE_TEST) {
auto table_id = "meta_test_table";
ms::engine::meta::TableSchema table;
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl_->CreateTable(table);
ASSERT_TRUE(status.ok());
@ -55,7 +48,7 @@ TEST_F(MySqlMetaTest, TABLE_TEST) {
table.table_id_ = table_id;
status = impl_->CreateTable(table);
ASSERT_EQ(status.code(), ms::DB_ALREADY_EXIST);
ASSERT_EQ(status.code(), milvus::DB_ALREADY_EXIST);
table.table_id_ = "";
status = impl_->CreateTable(table);
@ -68,19 +61,19 @@ TEST_F(MySqlMetaTest, TABLE_TEST) {
TEST_F(MySqlMetaTest, TABLE_FILE_TEST) {
auto table_id = "meta_test_table";
ms::engine::meta::TableSchema table;
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
table.dimension_ = 256;
auto status = impl_->CreateTable(table);
ms::engine::meta::TableFileSchema table_file;
milvus::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
status = impl_->CreateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type_, ms::engine::meta::TableFileSchema::NEW);
ASSERT_EQ(table_file.file_type_, milvus::engine::meta::TableFileSchema::NEW);
ms::engine::meta::DatesT dates;
dates.push_back(ms::engine::utils::GetDate());
milvus::engine::meta::DatesT dates;
dates.push_back(milvus::engine::utils::GetDate());
status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok());
@ -91,7 +84,7 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) {
auto file_id = table_file.file_id_;
auto new_file_type = ms::engine::meta::TableFileSchema::INDEX;
auto new_file_type = milvus::engine::meta::TableFileSchema::INDEX;
table_file.file_type_ = new_file_type;
status = impl_->UpdateTableFile(table_file);
@ -100,16 +93,16 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) {
dates.clear();
for (auto i = 2; i < 10; ++i) {
dates.push_back(ms::engine::utils::GetDateWithDelta(-1 * i));
dates.push_back(milvus::engine::utils::GetDateWithDelta(-1 * i));
}
status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok());
table_file.date_ = ms::engine::utils::GetDateWithDelta(-2);
table_file.date_ = milvus::engine::utils::GetDateWithDelta(-2);
status = impl_->UpdateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.date_, ms::engine::utils::GetDateWithDelta(-2));
ASSERT_FALSE(table_file.file_type_ == ms::engine::meta::TableFileSchema::TO_DELETE);
ASSERT_EQ(table_file.date_, milvus::engine::utils::GetDateWithDelta(-2));
ASSERT_FALSE(table_file.file_type_ == milvus::engine::meta::TableFileSchema::TO_DELETE);
dates.clear();
dates.push_back(table_file.date_);
@ -117,42 +110,42 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) {
ASSERT_TRUE(status.ok());
std::vector<size_t> ids = {table_file.id_};
ms::engine::meta::TableFilesSchema files;
milvus::engine::meta::TableFilesSchema files;
status = impl_->GetTableFiles(table_file.table_id_, ids, files);
ASSERT_EQ(files.size(), 0UL);
}
TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
srand(time(0));
ms::engine::DBMetaOptions options = GetOptions().meta_;
milvus::engine::DBMetaOptions options = GetOptions().meta_;
unsigned int seed = 1;
int days_num = rand_r(&seed) % 100;
std::stringstream ss;
ss << "days:" << days_num;
options.archive_conf_ = ms::engine::ArchiveConf("delete", ss.str());
int mode = ms::engine::DBOptions::MODE::SINGLE;
ms::engine::meta::MySQLMetaImpl impl(options, mode);
options.archive_conf_ = milvus::engine::ArchiveConf("delete", ss.str());
int mode = milvus::engine::DBOptions::MODE::SINGLE;
milvus::engine::meta::MySQLMetaImpl impl(options, mode);
auto table_id = "meta_test_table";
ms::engine::meta::TableSchema table;
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl.CreateTable(table);
ms::engine::meta::TableFilesSchema files;
ms::engine::meta::TableFileSchema table_file;
milvus::engine::meta::TableFilesSchema files;
milvus::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
auto cnt = 100;
int64_t ts = ms::engine::utils::GetMicroSecTimeStamp();
int64_t ts = milvus::engine::utils::GetMicroSecTimeStamp();
std::vector<int> days;
std::vector<size_t> ids;
for (auto i = 0; i < cnt; ++i) {
status = impl.CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW;
int day = rand_r(&seed) % (days_num * 2);
table_file.created_on_ = ts - day * ms::engine::meta::D_SEC * ms::engine::meta::US_PS - 10000;
table_file.created_on_ = ts - day * milvus::engine::meta::D_SEC * milvus::engine::meta::US_PS - 10000;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
days.push_back(day);
@ -162,19 +155,19 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
impl.Archive();
int i = 0;
ms::engine::meta::TableFilesSchema files_get;
milvus::engine::meta::TableFilesSchema files_get;
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for (auto &file : files_get) {
if (days[i] < days_num) {
ASSERT_EQ(file.file_type_, ms::engine::meta::TableFileSchema::NEW);
ASSERT_EQ(file.file_type_, milvus::engine::meta::TableFileSchema::NEW);
}
i++;
}
std::vector<int> file_types = {
(int) ms::engine::meta::TableFileSchema::NEW,
(int) milvus::engine::meta::TableFileSchema::NEW,
};
std::vector<std::string> file_ids;
status = impl.FilesByType(table_id, file_types, file_ids);
@ -188,23 +181,23 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
}
TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
ms::engine::DBMetaOptions options = GetOptions().meta_;
milvus::engine::DBMetaOptions options = GetOptions().meta_;
options.archive_conf_ = ms::engine::ArchiveConf("delete", "disk:11");
int mode = ms::engine::DBOptions::MODE::SINGLE;
auto impl = ms::engine::meta::MySQLMetaImpl(options, mode);
options.archive_conf_ = milvus::engine::ArchiveConf("delete", "disk:11");
int mode = milvus::engine::DBOptions::MODE::SINGLE;
auto impl = milvus::engine::meta::MySQLMetaImpl(options, mode);
auto table_id = "meta_test_group";
ms::engine::meta::TableSchema table;
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl.CreateTable(table);
ms::engine::meta::TableSchema table_schema;
milvus::engine::meta::TableSchema table_schema;
table_schema.table_id_ = "";
status = impl.CreateTable(table_schema);
ms::engine::meta::TableFilesSchema files;
ms::engine::meta::TableFileSchema table_file;
milvus::engine::meta::TableFilesSchema files;
milvus::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
auto cnt = 10;
@ -212,8 +205,8 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
std::vector<size_t> ids;
for (auto i = 0; i < cnt; ++i) {
status = impl.CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
table_file.file_size_ = each_size * ms::engine::G;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW;
table_file.file_size_ = each_size * milvus::engine::G;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
ids.push_back(table_file.id_);
@ -222,13 +215,13 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
impl.Archive();
int i = 0;
ms::engine::meta::TableFilesSchema files_get;
milvus::engine::meta::TableFilesSchema files_get;
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for (auto &file : files_get) {
if (i >= 5) {
ASSERT_EQ(file.file_type_, ms::engine::meta::TableFileSchema::NEW);
ASSERT_EQ(file.file_type_, milvus::engine::meta::TableFileSchema::NEW);
}
++i;
}
@ -240,7 +233,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
auto table_id = "meta_test_group";
ms::engine::meta::TableSchema table;
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl_->CreateTable(table);
@ -252,51 +245,51 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
uint64_t to_index_files_cnt = 6;
uint64_t index_files_cnt = 7;
ms::engine::meta::TableFileSchema table_file;
milvus::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
for (auto i = 0; i < new_merge_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_MERGE;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW_MERGE;
status = impl_->UpdateTableFile(table_file);
}
for (auto i = 0; i < new_index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_INDEX;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW_INDEX;
status = impl_->UpdateTableFile(table_file);
}
for (auto i = 0; i < backup_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::BACKUP;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::BACKUP;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
for (auto i = 0; i < new_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::NEW;
status = impl_->UpdateTableFile(table_file);
}
for (auto i = 0; i < raw_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::RAW;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::RAW;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
for (auto i = 0; i < to_index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::TO_INDEX;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::TO_INDEX;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
for (auto i = 0; i < index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = ms::engine::meta::TableFileSchema::INDEX;
table_file.file_type_ = milvus::engine::meta::TableFileSchema::INDEX;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
@ -306,28 +299,28 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
ASSERT_TRUE(status.ok());
ASSERT_EQ(total_row_count, raw_files_cnt + to_index_files_cnt + index_files_cnt);
ms::engine::meta::TableFilesSchema files;
milvus::engine::meta::TableFilesSchema files;
status = impl_->FilesToIndex(files);
ASSERT_EQ(files.size(), to_index_files_cnt);
ms::engine::meta::DatePartionedTableFilesSchema dated_files;
milvus::engine::meta::DatePartionedTableFilesSchema dated_files;
status = impl_->FilesToMerge(table.table_id_, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt);
status = impl_->FilesToIndex(files);
ASSERT_EQ(files.size(), to_index_files_cnt);
ms::engine::meta::DatesT dates = {table_file.date_};
milvus::engine::meta::DatesT dates = {table_file.date_};
std::vector<size_t> ids;
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files);
status = impl_->FilesToSearch(table_id, ids, milvus::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files);
status = impl_->FilesToSearch(table_id, ids, milvus::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt + raw_files_cnt + index_files_cnt);
@ -342,13 +335,13 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
ASSERT_FALSE(status.ok());
file_types = {
ms::engine::meta::TableFileSchema::NEW,
ms::engine::meta::TableFileSchema::NEW_MERGE,
ms::engine::meta::TableFileSchema::NEW_INDEX,
ms::engine::meta::TableFileSchema::TO_INDEX,
ms::engine::meta::TableFileSchema::INDEX,
ms::engine::meta::TableFileSchema::RAW,
ms::engine::meta::TableFileSchema::BACKUP,
milvus::engine::meta::TableFileSchema::NEW,
milvus::engine::meta::TableFileSchema::NEW_MERGE,
milvus::engine::meta::TableFileSchema::NEW_INDEX,
milvus::engine::meta::TableFileSchema::TO_INDEX,
milvus::engine::meta::TableFileSchema::INDEX,
milvus::engine::meta::TableFileSchema::RAW,
milvus::engine::meta::TableFileSchema::BACKUP,
};
status = impl_->FilesByType(table.table_id_, file_types, file_ids);
ASSERT_TRUE(status.ok());
@ -370,11 +363,11 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
TEST_F(MySqlMetaTest, INDEX_TEST) {
auto table_id = "index_test";
ms::engine::meta::TableSchema table;
milvus::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl_->CreateTable(table);
ms::engine::TableIndex index;
milvus::engine::TableIndex index;
index.metric_type_ = 2;
index.nlist_ = 1234;
index.engine_type_ = 3;
@ -385,12 +378,12 @@ TEST_F(MySqlMetaTest, INDEX_TEST) {
status = impl_->UpdateTableFlag(table_id, flag);
ASSERT_TRUE(status.ok());
ms::engine::meta::TableSchema table_info;
milvus::engine::meta::TableSchema table_info;
table_info.table_id_ = table_id;
status = impl_->DescribeTable(table_info);
ASSERT_EQ(table_info.flag_, flag);
ms::engine::TableIndex index_out;
milvus::engine::TableIndex index_out;
status = impl_->DescribeTableIndex(table_id, index_out);
ASSERT_EQ(index_out.metric_type_, index.metric_type_);
ASSERT_EQ(index_out.nlist_, index.nlist_);

42
cpp/unittest/db/test_misc.cpp
View File

@ -27,47 +27,41 @@
#include <boost/filesystem.hpp>
#include <vector>
namespace {
namespace ms = milvus;
} // namespace
TEST(DBMiscTest, EXCEPTION_TEST) {
ms::Exception ex1(100, "error");
milvus::Exception ex1(100, "error");
std::string what = ex1.what();
ASSERT_EQ(what, "error");
ASSERT_EQ(ex1.code(), 100);
ms::InvalidArgumentException ex2;
ASSERT_EQ(ex2.code(), ms::SERVER_INVALID_ARGUMENT);
milvus::InvalidArgumentException ex2;
ASSERT_EQ(ex2.code(), milvus::SERVER_INVALID_ARGUMENT);
}
TEST(DBMiscTest, OPTIONS_TEST) {
try {
ms::engine::ArchiveConf archive("$$##");
milvus::engine::ArchiveConf archive("$$##");
} catch (std::exception &ex) {
ASSERT_TRUE(true);
}
{
ms::engine::ArchiveConf archive("delete", "no");
milvus::engine::ArchiveConf archive("delete", "no");
ASSERT_TRUE(archive.GetCriterias().empty());
}
{
ms::engine::ArchiveConf archive("delete", "1:2");
milvus::engine::ArchiveConf archive("delete", "1:2");
ASSERT_TRUE(archive.GetCriterias().empty());
}
{
ms::engine::ArchiveConf archive("delete", "1:2:3");
milvus::engine::ArchiveConf archive("delete", "1:2:3");
ASSERT_TRUE(archive.GetCriterias().empty());
}
{
ms::engine::ArchiveConf archive("delete");
ms::engine::ArchiveConf::CriteriaT criterial = {
milvus::engine::ArchiveConf archive("delete");
milvus::engine::ArchiveConf::CriteriaT criterial = {
{"disk", 1024},
{"days", 100}
};
@ -80,25 +74,25 @@ TEST(DBMiscTest, OPTIONS_TEST) {
}
TEST(DBMiscTest, META_TEST) {
ms::engine::DBMetaOptions options;
milvus::engine::DBMetaOptions options;
options.path_ = "/tmp/milvus_test";
ms::engine::meta::SqliteMetaImpl impl(options);
milvus::engine::meta::SqliteMetaImpl impl(options);
time_t tt;
time(&tt);
int delta = 10;
ms::engine::meta::DateT dt = ms::engine::utils::GetDate(tt, delta);
milvus::engine::meta::DateT dt = milvus::engine::utils::GetDate(tt, delta);
ASSERT_GT(dt, 0);
}
TEST(DBMiscTest, UTILS_TEST) {
ms::engine::DBMetaOptions options;
milvus::engine::DBMetaOptions options;
options.path_ = "/tmp/milvus_test/main";
options.slave_paths_.push_back("/tmp/milvus_test/slave_1");
options.slave_paths_.push_back("/tmp/milvus_test/slave_2");
const std::string TABLE_NAME = "test_tbl";
auto status = ms::engine::utils::CreateTablePath(options, TABLE_NAME);
auto status = milvus::engine::utils::CreateTablePath(options, TABLE_NAME);
ASSERT_TRUE(status.ok());
ASSERT_TRUE(boost::filesystem::exists(options.path_));
for (auto &path : options.slave_paths_) {
@ -113,18 +107,18 @@ TEST(DBMiscTest, UTILS_TEST) {
// status = engine::utils::CreateTablePath(options, TABLE_NAME);
// ASSERT_FALSE(status.ok());
ms::engine::meta::TableFileSchema file;
milvus::engine::meta::TableFileSchema file;
file.id_ = 50;
file.table_id_ = TABLE_NAME;
file.file_type_ = 3;
file.date_ = 155000;
status = ms::engine::utils::GetTableFilePath(options, file);
status = milvus::engine::utils::GetTableFilePath(options, file);
ASSERT_FALSE(status.ok());
ASSERT_TRUE(file.location_.empty());
status = ms::engine::utils::DeleteTablePath(options, TABLE_NAME);
status = milvus::engine::utils::DeleteTablePath(options, TABLE_NAME);
ASSERT_TRUE(status.ok());
status = ms::engine::utils::DeleteTableFilePath(options, file);
status = milvus::engine::utils::DeleteTableFilePath(options, file);
ASSERT_TRUE(status.ok());
}

78
cpp/unittest/db/test_search.cpp
View File

@ -30,8 +30,8 @@ void
BuildResult(uint64_t nq,
uint64_t topk,
bool ascending,
std::vector<int64_t> &output_ids,
std::vector<float> &output_distence) {
std::vector<int64_t>& output_ids,
std::vector<float>& output_distence) {
output_ids.clear();
output_ids.resize(nq * topk);
output_distence.clear();
@ -39,20 +39,21 @@ BuildResult(uint64_t nq,
for (uint64_t i = 0; i < nq; i++) {
for (uint64_t j = 0; j < topk; j++) {
output_ids[i * topk + j] = (int64_t) (drand48() * 100000);
output_ids[i * topk + j] = (int64_t)(drand48() * 100000);
output_distence[i * topk + j] = ascending ? (j + drand48()) : ((topk - j) + drand48());
}
}
}
void CheckTopkResult(const std::vector<int64_t> &input_ids_1,
const std::vector<float> &input_distance_1,
const std::vector<int64_t> &input_ids_2,
const std::vector<float> &input_distance_2,
uint64_t nq,
uint64_t topk,
bool ascending,
const ms::ResultSet& result) {
void
CheckTopkResult(const std::vector<int64_t>& input_ids_1,
const std::vector<float>& input_distance_1,
const std::vector<int64_t>& input_ids_2,
const std::vector<float>& input_distance_2,
uint64_t nq,
uint64_t topk,
bool ascending,
const milvus::scheduler::ResultSet& result) {
ASSERT_EQ(result.size(), nq);
ASSERT_EQ(input_ids_1.size(), input_distance_1.size());
ASSERT_EQ(input_ids_2.size(), input_distance_2.size());
@ -61,15 +62,18 @@ void CheckTopkResult(const std::vector<int64_t> &input_ids_1,
uint64_t input_k2 = input_ids_2.size() / nq;
for (int64_t i = 0; i < nq; i++) {
std::vector<float> src_vec(input_distance_1.begin()+i*input_k1, input_distance_1.begin()+(i+1)*input_k1);
src_vec.insert(src_vec.end(), input_distance_2.begin()+i*input_k2, input_distance_2.begin()+(i+1)*input_k2);
std::vector<float>
src_vec(input_distance_1.begin() + i * input_k1, input_distance_1.begin() + (i + 1) * input_k1);
src_vec.insert(src_vec.end(),
input_distance_2.begin() + i * input_k2,
input_distance_2.begin() + (i + 1) * input_k2);
if (ascending) {
std::sort(src_vec.begin(), src_vec.end());
} else {
std::sort(src_vec.begin(), src_vec.end(), std::greater<float>());
}
uint64_t n = std::min(topk, input_k1+input_k2);
uint64_t n = std::min(topk, input_k1 + input_k2);
for (uint64_t j = 0; j < n; j++) {
if (src_vec[j] != result[i][j].second) {
std::cout << src_vec[j] << " " << result[i][j].second << std::endl;
@ -87,19 +91,19 @@ TEST(DBSearchTest, TOPK_TEST) {
bool ascending;
std::vector<int64_t> ids1, ids2;
std::vector<float> dist1, dist2;
ms::ResultSet result;
milvus::scheduler::ResultSet result;
milvus::Status status;
/* test1, id1/dist1 valid, id2/dist2 empty */
ascending = true;
BuildResult(NQ, TOP_K, ascending, ids1, dist1);
status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K, NQ, TOP_K, ascending, result);
status = milvus::scheduler::XSearchTask::TopkResult(ids1, dist1, TOP_K, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
/* test2, id1/dist1 valid, id2/dist2 valid */
BuildResult(NQ, TOP_K, ascending, ids2, dist2);
status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
status = milvus::scheduler::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
@ -107,10 +111,10 @@ TEST(DBSearchTest, TOPK_TEST) {
ids1.clear();
dist1.clear();
result.clear();
BuildResult(NQ, TOP_K/2, ascending, ids1, dist1);
status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
BuildResult(NQ, TOP_K / 2, ascending, ids1, dist1);
status = milvus::scheduler::XSearchTask::TopkResult(ids1, dist1, TOP_K / 2, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
status = milvus::scheduler::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
@ -118,10 +122,10 @@ TEST(DBSearchTest, TOPK_TEST) {
ids2.clear();
dist2.clear();
result.clear();
BuildResult(NQ, TOP_K/3, ascending, ids2, dist2);
status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
BuildResult(NQ, TOP_K / 3, ascending, ids2, dist2);
status = milvus::scheduler::XSearchTask::TopkResult(ids1, dist1, TOP_K / 2, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K/3, NQ, TOP_K, ascending, result);
status = milvus::scheduler::XSearchTask::TopkResult(ids2, dist2, TOP_K / 3, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
@ -135,13 +139,13 @@ TEST(DBSearchTest, TOPK_TEST) {
/* test1, id1/dist1 valid, id2/dist2 empty */
BuildResult(NQ, TOP_K, ascending, ids1, dist1);
status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K, NQ, TOP_K, ascending, result);
status = milvus::scheduler::XSearchTask::TopkResult(ids1, dist1, TOP_K, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
/* test2, id1/dist1 valid, id2/dist2 valid */
BuildResult(NQ, TOP_K, ascending, ids2, dist2);
status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
status = milvus::scheduler::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
@ -149,10 +153,10 @@ TEST(DBSearchTest, TOPK_TEST) {
ids1.clear();
dist1.clear();
result.clear();
BuildResult(NQ, TOP_K/2, ascending, ids1, dist1);
status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
BuildResult(NQ, TOP_K / 2, ascending, ids1, dist1);
status = milvus::scheduler::XSearchTask::TopkResult(ids1, dist1, TOP_K / 2, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
status = milvus::scheduler::XSearchTask::TopkResult(ids2, dist2, TOP_K, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
@ -160,10 +164,10 @@ TEST(DBSearchTest, TOPK_TEST) {
ids2.clear();
dist2.clear();
result.clear();
BuildResult(NQ, TOP_K/3, ascending, ids2, dist2);
status = ms::XSearchTask::TopkResult(ids1, dist1, TOP_K/2, NQ, TOP_K, ascending, result);
BuildResult(NQ, TOP_K / 3, ascending, ids2, dist2);
status = milvus::scheduler::XSearchTask::TopkResult(ids1, dist1, TOP_K / 2, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
status = ms::XSearchTask::TopkResult(ids2, dist2, TOP_K/3, NQ, TOP_K, ascending, result);
status = milvus::scheduler::XSearchTask::TopkResult(ids2, dist2, TOP_K / 3, NQ, TOP_K, ascending, result);
ASSERT_TRUE(status.ok());
CheckTopkResult(ids1, dist1, ids2, dist2, NQ, TOP_K, ascending, result);
}
@ -175,7 +179,7 @@ TEST(DBSearchTest, REDUCE_PERF_TEST) {
bool ascending = true;
std::vector<int64_t> input_ids;
std::vector<float> input_distance;
ms::ResultSet final_result;
milvus::scheduler::ResultSet final_result;
milvus::Status status;
double span, reduce_cost = 0.0;
@ -187,12 +191,18 @@ TEST(DBSearchTest, REDUCE_PERF_TEST) {
rc.RecordSection("do search for context: " + std::to_string(i));
// pick up topk result
status = ms::XSearchTask::TopkResult(input_ids, input_distance, top_k, nq, top_k, ascending, final_result);
status = milvus::scheduler::XSearchTask::TopkResult(input_ids,
input_distance,
top_k,
nq,
top_k,
ascending,
final_result);
ASSERT_TRUE(status.ok());
ASSERT_EQ(final_result.size(), nq);
span = rc.RecordSection("reduce topk for context: " + std::to_string(i));
reduce_cost += span;
}
std::cout << "total reduce time: " << reduce_cost/1000 << " ms" << std::endl;
std::cout << "total reduce time: " << reduce_cost / 1000 << " ms" << std::endl;
}

50
cpp/unittest/db/utils.cpp
View File

@ -33,8 +33,6 @@ INITIALIZE_EASYLOGGINGPP
namespace {
namespace ms = milvus;
class DBTestEnvironment : public ::testing::Environment {
public:
explicit DBTestEnvironment(const std::string& uri)
@ -83,9 +81,9 @@ BaseTest::TearDown() {
knowhere::FaissGpuResourceMgr::GetInstance().Free();
}
ms::engine::DBOptions
milvus::engine::DBOptions
BaseTest::GetOptions() {
auto options = ms::engine::DBFactory::BuildOption();
auto options = milvus::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.backend_uri_ = "sqlite://:@:/";
return options;
@ -96,23 +94,23 @@ void
DBTest::SetUp() {
BaseTest::SetUp();
auto res_mgr = ms::scheduler::ResMgrInst::GetInstance();
auto res_mgr = milvus::scheduler::ResMgrInst::GetInstance();
res_mgr->Clear();
res_mgr->Add(ms::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false));
res_mgr->Add(ms::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, false));
res_mgr->Add(ms::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true));
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false));
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, false));
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true));
auto default_conn = ms::scheduler::Connection("IO", 500.0);
auto PCIE = ms::scheduler::Connection("IO", 11000.0);
auto default_conn = milvus::scheduler::Connection("IO", 500.0);
auto PCIE = milvus::scheduler::Connection("IO", 11000.0);
res_mgr->Connect("disk", "cpu", default_conn);
res_mgr->Connect("cpu", "gtx1660", PCIE);
res_mgr->Start();
ms::scheduler::SchedInst::GetInstance()->Start();
milvus::scheduler::SchedInst::GetInstance()->Start();
ms::scheduler::JobMgrInst::GetInstance()->Start();
milvus::scheduler::JobMgrInst::GetInstance()->Start();
auto options = GetOptions();
db_ = ms::engine::DBFactory::Build(options);
db_ = milvus::engine::DBFactory::Build(options);
}
void
@ -120,10 +118,10 @@ DBTest::TearDown() {
db_->Stop();
db_->DropAll();
ms::scheduler::JobMgrInst::GetInstance()->Stop();
ms::scheduler::SchedInst::GetInstance()->Stop();
ms::scheduler::ResMgrInst::GetInstance()->Stop();
ms::scheduler::ResMgrInst::GetInstance()->Clear();
milvus::scheduler::JobMgrInst::GetInstance()->Stop();
milvus::scheduler::SchedInst::GetInstance()->Stop();
milvus::scheduler::ResMgrInst::GetInstance()->Stop();
milvus::scheduler::ResMgrInst::GetInstance()->Clear();
BaseTest::TearDown();
@ -132,11 +130,11 @@ DBTest::TearDown() {
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
ms::engine::DBOptions
milvus::engine::DBOptions
DBTest2::GetOptions() {
auto options = ms::engine::DBFactory::BuildOption();
auto options = milvus::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.archive_conf_ = ms::engine::ArchiveConf("delete", "disk:1");
options.meta_.archive_conf_ = milvus::engine::ArchiveConf("delete", "disk:1");
options.meta_.backend_uri_ = "sqlite://:@:/";
return options;
}
@ -147,7 +145,7 @@ MetaTest::SetUp() {
BaseTest::SetUp();
auto options = GetOptions();
impl_ = std::make_shared<ms::engine::meta::SqliteMetaImpl>(options.meta_);
impl_ = std::make_shared<milvus::engine::meta::SqliteMetaImpl>(options.meta_);
}
void
@ -161,9 +159,9 @@ MetaTest::TearDown() {
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
ms::engine::DBOptions
milvus::engine::DBOptions
MySqlDBTest::GetOptions() {
auto options = ms::engine::DBFactory::BuildOption();
auto options = milvus::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.backend_uri_ = test_env->getURI();
@ -176,7 +174,7 @@ MySqlMetaTest::SetUp() {
BaseTest::SetUp();
auto options = GetOptions();
impl_ = std::make_shared<ms::engine::meta::MySQLMetaImpl>(options.meta_, options.mode_);
impl_ = std::make_shared<milvus::engine::meta::MySQLMetaImpl>(options.meta_, options.mode_);
}
void
@ -189,9 +187,9 @@ MySqlMetaTest::TearDown() {
boost::filesystem::remove_all(options.meta_.path_);
}
ms::engine::DBOptions
milvus::engine::DBOptions
MySqlMetaTest::GetOptions() {
auto options = ms::engine::DBFactory::BuildOption();
auto options = milvus::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.backend_uri_ = test_env->getURI();

10
cpp/unittest/metrics/test_metricbase.cpp
View File

@ -21,16 +21,10 @@
#include <gtest/gtest.h>
#include <iostream>
namespace {
namespace ms = milvus;
} // namespace
TEST(MetricbaseTest, METRICBASE_TEST) {
ms::server::MetricsBase instance = ms::server::MetricsBase::GetInstance();
milvus::server::MetricsBase instance = milvus::server::MetricsBase::GetInstance();
instance.Init();
ms::server::SystemInfo::GetInstance().Init();
milvus::server::SystemInfo::GetInstance().Init();
instance.AddVectorsSuccessTotalIncrement();
instance.AddVectorsFailTotalIncrement();
instance.AddVectorsDurationHistogramOberve(1.0);

62
cpp/unittest/metrics/test_metrics.cpp
View File

@ -29,22 +29,16 @@
#include "db/DB.h"
#include "db/meta/SqliteMetaImpl.h"
namespace {
namespace ms = milvus;
} // namespace
TEST_F(MetricTest, METRIC_TEST) {
ms::server::Config::GetInstance().SetMetricConfigCollector("zabbix");
ms::server::Metrics::GetInstance();
ms::server::Config::GetInstance().SetMetricConfigCollector("prometheus");
ms::server::Metrics::GetInstance();
milvus::server::Config::GetInstance().SetMetricConfigCollector("zabbix");
milvus::server::Metrics::GetInstance();
milvus::server::Config::GetInstance().SetMetricConfigCollector("prometheus");
milvus::server::Metrics::GetInstance();
ms::server::SystemInfo::GetInstance().Init();
milvus::server::SystemInfo::GetInstance().Init();
// server::Metrics::GetInstance().Init();
// server::Metrics::GetInstance().exposer_ptr()->RegisterCollectable(server::Metrics::GetInstance().registry_ptr());
ms::server::Metrics::GetInstance().Init();
milvus::server::Metrics::GetInstance().Init();
// server::PrometheusMetrics::GetInstance().exposer_ptr()->RegisterCollectable(server::PrometheusMetrics::GetInstance().registry_ptr());
milvus::cache::CpuCacheMgr::GetInstance()->SetCapacity(1UL * 1024 * 1024 * 1024);
@ -53,17 +47,17 @@ TEST_F(MetricTest, METRIC_TEST) {
static const char *group_name = "test_group";
static const int group_dim = 256;
ms::engine::meta::TableSchema group_info;
milvus::engine::meta::TableSchema group_info;
group_info.dimension_ = group_dim;
group_info.table_id_ = group_name;
auto stat = db_->CreateTable(group_info);
ms::engine::meta::TableSchema group_info_get;
milvus::engine::meta::TableSchema group_info_get;
group_info_get.table_id_ = group_name;
stat = db_->DescribeTable(group_info_get);
ms::engine::IDNumbers vector_ids;
ms::engine::IDNumbers target_ids;
milvus::engine::IDNumbers vector_ids;
milvus::engine::IDNumbers target_ids;
int d = 256;
int nb = 50;
@ -81,7 +75,7 @@ TEST_F(MetricTest, METRIC_TEST) {
}
std::thread search([&]() {
ms::engine::QueryResults results;
milvus::engine::QueryResults results;
int k = 10;
std::this_thread::sleep_for(std::chrono::seconds(2));
@ -132,32 +126,32 @@ TEST_F(MetricTest, METRIC_TEST) {
}
TEST_F(MetricTest, COLLECTOR_METRICS_TEST) {
auto status = ms::Status::OK();
ms::server::CollectInsertMetrics insert_metrics0(0, status);
status = ms::Status(ms::DB_ERROR, "error");
ms::server::CollectInsertMetrics insert_metrics1(0, status);
auto status = milvus::Status::OK();
milvus::server::CollectInsertMetrics insert_metrics0(0, status);
status = milvus::Status(milvus::DB_ERROR, "error");
milvus::server::CollectInsertMetrics insert_metrics1(0, status);
ms::server::CollectQueryMetrics query_metrics(10);
milvus::server::CollectQueryMetrics query_metrics(10);
ms::server::CollectMergeFilesMetrics merge_metrics();
milvus::server::CollectMergeFilesMetrics merge_metrics();
ms::server::CollectBuildIndexMetrics build_index_metrics();
milvus::server::CollectBuildIndexMetrics build_index_metrics();
ms::server::CollectExecutionEngineMetrics execution_metrics(10);
milvus::server::CollectExecutionEngineMetrics execution_metrics(10);
ms::server::CollectSerializeMetrics serialize_metrics(10);
milvus::server::CollectSerializeMetrics serialize_metrics(10);
ms::server::CollectAddMetrics add_metrics(10, 128);
milvus::server::CollectAddMetrics add_metrics(10, 128);
ms::server::CollectDurationMetrics duration_metrics_raw(ms::engine::meta::TableFileSchema::RAW);
ms::server::CollectDurationMetrics duration_metrics_index(ms::engine::meta::TableFileSchema::TO_INDEX);
ms::server::CollectDurationMetrics duration_metrics_delete(ms::engine::meta::TableFileSchema::TO_DELETE);
milvus::server::CollectDurationMetrics duration_metrics_raw(milvus::engine::meta::TableFileSchema::RAW);
milvus::server::CollectDurationMetrics duration_metrics_index(milvus::engine::meta::TableFileSchema::TO_INDEX);
milvus::server::CollectDurationMetrics duration_metrics_delete(milvus::engine::meta::TableFileSchema::TO_DELETE);
ms::server::CollectSearchTaskMetrics search_metrics_raw(ms::engine::meta::TableFileSchema::RAW);
ms::server::CollectSearchTaskMetrics search_metrics_index(ms::engine::meta::TableFileSchema::TO_INDEX);
ms::server::CollectSearchTaskMetrics search_metrics_delete(ms::engine::meta::TableFileSchema::TO_DELETE);
milvus::server::CollectSearchTaskMetrics search_metrics_raw(milvus::engine::meta::TableFileSchema::RAW);
milvus::server::CollectSearchTaskMetrics search_metrics_index(milvus::engine::meta::TableFileSchema::TO_INDEX);
milvus::server::CollectSearchTaskMetrics search_metrics_delete(milvus::engine::meta::TableFileSchema::TO_DELETE);
ms::server::MetricCollector metric_collector();
milvus::server::MetricCollector metric_collector();
}

14
cpp/unittest/metrics/test_prometheus.cpp
View File

@ -22,19 +22,13 @@
#include <gtest/gtest.h>
#include <iostream>
namespace {
namespace ms = milvus;
} // namespace
TEST(PrometheusTest, PROMETHEUS_TEST) {
ms::server::Config::GetInstance().SetMetricConfigEnableMonitor("on");
milvus::server::Config::GetInstance().SetMetricConfigEnableMonitor("on");
ms::server::PrometheusMetrics instance = ms::server::PrometheusMetrics::GetInstance();
milvus::server::PrometheusMetrics instance = milvus::server::PrometheusMetrics::GetInstance();
instance.Init();
instance.SetStartup(true);
ms::server::SystemInfo::GetInstance().Init();
milvus::server::SystemInfo::GetInstance().Init();
instance.AddVectorsSuccessTotalIncrement();
instance.AddVectorsFailTotalIncrement();
instance.AddVectorsDurationHistogramOberve(1.0);
@ -80,7 +74,7 @@ TEST(PrometheusTest, PROMETHEUS_TEST) {
instance.GPUTemperature();
instance.CPUTemperature();
ms::server::Config::GetInstance().SetMetricConfigEnableMonitor("off");
milvus::server::Config::GetInstance().SetMetricConfigEnableMonitor("off");
instance.Init();
instance.CPUCoreUsagePercentSet();
instance.GPUTemperature();

8
cpp/unittest/metrics/utils.cpp
View File

@ -28,8 +28,6 @@ INITIALIZE_EASYLOGGINGPP
namespace {
namespace ms = milvus;
class DBTestEnvironment : public ::testing::Environment {
public:
explicit DBTestEnvironment(const std::string& uri) : uri_(uri) {}
@ -58,8 +56,8 @@ void MetricTest::InitLog() {
el::Loggers::reconfigureLogger("default", defaultConf);
}
ms::engine::DBOptions MetricTest::GetOptions() {
auto options = ms::engine::DBFactory::BuildOption();
milvus::engine::DBOptions MetricTest::GetOptions() {
auto options = milvus::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.backend_uri_ = "sqlite://:@:/";
return options;
@ -69,7 +67,7 @@ void MetricTest::SetUp() {
boost::filesystem::remove_all("/tmp/milvus_test");
InitLog();
auto options = GetOptions();
db_ = ms::engine::DBFactory::Build(options);
db_ = milvus::engine::DBFactory::Build(options);
}
void MetricTest::TearDown() {

178
cpp/unittest/scheduler/test_tasktable.cpp
View File

@ -21,53 +21,47 @@
#include "scheduler/tasklabel/DefaultLabel.h"
#include <gtest/gtest.h>
namespace {
namespace ms = milvus::scheduler;
} // namespace
/************ TaskTableBaseTest ************/
class TaskTableItemTest : public ::testing::Test {
protected:
void
SetUp() override {
std::vector<ms::TaskTableItemState> states{
ms::TaskTableItemState::INVALID,
ms::TaskTableItemState::START,
ms::TaskTableItemState::LOADING,
ms::TaskTableItemState::LOADED,
ms::TaskTableItemState::EXECUTING,
ms::TaskTableItemState::EXECUTED,
ms::TaskTableItemState::MOVING,
ms::TaskTableItemState::MOVED};
std::vector<milvus::scheduler::TaskTableItemState> states{
milvus::scheduler::TaskTableItemState::INVALID,
milvus::scheduler::TaskTableItemState::START,
milvus::scheduler::TaskTableItemState::LOADING,
milvus::scheduler::TaskTableItemState::LOADED,
milvus::scheduler::TaskTableItemState::EXECUTING,
milvus::scheduler::TaskTableItemState::EXECUTED,
milvus::scheduler::TaskTableItemState::MOVING,
milvus::scheduler::TaskTableItemState::MOVED};
for (auto &state : states) {
auto item = std::make_shared<ms::TaskTableItem>();
auto item = std::make_shared<milvus::scheduler::TaskTableItem>();
item->state = state;
items_.emplace_back(item);
}
}
ms::TaskTableItem default_;
std::vector<ms::TaskTableItemPtr> items_;
milvus::scheduler::TaskTableItem default_;
std::vector<milvus::scheduler::TaskTableItemPtr> items_;
};
TEST_F(TaskTableItemTest, CONSTRUCT) {
ASSERT_EQ(default_.id, 0);
ASSERT_EQ(default_.task, nullptr);
ASSERT_EQ(default_.state, ms::TaskTableItemState::INVALID);
ASSERT_EQ(default_.state, milvus::scheduler::TaskTableItemState::INVALID);
}
TEST_F(TaskTableItemTest, DESTRUCT) {
auto p_item = new ms::TaskTableItem();
auto p_item = new milvus::scheduler::TaskTableItem();
delete p_item;
}
TEST_F(TaskTableItemTest, IS_FINISH) {
for (auto &item : items_) {
if (item->state == ms::TaskTableItemState::EXECUTED
|| item->state == ms::TaskTableItemState::MOVED) {
if (item->state == milvus::scheduler::TaskTableItemState::EXECUTED
|| item->state == milvus::scheduler::TaskTableItemState::MOVED) {
ASSERT_TRUE(item->IsFinish());
} else {
ASSERT_FALSE(item->IsFinish());
@ -85,9 +79,9 @@ TEST_F(TaskTableItemTest, LOAD) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Load();
if (before_state == ms::TaskTableItemState::START) {
if (before_state == milvus::scheduler::TaskTableItemState::START) {
ASSERT_TRUE(ret);
ASSERT_EQ(item->state, ms::TaskTableItemState::LOADING);
ASSERT_EQ(item->state, milvus::scheduler::TaskTableItemState::LOADING);
} else {
ASSERT_FALSE(ret);
ASSERT_EQ(item->state, before_state);
@ -99,9 +93,9 @@ TEST_F(TaskTableItemTest, LOADED) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Loaded();
if (before_state == ms::TaskTableItemState::LOADING) {
if (before_state == milvus::scheduler::TaskTableItemState::LOADING) {
ASSERT_TRUE(ret);
ASSERT_EQ(item->state, ms::TaskTableItemState::LOADED);
ASSERT_EQ(item->state, milvus::scheduler::TaskTableItemState::LOADED);
} else {
ASSERT_FALSE(ret);
ASSERT_EQ(item->state, before_state);
@ -113,9 +107,9 @@ TEST_F(TaskTableItemTest, EXECUTE) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Execute();
if (before_state == ms::TaskTableItemState::LOADED) {
if (before_state == milvus::scheduler::TaskTableItemState::LOADED) {
ASSERT_TRUE(ret);
ASSERT_EQ(item->state, ms::TaskTableItemState::EXECUTING);
ASSERT_EQ(item->state, milvus::scheduler::TaskTableItemState::EXECUTING);
} else {
ASSERT_FALSE(ret);
ASSERT_EQ(item->state, before_state);
@ -127,9 +121,9 @@ TEST_F(TaskTableItemTest, EXECUTED) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Executed();
if (before_state == ms::TaskTableItemState::EXECUTING) {
if (before_state == milvus::scheduler::TaskTableItemState::EXECUTING) {
ASSERT_TRUE(ret);
ASSERT_EQ(item->state, ms::TaskTableItemState::EXECUTED);
ASSERT_EQ(item->state, milvus::scheduler::TaskTableItemState::EXECUTED);
} else {
ASSERT_FALSE(ret);
ASSERT_EQ(item->state, before_state);
@ -141,9 +135,9 @@ TEST_F(TaskTableItemTest, MOVE) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Move();
if (before_state == ms::TaskTableItemState::LOADED) {
if (before_state == milvus::scheduler::TaskTableItemState::LOADED) {
ASSERT_TRUE(ret);
ASSERT_EQ(item->state, ms::TaskTableItemState::MOVING);
ASSERT_EQ(item->state, milvus::scheduler::TaskTableItemState::MOVING);
} else {
ASSERT_FALSE(ret);
ASSERT_EQ(item->state, before_state);
@ -155,9 +149,9 @@ TEST_F(TaskTableItemTest, MOVED) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Moved();
if (before_state == ms::TaskTableItemState::MOVING) {
if (before_state == milvus::scheduler::TaskTableItemState::MOVING) {
ASSERT_TRUE(ret);
ASSERT_EQ(item->state, ms::TaskTableItemState::MOVED);
ASSERT_EQ(item->state, milvus::scheduler::TaskTableItemState::MOVED);
} else {
ASSERT_FALSE(ret);
ASSERT_EQ(item->state, before_state);
@ -171,17 +165,17 @@ class TaskTableBaseTest : public ::testing::Test {
protected:
void
SetUp() override {
ms::TableFileSchemaPtr dummy = nullptr;
milvus::scheduler::TableFileSchemaPtr dummy = nullptr;
invalid_task_ = nullptr;
auto label = std::make_shared<ms::DefaultLabel>();
task1_ = std::make_shared<ms::TestTask>(dummy, label);
task2_ = std::make_shared<ms::TestTask>(dummy, label);
auto label = std::make_shared<milvus::scheduler::DefaultLabel>();
task1_ = std::make_shared<milvus::scheduler::TestTask>(dummy, label);
task2_ = std::make_shared<milvus::scheduler::TestTask>(dummy, label);
}
ms::TaskPtr invalid_task_;
ms::TaskPtr task1_;
ms::TaskPtr task2_;
ms::TaskTable empty_table_;
milvus::scheduler::TaskPtr invalid_task_;
milvus::scheduler::TaskPtr task1_;
milvus::scheduler::TaskPtr task2_;
milvus::scheduler::TaskTable empty_table_;
};
TEST_F(TaskTableBaseTest, SUBSCRIBER) {
@ -205,14 +199,14 @@ TEST_F(TaskTableBaseTest, PUT_INVALID_TEST) {
}
TEST_F(TaskTableBaseTest, PUT_BATCH) {
std::vector<ms::TaskPtr> tasks{task1_, task2_};
std::vector<milvus::scheduler::TaskPtr> tasks{task1_, task2_};
empty_table_.Put(tasks);
ASSERT_EQ(empty_table_.Get(0)->task, task1_);
ASSERT_EQ(empty_table_.Get(1)->task, task2_);
}
TEST_F(TaskTableBaseTest, PUT_EMPTY_BATCH) {
std::vector<ms::TaskPtr> tasks{};
std::vector<milvus::scheduler::TaskPtr> tasks{};
empty_table_.Put(tasks);
}
@ -238,8 +232,8 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD) {
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
}
empty_table_[0]->state = ms::TaskTableItemState::MOVED;
empty_table_[1]->state = ms::TaskTableItemState::EXECUTED;
empty_table_[0]->state = milvus::scheduler::TaskTableItemState::MOVED;
empty_table_[1]->state = milvus::scheduler::TaskTableItemState::EXECUTED;
auto indexes = empty_table_.PickToLoad(1);
ASSERT_EQ(indexes.size(), 1);
@ -251,8 +245,8 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD_LIMIT) {
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
}
empty_table_[0]->state = ms::TaskTableItemState::MOVED;
empty_table_[1]->state = ms::TaskTableItemState::EXECUTED;
empty_table_[0]->state = milvus::scheduler::TaskTableItemState::MOVED;
empty_table_[1]->state = milvus::scheduler::TaskTableItemState::EXECUTED;
auto indexes = empty_table_.PickToLoad(3);
ASSERT_EQ(indexes.size(), 3);
@ -266,8 +260,8 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD_CACHE) {
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
}
empty_table_[0]->state = ms::TaskTableItemState::MOVED;
empty_table_[1]->state = ms::TaskTableItemState::EXECUTED;
empty_table_[0]->state = milvus::scheduler::TaskTableItemState::MOVED;
empty_table_[1]->state = milvus::scheduler::TaskTableItemState::EXECUTED;
// first pick, non-cache
auto indexes = empty_table_.PickToLoad(1);
@ -276,7 +270,7 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD_CACHE) {
// second pick, iterate from 2
// invalid state change
empty_table_[1]->state = ms::TaskTableItemState::START;
empty_table_[1]->state = milvus::scheduler::TaskTableItemState::START;
indexes = empty_table_.PickToLoad(1);
ASSERT_EQ(indexes.size(), 1);
ASSERT_EQ(indexes[0], 2);
@ -287,9 +281,9 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE) {
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
}
empty_table_[0]->state = ms::TaskTableItemState::MOVED;
empty_table_[1]->state = ms::TaskTableItemState::EXECUTED;
empty_table_[2]->state = ms::TaskTableItemState::LOADED;
empty_table_[0]->state = milvus::scheduler::TaskTableItemState::MOVED;
empty_table_[1]->state = milvus::scheduler::TaskTableItemState::EXECUTED;
empty_table_[2]->state = milvus::scheduler::TaskTableItemState::LOADED;
auto indexes = empty_table_.PickToExecute(1);
ASSERT_EQ(indexes.size(), 1);
@ -301,10 +295,10 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_LIMIT) {
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
}
empty_table_[0]->state = ms::TaskTableItemState::MOVED;
empty_table_[1]->state = ms::TaskTableItemState::EXECUTED;
empty_table_[2]->state = ms::TaskTableItemState::LOADED;
empty_table_[3]->state = ms::TaskTableItemState::LOADED;
empty_table_[0]->state = milvus::scheduler::TaskTableItemState::MOVED;
empty_table_[1]->state = milvus::scheduler::TaskTableItemState::EXECUTED;
empty_table_[2]->state = milvus::scheduler::TaskTableItemState::LOADED;
empty_table_[3]->state = milvus::scheduler::TaskTableItemState::LOADED;
auto indexes = empty_table_.PickToExecute(3);
ASSERT_EQ(indexes.size(), 2);
@ -317,9 +311,9 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_CACHE) {
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
}
empty_table_[0]->state = ms::TaskTableItemState::MOVED;
empty_table_[1]->state = ms::TaskTableItemState::EXECUTED;
empty_table_[2]->state = ms::TaskTableItemState::LOADED;
empty_table_[0]->state = milvus::scheduler::TaskTableItemState::MOVED;
empty_table_[1]->state = milvus::scheduler::TaskTableItemState::EXECUTED;
empty_table_[2]->state = milvus::scheduler::TaskTableItemState::LOADED;
// first pick, non-cache
auto indexes = empty_table_.PickToExecute(1);
@ -328,7 +322,7 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_CACHE) {
// second pick, iterate from 2
// invalid state change
empty_table_[1]->state = ms::TaskTableItemState::START;
empty_table_[1]->state = milvus::scheduler::TaskTableItemState::START;
indexes = empty_table_.PickToExecute(1);
ASSERT_EQ(indexes.size(), 1);
ASSERT_EQ(indexes[0], 2);
@ -340,28 +334,28 @@ class TaskTableAdvanceTest : public ::testing::Test {
protected:
void
SetUp() override {
ms::TableFileSchemaPtr dummy = nullptr;
milvus::scheduler::TableFileSchemaPtr dummy = nullptr;
for (uint64_t i = 0; i < 8; ++i) {
auto label = std::make_shared<ms::DefaultLabel>();
auto task = std::make_shared<ms::TestTask>(dummy, label);
auto label = std::make_shared<milvus::scheduler::DefaultLabel>();
auto task = std::make_shared<milvus::scheduler::TestTask>(dummy, label);
table1_.Put(task);
}
table1_.Get(0)->state = ms::TaskTableItemState::INVALID;
table1_.Get(1)->state = ms::TaskTableItemState::START;
table1_.Get(2)->state = ms::TaskTableItemState::LOADING;
table1_.Get(3)->state = ms::TaskTableItemState::LOADED;
table1_.Get(4)->state = ms::TaskTableItemState::EXECUTING;
table1_.Get(5)->state = ms::TaskTableItemState::EXECUTED;
table1_.Get(6)->state = ms::TaskTableItemState::MOVING;
table1_.Get(7)->state = ms::TaskTableItemState::MOVED;
table1_.Get(0)->state = milvus::scheduler::TaskTableItemState::INVALID;
table1_.Get(1)->state = milvus::scheduler::TaskTableItemState::START;
table1_.Get(2)->state = milvus::scheduler::TaskTableItemState::LOADING;
table1_.Get(3)->state = milvus::scheduler::TaskTableItemState::LOADED;
table1_.Get(4)->state = milvus::scheduler::TaskTableItemState::EXECUTING;
table1_.Get(5)->state = milvus::scheduler::TaskTableItemState::EXECUTED;
table1_.Get(6)->state = milvus::scheduler::TaskTableItemState::MOVING;
table1_.Get(7)->state = milvus::scheduler::TaskTableItemState::MOVED;
}
ms::TaskTable table1_;
milvus::scheduler::TaskTable table1_;
};
TEST_F(TaskTableAdvanceTest, LOAD) {
std::vector<ms::TaskTableItemState> before_state;
std::vector<milvus::scheduler::TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
}
@ -371,8 +365,8 @@ TEST_F(TaskTableAdvanceTest, LOAD) {
}
for (size_t i = 0; i < table1_.Size(); ++i) {
if (before_state[i] == ms::TaskTableItemState::START) {
ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::LOADING);
if (before_state[i] == milvus::scheduler::TaskTableItemState::START) {
ASSERT_EQ(table1_.Get(i)->state, milvus::scheduler::TaskTableItemState::LOADING);
} else {
ASSERT_EQ(table1_.Get(i)->state, before_state[i]);
}
@ -380,7 +374,7 @@ TEST_F(TaskTableAdvanceTest, LOAD) {
}
TEST_F(TaskTableAdvanceTest, LOADED) {
std::vector<ms::TaskTableItemState> before_state;
std::vector<milvus::scheduler::TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
}
@ -390,8 +384,8 @@ TEST_F(TaskTableAdvanceTest, LOADED) {
}
for (size_t i = 0; i < table1_.Size(); ++i) {
if (before_state[i] == ms::TaskTableItemState::LOADING) {
ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::LOADED);
if (before_state[i] == milvus::scheduler::TaskTableItemState::LOADING) {
ASSERT_EQ(table1_.Get(i)->state, milvus::scheduler::TaskTableItemState::LOADED);
} else {
ASSERT_EQ(table1_.Get(i)->state, before_state[i]);
}
@ -399,7 +393,7 @@ TEST_F(TaskTableAdvanceTest, LOADED) {
}
TEST_F(TaskTableAdvanceTest, EXECUTE) {
std::vector<ms::TaskTableItemState> before_state;
std::vector<milvus::scheduler::TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
}
@ -409,8 +403,8 @@ TEST_F(TaskTableAdvanceTest, EXECUTE) {
}
for (size_t i = 0; i < table1_.Size(); ++i) {
if (before_state[i] == ms::TaskTableItemState::LOADED) {
ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::EXECUTING);
if (before_state[i] == milvus::scheduler::TaskTableItemState::LOADED) {
ASSERT_EQ(table1_.Get(i)->state, milvus::scheduler::TaskTableItemState::EXECUTING);
} else {
ASSERT_EQ(table1_.Get(i)->state, before_state[i]);
}
@ -418,7 +412,7 @@ TEST_F(TaskTableAdvanceTest, EXECUTE) {
}
TEST_F(TaskTableAdvanceTest, EXECUTED) {
std::vector<ms::TaskTableItemState> before_state;
std::vector<milvus::scheduler::TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
}
@ -428,8 +422,8 @@ TEST_F(TaskTableAdvanceTest, EXECUTED) {
}
for (size_t i = 0; i < table1_.Size(); ++i) {
if (before_state[i] == ms::TaskTableItemState::EXECUTING) {
ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::EXECUTED);
if (before_state[i] == milvus::scheduler::TaskTableItemState::EXECUTING) {
ASSERT_EQ(table1_.Get(i)->state, milvus::scheduler::TaskTableItemState::EXECUTED);
} else {
ASSERT_EQ(table1_.Get(i)->state, before_state[i]);
}
@ -437,7 +431,7 @@ TEST_F(TaskTableAdvanceTest, EXECUTED) {
}
TEST_F(TaskTableAdvanceTest, MOVE) {
std::vector<ms::TaskTableItemState> before_state;
std::vector<milvus::scheduler::TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
}
@ -447,8 +441,8 @@ TEST_F(TaskTableAdvanceTest, MOVE) {
}
for (size_t i = 0; i < table1_.Size(); ++i) {
if (before_state[i] == ms::TaskTableItemState::LOADED) {
ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::MOVING);
if (before_state[i] == milvus::scheduler::TaskTableItemState::LOADED) {
ASSERT_EQ(table1_.Get(i)->state, milvus::scheduler::TaskTableItemState::MOVING);
} else {
ASSERT_EQ(table1_.Get(i)->state, before_state[i]);
}
@ -456,7 +450,7 @@ TEST_F(TaskTableAdvanceTest, MOVE) {
}
TEST_F(TaskTableAdvanceTest, MOVED) {
std::vector<ms::TaskTableItemState> before_state;
std::vector<milvus::scheduler::TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
}
@ -466,8 +460,8 @@ TEST_F(TaskTableAdvanceTest, MOVED) {
}
for (size_t i = 0; i < table1_.Size(); ++i) {
if (before_state[i] == ms::TaskTableItemState::MOVING) {
ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::MOVED);
if (before_state[i] == milvus::scheduler::TaskTableItemState::MOVING) {
ASSERT_EQ(table1_.Get(i)->state, milvus::scheduler::TaskTableItemState::MOVED);
} else {
ASSERT_EQ(table1_.Get(i)->state, before_state[i]);
}

72
cpp/unittest/server/test_cache.cpp
View File

@ -23,38 +23,36 @@
namespace {
namespace ms = milvus;
class InvalidCacheMgr : public ms::cache::CacheMgr<ms::cache::DataObjPtr> {
class InvalidCacheMgr : public milvus::cache::CacheMgr<milvus::cache::DataObjPtr> {
public:
InvalidCacheMgr() {
}
};
class LessItemCacheMgr : public ms::cache::CacheMgr<ms::cache::DataObjPtr> {
class LessItemCacheMgr : public milvus::cache::CacheMgr<milvus::cache::DataObjPtr> {
public:
LessItemCacheMgr() {
cache_ = std::make_shared<ms::cache::Cache<ms::cache::DataObjPtr>>(1UL << 12, 10);
cache_ = std::make_shared<milvus::cache::Cache<milvus::cache::DataObjPtr>>(1UL << 12, 10);
}
};
class MockVecIndex : public ms::engine::VecIndex {
class MockVecIndex : public milvus::engine::VecIndex {
public:
MockVecIndex(int64_t dim, int64_t total)
: dimension_(dim),
ntotal_(total) {
}
virtual ms::Status BuildAll(const int64_t &nb,
virtual milvus::Status BuildAll(const int64_t &nb,
const float *xb,
const int64_t *ids,
const ms::engine::Config &cfg,
const milvus::engine::Config &cfg,
const int64_t &nt = 0,
const float *xt = nullptr) {
return ms::Status();
return milvus::Status();
}
ms::engine::VecIndexPtr Clone() override {
milvus::engine::VecIndexPtr Clone() override {
return milvus::engine::VecIndexPtr();
}
@ -62,31 +60,31 @@ class MockVecIndex : public ms::engine::VecIndex {
return 0;
}
ms::engine::IndexType GetType() override {
return ms::engine::IndexType::INVALID;
milvus::engine::IndexType GetType() override {
return milvus::engine::IndexType::INVALID;
}
virtual ms::Status Add(const int64_t &nb,
virtual milvus::Status Add(const int64_t &nb,
const float *xb,
const int64_t *ids,
const ms::engine::Config &cfg = ms::engine::Config()) {
return ms::Status();
const milvus::engine::Config &cfg = milvus::engine::Config()) {
return milvus::Status();
}
virtual ms::Status Search(const int64_t &nq,
virtual milvus::Status Search(const int64_t &nq,
const float *xq,
float *dist,
int64_t *ids,
const ms::engine::Config &cfg = ms::engine::Config()) {
return ms::Status();
const milvus::engine::Config &cfg = milvus::engine::Config()) {
return milvus::Status();
}
ms::engine::VecIndexPtr CopyToGpu(const int64_t &device_id,
const ms::engine::Config &cfg) override {
milvus::engine::VecIndexPtr CopyToGpu(const int64_t &device_id,
const milvus::engine::Config &cfg) override {
return nullptr;
}
ms::engine::VecIndexPtr CopyToCpu(const ms::engine::Config &cfg) override {
milvus::engine::VecIndexPtr CopyToCpu(const milvus::engine::Config &cfg) override {
return nullptr;
}
@ -103,8 +101,8 @@ class MockVecIndex : public ms::engine::VecIndex {
return binset;
}
virtual ms::Status Load(const knowhere::BinarySet &index_binary) {
return ms::Status();
virtual milvus::Status Load(const knowhere::BinarySet &index_binary) {
return milvus::Status();
}
public:
@ -115,7 +113,7 @@ class MockVecIndex : public ms::engine::VecIndex {
} // namespace
TEST(CacheTest, DUMMY_TEST) {
ms::engine::Config cfg;
milvus::engine::Config cfg;
MockVecIndex mock_index(256, 1000);
mock_index.Dimension();
mock_index.Count();
@ -133,7 +131,7 @@ TEST(CacheTest, DUMMY_TEST) {
}
TEST(CacheTest, CPU_CACHE_TEST) {
auto cpu_mgr = ms::cache::CpuCacheMgr::GetInstance();
auto cpu_mgr = milvus::cache::CpuCacheMgr::GetInstance();
const int64_t gbyte = 1024 * 1024 * 1024;
int64_t g_num = 16;
@ -144,8 +142,8 @@ TEST(CacheTest, CPU_CACHE_TEST) {
uint64_t item_count = 20;
for (uint64_t i = 0; i < item_count; i++) {
//each vector is 1k byte, total size less than 1G
ms::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 1000000);
ms::cache::DataObjPtr data_obj = std::static_pointer_cast<ms::cache::DataObj>(mock_index);
milvus::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 1000000);
milvus::cache::DataObjPtr data_obj = std::static_pointer_cast<milvus::cache::DataObj>(mock_index);
cpu_mgr->InsertItem("index_" + std::to_string(i), data_obj);
}
ASSERT_LT(cpu_mgr->ItemCount(), g_num);
@ -168,8 +166,8 @@ TEST(CacheTest, CPU_CACHE_TEST) {
cpu_mgr->SetCapacity(g_num * gbyte);
//each vector is 1k byte, total size less than 6G
ms::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 6000000);
ms::cache::DataObjPtr data_obj = std::static_pointer_cast<ms::cache::DataObj>(mock_index);
milvus::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 6000000);
milvus::cache::DataObjPtr data_obj = std::static_pointer_cast<milvus::cache::DataObj>(mock_index);
cpu_mgr->InsertItem("index_6g", data_obj);
ASSERT_TRUE(cpu_mgr->ItemExists("index_6g"));
}
@ -178,12 +176,12 @@ TEST(CacheTest, CPU_CACHE_TEST) {
}
TEST(CacheTest, GPU_CACHE_TEST) {
auto gpu_mgr = ms::cache::GpuCacheMgr::GetInstance(0);
auto gpu_mgr = milvus::cache::GpuCacheMgr::GetInstance(0);
for (int i = 0; i < 20; i++) {
//each vector is 1k byte
ms::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 1000);
ms::cache::DataObjPtr data_obj = std::static_pointer_cast<ms::cache::DataObj>(mock_index);
milvus::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 1000);
milvus::cache::DataObjPtr data_obj = std::static_pointer_cast<milvus::cache::DataObj>(mock_index);
gpu_mgr->InsertItem("index_" + std::to_string(i), data_obj);
}
@ -195,8 +193,8 @@ TEST(CacheTest, GPU_CACHE_TEST) {
for (auto i = 0; i < 3; i++) {
// TODO(myh): use gpu index to mock
//each vector is 1k byte, total size less than 2G
ms::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 2000000);
ms::cache::DataObjPtr data_obj = std::static_pointer_cast<ms::cache::DataObj>(mock_index);
milvus::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 2000000);
milvus::cache::DataObjPtr data_obj = std::static_pointer_cast<milvus::cache::DataObj>(mock_index);
std::cout << data_obj->Size() << std::endl;
gpu_mgr->InsertItem("index_" + std::to_string(i), data_obj);
}
@ -212,7 +210,7 @@ TEST(CacheTest, INVALID_TEST) {
ASSERT_FALSE(mgr.ItemExists("test"));
ASSERT_EQ(mgr.GetItem("test"), nullptr);
mgr.InsertItem("test", ms::cache::DataObjPtr());
mgr.InsertItem("test", milvus::cache::DataObjPtr());
mgr.InsertItem("test", nullptr);
mgr.EraseItem("test");
mgr.PrintInfo();
@ -226,8 +224,8 @@ TEST(CacheTest, INVALID_TEST) {
LessItemCacheMgr mgr;
for (int i = 0; i < 20; i++) {
//each vector is 1k byte
ms::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 2);
ms::cache::DataObjPtr data_obj = std::static_pointer_cast<ms::cache::DataObj>(mock_index);
milvus::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 2);
milvus::cache::DataObjPtr data_obj = std::static_pointer_cast<milvus::cache::DataObj>(mock_index);
mgr.InsertItem("index_" + std::to_string(i), data_obj);
}
ASSERT_EQ(mgr.GetItem("index_0"), nullptr);

34
cpp/unittest/server/test_config.cpp
View File

@ -25,8 +25,6 @@
namespace {
namespace ms = milvus;
static const char *CONFIG_FILE_PATH = "./milvus/conf/server_config.yaml";
static const char *LOG_FILE_PATH = "./milvus/conf/log_config.conf";
@ -37,25 +35,25 @@ static constexpr uint64_t GB = MB * 1024;
} // namespace
TEST(ConfigTest, CONFIG_TEST) {
ms::server::ConfigMgr *config_mgr = ms::server::ConfigMgr::GetInstance();
milvus::server::ConfigMgr *config_mgr = milvus::server::ConfigMgr::GetInstance();
ms::ErrorCode err = config_mgr->LoadConfigFile("");
ASSERT_EQ(err, ms::SERVER_UNEXPECTED_ERROR);
milvus::ErrorCode err = config_mgr->LoadConfigFile("");
ASSERT_EQ(err, milvus::SERVER_UNEXPECTED_ERROR);
err = config_mgr->LoadConfigFile(LOG_FILE_PATH);
ASSERT_EQ(err, ms::SERVER_UNEXPECTED_ERROR);
ASSERT_EQ(err, milvus::SERVER_UNEXPECTED_ERROR);
err = config_mgr->LoadConfigFile(CONFIG_FILE_PATH);
ASSERT_EQ(err, ms::SERVER_SUCCESS);
ASSERT_EQ(err, milvus::SERVER_SUCCESS);
config_mgr->Print();
ms::server::ConfigNode &root_config = config_mgr->GetRootNode();
ms::server::ConfigNode &server_config = root_config.GetChild("server_config");
ms::server::ConfigNode &db_config = root_config.GetChild("db_config");
ms::server::ConfigNode &metric_config = root_config.GetChild("metric_config");
ms::server::ConfigNode &cache_config = root_config.GetChild("cache_config");
ms::server::ConfigNode invalid_config = root_config.GetChild("invalid_config");
milvus::server::ConfigNode &root_config = config_mgr->GetRootNode();
milvus::server::ConfigNode &server_config = root_config.GetChild("server_config");
milvus::server::ConfigNode &db_config = root_config.GetChild("db_config");
milvus::server::ConfigNode &metric_config = root_config.GetChild("metric_config");
milvus::server::ConfigNode &cache_config = root_config.GetChild("cache_config");
milvus::server::ConfigNode invalid_config = root_config.GetChild("invalid_config");
auto valus = invalid_config.GetSequence("not_exist");
float ff = invalid_config.GetFloatValue("not_exist", 3.0);
ASSERT_EQ(ff, 3.0);
@ -69,10 +67,10 @@ TEST(ConfigTest, CONFIG_TEST) {
double test = server_config.GetDoubleValue("test");
ASSERT_EQ(test, 2.5);
ms::server::ConfigNode fake;
milvus::server::ConfigNode fake;
server_config.AddChild("fake", fake);
fake = server_config.GetChild("fake");
ms::server::ConfigNodeArr arr;
milvus::server::ConfigNodeArr arr;
server_config.GetChildren(arr);
ASSERT_EQ(arr.size(), 1UL);
@ -89,7 +87,7 @@ TEST(ConfigTest, CONFIG_TEST) {
auto seq = server_config.GetSequence("seq");
ASSERT_EQ(seq.size(), 2UL);
ms::server::ConfigNode combine;
milvus::server::ConfigNode combine;
combine.Combine(server_config);
combine.PrintAll();
@ -102,8 +100,8 @@ TEST(ConfigTest, CONFIG_TEST) {
}
TEST(ConfigTest, SERVER_CONFIG_TEST) {
ms::server::Config &config = ms::server::Config::GetInstance();
ms::Status s = config.LoadConfigFile(CONFIG_FILE_PATH);
milvus::server::Config &config = milvus::server::Config::GetInstance();
milvus::Status s = config.LoadConfigFile(CONFIG_FILE_PATH);
ASSERT_TRUE(s.ok());
s = config.ValidateConfig();

72
cpp/unittest/server/test_rpc.cpp
View File

@ -36,8 +36,6 @@
namespace {
namespace ms = milvus;
static const char *TABLE_NAME = "test_grpc";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t INDEX_FILE_SIZE = 1024;
@ -49,29 +47,29 @@ class RpcHandlerTest : public testing::Test {
protected:
void
SetUp() override {
auto res_mgr = ms::scheduler::ResMgrInst::GetInstance();
auto res_mgr = milvus::scheduler::ResMgrInst::GetInstance();
res_mgr->Clear();
res_mgr->Add(ms::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false));
res_mgr->Add(ms::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true));
res_mgr->Add(ms::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true));
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false));
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true));
res_mgr->Add(milvus::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true));
auto default_conn = ms::scheduler::Connection("IO", 500.0);
auto PCIE = ms::scheduler::Connection("IO", 11000.0);
auto default_conn = milvus::scheduler::Connection("IO", 500.0);
auto PCIE = milvus::scheduler::Connection("IO", 11000.0);
res_mgr->Connect("disk", "cpu", default_conn);
res_mgr->Connect("cpu", "gtx1660", PCIE);
res_mgr->Start();
ms::scheduler::SchedInst::GetInstance()->Start();
ms::scheduler::JobMgrInst::GetInstance()->Start();
milvus::scheduler::SchedInst::GetInstance()->Start();
milvus::scheduler::JobMgrInst::GetInstance()->Start();
ms::engine::DBOptions opt;
milvus::engine::DBOptions opt;
ms::server::Config::GetInstance().SetDBConfigBackendUrl("sqlite://:@:/");
ms::server::Config::GetInstance().SetDBConfigPrimaryPath("/tmp/milvus_test");
ms::server::Config::GetInstance().SetDBConfigSecondaryPath("");
ms::server::Config::GetInstance().SetDBConfigArchiveDiskThreshold("");
ms::server::Config::GetInstance().SetDBConfigArchiveDaysThreshold("");
ms::server::Config::GetInstance().SetCacheConfigCacheInsertData("");
ms::server::Config::GetInstance().SetEngineConfigOmpThreadNum("");
milvus::server::Config::GetInstance().SetDBConfigBackendUrl("sqlite://:@:/");
milvus::server::Config::GetInstance().SetDBConfigPrimaryPath("/tmp/milvus_test");
milvus::server::Config::GetInstance().SetDBConfigSecondaryPath("");
milvus::server::Config::GetInstance().SetDBConfigArchiveDiskThreshold("");
milvus::server::Config::GetInstance().SetDBConfigArchiveDaysThreshold("");
milvus::server::Config::GetInstance().SetCacheConfigCacheInsertData("");
milvus::server::Config::GetInstance().SetEngineConfigOmpThreadNum("");
// serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "cluster");
// DBWrapper::GetInstance().GetInstance().StartService();
@ -81,11 +79,11 @@ class RpcHandlerTest : public testing::Test {
// DBWrapper::GetInstance().GetInstance().StartService();
// DBWrapper::GetInstance().GetInstance().StopService();
ms::server::Config::GetInstance().SetResourceConfigMode("single");
ms::server::DBWrapper::GetInstance().StartService();
milvus::server::Config::GetInstance().SetResourceConfigMode("single");
milvus::server::DBWrapper::GetInstance().StartService();
//initialize handler, create table
handler = std::make_shared<ms::server::grpc::GrpcRequestHandler>();
handler = std::make_shared<milvus::server::grpc::GrpcRequestHandler>();
::grpc::ServerContext context;
::milvus::grpc::TableSchema request;
::milvus::grpc::Status status;
@ -98,15 +96,15 @@ class RpcHandlerTest : public testing::Test {
void
TearDown() override {
ms::server::DBWrapper::GetInstance().StopService();
ms::scheduler::JobMgrInst::GetInstance()->Stop();
ms::scheduler::ResMgrInst::GetInstance()->Stop();
ms::scheduler::SchedInst::GetInstance()->Stop();
milvus::server::DBWrapper::GetInstance().StopService();
milvus::scheduler::JobMgrInst::GetInstance()->Stop();
milvus::scheduler::ResMgrInst::GetInstance()->Stop();
milvus::scheduler::SchedInst::GetInstance()->Stop();
boost::filesystem::remove_all("/tmp/milvus_test");
}
protected:
std::shared_ptr<ms::server::grpc::GrpcRequestHandler> handler;
std::shared_ptr<milvus::server::grpc::GrpcRequestHandler> handler;
};
void
@ -425,16 +423,16 @@ TEST_F(RpcHandlerTest, DELETE_BY_RANGE_TEST) {
//////////////////////////////////////////////////////////////////////
namespace {
class DummyTask : public ms::server::grpc::GrpcBaseTask {
class DummyTask : public milvus::server::grpc::GrpcBaseTask {
public:
ms::Status
milvus::Status
OnExecute() override {
return ms::Status::OK();
return milvus::Status::OK();
}
static ms::server::grpc::BaseTaskPtr
static milvus::server::grpc::BaseTaskPtr
Create(std::string &dummy) {
return std::shared_ptr<ms::server::grpc::GrpcBaseTask>(new DummyTask(dummy));
return std::shared_ptr<milvus::server::grpc::GrpcBaseTask>(new DummyTask(dummy));
}
public:
@ -459,16 +457,16 @@ TEST_F(RpcSchedulerTest, BASE_TASK_TEST) {
auto status = task_ptr->Execute();
ASSERT_TRUE(status.ok());
ms::server::grpc::GrpcRequestScheduler::GetInstance().Start();
milvus::server::grpc::GrpcRequestScheduler::GetInstance().Start();
::milvus::grpc::Status grpc_status;
std::string dummy = "dql";
ms::server::grpc::BaseTaskPtr base_task_ptr = DummyTask::Create(dummy);
ms::server::grpc::GrpcRequestScheduler::GetInstance().ExecTask(base_task_ptr, &grpc_status);
milvus::server::grpc::BaseTaskPtr base_task_ptr = DummyTask::Create(dummy);
milvus::server::grpc::GrpcRequestScheduler::GetInstance().ExecTask(base_task_ptr, &grpc_status);
ms::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr);
milvus::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr);
task_ptr = nullptr;
ms::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr);
milvus::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr);
ms::server::grpc::GrpcRequestScheduler::GetInstance().Stop();
milvus::server::grpc::GrpcRequestScheduler::GetInstance().Stop();
}

208
cpp/unittest/server/util_test.cpp
View File

@ -33,12 +33,10 @@
namespace {
namespace ms = milvus;
static const char *LOG_FILE_PATH = "./milvus/conf/log_config.conf";
static const char* LOG_FILE_PATH = "./milvus/conf/log_config.conf";
void
CopyStatus(ms::Status &st1, ms::Status &st2) {
CopyStatus(milvus::Status& st1, milvus::Status& st2) {
st1 = st2;
}
@ -46,61 +44,61 @@ CopyStatus(ms::Status &st1, ms::Status &st2) {
TEST(UtilTest, EXCEPTION_TEST) {
std::string err_msg = "failed";
ms::server::ServerException ex(ms::SERVER_UNEXPECTED_ERROR, err_msg);
ASSERT_EQ(ex.error_code(), ms::SERVER_UNEXPECTED_ERROR);
milvus::server::ServerException ex(milvus::SERVER_UNEXPECTED_ERROR, err_msg);
ASSERT_EQ(ex.error_code(), milvus::SERVER_UNEXPECTED_ERROR);
std::string msg = ex.what();
ASSERT_EQ(msg, err_msg);
}
TEST(UtilTest, SIGNAL_TEST) {
ms::server::SignalUtil::PrintStacktrace();
milvus::server::SignalUtil::PrintStacktrace();
}
TEST(UtilTest, COMMON_TEST) {
uint64_t total_mem = 0, free_mem = 0;
ms::server::CommonUtil::GetSystemMemInfo(total_mem, free_mem);
milvus::server::CommonUtil::GetSystemMemInfo(total_mem, free_mem);
ASSERT_GT(total_mem, 0);
ASSERT_GT(free_mem, 0);
uint32_t thread_cnt = 0;
ms::server::CommonUtil::GetSystemAvailableThreads(thread_cnt);
milvus::server::CommonUtil::GetSystemAvailableThreads(thread_cnt);
ASSERT_GT(thread_cnt, 0);
std::string path1 = "/tmp/milvus_test/";
std::string path2 = path1 + "common_test_12345/";
std::string path3 = path2 + "abcdef";
ms::Status status = ms::server::CommonUtil::CreateDirectory(path3);
milvus::Status status = milvus::server::CommonUtil::CreateDirectory(path3);
ASSERT_TRUE(status.ok());
//test again
status = ms::server::CommonUtil::CreateDirectory(path3);
status = milvus::server::CommonUtil::CreateDirectory(path3);
ASSERT_TRUE(status.ok());
ASSERT_TRUE(ms::server::CommonUtil::IsDirectoryExist(path3));
ASSERT_TRUE(milvus::server::CommonUtil::IsDirectoryExist(path3));
status = ms::server::CommonUtil::DeleteDirectory(path1);
status = milvus::server::CommonUtil::DeleteDirectory(path1);
ASSERT_TRUE(status.ok());
//test again
status = ms::server::CommonUtil::DeleteDirectory(path1);
status = milvus::server::CommonUtil::DeleteDirectory(path1);
ASSERT_TRUE(status.ok());
ASSERT_FALSE(ms::server::CommonUtil::IsDirectoryExist(path1));
ASSERT_FALSE(ms::server::CommonUtil::IsFileExist(path1));
ASSERT_FALSE(milvus::server::CommonUtil::IsDirectoryExist(path1));
ASSERT_FALSE(milvus::server::CommonUtil::IsFileExist(path1));
std::string exe_path = ms::server::CommonUtil::GetExePath();
std::string exe_path = milvus::server::CommonUtil::GetExePath();
ASSERT_FALSE(exe_path.empty());
time_t tt;
time(&tt);
tm time_struct;
memset(&time_struct, 0, sizeof(tm));
ms::server::CommonUtil::ConvertTime(tt, time_struct);
milvus::server::CommonUtil::ConvertTime(tt, time_struct);
ASSERT_GT(time_struct.tm_year, 0);
ASSERT_GT(time_struct.tm_mon, 0);
ASSERT_GT(time_struct.tm_mday, 0);
ms::server::CommonUtil::ConvertTime(time_struct, tt);
milvus::server::CommonUtil::ConvertTime(time_struct, tt);
ASSERT_GT(tt, 0);
bool res = ms::server::CommonUtil::TimeStrToTime("2019-03-23", tt, time_struct);
bool res = milvus::server::CommonUtil::TimeStrToTime("2019-03-23", tt, time_struct);
ASSERT_EQ(time_struct.tm_year, 119);
ASSERT_EQ(time_struct.tm_mon, 2);
ASSERT_EQ(time_struct.tm_mday, 23);
@ -110,38 +108,38 @@ TEST(UtilTest, COMMON_TEST) {
TEST(UtilTest, STRINGFUNCTIONS_TEST) {
std::string str = " test zilliz";
ms::server::StringHelpFunctions::TrimStringBlank(str);
milvus::server::StringHelpFunctions::TrimStringBlank(str);
ASSERT_EQ(str, "test zilliz");
str = "\"test zilliz\"";
ms::server::StringHelpFunctions::TrimStringQuote(str, "\"");
milvus::server::StringHelpFunctions::TrimStringQuote(str, "\"");
ASSERT_EQ(str, "test zilliz");
str = "a,b,c";
std::vector<std::string> result;
auto status = ms::server::StringHelpFunctions::SplitStringByDelimeter(str, ",", result);
auto status = milvus::server::StringHelpFunctions::SplitStringByDelimeter(str, ",", result);
ASSERT_TRUE(status.ok());
ASSERT_EQ(result.size(), 3UL);
result.clear();
status = ms::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
ASSERT_TRUE(status.ok());
ASSERT_EQ(result.size(), 3UL);
result.clear();
status = ms::server::StringHelpFunctions::SplitStringByQuote(str, ",", "", result);
status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "", result);
ASSERT_TRUE(status.ok());
ASSERT_EQ(result.size(), 3UL);
str = "55,\"aa,gg,yy\",b";
result.clear();
status = ms::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
status = milvus::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
ASSERT_TRUE(status.ok());
ASSERT_EQ(result.size(), 3UL);
}
TEST(UtilTest, BLOCKINGQUEUE_TEST) {
ms::server::BlockingQueue<std::string> bq;
milvus::server::BlockingQueue<std::string> bq;
static const size_t count = 10;
bq.SetCapacity(count);
@ -170,13 +168,13 @@ TEST(UtilTest, BLOCKINGQUEUE_TEST) {
}
TEST(UtilTest, LOG_TEST) {
auto status = ms::server::InitLog(LOG_FILE_PATH);
auto status = milvus::server::InitLog(LOG_FILE_PATH);
ASSERT_TRUE(status.ok());
EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer));
EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::LogDetailedCrashReason));
std::string fname = ms::server::CommonUtil::GetFileName(LOG_FILE_PATH);
std::string fname = milvus::server::CommonUtil::GetFileName(LOG_FILE_PATH);
ASSERT_EQ(fname, "log_config.conf");
}
@ -185,39 +183,39 @@ TEST(UtilTest, TIMERECORDER_TEST) {
if (log_level == 5) {
continue; //skip fatal
}
ms::TimeRecorder rc("time", log_level);
milvus::TimeRecorder rc("time", log_level);
rc.RecordSection("end");
}
}
TEST(UtilTest, STATUS_TEST) {
auto status = ms::Status::OK();
auto status = milvus::Status::OK();
std::string str = status.ToString();
ASSERT_FALSE(str.empty());
status = ms::Status(ms::DB_ERROR, "mistake");
ASSERT_EQ(status.code(), ms::DB_ERROR);
status = milvus::Status(milvus::DB_ERROR, "mistake");
ASSERT_EQ(status.code(), milvus::DB_ERROR);
str = status.ToString();
ASSERT_FALSE(str.empty());
status = ms::Status(ms::DB_NOT_FOUND, "mistake");
ASSERT_EQ(status.code(), ms::DB_NOT_FOUND);
status = milvus::Status(milvus::DB_NOT_FOUND, "mistake");
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
str = status.ToString();
ASSERT_FALSE(str.empty());
status = ms::Status(ms::DB_ALREADY_EXIST, "mistake");
ASSERT_EQ(status.code(), ms::DB_ALREADY_EXIST);
status = milvus::Status(milvus::DB_ALREADY_EXIST, "mistake");
ASSERT_EQ(status.code(), milvus::DB_ALREADY_EXIST);
str = status.ToString();
ASSERT_FALSE(str.empty());
status = ms::Status(ms::DB_META_TRANSACTION_FAILED, "mistake");
ASSERT_EQ(status.code(), ms::DB_META_TRANSACTION_FAILED);
status = milvus::Status(milvus::DB_META_TRANSACTION_FAILED, "mistake");
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
str = status.ToString();
ASSERT_FALSE(str.empty());
auto status_copy = ms::Status::OK();
auto status_copy = milvus::Status::OK();
CopyStatus(status_copy, status);
ASSERT_EQ(status.code(), ms::DB_META_TRANSACTION_FAILED);
ASSERT_EQ(status.code(), milvus::DB_META_TRANSACTION_FAILED);
auto status_ref(status);
ASSERT_EQ(status_ref.code(), status.code());
@ -230,120 +228,126 @@ TEST(UtilTest, STATUS_TEST) {
TEST(ValidationUtilTest, VALIDATE_TABLENAME_TEST) {
std::string table_name = "Normal123_";
auto status = ms::server::ValidationUtil::ValidateTableName(table_name);
auto status = milvus::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_TRUE(status.ok());
table_name = "12sds";
status = ms::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME);
status = milvus::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME);
table_name = "";
status = ms::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME);
status = milvus::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME);
table_name = "_asdasd";
status = ms::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), ms::SERVER_SUCCESS);
status = milvus::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), milvus::SERVER_SUCCESS);
table_name = "!@#!@";
status = ms::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME);
status = milvus::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME);
table_name = "_!@#!@";
status = ms::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME);
status = milvus::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME);
table_name = "中文";
status = ms::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME);
status = milvus::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME);
table_name = std::string(10000, 'a');
status = ms::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME);
status = milvus::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), milvus::SERVER_INVALID_TABLE_NAME);
}
TEST(ValidationUtilTest, VALIDATE_DIMENSION_TEST) {
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(-1).code(), ms::SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(0).code(), ms::SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(16385).code(), ms::SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(16384).code(), ms::SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(1).code(), ms::SERVER_SUCCESS);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableDimension(-1).code(),
milvus::SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableDimension(0).code(),
milvus::SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableDimension(16385).code(),
milvus::SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableDimension(16384).code(), milvus::SERVER_SUCCESS);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableDimension(1).code(), milvus::SERVER_SUCCESS);
}
TEST(ValidationUtilTest, VALIDATE_INDEX_TEST) {
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexType((int) ms::engine::EngineType::INVALID).code(),
ms::SERVER_INVALID_INDEX_TYPE);
for (int i = 1; i <= (int) ms::engine::EngineType::MAX_VALUE; i++) {
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexType(i).code(), ms::SERVER_SUCCESS);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType((int)milvus::engine::EngineType::INVALID).code(),
milvus::SERVER_INVALID_INDEX_TYPE);
for (int i = 1; i <= (int)milvus::engine::EngineType::MAX_VALUE; i++) {
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType(i).code(), milvus::SERVER_SUCCESS);
}
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexType((int) ms::engine::EngineType::MAX_VALUE + 1).code(),
ms::SERVER_INVALID_INDEX_TYPE);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexType(
(int)milvus::engine::EngineType::MAX_VALUE + 1).code(),
milvus::SERVER_INVALID_INDEX_TYPE);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexNlist(0).code(), ms::SERVER_INVALID_INDEX_NLIST);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexNlist(100).code(), ms::SERVER_SUCCESS);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexNlist(0).code(), milvus::SERVER_INVALID_INDEX_NLIST);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexNlist(100).code(), milvus::SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexFileSize(0).code(), ms::SERVER_INVALID_INDEX_FILE_SIZE);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexFileSize(100).code(), ms::SERVER_SUCCESS);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexFileSize(0).code(),
milvus::SERVER_INVALID_INDEX_FILE_SIZE);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexFileSize(100).code(), milvus::SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexMetricType(0).code(), ms::SERVER_INVALID_INDEX_METRIC_TYPE);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexMetricType(1).code(), ms::SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexMetricType(2).code(), ms::SERVER_SUCCESS);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexMetricType(0).code(),
milvus::SERVER_INVALID_INDEX_METRIC_TYPE);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexMetricType(1).code(), milvus::SERVER_SUCCESS);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateTableIndexMetricType(2).code(), milvus::SERVER_SUCCESS);
}
TEST(ValidationUtilTest, VALIDATE_TOPK_TEST) {
ms::engine::meta::TableSchema schema;
ASSERT_EQ(ms::server::ValidationUtil::ValidateSearchTopk(10, schema).code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateSearchTopk(65536, schema).code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateSearchTopk(0, schema).code(), ms::SERVER_SUCCESS);
milvus::engine::meta::TableSchema schema;
ASSERT_EQ(milvus::server::ValidationUtil::ValidateSearchTopk(10, schema).code(), milvus::SERVER_SUCCESS);
ASSERT_NE(milvus::server::ValidationUtil::ValidateSearchTopk(65536, schema).code(), milvus::SERVER_SUCCESS);
ASSERT_NE(milvus::server::ValidationUtil::ValidateSearchTopk(0, schema).code(), milvus::SERVER_SUCCESS);
}
TEST(ValidationUtilTest, VALIDATE_NPROBE_TEST) {
ms::engine::meta::TableSchema schema;
milvus::engine::meta::TableSchema schema;
schema.nlist_ = 100;
ASSERT_EQ(ms::server::ValidationUtil::ValidateSearchNprobe(10, schema).code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateSearchNprobe(0, schema).code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateSearchNprobe(101, schema).code(), ms::SERVER_SUCCESS);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateSearchNprobe(10, schema).code(), milvus::SERVER_SUCCESS);
ASSERT_NE(milvus::server::ValidationUtil::ValidateSearchNprobe(0, schema).code(), milvus::SERVER_SUCCESS);
ASSERT_NE(milvus::server::ValidationUtil::ValidateSearchNprobe(101, schema).code(), milvus::SERVER_SUCCESS);
}
TEST(ValidationUtilTest, VALIDATE_GPU_TEST) {
ASSERT_EQ(ms::server::ValidationUtil::ValidateGpuIndex(0).code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateGpuIndex(100).code(), ms::SERVER_SUCCESS);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateGpuIndex(0).code(), milvus::SERVER_SUCCESS);
ASSERT_NE(milvus::server::ValidationUtil::ValidateGpuIndex(100).code(), milvus::SERVER_SUCCESS);
size_t memory = 0;
ASSERT_EQ(ms::server::ValidationUtil::GetGpuMemory(0, memory).code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::GetGpuMemory(100, memory).code(), ms::SERVER_SUCCESS);
ASSERT_EQ(milvus::server::ValidationUtil::GetGpuMemory(0, memory).code(), milvus::SERVER_SUCCESS);
ASSERT_NE(milvus::server::ValidationUtil::GetGpuMemory(100, memory).code(), milvus::SERVER_SUCCESS);
}
TEST(ValidationUtilTest, VALIDATE_IPADDRESS_TEST) {
ASSERT_EQ(ms::server::ValidationUtil::ValidateIpAddress("127.0.0.1").code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateIpAddress("not ip").code(), ms::SERVER_SUCCESS);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateIpAddress("127.0.0.1").code(), milvus::SERVER_SUCCESS);
ASSERT_NE(milvus::server::ValidationUtil::ValidateIpAddress("not ip").code(), milvus::SERVER_SUCCESS);
}
TEST(ValidationUtilTest, VALIDATE_NUMBER_TEST) {
ASSERT_EQ(ms::server::ValidationUtil::ValidateStringIsNumber("1234").code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateStringIsNumber("not number").code(), ms::SERVER_SUCCESS);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateStringIsNumber("1234").code(), milvus::SERVER_SUCCESS);
ASSERT_NE(milvus::server::ValidationUtil::ValidateStringIsNumber("not number").code(), milvus::SERVER_SUCCESS);
}
TEST(ValidationUtilTest, VALIDATE_BOOL_TEST) {
std::string str = "true";
ASSERT_EQ(ms::server::ValidationUtil::ValidateStringIsBool(str).code(), ms::SERVER_SUCCESS);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateStringIsBool(str).code(), milvus::SERVER_SUCCESS);
str = "not bool";
ASSERT_NE(ms::server::ValidationUtil::ValidateStringIsBool(str).code(), ms::SERVER_SUCCESS);
ASSERT_NE(milvus::server::ValidationUtil::ValidateStringIsBool(str).code(), milvus::SERVER_SUCCESS);
}
TEST(ValidationUtilTest, VALIDATE_DOUBLE_TEST) {
ASSERT_EQ(ms::server::ValidationUtil::ValidateStringIsFloat("2.5").code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateStringIsFloat("not double").code(), ms::SERVER_SUCCESS);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateStringIsFloat("2.5").code(), milvus::SERVER_SUCCESS);
ASSERT_NE(milvus::server::ValidationUtil::ValidateStringIsFloat("not double").code(), milvus::SERVER_SUCCESS);
}
TEST(ValidationUtilTest, VALIDATE_DBURI_TEST) {
ASSERT_EQ(ms::server::ValidationUtil::ValidateDbURI("sqlite://:@:/").code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateDbURI("xxx://:@:/").code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateDbURI("not uri").code(), ms::SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:3303/milvus").code(),
ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:port/milvus").code(),
ms::SERVER_SUCCESS);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateDbURI("sqlite://:@:/").code(), milvus::SERVER_SUCCESS);
ASSERT_NE(milvus::server::ValidationUtil::ValidateDbURI("xxx://:@:/").code(), milvus::SERVER_SUCCESS);
ASSERT_NE(milvus::server::ValidationUtil::ValidateDbURI("not uri").code(), milvus::SERVER_SUCCESS);
ASSERT_EQ(milvus::server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:3303/milvus").code(),
milvus::SERVER_SUCCESS);
ASSERT_NE(milvus::server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:port/milvus").code(),
milvus::SERVER_SUCCESS);
}
TEST(UtilTest, ROLLOUTHANDLER_TEST) {
@ -374,7 +378,7 @@ TEST(UtilTest, ROLLOUTHANDLER_TEST) {
file.open(tmp.c_str());
file << "zilliz" << std::endl;
ms::server::RolloutHandler(tmp.c_str(), 0, list[i]);
milvus::server::RolloutHandler(tmp.c_str(), 0, list[i]);
tmp.append(".1");
std::ifstream file2;

68
cpp/unittest/wrapper/test_wrapper.cpp
View File

@ -42,30 +42,30 @@ constexpr int64_t DEVICE_ID = 0;
class ParamGenerator {
public:
static ParamGenerator &GetInstance() {
static ParamGenerator& GetInstance() {
static ParamGenerator instance;
return instance;
}
kw::Config Gen(const ms::IndexType &type) {
knowhere::Config Gen(const milvus::engine::IndexType& type) {
switch (type) {
case ms::IndexType::FAISS_IDMAP: {
case milvus::engine::IndexType::FAISS_IDMAP: {
auto tempconf = std::make_shared<knowhere::Cfg>();
tempconf->metric_type = knowhere::METRICTYPE::L2;
return tempconf;
}
case ms::IndexType::FAISS_IVFFLAT_CPU:
case ms::IndexType::FAISS_IVFFLAT_GPU:
case ms::IndexType::FAISS_IVFFLAT_MIX: {
case milvus::engine::IndexType::FAISS_IVFFLAT_CPU:
case milvus::engine::IndexType::FAISS_IVFFLAT_GPU:
case milvus::engine::IndexType::FAISS_IVFFLAT_MIX: {
auto tempconf = std::make_shared<knowhere::IVFCfg>();
tempconf->nlist = 100;
tempconf->nprobe = 16;
tempconf->metric_type = knowhere::METRICTYPE::L2;
return tempconf;
}
case ms::IndexType::FAISS_IVFSQ8_CPU:
case ms::IndexType::FAISS_IVFSQ8_GPU:
case ms::IndexType::FAISS_IVFSQ8_MIX: {
case milvus::engine::IndexType::FAISS_IVFSQ8_CPU:
case milvus::engine::IndexType::FAISS_IVFSQ8_GPU:
case milvus::engine::IndexType::FAISS_IVFSQ8_MIX: {
auto tempconf = std::make_shared<knowhere::IVFSQCfg>();
tempconf->nlist = 100;
tempconf->nprobe = 16;
@ -73,8 +73,8 @@ class ParamGenerator {
tempconf->metric_type = knowhere::METRICTYPE::L2;
return tempconf;
}
case ms::IndexType::FAISS_IVFPQ_CPU:
case ms::IndexType::FAISS_IVFPQ_GPU: {
case milvus::engine::IndexType::FAISS_IVFPQ_CPU:
case milvus::engine::IndexType::FAISS_IVFPQ_GPU: {
auto tempconf = std::make_shared<knowhere::IVFPQCfg>();
tempconf->nlist = 100;
tempconf->nprobe = 16;
@ -83,7 +83,7 @@ class ParamGenerator {
tempconf->metric_type = knowhere::METRICTYPE::L2;
return tempconf;
}
case ms::IndexType::NSG_MIX: {
case milvus::engine::IndexType::NSG_MIX: {
auto tempconf = std::make_shared<knowhere::NSGCfg>();
tempconf->nlist = 100;
tempconf->nprobe = 16;
@ -100,13 +100,13 @@ class ParamGenerator {
};
class KnowhereWrapperTest
: public TestWithParam<::std::tuple<ms::IndexType, std::string, int, int, int, int>> {
: public TestWithParam<::std::tuple<milvus::engine::IndexType, std::string, int, int, int, int>> {
protected:
void SetUp() override {
knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID,
1024 * 1024 * 200,
1024 * 1024 * 300,
2);
1024 * 1024 * 200,
1024 * 1024 * 300,
2);
std::string generator_type;
std::tie(index_type, generator_type, dim, nb, nq, k) = GetParam();
@ -126,7 +126,7 @@ class KnowhereWrapperTest
knowhere::FaissGpuResourceMgr::GetInstance().Free();
}
void AssertResult(const std::vector<int64_t> &ids, const std::vector<float> &dis) {
void AssertResult(const std::vector<int64_t>& ids, const std::vector<float>& dis) {
EXPECT_EQ(ids.size(), nq * k);
EXPECT_EQ(dis.size(), nq * k);
@ -153,8 +153,8 @@ class KnowhereWrapperTest
}
protected:
ms::IndexType index_type;
kw::Config conf;
milvus::engine::IndexType index_type;
knowhere::Config conf;
int dim = DIM;
int nb = NB;
@ -164,7 +164,7 @@ class KnowhereWrapperTest
std::vector<float> xq;
std::vector<int64_t> ids;
ms::VecIndexPtr index_ = nullptr;
milvus::engine::VecIndexPtr index_ = nullptr;
// Ground Truth
std::vector<int64_t> gt_ids;
@ -174,15 +174,25 @@ class KnowhereWrapperTest
INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest,
Values(
//["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"]
std::make_tuple(ms::IndexType::FAISS_IVFFLAT_CPU, "Default", 64, 100000, 10, 10),
std::make_tuple(ms::IndexType::FAISS_IVFFLAT_GPU, "Default", DIM, NB, 10, 10),
std::make_tuple(ms::IndexType::FAISS_IVFFLAT_MIX, "Default", 64, 100000, 10, 10),
std::make_tuple(ms::IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10),
std::make_tuple(ms::IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10),
std::make_tuple(ms::IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_CPU,
"Default",
64,
100000,
10,
10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_GPU, "Default", DIM, NB, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_MIX,
"Default",
64,
100000,
10,
10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10),
std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10),
// std::make_tuple(IndexType::NSG_MIX, "Default", 128, 250000, 10, 10),
// std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 250000, 10, 10),
std::make_tuple(ms::IndexType::FAISS_IDMAP, "Default", 64, 100000, 10, 10)
std::make_tuple(milvus::engine::IndexType::FAISS_IDMAP, "Default", 64, 100000, 10, 10)
)
);
@ -220,7 +230,7 @@ TEST_P(KnowhereWrapperTest, TO_GPU_TEST) {
{
std::string file_location = "/tmp/knowhere_gpu_file";
write_index(index_, file_location);
auto new_index = ms::read_index(file_location);
auto new_index = milvus::engine::read_index(file_location);
auto dev_idx = new_index->CopyToGpu(DEVICE_ID);
for (int i = 0; i < 10; ++i) {
@ -261,7 +271,7 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) {
{
std::string file_location = "/tmp/knowhere";
write_index(index_, file_location);
auto new_index = ms::read_index(file_location);
auto new_index = milvus::engine::read_index(file_location);
EXPECT_EQ(new_index->GetType(), ConvertToCpuIndexType(index_type));
EXPECT_EQ(new_index->Dimension(), index_->Dimension());
EXPECT_EQ(new_index->Count(), index_->Count());