milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'branch-0.5.0' into 'branch-0.5.0' 

format unittest code

See merge request megasearch/milvus!634

Former-commit-id: 38c8befef43f04c9c60534769d6333f57aadb8ce
pull/191/head
peng.xu 2019-09-27 19:25:01 +08:00
parent 37e26cce72 5220a39362
commit ae9e7802b9
40 changed files with 1397 additions and 1315 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cpp/CMakeLists.txt
View File

@ -189,7 +189,7 @@ add_custom_target(lint
--exclude_globs
${LINT_EXCLUSIONS_FILE}
--source_dir
${CMAKE_CURRENT_SOURCE_DIR}/src
${CMAKE_CURRENT_SOURCE_DIR}
${MILVUS_LINT_QUIET})
#

8
cpp/build-support/lint_exclusions.txt
View File

@ -1,8 +1,10 @@
*cmake-build-debug*
*cmake-build-release*
*cmake_build*
*src/thirdparty*
*src/core/thirdparty*
*src/grpc*
*thirdparty*
*easylogging++*
*SqliteMetaImpl.cpp
*SqliteMetaImpl.cpp
*src/grpc*
*src/core*
*src/wrapper*

31
cpp/build.sh
View File

@ -99,21 +99,26 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then
# cpplint check
make lint
if [ $? -ne 0 ]; then
echo "ERROR! cpplint check not pass"
exit 1
fi
# clang-format check
make check-clang-format
if [ $? -ne 0 ]; then
echo "ERROR! clang-format check failed"
exit 1
fi
# clang-tidy check
make check-clang-tidy
if [ $? -ne 0 ]; then
echo "ERROR! clang-tidy check failed"
echo "ERROR! cpplint check failed"
exit 1
fi
echo "cpplint check passed!"
# # clang-format check
# make check-clang-format
# if [ $? -ne 0 ]; then
# echo "ERROR! clang-format check failed"
# exit 1
# fi
# echo "clang-format check passed!"
#
# # clang-tidy check
# make check-clang-tidy
# if [ $? -ne 0 ]; then
# echo "ERROR! clang-tidy check failed"
# exit 1
# fi
# echo "clang-tidy check passed!"
else
# compile and build
make -j 4 || exit 1

5
cpp/src/db/meta/MetaFactory.cpp
View File

@ -40,9 +40,8 @@ MetaFactory::BuildOption(const std::string &path) {
if (p == "") {
srand(time(nullptr));
std::stringstream ss;
uint32_t rd = 0;
rand_r(&rd);
ss << "/tmp/" << rd;
uint32_t seed = 1;
ss << "/tmp/" << rand_r(&seed);
p = ss.str();
}

2
cpp/unittest/CMakeLists.txt
View File

@ -118,6 +118,8 @@ set(unittest_libs
cublas
)
include_directories(${CMAKE_CURRENT_SOURCE_DIR})
message(STATUS "CMAKE_CURRENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}")
foreach(dir ${CORE_INCLUDE_DIRS})
include_directories(${dir})
endforeach()

331
cpp/unittest/db/db_tests.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "utils.h"
#include "db/utils.h"
#include "db/DB.h"
#include "db/DBImpl.h"
#include "db/meta/MetaConsts.h"
@ -28,139 +28,142 @@
#include <thread>
#include <random>
using namespace zilliz::milvus;
namespace {
static const char* TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 25000;
static constexpr int64_t INSERT_LOOP = 1000;
static constexpr int64_t SECONDS_EACH_HOUR = 3600;
static constexpr int64_t DAY_SECONDS = 24 * 60 * 60;
namespace ms = zilliz::milvus;
engine::meta::TableSchema BuildTableSchema() {
engine::meta::TableSchema table_info;
table_info.dimension_ = TABLE_DIM;
table_info.table_id_ = TABLE_NAME;
return table_info;
}
void BuildVectors(int64_t n, std::vector<float>& vectors) {
vectors.clear();
vectors.resize(n*TABLE_DIM);
float* data = vectors.data();
for(int i = 0; i < n; i++) {
for(int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48();
data[TABLE_DIM * i] += i / 2000.;
}
}
std::string CurrentTmDate(int64_t offset_day = 0) {
time_t tt;
time( &tt );
tt = tt + 8*SECONDS_EACH_HOUR;
tt = tt + 24*SECONDS_EACH_HOUR*offset_day;
tm* t= gmtime( &tt );
std::string str = std::to_string(t->tm_year + 1900) + "-" + std::to_string(t->tm_mon + 1)
+ "-" + std::to_string(t->tm_mday);
return str;
}
void
ConvertTimeRangeToDBDates(const std::string &start_value,
const std::string &end_value,
std::vector<engine::meta::DateT > &dates) {
dates.clear();
time_t tt_start, tt_end;
tm tm_start, tm_end;
if (!zilliz::milvus::server::CommonUtil::TimeStrToTime(start_value, tt_start, tm_start)) {
return;
}
if (!zilliz::milvus::server::CommonUtil::TimeStrToTime(end_value, tt_end, tm_end)) {
return;
}
long days = (tt_end > tt_start) ? (tt_end - tt_start) / DAY_SECONDS : (tt_start - tt_end) /
DAY_SECONDS;
if (days == 0) {
return;
}
for (long i = 0; i < days; i++) {
time_t tt_day = tt_start + DAY_SECONDS * i;
tm tm_day;
zilliz::milvus::server::CommonUtil::ConvertTime(tt_day, tm_day);
long date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 +
tm_day.tm_mday;//according to db logic
dates.push_back(date);
}
}
static const char *TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 25000;
static constexpr int64_t INSERT_LOOP = 1000;
static constexpr int64_t SECONDS_EACH_HOUR = 3600;
static constexpr int64_t DAY_SECONDS = 24 * 60 * 60;
ms::engine::meta::TableSchema
BuildTableSchema() {
ms::engine::meta::TableSchema table_info;
table_info.dimension_ = TABLE_DIM;
table_info.table_id_ = TABLE_NAME;
return table_info;
}
void
BuildVectors(int64_t n, std::vector<float> &vectors) {
vectors.clear();
vectors.resize(n * TABLE_DIM);
float *data = vectors.data();
for (int i = 0; i < n; i++) {
for (int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48();
data[TABLE_DIM * i] += i / 2000.;
}
}
std::string
CurrentTmDate(int64_t offset_day = 0) {
time_t tt;
time(&tt);
tt = tt + 8 * SECONDS_EACH_HOUR;
tt = tt + 24 * SECONDS_EACH_HOUR * offset_day;
tm t;
gmtime_r(&tt, &t);
std::string str = std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1)
+ "-" + std::to_string(t.tm_mday);
return str;
}
void
ConvertTimeRangeToDBDates(const std::string &start_value,
const std::string &end_value,
std::vector<ms::engine::meta::DateT> &dates) {
dates.clear();
time_t tt_start, tt_end;
tm tm_start, tm_end;
if (!zilliz::milvus::server::CommonUtil::TimeStrToTime(start_value, tt_start, tm_start)) {
return;
}
if (!zilliz::milvus::server::CommonUtil::TimeStrToTime(end_value, tt_end, tm_end)) {
return;
}
int64_t days = (tt_end > tt_start) ? (tt_end - tt_start) / DAY_SECONDS : (tt_start - tt_end) /
DAY_SECONDS;
if (days == 0) {
return;
}
for (int64_t i = 0; i < days; i++) {
time_t tt_day = tt_start + DAY_SECONDS * i;
tm tm_day;
zilliz::milvus::server::CommonUtil::ConvertTime(tt_day, tm_day);
int64_t date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 +
tm_day.tm_mday;//according to db logic
dates.push_back(date);
}
}
} // namespace
TEST_F(DBTest, CONFIG_TEST) {
{
ASSERT_ANY_THROW(engine::ArchiveConf conf("wrong"));
ASSERT_ANY_THROW(ms::engine::ArchiveConf conf("wrong"));
/* EXPECT_DEATH(engine::ArchiveConf conf("wrong"), ""); */
}
{
engine::ArchiveConf conf("delete");
ms::engine::ArchiveConf conf("delete");
ASSERT_EQ(conf.GetType(), "delete");
auto criterias = conf.GetCriterias();
ASSERT_TRUE(criterias.size() == 0);
ASSERT_EQ(criterias.size(), 0);
}
{
engine::ArchiveConf conf("swap");
ms::engine::ArchiveConf conf("swap");
ASSERT_EQ(conf.GetType(), "swap");
auto criterias = conf.GetCriterias();
ASSERT_TRUE(criterias.size() == 0);
ASSERT_EQ(criterias.size(), 0);
}
{
ASSERT_ANY_THROW(engine::ArchiveConf conf1("swap", "disk:"));
ASSERT_ANY_THROW(engine::ArchiveConf conf2("swap", "disk:a"));
engine::ArchiveConf conf("swap", "disk:1024");
ASSERT_ANY_THROW(ms::engine::ArchiveConf conf1("swap", "disk:"));
ASSERT_ANY_THROW(ms::engine::ArchiveConf conf2("swap", "disk:a"));
ms::engine::ArchiveConf conf("swap", "disk:1024");
auto criterias = conf.GetCriterias();
ASSERT_TRUE(criterias.size() == 1);
ASSERT_TRUE(criterias["disk"] == 1024);
ASSERT_EQ(criterias.size(), 1);
ASSERT_EQ(criterias["disk"], 1024);
}
{
ASSERT_ANY_THROW(engine::ArchiveConf conf1("swap", "days:"));
ASSERT_ANY_THROW(engine::ArchiveConf conf2("swap", "days:a"));
engine::ArchiveConf conf("swap", "days:100");
ASSERT_ANY_THROW(ms::engine::ArchiveConf conf1("swap", "days:"));
ASSERT_ANY_THROW(ms::engine::ArchiveConf conf2("swap", "days:a"));
ms::engine::ArchiveConf conf("swap", "days:100");
auto criterias = conf.GetCriterias();
ASSERT_TRUE(criterias.size() == 1);
ASSERT_TRUE(criterias["days"] == 100);
ASSERT_EQ(criterias.size(), 1);
ASSERT_EQ(criterias["days"], 100);
}
{
ASSERT_ANY_THROW(engine::ArchiveConf conf1("swap", "days:"));
ASSERT_ANY_THROW(engine::ArchiveConf conf2("swap", "days:a"));
engine::ArchiveConf conf("swap", "days:100;disk:200");
ASSERT_ANY_THROW(ms::engine::ArchiveConf conf1("swap", "days:"));
ASSERT_ANY_THROW(ms::engine::ArchiveConf conf2("swap", "days:a"));
ms::engine::ArchiveConf conf("swap", "days:100;disk:200");
auto criterias = conf.GetCriterias();
ASSERT_TRUE(criterias.size() == 2);
ASSERT_TRUE(criterias["days"] == 100);
ASSERT_TRUE(criterias["disk"] == 200);
ASSERT_EQ(criterias.size(), 2);
ASSERT_EQ(criterias["days"], 100);
ASSERT_EQ(criterias["disk"], 200);
}
}
TEST_F(DBTest, DB_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
engine::IDNumbers vector_ids;
engine::IDNumbers target_ids;
ms::engine::IDNumbers vector_ids;
ms::engine::IDNumbers target_ids;
int64_t nb = 50;
std::vector<float> xb;
@ -171,7 +174,7 @@ TEST_F(DBTest, DB_TEST) {
BuildVectors(qb, qxb);
std::thread search([&]() {
engine::QueryResults results;
ms::engine::QueryResults results;
int k = 10;
std::this_thread::sleep_for(std::chrono::seconds(2));
@ -180,18 +183,18 @@ TEST_F(DBTest, DB_TEST) {
uint64_t count = 0;
uint64_t prev_count = 0;
for (auto j=0; j<10; ++j) {
for (auto j = 0; j < 10; ++j) {
ss.str("");
db_->Size(count);
prev_count = count;
START_TIMER;
stat = db_->Query(TABLE_NAME, k, qb, 10, qxb.data(), results);
ss << "Search " << j << " With Size " << count/engine::meta::M << " M";
ss << "Search " << j << " With Size " << count / ms::engine::meta::M << " M";
STOP_TIMER(ss.str());
ASSERT_TRUE(stat.ok());
for (auto k=0; k<qb; ++k) {
for (auto k = 0; k < qb; ++k) {
ASSERT_EQ(results[k][0].first, target_ids[k]);
ss.str("");
ss << "Result [" << k << "]:";
@ -207,8 +210,8 @@ TEST_F(DBTest, DB_TEST) {
int loop = INSERT_LOOP;
for (auto i=0; i<loop; ++i) {
if (i==40) {
for (auto i = 0; i < loop; ++i) {
if (i == 40) {
db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids);
ASSERT_EQ(target_ids.size(), qb);
} else {
@ -222,14 +225,14 @@ TEST_F(DBTest, DB_TEST) {
uint64_t count;
stat = db_->GetTableRowCount(TABLE_NAME, count);
ASSERT_TRUE(stat.ok());
ASSERT_TRUE(count > 0);
};
ASSERT_GT(count, 0);
}
TEST_F(DBTest, SEARCH_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
@ -239,63 +242,63 @@ TEST_F(DBTest, SEARCH_TEST) {
size_t nb = VECTOR_COUNT;
size_t nq = 10;
size_t k = 5;
std::vector<float> xb(nb*TABLE_DIM);
std::vector<float> xq(nq*TABLE_DIM);
std::vector<long> ids(nb);
std::vector<float> xb(nb * TABLE_DIM);
std::vector<float> xq(nq * TABLE_DIM);
std::vector<int64_t> ids(nb);
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_real_distribution<> dis_xt(-1.0, 1.0);
for (size_t i = 0; i < nb*TABLE_DIM; i++) {
for (size_t i = 0; i < nb * TABLE_DIM; i++) {
xb[i] = dis_xt(gen);
if (i < nb){
if (i < nb) {
ids[i] = i;
}
}
for (size_t i = 0; i < nq*TABLE_DIM; i++) {
for (size_t i = 0; i < nq * TABLE_DIM; i++) {
xq[i] = dis_xt(gen);
}
// result data
//std::vector<long> nns_gt(k*nq);
std::vector<long> nns(k*nq); // nns = nearst neg search
std::vector<int64_t> nns(k * nq); // nns = nearst neg search
//std::vector<float> dis_gt(k*nq);
std::vector<float> dis(k*nq);
std::vector<float> dis(k * nq);
// insert data
const int batch_size = 100;
for (int j = 0; j < nb / batch_size; ++j) {
stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data()+batch_size*j*TABLE_DIM, ids);
if (j == 200){ sleep(1);}
stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data() + batch_size * j * TABLE_DIM, ids);
if (j == 200) { sleep(1); }
ASSERT_TRUE(stat.ok());
}
engine::TableIndex index;
index.engine_type_ = (int)engine::EngineType::FAISS_IDMAP;
ms::engine::TableIndex index;
index.engine_type_ = (int) ms::engine::EngineType::FAISS_IDMAP;
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
{
engine::QueryResults results;
ms::engine::QueryResults results;
stat = db_->Query(TABLE_NAME, k, nq, 10, xq.data(), results);
ASSERT_TRUE(stat.ok());
}
{//search by specify index file
engine::meta::DatesT dates;
ms::engine::meta::DatesT dates;
std::vector<std::string> file_ids = {"1", "2", "3", "4", "5", "6"};
engine::QueryResults results;
ms::engine::QueryResults results;
stat = db_->Query(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, results);
ASSERT_TRUE(stat.ok());
}
// TODO(linxj): add groundTruth assert
};
}
TEST_F(DBTest, PRELOADTABLE_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
@ -306,28 +309,27 @@ TEST_F(DBTest, PRELOADTABLE_TEST) {
BuildVectors(nb, xb);
int loop = 5;
for (auto i=0; i<loop; ++i) {
engine::IDNumbers vector_ids;
for (auto i = 0; i < loop; ++i) {
ms::engine::IDNumbers vector_ids;
db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
ASSERT_EQ(vector_ids.size(), nb);
}
engine::TableIndex index;
index.engine_type_ = (int)engine::EngineType::FAISS_IDMAP;
ms::engine::TableIndex index;
index.engine_type_ = (int) ms::engine::EngineType::FAISS_IDMAP;
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
int64_t prev_cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage();
int64_t prev_cache_usage = ms::cache::CpuCacheMgr::GetInstance()->CacheUsage();
stat = db_->PreloadTable(TABLE_NAME);
ASSERT_TRUE(stat.ok());
int64_t cur_cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage();
int64_t cur_cache_usage = ms::cache::CpuCacheMgr::GetInstance()->CacheUsage();
ASSERT_TRUE(prev_cache_usage < cur_cache_usage);
}
TEST_F(DBTest, SHUTDOWN_TEST) {
db_->Stop();
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
ASSERT_FALSE(stat.ok());
@ -338,7 +340,7 @@ TEST_F(DBTest, SHUTDOWN_TEST) {
stat = db_->HasTable(table_info.table_id_, has_table);
ASSERT_FALSE(stat.ok());
engine::IDNumbers ids;
ms::engine::IDNumbers ids;
stat = db_->InsertVectors(table_info.table_id_, 0, nullptr, ids);
ASSERT_FALSE(stat.ok());
@ -349,15 +351,15 @@ TEST_F(DBTest, SHUTDOWN_TEST) {
stat = db_->GetTableRowCount(table_info.table_id_, row_count);
ASSERT_FALSE(stat.ok());
engine::TableIndex index;
ms::engine::TableIndex index;
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_FALSE(stat.ok());
stat = db_->DescribeIndex(table_info.table_id_, index);
ASSERT_FALSE(stat.ok());
engine::meta::DatesT dates;
engine::QueryResults results;
ms::engine::meta::DatesT dates;
ms::engine::QueryResults results;
stat = db_->Query(table_info.table_id_, 1, 1, 1, nullptr, dates, results);
ASSERT_FALSE(stat.ok());
std::vector<std::string> file_ids;
@ -369,24 +371,24 @@ TEST_F(DBTest, SHUTDOWN_TEST) {
}
TEST_F(DBTest, INDEX_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
int64_t nb = VECTOR_COUNT;
std::vector<float> xb;
BuildVectors(nb, xb);
engine::IDNumbers vector_ids;
ms::engine::IDNumbers vector_ids;
db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
ASSERT_EQ(vector_ids.size(), nb);
engine::TableIndex index;
index.engine_type_ = (int)engine::EngineType::FAISS_IVFSQ8;
index.metric_type_ = (int)engine::MetricType::IP;
ms::engine::TableIndex index;
index.engine_type_ = (int) ms::engine::EngineType::FAISS_IVFSQ8;
index.metric_type_ = (int) ms::engine::MetricType::IP;
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_TRUE(stat.ok());
engine::TableIndex index_out;
ms::engine::TableIndex index_out;
stat = db_->DescribeIndex(table_info.table_id_, index_out);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(index.engine_type_, index_out.engine_type_);
@ -398,23 +400,22 @@ TEST_F(DBTest, INDEX_TEST) {
}
TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
std::vector<engine::meta::TableSchema> table_schema_array;
std::vector<ms::engine::meta::TableSchema> table_schema_array;
stat = db_->AllTables(table_schema_array);
ASSERT_TRUE(stat.ok());
bool bfound = false;
for(auto& schema : table_schema_array) {
if(schema.table_id_ == TABLE_NAME) {
for (auto &schema : table_schema_array) {
if (schema.table_id_ == TABLE_NAME) {
bfound = true;
break;
}
}
ASSERT_TRUE(bfound);
engine::meta::TableSchema table_info_get;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
@ -428,8 +429,8 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
BuildVectors(nb, xb);
int loop = INSERT_LOOP;
for (auto i=0; i<loop; ++i) {
engine::IDNumbers vector_ids;
for (auto i = 0; i < loop; ++i) {
ms::engine::IDNumbers vector_ids;
db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
std::this_thread::sleep_for(std::chrono::microseconds(1));
}
@ -438,14 +439,14 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
db_->Size(size);
LOG(DEBUG) << "size=" << size;
ASSERT_LE(size, 1 * engine::meta::G);
};
ASSERT_LE(size, 1 * ms::engine::meta::G);
}
TEST_F(DBTest2, DELETE_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
@ -461,25 +462,25 @@ TEST_F(DBTest2, DELETE_TEST) {
std::vector<float> xb;
BuildVectors(nb, xb);
engine::IDNumbers vector_ids;
ms::engine::IDNumbers vector_ids;
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
engine::TableIndex index;
ms::engine::TableIndex index;
stat = db_->CreateIndex(TABLE_NAME, index);
std::vector<engine::meta::DateT> dates;
std::vector<ms::engine::meta::DateT> dates;
stat = db_->DeleteTable(TABLE_NAME, dates);
std::this_thread::sleep_for(std::chrono::seconds(2));
ASSERT_TRUE(stat.ok());
db_->HasTable(TABLE_NAME, has_table);
ASSERT_FALSE(has_table);
};
}
TEST_F(DBTest2, DELETE_BY_RANGE_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
@ -496,15 +497,15 @@ TEST_F(DBTest2, DELETE_BY_RANGE_TEST) {
std::vector<float> xb;
BuildVectors(nb, xb);
engine::IDNumbers vector_ids;
ms::engine::IDNumbers vector_ids;
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
engine::TableIndex index;
ms::engine::TableIndex index;
stat = db_->CreateIndex(TABLE_NAME, index);
db_->Size(size);
ASSERT_NE(size, 0UL);
std::vector<engine::meta::DateT> dates;
std::vector<ms::engine::meta::DateT> dates;
std::string start_value = CurrentTmDate();
std::string end_value = CurrentTmDate(1);
ConvertTimeRangeToDBDates(start_value, end_value, dates);
@ -515,4 +516,4 @@ TEST_F(DBTest2, DELETE_BY_RANGE_TEST) {
uint64_t row_count = 0;
db_->GetTableRowCount(TABLE_NAME, row_count);
ASSERT_EQ(row_count, 0UL);
}
}

71
cpp/unittest/db/engine_test.cpp
View File

@ -21,67 +21,66 @@
#include "db/engine/EngineFactory.h"
#include "db/engine/ExecutionEngineImpl.h"
#include "utils.h"
#include "db/utils.h"
using namespace zilliz::milvus;
namespace {
namespace ms = zilliz::milvus;
}
TEST_F(EngineTest, FACTORY_TEST) {
{
auto engine_ptr = engine::EngineFactory::Build(
auto engine_ptr = ms::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
engine::EngineType::INVALID,
engine::MetricType::IP,
1024
);
ms::engine::EngineType::INVALID,
ms::engine::MetricType::IP,
1024);
ASSERT_TRUE(engine_ptr == nullptr);
}
{
auto engine_ptr = engine::EngineFactory::Build(
auto engine_ptr = ms::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
engine::EngineType::FAISS_IDMAP,
engine::MetricType::IP,
1024
);
ms::engine::EngineType::FAISS_IDMAP,
ms::engine::MetricType::IP,
1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = engine::EngineFactory::Build(
auto engine_ptr = ms::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
engine::EngineType::FAISS_IVFFLAT,
engine::MetricType::IP,
1024
);
ms::engine::EngineType::FAISS_IVFFLAT,
ms::engine::MetricType::IP,
1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = engine::EngineFactory::Build(
auto engine_ptr = ms::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
engine::EngineType::FAISS_IVFSQ8,
engine::MetricType::IP,
1024
);
ms::engine::EngineType::FAISS_IVFSQ8,
ms::engine::MetricType::IP,
1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
{
auto engine_ptr = engine::EngineFactory::Build(
auto engine_ptr = ms::engine::EngineFactory::Build(
512,
"/tmp/milvus_index_1",
engine::EngineType::NSG_MIX,
engine::MetricType::IP,
1024
);
ms::engine::EngineType::NSG_MIX,
ms::engine::MetricType::IP,
1024);
ASSERT_TRUE(engine_ptr != nullptr);
}
@ -90,27 +89,26 @@ TEST_F(EngineTest, FACTORY_TEST) {
TEST_F(EngineTest, ENGINE_IMPL_TEST) {
uint16_t dimension = 64;
std::string file_path = "/tmp/milvus_index_1";
auto engine_ptr = engine::EngineFactory::Build(
auto engine_ptr = ms::engine::EngineFactory::Build(
dimension,
file_path,
engine::EngineType::FAISS_IVFFLAT,
engine::MetricType::IP,
1024
);
ms::engine::EngineType::FAISS_IVFFLAT,
ms::engine::MetricType::IP,
1024);
std::vector<float> data;
std::vector<long> ids;
std::vector<int64_t> ids;
const int row_count = 10000;
data.reserve(row_count*dimension);
ids.reserve(row_count);
for(long i = 0; i < row_count; i++) {
for (int64_t i = 0; i < row_count; i++) {
ids.push_back(i);
for(uint16_t k = 0; k < dimension; k++) {
for (uint16_t k = 0; k < dimension; k++) {
data.push_back(i*dimension + k);
}
}
auto status = engine_ptr->AddWithIds((long)ids.size(), data.data(), ids.data());
auto status = engine_ptr->AddWithIds((int64_t)ids.size(), data.data(), ids.data());
ASSERT_TRUE(status.ok());
ASSERT_EQ(engine_ptr->Dimension(), dimension);
@ -127,5 +125,4 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) {
//
// auto engine_build = new_engine->BuildIndex("/tmp/milvus_index_2", engine::EngineType::FAISS_IVFSQ8);
// //ASSERT_TRUE(status.ok());
}

152
cpp/unittest/db/mem_test.cpp
View File

@ -25,7 +25,7 @@
#include "db/engine/EngineFactory.h"
#include "db/meta/MetaConsts.h"
#include "metrics/Metrics.h"
#include "utils.h"
#include "db/utils.h"
#include <boost/filesystem.hpp>
#include <thread>
@ -35,30 +35,32 @@
#include <random>
#include <chrono>
using namespace zilliz::milvus;
namespace {
static std::string TABLE_NAME = "test_group";
namespace ms = zilliz::milvus;
static constexpr int64_t TABLE_DIM = 256;
std::string GenTableName() {
std::string
GetTableName() {
auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count();
TABLE_NAME = std::to_string(micros);
return TABLE_NAME;
now.time_since_epoch()).count();
static std::string table_name = std::to_string(micros);
return table_name;
}
engine::meta::TableSchema BuildTableSchema() {
engine::meta::TableSchema table_info;
ms::engine::meta::TableSchema
BuildTableSchema() {
ms::engine::meta::TableSchema table_info;
table_info.dimension_ = TABLE_DIM;
table_info.table_id_ = GenTableName();
table_info.engine_type_ = (int) engine::EngineType::FAISS_IDMAP;
table_info.table_id_ = GetTableName();
table_info.engine_type_ = (int) ms::engine::EngineType::FAISS_IDMAP;
return table_info;
}
void BuildVectors(int64_t n, std::vector<float> &vectors) {
void
BuildVectors(int64_t n, std::vector<float> &vectors) {
vectors.clear();
vectors.resize(n * TABLE_DIM);
float *data = vectors.data();
@ -67,15 +69,15 @@ void BuildVectors(int64_t n, std::vector<float> &vectors) {
data[TABLE_DIM * i + j] = drand48();
}
}
}
} // namespace
TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) {
engine::meta::TableSchema table_schema = BuildTableSchema();
ms::engine::meta::TableSchema table_schema = BuildTableSchema();
auto status = impl_->CreateTable(table_schema);
ASSERT_TRUE(status.ok());
engine::meta::TableFileSchema table_file_schema;
table_file_schema.table_id_ = TABLE_NAME;
ms::engine::meta::TableFileSchema table_file_schema;
table_file_schema.table_id_ = GetTableName();
status = impl_->CreateTableFile(table_file_schema);
ASSERT_TRUE(status.ok());
@ -83,17 +85,17 @@ TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) {
std::vector<float> vectors;
BuildVectors(n, vectors);
engine::VectorSource source(n, vectors.data());
ms::engine::VectorSource source(n, vectors.data());
size_t num_vectors_added;
engine::ExecutionEnginePtr execution_engine_ =
engine::EngineFactory::Build(table_file_schema.dimension_,
table_file_schema.location_,
(engine::EngineType) table_file_schema.engine_type_,
(engine::MetricType)table_file_schema.metric_type_,
table_schema.nlist_);
ms::engine::ExecutionEnginePtr execution_engine_ =
ms::engine::EngineFactory::Build(table_file_schema.dimension_,
table_file_schema.location_,
(ms::engine::EngineType) table_file_schema.engine_type_,
(ms::engine::MetricType) table_file_schema.metric_type_,
table_schema.nlist_);
engine::IDNumbers vector_ids;
ms::engine::IDNumbers vector_ids;
status = source.Add(execution_engine_, table_file_schema, 50, num_vectors_added, vector_ids);
ASSERT_TRUE(status.ok());
vector_ids = source.GetVectorIds();
@ -113,19 +115,19 @@ TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) {
TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
auto options = GetOptions();
engine::meta::TableSchema table_schema = BuildTableSchema();
ms::engine::meta::TableSchema table_schema = BuildTableSchema();
auto status = impl_->CreateTable(table_schema);
ASSERT_TRUE(status.ok());
engine::MemTableFile mem_table_file(TABLE_NAME, impl_, options);
ms::engine::MemTableFile mem_table_file(GetTableName(), impl_, options);
int64_t n_100 = 100;
std::vector<float> vectors_100;
BuildVectors(n_100, vectors_100);
engine::VectorSourcePtr source = std::make_shared<engine::VectorSource>(n_100, vectors_100.data());
ms::engine::VectorSourcePtr source = std::make_shared<ms::engine::VectorSource>(n_100, vectors_100.data());
engine::IDNumbers vector_ids;
ms::engine::IDNumbers vector_ids;
status = mem_table_file.Add(source, vector_ids);
ASSERT_TRUE(status.ok());
@ -137,11 +139,11 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
size_t singleVectorMem = sizeof(float) * TABLE_DIM;
ASSERT_EQ(mem_table_file.GetCurrentMem(), n_100 * singleVectorMem);
int64_t n_max = engine::MAX_TABLE_FILE_MEM / singleVectorMem;
int64_t n_max = ms::engine::MAX_TABLE_FILE_MEM / singleVectorMem;
std::vector<float> vectors_128M;
BuildVectors(n_max, vectors_128M);
engine::VectorSourcePtr source_128M = std::make_shared<engine::VectorSource>(n_max, vectors_128M.data());
ms::engine::VectorSourcePtr source_128M = std::make_shared<ms::engine::VectorSource>(n_max, vectors_128M.data());
vector_ids.clear();
status = mem_table_file.Add(source_128M, vector_ids);
@ -154,7 +156,7 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
TEST_F(MemManagerTest, MEM_TABLE_TEST) {
auto options = GetOptions();
engine::meta::TableSchema table_schema = BuildTableSchema();
ms::engine::meta::TableSchema table_schema = BuildTableSchema();
auto status = impl_->CreateTable(table_schema);
ASSERT_TRUE(status.ok());
@ -162,27 +164,27 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
std::vector<float> vectors_100;
BuildVectors(n_100, vectors_100);
engine::VectorSourcePtr source_100 = std::make_shared<engine::VectorSource>(n_100, vectors_100.data());
ms::engine::VectorSourcePtr source_100 = std::make_shared<ms::engine::VectorSource>(n_100, vectors_100.data());
engine::MemTable mem_table(TABLE_NAME, impl_, options);
ms::engine::MemTable mem_table(GetTableName(), impl_, options);
engine::IDNumbers vector_ids;
ms::engine::IDNumbers vector_ids;
status = mem_table.Add(source_100, vector_ids);
ASSERT_TRUE(status.ok());
vector_ids = source_100->GetVectorIds();
ASSERT_EQ(vector_ids.size(), 100);
engine::MemTableFilePtr mem_table_file;
ms::engine::MemTableFilePtr mem_table_file;
mem_table.GetCurrentMemTableFile(mem_table_file);
size_t singleVectorMem = sizeof(float) * TABLE_DIM;
ASSERT_EQ(mem_table_file->GetCurrentMem(), n_100 * singleVectorMem);
int64_t n_max = engine::MAX_TABLE_FILE_MEM / singleVectorMem;
int64_t n_max = ms::engine::MAX_TABLE_FILE_MEM / singleVectorMem;
std::vector<float> vectors_128M;
BuildVectors(n_max, vectors_128M);
vector_ids.clear();
engine::VectorSourcePtr source_128M = std::make_shared<engine::VectorSource>(n_max, vectors_128M.data());
ms::engine::VectorSourcePtr source_128M = std::make_shared<ms::engine::VectorSource>(n_max, vectors_128M.data());
status = mem_table.Add(source_128M, vector_ids);
ASSERT_TRUE(status.ok());
@ -198,7 +200,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
std::vector<float> vectors_1G;
BuildVectors(n_1G, vectors_1G);
engine::VectorSourcePtr source_1G = std::make_shared<engine::VectorSource>(n_1G, vectors_1G.data());
ms::engine::VectorSourcePtr source_1G = std::make_shared<ms::engine::VectorSource>(n_1G, vectors_1G.data());
vector_ids.clear();
status = mem_table.Add(source_1G, vector_ids);
@ -207,7 +209,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
vector_ids = source_1G->GetVectorIds();
ASSERT_EQ(vector_ids.size(), n_1G);
int expectedTableFileCount = 2 + std::ceil((n_1G - n_100) * singleVectorMem / engine::MAX_TABLE_FILE_MEM);
int expectedTableFileCount = 2 + std::ceil((n_1G - n_100) * singleVectorMem / ms::engine::MAX_TABLE_FILE_MEM);
ASSERT_EQ(mem_table.GetTableFileCount(), expectedTableFileCount);
status = mem_table.Serialize();
@ -215,11 +217,11 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
}
TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
@ -228,12 +230,12 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
std::vector<float> xb;
BuildVectors(nb, xb);
engine::IDNumbers vector_ids;
for(int64_t i = 0; i < nb; i++) {
ms::engine::IDNumbers vector_ids;
for (int64_t i = 0; i < nb; i++) {
vector_ids.push_back(i);
}
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
ASSERT_TRUE(stat.ok());
std::this_thread::sleep_for(std::chrono::seconds(3));//ensure raw data write to disk
@ -256,19 +258,19 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
int topk = 10, nprobe = 10;
for (auto &pair : search_vectors) {
auto &search = pair.second;
engine::QueryResults results;
stat = db_->Query(TABLE_NAME, topk, 1, nprobe, search.data(), results);
ms::engine::QueryResults results;
stat = db_->Query(GetTableName(), topk, 1, nprobe, search.data(), results);
ASSERT_EQ(results[0][0].first, pair.first);
ASSERT_LT(results[0][0].second, 1e-4);
}
}
TEST_F(MemManagerTest2, INSERT_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
@ -280,8 +282,8 @@ TEST_F(MemManagerTest2, INSERT_TEST) {
int64_t nb = 40960;
std::vector<float> xb;
BuildVectors(nb, xb);
engine::IDNumbers vector_ids;
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
ms::engine::IDNumbers vector_ids;
stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
ASSERT_TRUE(stat.ok());
}
auto end_time = METRICS_NOW_TIME;
@ -290,17 +292,17 @@ TEST_F(MemManagerTest2, INSERT_TEST) {
}
TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
engine::IDNumbers vector_ids;
engine::IDNumbers target_ids;
ms::engine::IDNumbers vector_ids;
ms::engine::IDNumbers target_ids;
int64_t nb = 40960;
std::vector<float> xb;
@ -311,7 +313,7 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) {
BuildVectors(qb, qxb);
std::thread search([&]() {
engine::QueryResults results;
ms::engine::QueryResults results;
int k = 10;
std::this_thread::sleep_for(std::chrono::seconds(2));
@ -326,8 +328,8 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) {
prev_count = count;
START_TIMER;
stat = db_->Query(TABLE_NAME, k, qb, 10, qxb.data(), results);
ss << "Search " << j << " With Size " << count / engine::meta::M << " M";
stat = db_->Query(GetTableName(), k, qb, 10, qxb.data(), results);
ss << "Search " << j << " With Size " << count / ms::engine::meta::M << " M";
STOP_TIMER(ss.str());
ASSERT_TRUE(stat.ok());
@ -349,29 +351,28 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) {
for (auto i = 0; i < loop; ++i) {
if (i == 0) {
db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids);
db_->InsertVectors(GetTableName(), qb, qxb.data(), target_ids);
ASSERT_EQ(target_ids.size(), qb);
} else {
db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
}
std::this_thread::sleep_for(std::chrono::microseconds(1));
}
search.join();
};
}
TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
engine::IDNumbers vector_ids;
ms::engine::IDNumbers vector_ids;
int64_t nb = 100000;
std::vector<float> xb;
@ -382,7 +383,7 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
vector_ids[i] = i;
}
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
ASSERT_EQ(vector_ids[0], 0);
ASSERT_TRUE(stat.ok());
@ -394,7 +395,7 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
for (auto i = 0; i < nb; i++) {
vector_ids[i] = i + nb;
}
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
ASSERT_EQ(vector_ids[0], nb);
ASSERT_TRUE(stat.ok());
@ -406,15 +407,15 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
for (auto i = 0; i < nb; i++) {
vector_ids[i] = i + nb / 2;
}
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
ASSERT_EQ(vector_ids[0], nb/2);
stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
ASSERT_EQ(vector_ids[0], nb / 2);
ASSERT_TRUE(stat.ok());
nb = 65536; //128M
xb.clear();
BuildVectors(nb, xb);
vector_ids.clear();
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
ASSERT_TRUE(stat.ok());
nb = 100;
@ -425,8 +426,9 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
for (auto i = 0; i < nb; i++) {
vector_ids[i] = i + nb;
}
stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids);
for (auto i = 0; i < nb; i++) {
ASSERT_EQ(vector_ids[i], i + nb);
}
}

174
cpp/unittest/db/meta_tests.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "utils.h"
#include "db/utils.h"
#include "db/meta/SqliteMetaImpl.h"
#include "db/Utils.h"
#include "db/meta/MetaConsts.h"
@ -25,13 +25,16 @@
#include <stdlib.h>
#include <time.h>
using namespace zilliz::milvus;
using namespace zilliz::milvus::engine;
namespace {
namespace ms = zilliz::milvus;
} // namespace
TEST_F(MetaTest, TABLE_TEST) {
auto table_id = "meta_test_table";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl_->CreateTable(table);
ASSERT_TRUE(status.ok());
@ -49,7 +52,7 @@ TEST_F(MetaTest, TABLE_TEST) {
table.table_id_ = table_id;
status = impl_->CreateTable(table);
ASSERT_EQ(status.code(), DB_ALREADY_EXIST);
ASSERT_EQ(status.code(), ms::DB_ALREADY_EXIST);
table.table_id_ = "";
status = impl_->CreateTable(table);
@ -59,16 +62,16 @@ TEST_F(MetaTest, TABLE_TEST) {
TEST_F(MetaTest, TABLE_FILE_TEST) {
auto table_id = "meta_test_table";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
table.dimension_ = 256;
auto status = impl_->CreateTable(table);
meta::TableFileSchema table_file;
ms::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
status = impl_->CreateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW);
ASSERT_EQ(table_file.file_type_, ms::engine::meta::TableFileSchema::NEW);
uint64_t cnt = 0;
status = impl_->Count(table_id, cnt);
@ -77,30 +80,30 @@ TEST_F(MetaTest, TABLE_FILE_TEST) {
auto file_id = table_file.file_id_;
auto new_file_type = meta::TableFileSchema::INDEX;
auto new_file_type = ms::engine::meta::TableFileSchema::INDEX;
table_file.file_type_ = new_file_type;
status = impl_->UpdateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type_, new_file_type);
meta::DatesT dates;
dates.push_back(utils::GetDate());
ms::engine::meta::DatesT dates;
dates.push_back(ms::engine::utils::GetDate());
status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok());
dates.clear();
for (auto i=2; i < 10; ++i) {
dates.push_back(utils::GetDateWithDelta(-1*i));
for (auto i = 2; i < 10; ++i) {
dates.push_back(ms::engine::utils::GetDateWithDelta(-1 * i));
}
status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok());
table_file.date_ = utils::GetDateWithDelta(-2);
table_file.date_ = ms::engine::utils::GetDateWithDelta(-2);
status = impl_->UpdateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.date_, utils::GetDateWithDelta(-2));
ASSERT_FALSE(table_file.file_type_ == meta::TableFileSchema::TO_DELETE);
ASSERT_EQ(table_file.date_, ms::engine::utils::GetDateWithDelta(-2));
ASSERT_FALSE(table_file.file_type_ == ms::engine::meta::TableFileSchema::TO_DELETE);
dates.clear();
dates.push_back(table_file.date_);
@ -108,7 +111,7 @@ TEST_F(MetaTest, TABLE_FILE_TEST) {
ASSERT_TRUE(status.ok());
std::vector<size_t> ids = {table_file.id_};
meta::TableFilesSchema files;
ms::engine::meta::TableFilesSchema files;
status = impl_->GetTableFiles(table_file.table_id_, ids, files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(files.size(), 0UL);
@ -116,33 +119,34 @@ TEST_F(MetaTest, TABLE_FILE_TEST) {
TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
srand(time(0));
DBMetaOptions options;
ms::engine::DBMetaOptions options;
options.path_ = "/tmp/milvus_test";
int days_num = rand() % 100;
unsigned int seed = 1;
int days_num = rand_r(&seed) % 100;
std::stringstream ss;
ss << "days:" << days_num;
options.archive_conf_ = ArchiveConf("delete", ss.str());
options.archive_conf_ = ms::engine::ArchiveConf("delete", ss.str());
meta::SqliteMetaImpl impl(options);
ms::engine::meta::SqliteMetaImpl impl(options);
auto table_id = "meta_test_table";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl.CreateTable(table);
meta::TableFilesSchema files;
meta::TableFileSchema table_file;
ms::engine::meta::TableFilesSchema files;
ms::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
auto cnt = 100;
long ts = utils::GetMicroSecTimeStamp();
int64_t ts = ms::engine::utils::GetMicroSecTimeStamp();
std::vector<int> days;
std::vector<size_t> ids;
for (auto i=0; i<cnt; ++i) {
for (auto i = 0; i < cnt; ++i) {
status = impl.CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW;
int day = rand() % (days_num*2);
table_file.created_on_ = ts - day*meta::D_SEC*meta::US_PS - 10000;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
int day = rand_r(&seed) % (days_num * 2);
table_file.created_on_ = ts - day * ms::engine::meta::D_SEC * ms::engine::meta::US_PS - 10000;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
days.push_back(day);
@ -152,13 +156,13 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
impl.Archive();
int i = 0;
meta::TableFilesSchema files_get;
ms::engine::meta::TableFilesSchema files_get;
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for(auto& file : files_get) {
for (auto &file : files_get) {
if (days[i] < days_num) {
ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW);
ASSERT_EQ(file.file_type_, ms::engine::meta::TableFileSchema::NEW);
}
i++;
}
@ -167,28 +171,28 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
}
TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
DBMetaOptions options;
ms::engine::DBMetaOptions options;
options.path_ = "/tmp/milvus_test";
options.archive_conf_ = ArchiveConf("delete", "disk:11");
options.archive_conf_ = ms::engine::ArchiveConf("delete", "disk:11");
meta::SqliteMetaImpl impl(options);
ms::engine::meta::SqliteMetaImpl impl(options);
auto table_id = "meta_test_group";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl.CreateTable(table);
meta::TableFilesSchema files;
meta::TableFileSchema table_file;
ms::engine::meta::TableFilesSchema files;
ms::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
auto cnt = 10;
auto each_size = 2UL;
std::vector<size_t> ids;
for (auto i=0; i<cnt; ++i) {
for (auto i = 0; i < cnt; ++i) {
status = impl.CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW;
table_file.file_size_ = each_size * meta::G;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
table_file.file_size_ = each_size * ms::engine::meta::G;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
ids.push_back(table_file.id_);
@ -197,13 +201,13 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
impl.Archive();
int i = 0;
meta::TableFilesSchema files_get;
ms::engine::meta::TableFilesSchema files_get;
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for(auto& file : files_get) {
for (auto &file : files_get) {
if (i >= 5) {
ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW);
ASSERT_EQ(file.file_type_, ms::engine::meta::TableFileSchema::NEW);
}
++i;
}
@ -214,7 +218,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
TEST_F(MetaTest, TABLE_FILES_TEST) {
auto table_id = "meta_test_group";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl_->CreateTable(table);
@ -226,51 +230,51 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
uint64_t to_index_files_cnt = 6;
uint64_t index_files_cnt = 7;
meta::TableFileSchema table_file;
ms::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
for (auto i=0; i<new_merge_files_cnt; ++i) {
for (auto i = 0; i < new_merge_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW_MERGE;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_MERGE;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<new_index_files_cnt; ++i) {
for (auto i = 0; i < new_index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW_INDEX;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_INDEX;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<backup_files_cnt; ++i) {
for (auto i = 0; i < backup_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::BACKUP;
table_file.file_type_ = ms::engine::meta::TableFileSchema::BACKUP;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<new_files_cnt; ++i) {
for (auto i = 0; i < new_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<raw_files_cnt; ++i) {
for (auto i = 0; i < raw_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::RAW;
table_file.file_type_ = ms::engine::meta::TableFileSchema::RAW;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<to_index_files_cnt; ++i) {
for (auto i = 0; i < to_index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::TO_INDEX;
table_file.file_type_ = ms::engine::meta::TableFileSchema::TO_INDEX;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<index_files_cnt; ++i) {
for (auto i = 0; i < index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::INDEX;
table_file.file_type_ = ms::engine::meta::TableFileSchema::INDEX;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
@ -278,36 +282,36 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
uint64_t total_row_count = 0;
status = impl_->Count(table_id, total_row_count);
ASSERT_TRUE(status.ok());
ASSERT_EQ(total_row_count, raw_files_cnt+to_index_files_cnt+index_files_cnt);
ASSERT_EQ(total_row_count, raw_files_cnt + to_index_files_cnt + index_files_cnt);
meta::TableFilesSchema files;
ms::engine::meta::TableFilesSchema files;
status = impl_->FilesToIndex(files);
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatePartionedTableFilesSchema dated_files;
ms::engine::meta::DatePartionedTableFilesSchema dated_files;
status = impl_->FilesToMerge(table.table_id_, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt);
status = impl_->FilesToIndex(files);
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatesT dates = {table_file.date_};
ms::engine::meta::DatesT dates = {table_file.date_};
std::vector<size_t> ids;
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files);
status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files);
status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
to_index_files_cnt + raw_files_cnt + index_files_cnt);
ids.push_back(size_t(9999999999));
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),0);
ASSERT_EQ(dated_files[table_file.date_].size(), 0);
std::vector<int> file_types;
std::vector<std::string> file_ids;
@ -316,26 +320,26 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
ASSERT_FALSE(status.ok());
file_types = {
meta::TableFileSchema::NEW,
meta::TableFileSchema::NEW_MERGE,
meta::TableFileSchema::NEW_INDEX,
meta::TableFileSchema::TO_INDEX,
meta::TableFileSchema::INDEX,
meta::TableFileSchema::RAW,
meta::TableFileSchema::BACKUP,
ms::engine::meta::TableFileSchema::NEW,
ms::engine::meta::TableFileSchema::NEW_MERGE,
ms::engine::meta::TableFileSchema::NEW_INDEX,
ms::engine::meta::TableFileSchema::TO_INDEX,
ms::engine::meta::TableFileSchema::INDEX,
ms::engine::meta::TableFileSchema::RAW,
ms::engine::meta::TableFileSchema::BACKUP,
};
status = impl_->FilesByType(table.table_id_, file_types, file_ids);
ASSERT_TRUE(status.ok());
uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt +
backup_files_cnt + new_files_cnt + raw_files_cnt +
to_index_files_cnt + index_files_cnt;
backup_files_cnt + new_files_cnt + raw_files_cnt +
to_index_files_cnt + index_files_cnt;
ASSERT_EQ(file_ids.size(), total_cnt);
status = impl_->DeleteTableFiles(table_id);
ASSERT_TRUE(status.ok());
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
status = impl_->UpdateTableFile(table_file);
status = impl_->CleanUp();
ASSERT_TRUE(status.ok());
@ -350,11 +354,11 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
TEST_F(MetaTest, INDEX_TEST) {
auto table_id = "index_test";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl_->CreateTable(table);
TableIndex index;
ms::engine::TableIndex index;
index.metric_type_ = 2;
index.nlist_ = 1234;
index.engine_type_ = 3;
@ -365,12 +369,12 @@ TEST_F(MetaTest, INDEX_TEST) {
status = impl_->UpdateTableFlag(table_id, flag);
ASSERT_TRUE(status.ok());
engine::meta::TableSchema table_info;
ms::engine::meta::TableSchema table_info;
table_info.table_id_ = table_id;
status = impl_->DescribeTable(table_info);
ASSERT_EQ(table_info.flag_, flag);
TableIndex index_out;
ms::engine::TableIndex index_out;
status = impl_->DescribeTableIndex(table_id, index_out);
ASSERT_EQ(index_out.metric_type_, index.metric_type_);
ASSERT_EQ(index_out.nlist_, index.nlist_);
@ -385,4 +389,4 @@ TEST_F(MetaTest, INDEX_TEST) {
status = impl_->UpdateTableFilesToIndex(table_id);
ASSERT_TRUE(status.ok());
}
}

52
cpp/unittest/db/misc_test.cpp
View File

@ -27,43 +27,47 @@
#include <boost/filesystem.hpp>
#include <vector>
using namespace zilliz::milvus;
namespace {
namespace ms = zilliz::milvus;
} // namespace
TEST(DBMiscTest, EXCEPTION_TEST) {
Exception ex1(100, "error");
ms::Exception ex1(100, "error");
std::string what = ex1.what();
ASSERT_EQ(what, "error");
ASSERT_EQ(ex1.code(), 100);
InvalidArgumentException ex2;
ASSERT_EQ(ex2.code(), SERVER_INVALID_ARGUMENT);
ms::InvalidArgumentException ex2;
ASSERT_EQ(ex2.code(), ms::SERVER_INVALID_ARGUMENT);
}
TEST(DBMiscTest, OPTIONS_TEST) {
try {
engine::ArchiveConf archive("$$##");
} catch (std::exception& ex) {
ms::engine::ArchiveConf archive("$$##");
} catch (std::exception &ex) {
ASSERT_TRUE(true);
}
{
engine::ArchiveConf archive("delete", "no");
ms::engine::ArchiveConf archive("delete", "no");
ASSERT_TRUE(archive.GetCriterias().empty());
}
{
engine::ArchiveConf archive("delete", "1:2");
ms::engine::ArchiveConf archive("delete", "1:2");
ASSERT_TRUE(archive.GetCriterias().empty());
}
{
engine::ArchiveConf archive("delete", "1:2:3");
ms::engine::ArchiveConf archive("delete", "1:2:3");
ASSERT_TRUE(archive.GetCriterias().empty());
}
{
engine::ArchiveConf archive("delete");
engine::ArchiveConf::CriteriaT criterial = {
ms::engine::ArchiveConf archive("delete");
ms::engine::ArchiveConf::CriteriaT criterial = {
{"disk", 1024},
{"days", 100}
};
@ -76,29 +80,29 @@ TEST(DBMiscTest, OPTIONS_TEST) {
}
TEST(DBMiscTest, META_TEST) {
engine::DBMetaOptions options;
ms::engine::DBMetaOptions options;
options.path_ = "/tmp/milvus_test";
engine::meta::SqliteMetaImpl impl(options);
ms::engine::meta::SqliteMetaImpl impl(options);
time_t tt;
time( &tt );
time(&tt);
int delta = 10;
engine::meta::DateT dt = engine::utils::GetDate(tt, delta);
ms::engine::meta::DateT dt = ms::engine::utils::GetDate(tt, delta);
ASSERT_GT(dt, 0);
}
TEST(DBMiscTest, UTILS_TEST) {
engine::DBMetaOptions options;
ms::engine::DBMetaOptions options;
options.path_ = "/tmp/milvus_test/main";
options.slave_paths_.push_back("/tmp/milvus_test/slave_1");
options.slave_paths_.push_back("/tmp/milvus_test/slave_2");
const std::string TABLE_NAME = "test_tbl";
auto status = engine::utils::CreateTablePath(options, TABLE_NAME);
auto status = ms::engine::utils::CreateTablePath(options, TABLE_NAME);
ASSERT_TRUE(status.ok());
ASSERT_TRUE(boost::filesystem::exists(options.path_));
for(auto& path : options.slave_paths_) {
ASSERT_TRUE(boost::filesystem::exists(path));
for (auto &path : options.slave_paths_) {
ASSERT_TRUE(boost::filesystem::exists(path));
}
// options.slave_paths.push_back("/");
@ -109,18 +113,18 @@ TEST(DBMiscTest, UTILS_TEST) {
// status = engine::utils::CreateTablePath(options, TABLE_NAME);
// ASSERT_FALSE(status.ok());
engine::meta::TableFileSchema file;
ms::engine::meta::TableFileSchema file;
file.id_ = 50;
file.table_id_ = TABLE_NAME;
file.file_type_ = 3;
file.date_ = 155000;
status = engine::utils::GetTableFilePath(options, file);
status = ms::engine::utils::GetTableFilePath(options, file);
ASSERT_FALSE(status.ok());
ASSERT_TRUE(file.location_.empty());
status = engine::utils::DeleteTablePath(options, TABLE_NAME);
status = ms::engine::utils::DeleteTablePath(options, TABLE_NAME);
ASSERT_TRUE(status.ok());
status = engine::utils::DeleteTableFilePath(options, file);
status = ms::engine::utils::DeleteTableFilePath(options, file);
ASSERT_TRUE(status.ok());
}
}

130
cpp/unittest/db/mysql_db_test.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "utils.h"
#include "db/utils.h"
#include "db/DB.h"
#include "db/DBImpl.h"
#include "db/meta/MetaConsts.h"
@ -26,48 +26,49 @@
#include <thread>
#include <random>
using namespace zilliz::milvus;
namespace {
static const char* TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 25000;
static constexpr int64_t INSERT_LOOP = 1000;
namespace ms = zilliz::milvus;
engine::meta::TableSchema BuildTableSchema() {
engine::meta::TableSchema table_info;
table_info.dimension_ = TABLE_DIM;
table_info.table_id_ = TABLE_NAME;
table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP;
return table_info;
}
void BuildVectors(int64_t n, std::vector<float>& vectors) {
vectors.clear();
vectors.resize(n*TABLE_DIM);
float* data = vectors.data();
for(int i = 0; i < n; i++) {
for(int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48();
data[TABLE_DIM * i] += i / 2000.;
}
}
static const char *TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 25000;
static constexpr int64_t INSERT_LOOP = 1000;
ms::engine::meta::TableSchema
BuildTableSchema() {
ms::engine::meta::TableSchema table_info;
table_info.dimension_ = TABLE_DIM;
table_info.table_id_ = TABLE_NAME;
table_info.engine_type_ = (int) ms::engine::EngineType::FAISS_IDMAP;
return table_info;
}
void
BuildVectors(int64_t n, std::vector<float> &vectors) {
vectors.clear();
vectors.resize(n * TABLE_DIM);
float *data = vectors.data();
for (int i = 0; i < n; i++) {
for (int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48();
data[TABLE_DIM * i] += i / 2000.;
}
}
} // namespace
TEST_F(MySqlDBTest, DB_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
engine::IDNumbers vector_ids;
engine::IDNumbers target_ids;
ms::engine::IDNumbers vector_ids;
ms::engine::IDNumbers target_ids;
int64_t nb = 50;
std::vector<float> xb;
@ -81,7 +82,7 @@ TEST_F(MySqlDBTest, DB_TEST) {
ASSERT_EQ(target_ids.size(), qb);
std::thread search([&]() {
engine::QueryResults results;
ms::engine::QueryResults results;
int k = 10;
std::this_thread::sleep_for(std::chrono::seconds(5));
@ -90,22 +91,22 @@ TEST_F(MySqlDBTest, DB_TEST) {
uint64_t count = 0;
uint64_t prev_count = 0;
for (auto j=0; j<10; ++j) {
for (auto j = 0; j < 10; ++j) {
ss.str("");
db_->Size(count);
prev_count = count;
START_TIMER;
stat = db_->Query(TABLE_NAME, k, qb, 10, qxb.data(), results);
ss << "Search " << j << " With Size " << count/engine::meta::M << " M";
ss << "Search " << j << " With Size " << count / ms::engine::meta::M << " M";
STOP_TIMER(ss.str());
ASSERT_TRUE(stat.ok());
for (auto k=0; k<qb; ++k) {
for (auto k = 0; k < qb; ++k) {
// std::cout << results[k][0].first << " " << target_ids[k] << std::endl;
// ASSERT_EQ(results[k][0].first, target_ids[k]);
bool exists = false;
for (auto& result : results[k]) {
for (auto &result : results[k]) {
if (result.first == target_ids[k]) {
exists = true;
}
@ -127,7 +128,7 @@ TEST_F(MySqlDBTest, DB_TEST) {
int loop = INSERT_LOOP;
for (auto i=0; i<loop; ++i) {
for (auto i = 0; i < loop; ++i) {
// if (i==10) {
// db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids);
// ASSERT_EQ(target_ids.size(), qb);
@ -139,13 +140,13 @@ TEST_F(MySqlDBTest, DB_TEST) {
}
search.join();
};
}
TEST_F(MySqlDBTest, SEARCH_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
engine::meta::TableSchema table_info_get;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
@ -155,68 +156,68 @@ TEST_F(MySqlDBTest, SEARCH_TEST) {
size_t nb = VECTOR_COUNT;
size_t nq = 10;
size_t k = 5;
std::vector<float> xb(nb*TABLE_DIM);
std::vector<float> xq(nq*TABLE_DIM);
std::vector<long> ids(nb);
std::vector<float> xb(nb * TABLE_DIM);
std::vector<float> xq(nq * TABLE_DIM);
std::vector<int64_t> ids(nb);
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_real_distribution<> dis_xt(-1.0, 1.0);
for (size_t i = 0; i < nb*TABLE_DIM; i++) {
for (size_t i = 0; i < nb * TABLE_DIM; i++) {
xb[i] = dis_xt(gen);
if (i < nb){
if (i < nb) {
ids[i] = i;
}
}
for (size_t i = 0; i < nq*TABLE_DIM; i++) {
for (size_t i = 0; i < nq * TABLE_DIM; i++) {
xq[i] = dis_xt(gen);
}
// result data
//std::vector<long> nns_gt(k*nq);
std::vector<long> nns(k*nq); // nns = nearst neg search
std::vector<int64_t> nns(k * nq); // nns = nearst neg search
//std::vector<float> dis_gt(k*nq);
std::vector<float> dis(k*nq);
std::vector<float> dis(k * nq);
// insert data
const int batch_size = 100;
for (int j = 0; j < nb / batch_size; ++j) {
stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data()+batch_size*j*TABLE_DIM, ids);
if (j == 200){ sleep(1);}
stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data() + batch_size * j * TABLE_DIM, ids);
if (j == 200) { sleep(1); }
ASSERT_TRUE(stat.ok());
}
sleep(2); // wait until build index finish
engine::QueryResults results;
ms::engine::QueryResults results;
stat = db_->Query(TABLE_NAME, k, nq, 10, xq.data(), results);
ASSERT_TRUE(stat.ok());
};
}
TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
std::vector<engine::meta::TableSchema> table_schema_array;
std::vector<ms::engine::meta::TableSchema> table_schema_array;
stat = db_->AllTables(table_schema_array);
ASSERT_TRUE(stat.ok());
bool bfound = false;
for(auto& schema : table_schema_array) {
if(schema.table_id_ == TABLE_NAME) {
for (auto &schema : table_schema_array) {
if (schema.table_id_ == TABLE_NAME) {
bfound = true;
break;
}
}
ASSERT_TRUE(bfound);
engine::meta::TableSchema table_info_get;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
engine::IDNumbers vector_ids;
engine::IDNumbers target_ids;
ms::engine::IDNumbers vector_ids;
ms::engine::IDNumbers target_ids;
uint64_t size;
db_->Size(size);
@ -226,7 +227,7 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
BuildVectors(nb, xb);
int loop = INSERT_LOOP;
for (auto i=0; i<loop; ++i) {
for (auto i = 0; i < loop; ++i) {
db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
std::this_thread::sleep_for(std::chrono::microseconds(1));
}
@ -235,15 +236,15 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
db_->Size(size);
LOG(DEBUG) << "size=" << size;
ASSERT_LE(size, 1 * engine::meta::G);
};
ASSERT_LE(size, 1 * ms::engine::meta::G);
}
TEST_F(MySqlDBTest, DELETE_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
ms::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
// std::cout << stat.ToString() << std::endl;
engine::meta::TableSchema table_info_get;
ms::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = TABLE_NAME;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
@ -252,7 +253,7 @@ TEST_F(MySqlDBTest, DELETE_TEST) {
db_->HasTable(TABLE_NAME, has_table);
ASSERT_TRUE(has_table);
engine::IDNumbers vector_ids;
ms::engine::IDNumbers vector_ids;
uint64_t size;
db_->Size(size);
@ -262,7 +263,7 @@ TEST_F(MySqlDBTest, DELETE_TEST) {
BuildVectors(nb, xb);
int loop = 20;
for (auto i=0; i<loop; ++i) {
for (auto i = 0; i < loop; ++i) {
db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
std::this_thread::sleep_for(std::chrono::microseconds(1));
}
@ -276,4 +277,5 @@ TEST_F(MySqlDBTest, DELETE_TEST) {
//
// db_->HasTable(TABLE_NAME, has_table);
// ASSERT_FALSE(has_table);
};
}

180
cpp/unittest/db/mysql_meta_test.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "utils.h"
#include "db/utils.h"
#include "db/meta/MySQLMetaImpl.h"
#include "db/Utils.h"
#include "db/meta/MetaConsts.h"
@ -27,13 +27,17 @@
#include <gtest/gtest.h>
#include <mysql++/mysql++.h>
using namespace zilliz::milvus;
using namespace zilliz::milvus::engine;
namespace {
namespace ms = zilliz::milvus;
}
TEST_F(MySqlMetaTest, TABLE_TEST) {
auto table_id = "meta_test_table";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl_->CreateTable(table);
ASSERT_TRUE(status.ok());
@ -51,7 +55,7 @@ TEST_F(MySqlMetaTest, TABLE_TEST) {
table.table_id_ = table_id;
status = impl_->CreateTable(table);
ASSERT_EQ(status.code(), DB_ALREADY_EXIST);
ASSERT_EQ(status.code(), ms::DB_ALREADY_EXIST);
table.table_id_ = "";
status = impl_->CreateTable(table);
@ -64,20 +68,19 @@ TEST_F(MySqlMetaTest, TABLE_TEST) {
TEST_F(MySqlMetaTest, TABLE_FILE_TEST) {
auto table_id = "meta_test_table";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
table.dimension_ = 256;
auto status = impl_->CreateTable(table);
meta::TableFileSchema table_file;
ms::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
status = impl_->CreateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW);
ASSERT_EQ(table_file.file_type_, ms::engine::meta::TableFileSchema::NEW);
meta::DatesT dates;
dates.push_back(utils::GetDate());
ms::engine::meta::DatesT dates;
dates.push_back(ms::engine::utils::GetDate());
status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok());
@ -88,7 +91,7 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) {
auto file_id = table_file.file_id_;
auto new_file_type = meta::TableFileSchema::INDEX;
auto new_file_type = ms::engine::meta::TableFileSchema::INDEX;
table_file.file_type_ = new_file_type;
status = impl_->UpdateTableFile(table_file);
@ -96,17 +99,17 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) {
ASSERT_EQ(table_file.file_type_, new_file_type);
dates.clear();
for (auto i=2; i < 10; ++i) {
dates.push_back(utils::GetDateWithDelta(-1*i));
for (auto i = 2; i < 10; ++i) {
dates.push_back(ms::engine::utils::GetDateWithDelta(-1 * i));
}
status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok());
table_file.date_ = utils::GetDateWithDelta(-2);
table_file.date_ = ms::engine::utils::GetDateWithDelta(-2);
status = impl_->UpdateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.date_, utils::GetDateWithDelta(-2));
ASSERT_FALSE(table_file.file_type_ == meta::TableFileSchema::TO_DELETE);
ASSERT_EQ(table_file.date_, ms::engine::utils::GetDateWithDelta(-2));
ASSERT_FALSE(table_file.file_type_ == ms::engine::meta::TableFileSchema::TO_DELETE);
dates.clear();
dates.push_back(table_file.date_);
@ -114,41 +117,42 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) {
ASSERT_TRUE(status.ok());
std::vector<size_t> ids = {table_file.id_};
meta::TableFilesSchema files;
ms::engine::meta::TableFilesSchema files;
status = impl_->GetTableFiles(table_file.table_id_, ids, files);
ASSERT_EQ(files.size(), 0UL);
}
TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
srand(time(0));
DBMetaOptions options = GetOptions().meta_;
ms::engine::DBMetaOptions options = GetOptions().meta_;
int days_num = rand() % 100;
unsigned int seed = 1;
int days_num = rand_r(&seed) % 100;
std::stringstream ss;
ss << "days:" << days_num;
options.archive_conf_ = ArchiveConf("delete", ss.str());
int mode = DBOptions::MODE::SINGLE;
meta::MySQLMetaImpl impl(options, mode);
options.archive_conf_ = ms::engine::ArchiveConf("delete", ss.str());
int mode = ms::engine::DBOptions::MODE::SINGLE;
ms::engine::meta::MySQLMetaImpl impl(options, mode);
auto table_id = "meta_test_table";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl.CreateTable(table);
meta::TableFilesSchema files;
meta::TableFileSchema table_file;
ms::engine::meta::TableFilesSchema files;
ms::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
auto cnt = 100;
long ts = utils::GetMicroSecTimeStamp();
int64_t ts = ms::engine::utils::GetMicroSecTimeStamp();
std::vector<int> days;
std::vector<size_t> ids;
for (auto i=0; i<cnt; ++i) {
for (auto i = 0; i < cnt; ++i) {
status = impl.CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW;
int day = rand() % (days_num*2);
table_file.created_on_ = ts - day*meta::D_SEC*meta::US_PS - 10000;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
int day = rand_r(&seed) % (days_num * 2);
table_file.created_on_ = ts - day * ms::engine::meta::D_SEC * ms::engine::meta::US_PS - 10000;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
days.push_back(day);
@ -158,19 +162,19 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
impl.Archive();
int i = 0;
meta::TableFilesSchema files_get;
ms::engine::meta::TableFilesSchema files_get;
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for(auto& file : files_get) {
for (auto &file : files_get) {
if (days[i] < days_num) {
ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW);
ASSERT_EQ(file.file_type_, ms::engine::meta::TableFileSchema::NEW);
}
i++;
}
std::vector<int> file_types = {
(int) meta::TableFileSchema::NEW,
(int) ms::engine::meta::TableFileSchema::NEW,
};
std::vector<std::string> file_ids;
status = impl.FilesByType(table_id, file_types, file_ids);
@ -184,32 +188,32 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) {
}
TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
DBMetaOptions options = GetOptions().meta_;
ms::engine::DBMetaOptions options = GetOptions().meta_;
options.archive_conf_ = ArchiveConf("delete", "disk:11");
int mode = DBOptions::MODE::SINGLE;
auto impl = meta::MySQLMetaImpl(options, mode);
options.archive_conf_ = ms::engine::ArchiveConf("delete", "disk:11");
int mode = ms::engine::DBOptions::MODE::SINGLE;
auto impl = ms::engine::meta::MySQLMetaImpl(options, mode);
auto table_id = "meta_test_group";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl.CreateTable(table);
meta::TableSchema table_schema;
ms::engine::meta::TableSchema table_schema;
table_schema.table_id_ = "";
status = impl.CreateTable(table_schema);
meta::TableFilesSchema files;
meta::TableFileSchema table_file;
ms::engine::meta::TableFilesSchema files;
ms::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
auto cnt = 10;
auto each_size = 2UL;
std::vector<size_t> ids;
for (auto i=0; i<cnt; ++i) {
for (auto i = 0; i < cnt; ++i) {
status = impl.CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW;
table_file.file_size_ = each_size * meta::G;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
table_file.file_size_ = each_size * ms::engine::meta::G;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
ids.push_back(table_file.id_);
@ -218,13 +222,13 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
impl.Archive();
int i = 0;
meta::TableFilesSchema files_get;
ms::engine::meta::TableFilesSchema files_get;
status = impl.GetTableFiles(table_file.table_id_, ids, files_get);
ASSERT_TRUE(status.ok());
for(auto& file : files_get) {
for (auto &file : files_get) {
if (i >= 5) {
ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW);
ASSERT_EQ(file.file_type_, ms::engine::meta::TableFileSchema::NEW);
}
++i;
}
@ -236,7 +240,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
auto table_id = "meta_test_group";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl_->CreateTable(table);
@ -248,51 +252,51 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
uint64_t to_index_files_cnt = 6;
uint64_t index_files_cnt = 7;
meta::TableFileSchema table_file;
ms::engine::meta::TableFileSchema table_file;
table_file.table_id_ = table.table_id_;
for (auto i=0; i<new_merge_files_cnt; ++i) {
for (auto i = 0; i < new_merge_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW_MERGE;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_MERGE;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<new_index_files_cnt; ++i) {
for (auto i = 0; i < new_index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW_INDEX;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_INDEX;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<backup_files_cnt; ++i) {
for (auto i = 0; i < backup_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::BACKUP;
table_file.file_type_ = ms::engine::meta::TableFileSchema::BACKUP;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<new_files_cnt; ++i) {
for (auto i = 0; i < new_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::NEW;
table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<raw_files_cnt; ++i) {
for (auto i = 0; i < raw_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::RAW;
table_file.file_type_ = ms::engine::meta::TableFileSchema::RAW;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<to_index_files_cnt; ++i) {
for (auto i = 0; i < to_index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::TO_INDEX;
table_file.file_type_ = ms::engine::meta::TableFileSchema::TO_INDEX;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<index_files_cnt; ++i) {
for (auto i = 0; i < index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type_ = meta::TableFileSchema::INDEX;
table_file.file_type_ = ms::engine::meta::TableFileSchema::INDEX;
table_file.row_count_ = 1;
status = impl_->UpdateTableFile(table_file);
}
@ -300,36 +304,36 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
uint64_t total_row_count = 0;
status = impl_->Count(table_id, total_row_count);
ASSERT_TRUE(status.ok());
ASSERT_EQ(total_row_count, raw_files_cnt+to_index_files_cnt+index_files_cnt);
ASSERT_EQ(total_row_count, raw_files_cnt + to_index_files_cnt + index_files_cnt);
meta::TableFilesSchema files;
ms::engine::meta::TableFilesSchema files;
status = impl_->FilesToIndex(files);
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatePartionedTableFilesSchema dated_files;
ms::engine::meta::DatePartionedTableFilesSchema dated_files;
status = impl_->FilesToMerge(table.table_id_, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt);
status = impl_->FilesToIndex(files);
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatesT dates = {table_file.date_};
ms::engine::meta::DatesT dates = {table_file.date_};
std::vector<size_t> ids;
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files);
status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
to_index_files_cnt + raw_files_cnt + index_files_cnt);
status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files);
status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
to_index_files_cnt + raw_files_cnt + index_files_cnt);
ids.push_back(size_t(9999999999));
status = impl_->FilesToSearch(table_id, ids, dates, dated_files);
ASSERT_EQ(dated_files[table_file.date_].size(),0);
ASSERT_EQ(dated_files[table_file.date_].size(), 0);
std::vector<int> file_types;
std::vector<std::string> file_ids;
@ -338,19 +342,19 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
ASSERT_FALSE(status.ok());
file_types = {
meta::TableFileSchema::NEW,
meta::TableFileSchema::NEW_MERGE,
meta::TableFileSchema::NEW_INDEX,
meta::TableFileSchema::TO_INDEX,
meta::TableFileSchema::INDEX,
meta::TableFileSchema::RAW,
meta::TableFileSchema::BACKUP,
ms::engine::meta::TableFileSchema::NEW,
ms::engine::meta::TableFileSchema::NEW_MERGE,
ms::engine::meta::TableFileSchema::NEW_INDEX,
ms::engine::meta::TableFileSchema::TO_INDEX,
ms::engine::meta::TableFileSchema::INDEX,
ms::engine::meta::TableFileSchema::RAW,
ms::engine::meta::TableFileSchema::BACKUP,
};
status = impl_->FilesByType(table.table_id_, file_types, file_ids);
ASSERT_TRUE(status.ok());
uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt +
backup_files_cnt + new_files_cnt + raw_files_cnt +
to_index_files_cnt + index_files_cnt;
backup_files_cnt + new_files_cnt + raw_files_cnt +
to_index_files_cnt + index_files_cnt;
ASSERT_EQ(file_ids.size(), total_cnt);
status = impl_->DeleteTableFiles(table_id);
@ -366,11 +370,11 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) {
TEST_F(MySqlMetaTest, INDEX_TEST) {
auto table_id = "index_test";
meta::TableSchema table;
ms::engine::meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl_->CreateTable(table);
TableIndex index;
ms::engine::TableIndex index;
index.metric_type_ = 2;
index.nlist_ = 1234;
index.engine_type_ = 3;
@ -381,12 +385,12 @@ TEST_F(MySqlMetaTest, INDEX_TEST) {
status = impl_->UpdateTableFlag(table_id, flag);
ASSERT_TRUE(status.ok());
engine::meta::TableSchema table_info;
ms::engine::meta::TableSchema table_info;
table_info.table_id_ = table_id;
status = impl_->DescribeTable(table_info);
ASSERT_EQ(table_info.flag_, flag);
TableIndex index_out;
ms::engine::TableIndex index_out;
status = impl_->DescribeTableIndex(table_id, index_out);
ASSERT_EQ(index_out.metric_type_, index.metric_type_);
ASSERT_EQ(index_out.nlist_, index.nlist_);

182
cpp/unittest/db/search_test.cpp
View File

@ -22,53 +22,55 @@
#include "scheduler/task/SearchTask.h"
#include "utils/TimeRecorder.h"
using namespace zilliz::milvus;
namespace {
namespace ms = zilliz::milvus;
static constexpr uint64_t NQ = 15;
static constexpr uint64_t TOP_K = 64;
void BuildResult(uint64_t nq,
uint64_t topk,
bool ascending,
std::vector<long> &output_ids,
std::vector<float> &output_distence) {
void
BuildResult(uint64_t nq,
uint64_t topk,
bool ascending,
std::vector<int64_t> &output_ids,
std::vector<float> &output_distence) {
output_ids.clear();
output_ids.resize(nq*topk);
output_ids.resize(nq * topk);
output_distence.clear();
output_distence.resize(nq*topk);
output_distence.resize(nq * topk);
for(uint64_t i = 0; i < nq; i++) {
for(uint64_t j = 0; j < topk; j++) {
output_ids[i * topk + j] = (long)(drand48()*100000);
for (uint64_t i = 0; i < nq; i++) {
for (uint64_t j = 0; j < topk; j++) {
output_ids[i * topk + j] = (int64_t) (drand48() * 100000);
output_distence[i * topk + j] = ascending ? (j + drand48()) : ((topk - j) + drand48());
}
}
}
void CheckResult(const scheduler::Id2DistanceMap& src_1,
const scheduler::Id2DistanceMap& src_2,
const scheduler::Id2DistanceMap& target,
bool ascending) {
for(uint64_t i = 0; i < target.size() - 1; i++) {
if(ascending) {
void
CheckResult(const ms::scheduler::Id2DistanceMap &src_1,
const ms::scheduler::Id2DistanceMap &src_2,
const ms::scheduler::Id2DistanceMap &target,
bool ascending) {
for (uint64_t i = 0; i < target.size() - 1; i++) {
if (ascending) {
ASSERT_LE(target[i].second, target[i + 1].second);
} else {
ASSERT_GE(target[i].second, target[i + 1].second);
}
}
using ID2DistMap = std::map<long, float>;
using ID2DistMap = std::map<int64_t, float>;
ID2DistMap src_map_1, src_map_2;
for(const auto& pair : src_1) {
for (const auto &pair : src_1) {
src_map_1.insert(pair);
}
for(const auto& pair : src_2) {
for (const auto &pair : src_2) {
src_map_2.insert(pair);
}
for(const auto& pair : target) {
for (const auto &pair : target) {
ASSERT_TRUE(src_map_1.find(pair.first) != src_map_1.end() || src_map_2.find(pair.first) != src_map_2.end());
float dist = src_map_1.find(pair.first) != src_map_1.end() ? src_map_1[pair.first] : src_map_2[pair.first];
@ -76,40 +78,42 @@ void CheckResult(const scheduler::Id2DistanceMap& src_1,
}
}
void CheckCluster(const std::vector<long>& target_ids,
const std::vector<float>& target_distence,
const scheduler::ResultSet& src_result,
int64_t nq,
int64_t topk) {
void
CheckCluster(const std::vector<int64_t> &target_ids,
const std::vector<float> &target_distence,
const ms::scheduler::ResultSet &src_result,
int64_t nq,
int64_t topk) {
ASSERT_EQ(src_result.size(), nq);
for(int64_t i = 0; i < nq; i++) {
auto& res = src_result[i];
for (int64_t i = 0; i < nq; i++) {
auto &res = src_result[i];
ASSERT_EQ(res.size(), topk);
if(res.empty()) {
if (res.empty()) {
continue;
}
ASSERT_EQ(res[0].first, target_ids[i*topk]);
ASSERT_EQ(res[topk - 1].first, target_ids[i*topk + topk - 1]);
ASSERT_EQ(res[0].first, target_ids[i * topk]);
ASSERT_EQ(res[topk - 1].first, target_ids[i * topk + topk - 1]);
}
}
void CheckTopkResult(const scheduler::ResultSet& src_result,
bool ascending,
int64_t nq,
int64_t topk) {
void
CheckTopkResult(const ms::scheduler::ResultSet &src_result,
bool ascending,
int64_t nq,
int64_t topk) {
ASSERT_EQ(src_result.size(), nq);
for(int64_t i = 0; i < nq; i++) {
auto& res = src_result[i];
for (int64_t i = 0; i < nq; i++) {
auto &res = src_result[i];
ASSERT_EQ(res.size(), topk);
if(res.empty()) {
if (res.empty()) {
continue;
}
for(int64_t k = 0; k < topk - 1; k++) {
if(ascending) {
for (int64_t k = 0; k < topk - 1; k++) {
if (ascending) {
ASSERT_LE(res[k].second, res[k + 1].second);
} else {
ASSERT_GE(res[k].second, res[k + 1].second);
@ -118,87 +122,87 @@ void CheckTopkResult(const scheduler::ResultSet& src_result,
}
}
}
} // namespace
TEST(DBSearchTest, TOPK_TEST) {
bool ascending = true;
std::vector<long> target_ids;
std::vector<int64_t> target_ids;
std::vector<float> target_distence;
scheduler::ResultSet src_result;
auto status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result);
ms::scheduler::ResultSet src_result;
auto status = ms::scheduler::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result);
ASSERT_FALSE(status.ok());
ASSERT_TRUE(src_result.empty());
BuildResult(NQ, TOP_K, ascending, target_ids, target_distence);
status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result);
status = ms::scheduler::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result);
ASSERT_TRUE(status.ok());
ASSERT_EQ(src_result.size(), NQ);
scheduler::ResultSet target_result;
status = scheduler::XSearchTask::TopkResult(target_result, TOP_K, ascending, target_result);
ms::scheduler::ResultSet target_result;
status = ms::scheduler::XSearchTask::TopkResult(target_result, TOP_K, ascending, target_result);
ASSERT_TRUE(status.ok());
status = scheduler::XSearchTask::TopkResult(target_result, TOP_K, ascending, src_result);
status = ms::scheduler::XSearchTask::TopkResult(target_result, TOP_K, ascending, src_result);
ASSERT_FALSE(status.ok());
status = scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result);
status = ms::scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result);
ASSERT_TRUE(status.ok());
ASSERT_TRUE(src_result.empty());
ASSERT_EQ(target_result.size(), NQ);
std::vector<long> src_ids;
std::vector<int64_t> src_ids;
std::vector<float> src_distence;
uint64_t wrong_topk = TOP_K - 10;
BuildResult(NQ, wrong_topk, ascending, src_ids, src_distence);
status = scheduler::XSearchTask::ClusterResult(src_ids, src_distence, NQ, wrong_topk, src_result);
status = ms::scheduler::XSearchTask::ClusterResult(src_ids, src_distence, NQ, wrong_topk, src_result);
ASSERT_TRUE(status.ok());
status = scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result);
status = ms::scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result);
ASSERT_TRUE(status.ok());
for(uint64_t i = 0; i < NQ; i++) {
for (uint64_t i = 0; i < NQ; i++) {
ASSERT_EQ(target_result[i].size(), TOP_K);
}
wrong_topk = TOP_K + 10;
BuildResult(NQ, wrong_topk, ascending, src_ids, src_distence);
status = scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result);
status = ms::scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result);
ASSERT_TRUE(status.ok());
for(uint64_t i = 0; i < NQ; i++) {
for (uint64_t i = 0; i < NQ; i++) {
ASSERT_EQ(target_result[i].size(), TOP_K);
}
}
TEST(DBSearchTest, MERGE_TEST) {
bool ascending = true;
std::vector<long> target_ids;
std::vector<int64_t> target_ids;
std::vector<float> target_distence;
std::vector<long> src_ids;
std::vector<int64_t> src_ids;
std::vector<float> src_distence;
scheduler::ResultSet src_result, target_result;
ms::scheduler::ResultSet src_result, target_result;
uint64_t src_count = 5, target_count = 8;
BuildResult(1, src_count, ascending, src_ids, src_distence);
BuildResult(1, target_count, ascending, target_ids, target_distence);
auto status = scheduler::XSearchTask::ClusterResult(src_ids, src_distence, 1, src_count, src_result);
auto status = ms::scheduler::XSearchTask::ClusterResult(src_ids, src_distence, 1, src_count, src_result);
ASSERT_TRUE(status.ok());
status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, 1, target_count, target_result);
status = ms::scheduler::XSearchTask::ClusterResult(target_ids, target_distence, 1, target_count, target_result);
ASSERT_TRUE(status.ok());
{
scheduler::Id2DistanceMap src = src_result[0];
scheduler::Id2DistanceMap target = target_result[0];
status = scheduler::XSearchTask::MergeResult(src, target, 10, ascending);
ms::scheduler::Id2DistanceMap src = src_result[0];
ms::scheduler::Id2DistanceMap target = target_result[0];
status = ms::scheduler::XSearchTask::MergeResult(src, target, 10, ascending);
ASSERT_TRUE(status.ok());
ASSERT_EQ(target.size(), 10);
CheckResult(src_result[0], target_result[0], target, ascending);
}
{
scheduler::Id2DistanceMap src = src_result[0];
scheduler::Id2DistanceMap target;
status = scheduler::XSearchTask::MergeResult(src, target, 10, ascending);
ms::scheduler::Id2DistanceMap src = src_result[0];
ms::scheduler::Id2DistanceMap target;
status = ms::scheduler::XSearchTask::MergeResult(src, target, 10, ascending);
ASSERT_TRUE(status.ok());
ASSERT_EQ(target.size(), src_count);
ASSERT_TRUE(src.empty());
@ -206,18 +210,18 @@ TEST(DBSearchTest, MERGE_TEST) {
}
{
scheduler::Id2DistanceMap src = src_result[0];
scheduler::Id2DistanceMap target = target_result[0];
status = scheduler::XSearchTask::MergeResult(src, target, 30, ascending);
ms::scheduler::Id2DistanceMap src = src_result[0];
ms::scheduler::Id2DistanceMap target = target_result[0];
status = ms::scheduler::XSearchTask::MergeResult(src, target, 30, ascending);
ASSERT_TRUE(status.ok());
ASSERT_EQ(target.size(), src_count + target_count);
CheckResult(src_result[0], target_result[0], target, ascending);
}
{
scheduler::Id2DistanceMap target = src_result[0];
scheduler::Id2DistanceMap src = target_result[0];
status = scheduler::XSearchTask::MergeResult(src, target, 30, ascending);
ms::scheduler::Id2DistanceMap target = src_result[0];
ms::scheduler::Id2DistanceMap src = target_result[0];
status = ms::scheduler::XSearchTask::MergeResult(src, target, 30, ascending);
ASSERT_TRUE(status.ok());
ASSERT_EQ(target.size(), src_count + target_count);
CheckResult(src_result[0], target_result[0], target, ascending);
@ -226,17 +230,17 @@ TEST(DBSearchTest, MERGE_TEST) {
TEST(DBSearchTest, PARALLEL_CLUSTER_TEST) {
bool ascending = true;
std::vector<long> target_ids;
std::vector<int64_t> target_ids;
std::vector<float> target_distence;
scheduler::ResultSet src_result;
ms::scheduler::ResultSet src_result;
auto DoCluster = [&](int64_t nq, int64_t topk) {
TimeRecorder rc("DoCluster");
ms::TimeRecorder rc("DoCluster");
src_result.clear();
BuildResult(nq, topk, ascending, target_ids, target_distence);
rc.RecordSection("build id/dietance map");
auto status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result);
auto status = ms::scheduler::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result);
ASSERT_TRUE(status.ok());
ASSERT_EQ(src_result.size(), nq);
@ -257,29 +261,33 @@ TEST(DBSearchTest, PARALLEL_CLUSTER_TEST) {
}
TEST(DBSearchTest, PARALLEL_TOPK_TEST) {
std::vector<long> target_ids;
std::vector<int64_t> target_ids;
std::vector<float> target_distence;
scheduler::ResultSet src_result;
ms::scheduler::ResultSet src_result;
std::vector<long> insufficient_ids;
std::vector<int64_t> insufficient_ids;
std::vector<float> insufficient_distence;
scheduler::ResultSet insufficient_result;
ms::scheduler::ResultSet insufficient_result;
auto DoTopk = [&](int64_t nq, int64_t topk,int64_t insufficient_topk, bool ascending) {
auto DoTopk = [&](int64_t nq, int64_t topk, int64_t insufficient_topk, bool ascending) {
src_result.clear();
insufficient_result.clear();
TimeRecorder rc("DoCluster");
ms::TimeRecorder rc("DoCluster");
BuildResult(nq, topk, ascending, target_ids, target_distence);
auto status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result);
auto status = ms::scheduler::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result);
rc.RecordSection("cluster result");
BuildResult(nq, insufficient_topk, ascending, insufficient_ids, insufficient_distence);
status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, nq, insufficient_topk, insufficient_result);
status = ms::scheduler::XSearchTask::ClusterResult(target_ids,
target_distence,
nq,
insufficient_topk,
insufficient_result);
rc.RecordSection("cluster result");
scheduler::XSearchTask::TopkResult(insufficient_result, topk, ascending, src_result);
ms::scheduler::XSearchTask::TopkResult(insufficient_result, topk, ascending, src_result);
ASSERT_TRUE(status.ok());
rc.RecordSection("topk");
@ -292,4 +300,4 @@ TEST(DBSearchTest, PARALLEL_TOPK_TEST) {
// DoTopk(9987, 12, 10, false);
// DoTopk(77777, 1000, 1, false);
// DoTopk(5432, 8899, 8899, true);
}
}

132
cpp/unittest/db/utils.cpp
View File

@ -18,9 +18,11 @@
#include <iostream>
#include <thread>
#include <memory>
#include <string>
#include <boost/filesystem.hpp>
#include "utils.h"
#include "db/utils.h"
#include "cache/GpuCacheMgr.h"
#include "cache/CpuCacheMgr.h"
#include "db/DBFactory.h"
@ -29,83 +31,98 @@
INITIALIZE_EASYLOGGINGPP
using namespace zilliz::milvus;
namespace {
static std::string uri;
namespace ms = zilliz::milvus;
class DBTestEnvironment : public ::testing::Environment {
public:
public:
explicit DBTestEnvironment(const std::string& uri)
: uri_(uri) {
}
// explicit DBTestEnvironment(std::string uri) : uri_(uri) {}
static std::string getURI() {
return uri;
std::string getURI() const {
return uri_;
}
void SetUp() override {
getURI();
}
private:
std::string uri_;
};
DBTestEnvironment* test_env = nullptr;
} // namespace
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void BaseTest::InitLog() {
void
BaseTest::InitLog() {
el::Configurations defaultConf;
defaultConf.setToDefault();
defaultConf.set(el::Level::Debug,
el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)");
el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)");
el::Loggers::reconfigureLogger("default", defaultConf);
}
void BaseTest::SetUp() {
void
BaseTest::SetUp() {
InitLog();
zilliz::knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(0, 1024*1024*200, 1024*1024*300, 2);
zilliz::knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(0, 1024 * 1024 * 200, 1024 * 1024 * 300, 2);
}
void BaseTest::TearDown() {
void
BaseTest::TearDown() {
zilliz::milvus::cache::CpuCacheMgr::GetInstance()->ClearCache();
zilliz::milvus::cache::GpuCacheMgr::GetInstance(0)->ClearCache();
zilliz::knowhere::FaissGpuResourceMgr::GetInstance().Free();
}
engine::DBOptions BaseTest::GetOptions() {
auto options = engine::DBFactory::BuildOption();
ms::engine::DBOptions
BaseTest::GetOptions() {
auto options = ms::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.backend_uri_ = "sqlite://:@:/";
return options;
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void DBTest::SetUp() {
void
DBTest::SetUp() {
BaseTest::SetUp();
auto res_mgr = scheduler::ResMgrInst::GetInstance();
auto res_mgr = ms::scheduler::ResMgrInst::GetInstance();
res_mgr->Clear();
res_mgr->Add(scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false));
res_mgr->Add(scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, false));
res_mgr->Add(scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true));
res_mgr->Add(ms::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false));
res_mgr->Add(ms::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, false));
res_mgr->Add(ms::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true));
auto default_conn = scheduler::Connection("IO", 500.0);
auto PCIE = scheduler::Connection("IO", 11000.0);
auto default_conn = ms::scheduler::Connection("IO", 500.0);
auto PCIE = ms::scheduler::Connection("IO", 11000.0);
res_mgr->Connect("disk", "cpu", default_conn);
res_mgr->Connect("cpu", "gtx1660", PCIE);
res_mgr->Start();
scheduler::SchedInst::GetInstance()->Start();
ms::scheduler::SchedInst::GetInstance()->Start();
scheduler::JobMgrInst::GetInstance()->Start();
ms::scheduler::JobMgrInst::GetInstance()->Start();
auto options = GetOptions();
db_ = engine::DBFactory::Build(options);
db_ = ms::engine::DBFactory::Build(options);
}
void DBTest::TearDown() {
void
DBTest::TearDown() {
db_->Stop();
db_->DropAll();
scheduler::JobMgrInst::GetInstance()->Stop();
scheduler::SchedInst::GetInstance()->Stop();
scheduler::ResMgrInst::GetInstance()->Stop();
ms::scheduler::JobMgrInst::GetInstance()->Stop();
ms::scheduler::SchedInst::GetInstance()->Stop();
ms::scheduler::ResMgrInst::GetInstance()->Stop();
BaseTest::TearDown();
@ -114,23 +131,26 @@ void DBTest::TearDown() {
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
engine::DBOptions DBTest2::GetOptions() {
auto options = engine::DBFactory::BuildOption();
ms::engine::DBOptions
DBTest2::GetOptions() {
auto options = ms::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.archive_conf_ = engine::ArchiveConf("delete", "disk:1");
options.meta_.archive_conf_ = ms::engine::ArchiveConf("delete", "disk:1");
options.meta_.backend_uri_ = "sqlite://:@:/";
return options;
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void MetaTest::SetUp() {
void
MetaTest::SetUp() {
BaseTest::SetUp();
auto options = GetOptions();
impl_ = std::make_shared<engine::meta::SqliteMetaImpl>(options.meta_);
impl_ = std::make_shared<ms::engine::meta::SqliteMetaImpl>(options.meta_);
}
void MetaTest::TearDown() {
void
MetaTest::TearDown() {
impl_->DropAll();
BaseTest::TearDown();
@ -140,27 +160,26 @@ void MetaTest::TearDown() {
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
engine::DBOptions MySqlDBTest::GetOptions() {
auto options = engine::DBFactory::BuildOption();
ms::engine::DBOptions
MySqlDBTest::GetOptions() {
auto options = ms::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.backend_uri_ = DBTestEnvironment::getURI();
if(options.meta_.backend_uri_.empty()) {
options.meta_.backend_uri_ = "mysql://root:Fantast1c@192.168.1.194:3306/";
}
options.meta_.backend_uri_ = test_env->getURI();
return options;
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void MySqlMetaTest::SetUp() {
void
MySqlMetaTest::SetUp() {
BaseTest::SetUp();
auto options = GetOptions();
impl_ = std::make_shared<engine::meta::MySQLMetaImpl>(options.meta_, options.mode_);
impl_ = std::make_shared<ms::engine::meta::MySQLMetaImpl>(options.meta_, options.mode_);
}
void MySqlMetaTest::TearDown() {
void
MySqlMetaTest::TearDown() {
impl_->DropAll();
BaseTest::TearDown();
@ -169,29 +188,26 @@ void MySqlMetaTest::TearDown() {
boost::filesystem::remove_all(options.meta_.path_);
}
engine::DBOptions MySqlMetaTest::GetOptions() {
auto options = engine::DBFactory::BuildOption();
ms::engine::DBOptions
MySqlMetaTest::GetOptions() {
auto options = ms::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.backend_uri_ = DBTestEnvironment::getURI();
if(options.meta_.backend_uri_.empty()) {
options.meta_.backend_uri_ = "mysql://root:Fantast1c@192.168.1.194:3306/";
}
options.meta_.backend_uri_ = test_env->getURI();
return options;
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
int main(int argc, char **argv) {
int
main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
std::string uri;
if (argc > 1) {
uri = argv[1];
}
// if(uri.empty()) {
// uri = "mysql://root:Fantast1c@192.168.1.194:3306/";
// }
// std::cout << uri << std::endl;
::testing::AddGlobalTestEnvironment(new DBTestEnvironment);
test_env = new DBTestEnvironment(uri);
::testing::AddGlobalTestEnvironment(test_env);
return RUN_ALL_TESTS();
}

33
cpp/unittest/db/utils.h
View File

@ -20,7 +20,7 @@
#include <gtest/gtest.h>
#include <chrono>
//#include <src/db/MySQLMetaImpl.h>
#include <memory>
#include "db/DB.h"
#include "db/meta/SqliteMetaImpl.h"
@ -28,7 +28,6 @@
#include "scheduler/SchedInst.h"
#include "scheduler/ResourceFactory.h"
#define TIMING
#ifdef TIMING
@ -36,8 +35,7 @@
#define START_TIMER start = std::chrono::high_resolution_clock::now();
#define STOP_TIMER(name) LOG(DEBUG) << "RUNTIME of " << name << ": " << \
std::chrono::duration_cast<std::chrono::milliseconds>( \
std::chrono::high_resolution_clock::now()-start \
).count() << " ms ";
std::chrono::high_resolution_clock::now()-start).count() << " ms ";
#else
#define INIT_TIMER
#define START_TIMER
@ -45,11 +43,11 @@
#endif
class BaseTest : public ::testing::Test {
protected:
protected:
void InitLog();
virtual void SetUp() override;
virtual void TearDown() override;
void SetUp() override;
void TearDown() override;
virtual zilliz::milvus::engine::DBOptions GetOptions();
};
@ -58,14 +56,14 @@ class DBTest : public BaseTest {
protected:
zilliz::milvus::engine::DBPtr db_;
virtual void SetUp() override;
virtual void TearDown() override;
void SetUp() override;
void TearDown() override;
};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class DBTest2 : public DBTest {
protected:
virtual zilliz::milvus::engine::DBOptions GetOptions() override;
zilliz::milvus::engine::DBOptions GetOptions() override;
};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -77,14 +75,14 @@ class MetaTest : public BaseTest {
protected:
std::shared_ptr<zilliz::milvus::engine::meta::SqliteMetaImpl> impl_;
virtual void SetUp() override;
virtual void TearDown() override;
void SetUp() override;
void TearDown() override;
};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class MySqlDBTest : public DBTest {
protected:
zilliz::milvus::engine::DBOptions GetOptions();
protected:
zilliz::milvus::engine::DBOptions GetOptions() override;
};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -92,12 +90,11 @@ class MySqlMetaTest : public BaseTest {
protected:
std::shared_ptr<zilliz::milvus::engine::meta::MySQLMetaImpl> impl_;
virtual void SetUp() override;
virtual void TearDown() override;
zilliz::milvus::engine::DBOptions GetOptions();
void SetUp() override;
void TearDown() override;
zilliz::milvus::engine::DBOptions GetOptions() override;
};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class MemManagerTest : public MetaTest {
};

6
cpp/unittest/main.cpp
View File

@ -19,13 +19,11 @@
#include <gmock/gmock.h>
#include "utils/easylogging++.h"
#include "utils/CommonUtil.h"
INITIALIZE_EASYLOGGINGPP
using namespace zilliz::milvus;
int main(int argc, char **argv) {
int
main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

16
cpp/unittest/metrics/metricbase_test.cpp
View File

@ -21,12 +21,16 @@
#include <gtest/gtest.h>
#include <iostream>
using namespace zilliz::milvus;
namespace {
TEST(MetricbaseTest, METRICBASE_TEST){
server::MetricsBase instance = server::MetricsBase::GetInstance();
namespace ms = zilliz::milvus;
} // namespace
TEST(MetricbaseTest, METRICBASE_TEST) {
ms::server::MetricsBase instance = ms::server::MetricsBase::GetInstance();
instance.Init();
server::SystemInfo::GetInstance().Init();
ms::server::SystemInfo::GetInstance().Init();
instance.AddVectorsSuccessTotalIncrement();
instance.AddVectorsFailTotalIncrement();
instance.AddVectorsDurationHistogramOberve(1.0);
@ -60,10 +64,10 @@ TEST(MetricbaseTest, METRICBASE_TEST){
instance.QueryResponsePerSecondGaugeSet(1.0);
instance.GPUPercentGaugeSet();
instance.GPUMemoryUsageGaugeSet();
instance.AddVectorsPerSecondGaugeSet(1,1,1);
instance.AddVectorsPerSecondGaugeSet(1, 1, 1);
instance.QueryIndexTypePerSecondSet("IVF", 1.0);
instance.ConnectionGaugeIncrement();
instance.ConnectionGaugeDecrement();
instance.KeepingAliveCounterIncrement();
instance.OctetsSet();
}
}

104
cpp/unittest/metrics/metrics_test.cpp
View File

@ -15,75 +15,73 @@
// specific language governing permissions and limitations
// under the License.
#include <chrono>
#include <chrono>
#include <map>
#include <memory>
#include <string>
#include <thread>
#include <gtest/gtest.h>
//#include "prometheus/registry.h"
//#include "prometheus/exposer.h"
#include <cache/CpuCacheMgr.h>
#include <src/server/Config.h>
#include "cache/CpuCacheMgr.h"
#include "server/Config.h"
#include "metrics/Metrics.h"
#include "utils.h"
#include "metrics/utils.h"
#include "db/DB.h"
#include "db/meta/SqliteMetaImpl.h"
namespace {
using namespace zilliz::milvus;
namespace ms = zilliz::milvus;
} // namespace
TEST_F(MetricTest, METRIC_TEST) {
server::Config::GetInstance().SetMetricConfigCollector("zabbix");
server::Metrics::GetInstance();
server::Config::GetInstance().SetMetricConfigCollector("prometheus");
server::Metrics::GetInstance();
ms::server::Config::GetInstance().SetMetricConfigCollector("zabbix");
ms::server::Metrics::GetInstance();
ms::server::Config::GetInstance().SetMetricConfigCollector("prometheus");
ms::server::Metrics::GetInstance();
server::SystemInfo::GetInstance().Init();
ms::server::SystemInfo::GetInstance().Init();
// server::Metrics::GetInstance().Init();
// server::Metrics::GetInstance().exposer_ptr()->RegisterCollectable(server::Metrics::GetInstance().registry_ptr());
server::Metrics::GetInstance().Init();
ms::server::Metrics::GetInstance().Init();
// server::PrometheusMetrics::GetInstance().exposer_ptr()->RegisterCollectable(server::PrometheusMetrics::GetInstance().registry_ptr());
zilliz::milvus::cache::CpuCacheMgr::GetInstance()->SetCapacity(1UL*1024*1024*1024);
std::cout<<zilliz::milvus::cache::CpuCacheMgr::GetInstance()->CacheCapacity()<<std::endl;
zilliz::milvus::cache::CpuCacheMgr::GetInstance()->SetCapacity(1UL * 1024 * 1024 * 1024);
std::cout << zilliz::milvus::cache::CpuCacheMgr::GetInstance()->CacheCapacity() << std::endl;
static const char* group_name = "test_group";
static const char *group_name = "test_group";
static const int group_dim = 256;
engine::meta::TableSchema group_info;
ms::engine::meta::TableSchema group_info;
group_info.dimension_ = group_dim;
group_info.table_id_ = group_name;
auto stat = db_->CreateTable(group_info);
engine::meta::TableSchema group_info_get;
ms::engine::meta::TableSchema group_info_get;
group_info_get.table_id_ = group_name;
stat = db_->DescribeTable(group_info_get);
engine::IDNumbers vector_ids;
engine::IDNumbers target_ids;
ms::engine::IDNumbers vector_ids;
ms::engine::IDNumbers target_ids;
int d = 256;
int nb = 50;
float *xb = new float[d * nb];
for(int i = 0; i < nb; i++) {
for(int j = 0; j < d; j++) xb[d * i + j] = drand48();
for (int i = 0; i < nb; i++) {
for (int j = 0; j < d; j++) xb[d * i + j] = drand48();
xb[d * i] += i / 2000.;
}
int qb = 5;
float *qxb = new float[d * qb];
for(int i = 0; i < qb; i++) {
for(int j = 0; j < d; j++) qxb[d * i + j] = drand48();
for (int i = 0; i < qb; i++) {
for (int j = 0; j < d; j++) qxb[d * i + j] = drand48();
qxb[d * i] += i / 2000.;
}
std::thread search([&]() {
engine::QueryResults results;
ms::engine::QueryResults results;
int k = 10;
std::this_thread::sleep_for(std::chrono::seconds(2));
@ -92,23 +90,23 @@ TEST_F(MetricTest, METRIC_TEST) {
uint64_t count = 0;
uint64_t prev_count = 0;
for (auto j=0; j<10; ++j) {
for (auto j = 0; j < 10; ++j) {
ss.str("");
db_->Size(count);
prev_count = count;
START_TIMER;
// stat = db_->Query(group_name, k, qb, qxb, results);
ss << "Search " << j << " With Size " << (float)(count*group_dim*sizeof(float))/(1024*1024) << " M";
ss << "Search " << j << " With Size " << (float) (count * group_dim * sizeof(float)) / (1024 * 1024)
<< " M";
for (auto k=0; k<qb; ++k) {
for (auto k = 0; k < qb; ++k) {
// ASSERT_EQ(results[k][0].first, target_ids[k]);
ss.str("");
ss << "Result [" << k << "]:";
// for (auto result : results[k]) {
// ss << result.first << " ";
// }
}
ASSERT_TRUE(count >= prev_count);
std::this_thread::sleep_for(std::chrono::seconds(1));
@ -117,8 +115,8 @@ TEST_F(MetricTest, METRIC_TEST) {
int loop = 10000;
for (auto i=0; i<loop; ++i) {
if (i==40) {
for (auto i = 0; i < loop; ++i) {
if (i == 40) {
db_->InsertVectors(group_name, qb, qxb, target_ids);
ASSERT_EQ(target_ids.size(), qb);
} else {
@ -129,37 +127,37 @@ TEST_F(MetricTest, METRIC_TEST) {
search.join();
delete [] xb;
delete [] qxb;
};
delete[] xb;
delete[] qxb;
}
TEST_F(MetricTest, COLLECTOR_METRICS_TEST){
auto status = Status::OK();
server::CollectInsertMetrics insert_metrics0(0, status);
status = Status(DB_ERROR, "error");
server::CollectInsertMetrics insert_metrics1(0, status);
TEST_F(MetricTest, COLLECTOR_METRICS_TEST) {
auto status = ms::Status::OK();
ms::server::CollectInsertMetrics insert_metrics0(0, status);
status = ms::Status(ms::DB_ERROR, "error");
ms::server::CollectInsertMetrics insert_metrics1(0, status);
server::CollectQueryMetrics query_metrics(10);
ms::server::CollectQueryMetrics query_metrics(10);
server::CollectMergeFilesMetrics merge_metrics();
ms::server::CollectMergeFilesMetrics merge_metrics();
server::CollectBuildIndexMetrics build_index_metrics();
ms::server::CollectBuildIndexMetrics build_index_metrics();
server::CollectExecutionEngineMetrics execution_metrics(10);
ms::server::CollectExecutionEngineMetrics execution_metrics(10);
server::CollectSerializeMetrics serialize_metrics(10);
ms::server::CollectSerializeMetrics serialize_metrics(10);
server::CollectAddMetrics add_metrics(10, 128);
ms::server::CollectAddMetrics add_metrics(10, 128);
server::CollectDurationMetrics duration_metrics_raw(engine::meta::TableFileSchema::RAW);
server::CollectDurationMetrics duration_metrics_index(engine::meta::TableFileSchema::TO_INDEX);
server::CollectDurationMetrics duration_metrics_delete(engine::meta::TableFileSchema::TO_DELETE);
ms::server::CollectDurationMetrics duration_metrics_raw(ms::engine::meta::TableFileSchema::RAW);
ms::server::CollectDurationMetrics duration_metrics_index(ms::engine::meta::TableFileSchema::TO_INDEX);
ms::server::CollectDurationMetrics duration_metrics_delete(ms::engine::meta::TableFileSchema::TO_DELETE);
server::CollectSearchTaskMetrics search_metrics_raw(engine::meta::TableFileSchema::RAW);
server::CollectSearchTaskMetrics search_metrics_index(engine::meta::TableFileSchema::TO_INDEX);
server::CollectSearchTaskMetrics search_metrics_delete(engine::meta::TableFileSchema::TO_DELETE);
ms::server::CollectSearchTaskMetrics search_metrics_raw(ms::engine::meta::TableFileSchema::RAW);
ms::server::CollectSearchTaskMetrics search_metrics_index(ms::engine::meta::TableFileSchema::TO_INDEX);
ms::server::CollectSearchTaskMetrics search_metrics_delete(ms::engine::meta::TableFileSchema::TO_DELETE);
server::MetricCollector metric_collector();
ms::server::MetricCollector metric_collector();
}

21
cpp/unittest/metrics/prometheus_test.cpp
View File

@ -22,15 +22,19 @@
#include <gtest/gtest.h>
#include <iostream>
using namespace zilliz::milvus;
namespace {
TEST(PrometheusTest, PROMETHEUS_TEST){
server::Config::GetInstance().SetMetricConfigEnableMonitor("on");
namespace ms = zilliz::milvus;
server::PrometheusMetrics instance = server::PrometheusMetrics::GetInstance();
} // namespace
TEST(PrometheusTest, PROMETHEUS_TEST) {
ms::server::Config::GetInstance().SetMetricConfigEnableMonitor("on");
ms::server::PrometheusMetrics instance = ms::server::PrometheusMetrics::GetInstance();
instance.Init();
instance.SetStartup(true);
server::SystemInfo::GetInstance().Init();
ms::server::SystemInfo::GetInstance().Init();
instance.AddVectorsSuccessTotalIncrement();
instance.AddVectorsFailTotalIncrement();
instance.AddVectorsDurationHistogramOberve(1.0);
@ -64,7 +68,7 @@ TEST(PrometheusTest, PROMETHEUS_TEST){
instance.QueryResponsePerSecondGaugeSet(1.0);
instance.GPUPercentGaugeSet();
instance.GPUMemoryUsageGaugeSet();
instance.AddVectorsPerSecondGaugeSet(1,1,1);
instance.AddVectorsPerSecondGaugeSet(1, 1, 1);
instance.QueryIndexTypePerSecondSet("IVF", 1.0);
instance.QueryIndexTypePerSecondSet("IDMap", 1.0);
instance.ConnectionGaugeIncrement();
@ -76,10 +80,9 @@ TEST(PrometheusTest, PROMETHEUS_TEST){
instance.GPUTemperature();
instance.CPUTemperature();
server::Config::GetInstance().SetMetricConfigEnableMonitor("off");
ms::server::Config::GetInstance().SetMetricConfigEnableMonitor("off");
instance.Init();
instance.CPUCoreUsagePercentSet();
instance.GPUTemperature();
instance.CPUTemperature();
}
}

34
cpp/unittest/metrics/utils.cpp
View File

@ -18,32 +18,38 @@
#include <iostream>
#include <thread>
#include <string>
#include <boost/filesystem.hpp>
#include "utils.h"
#include "metrics/utils.h"
#include "db/DBFactory.h"
INITIALIZE_EASYLOGGINGPP
using namespace zilliz::milvus;
namespace {
static std::string uri;
namespace ms = zilliz::milvus;
class DBTestEnvironment : public ::testing::Environment {
public:
public:
explicit DBTestEnvironment(const std::string& uri) : uri_(uri) {}
// explicit DBTestEnvironment(std::string uri) : uri_(uri) {}
static std::string getURI() {
return uri;
std::string getURI() const {
return uri_;
}
void SetUp() override {
getURI();
}
private:
std::string uri_;
};
DBTestEnvironment* test_env = nullptr;
} // namespace
void MetricTest::InitLog() {
el::Configurations defaultConf;
defaultConf.setToDefault();
@ -52,8 +58,8 @@ void MetricTest::InitLog() {
el::Loggers::reconfigureLogger("default", defaultConf);
}
engine::DBOptions MetricTest::GetOptions() {
auto options = engine::DBFactory::BuildOption();
ms::engine::DBOptions MetricTest::GetOptions() {
auto options = ms::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.backend_uri_ = "sqlite://:@:/";
return options;
@ -62,7 +68,7 @@ engine::DBOptions MetricTest::GetOptions() {
void MetricTest::SetUp() {
InitLog();
auto options = GetOptions();
db_ = engine::DBFactory::Build(options);
db_ = ms::engine::DBFactory::Build(options);
}
void MetricTest::TearDown() {
@ -72,10 +78,12 @@ void MetricTest::TearDown() {
int main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
std::string uri;
if (argc > 1) {
uri = argv[1];
}
// std::cout << uri << std::endl;
::testing::AddGlobalTestEnvironment(new DBTestEnvironment);
test_env = new DBTestEnvironment(uri);
::testing::AddGlobalTestEnvironment(test_env);
return RUN_ALL_TESTS();
}

15
cpp/unittest/metrics/utils.h
View File

@ -26,7 +26,6 @@
#include "db/meta/SqliteMetaImpl.h"
#include "db/meta/MySQLMetaImpl.h"
#define TIMING
#ifdef TIMING
@ -34,15 +33,15 @@
#define START_TIMER start = std::chrono::high_resolution_clock::now();
#define STOP_TIMER(name) LOG(DEBUG) << "RUNTIME of " << name << ": " << \
std::chrono::duration_cast<std::chrono::milliseconds>( \
std::chrono::high_resolution_clock::now()-start \
).count() << " ms ";
std::chrono::high_resolution_clock::now()-start).count() << " ms ";
#else
#define INIT_TIMER
#define START_TIMER
#define STOP_TIMER(name)
#endif
void ASSERT_STATS(zilliz::milvus::Status& stat);
void
ASSERT_STATS(zilliz::milvus::Status &stat);
//class TestEnv : public ::testing::Environment {
//public:
@ -66,11 +65,11 @@ void ASSERT_STATS(zilliz::milvus::Status& stat);
// ::testing::AddGlobalTestEnvironment(new TestEnv);
class MetricTest : public ::testing::Test {
protected:
protected:
zilliz::milvus::engine::DBPtr db_;
void InitLog();
virtual void SetUp() override;
virtual void TearDown() override;
void SetUp() override;
void TearDown() override;
virtual zilliz::milvus::engine::DBOptions GetOptions();
};
};

9
cpp/unittest/scheduler/algorithm_test.cpp
View File

@ -101,11 +101,8 @@ TEST_F(AlgorithmTest, SHORTESTPATH_TEST) {
std::cout << sp[sp.size() - 1] << std::endl;
sp.pop_back();
}
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

6
cpp/unittest/scheduler/event_test.cpp
View File

@ -60,7 +60,7 @@ TEST(EventTest, TASKTABLE_UPDATED_EVENT) {
std::cout << *EventPtr(event);
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

34
cpp/unittest/scheduler/node_test.cpp
View File

@ -19,37 +19,40 @@
#include "scheduler/resource/Node.h"
#include <gtest/gtest.h>
namespace {
using namespace zilliz::milvus::scheduler;
namespace ms = zilliz::milvus::scheduler;
} // namespace
class NodeTest : public ::testing::Test {
protected:
protected:
void
SetUp() override {
node1_ = std::make_shared<Node>();
node2_ = std::make_shared<Node>();
node3_ = std::make_shared<Node>();
isolated_node1_ = std::make_shared<Node>();
isolated_node2_ = std::make_shared<Node>();
node1_ = std::make_shared<ms::Node>();
node2_ = std::make_shared<ms::Node>();
node3_ = std::make_shared<ms::Node>();
isolated_node1_ = std::make_shared<ms::Node>();
isolated_node2_ = std::make_shared<ms::Node>();
auto pcie = Connection("PCIe", 11.0);
auto pcie = ms::Connection("PCIe", 11.0);
node1_->AddNeighbour(node2_, pcie);
node1_->AddNeighbour(node3_, pcie);
node2_->AddNeighbour(node1_, pcie);
}
NodePtr node1_;
NodePtr node2_;
NodePtr node3_;
NodePtr isolated_node1_;
NodePtr isolated_node2_;
ms::NodePtr node1_;
ms::NodePtr node2_;
ms::NodePtr node3_;
ms::NodePtr isolated_node1_;
ms::NodePtr isolated_node2_;
};
TEST_F(NodeTest, ADD_NEIGHBOUR) {
ASSERT_EQ(isolated_node1_->GetNeighbours().size(), 0);
ASSERT_EQ(isolated_node2_->GetNeighbours().size(), 0);
auto pcie = Connection("PCIe", 11.0);
auto pcie = ms::Connection("PCIe", 11.0);
isolated_node1_->AddNeighbour(isolated_node2_, pcie);
ASSERT_EQ(isolated_node1_->GetNeighbours().size(), 1);
ASSERT_EQ(isolated_node2_->GetNeighbours().size(), 0);
@ -58,7 +61,7 @@ TEST_F(NodeTest, ADD_NEIGHBOUR) {
TEST_F(NodeTest, REPEAT_ADD_NEIGHBOUR) {
ASSERT_EQ(isolated_node1_->GetNeighbours().size(), 0);
ASSERT_EQ(isolated_node2_->GetNeighbours().size(), 0);
auto pcie = Connection("PCIe", 11.0);
auto pcie = ms::Connection("PCIe", 11.0);
isolated_node1_->AddNeighbour(isolated_node2_, pcie);
isolated_node1_->AddNeighbour(isolated_node2_, pcie);
ASSERT_EQ(isolated_node1_->GetNeighbours().size(), 1);
@ -97,3 +100,4 @@ TEST_F(NodeTest, DUMP) {
std::cout << node2_->Dump();
ASSERT_FALSE(node2_->Dump().empty());
}

23
cpp/unittest/scheduler/normal_test.cpp
View File

@ -25,35 +25,37 @@
#include "utils/Log.h"
#include <gtest/gtest.h>
namespace {
using namespace zilliz::milvus::scheduler;
namespace ms = zilliz::milvus::scheduler;
} // namespace
TEST(NormalTest, INST_TEST) {
// ResourceMgr only compose resources, provide unified event
auto res_mgr = ResMgrInst::GetInstance();
auto res_mgr = ms::ResMgrInst::GetInstance();
res_mgr->Add(ResourceFactory::Create("disk", "DISK", 0, true, false));
res_mgr->Add(ResourceFactory::Create("cpu", "CPU", 0, true, true));
res_mgr->Add(ms::ResourceFactory::Create("disk", "DISK", 0, true, false));
res_mgr->Add(ms::ResourceFactory::Create("cpu", "CPU", 0, true, true));
auto IO = Connection("IO", 500.0);
auto IO = ms::Connection("IO", 500.0);
res_mgr->Connect("disk", "cpu", IO);
auto scheduler = SchedInst::GetInstance();
auto scheduler = ms::SchedInst::GetInstance();
res_mgr->Start();
scheduler->Start();
const uint64_t NUM_TASK = 1000;
std::vector<std::shared_ptr<TestTask>> tasks;
TableFileSchemaPtr dummy = nullptr;
std::vector<std::shared_ptr<ms::TestTask>> tasks;
ms::TableFileSchemaPtr dummy = nullptr;
auto disks = res_mgr->GetDiskResources();
ASSERT_FALSE(disks.empty());
if (auto observe = disks[0].lock()) {
for (uint64_t i = 0; i < NUM_TASK; ++i) {
auto task = std::make_shared<TestTask>(dummy);
task->label() = std::make_shared<DefaultLabel>();
auto task = std::make_shared<ms::TestTask>(dummy);
task->label() = std::make_shared<ms::DefaultLabel>();
tasks.push_back(task);
observe->task_table().Put(task);
}
@ -67,5 +69,4 @@ TEST(NormalTest, INST_TEST) {
scheduler->Stop();
res_mgr->Stop();
}

17
cpp/unittest/scheduler/resource_factory_test.cpp
View File

@ -19,15 +19,18 @@
#include "scheduler/ResourceFactory.h"
#include <gtest/gtest.h>
namespace {
using namespace zilliz::milvus::scheduler;
namespace ms = zilliz::milvus::scheduler;
} // namespace
TEST(ResourceFactoryTest, CREATE) {
auto disk = ResourceFactory::Create("ssd", "DISK", 0);
auto cpu = ResourceFactory::Create("cpu", "CPU", 0);
auto gpu = ResourceFactory::Create("gpu", "GPU", 0);
auto disk = ms::ResourceFactory::Create("ssd", "DISK", 0);
auto cpu = ms::ResourceFactory::Create("cpu", "CPU", 0);
auto gpu = ms::ResourceFactory::Create("gpu", "GPU", 0);
ASSERT_TRUE(std::dynamic_pointer_cast<DiskResource>(disk));
ASSERT_TRUE(std::dynamic_pointer_cast<CpuResource>(cpu));
ASSERT_TRUE(std::dynamic_pointer_cast<GpuResource>(gpu));
ASSERT_TRUE(std::dynamic_pointer_cast<ms::DiskResource>(disk));
ASSERT_TRUE(std::dynamic_pointer_cast<ms::CpuResource>(cpu));
ASSERT_TRUE(std::dynamic_pointer_cast<ms::GpuResource>(gpu));
}

15
cpp/unittest/scheduler/resource_mgr_test.cpp
View File

@ -24,15 +24,13 @@
#include "scheduler/ResourceMgr.h"
#include <gtest/gtest.h>
namespace zilliz {
namespace milvus {
namespace scheduler {
/************ ResourceMgrBaseTest ************/
class ResourceMgrBaseTest : public testing::Test {
protected:
protected:
void
SetUp() override {
empty_mgr_ = std::make_shared<ResourceMgr>();
@ -77,7 +75,6 @@ TEST_F(ResourceMgrBaseTest, CONNECT) {
ASSERT_TRUE(empty_mgr_->Connect("resource1", "resource2", io));
}
TEST_F(ResourceMgrBaseTest, INVALID_CONNECT) {
auto resource1 = std::make_shared<TestResource>("resource1", 0, true, true);
auto resource2 = std::make_shared<TestResource>("resource2", 2, true, true);
@ -87,7 +84,6 @@ TEST_F(ResourceMgrBaseTest, INVALID_CONNECT) {
ASSERT_FALSE(empty_mgr_->Connect("xx", "yy", io));
}
TEST_F(ResourceMgrBaseTest, CLEAR) {
ASSERT_EQ(mgr1_->GetNumOfResource(), 3);
mgr1_->Clear();
@ -163,7 +159,7 @@ TEST_F(ResourceMgrBaseTest, DUMP_TASKTABLES) {
/************ ResourceMgrAdvanceTest ************/
class ResourceMgrAdvanceTest : public testing::Test {
protected:
protected:
void
SetUp() override {
mgr1_ = std::make_shared<ResourceMgr>();
@ -193,7 +189,6 @@ TEST_F(ResourceMgrAdvanceTest, REGISTER_SUBSCRIBER) {
ASSERT_TRUE(flag);
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

27
cpp/unittest/scheduler/resource_test.cpp
View File

@ -26,20 +26,19 @@
#include "scheduler/ResourceFactory.h"
#include <gtest/gtest.h>
namespace zilliz {
namespace milvus {
namespace scheduler {
/************ ResourceBaseTest ************/
class ResourceBaseTest : public testing::Test {
protected:
protected:
void
SetUp() override {
only_loader_ = std::make_shared<DiskResource>(name1, id1, true, false);
only_executor_ = std::make_shared<CpuResource>(name2, id2, false, true);
both_enable_ = std::make_shared<GpuResource>(name3, id3, true, true);
both_disable_ = std::make_shared<TestResource>(name4, id4, false, false);
only_loader_ = std::make_shared<DiskResource>(name1, id1, true, false);
only_executor_ = std::make_shared<CpuResource>(name2, id2, false, true);
both_enable_ = std::make_shared<GpuResource>(name3, id3, true, true);
both_disable_ = std::make_shared<TestResource>(name4, id4, false, false);
}
const std::string name1 = "only_loader_";
@ -104,7 +103,7 @@ TEST_F(ResourceBaseTest, DUMP) {
/************ ResourceAdvanceTest ************/
class ResourceAdvanceTest : public testing::Test {
protected:
protected:
void
SetUp() override {
disk_resource_ = ResourceFactory::Create("ssd", "DISK", 0);
@ -156,13 +155,17 @@ protected:
void
WaitLoader(uint64_t count) {
std::unique_lock<std::mutex> lock(load_mutex_);
cv_.wait(lock, [&] { return load_count_ == count; });
cv_.wait(lock, [&] {
return load_count_ == count;
});
}
void
WaitExecutor(uint64_t count) {
std::unique_lock<std::mutex> lock(exec_mutex_);
cv_.wait(lock, [&] { return exec_count_ == count; });
cv_.wait(lock, [&] {
return exec_count_ == count;
});
}
ResourcePtr disk_resource_;
@ -277,6 +280,6 @@ TEST_F(ResourceAdvanceTest, TEST_RESOURCE_TEST) {
}
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

12
cpp/unittest/scheduler/schedinst_test.cpp
View File

@ -25,10 +25,8 @@ namespace zilliz {
namespace milvus {
namespace scheduler {
class SchedInstTest : public testing::Test {
protected:
protected:
void
SetUp() override {
boost::filesystem::create_directory(TMP_DIR);
@ -83,6 +81,8 @@ TEST_F(SchedInstTest, SIMPLE_GPU) {
StartSchedulerService();
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

55
cpp/unittest/scheduler/scheduler_test.cpp
View File

@ -28,19 +28,18 @@
#include "utils/Error.h"
#include "wrapper/VecIndex.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
class MockVecIndex : public engine::VecIndex {
public:
virtual Status BuildAll(const long &nb,
const float *xb,
const long *ids,
const engine::Config &cfg,
const long &nt = 0,
const float *xt = nullptr) {
public:
virtual Status BuildAll(const int64_t &nb,
const float *xb,
const int64_t *ids,
const engine::Config &cfg,
const int64_t &nt = 0,
const float *xt = nullptr) {
}
engine::VecIndexPtr Clone() override {
@ -55,27 +54,23 @@ public:
return engine::IndexType::INVALID;
}
virtual Status Add(const long &nb,
const float *xb,
const long *ids,
const engine::Config &cfg = engine::Config()) {
virtual Status Add(const int64_t &nb,
const float *xb,
const int64_t *ids,
const engine::Config &cfg = engine::Config()) {
}
virtual Status Search(const long &nq,
const float *xq,
float *dist,
long *ids,
const engine::Config &cfg = engine::Config()) {
virtual Status Search(const int64_t &nq,
const float *xq,
float *dist,
int64_t *ids,
const engine::Config &cfg = engine::Config()) {
}
engine::VecIndexPtr CopyToGpu(const int64_t &device_id, const engine::Config &cfg) override {
}
engine::VecIndexPtr CopyToCpu(const engine::Config &cfg) override {
}
virtual int64_t Dimension() {
@ -92,20 +87,18 @@ public:
}
virtual Status Load(const zilliz::knowhere::BinarySet &index_binary) {
}
public:
public:
int64_t dimension_ = 512;
int64_t ntotal_ = 0;
};
class SchedulerTest : public testing::Test {
protected:
protected:
void
SetUp() override {
constexpr int64_t cache_cap = 1024*1024*1024;
constexpr int64_t cache_cap = 1024 * 1024 * 1024;
cache::GpuCacheMgr::GetInstance(0)->SetCapacity(cache_cap);
cache::GpuCacheMgr::GetInstance(1)->SetCapacity(cache_cap);
@ -170,7 +163,6 @@ TEST_F(SchedulerTest, ON_LOAD_COMPLETED) {
sleep(3);
ASSERT_EQ(res_mgr_->GetResource(ResourceType::GPU, 1)->task_table().Size(), NUM);
}
TEST_F(SchedulerTest, PUSH_TASK_TO_NEIGHBOUR_RANDOMLY_TEST) {
@ -193,7 +185,7 @@ TEST_F(SchedulerTest, PUSH_TASK_TO_NEIGHBOUR_RANDOMLY_TEST) {
}
class SchedulerTest2 : public testing::Test {
protected:
protected:
void
SetUp() override {
ResourcePtr disk = ResourceFactory::Create("disk", "DISK", 0, true, false);
@ -243,7 +235,6 @@ protected:
std::shared_ptr<Scheduler> scheduler_;
};
TEST_F(SchedulerTest2, SPECIFIED_RESOURCE_TEST) {
const uint64_t NUM = 10;
std::vector<std::shared_ptr<TestTask>> tasks;
@ -260,6 +251,6 @@ TEST_F(SchedulerTest2, SPECIFIED_RESOURCE_TEST) {
// ASSERT_EQ(res_mgr_->GetResource(ResourceType::GPU, 1)->task_table().Size(), NUM);
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

9
cpp/unittest/scheduler/task_test.cpp
View File

@ -19,17 +19,16 @@
#include "scheduler/task/SearchTask.h"
#include <gtest/gtest.h>
namespace zilliz {
namespace milvus {
namespace scheduler {
TEST(TaskTest, INVALID_INDEX) {
auto search_task = std::make_shared<XSearchTask>(nullptr);
search_task->Load(LoadType::TEST, 10);
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

181
cpp/unittest/scheduler/tasktable_test.cpp
View File

@ -20,51 +20,53 @@
#include "scheduler/task/TestTask.h"
#include <gtest/gtest.h>
namespace {
using namespace zilliz::milvus::scheduler;
namespace ms = zilliz::milvus::scheduler;
} // namespace
/************ TaskTableBaseTest ************/
class TaskTableItemTest : public ::testing::Test {
protected:
protected:
void
SetUp() override {
std::vector<TaskTableItemState> states{
TaskTableItemState::INVALID,
TaskTableItemState::START,
TaskTableItemState::LOADING,
TaskTableItemState::LOADED,
TaskTableItemState::EXECUTING,
TaskTableItemState::EXECUTED,
TaskTableItemState::MOVING,
TaskTableItemState::MOVED};
std::vector<ms::TaskTableItemState> states{
ms::TaskTableItemState::INVALID,
ms::TaskTableItemState::START,
ms::TaskTableItemState::LOADING,
ms::TaskTableItemState::LOADED,
ms::TaskTableItemState::EXECUTING,
ms::TaskTableItemState::EXECUTED,
ms::TaskTableItemState::MOVING,
ms::TaskTableItemState::MOVED};
for (auto &state : states) {
auto item = std::make_shared<TaskTableItem>();
auto item = std::make_shared<ms::TaskTableItem>();
item->state = state;
items_.emplace_back(item);
}
}
TaskTableItem default_;
std::vector<TaskTableItemPtr> items_;
ms::TaskTableItem default_;
std::vector<ms::TaskTableItemPtr> items_;
};
TEST_F(TaskTableItemTest, CONSTRUCT) {
ASSERT_EQ(default_.id, 0);
ASSERT_EQ(default_.task, nullptr);
ASSERT_EQ(default_.state, TaskTableItemState::INVALID);
ASSERT_EQ(default_.state, ms::TaskTableItemState::INVALID);
}
TEST_F(TaskTableItemTest, DESTRUCT) {
auto p_item = new TaskTableItem();
auto p_item = new ms::TaskTableItem();
delete p_item;
}
TEST_F(TaskTableItemTest, IS_FINISH) {
for (auto &item : items_) {
if (item->state == TaskTableItemState::EXECUTED
|| item->state == TaskTableItemState::MOVED) {
if (item->state == ms::TaskTableItemState::EXECUTED
|| item->state == ms::TaskTableItemState::MOVED) {
ASSERT_TRUE(item->IsFinish());
} else {
ASSERT_FALSE(item->IsFinish());
@ -82,9 +84,9 @@ TEST_F(TaskTableItemTest, LOAD) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Load();
if (before_state == TaskTableItemState::START) {
if (before_state == ms::TaskTableItemState::START) {
ASSERT_TRUE(ret);
ASSERT_EQ(item->state, TaskTableItemState::LOADING);
ASSERT_EQ(item->state, ms::TaskTableItemState::LOADING);
} else {
ASSERT_FALSE(ret);
ASSERT_EQ(item->state, before_state);
@ -96,9 +98,9 @@ TEST_F(TaskTableItemTest, LOADED) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Loaded();
if (before_state == TaskTableItemState::LOADING) {
if (before_state == ms::TaskTableItemState::LOADING) {
ASSERT_TRUE(ret);
ASSERT_EQ(item->state, TaskTableItemState::LOADED);
ASSERT_EQ(item->state, ms::TaskTableItemState::LOADED);
} else {
ASSERT_FALSE(ret);
ASSERT_EQ(item->state, before_state);
@ -110,9 +112,9 @@ TEST_F(TaskTableItemTest, EXECUTE) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Execute();
if (before_state == TaskTableItemState::LOADED) {
if (before_state == ms::TaskTableItemState::LOADED) {
ASSERT_TRUE(ret);
ASSERT_EQ(item->state, TaskTableItemState::EXECUTING);
ASSERT_EQ(item->state, ms::TaskTableItemState::EXECUTING);
} else {
ASSERT_FALSE(ret);
ASSERT_EQ(item->state, before_state);
@ -120,14 +122,13 @@ TEST_F(TaskTableItemTest, EXECUTE) {
}
}
TEST_F(TaskTableItemTest, EXECUTED) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Executed();
if (before_state == TaskTableItemState::EXECUTING) {
if (before_state == ms::TaskTableItemState::EXECUTING) {
ASSERT_TRUE(ret);
ASSERT_EQ(item->state, TaskTableItemState::EXECUTED);
ASSERT_EQ(item->state, ms::TaskTableItemState::EXECUTED);
} else {
ASSERT_FALSE(ret);
ASSERT_EQ(item->state, before_state);
@ -139,9 +140,9 @@ TEST_F(TaskTableItemTest, MOVE) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Move();
if (before_state == TaskTableItemState::LOADED) {
if (before_state == ms::TaskTableItemState::LOADED) {
ASSERT_TRUE(ret);
ASSERT_EQ(item->state, TaskTableItemState::MOVING);
ASSERT_EQ(item->state, ms::TaskTableItemState::MOVING);
} else {
ASSERT_FALSE(ret);
ASSERT_EQ(item->state, before_state);
@ -153,9 +154,9 @@ TEST_F(TaskTableItemTest, MOVED) {
for (auto &item : items_) {
auto before_state = item->state;
auto ret = item->Moved();
if (before_state == TaskTableItemState::MOVING) {
if (before_state == ms::TaskTableItemState::MOVING) {
ASSERT_TRUE(ret);
ASSERT_EQ(item->state, TaskTableItemState::MOVED);
ASSERT_EQ(item->state, ms::TaskTableItemState::MOVED);
} else {
ASSERT_FALSE(ret);
ASSERT_EQ(item->state, before_state);
@ -166,19 +167,19 @@ TEST_F(TaskTableItemTest, MOVED) {
/************ TaskTableBaseTest ************/
class TaskTableBaseTest : public ::testing::Test {
protected:
protected:
void
SetUp() override {
TableFileSchemaPtr dummy = nullptr;
ms::TableFileSchemaPtr dummy = nullptr;
invalid_task_ = nullptr;
task1_ = std::make_shared<TestTask>(dummy);
task2_ = std::make_shared<TestTask>(dummy);
task1_ = std::make_shared<ms::TestTask>(dummy);
task2_ = std::make_shared<ms::TestTask>(dummy);
}
TaskPtr invalid_task_;
TaskPtr task1_;
TaskPtr task2_;
TaskTable empty_table_;
ms::TaskPtr invalid_task_;
ms::TaskPtr task1_;
ms::TaskPtr task2_;
ms::TaskTable empty_table_;
};
TEST_F(TaskTableBaseTest, SUBSCRIBER) {
@ -191,7 +192,6 @@ TEST_F(TaskTableBaseTest, SUBSCRIBER) {
ASSERT_TRUE(flag);
}
TEST_F(TaskTableBaseTest, PUT_TASK) {
empty_table_.Put(task1_);
ASSERT_EQ(empty_table_.Get(0)->task, task1_);
@ -203,14 +203,14 @@ TEST_F(TaskTableBaseTest, PUT_INVALID_TEST) {
}
TEST_F(TaskTableBaseTest, PUT_BATCH) {
std::vector<TaskPtr> tasks{task1_, task2_};
std::vector<ms::TaskPtr> tasks{task1_, task2_};
empty_table_.Put(tasks);
ASSERT_EQ(empty_table_.Get(0)->task, task1_);
ASSERT_EQ(empty_table_.Get(1)->task, task2_);
}
TEST_F(TaskTableBaseTest, PUT_EMPTY_BATCH) {
std::vector<TaskPtr> tasks{};
std::vector<ms::TaskPtr> tasks{};
empty_table_.Put(tasks);
}
@ -236,8 +236,8 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD) {
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
}
empty_table_[0]->state = TaskTableItemState::MOVED;
empty_table_[1]->state = TaskTableItemState::EXECUTED;
empty_table_[0]->state = ms::TaskTableItemState::MOVED;
empty_table_[1]->state = ms::TaskTableItemState::EXECUTED;
auto indexes = empty_table_.PickToLoad(1);
ASSERT_EQ(indexes.size(), 1);
@ -249,8 +249,8 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD_LIMIT) {
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
}
empty_table_[0]->state = TaskTableItemState::MOVED;
empty_table_[1]->state = TaskTableItemState::EXECUTED;
empty_table_[0]->state = ms::TaskTableItemState::MOVED;
empty_table_[1]->state = ms::TaskTableItemState::EXECUTED;
auto indexes = empty_table_.PickToLoad(3);
ASSERT_EQ(indexes.size(), 3);
@ -264,8 +264,8 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD_CACHE) {
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
}
empty_table_[0]->state = TaskTableItemState::MOVED;
empty_table_[1]->state = TaskTableItemState::EXECUTED;
empty_table_[0]->state = ms::TaskTableItemState::MOVED;
empty_table_[1]->state = ms::TaskTableItemState::EXECUTED;
// first pick, non-cache
auto indexes = empty_table_.PickToLoad(1);
@ -274,7 +274,7 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD_CACHE) {
// second pick, iterate from 2
// invalid state change
empty_table_[1]->state = TaskTableItemState::START;
empty_table_[1]->state = ms::TaskTableItemState::START;
indexes = empty_table_.PickToLoad(1);
ASSERT_EQ(indexes.size(), 1);
ASSERT_EQ(indexes[0], 2);
@ -285,9 +285,9 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE) {
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
}
empty_table_[0]->state = TaskTableItemState::MOVED;
empty_table_[1]->state = TaskTableItemState::EXECUTED;
empty_table_[2]->state = TaskTableItemState::LOADED;
empty_table_[0]->state = ms::TaskTableItemState::MOVED;
empty_table_[1]->state = ms::TaskTableItemState::EXECUTED;
empty_table_[2]->state = ms::TaskTableItemState::LOADED;
auto indexes = empty_table_.PickToExecute(1);
ASSERT_EQ(indexes.size(), 1);
@ -299,10 +299,10 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_LIMIT) {
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
}
empty_table_[0]->state = TaskTableItemState::MOVED;
empty_table_[1]->state = TaskTableItemState::EXECUTED;
empty_table_[2]->state = TaskTableItemState::LOADED;
empty_table_[3]->state = TaskTableItemState::LOADED;
empty_table_[0]->state = ms::TaskTableItemState::MOVED;
empty_table_[1]->state = ms::TaskTableItemState::EXECUTED;
empty_table_[2]->state = ms::TaskTableItemState::LOADED;
empty_table_[3]->state = ms::TaskTableItemState::LOADED;
auto indexes = empty_table_.PickToExecute(3);
ASSERT_EQ(indexes.size(), 2);
@ -315,9 +315,9 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_CACHE) {
for (size_t i = 0; i < NUM_TASKS; ++i) {
empty_table_.Put(task1_);
}
empty_table_[0]->state = TaskTableItemState::MOVED;
empty_table_[1]->state = TaskTableItemState::EXECUTED;
empty_table_[2]->state = TaskTableItemState::LOADED;
empty_table_[0]->state = ms::TaskTableItemState::MOVED;
empty_table_[1]->state = ms::TaskTableItemState::EXECUTED;
empty_table_[2]->state = ms::TaskTableItemState::LOADED;
// first pick, non-cache
auto indexes = empty_table_.PickToExecute(1);
@ -326,40 +326,39 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_CACHE) {
// second pick, iterate from 2
// invalid state change
empty_table_[1]->state = TaskTableItemState::START;
empty_table_[1]->state = ms::TaskTableItemState::START;
indexes = empty_table_.PickToExecute(1);
ASSERT_EQ(indexes.size(), 1);
ASSERT_EQ(indexes[0], 2);
}
/************ TaskTableAdvanceTest ************/
class TaskTableAdvanceTest : public ::testing::Test {
protected:
protected:
void
SetUp() override {
TableFileSchemaPtr dummy = nullptr;
ms::TableFileSchemaPtr dummy = nullptr;
for (uint64_t i = 0; i < 8; ++i) {
auto task = std::make_shared<TestTask>(dummy);
auto task = std::make_shared<ms::TestTask>(dummy);
table1_.Put(task);
}
table1_.Get(0)->state = TaskTableItemState::INVALID;
table1_.Get(1)->state = TaskTableItemState::START;
table1_.Get(2)->state = TaskTableItemState::LOADING;
table1_.Get(3)->state = TaskTableItemState::LOADED;
table1_.Get(4)->state = TaskTableItemState::EXECUTING;
table1_.Get(5)->state = TaskTableItemState::EXECUTED;
table1_.Get(6)->state = TaskTableItemState::MOVING;
table1_.Get(7)->state = TaskTableItemState::MOVED;
table1_.Get(0)->state = ms::TaskTableItemState::INVALID;
table1_.Get(1)->state = ms::TaskTableItemState::START;
table1_.Get(2)->state = ms::TaskTableItemState::LOADING;
table1_.Get(3)->state = ms::TaskTableItemState::LOADED;
table1_.Get(4)->state = ms::TaskTableItemState::EXECUTING;
table1_.Get(5)->state = ms::TaskTableItemState::EXECUTED;
table1_.Get(6)->state = ms::TaskTableItemState::MOVING;
table1_.Get(7)->state = ms::TaskTableItemState::MOVED;
}
TaskTable table1_;
ms::TaskTable table1_;
};
TEST_F(TaskTableAdvanceTest, LOAD) {
std::vector<TaskTableItemState> before_state;
std::vector<ms::TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
}
@ -369,8 +368,8 @@ TEST_F(TaskTableAdvanceTest, LOAD) {
}
for (size_t i = 0; i < table1_.Size(); ++i) {
if (before_state[i] == TaskTableItemState::START) {
ASSERT_EQ(table1_.Get(i)->state, TaskTableItemState::LOADING);
if (before_state[i] == ms::TaskTableItemState::START) {
ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::LOADING);
} else {
ASSERT_EQ(table1_.Get(i)->state, before_state[i]);
}
@ -378,7 +377,7 @@ TEST_F(TaskTableAdvanceTest, LOAD) {
}
TEST_F(TaskTableAdvanceTest, LOADED) {
std::vector<TaskTableItemState> before_state;
std::vector<ms::TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
}
@ -388,8 +387,8 @@ TEST_F(TaskTableAdvanceTest, LOADED) {
}
for (size_t i = 0; i < table1_.Size(); ++i) {
if (before_state[i] == TaskTableItemState::LOADING) {
ASSERT_EQ(table1_.Get(i)->state, TaskTableItemState::LOADED);
if (before_state[i] == ms::TaskTableItemState::LOADING) {
ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::LOADED);
} else {
ASSERT_EQ(table1_.Get(i)->state, before_state[i]);
}
@ -397,7 +396,7 @@ TEST_F(TaskTableAdvanceTest, LOADED) {
}
TEST_F(TaskTableAdvanceTest, EXECUTE) {
std::vector<TaskTableItemState> before_state;
std::vector<ms::TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
}
@ -407,8 +406,8 @@ TEST_F(TaskTableAdvanceTest, EXECUTE) {
}
for (size_t i = 0; i < table1_.Size(); ++i) {
if (before_state[i] == TaskTableItemState::LOADED) {
ASSERT_EQ(table1_.Get(i)->state, TaskTableItemState::EXECUTING);
if (before_state[i] == ms::TaskTableItemState::LOADED) {
ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::EXECUTING);
} else {
ASSERT_EQ(table1_.Get(i)->state, before_state[i]);
}
@ -416,7 +415,7 @@ TEST_F(TaskTableAdvanceTest, EXECUTE) {
}
TEST_F(TaskTableAdvanceTest, EXECUTED) {
std::vector<TaskTableItemState> before_state;
std::vector<ms::TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
}
@ -426,8 +425,8 @@ TEST_F(TaskTableAdvanceTest, EXECUTED) {
}
for (size_t i = 0; i < table1_.Size(); ++i) {
if (before_state[i] == TaskTableItemState::EXECUTING) {
ASSERT_EQ(table1_.Get(i)->state, TaskTableItemState::EXECUTED);
if (before_state[i] == ms::TaskTableItemState::EXECUTING) {
ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::EXECUTED);
} else {
ASSERT_EQ(table1_.Get(i)->state, before_state[i]);
}
@ -435,7 +434,7 @@ TEST_F(TaskTableAdvanceTest, EXECUTED) {
}
TEST_F(TaskTableAdvanceTest, MOVE) {
std::vector<TaskTableItemState> before_state;
std::vector<ms::TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
}
@ -445,8 +444,8 @@ TEST_F(TaskTableAdvanceTest, MOVE) {
}
for (size_t i = 0; i < table1_.Size(); ++i) {
if (before_state[i] == TaskTableItemState::LOADED) {
ASSERT_EQ(table1_.Get(i)->state, TaskTableItemState::MOVING);
if (before_state[i] == ms::TaskTableItemState::LOADED) {
ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::MOVING);
} else {
ASSERT_EQ(table1_.Get(i)->state, before_state[i]);
}
@ -454,7 +453,7 @@ TEST_F(TaskTableAdvanceTest, MOVE) {
}
TEST_F(TaskTableAdvanceTest, MOVED) {
std::vector<TaskTableItemState> before_state;
std::vector<ms::TaskTableItemState> before_state;
for (auto &task : table1_) {
before_state.push_back(task->state);
}
@ -464,8 +463,8 @@ TEST_F(TaskTableAdvanceTest, MOVED) {
}
for (size_t i = 0; i < table1_.Size(); ++i) {
if (before_state[i] == TaskTableItemState::MOVING) {
ASSERT_EQ(table1_.Get(i)->state, TaskTableItemState::MOVED);
if (before_state[i] == ms::TaskTableItemState::MOVING) {
ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::MOVED);
} else {
ASSERT_EQ(table1_.Get(i)->state, before_state[i]);
}

116
cpp/unittest/server/cache_test.cpp
View File

@ -21,41 +21,40 @@
#include "utils/Error.h"
#include "wrapper/VecIndex.h"
using namespace zilliz::milvus;
namespace {
class InvalidCacheMgr : public cache::CacheMgr<cache::DataObjPtr> {
public:
namespace ms = zilliz::milvus;
class InvalidCacheMgr : public ms::cache::CacheMgr<ms::cache::DataObjPtr> {
public:
InvalidCacheMgr() {
}
};
class LessItemCacheMgr : public cache::CacheMgr<cache::DataObjPtr> {
public:
class LessItemCacheMgr : public ms::cache::CacheMgr<ms::cache::DataObjPtr> {
public:
LessItemCacheMgr() {
cache_ = std::make_shared<cache::Cache<cache::DataObjPtr>>(1UL << 12, 10);
cache_ = std::make_shared<ms::cache::Cache<ms::cache::DataObjPtr>>(1UL << 12, 10);
}
};
class MockVecIndex : public engine::VecIndex {
public:
class MockVecIndex : public ms::engine::VecIndex {
public:
MockVecIndex(int64_t dim, int64_t total)
: dimension_(dim),
ntotal_(total){
ntotal_(total) {
}
virtual Status BuildAll(const long &nb,
const float *xb,
const long *ids,
const engine::Config &cfg,
const long &nt = 0,
const float *xt = nullptr) {
return Status();
virtual ms::Status BuildAll(const int64_t &nb,
const float *xb,
const int64_t *ids,
const ms::engine::Config &cfg,
const int64_t &nt = 0,
const float *xt = nullptr) {
return ms::Status();
}
engine::VecIndexPtr Clone() override {
ms::engine::VecIndexPtr Clone() override {
return zilliz::milvus::engine::VecIndexPtr();
}
@ -63,31 +62,31 @@ public:
return 0;
}
engine::IndexType GetType() override {
return engine::IndexType::INVALID;
ms::engine::IndexType GetType() override {
return ms::engine::IndexType::INVALID;
}
virtual Status Add(const long &nb,
const float *xb,
const long *ids,
const engine::Config &cfg = engine::Config()) {
return Status();
virtual ms::Status Add(const int64_t &nb,
const float *xb,
const int64_t *ids,
const ms::engine::Config &cfg = ms::engine::Config()) {
return ms::Status();
}
virtual Status Search(const long &nq,
const float *xq,
float *dist,
long *ids,
const engine::Config &cfg = engine::Config()) {
return Status();
virtual ms::Status Search(const int64_t &nq,
const float *xq,
float *dist,
int64_t *ids,
const ms::engine::Config &cfg = ms::engine::Config()) {
return ms::Status();
}
engine::VecIndexPtr CopyToGpu(const int64_t &device_id,
const engine::Config &cfg) override {
ms::engine::VecIndexPtr CopyToGpu(const int64_t &device_id,
const ms::engine::Config &cfg) override {
return nullptr;
}
engine::VecIndexPtr CopyToCpu(const engine::Config &cfg) override {
ms::engine::VecIndexPtr CopyToCpu(const ms::engine::Config &cfg) override {
return nullptr;
}
@ -104,19 +103,19 @@ public:
return binset;
}
virtual Status Load(const zilliz::knowhere::BinarySet &index_binary) {
return Status();
virtual ms::Status Load(const zilliz::knowhere::BinarySet &index_binary) {
return ms::Status();
}
public:
public:
int64_t dimension_ = 256;
int64_t ntotal_ = 0;
};
}
} // namespace
TEST(CacheTest, DUMMY_TEST) {
engine::Config cfg;
ms::engine::Config cfg;
MockVecIndex mock_index(256, 1000);
mock_index.Dimension();
mock_index.Count();
@ -134,9 +133,9 @@ TEST(CacheTest, DUMMY_TEST) {
}
TEST(CacheTest, CPU_CACHE_TEST) {
auto cpu_mgr = cache::CpuCacheMgr::GetInstance();
auto cpu_mgr = ms::cache::CpuCacheMgr::GetInstance();
const int64_t gbyte = 1024*1024*1024;
const int64_t gbyte = 1024 * 1024 * 1024;
int64_t g_num = 16;
int64_t cap = g_num * gbyte;
cpu_mgr->SetCapacity(cap);
@ -145,8 +144,8 @@ TEST(CacheTest, CPU_CACHE_TEST) {
uint64_t item_count = 20;
for (uint64_t i = 0; i < item_count; i++) {
//each vector is 1k byte, total size less than 1G
engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 1000000);
cache::DataObjPtr data_obj = std::make_shared<cache::DataObj>(mock_index);
ms::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 1000000);
ms::cache::DataObjPtr data_obj = std::make_shared<ms::cache::DataObj>(mock_index);
cpu_mgr->InsertItem("index_" + std::to_string(i), data_obj);
}
ASSERT_LT(cpu_mgr->ItemCount(), g_num);
@ -169,8 +168,8 @@ TEST(CacheTest, CPU_CACHE_TEST) {
cpu_mgr->SetCapacity(g_num * gbyte);
//each vector is 1k byte, total size less than 6G
engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 6000000);
cache::DataObjPtr data_obj = std::make_shared<cache::DataObj>(mock_index);
ms::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 6000000);
ms::cache::DataObjPtr data_obj = std::make_shared<ms::cache::DataObj>(mock_index);
cpu_mgr->InsertItem("index_6g", data_obj);
ASSERT_TRUE(cpu_mgr->ItemExists("index_6g"));
}
@ -179,12 +178,12 @@ TEST(CacheTest, CPU_CACHE_TEST) {
}
TEST(CacheTest, GPU_CACHE_TEST) {
auto gpu_mgr = cache::GpuCacheMgr::GetInstance(0);
auto gpu_mgr = ms::cache::GpuCacheMgr::GetInstance(0);
for(int i = 0; i < 20; i++) {
for (int i = 0; i < 20; i++) {
//each vector is 1k byte
engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 1000);
cache::DataObjPtr data_obj = std::make_shared<cache::DataObj>(mock_index);
ms::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 1000);
ms::cache::DataObjPtr data_obj = std::make_shared<ms::cache::DataObj>(mock_index);
gpu_mgr->InsertItem("index_" + std::to_string(i), data_obj);
}
@ -196,15 +195,14 @@ TEST(CacheTest, GPU_CACHE_TEST) {
for (auto i = 0; i < 3; i++) {
// TODO: use gpu index to mock
//each vector is 1k byte, total size less than 2G
engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 2000000);
cache::DataObjPtr data_obj = std::make_shared<cache::DataObj>(mock_index);
std::cout << data_obj->size() <<std::endl;
ms::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 2000000);
ms::cache::DataObjPtr data_obj = std::make_shared<ms::cache::DataObj>(mock_index);
std::cout << data_obj->size() << std::endl;
gpu_mgr->InsertItem("index_" + std::to_string(i), data_obj);
}
gpu_mgr->ClearCache();
ASSERT_EQ(gpu_mgr->ItemCount(), 0);
}
TEST(CacheTest, INVALID_TEST) {
@ -214,7 +212,7 @@ TEST(CacheTest, INVALID_TEST) {
ASSERT_FALSE(mgr.ItemExists("test"));
ASSERT_EQ(mgr.GetItem("test"), nullptr);
mgr.InsertItem("test", cache::DataObjPtr());
mgr.InsertItem("test", ms::cache::DataObjPtr());
mgr.InsertItem("test", nullptr);
mgr.EraseItem("test");
mgr.PrintInfo();
@ -226,12 +224,12 @@ TEST(CacheTest, INVALID_TEST) {
{
LessItemCacheMgr mgr;
for(int i = 0; i < 20; i++) {
for (int i = 0; i < 20; i++) {
//each vector is 1k byte
engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 2);
cache::DataObjPtr data_obj = std::make_shared<cache::DataObj>(mock_index);
ms::engine::VecIndexPtr mock_index = std::make_shared<MockVecIndex>(256, 2);
ms::cache::DataObjPtr data_obj = std::make_shared<ms::cache::DataObj>(mock_index);
mgr.InsertItem("index_" + std::to_string(i), data_obj);
}
ASSERT_EQ(mgr.GetItem("index_0"), nullptr);
}
}
}

50
cpp/unittest/server/config_test.cpp
View File

@ -23,39 +23,39 @@
#include "utils/ValidationUtil.h"
#include "server/Config.h"
using namespace zilliz::milvus;
namespace {
static const char* CONFIG_FILE_PATH = "./milvus/conf/server_config.yaml";
static const char* LOG_FILE_PATH = "./milvus/conf/log_config.conf";
namespace ms = zilliz::milvus;
static const char *CONFIG_FILE_PATH = "./milvus/conf/server_config.yaml";
static const char *LOG_FILE_PATH = "./milvus/conf/log_config.conf";
static constexpr uint64_t KB = 1024;
static constexpr uint64_t MB = KB*1024;
static constexpr uint64_t GB = MB*1024;
static constexpr uint64_t MB = KB * 1024;
static constexpr uint64_t GB = MB * 1024;
}
} // namespace
TEST(ConfigTest, CONFIG_TEST) {
server::ConfigMgr* config_mgr = server::ConfigMgr::GetInstance();
ms::server::ConfigMgr *config_mgr = ms::server::ConfigMgr::GetInstance();
ErrorCode err = config_mgr->LoadConfigFile("");
ASSERT_EQ(err, SERVER_UNEXPECTED_ERROR);
ms::ErrorCode err = config_mgr->LoadConfigFile("");
ASSERT_EQ(err, ms::SERVER_UNEXPECTED_ERROR);
err = config_mgr->LoadConfigFile(LOG_FILE_PATH);
ASSERT_EQ(err, SERVER_UNEXPECTED_ERROR);
ASSERT_EQ(err, ms::SERVER_UNEXPECTED_ERROR);
err = config_mgr->LoadConfigFile(CONFIG_FILE_PATH);
ASSERT_EQ(err, SERVER_SUCCESS);
ASSERT_EQ(err, ms::SERVER_SUCCESS);
config_mgr->Print();
server::ConfigNode& root_config = config_mgr->GetRootNode();
server::ConfigNode& server_config = root_config.GetChild("server_config");
server::ConfigNode& db_config = root_config.GetChild("db_config");
server::ConfigNode& metric_config = root_config.GetChild("metric_config");
server::ConfigNode& cache_config = root_config.GetChild("cache_config");
server::ConfigNode invalid_config = root_config.GetChild("invalid_config");
ms::server::ConfigNode &root_config = config_mgr->GetRootNode();
ms::server::ConfigNode &server_config = root_config.GetChild("server_config");
ms::server::ConfigNode &db_config = root_config.GetChild("db_config");
ms::server::ConfigNode &metric_config = root_config.GetChild("metric_config");
ms::server::ConfigNode &cache_config = root_config.GetChild("cache_config");
ms::server::ConfigNode invalid_config = root_config.GetChild("invalid_config");
auto valus = invalid_config.GetSequence("not_exist");
float ff = invalid_config.GetFloatValue("not_exist", 3.0);
ASSERT_EQ(ff, 3.0);
@ -63,16 +63,16 @@ TEST(ConfigTest, CONFIG_TEST) {
std::string address = server_config.GetValue("address");
ASSERT_TRUE(!address.empty());
int64_t port = server_config.GetInt64Value("port");
ASSERT_TRUE(port != 0);
ASSERT_NE(port, 0);
server_config.SetValue("test", "2.5");
double test = server_config.GetDoubleValue("test");
ASSERT_EQ(test, 2.5);
server::ConfigNode fake;
ms::server::ConfigNode fake;
server_config.AddChild("fake", fake);
fake = server_config.GetChild("fake");
server::ConfigNodeArr arr;
ms::server::ConfigNodeArr arr;
server_config.GetChildren(arr);
ASSERT_EQ(arr.size(), 1UL);
@ -89,7 +89,7 @@ TEST(ConfigTest, CONFIG_TEST) {
auto seq = server_config.GetSequence("seq");
ASSERT_EQ(seq.size(), 2UL);
server::ConfigNode combine;
ms::server::ConfigNode combine;
combine.Combine(server_config);
combine.PrintAll();
@ -102,8 +102,8 @@ TEST(ConfigTest, CONFIG_TEST) {
}
TEST(ConfigTest, SERVER_CONFIG_TEST) {
server::Config& config = server::Config::GetInstance();
Status s = config.LoadConfigFile(CONFIG_FILE_PATH);
ms::server::Config &config = ms::server::Config::GetInstance();
ms::Status s = config.LoadConfigFile(CONFIG_FILE_PATH);
ASSERT_TRUE(s.ok());
s = config.ValidateConfig();
@ -113,4 +113,4 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) {
s = config.ResetDefaultConfig();
ASSERT_TRUE(s.ok());
}
}

108
cpp/unittest/server/rpc_test.cpp
View File

@ -23,7 +23,7 @@
#include "server/grpc_impl/GrpcRequestHandler.h"
#include "server/grpc_impl/GrpcRequestScheduler.h"
#include "server/grpc_impl/GrpcRequestTask.h"
#include "version.h"
#include "../version.h"
#include "grpc/gen-milvus/milvus.grpc.pb.h"
#include "grpc/gen-status/status.pb.h"
@ -34,10 +34,10 @@
#include "scheduler/ResourceFactory.h"
#include "utils/CommonUtil.h"
using namespace zilliz::milvus;
namespace {
namespace ms = zilliz::milvus;
static const char *TABLE_NAME = "test_grpc";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t INDEX_FILE_SIZE = 1024;
@ -49,30 +49,29 @@ class RpcHandlerTest : public testing::Test {
protected:
void
SetUp() override {
auto res_mgr = scheduler::ResMgrInst::GetInstance();
auto res_mgr = ms::scheduler::ResMgrInst::GetInstance();
res_mgr->Clear();
res_mgr->Add(scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false));
res_mgr->Add(scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true));
res_mgr->Add(scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true));
res_mgr->Add(ms::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false));
res_mgr->Add(ms::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true));
res_mgr->Add(ms::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true));
auto default_conn = scheduler::Connection("IO", 500.0);
auto PCIE = scheduler::Connection("IO", 11000.0);
auto default_conn = ms::scheduler::Connection("IO", 500.0);
auto PCIE = ms::scheduler::Connection("IO", 11000.0);
res_mgr->Connect("disk", "cpu", default_conn);
res_mgr->Connect("cpu", "gtx1660", PCIE);
res_mgr->Start();
scheduler::SchedInst::GetInstance()->Start();
scheduler::JobMgrInst::GetInstance()->Start();
ms::scheduler::SchedInst::GetInstance()->Start();
ms::scheduler::JobMgrInst::GetInstance()->Start();
engine::DBOptions opt;
ms::engine::DBOptions opt;
server::Config::GetInstance().SetDBConfigBackendUrl("sqlite://:@:/");
server::Config::GetInstance().SetDBConfigPrimaryPath("/tmp/milvus_test");
server::Config::GetInstance().SetDBConfigSecondaryPath("");
server::Config::GetInstance().SetDBConfigArchiveDiskThreshold("");
server::Config::GetInstance().SetDBConfigArchiveDaysThreshold("");
server::Config::GetInstance().SetCacheConfigCacheInsertData("");
server::Config::GetInstance().SetEngineConfigOmpThreadNum("");
ms::server::Config::GetInstance().SetDBConfigBackendUrl("sqlite://:@:/");
ms::server::Config::GetInstance().SetDBConfigPrimaryPath("/tmp/milvus_test");
ms::server::Config::GetInstance().SetDBConfigSecondaryPath("");
ms::server::Config::GetInstance().SetDBConfigArchiveDiskThreshold("");
ms::server::Config::GetInstance().SetDBConfigArchiveDaysThreshold("");
ms::server::Config::GetInstance().SetCacheConfigCacheInsertData("");
ms::server::Config::GetInstance().SetEngineConfigOmpThreadNum("");
// serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "cluster");
// DBWrapper::GetInstance().GetInstance().StartService();
@ -82,11 +81,11 @@ class RpcHandlerTest : public testing::Test {
// DBWrapper::GetInstance().GetInstance().StartService();
// DBWrapper::GetInstance().GetInstance().StopService();
server::Config::GetInstance().SetResourceConfigMode("single");
server::DBWrapper::GetInstance().StartService();
ms::server::Config::GetInstance().SetResourceConfigMode("single");
ms::server::DBWrapper::GetInstance().StartService();
//initialize handler, create table
handler = std::make_shared<server::grpc::GrpcRequestHandler>();
handler = std::make_shared<ms::server::grpc::GrpcRequestHandler>();
::grpc::ServerContext context;
::milvus::grpc::TableSchema request;
::milvus::grpc::Status status;
@ -99,18 +98,20 @@ class RpcHandlerTest : public testing::Test {
void
TearDown() override {
server::DBWrapper::GetInstance().StopService();
scheduler::JobMgrInst::GetInstance()->Stop();
scheduler::ResMgrInst::GetInstance()->Stop();
scheduler::SchedInst::GetInstance()->Stop();
ms::server::DBWrapper::GetInstance().StopService();
ms::scheduler::JobMgrInst::GetInstance()->Stop();
ms::scheduler::ResMgrInst::GetInstance()->Stop();
ms::scheduler::SchedInst::GetInstance()->Stop();
boost::filesystem::remove_all("/tmp/milvus_test");
}
protected:
std::shared_ptr<server::grpc::GrpcRequestHandler> handler;
std::shared_ptr<ms::server::grpc::GrpcRequestHandler> handler;
};
void BuildVectors(int64_t from, int64_t to,
std::vector<std::vector<float >> &vector_record_array) {
void
BuildVectors(int64_t from, int64_t to,
std::vector<std::vector<float >> &vector_record_array) {
if (to <= from) {
return;
}
@ -127,20 +128,22 @@ void BuildVectors(int64_t from, int64_t to,
}
}
std::string CurrentTmDate(int64_t offset_day = 0) {
std::string
CurrentTmDate(int64_t offset_day = 0) {
time_t tt;
time(&tt);
tt = tt + 8 * SECONDS_EACH_HOUR;
tt = tt + 24 * SECONDS_EACH_HOUR * offset_day;
tm *t = gmtime(&tt);
tm t;
gmtime_r(&tt, &t);
std::string str = std::to_string(t->tm_year + 1900) + "-" + std::to_string(t->tm_mon + 1)
+ "-" + std::to_string(t->tm_mday);
std::string str = std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1)
+ "-" + std::to_string(t.tm_mday);
return str;
}
}
} // namespace
TEST_F(RpcHandlerTest, HAS_TABLE_TEST) {
::grpc::ServerContext context;
@ -311,7 +314,6 @@ TEST_F(RpcHandlerTest, TABLES_TEST) {
::grpc::Status status = handler->DescribeTable(&context, &table_name, &table_schema);
ASSERT_EQ(status.error_code(), ::grpc::Status::OK.error_code());
::milvus::grpc::InsertParam request;
std::vector<std::vector<float>> record_array;
BuildVectors(0, VECTOR_COUNT, record_array);
@ -419,31 +421,29 @@ TEST_F(RpcHandlerTest, DELETE_BY_RANGE_TEST) {
grpc_status = handler->DeleteByRange(&context, &request, &status);
request.mutable_range()->set_end_value(CurrentTmDate(-2));
grpc_status = handler->DeleteByRange(&context, &request, &status);
}
//////////////////////////////////////////////////////////////////////
namespace {
class DummyTask : public server::grpc::GrpcBaseTask {
public:
Status
class DummyTask : public ms::server::grpc::GrpcBaseTask {
public:
ms::Status
OnExecute() override {
return Status::OK();
return ms::Status::OK();
}
static server::grpc::BaseTaskPtr
static ms::server::grpc::BaseTaskPtr
Create(std::string &dummy) {
return std::shared_ptr<server::grpc::GrpcBaseTask>(new DummyTask(dummy));
return std::shared_ptr<ms::server::grpc::GrpcBaseTask>(new DummyTask(dummy));
}
public:
public:
explicit DummyTask(std::string &dummy) : GrpcBaseTask(dummy) {
}
};
class RpcSchedulerTest : public testing::Test {
protected:
protected:
void
SetUp() override {
std::string dummy = "dql";
@ -453,22 +453,22 @@ protected:
std::shared_ptr<DummyTask> task_ptr;
};
}
} // namespace
TEST_F(RpcSchedulerTest, BASE_TASK_TEST){
TEST_F(RpcSchedulerTest, BASE_TASK_TEST) {
auto status = task_ptr->Execute();
ASSERT_TRUE(status.ok());
server::grpc::GrpcRequestScheduler::GetInstance().Start();
ms::server::grpc::GrpcRequestScheduler::GetInstance().Start();
::milvus::grpc::Status grpc_status;
std::string dummy = "dql";
server::grpc::BaseTaskPtr base_task_ptr = DummyTask::Create(dummy);
server::grpc::GrpcRequestScheduler::GetInstance().ExecTask(base_task_ptr, &grpc_status);
ms::server::grpc::BaseTaskPtr base_task_ptr = DummyTask::Create(dummy);
ms::server::grpc::GrpcRequestScheduler::GetInstance().ExecTask(base_task_ptr, &grpc_status);
server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr);
ms::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr);
task_ptr = nullptr;
server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr);
ms::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr);
server::grpc::GrpcRequestScheduler::GetInstance().Stop();
ms::server::grpc::GrpcRequestScheduler::GetInstance().Stop();
}

240
cpp/unittest/server/util_test.cpp
View File

@ -31,75 +31,76 @@
#include <boost/filesystem.hpp>
#include <gtest/gtest.h>
using namespace zilliz::milvus;
namespace {
static const char* LOG_FILE_PATH = "./milvus/conf/log_config.conf";
namespace ms = zilliz::milvus;
void CopyStatus(Status& st1, Status& st2) {
static const char *LOG_FILE_PATH = "./milvus/conf/log_config.conf";
void
CopyStatus(ms::Status &st1, ms::Status &st2) {
st1 = st2;
}
}
} // namespace
TEST(UtilTest, EXCEPTION_TEST) {
std::string err_msg = "failed";
server::ServerException ex(SERVER_UNEXPECTED_ERROR, err_msg);
ASSERT_EQ(ex.error_code(), SERVER_UNEXPECTED_ERROR);
ms::server::ServerException ex(ms::SERVER_UNEXPECTED_ERROR, err_msg);
ASSERT_EQ(ex.error_code(), ms::SERVER_UNEXPECTED_ERROR);
std::string msg = ex.what();
ASSERT_EQ(msg, err_msg);
}
TEST(UtilTest, SIGNAL_TEST) {
server::SignalUtil::PrintStacktrace();
ms::server::SignalUtil::PrintStacktrace();
}
TEST(UtilTest, COMMON_TEST) {
unsigned long total_mem = 0, free_mem = 0;
server::CommonUtil::GetSystemMemInfo(total_mem, free_mem);
uint64_t total_mem = 0, free_mem = 0;
ms::server::CommonUtil::GetSystemMemInfo(total_mem, free_mem);
ASSERT_GT(total_mem, 0);
ASSERT_GT(free_mem, 0);
unsigned int thread_cnt = 0;
server::CommonUtil::GetSystemAvailableThreads(thread_cnt);
uint32_t thread_cnt = 0;
ms::server::CommonUtil::GetSystemAvailableThreads(thread_cnt);
ASSERT_GT(thread_cnt, 0);
std::string path1 = "/tmp/milvus_test/";
std::string path2 = path1 + "common_test_12345/";
std::string path3 = path2 + "abcdef";
Status status = server::CommonUtil::CreateDirectory(path3);
ms::Status status = ms::server::CommonUtil::CreateDirectory(path3);
ASSERT_TRUE(status.ok());
//test again
status = server::CommonUtil::CreateDirectory(path3);
status = ms::server::CommonUtil::CreateDirectory(path3);
ASSERT_TRUE(status.ok());
ASSERT_TRUE(server::CommonUtil::IsDirectoryExist(path3));
ASSERT_TRUE(ms::server::CommonUtil::IsDirectoryExist(path3));
status = server::CommonUtil::DeleteDirectory(path1);
status = ms::server::CommonUtil::DeleteDirectory(path1);
ASSERT_TRUE(status.ok());
//test again
status = server::CommonUtil::DeleteDirectory(path1);
status = ms::server::CommonUtil::DeleteDirectory(path1);
ASSERT_TRUE(status.ok());
ASSERT_FALSE(server::CommonUtil::IsDirectoryExist(path1));
ASSERT_FALSE(server::CommonUtil::IsFileExist(path1));
ASSERT_FALSE(ms::server::CommonUtil::IsDirectoryExist(path1));
ASSERT_FALSE(ms::server::CommonUtil::IsFileExist(path1));
std::string exe_path = server::CommonUtil::GetExePath();
std::string exe_path = ms::server::CommonUtil::GetExePath();
ASSERT_FALSE(exe_path.empty());
time_t tt;
time( &tt );
time(&tt);
tm time_struct;
memset(&time_struct, 0, sizeof(tm));
server::CommonUtil::ConvertTime(tt, time_struct);
ms::server::CommonUtil::ConvertTime(tt, time_struct);
ASSERT_GT(time_struct.tm_year, 0);
ASSERT_GT(time_struct.tm_mon, 0);
ASSERT_GT(time_struct.tm_mday, 0);
server::CommonUtil::ConvertTime(time_struct, tt);
ms::server::CommonUtil::ConvertTime(time_struct, tt);
ASSERT_GT(tt, 0);
bool res = server::CommonUtil::TimeStrToTime("2019-03-23", tt, time_struct);
bool res = ms::server::CommonUtil::TimeStrToTime("2019-03-23", tt, time_struct);
ASSERT_EQ(time_struct.tm_year, 119);
ASSERT_EQ(time_struct.tm_mon, 2);
ASSERT_EQ(time_struct.tm_mday, 23);
@ -109,45 +110,43 @@ TEST(UtilTest, COMMON_TEST) {
TEST(UtilTest, STRINGFUNCTIONS_TEST) {
std::string str = " test zilliz";
server::StringHelpFunctions::TrimStringBlank(str);
ms::server::StringHelpFunctions::TrimStringBlank(str);
ASSERT_EQ(str, "test zilliz");
str = "\"test zilliz\"";
server::StringHelpFunctions::TrimStringQuote(str, "\"");
ms::server::StringHelpFunctions::TrimStringQuote(str, "\"");
ASSERT_EQ(str, "test zilliz");
str = "a,b,c";
std::vector<std::string> result;
auto status = server::StringHelpFunctions::SplitStringByDelimeter(str , ",", result);
auto status = ms::server::StringHelpFunctions::SplitStringByDelimeter(str, ",", result);
ASSERT_TRUE(status.ok());
ASSERT_EQ(result.size(), 3UL);
result.clear();
status = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result);
status = ms::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
ASSERT_TRUE(status.ok());
ASSERT_EQ(result.size(), 3UL);
result.clear();
status = server::StringHelpFunctions::SplitStringByQuote(str , ",", "", result);
status = ms::server::StringHelpFunctions::SplitStringByQuote(str, ",", "", result);
ASSERT_TRUE(status.ok());
ASSERT_EQ(result.size(), 3UL);
str = "55,\"aa,gg,yy\",b";
result.clear();
status = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result);
status = ms::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result);
ASSERT_TRUE(status.ok());
ASSERT_EQ(result.size(), 3UL);
}
TEST(UtilTest, BLOCKINGQUEUE_TEST) {
server::BlockingQueue<std::string> bq;
ms::server::BlockingQueue<std::string> bq;
static const size_t count = 10;
bq.SetCapacity(count);
for(size_t i = 1; i <= count; i++) {
for (size_t i = 1; i <= count; i++) {
std::string id = "No." + std::to_string(i);
bq.Put(id);
}
@ -161,7 +160,7 @@ TEST(UtilTest, BLOCKINGQUEUE_TEST) {
str = bq.Back();
ASSERT_EQ(str, "No." + std::to_string(count));
for(size_t i = 1; i <= count; i++) {
for (size_t i = 1; i <= count; i++) {
std::string id = "No." + std::to_string(i);
str = bq.Take();
ASSERT_EQ(id, str);
@ -171,54 +170,54 @@ TEST(UtilTest, BLOCKINGQUEUE_TEST) {
}
TEST(UtilTest, LOG_TEST) {
auto status = server::InitLog(LOG_FILE_PATH);
auto status = ms::server::InitLog(LOG_FILE_PATH);
ASSERT_TRUE(status.ok());
EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer));
EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::LogDetailedCrashReason));
std::string fname = server::CommonUtil::GetFileName(LOG_FILE_PATH);
std::string fname = ms::server::CommonUtil::GetFileName(LOG_FILE_PATH);
ASSERT_EQ(fname, "log_config.conf");
}
TEST(UtilTest, TIMERECORDER_TEST) {
for(int64_t log_level = 0; log_level <= 6; log_level++) {
if(log_level == 5) {
for (int64_t log_level = 0; log_level <= 6; log_level++) {
if (log_level == 5) {
continue; //skip fatal
}
TimeRecorder rc("time", log_level);
ms::TimeRecorder rc("time", log_level);
rc.RecordSection("end");
}
}
TEST(UtilTest, STATUS_TEST) {
auto status = Status::OK();
auto status = ms::Status::OK();
std::string str = status.ToString();
ASSERT_FALSE(str.empty());
status = Status(DB_ERROR, "mistake");
ASSERT_EQ(status.code(), DB_ERROR);
status = ms::Status(ms::DB_ERROR, "mistake");
ASSERT_EQ(status.code(), ms::DB_ERROR);
str = status.ToString();
ASSERT_FALSE(str.empty());
status = Status(DB_NOT_FOUND, "mistake");
ASSERT_EQ(status.code(), DB_NOT_FOUND);
status = ms::Status(ms::DB_NOT_FOUND, "mistake");
ASSERT_EQ(status.code(), ms::DB_NOT_FOUND);
str = status.ToString();
ASSERT_FALSE(str.empty());
status = Status(DB_ALREADY_EXIST, "mistake");
ASSERT_EQ(status.code(), DB_ALREADY_EXIST);
status = ms::Status(ms::DB_ALREADY_EXIST, "mistake");
ASSERT_EQ(status.code(), ms::DB_ALREADY_EXIST);
str = status.ToString();
ASSERT_FALSE(str.empty());
status = Status(DB_META_TRANSACTION_FAILED, "mistake");
ASSERT_EQ(status.code(), DB_META_TRANSACTION_FAILED);
status = ms::Status(ms::DB_META_TRANSACTION_FAILED, "mistake");
ASSERT_EQ(status.code(), ms::DB_META_TRANSACTION_FAILED);
str = status.ToString();
ASSERT_FALSE(str.empty());
auto status_copy = Status::OK();
auto status_copy = ms::Status::OK();
CopyStatus(status_copy, status);
ASSERT_EQ(status.code(), DB_META_TRANSACTION_FAILED);
ASSERT_EQ(status.code(), ms::DB_META_TRANSACTION_FAILED);
auto status_ref(status);
ASSERT_EQ(status_ref.code(), status.code());
@ -231,123 +230,140 @@ TEST(UtilTest, STATUS_TEST) {
TEST(ValidationUtilTest, VALIDATE_TABLENAME_TEST) {
std::string table_name = "Normal123_";
auto status = server::ValidationUtil::ValidateTableName(table_name);
auto status = ms::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_TRUE(status.ok());
table_name = "12sds";
status = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), SERVER_INVALID_TABLE_NAME);
status = ms::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME);
table_name = "";
status = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), SERVER_INVALID_TABLE_NAME);
status = ms::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME);
table_name = "_asdasd";
status = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), SERVER_SUCCESS);
status = ms::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), ms::SERVER_SUCCESS);
table_name = "!@#!@";
status = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), SERVER_INVALID_TABLE_NAME);
status = ms::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME);
table_name = "_!@#!@";
status = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), SERVER_INVALID_TABLE_NAME);
status = ms::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME);
table_name = "中文";
status = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), SERVER_INVALID_TABLE_NAME);
status = ms::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME);
table_name = std::string(10000, 'a');
status = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), SERVER_INVALID_TABLE_NAME);
status = ms::server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME);
}
TEST(ValidationUtilTest, VALIDATE_DIMENSION_TEST) {
ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(-1).code(), SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(0).code(), SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16385).code(), SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16384).code(), SERVER_SUCCESS);
ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(1).code(), SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(-1).code(), ms::SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(0).code(), ms::SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(16385).code(), ms::SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(16384).code(), ms::SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(1).code(), ms::SERVER_SUCCESS);
}
TEST(ValidationUtilTest, VALIDATE_INDEX_TEST) {
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::INVALID).code(), SERVER_INVALID_INDEX_TYPE);
for(int i = 1; i <= (int)engine::EngineType::MAX_VALUE; i++) {
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType(i).code(), SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexType((int) ms::engine::EngineType::INVALID).code(),
ms::SERVER_INVALID_INDEX_TYPE);
for (int i = 1; i <= (int) ms::engine::EngineType::MAX_VALUE; i++) {
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexType(i).code(), ms::SERVER_SUCCESS);
}
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1).code(), SERVER_INVALID_INDEX_TYPE);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexType((int) ms::engine::EngineType::MAX_VALUE + 1).code(),
ms::SERVER_INVALID_INDEX_TYPE);
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexNlist(0).code(), SERVER_INVALID_INDEX_NLIST);
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexNlist(100).code(), SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexNlist(0).code(), ms::SERVER_INVALID_INDEX_NLIST);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexNlist(100).code(), ms::SERVER_SUCCESS);
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexFileSize(0).code(), SERVER_INVALID_INDEX_FILE_SIZE);
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexFileSize(100).code(), SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexFileSize(0).code(), ms::SERVER_INVALID_INDEX_FILE_SIZE);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexFileSize(100).code(), ms::SERVER_SUCCESS);
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(0).code(), SERVER_INVALID_INDEX_METRIC_TYPE);
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(1).code(), SERVER_SUCCESS);
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(2).code(), SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexMetricType(0).code(), ms::SERVER_INVALID_INDEX_METRIC_TYPE);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexMetricType(1).code(), ms::SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexMetricType(2).code(), ms::SERVER_SUCCESS);
}
TEST(ValidationUtilTest, VALIDATE_TOPK_TEST) {
engine::meta::TableSchema schema;
ASSERT_EQ(server::ValidationUtil::ValidateSearchTopk(10, schema).code(), SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(65536, schema).code(), SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(0, schema).code(), SERVER_SUCCESS);
ms::engine::meta::TableSchema schema;
ASSERT_EQ(ms::server::ValidationUtil::ValidateSearchTopk(10, schema).code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateSearchTopk(65536, schema).code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateSearchTopk(0, schema).code(), ms::SERVER_SUCCESS);
}
TEST(ValidationUtilTest, VALIDATE_NPROBE_TEST) {
engine::meta::TableSchema schema;
ms::engine::meta::TableSchema schema;
schema.nlist_ = 100;
ASSERT_EQ(server::ValidationUtil::ValidateSearchNprobe(10, schema).code(), SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(0, schema).code(), SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(101, schema).code(), SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateSearchNprobe(10, schema).code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateSearchNprobe(0, schema).code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateSearchNprobe(101, schema).code(), ms::SERVER_SUCCESS);
}
TEST(ValidationUtilTest, VALIDATE_GPU_TEST) {
ASSERT_EQ(server::ValidationUtil::ValidateGpuIndex(0).code(), SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::ValidateGpuIndex(100).code(), SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateGpuIndex(0).code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateGpuIndex(100).code(), ms::SERVER_SUCCESS);
size_t memory = 0;
ASSERT_EQ(server::ValidationUtil::GetGpuMemory(0, memory).code(), SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::GetGpuMemory(100, memory).code(), SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::GetGpuMemory(0, memory).code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::GetGpuMemory(100, memory).code(), ms::SERVER_SUCCESS);
}
TEST(ValidationUtilTest, VALIDATE_IPADDRESS_TEST) {
ASSERT_EQ(server::ValidationUtil::ValidateIpAddress("127.0.0.1").code(), SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::ValidateIpAddress("not ip").code(), SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateIpAddress("127.0.0.1").code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateIpAddress("not ip").code(), ms::SERVER_SUCCESS);
}
TEST(ValidationUtilTest, VALIDATE_NUMBER_TEST) {
ASSERT_EQ(server::ValidationUtil::ValidateStringIsNumber("1234").code(), SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::ValidateStringIsNumber("not number").code(), SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateStringIsNumber("1234").code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateStringIsNumber("not number").code(), ms::SERVER_SUCCESS);
}
TEST(ValidationUtilTest, VALIDATE_BOOL_TEST) {
std::string str = "true";
ASSERT_EQ(server::ValidationUtil::ValidateStringIsBool(str).code(), SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateStringIsBool(str).code(), ms::SERVER_SUCCESS);
str = "not bool";
ASSERT_NE(server::ValidationUtil::ValidateStringIsBool(str).code(), SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateStringIsBool(str).code(), ms::SERVER_SUCCESS);
}
TEST(ValidationUtilTest, VALIDATE_DOUBLE_TEST) {
ASSERT_EQ(server::ValidationUtil::ValidateStringIsFloat("2.5").code(), SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::ValidateStringIsFloat("not double").code(), SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateStringIsFloat("2.5").code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateStringIsFloat("not double").code(), ms::SERVER_SUCCESS);
}
TEST(ValidationUtilTest, VALIDATE_DBURI_TEST) {
ASSERT_EQ(server::ValidationUtil::ValidateDbURI("sqlite://:@:/").code(), SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::ValidateDbURI("xxx://:@:/").code(), SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::ValidateDbURI("not uri").code(), SERVER_SUCCESS);
ASSERT_EQ(server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:3303/milvus").code(), SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:port/milvus").code(), SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateDbURI("sqlite://:@:/").code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateDbURI("xxx://:@:/").code(), ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateDbURI("not uri").code(), ms::SERVER_SUCCESS);
ASSERT_EQ(ms::server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:3303/milvus").code(),
ms::SERVER_SUCCESS);
ASSERT_NE(ms::server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:port/milvus").code(),
ms::SERVER_SUCCESS);
}
TEST(UtilTest, ROLLOUTHANDLER_TEST){
TEST(UtilTest, ROLLOUTHANDLER_TEST) {
std::string dir1 = "/tmp/milvus_test";
std::string dir2 = "/tmp/milvus_test/log_test";
std::string filename[6] = {"log_global.log", "log_debug.log", "log_warning.log", "log_trace.log", "log_error.log", "log_fatal.log"};
el::Level list[6] = {el::Level::Global, el::Level::Debug, el::Level::Warning, el::Level::Trace, el::Level::Error, el::Level::Fatal};
std::string filename[6] = {
"log_global.log",
"log_debug.log",
"log_warning.log",
"log_trace.log",
"log_error.log",
"log_fatal.log"};
el::Level list[6] = {
el::Level::Global,
el::Level::Debug,
el::Level::Warning,
el::Level::Trace,
el::Level::Error,
el::Level::Fatal};
mkdir(dir1.c_str(), S_IRWXU);
mkdir(dir2.c_str(), S_IRWXU);
@ -358,7 +374,7 @@ TEST(UtilTest, ROLLOUTHANDLER_TEST){
file.open(tmp.c_str());
file << "zilliz" << std::endl;
server::RolloutHandler(tmp.c_str(), 0, list[i]);
ms::server::RolloutHandler(tmp.c_str(), 0, list[i]);
tmp.append(".1");
std::ifstream file2;
@ -369,4 +385,4 @@ TEST(UtilTest, ROLLOUTHANDLER_TEST){
ASSERT_EQ(tmp2, "zilliz");
}
boost::filesystem::remove_all(dir2);
}
}

29
cpp/unittest/wrapper/utils.cpp
View File

@ -18,12 +18,12 @@
#include <faiss/IndexFlat.h>
#include "utils.h"
#include "wrapper/utils.h"
void DataGenBase::GenData(const int &dim, const int &nb, const int &nq,
float *xb, float *xq, long *ids,
const int &k, long *gt_ids, float *gt_dis) {
void
DataGenBase::GenData(const int &dim, const int &nb, const int &nq,
float *xb, float *xq, int64_t *ids,
const int &k, int64_t *gt_ids, float *gt_dis) {
for (auto i = 0; i < nb; ++i) {
for (auto j = 0; j < dim; ++j) {
//p_data[i * d + j] = float(base + i);
@ -42,15 +42,16 @@ void DataGenBase::GenData(const int &dim, const int &nb, const int &nq,
index.search(nq, xq, k, gt_dis, gt_ids);
}
void DataGenBase::GenData(const int &dim,
const int &nb,
const int &nq,
std::vector<float> &xb,
std::vector<float> &xq,
std::vector<long> &ids,
const int &k,
std::vector<long> &gt_ids,
std::vector<float> &gt_dis) {
void
DataGenBase::GenData(const int &dim,
const int &nb,
const int &nq,
std::vector<float> &xb,
std::vector<float> &xq,
std::vector<int64_t> &ids,
const int &k,
std::vector<int64_t> &gt_ids,
std::vector<float> &gt_dis) {
xb.resize(nb * dim);
xq.resize(nq * dim);
ids.resize(nb);

10
cpp/unittest/wrapper/utils.h
View File

@ -24,25 +24,23 @@
#include <cstdio>
#include <fstream>
class DataGenBase;
using DataGenPtr = std::shared_ptr<DataGenBase>;
class DataGenBase {
public:
virtual void GenData(const int &dim, const int &nb, const int &nq, float *xb, float *xq, long *ids,
const int &k, long *gt_ids, float *gt_dis);
virtual void GenData(const int &dim, const int &nb, const int &nq, float *xb, float *xq, int64_t *ids,
const int &k, int64_t *gt_ids, float *gt_dis);
virtual void GenData(const int &dim,
const int &nb,
const int &nq,
std::vector<float> &xb,
std::vector<float> &xq,
std::vector<long> &ids,
std::vector<int64_t> &ids,
const int &k,
std::vector<long> &gt_ids,
std::vector<int64_t> &gt_ids,
std::vector<float> &gt_dis);
};

73
cpp/unittest/wrapper/wrapper_test.cpp
View File

@ -19,15 +19,18 @@
#include "src/wrapper/VecIndex.h"
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
#include "utils.h"
#include "wrapper/utils.h"
#include <gtest/gtest.h>
INITIALIZE_EASYLOGGINGPP
using namespace zilliz::milvus::engine;
//using namespace zilliz::knowhere;
namespace {
namespace ms = zilliz::milvus::engine;
namespace kw = zilliz::knowhere;
} // namespace
using ::testing::TestWithParam;
using ::testing::Values;
@ -39,30 +42,30 @@ constexpr int64_t DEVICE_ID = 0;
class ParamGenerator {
public:
static ParamGenerator& GetInstance(){
static ParamGenerator &GetInstance() {
static ParamGenerator instance;
return instance;
}
Config Gen(const IndexType& type) {
kw::Config Gen(const ms::IndexType &type) {
switch (type) {
case IndexType::FAISS_IDMAP: {
case ms::IndexType::FAISS_IDMAP: {
auto tempconf = std::make_shared<zilliz::knowhere::Cfg>();
tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2;
return tempconf;
}
case IndexType::FAISS_IVFFLAT_CPU:
case IndexType::FAISS_IVFFLAT_GPU:
case IndexType::FAISS_IVFFLAT_MIX: {
case ms::IndexType::FAISS_IVFFLAT_CPU:
case ms::IndexType::FAISS_IVFFLAT_GPU:
case ms::IndexType::FAISS_IVFFLAT_MIX: {
auto tempconf = std::make_shared<zilliz::knowhere::IVFCfg>();
tempconf->nlist = 100;
tempconf->nprobe = 16;
tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2;
return tempconf;
}
case IndexType::FAISS_IVFSQ8_CPU:
case IndexType::FAISS_IVFSQ8_GPU:
case IndexType::FAISS_IVFSQ8_MIX: {
case ms::IndexType::FAISS_IVFSQ8_CPU:
case ms::IndexType::FAISS_IVFSQ8_GPU:
case ms::IndexType::FAISS_IVFSQ8_MIX: {
auto tempconf = std::make_shared<zilliz::knowhere::IVFSQCfg>();
tempconf->nlist = 100;
tempconf->nprobe = 16;
@ -70,8 +73,8 @@ class ParamGenerator {
tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2;
return tempconf;
}
case IndexType::FAISS_IVFPQ_CPU:
case IndexType::FAISS_IVFPQ_GPU: {
case ms::IndexType::FAISS_IVFPQ_CPU:
case ms::IndexType::FAISS_IVFPQ_GPU: {
auto tempconf = std::make_shared<zilliz::knowhere::IVFPQCfg>();
tempconf->nlist = 100;
tempconf->nprobe = 16;
@ -80,7 +83,7 @@ class ParamGenerator {
tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2;
return tempconf;
}
case IndexType::NSG_MIX: {
case ms::IndexType::NSG_MIX: {
auto tempconf = std::make_shared<zilliz::knowhere::NSGCfg>();
tempconf->nlist = 100;
tempconf->nprobe = 16;
@ -97,10 +100,13 @@ class ParamGenerator {
};
class KnowhereWrapperTest
: public TestWithParam<::std::tuple<IndexType, std::string, int, int, int, int>> {
: public TestWithParam<::std::tuple<ms::IndexType, std::string, int, int, int, int>> {
protected:
void SetUp() override {
zilliz::knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID,1024*1024*200, 1024*1024*300, 2);
zilliz::knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID,
1024 * 1024 * 200,
1024 * 1024 * 300,
2);
std::string generator_type;
std::tie(index_type, generator_type, dim, nb, nq, k) = GetParam();
@ -115,11 +121,12 @@ class KnowhereWrapperTest
conf->d = dim;
conf->gpu_id = DEVICE_ID;
}
void TearDown() override {
zilliz::knowhere::FaissGpuResourceMgr::GetInstance().Free();
}
void AssertResult(const std::vector<long> &ids, const std::vector<float> &dis) {
void AssertResult(const std::vector<int64_t> &ids, const std::vector<float> &dis) {
EXPECT_EQ(ids.size(), nq * k);
EXPECT_EQ(dis.size(), nq * k);
@ -146,8 +153,8 @@ class KnowhereWrapperTest
}
protected:
IndexType index_type;
Config conf;
ms::IndexType index_type;
kw::Config conf;
int dim = DIM;
int nb = NB;
@ -155,27 +162,27 @@ class KnowhereWrapperTest
int k = 10;
std::vector<float> xb;
std::vector<float> xq;
std::vector<long> ids;
std::vector<int64_t> ids;
VecIndexPtr index_ = nullptr;
ms::VecIndexPtr index_ = nullptr;
// Ground Truth
std::vector<long> gt_ids;
std::vector<int64_t> gt_ids;
std::vector<float> gt_dis;
};
INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest,
Values(
//["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"]
std::make_tuple(IndexType::FAISS_IVFFLAT_CPU, "Default", 64, 100000, 10, 10),
std::make_tuple(IndexType::FAISS_IVFFLAT_GPU, "Default", DIM, NB, 10, 10),
std::make_tuple(IndexType::FAISS_IVFFLAT_MIX, "Default", 64, 100000, 10, 10),
std::make_tuple(IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10),
std::make_tuple(IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10),
std::make_tuple(IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10),
std::make_tuple(ms::IndexType::FAISS_IVFFLAT_CPU, "Default", 64, 100000, 10, 10),
std::make_tuple(ms::IndexType::FAISS_IVFFLAT_GPU, "Default", DIM, NB, 10, 10),
std::make_tuple(ms::IndexType::FAISS_IVFFLAT_MIX, "Default", 64, 100000, 10, 10),
std::make_tuple(ms::IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10),
std::make_tuple(ms::IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10),
std::make_tuple(ms::IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10),
// std::make_tuple(IndexType::NSG_MIX, "Default", 128, 250000, 10, 10),
// std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 250000, 10, 10),
std::make_tuple(IndexType::FAISS_IDMAP, "Default", 64, 100000, 10, 10)
std::make_tuple(ms::IndexType::FAISS_IDMAP, "Default", 64, 100000, 10, 10)
)
);
@ -213,7 +220,7 @@ TEST_P(KnowhereWrapperTest, TO_GPU_TEST) {
{
std::string file_location = "/tmp/knowhere_gpu_file";
write_index(index_, file_location);
auto new_index = read_index(file_location);
auto new_index = ms::read_index(file_location);
auto dev_idx = new_index->CopyToGpu(DEVICE_ID);
for (int i = 0; i < 10; ++i) {
@ -254,7 +261,7 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) {
{
std::string file_location = "/tmp/knowhere";
write_index(index_, file_location);
auto new_index = read_index(file_location);
auto new_index = ms::read_index(file_location);
EXPECT_EQ(new_index->GetType(), ConvertToCpuIndexType(index_type));
EXPECT_EQ(new_index->Dimension(), index_->Dimension());
EXPECT_EQ(new_index->Count(), index_->Count());

17
cpp/version.h.macro
View File

@ -1,3 +1,20 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#define MILVUS_VERSION "@MILVUS_VERSION@"