diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.cpp index 31b2070b49..d7a2087ee0 100644 --- a/cpp/src/db/MemManager.cpp +++ b/cpp/src/db/MemManager.cpp @@ -62,7 +62,7 @@ Status MemVectors::Serialize(std::string& table_id) { auto status = pMeta_->UpdateTableFile(schema_); LOG(DEBUG) << "New " << ((schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index") - << " file " << schema_.file_id_ << " of size " << pEE_->Size() / meta::M << " M"; + << " file " << schema_.file_id_ << " of size " << (double)(pEE_->Size()) / (double)meta::M << " M"; pEE_->Cache(); diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 6cfbe9139e..aa311550ee 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include "utils.h" #include "db/DB.h" @@ -14,6 +15,31 @@ using namespace zilliz::milvus; +namespace { + +static const std::string TABLE_NAME = "test_group"; +static constexpr int64_t TABLE_DIM = 256; + +engine::meta::TableSchema BuildTableSchema() { + engine::meta::TableSchema table_info; + table_info.dimension_ = TABLE_DIM; + table_info.table_id_ = TABLE_NAME; + table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; + return table_info; +} + +void BuildVectors(int64_t n, std::vector& vectors) { + vectors.clear(); + vectors.resize(n*TABLE_DIM); + float* data = vectors.data(); + for(int i = 0; i < n; i++) { + for(int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); + data[TABLE_DIM * i] += i / 2000.; + } +} + +} + TEST_F(DBTest, CONFIG_TEST) { { ASSERT_ANY_THROW(engine::ArchiveConf conf("wrong")); @@ -60,52 +86,6 @@ TEST_F(DBTest, CONFIG_TEST) { } } -TEST_F(DBTest2, ARHIVE_DISK_CHECK) { - - static const std::string group_name = "test_group"; - static const int group_dim = 256; - uint64_t size; - - engine::meta::TableSchema group_info; - group_info.dimension_ = group_dim; - group_info.table_id_ = group_name; - group_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; - engine::Status stat = db_->CreateTable(group_info); - - engine::meta::TableSchema group_info_get; - group_info_get.table_id_ = group_name; - stat = db_->DescribeTable(group_info_get); - ASSERT_STATS(stat); - ASSERT_EQ(group_info_get.dimension_, group_dim); - - engine::IDNumbers vector_ids; - engine::IDNumbers target_ids; - - db_->Size(size); - int d = 256; - int nb = 20; - float *xb = new float[d * nb]; - for(int i = 0; i < nb; i++) { - for(int j = 0; j < d; j++) xb[d * i + j] = drand48(); - xb[d * i] += i / 2000.; - } - - int loop = 100000; - - for (auto i=0; iInsertVectors(group_name, nb, xb, vector_ids); - std::this_thread::sleep_for(std::chrono::microseconds(1)); - } - - std::this_thread::sleep_for(std::chrono::seconds(1)); - - db_->Size(size); - LOG(DEBUG) << "size=" << size; - ASSERT_LT(size, 1 * engine::meta::G); - - delete [] xb; -}; - TEST_F(DBTest, DB_TEST) { static const std::string table_name = "test_group"; @@ -126,20 +106,13 @@ TEST_F(DBTest, DB_TEST) { engine::IDNumbers vector_ids; engine::IDNumbers target_ids; - int d = 256; - int nb = 50; - float *xb = new float[d * nb]; - for(int i = 0; i < nb; i++) { - for(int j = 0; j < d; j++) xb[d * i + j] = drand48(); - xb[d * i] += i / 2000.; - } + int64_t nb = 50; + std::vector xb; + BuildVectors(nb, xb); - int qb = 5; - float *qxb = new float[d * qb]; - for(int i = 0; i < qb; i++) { - for(int j = 0; j < d; j++) qxb[d * i + j] = drand48(); - qxb[d * i] += i / 2000.; - } + int64_t qb = 5; + std::vector qxb; + BuildVectors(qb, qxb); std::thread search([&]() { engine::QueryResults results; @@ -157,7 +130,7 @@ TEST_F(DBTest, DB_TEST) { prev_count = count; START_TIMER; - stat = db_->Query(table_name, k, qb, qxb, results); + stat = db_->Query(table_name, k, qb, qxb.data(), results); ss << "Search " << j << " With Size " << count/engine::meta::M << " M"; STOP_TIMER(ss.str()); @@ -180,54 +153,45 @@ TEST_F(DBTest, DB_TEST) { for (auto i=0; iInsertVectors(table_name, qb, qxb, target_ids); + db_->InsertVectors(table_name, qb, qxb.data(), target_ids); ASSERT_EQ(target_ids.size(), qb); } else { - db_->InsertVectors(table_name, nb, xb, vector_ids); + db_->InsertVectors(table_name, nb, xb.data(), vector_ids); } std::this_thread::sleep_for(std::chrono::microseconds(1)); } search.join(); - - delete [] xb; - delete [] qxb; }; TEST_F(DBTest, SEARCH_TEST) { - static const std::string table_name = "test_group"; - static const int group_dim = 256; - - engine::meta::TableSchema table_info; - table_info.dimension_ = group_dim; - table_info.table_id_ = table_name; - table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; + engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = table_name; + table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_STATS(stat); - ASSERT_EQ(table_info_get.dimension_, group_dim); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); // prepare raw data size_t nb = 250000; size_t nq = 10; size_t k = 5; - std::vector xb(nb*group_dim); - std::vector xq(nq*group_dim); + std::vector xb(nb*TABLE_DIM); + std::vector xq(nq*TABLE_DIM); std::vector ids(nb); std::random_device rd; std::mt19937 gen(rd()); std::uniform_real_distribution<> dis_xt(-1.0, 1.0); - for (size_t i = 0; i < nb*group_dim; i++) { + for (size_t i = 0; i < nb*TABLE_DIM; i++) { xb[i] = dis_xt(gen); if (i < nb){ ids[i] = i; } } - for (size_t i = 0; i < nq*group_dim; i++) { + for (size_t i = 0; i < nq*TABLE_DIM; i++) { xq[i] = dis_xt(gen); } @@ -240,7 +204,7 @@ TEST_F(DBTest, SEARCH_TEST) { // insert data const int batch_size = 100; for (int j = 0; j < nb / batch_size; ++j) { - stat = db_->InsertVectors(table_name, batch_size, xb.data()+batch_size*j*group_dim, ids); + stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data()+batch_size*j*TABLE_DIM, ids); if (j == 200){ sleep(1);} ASSERT_STATS(stat); } @@ -248,9 +212,77 @@ TEST_F(DBTest, SEARCH_TEST) { sleep(2); // wait until build index finish engine::QueryResults results; - stat = db_->Query(table_name, k, nq, xq.data(), results); + stat = db_->Query(TABLE_NAME, k, nq, xq.data(), results); ASSERT_STATS(stat); // TODO(linxj): add groundTruth assert }; +TEST_F(DBTest2, ARHIVE_DISK_CHECK) { + + engine::meta::TableSchema table_info = BuildTableSchema(); + engine::Status stat = db_->CreateTable(table_info); + + engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = TABLE_NAME; + stat = db_->DescribeTable(table_info_get); + ASSERT_STATS(stat); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + engine::IDNumbers vector_ids; + engine::IDNumbers target_ids; + + uint64_t size; + db_->Size(size); + + int64_t nb = 10; + std::vector xb; + BuildVectors(nb, xb); + + int loop = 100000; + for (auto i=0; iInsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } + + std::this_thread::sleep_for(std::chrono::seconds(1)); + + db_->Size(size); + LOG(DEBUG) << "size=" << size; + ASSERT_LE(size, 1 * engine::meta::G); +}; + +TEST_F(DBTest2, DELETE_TEST) { + + + engine::meta::TableSchema table_info = BuildTableSchema(); + engine::Status stat = db_->CreateTable(table_info); + + engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = TABLE_NAME; + stat = db_->DescribeTable(table_info_get); + ASSERT_STATS(stat); + + ASSERT_TRUE(boost::filesystem::exists(table_info_get.location_)); + + engine::IDNumbers vector_ids; + + uint64_t size; + db_->Size(size); + + int64_t nb = 100000; + std::vector xb; + BuildVectors(nb, xb); + + int loop = 20; + for (auto i=0; iInsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } + + std::vector dates; + stat = db_->DeleteTable(TABLE_NAME, dates); + std::this_thread::sleep_for(std::chrono::seconds(2)); + ASSERT_TRUE(stat.ok()); + ASSERT_FALSE(boost::filesystem::exists(table_info_get.location_)); +};