diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 6b432056c5..6339b0c440 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -15,6 +15,7 @@ db_config: index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB archive_days_threshold: 30 # files older than x days will be archived, unit: day + sql_echo: on # print sql statement in debug log metric_config: is_startup: off # if monitoring start: on, off diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index 0978d77a7c..143e68be06 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -64,6 +64,10 @@ public: max_idle_time_ = max_idle; } + std::string getDB() { + return db_; + } + protected: // Superclass overrides diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 4d5fc00050..2051c12f49 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -1733,17 +1733,30 @@ namespace meta { // ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } - ENGINE_LOG_DEBUG << "Remove table file type as NEW"; Query cleanUpQuery = connectionPtr->query(); - cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; + cleanUpQuery << "SELECT table_name " << + "FROM information_schema.tables " << + "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << quote << " " << + "AND table_name = " << quote << "TableFiles" << quote << ";"; if (options_.sql_echo) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); } - if (!cleanUpQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; - return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); + StoreQueryResult res = cleanUpQuery.store(); + assert(res); + if (!res.empty()) { + ENGINE_LOG_DEBUG << "Remove table file type as NEW"; + cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); + } + + if (!cleanUpQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; + return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); + } } } catch (const BadQuery& er) { diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index 14d9446d47..85fe749460 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -72,7 +72,7 @@ TEST_F(MySQLDBTest, DB_TEST) { std::thread search([&]() { engine::QueryResults results; int k = 10; - std::this_thread::sleep_for(std::chrono::seconds(3)); + std::this_thread::sleep_for(std::chrono::seconds(5)); INIT_TIMER; std::stringstream ss; @@ -119,8 +119,6 @@ TEST_F(MySQLDBTest, DB_TEST) { search.join(); -// db_->DropAll(); - delete db_; auto dummyDB = engine::DBFactory::Build(options); diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index 436086acb3..76d7846362 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -21,184 +21,7 @@ using namespace zilliz::milvus::engine; -//TEST_F(MySQLTest, InitializeTest) { -// DBMetaOptions options; -// //dialect+driver://username:password@host:port/database -// options.backend_uri = "mysql://root:1234@:/test"; -// meta::MySQLMetaImpl impl(options); -// auto status = impl.Initialize(); -// std::cout << status.ToString() << std::endl; -// ASSERT_TRUE(status.ok()); -//} - -TEST_F(MySQLTest, core) { - DBMetaOptions options; -// //dialect+driver://username:password@host:port/database -// options.backend_uri = "mysql://root:1234@:/test"; -// options.path = "/tmp/vecwise_test"; - try { - options = getDBMetaOptions(); - } catch(std::exception& ex) { - ASSERT_TRUE(false); - return; - } - - int mode = Options::MODE::SINGLE; - meta::MySQLMetaImpl impl(options, mode); -// auto status = impl.Initialize(); -// ASSERT_TRUE(status.ok()); - - meta::TableSchema schema1; - schema1.table_id_ = "test1"; - schema1.dimension_ = 123; - - auto status = impl.CreateTable(schema1); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - meta::TableSchema schema2; - schema2.table_id_ = "test2"; - schema2.dimension_ = 321; - status = impl.CreateTable(schema2); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - status = impl.CreateTable(schema2); -// std::cout << status.ToString() << std::endl; -// ASSERT_THROW(impl.CreateTable(schema), mysqlpp::BadQuery); - ASSERT_TRUE(status.ok()); - - status = impl.DeleteTable(schema2.table_id_); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - size_t id1 = schema1.id_; - long created_on1 = schema1.created_on_; - status = impl.DescribeTable(schema1); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(schema1.id_, id1); - ASSERT_EQ(schema1.table_id_, "test1"); - ASSERT_EQ(schema1.created_on_, created_on1); - ASSERT_EQ(schema1.files_cnt_, 0); - ASSERT_EQ(schema1.engine_type_, 1); - ASSERT_EQ(schema1.store_raw_data_, false); - - bool check; - status = impl.HasTable("test1", check); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(check, true); - - std::vector table_schema_array; - status = impl.AllTables(table_schema_array); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(table_schema_array.size(), 1); - meta::TableSchema resultSchema = table_schema_array[0]; - ASSERT_EQ(resultSchema.id_, id1); - ASSERT_EQ(resultSchema.table_id_, "test1"); - ASSERT_EQ(resultSchema.dimension_, 123); - ASSERT_EQ(resultSchema.files_cnt_, 0); - ASSERT_EQ(resultSchema.engine_type_, 1); - ASSERT_EQ(resultSchema.store_raw_data_, false); - - meta::TableFileSchema tableFileSchema; - tableFileSchema.table_id_ = "test1"; - - status = impl.CreateTableFile(tableFileSchema); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - tableFileSchema.file_type_ = meta::TableFileSchema::TO_INDEX; - status = impl.UpdateTableFile(tableFileSchema); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - meta::TableFilesSchema filesToIndex; - status = impl.FilesToIndex(filesToIndex); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(filesToIndex.size(), 1); - meta::TableFileSchema fileToIndex = filesToIndex[0]; - ASSERT_EQ(fileToIndex.table_id_, "test1"); - ASSERT_EQ(fileToIndex.dimension_, 123); - -// meta::TableFilesSchema filesToIndex; -// status = impl.FilesToIndex(filesToIndex); -// ASSERT_TRUE(status.ok()); -// ASSERT_EQ(filesToIndex.size(), 0); - - meta::DatesT partition; - partition.push_back(tableFileSchema.date_); - meta::DatePartionedTableFilesSchema filesToSearch; - status = impl.FilesToSearch(tableFileSchema.table_id_, partition, filesToSearch); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(filesToSearch.size(), 1); - ASSERT_EQ(filesToSearch[tableFileSchema.date_].size(), 1); - meta::TableFileSchema fileToSearch = filesToSearch[tableFileSchema.date_][0]; - ASSERT_EQ(fileToSearch.table_id_, "test1"); - ASSERT_EQ(fileToSearch.dimension_, 123); - - tableFileSchema.file_type_ = meta::TableFileSchema::RAW; - status = impl.UpdateTableFile(tableFileSchema); - ASSERT_TRUE(status.ok()); - - meta::DatePartionedTableFilesSchema filesToMerge; - status = impl.FilesToMerge(tableFileSchema.table_id_, filesToMerge); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - ASSERT_EQ(filesToMerge.size(), 1); - ASSERT_EQ(filesToMerge[tableFileSchema.date_].size(), 1); - meta::TableFileSchema fileToMerge = filesToMerge[tableFileSchema.date_][0]; - ASSERT_EQ(fileToMerge.table_id_, "test1"); - ASSERT_EQ(fileToMerge.dimension_, 123); - - meta::TableFilesSchema resultTableFilesSchema; - std::vector ids; - ids.push_back(tableFileSchema.id_); - status = impl.GetTableFiles(tableFileSchema.table_id_, ids, resultTableFilesSchema); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(resultTableFilesSchema.size(), 1); - meta::TableFileSchema resultTableFileSchema = resultTableFilesSchema[0]; -// ASSERT_EQ(resultTableFileSchema.id_, tableFileSchema.id_); - ASSERT_EQ(resultTableFileSchema.table_id_, tableFileSchema.table_id_); - ASSERT_EQ(resultTableFileSchema.file_id_, tableFileSchema.file_id_); - ASSERT_EQ(resultTableFileSchema.file_type_, tableFileSchema.file_type_); - ASSERT_EQ(resultTableFileSchema.size_, tableFileSchema.size_); - ASSERT_EQ(resultTableFileSchema.date_, tableFileSchema.date_); - ASSERT_EQ(resultTableFileSchema.engine_type_, tableFileSchema.engine_type_); - ASSERT_EQ(resultTableFileSchema.dimension_, tableFileSchema.dimension_); - - tableFileSchema.size_ = 234; - meta::TableSchema schema3; - schema3.table_id_ = "test3"; - schema3.dimension_ = 321; - status = impl.CreateTable(schema3); - ASSERT_TRUE(status.ok()); - meta::TableFileSchema tableFileSchema2; - tableFileSchema2.table_id_ = "test3"; - tableFileSchema2.size_ = 345; - status = impl.CreateTableFile(tableFileSchema2); - ASSERT_TRUE(status.ok()); - meta::TableFilesSchema filesToUpdate; - filesToUpdate.emplace_back(tableFileSchema); - filesToUpdate.emplace_back(tableFileSchema2); - status = impl.UpdateTableFile(tableFileSchema); - ASSERT_TRUE(status.ok()); - - uint64_t resultSize; - status = impl.Size(resultSize); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - ASSERT_EQ(resultSize, tableFileSchema.size_ + tableFileSchema2.size_); - - uint64_t countResult; - status = impl.Count(tableFileSchema.table_id_, countResult); - ASSERT_TRUE(status.ok()); - - status = impl.DropAll(); - ASSERT_TRUE(status.ok()); - -} - -TEST_F(MySQLTest, GROUP_TEST) { +TEST_F(MySQLTest, TABLE_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -210,38 +33,37 @@ TEST_F(MySQLTest, GROUP_TEST) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); ASSERT_TRUE(status.ok()); - auto gid = group.id_; - group.id_ = -1; - status = impl.DescribeTable(group); + auto gid = table.id_; + table.id_ = -1; + status = impl.DescribeTable(table); ASSERT_TRUE(status.ok()); - ASSERT_EQ(group.id_, gid); - ASSERT_EQ(group.table_id_, table_id); + ASSERT_EQ(table.id_, gid); + ASSERT_EQ(table.table_id_, table_id); - group.table_id_ = "not_found"; - status = impl.DescribeTable(group); + table.table_id_ = "not_found"; + status = impl.DescribeTable(table); ASSERT_TRUE(!status.ok()); - group.table_id_ = table_id; - status = impl.CreateTable(group); + table.table_id_ = table_id; + status = impl.CreateTable(table); ASSERT_TRUE(status.ok()); - group.table_id_ = ""; - status = impl.CreateTable(group); + table.table_id_ = ""; + status = impl.CreateTable(table); ASSERT_TRUE(status.ok()); - status = impl.DropAll(); ASSERT_TRUE(status.ok()); } -TEST_F(MySQLTest, table_file_TEST) { +TEST_F(MySQLTest, TABLE_FILE_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -253,17 +75,16 @@ TEST_F(MySQLTest, table_file_TEST) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - group.dimension_ = 256; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + table.dimension_ = 256; + auto status = impl.CreateTable(table); meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; status = impl.CreateTableFile(table_file); -// std::cout << status.ToString() << std::endl; ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); @@ -332,15 +153,15 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); @@ -391,13 +212,13 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { auto impl = meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; auto cnt = 10; auto each_size = 2UL; @@ -445,9 +266,9 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { auto table_id = "meta_test_group"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); int new_files_cnt = 4; int raw_files_cnt = 5; @@ -455,7 +276,7 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { int index_files_cnt = 7; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; for (auto i=0; i