From 5bb1d001e26230f1dc2ace92012c82fb66bf8045 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Fri, 6 Mar 2020 15:21:39 +0800 Subject: [PATCH] Fix#1530 (#1533) * update Signed-off-by: youny626 * update Signed-off-by: Zhiru Zhu * update Signed-off-by: Zhiru Zhu Co-authored-by: Jin Hai --- CHANGELOG.md | 1 + core/src/db/meta/MySQLMetaImpl.cpp | 12 ++++- core/src/db/meta/SqliteMetaImpl.cpp | 79 +++++++++++++++++++---------- core/src/segment/SegmentWriter.cpp | 2 + 4 files changed, 65 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b100be4ed..2df6e52a77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ Please mark all change in change log and use the issue from GitHub - \#1510 Add set interfaces for WAL configurations - \#1511 Fix big integer cannot pass to server correctly - \#1518 Table count did not match after deleting vectors and compact +- \#1530 Set table file with correct engine type in meta - \#1525 Add setter API for config preload_table ## Feature diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index a80371c1fb..0016c47ef4 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -37,6 +37,7 @@ #include "utils/Exception.h" #include "utils/Log.h" #include "utils/StringHelpFunctions.h" +#include "utils/ValidationUtil.h" namespace milvus { namespace engine { @@ -672,7 +673,16 @@ MySQLMetaImpl::CreateTableFile(TableFileSchema& file_schema) { file_schema.created_on_ = utils::GetMicroSecTimeStamp(); file_schema.updated_time_ = file_schema.created_on_; file_schema.index_file_size_ = table_schema.index_file_size_; - file_schema.engine_type_ = table_schema.engine_type_; + + if (file_schema.file_type_ == TableFileSchema::FILE_TYPE::NEW || + file_schema.file_type_ == TableFileSchema::FILE_TYPE::NEW_MERGE) { + file_schema.engine_type_ = server::ValidationUtil::IsBinaryMetricType(table_schema.metric_type_) + ? (int32_t)EngineType::FAISS_BIN_IDMAP + : (int32_t)EngineType::FAISS_IDMAP; + } else { + file_schema.engine_type_ = table_schema.engine_type_; + } + file_schema.nlist_ = table_schema.nlist_; file_schema.metric_type_ = table_schema.metric_type_; diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index ba029a53cf..748cbb4944 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -22,7 +23,6 @@ #include #include #include -#include #include "MetaConsts.h" #include "db/IDGenerator.h" @@ -33,6 +33,7 @@ #include "utils/Exception.h" #include "utils/Log.h" #include "utils/StringHelpFunctions.h" +#include "utils/ValidationUtil.h" namespace milvus { namespace engine { @@ -372,7 +373,16 @@ SqliteMetaImpl::CreateTableFile(TableFileSchema& file_schema) { file_schema.created_on_ = utils::GetMicroSecTimeStamp(); file_schema.updated_time_ = file_schema.created_on_; file_schema.index_file_size_ = table_schema.index_file_size_; - file_schema.engine_type_ = table_schema.engine_type_; + + if (file_schema.file_type_ == TableFileSchema::FILE_TYPE::NEW || + file_schema.file_type_ == TableFileSchema::FILE_TYPE::NEW_MERGE) { + file_schema.engine_type_ = server::ValidationUtil::IsBinaryMetricType(table_schema.metric_type_) + ? (int32_t)EngineType::FAISS_BIN_IDMAP + : (int32_t)EngineType::FAISS_IDMAP; + } else { + file_schema.engine_type_ = table_schema.engine_type_; + } + file_schema.nlist_ = table_schema.nlist_; file_schema.metric_type_ = table_schema.metric_type_; @@ -447,9 +457,10 @@ SqliteMetaImpl::GetTableFilesBySegmentId(const std::string& segment_id, try { table_files.clear(); auto files = ConnectorPtr->select( - columns(&TableFileSchema::id_, &TableFileSchema::table_id_, &TableFileSchema::segment_id_, &TableFileSchema::file_id_, - &TableFileSchema::file_type_, &TableFileSchema::file_size_, &TableFileSchema::row_count_, - &TableFileSchema::date_, &TableFileSchema::engine_type_, &TableFileSchema::created_on_), + columns(&TableFileSchema::id_, &TableFileSchema::table_id_, &TableFileSchema::segment_id_, + &TableFileSchema::file_id_, &TableFileSchema::file_type_, &TableFileSchema::file_size_, + &TableFileSchema::row_count_, &TableFileSchema::date_, &TableFileSchema::engine_type_, + &TableFileSchema::created_on_), where(c(&TableFileSchema::segment_id_) == segment_id and c(&TableFileSchema::file_type_) != (int)TableFileSchema::TO_DELETE)); @@ -875,13 +886,13 @@ SqliteMetaImpl::ShowPartitions(const std::string& table_id, std::vectorselect( - columns(&TableSchema::id_, &TableSchema::state_, &TableSchema::dimension_, &TableSchema::created_on_, - &TableSchema::flag_, &TableSchema::index_file_size_, &TableSchema::engine_type_, - &TableSchema::nlist_, &TableSchema::metric_type_, &TableSchema::partition_tag_, - &TableSchema::version_, &TableSchema::table_id_), - where(c(&TableSchema::owner_table_) == table_id and - c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); + auto partitions = + ConnectorPtr->select(columns(&TableSchema::id_, &TableSchema::state_, &TableSchema::dimension_, + &TableSchema::created_on_, &TableSchema::flag_, &TableSchema::index_file_size_, + &TableSchema::engine_type_, &TableSchema::nlist_, &TableSchema::metric_type_, + &TableSchema::partition_tag_, &TableSchema::version_, &TableSchema::table_id_), + where(c(&TableSchema::owner_table_) == table_id and + c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); for (size_t i = 0; i < partitions.size(); i++) { meta::TableSchema partition_schema; partition_schema.id_ = std::get<0>(partitions[i]); @@ -1185,19 +1196,26 @@ SqliteMetaImpl::FilesByType(const std::string& table_id, const std::vector& file_schema.metric_type_ = table_schema.metric_type_; switch (file_schema.file_type_) { - case (int)TableFileSchema::RAW:++raw_count; + case (int)TableFileSchema::RAW: + ++raw_count; break; - case (int)TableFileSchema::NEW:++new_count; + case (int)TableFileSchema::NEW: + ++new_count; break; - case (int)TableFileSchema::NEW_MERGE:++new_merge_count; + case (int)TableFileSchema::NEW_MERGE: + ++new_merge_count; break; - case (int)TableFileSchema::NEW_INDEX:++new_index_count; + case (int)TableFileSchema::NEW_INDEX: + ++new_index_count; break; - case (int)TableFileSchema::TO_INDEX:++to_index_count; + case (int)TableFileSchema::TO_INDEX: + ++to_index_count; break; - case (int)TableFileSchema::INDEX:++index_count; + case (int)TableFileSchema::INDEX: + ++index_count; break; - case (int)TableFileSchema::BACKUP:++backup_count; + case (int)TableFileSchema::BACKUP: + ++backup_count; break; default: break; @@ -1214,21 +1232,26 @@ SqliteMetaImpl::FilesByType(const std::string& table_id, const std::vector& std::string msg = "Get table files by type."; for (int file_type : file_types) { switch (file_type) { - case (int)TableFileSchema::RAW:msg = msg + " raw files:" + std::to_string(raw_count); + case (int)TableFileSchema::RAW: + msg = msg + " raw files:" + std::to_string(raw_count); break; - case (int)TableFileSchema::NEW:msg = msg + " new files:" + std::to_string(new_count); + case (int)TableFileSchema::NEW: + msg = msg + " new files:" + std::to_string(new_count); break; - case (int)TableFileSchema::NEW_MERGE:msg = msg + " new_merge files:" - + std::to_string(new_merge_count); + case (int)TableFileSchema::NEW_MERGE: + msg = msg + " new_merge files:" + std::to_string(new_merge_count); break; - case (int)TableFileSchema::NEW_INDEX:msg = msg + " new_index files:" - + std::to_string(new_index_count); + case (int)TableFileSchema::NEW_INDEX: + msg = msg + " new_index files:" + std::to_string(new_index_count); break; - case (int)TableFileSchema::TO_INDEX:msg = msg + " to_index files:" + std::to_string(to_index_count); + case (int)TableFileSchema::TO_INDEX: + msg = msg + " to_index files:" + std::to_string(to_index_count); break; - case (int)TableFileSchema::INDEX:msg = msg + " index files:" + std::to_string(index_count); + case (int)TableFileSchema::INDEX: + msg = msg + " index files:" + std::to_string(index_count); break; - case (int)TableFileSchema::BACKUP:msg = msg + " backup files:" + std::to_string(backup_count); + case (int)TableFileSchema::BACKUP: + msg = msg + " backup files:" + std::to_string(backup_count); break; default: break; diff --git a/core/src/segment/SegmentWriter.cpp b/core/src/segment/SegmentWriter.cpp index 1e8c509eeb..c80776e2f5 100644 --- a/core/src/segment/SegmentWriter.cpp +++ b/core/src/segment/SegmentWriter.cpp @@ -243,9 +243,11 @@ size_t SegmentWriter::Size() { // TODO(zhiru): switch to actual directory size size_t ret = segment_ptr_->vectors_ptr_->Size(); + /* if (segment_ptr_->id_bloom_filter_ptr_) { ret += segment_ptr_->id_bloom_filter_ptr_->Size(); } + */ return ret; }