milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

remove dependency of sqlite_orm (#4129) 

Signed-off-by: yhmo <yihua.mo@zilliz.com>
pull/4138/head
groot 2020-10-30 09:59:11 +08:00 committed by GitHub
parent a9e82f37a9
commit d41835585a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 63 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/cmake/DefineOptions.cmake
View File

@ -69,8 +69,6 @@ define_option(MILVUS_WITH_PROMETHEUS "Build with PROMETHEUS library" ON)
define_option(MILVUS_WITH_SQLITE "Build with SQLite library" ON)
define_option(MILVUS_WITH_SQLITE_ORM "Build with SQLite ORM library" ON)
define_option(MILVUS_WITH_MYSQLPP "Build with MySQL++" ON)
define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON)

61
core/cmake/ThirdPartyPackages.cmake
View File

@ -15,7 +15,6 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES
MySQLPP
Prometheus
SQLite
SQLite_ORM
yaml-cpp
libunwind
gperftools
@ -45,8 +44,6 @@ macro(build_dependency DEPENDENCY_NAME)
build_prometheus()
elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite")
build_sqlite()
elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite_ORM")
build_sqlite_orm()
elseif ("${DEPENDENCY_NAME}" STREQUAL "yaml-cpp")
build_yamlcpp()
elseif ("${DEPENDENCY_NAME}" STREQUAL "libunwind")
@ -611,64 +608,6 @@ if (MILVUS_WITH_SQLITE)
link_directories(SYSTEM ${SQLITE_PREFIX}/lib/)
endif ()
# ----------------------------------------------------------------------
# SQLite_ORM
macro(build_sqlite_orm)
message(STATUS "Building SQLITE_ORM-${SQLITE_ORM_VERSION} from source")
set(SQLITE_ORM_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/sqlite_orm_ep-prefix")
set(SQLITE_ORM_TAR_NAME "${SQLITE_ORM_PREFIX}/sqlite_orm-${SQLITE_ORM_VERSION}.tar.gz")
set(SQLITE_ORM_INCLUDE_DIR "${SQLITE_ORM_PREFIX}/sqlite_orm-${SQLITE_ORM_VERSION}/include/sqlite_orm")
if (NOT EXISTS ${SQLITE_ORM_INCLUDE_DIR})
file(MAKE_DIRECTORY ${SQLITE_ORM_PREFIX})
set(IS_EXIST_FILE FALSE)
foreach(url ${SQLITE_ORM_SOURCE_URLS})
file(DOWNLOAD ${url}
${SQLITE_ORM_TAR_NAME}
TIMEOUT 60
STATUS status
LOG log)
list(GET status 0 status_code)
list(GET status 1 status_string)
if(status_code EQUAL 0)
message(STATUS "Downloading SQLITE_ORM ... done")
set(IS_EXIST_FILE TRUE)
break()
else()
string(APPEND logFailedURLs "error: downloading '${url}' failed
status_code: ${status_code}
status_string: ${status_string}
log:
--- LOG BEGIN ---
${log}
--- LOG END ---
"
)
endif()
endforeach()
if(IS_EXIST_FILE STREQUAL "FALSE")
message(FATAL_ERROR "Each download failed!
${logFailedURLs}
"
)
endif()
execute_process(COMMAND ${CMAKE_COMMAND} -E tar -xf ${SQLITE_ORM_TAR_NAME}
WORKING_DIRECTORY ${SQLITE_ORM_PREFIX})
endif ()
endmacro()
if (MILVUS_WITH_SQLITE_ORM)
resolve_dependency(SQLite_ORM)
include_directories(SYSTEM "${SQLITE_ORM_INCLUDE_DIR}")
endif ()
# ----------------------------------------------------------------------
# yaml-cpp

15
core/src/db/meta/MetaSchema.h
View File

@ -29,6 +29,16 @@ class MetaField {
return name_;
}
const std::string&
type() const {
return type_;
}
const std::string&
setting() const {
return setting_;
}
std::string
ToString() const {
return name_ + " " + type_ + " " + setting_;
@ -61,6 +71,11 @@ class MetaSchema {
bool
IsEqual(const MetaFields& fields) const;
const MetaFields&
Fields() const {
return fields_;
}
private:
std::string name_;
MetaFields fields_;

62
core/src/db/meta/SqliteMetaImpl.cpp
View File

@ -195,20 +195,54 @@ SqliteMetaImpl::ValidateMetaSchema() {
throw Exception(DB_ERROR, "Connector is null pointer");
}
// // old meta could be recreated since schema changed, throw exception if meta schema is not compatible
// auto ret = ConnectorPtr->sync_schema_simulate();
// if (ret.find(META_TABLES) != ret.end() &&
// sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLES]) {
// throw Exception(DB_INCOMPATIB_META, "Meta Tables schema is created by Milvus old version");
// }
// if (ret.find(META_FIELDS) != ret.end() &&
// sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_FIELDS]) {
// throw Exception(DB_INCOMPATIB_META, "Meta Tables schema is created by Milvus old version");
// }
// if (ret.find(META_TABLEFILES) != ret.end() &&
// sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLEFILES]) {
// throw Exception(DB_INCOMPATIB_META, "Meta TableFiles schema is created by Milvus old version");
// }
auto validate_schema = [&](const MetaSchema& schema) -> bool {
auto& fields = schema.Fields();
for (auto& field : fields) {
const char *data_type = nullptr;
const char *collseq = nullptr;
int not_null = 0, primary_key = 0, autoinc = 0;
int ret = sqlite3_table_column_metadata(db_, nullptr, schema.name().c_str(), field.name().c_str(),
&data_type, &collseq, &not_null, &primary_key, &autoinc);
if (ret == SQLITE_OK) {
std::string str_type(data_type);
if (str_type != field.type()) {
return false;
}
std::string settings = field.setting();
if (primary_key) {
auto poz = settings.find("PRIMARY KEY");
if (poz == std::string::npos) {
return false;
}
}
if (not_null) {
auto poz = settings.find("NOT NULL");
if (poz == std::string::npos) {
return false;
}
}
}
}
return true;
};
// old meta could be recreated since schema changed, throw exception if meta schema is not compatible
if (!validate_schema(TABLES_SCHEMA)) {
throw Exception(DB_INCOMPATIB_META, "Meta Tables schema is created by Milvus old version");
}
if (!validate_schema(TABLEFILES_SCHEMA)) {
throw Exception(DB_INCOMPATIB_META, "Meta TableFiles schema is created by Milvus old version");
}
if (!validate_schema(FIELDS_SCHEMA)) {
throw Exception(DB_INCOMPATIB_META, "Meta Fields schema is created by Milvus old version");
}
if (!validate_schema(ENVIRONMENT_SCHEMA)) {
throw Exception(DB_INCOMPATIB_META, "Meta Environment schema is created by Milvus old version");
}
}
Status