diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index cf202ce8b5..fd2bf9d658 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -26,6 +26,9 @@ Please mark all change in change log and use the ticket from JIRA. - MS-467 - mysql db test failed - MS-470 - Drop index success, which table not created - MS-471 - code coverage run failed +- MS-492 - Drop index failed if index have been created with index_type: FLAT +- MS-493 - Knowhere unittest crash +- MS-453 - GPU search error when nprobe set more than 1024 ## Improvement - MS-327 - Clean code for milvus @@ -84,11 +87,14 @@ Please mark all change in change log and use the ticket from JIRA. - MS-455 - Distribute tasks by minimal cost in scheduler - MS-460 - Put transport speed as weight when choosing neighbour to execute task - MS-459 - Add cache for pick function in tasktable +- MS-476 - Improve search performance - MS-482 - Change search stream transport to unary in grpc - MS-487 - Define metric type in CreateTable - MS-488 - Improve code format in scheduler +- MS-495 - cmake: integrated knowhere - MS-502 - Update tasktable_test in scheduler - MS-504 - Update node_test in scheduler +- MS-505 - Install core unit test and add to coverage - MS-508 - Update normal_test in scheduler ## New Feature diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 08898aacdf..16e8041c29 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -31,6 +31,7 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release") set(BUILD_TYPE "release") else() set(BUILD_TYPE "debug") + SET(CMAKE_VERBOSE_MAKEFILE on) endif() message(STATUS "Build type = ${BUILD_TYPE}") @@ -70,21 +71,21 @@ set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR}) find_package(CUDA) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda") set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") -message("CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}") -message("CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}") +message(STATUS "CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}") +message(STATUS "CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}") set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED on) if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") - message("building milvus_engine on x86 architecture") + message(STATUS "building milvus_engine on x86 architecture") set(MILVUS_BUILD_ARCH x86_64) elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)") - message("building milvus_engine on ppc architecture") + message(STATUS "building milvus_engine on ppc architecture") set(MILVUS_BUILD_ARCH ppc64le) else() - message("unknown processor type") - message("CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}") + message(WARNING "unknown processor type") + message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}") set(MILVUS_BUILD_ARCH unknown) endif() @@ -94,6 +95,8 @@ else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp") endif() +set(ALLOW_DUPLICATE_CUSTOM_TARGETS TRUE) + set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") include(ExternalProject) @@ -104,14 +107,6 @@ include(ThirdPartyPackages) include_directories(${MILVUS_SOURCE_DIR}) link_directories(${MILVUS_BINARY_DIR}) -if (NOT DEFINED KNOWHERE_BUILD_DIR) - message(FATAL_ERROR "You must set environment variable KNOWHERE_BUILD_DIR") -endif() -message(STATUS "Build with ${KNOWHERE_BUILD_DIR}") -include_directories(${KNOWHERE_BUILD_DIR}/include) -include_directories(${KNOWHERE_BUILD_DIR}/include/SPTAG/AnnService) -link_directories(${KNOWHERE_BUILD_DIR}/lib) - ## Following should be check set(MILVUS_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include) @@ -119,7 +114,7 @@ set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) add_compile_definitions(PROFILER=${PROFILER}) -message("MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}") +message(STATUS "MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}") if (MILVUS_ENABLE_PROFILING STREQUAL "ON") ADD_DEFINITIONS(-DMILVUS_ENABLE_PROFILING) endif() diff --git a/cpp/build.sh b/cpp/build.sh index b2dba3c870..f21e2234fa 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -83,10 +83,6 @@ if [[ ! -d cmake_build ]]; then MAKE_CLEAN="ON" fi -pushd `pwd`/src/core -./build.sh ${KNOWHERE_OPTIONS} -popd - cd cmake_build CUDA_COMPILER=/usr/local/cuda/bin/nvcc diff --git a/cpp/coverage.sh b/cpp/coverage.sh index 970394116e..701416921f 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -95,6 +95,7 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \ "*/boost/*" \ "*/cmake_build/*_ep-prefix/*" \ "src/core/cmake_build*" \ + "src/core/thirdparty*" \ # gen html report ${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ \ No newline at end of file diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 80758bfaad..387693a30c 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -4,6 +4,13 @@ # Proprietary and confidential. #------------------------------------------------------------------------------- +add_subdirectory(core) + +set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) +foreach(dir ${CORE_INCLUDE_DIRS}) + include_directories(${dir}) +endforeach() + aux_source_directory(cache cache_files) aux_source_directory(config config_files) aux_source_directory(server server_files) @@ -87,12 +94,6 @@ set(client_grpc_lib set(third_party_libs knowhere - SPTAGLibStatic - arrow - jemalloc_pic - faiss - openblas - lapack easyloggingpp sqlite ${client_grpc_lib} @@ -115,15 +116,6 @@ set(third_party_libs cudart ) -if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - set(third_party_libs ${third_party_libs} - ${MKL_LIBS}) -else() - set(third_party_libs ${third_party_libs} - lapack - openblas) -endif() - if (MILVUS_ENABLE_PROFILING STREQUAL "ON") set(third_party_libs ${third_party_libs} gperftools @@ -171,13 +163,6 @@ set(server_libs set(knowhere_libs knowhere - SPTAGLibStatic - arrow - jemalloc_pic - faiss - openblas - lapack - tbb ) add_executable(milvus_server @@ -195,8 +180,8 @@ target_link_libraries(milvus_server ${server_libs} ${knowhere_libs} ${third_part install(TARGETS milvus_server DESTINATION bin) install(FILES - ${KNOWHERE_BUILD_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX} - ${KNOWHERE_BUILD_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}.2 + ${CMAKE_SOURCE_DIR}/src/core/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX} + ${CMAKE_SOURCE_DIR}/src/core/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}.2 ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3 ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4 diff --git a/cpp/src/config/ConfigMgr.h b/cpp/src/config/ConfigMgr.h index 1aa9057fb6..e14ea43b36 100644 --- a/cpp/src/config/ConfigMgr.h +++ b/cpp/src/config/ConfigMgr.h @@ -30,7 +30,7 @@ class ConfigMgr { public: static ConfigMgr* GetInstance(); - virtual ServerError LoadConfigFile(const std::string &filename) = 0; + virtual ErrorCode LoadConfigFile(const std::string &filename) = 0; virtual void Print() const = 0;//will be deleted virtual std::string DumpString() const = 0; diff --git a/cpp/src/config/YamlConfigMgr.cpp b/cpp/src/config/YamlConfigMgr.cpp index 9a34ef3e63..bb395a5003 100644 --- a/cpp/src/config/YamlConfigMgr.cpp +++ b/cpp/src/config/YamlConfigMgr.cpp @@ -12,7 +12,7 @@ namespace zilliz { namespace milvus { namespace server { -ServerError YamlConfigMgr::LoadConfigFile(const std::string &filename) { +ErrorCode YamlConfigMgr::LoadConfigFile(const std::string &filename) { struct stat directoryStat; int statOK = stat(filename.c_str(), &directoryStat); if (statOK != 0) { diff --git a/cpp/src/config/YamlConfigMgr.h b/cpp/src/config/YamlConfigMgr.h index 354fd6aa8e..412694dab1 100644 --- a/cpp/src/config/YamlConfigMgr.h +++ b/cpp/src/config/YamlConfigMgr.h @@ -17,7 +17,7 @@ namespace server { class YamlConfigMgr : public ConfigMgr { public: - virtual ServerError LoadConfigFile(const std::string &filename); + virtual ErrorCode LoadConfigFile(const std::string &filename); virtual void Print() const; virtual std::string DumpString() const; diff --git a/cpp/src/core/CMakeLists.txt b/cpp/src/core/CMakeLists.txt index 1024a71e71..d406f41ce3 100644 --- a/cpp/src/core/CMakeLists.txt +++ b/cpp/src/core/CMakeLists.txt @@ -5,6 +5,7 @@ #------------------------------------------------------------------------------- cmake_minimum_required(VERSION 3.14) +message(STATUS "---------------core--------------") message(STATUS "Building using CMake version: ${CMAKE_VERSION}") set(KNOWHERE_VERSION "0.1.0") @@ -38,18 +39,16 @@ endif() MESSAGE(STATUS "CMAKE_CXX_FLAGS" ${CMAKE_CXX_FLAGS}) find_package(CUDA) -#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda") -#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") - message("building milvus_engine on x86 architecture") + message(STATUS "building milvus_engine on x86 architecture") set(KNOWHERE_BUILD_ARCH x86_64) elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)") - message("building milvus_engine on ppc architecture") + message(STATUS "building milvus_engine on ppc architecture") set(KNOWHERE_BUILD_ARCH ppc64le) else() - message("unknown processor type") - message("CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}") + message(WARNING "unknown processor type") + message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}") set(KNOWHERE_BUILD_ARCH unknown) endif() @@ -60,14 +59,25 @@ else() endif() message(STATUS "Build type = ${BUILD_TYPE}") -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") +set(CORE_SOURCE_DIR ${PROJECT_SOURCE_DIR}) +set(CORE_BINARY_DIR ${PROJECT_BINARY_DIR}) +message(STATUS "Core source dir: ${PROJECT_SOURCE_DIR}") +message(STATUS "Core binary dir: ${PROJECT_BINARY_DIR}") +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CORE_SOURCE_DIR}/cmake") + include(ExternalProject) -include(DefineOptions) -include(BuildUtils) -include(ThirdPartyPackages) +include(DefineOptionsCore) +include(BuildUtilsCore) +include(ThirdPartyPackagesCore) add_subdirectory(src) +if (BUILD_COVERAGE STREQUAL "ON") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") +endif() + +set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) + if(BUILD_UNIT_TEST STREQUAL "ON") add_subdirectory(test) endif() diff --git a/cpp/src/core/cmake/BuildUtils.cmake b/cpp/src/core/cmake/BuildUtilsCore.cmake similarity index 99% rename from cpp/src/core/cmake/BuildUtils.cmake rename to cpp/src/core/cmake/BuildUtilsCore.cmake index 2d778bf703..ce798c4bb0 100644 --- a/cpp/src/core/cmake/BuildUtils.cmake +++ b/cpp/src/core/cmake/BuildUtilsCore.cmake @@ -75,7 +75,7 @@ function(ExternalProject_Use_Cache project_name package_file install_path) "Extracting ${package_file} to ${install_path}" COMMAND ${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + WORKING_DIRECTORY ${CORE_BINARY_DIR} ) ExternalProject_Add_StepTargets(${project_name} extract) diff --git a/cpp/src/core/cmake/DefineOptions.cmake b/cpp/src/core/cmake/DefineOptionsCore.cmake similarity index 98% rename from cpp/src/core/cmake/DefineOptions.cmake rename to cpp/src/core/cmake/DefineOptionsCore.cmake index 589e2d9ae8..e20d3d77ab 100644 --- a/cpp/src/core/cmake/DefineOptions.cmake +++ b/cpp/src/core/cmake/DefineOptionsCore.cmake @@ -104,7 +104,7 @@ macro(config_summary) message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}") if(${CMAKE_EXPORT_COMPILE_COMMANDS}) message( - STATUS " Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json") + STATUS " Compile commands: ${CORE_BINARY_DIR}/compile_commands.json") endif() foreach(category ${KNOWHERE_OPTION_CATEGORIES}) diff --git a/cpp/src/core/cmake/ThirdPartyPackages.cmake b/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake similarity index 70% rename from cpp/src/core/cmake/ThirdPartyPackages.cmake rename to cpp/src/core/cmake/ThirdPartyPackagesCore.cmake index 4a5e47f953..fad0869021 100644 --- a/cpp/src/core/cmake/ThirdPartyPackages.cmake +++ b/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake @@ -6,7 +6,6 @@ # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an @@ -18,7 +17,6 @@ set(KNOWHERE_THIRDPARTY_DEPENDENCIES ARROW - BOOST FAISS GTest LAPACK @@ -53,11 +51,8 @@ endmacro() macro(resolve_dependency DEPENDENCY_NAME) if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") #message(STATUS "Finding ${DEPENDENCY_NAME} package") -# find_package(${DEPENDENCY_NAME} QUIET) -# if (NOT ${DEPENDENCY_NAME}_FOUND) #message(STATUS "${DEPENDENCY_NAME} package not found") build_dependency(${DEPENDENCY_NAME}) -# endif () elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") build_dependency(${DEPENDENCY_NAME}) elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") @@ -120,7 +115,7 @@ endif (UNIX) # ---------------------------------------------------------------------- # thirdparty directory -set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty") +set(THIRDPARTY_DIR "${CORE_SOURCE_DIR}/thirdparty") # ---------------------------------------------------------------------- # JFrog @@ -202,14 +197,6 @@ endif() set(MAKE_BUILD_ARGS "-j8") -## Using make -j in sub-make is fragile -## see discussion https://github.com/apache/KNOWHERE/pull/2779 -#if(${CMAKE_GENERATOR} MATCHES "Makefiles") -# set(MAKE_BUILD_ARGS "") -#else() -# # limit the maximum number of jobs for ninja -# set(MAKE_BUILD_ARGS "-j4") -#endif() # ---------------------------------------------------------------------- # Find pthreads @@ -243,22 +230,11 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) set(${_LIB_NAME} "${_LIB_VERSION}") endforeach() -if(DEFINED ENV{KNOWHERE_BOOST_URL}) - set(BOOST_SOURCE_URL "$ENV{KNOWHERE_BOOST_URL}") -else() - string(REPLACE "." "_" BOOST_VERSION_UNDERSCORES ${BOOST_VERSION}) - set(BOOST_SOURCE_URL - "https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz" - ) -endif() - if(DEFINED ENV{KNOWHERE_FAISS_URL}) set(FAISS_SOURCE_URL "$ENV{KNOWHERE_FAISS_URL}") else() set(FAISS_SOURCE_URL "http://192.168.1.105:6060/jinhai/faiss/-/archive/${FAISS_VERSION}/faiss-${FAISS_VERSION}.tar.gz") -# set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz") -# set(FAISS_SOURCE_URL "${CMAKE_SOURCE_DIR}/thirdparty/faiss-1.5.3") - message(STATUS ${FAISS_SOURCE_URL}) + message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}") endif() # set(FAISS_MD5 "a589663865a8558205533c8ac414278c") # set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c @@ -297,28 +273,18 @@ set(OPENBLAS_MD5 "8a110a25b819a4b94e8a9580702b6495") # ---------------------------------------------------------------------- # ARROW +set(ARROW_PREFIX "${CORE_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp") macro(build_arrow) message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source") - set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp") set(ARROW_STATIC_LIB_NAME arrow) -# set(PARQUET_STATIC_LIB_NAME parquet) - # set(ARROW_CUDA_STATIC_LIB_NAME arrow_cuda) - set(ARROW_STATIC_LIB + set(ARROW_STATIC_LIB "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) -# set(PARQUET_STATIC_LIB -# "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${PARQUET_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" -# ) - # set(ARROW_CUDA_STATIC_LIB - # "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_CUDA_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" - # ) - set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include") + set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include") set(ARROW_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - # "-DARROW_THRIFT_URL=${THRIFT_SOURCE_URL}" - #"env ARROW_THRIFT_URL=${THRIFT_SOURCE_URL}" -DARROW_BUILD_STATIC=ON -DARROW_BUILD_SHARED=OFF -DARROW_PARQUET=OFF @@ -328,8 +294,7 @@ macro(build_arrow) -DCMAKE_BUILD_TYPE=Release -DARROW_DEPENDENCY_SOURCE=BUNDLED) #Build all arrow dependencies from source instead of calling find_package first - # set($ENV{ARROW_THRIFT_URL} ${THRIFT_SOURCE_URL}) - + if(USE_JFROG_CACHE STREQUAL "ON") execute_process(COMMAND sh -c "git ls-remote --heads --tags ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID) if(${ARROW_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+") @@ -362,13 +327,13 @@ macro(build_arrow) "${ARROW_STATIC_LIB}" ) - ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL}) + ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL}) else() file(DOWNLOAD ${ARROW_CACHE_URL} ${ARROW_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${ARROW_CACHE_URL} TO ${ARROW_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) - ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR}) endif() endif() else() @@ -402,10 +367,9 @@ macro(build_arrow) set_target_properties(arrow PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") - # INTERFACE_LINK_LIBRARIES thrift) - add_dependencies(arrow arrow_ep) + add_dependencies(arrow arrow_ep) - set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep") + set(JEMALLOC_PREFIX "${CORE_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep") add_custom_command(TARGET arrow_ep POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_PREFIX}/lib/ @@ -414,7 +378,7 @@ macro(build_arrow) endmacro() -if(KNOWHERE_WITH_ARROW) +if(KNOWHERE_WITH_ARROW AND NOT TARGET arrow_ep) resolve_dependency(ARROW) @@ -422,191 +386,12 @@ if(KNOWHERE_WITH_ARROW) include_directories(SYSTEM ${ARROW_INCLUDE_DIR}) endif() -# ---------------------------------------------------------------------- -# Add Boost dependencies (code adapted from Apache Kudu (incubating)) - -set(Boost_USE_MULTITHREADED ON) -if(MSVC AND KNOWHERE_USE_STATIC_CRT) - set(Boost_USE_STATIC_RUNTIME ON) -endif() -set(Boost_ADDITIONAL_VERSIONS - "1.70.0" - "1.70" - "1.69.0" - "1.69" - "1.68.0" - "1.68" - "1.67.0" - "1.67" - "1.66.0" - "1.66" - "1.65.0" - "1.65" - "1.64.0" - "1.64" - "1.63.0" - "1.63" - "1.62.0" - "1.61" - "1.61.0" - "1.62" - "1.60.0" - "1.60") - -# TODO -if(KNOWHERE_BOOST_VENDORED) -# system thread serialization wserialization regex - set(BOOST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/boost_ep-prefix/src/boost_ep") - set(BOOST_LIB_DIR "${BOOST_PREFIX}/stage/lib") - set(BOOST_BUILD_LINK "static") - set(BOOST_STATIC_SYSTEM_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(BOOST_STATIC_FILESYSTEM_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(BOOST_STATIC_SERIALIZATION_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_serialization${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(BOOST_STATIC_WSERIALIZATION_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_wserialization${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(BOOST_STATIC_REGEX_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(BOOST_STATIC_THREAD_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_thread${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(BOOST_SYSTEM_LIBRARY boost_system_static) - set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static) - set(BOOST_SERIALIZATION_LIBRARY boost_serialization_static) - set(BOOST_WSERIALIZATION_LIBRARY boost_wserialization_static) - set(BOOST_REGEX_LIBRARY boost_regex_static) - set(BOOST_THREAD_LIBRARY boost_thread_static) - - if(KNOWHERE_BOOST_HEADER_ONLY) - set(BOOST_BUILD_PRODUCTS) - set(BOOST_CONFIGURE_COMMAND "") - set(BOOST_BUILD_COMMAND "") - else() - set(BOOST_BUILD_PRODUCTS ${BOOST_STATIC_SYSTEM_LIBRARY} - ${BOOST_STATIC_FILESYSTEM_LIBRARY} ${BOOST_STATIC_SERIALIZATION_LIBRARY} - ${BOOST_STATIC_WSERIALIZATION_LIBRARY} ${BOOST_STATIC_REGEX_LIBRARY} - ${BOOST_STATIC_THREAD_LIBRARY}) - set(BOOST_CONFIGURE_COMMAND "./bootstrap.sh" "--prefix=${BOOST_PREFIX}" - "--with-libraries=filesystem,serialization,wserialization,system,thread,regex") - if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") - set(BOOST_BUILD_VARIANT "debug") - else() - set(BOOST_BUILD_VARIANT "release") - endif() - set(BOOST_BUILD_COMMAND - "./b2" - "link=${BOOST_BUILD_LINK}" - "variant=${BOOST_BUILD_VARIANT}" - "cxxflags=-fPIC") - - add_thirdparty_lib(boost_system STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}") - - add_thirdparty_lib(boost_filesystem STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}") - - add_thirdparty_lib(boost_serialization STATIC_LIB "${BOOST_STATIC_SERIALIZATION_LIBRARY}") - - add_thirdparty_lib(boost_wserialization STATIC_LIB "${BOOST_STATIC_WSERIALIZATION_LIBRARY}") - - add_thirdparty_lib(boost_regex STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}") - - add_thirdparty_lib(boost_thread STATIC_LIB "${BOOST_STATIC_THREAD_LIBRARY}") - - set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY} ${BOOST_SERIALIZATION_LIBRARY} - ${BOOST_WSERIALIZATION_LIBRARY} ${BOOST_REGEX_LIBRARY} ${BOOST_THREAD_LIBRARY}) - endif() - externalproject_add(boost_ep - URL - ${BOOST_SOURCE_URL} - BUILD_BYPRODUCTS - ${BOOST_BUILD_PRODUCTS} - BUILD_IN_SOURCE - 1 - CONFIGURE_COMMAND - ${BOOST_CONFIGURE_COMMAND} - BUILD_COMMAND - ${BOOST_BUILD_COMMAND} - INSTALL_COMMAND - "" - ${EP_LOG_OPTIONS}) - set(Boost_INCLUDE_DIR "${BOOST_PREFIX}") - set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}") - add_dependencies(boost_system_static boost_ep) - add_dependencies(boost_filesystem_static boost_ep) - add_dependencies(boost_serialization_static boost_ep) - add_dependencies(boost_wserialization_static boost_ep) - add_dependencies(boost_regex_static boost_ep) - add_dependencies(boost_thread_static boost_ep) - -#else() -# if(MSVC) -# # disable autolinking in boost -# add_definitions(-DBOOST_ALL_NO_LIB) -# endif() - -# if(DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT) -# # In older versions of CMake (such as 3.2), the system paths for Boost will -# # In older versions of CMake (such as 3.2), the system paths for Boost will -# # be looked in first even if we set $BOOST_ROOT or pass -DBOOST_ROOT -# set(Boost_NO_SYSTEM_PATHS ON) -# endif() - -# if(KNOWHERE_BOOST_USE_SHARED) -# # Find shared Boost libraries. -# set(Boost_USE_STATIC_LIBS OFF) -# set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS}) -# set(BUILD_SHARED_LIBS ON) -# -# if(MSVC) -# # force all boost libraries to dynamic link -# add_definitions(-DBOOST_ALL_DYN_LINK) -# endif() -# -# if(KNOWHERE_BOOST_HEADER_ONLY) -# find_package(Boost REQUIRED) -# else() -# find_package(Boost COMPONENTS serialization system filesystem REQUIRED) -# set(BOOST_SYSTEM_LIBRARY Boost::system) -# set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) -# set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) -# set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) -# endif() -# set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP}) -# unset(BUILD_SHARED_LIBS_KEEP) -# else() -# # Find static boost headers and libs -# # TODO Differentiate here between release and debug builds -# set(Boost_USE_STATIC_LIBS ON) -# if(KNOWHERE_BOOST_HEADER_ONLY) -# find_package(Boost REQUIRED) -# else() -# find_package(Boost COMPONENTS serialization system filesystem REQUIRED) -# set(BOOST_SYSTEM_LIBRARY Boost::system) -# set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) -# set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) -# set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) -# endif() -# endif() -endif() - -#message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR}) -#message(STATUS "Boost libraries: " ${Boost_LIBRARIES}) - -include_directories(SYSTEM ${Boost_INCLUDE_DIR}) -link_directories(SYSTEM ${BOOST_LIB_DIR}) - # ---------------------------------------------------------------------- # OpenBLAS macro(build_openblas) message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source") - set(OPENBLAS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep") + set(OPENBLAS_PREFIX "${CORE_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep") set(OPENBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include") set(OPENBLAS_STATIC_LIB "${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}") @@ -639,13 +424,13 @@ macro(build_openblas) BUILD_BYPRODUCTS ${OPENBLAS_STATIC_LIB}) - ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL}) + ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL}) else() file(DOWNLOAD ${OPENBLAS_CACHE_URL} ${OPENBLAS_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${OPENBLAS_CACHE_URL} TO ${OPENBLAS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) - ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR}) endif() endif() else() @@ -678,20 +463,12 @@ macro(build_openblas) add_dependencies(openblas openblas_ep) endmacro() -#if(KNOWHERE_WITH_OPENBLAS) -# resolve_dependency(OpenBLAS) -# -# get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) -# include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") -# link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib) -#endif() - # ---------------------------------------------------------------------- # LAPACK macro(build_lapack) message(STATUS "Building LAPACK-${LAPACK_VERSION} from source") - set(LAPACK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep") + set(LAPACK_PREFIX "${CORE_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep") set(LAPACK_INCLUDE_DIR "${LAPACK_PREFIX}/include") set(LAPACK_STATIC_LIB "${LAPACK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lapack${CMAKE_STATIC_LIBRARY_SUFFIX}") @@ -723,13 +500,13 @@ macro(build_lapack) BUILD_BYPRODUCTS ${LAPACK_STATIC_LIB}) - ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL}) + ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL}) else() file(DOWNLOAD ${LAPACK_CACHE_URL} ${LAPACK_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${LAPACK_CACHE_URL} TO ${LAPACK_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) - ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR}) endif() endif() else() @@ -756,14 +533,6 @@ macro(build_lapack) add_dependencies(lapack lapack_ep) endmacro() -#if(KNOWHERE_WITH_LAPACK) -# resolve_dependency(LAPACK) -# -# get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) -# include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") -# link_directories(SYSTEM ${LAPACK_PREFIX}/lib) -#endif() - # ---------------------------------------------------------------------- # Google gtest @@ -780,7 +549,7 @@ macro(build_gtest) -Wno-ignored-attributes) endif() - set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") + set(GTEST_PREFIX "${CORE_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") set(GTEST_STATIC_LIB "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}") @@ -822,13 +591,13 @@ macro(build_gtest) ${GTEST_CMAKE_ARGS} ${EP_LOG_OPTIONS}) - ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL}) + ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL}) else() file(DOWNLOAD ${GTEST_CACHE_URL} ${GTEST_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${GTEST_CACHE_URL} TO ${GTEST_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) - ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR}) endif() endif() else() @@ -871,8 +640,7 @@ macro(build_gtest) endmacro() -if (KNOWHERE_BUILD_TESTS) - #message(STATUS "Resolving gtest dependency") +if (KNOWHERE_BUILD_TESTS AND NOT TARGET googletest_ep) resolve_dependency(GTest) if(NOT GTEST_VENDORED) @@ -889,17 +657,12 @@ endif() macro(build_faiss) message(STATUS "Building FAISS-${FAISS_VERSION} from source") - set(FAISS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep") + set(FAISS_PREFIX "${CORE_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep") set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include") set(FAISS_STATIC_LIB "${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}") - # add_custom_target(faiss_dependencies) - # add_dependencies(faiss_dependencies openblas_ep) - # add_dependencies(faiss_dependencies openblas) - # get_target_property(FAISS_OPENBLAS_LIB_DIR openblas IMPORTED_LOCATION) - # get_filename_component(FAISS_OPENBLAS_LIB "${FAISS_OPENBLAS_LIB_DIR}" DIRECTORY) - + set(FAISS_CONFIGURE_ARGS "--prefix=${FAISS_PREFIX}" "CFLAGS=${EP_C_FLAGS}" @@ -907,13 +670,7 @@ macro(build_faiss) "LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib -lopenblas -llapack" --without-python) - # if(OPENBLAS_STATIC_LIB) - # set(OPENBLAS_LIBRARY ${OPENBLAS_STATIC_LIB}) - # else() - # set(OPENBLAS_LIBRARY ${OPENBLAS_SHARED_LIB}) - # endif() - # set(FAISS_DEPENDENCIES ${FAISS_DEPENDENCIES} ${OPENBLAS_LIBRARY}) - + if(${KNOWHERE_WITH_FAISS_GPU_VERSION} STREQUAL "ON") set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}" @@ -924,9 +681,7 @@ macro(build_faiss) endif() if(USE_JFROG_CACHE STREQUAL "ON") -# Check_Last_Modify("${CMAKE_SOURCE_DIR}/thirdparty/faiss_cache_check_lists.txt" "${CMAKE_SOURCE_DIR}" FAISS_LAST_MODIFIED_COMMIT_ID) string(MD5 FAISS_COMBINE_MD5 "${FAISS_MD5}${LAPACK_MD5}${OPENBLAS_MD5}") - # string(MD5 FAISS_COMBINE_MD5 "${FAISS_LAST_MODIFIED_COMMIT_ID}${LAPACK_MD5}${OPENBLAS_MD5}") set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMBINE_MD5}.tar.gz") set(FAISS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${FAISS_CACHE_PACKAGE_NAME}") set(FAISS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${FAISS_CACHE_PACKAGE_NAME}") @@ -952,13 +707,13 @@ macro(build_faiss) ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) - ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL}) + ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL}) else() file(DOWNLOAD ${FAISS_CACHE_URL} ${FAISS_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${FAISS_CACHE_URL} TO ${FAISS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) - ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR}) endif() endif() else() @@ -990,14 +745,10 @@ macro(build_faiss) INTERFACE_LINK_LIBRARIES "openblas;lapack" ) add_dependencies(faiss faiss_ep) - #add_dependencies(faiss openblas_ep) - #add_dependencies(faiss lapack_ep) - #target_link_libraries(faiss ${OPENBLAS_PREFIX}/lib) - #target_link_libraries(faiss ${LAPACK_PREFIX}/lib) endmacro() -if(KNOWHERE_WITH_FAISS) +if(KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep) resolve_dependency(OpenBLAS) get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) @@ -1012,8 +763,5 @@ if(KNOWHERE_WITH_FAISS) resolve_dependency(FAISS) get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM "${FAISS_INCLUDE_DIR}") -# include_directories(SYSTEM "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/") -# link_directories(SYSTEM ${FAISS_PREFIX}/) link_directories(SYSTEM ${FAISS_PREFIX}/lib/) -# link_directories(SYSTEM ${FAISS_PREFIX}/gpu/) endif() diff --git a/cpp/src/core/include/knowhere/index/vector_index/gpu_ivf.h b/cpp/src/core/include/knowhere/index/vector_index/gpu_ivf.h index 22c43292da..3a62addf94 100644 --- a/cpp/src/core/include/knowhere/index/vector_index/gpu_ivf.h +++ b/cpp/src/core/include/knowhere/index/vector_index/gpu_ivf.h @@ -9,7 +9,7 @@ namespace zilliz { namespace knowhere { struct Resource { - Resource(std::shared_ptr &r): faiss_res(r) { + explicit Resource(std::shared_ptr &r): faiss_res(r) { static int64_t global_id = 0; id = global_id++; } @@ -32,6 +32,11 @@ class FaissGpuResourceMgr { static FaissGpuResourceMgr & GetInstance(); + // Free gpu resource, avoid cudaGetDevice error when deallocate. + // this func should be invoke before main return + void + Free(); + void AllocateTempMem(ResPtr &resource, const int64_t& device_id, const int64_t& size); diff --git a/cpp/src/core/src/CMakeLists.txt b/cpp/src/core/src/CMakeLists.txt index 5b119f8b2b..a51a62b44a 100644 --- a/cpp/src/core/src/CMakeLists.txt +++ b/cpp/src/core/src/CMakeLists.txt @@ -1,16 +1,16 @@ -set(TBB_DIR ${CMAKE_SOURCE_DIR}/thirdparty/tbb) +set(TBB_DIR ${CORE_SOURCE_DIR}/thirdparty/tbb) set(TBB_LIBRARIES ${TBB_DIR}/libtbb.so) include_directories(${TBB_DIR}/include) include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64) -include_directories(${CMAKE_SOURCE_DIR}/include) -include_directories(${CMAKE_SOURCE_DIR}/thirdparty) -include_directories(${CMAKE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) -include_directories(${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) +include_directories(${CORE_SOURCE_DIR}/include) +include_directories(${CORE_SOURCE_DIR}/thirdparty) +include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) +include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) -set(SPTAG_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/SPTAG) +set(SPTAG_SOURCE_DIR ${CORE_SOURCE_DIR}/thirdparty/SPTAG) file(GLOB HDR_FILES ${SPTAG_SOURCE_DIR}/AnnService/inc/Core/*.h ${SPTAG_SOURCE_DIR}/AnnService/inc/Core/Common/*.h @@ -24,9 +24,9 @@ file(GLOB SRC_FILES ${SPTAG_SOURCE_DIR}/AnnService/src/Core/KDT/*.cpp ${SPTAG_SOURCE_DIR}/AnnService/src/Helper/*.cpp) -#add_library(SPTAGLib SHARED ${SRC_FILES} ${HDR_FILES}) -#target_link_libraries(SPTAGLib ${TBB_LIBRARIES}) -add_library(SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES}) +if(NOT TARGET SPTAGLibStatic) + add_library(SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES}) +endif() set(external_srcs knowhere/adapter/sptag.cpp @@ -51,14 +51,13 @@ set(index_srcs ) set(depend_libs -# libtcmalloc.a SPTAGLibStatic ${TBB_LIBRARIES} faiss openblas lapack arrow - jemalloc_pic + ${ARROW_PREFIX}/lib/libjemalloc_pic.a cudart cublas gomp @@ -66,30 +65,19 @@ set(depend_libs pthread ) -add_library( - knowhere STATIC - ${external_srcs} - ${index_srcs} -) -#target_compile_options(knowhere PUBLIC "-fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free") +if(NOT TARGET knowhere) + add_library( + knowhere STATIC + ${external_srcs} + ${index_srcs} + ) +endif() target_link_libraries( knowhere ${depend_libs} ) -#add_library( -# knowhereS SHARED -# ${external_srcs} -# ${index_srcs} -#) -# -#target_link_libraries( -# knowhereS -## ${TBB_LIBRARIES} -# ${depend_libs} -#) - INSTALL(TARGETS knowhere SPTAGLibStatic @@ -98,7 +86,6 @@ INSTALL(TARGETS INSTALL(FILES ${ARROW_STATIC_LIB} -# ${PARQUET_STATIC_LIB} ${ARROW_PREFIX}/lib/libjemalloc_pic.a ${FAISS_STATIC_LIB} ${LAPACK_STATIC_LIB} @@ -112,24 +99,35 @@ INSTALL(FILES ${OPENBLAS_REAL_STATIC_LIB} DESTINATION lib ) -INSTALL(FILES ${CMAKE_SOURCE_DIR}/thirdparty/tbb/libtbb.so.2 -# RENAME "libtbb.so.2" +INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so.2 DESTINATION lib ) -INSTALL(FILES ${CMAKE_SOURCE_DIR}/thirdparty/tbb/libtbb.so -# RENAME "libtbb.so" +INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so DESTINATION lib ) +set(CORE_INCLUDE_DIRS + ${CORE_SOURCE_DIR}/include + ${CORE_SOURCE_DIR}/thirdparty + ${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService + ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include + ${ARROW_INCLUDE_DIR} + ${FAISS_INCLUDE_DIR} + ${OPENBLAS_INCLUDE_DIR} + ${LAPACK_INCLUDE_DIR} + ${CORE_SOURCE_DIR}/thirdparty/tbb/include + ) + +set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) + INSTALL(DIRECTORY - ${CMAKE_SOURCE_DIR}/include/knowhere - ${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons - ${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext + ${CORE_SOURCE_DIR}/include/knowhere + ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons + ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext ${ARROW_INCLUDE_DIR}/arrow -# ${ARROW_INCLUDE_DIR}/parquet ${FAISS_PREFIX}/include/faiss ${OPENBLAS_INCLUDE_DIR}/ - ${CMAKE_SOURCE_DIR}/thirdparty/tbb/include/tbb + ${CORE_SOURCE_DIR}/thirdparty/tbb/include/tbb DESTINATION include) diff --git a/cpp/src/core/src/knowhere/index/vector_index/gpu_ivf.cpp b/cpp/src/core/src/knowhere/index/vector_index/gpu_ivf.cpp index 26fcd6a749..0546674e78 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/gpu_ivf.cpp +++ b/cpp/src/core/src/knowhere/index/vector_index/gpu_ivf.cpp @@ -282,7 +282,7 @@ void FaissGpuResourceMgr::InitResource() { for(auto& device : devices_params_) { auto& resource_vec = idle_[device.first]; - for (int i = 0; i < device.second.resource_num; ++i) { + for (int64_t i = 0; i < device.second.resource_num; ++i) { auto res = std::make_shared(); // TODO(linxj): enable set pinned memory @@ -351,6 +351,18 @@ void FaissGpuResourceMgr::MoveToIdle(const int64_t &device_id, const ResPtr &res idle_[device_id].insert(it, res); } +void FaissGpuResourceMgr::Free() { + for (auto &item : in_use_) { + auto& res_vec = item.second; + res_vec.clear(); + } + for (auto &item : idle_) { + auto& res_vec = item.second; + res_vec.clear(); + } + is_init = false; +} + void GPUIndex::SetGpuDevice(const int &gpu_id) { gpu_id_ = gpu_id; } diff --git a/cpp/src/core/test/CMakeLists.txt b/cpp/src/core/test/CMakeLists.txt index fbbf151264..ba1501084c 100644 --- a/cpp/src/core/test/CMakeLists.txt +++ b/cpp/src/core/test/CMakeLists.txt @@ -1,17 +1,19 @@ -include_directories(${CMAKE_SOURCE_DIR}/thirdparty) -include_directories(${CMAKE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) -include_directories(${CMAKE_SOURCE_DIR}/include) -include_directories(${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) +include_directories(${CORE_SOURCE_DIR}/thirdparty) +include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) +include_directories(${CORE_SOURCE_DIR}/include) +include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) include_directories(/usr/local/cuda/include) link_directories(/usr/local/cuda/lib64) -link_directories(${CMAKE_SOURCE_DIR}/thirdparty/tbb) +link_directories(${CORE_SOURCE_DIR}/thirdparty/tbb) set(unittest_libs gtest gmock gtest_main gmock_main) +message(STATUS "arrow prefix: ${ARROW_PREFIX}") +message(STATUS "libjemalloc_pic path: ${ARROW_PREFIX}/lib/libjemalloc_pic.a") set(depend_libs faiss openblas lapack - arrow jemalloc_pic + arrow "${ARROW_PREFIX}/lib/libjemalloc_pic.a" tbb ) @@ -22,57 +24,59 @@ set(basic_libs # -#<<<<<<<<<< set(ivf_srcs - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp + ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp utils.cpp ) -add_executable(test_ivf test_ivf.cpp ${ivf_srcs}) +if(NOT TARGET test_ivf) + add_executable(test_ivf test_ivf.cpp ${ivf_srcs}) +endif() target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs}) -#<<<<<<<<<< # -#<<<<<<<<<< set(idmap_srcs - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp + ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp utils.cpp ) -add_executable(test_idmap test_idmap.cpp ${idmap_srcs}) +if(NOT TARGET test_idmap) + add_executable(test_idmap test_idmap.cpp ${idmap_srcs}) +endif() target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) -#<<<<<<<<<< - # -#<<<<<<<<<< set(kdt_srcs - ${CMAKE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp + ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp + ${CORE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp + ${CORE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp utils.cpp ) -add_executable(test_kdt test_kdt.cpp ${kdt_srcs}) +if(NOT TARGET test_kdt) + add_executable(test_kdt test_kdt.cpp ${kdt_srcs}) +endif() target_link_libraries(test_kdt SPTAGLibStatic ${depend_libs} ${unittest_libs} ${basic_libs}) -#<<<<<<<<<< +install(TARGETS test_ivf DESTINATION unittest) +install(TARGETS test_idmap DESTINATION unittest) +install(TARGETS test_kdt DESTINATION unittest) add_subdirectory(faiss_ori) add_subdirectory(test_nsg) diff --git a/cpp/src/core/test/faiss_ori/CMakeLists.txt b/cpp/src/core/test/faiss_ori/CMakeLists.txt index a6b20a2373..6cd490aff9 100644 --- a/cpp/src/core/test/faiss_ori/CMakeLists.txt +++ b/cpp/src/core/test/faiss_ori/CMakeLists.txt @@ -1,15 +1,15 @@ -include_directories(${CMAKE_SOURCE_DIR}/thirdparty) -include_directories(${CMAKE_SOURCE_DIR}/include) +include_directories(${CORE_SOURCE_DIR}/thirdparty) +include_directories(${CORE_SOURCE_DIR}/include) include_directories(/usr/local/cuda/include) link_directories(/usr/local/cuda/lib64) -link_directories(${CMAKE_SOURCE_DIR}/thirdparty/tbb) +link_directories(${CORE_SOURCE_DIR}/thirdparty/tbb) set(unittest_libs gtest gmock gtest_main gmock_main) set(depend_libs faiss openblas lapack - arrow jemalloc_pic + arrow ${ARROW_PREFIX}/lib/libjemalloc_pic.a tbb ) @@ -20,7 +20,9 @@ set(basic_libs # -#<<<<<<<<<< -add_executable(test_gpu gpuresource_test.cpp) +if(NOT TARGET test_gpu) + add_executable(test_gpu gpuresource_test.cpp) +endif() target_link_libraries(test_gpu ${depend_libs} ${unittest_libs} ${basic_libs}) -#<<<<<<<<<< + +install(TARGETS test_gpu DESTINATION unittest) \ No newline at end of file diff --git a/cpp/src/core/test/test_idmap.cpp b/cpp/src/core/test/test_idmap.cpp index 25f8f762f3..90546429b6 100644 --- a/cpp/src/core/test/test_idmap.cpp +++ b/cpp/src/core/test/test_idmap.cpp @@ -26,6 +26,11 @@ class IDMAPTest : public DataGen, public ::testing::Test { Init_with_default(); index_ = std::make_shared(); } + + void TearDown() override { + FaissGpuResourceMgr::GetInstance().Free(); + } + protected: IDMAPPtr index_ = nullptr; }; diff --git a/cpp/src/core/test/test_ivf.cpp b/cpp/src/core/test/test_ivf.cpp index 23847f7a16..9ebcf767d9 100644 --- a/cpp/src/core/test/test_ivf.cpp +++ b/cpp/src/core/test/test_ivf.cpp @@ -7,13 +7,11 @@ #include #include -#include #include #include #include #include -#include #include "knowhere/index/vector_index/gpu_ivf.h" #include "knowhere/index/vector_index/ivf.h" @@ -58,6 +56,9 @@ class IVFTest index_ = IndexFactory(index_type); FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024*1024*200, 1024*1024*300, 2); } + void TearDown() override { + FaissGpuResourceMgr::GetInstance().Free(); + } protected: std::string index_type; @@ -369,6 +370,7 @@ class GPURESTEST void TearDown() override { delete ids; delete dis; + FaissGpuResourceMgr::GetInstance().Free(); } protected: diff --git a/cpp/src/core/test/test_nsg/CMakeLists.txt b/cpp/src/core/test/test_nsg/CMakeLists.txt index 0ec275d9ee..9ee8e76525 100644 --- a/cpp/src/core/test/test_nsg/CMakeLists.txt +++ b/cpp/src/core/test/test_nsg/CMakeLists.txt @@ -2,7 +2,6 @@ include_directories(/usr/local/include/gperftools) link_directories(/usr/local/lib) -#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free") add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO) find_package(OpenMP) @@ -14,37 +13,31 @@ else () endif () message(${OpenMP_CXX_FLAGS}) -include_directories(${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg) -#include_directories(/home/zilliz/opt/app/pyenv/versions/3.6.8/include/python3.6m) -#include_directories(/home/zilliz/opt/app/pyenv/versions/3.6.8/envs/megasearch_testframework_dev/lib/python3.6/site-packages/numpy/core/include) +include_directories(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg) -aux_source_directory(${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src) - -#add_library(nsg_raw SHARED ${nsg_src}) -#target_link_libraries(nsg_raw -# gomp) +aux_source_directory(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src) set(interface_src - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp + ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp ../utils.cpp ) -add_executable(test_nsg - test_nsg.cpp - ${interface_src} - ${nsg_src} - ) -#target_link_libraries(test_nsg -# # libprofiler.so -## -ltcmalloc -## gomp -# ) +if(NOT TARGET test_nsg) + add_executable(test_nsg + test_nsg.cpp + ${interface_src} + ${nsg_src} + ) +endif() + target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs}) -############################## \ No newline at end of file +############################## + +install(TARGETS test_nsg DESTINATION unittest) \ No newline at end of file diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 17272c588c..a443e81fdc 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -100,17 +100,17 @@ Status DBImpl::DropAll() { Status DBImpl::CreateTable(meta::TableSchema& table_schema) { if (shutting_down_.load(std::memory_order_acquire)){ - return Status::Error("Milsvus server is shutdown!"); + return Status(DB_ERROR, "Milsvus server is shutdown!"); } meta::TableSchema temp_schema = table_schema; - temp_schema.index_file_size_ *= ONE_MB; + temp_schema.index_file_size_ *= ONE_MB; //store as MB return meta_ptr_->CreateTable(temp_schema); } Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) { if (shutting_down_.load(std::memory_order_acquire)){ - return Status::Error("Milsvus server is shutdown!"); + return Status(DB_ERROR, "Milsvus server is shutdown!"); } //dates partly delete files of the table but currently we don't support @@ -136,15 +136,17 @@ Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& date Status DBImpl::DescribeTable(meta::TableSchema& table_schema) { if (shutting_down_.load(std::memory_order_acquire)){ - return Status::Error("Milsvus server is shutdown!"); + return Status(DB_ERROR, "Milsvus server is shutdown!"); } - return meta_ptr_->DescribeTable(table_schema); + auto stat = meta_ptr_->DescribeTable(table_schema); + table_schema.index_file_size_ /= ONE_MB; //return as MB + return stat; } Status DBImpl::HasTable(const std::string& table_id, bool& has_or_not) { if (shutting_down_.load(std::memory_order_acquire)){ - return Status::Error("Milsvus server is shutdown!"); + return Status(DB_ERROR, "Milsvus server is shutdown!"); } return meta_ptr_->HasTable(table_id, has_or_not); @@ -152,7 +154,7 @@ Status DBImpl::HasTable(const std::string& table_id, bool& has_or_not) { Status DBImpl::AllTables(std::vector& table_schema_array) { if (shutting_down_.load(std::memory_order_acquire)){ - return Status::Error("Milsvus server is shutdown!"); + return Status(DB_ERROR, "Milsvus server is shutdown!"); } return meta_ptr_->AllTables(table_schema_array); @@ -160,7 +162,7 @@ Status DBImpl::AllTables(std::vector& table_schema_array) { Status DBImpl::PreloadTable(const std::string &table_id) { if (shutting_down_.load(std::memory_order_acquire)){ - return Status::Error("Milsvus server is shutdown!"); + return Status(DB_ERROR, "Milsvus server is shutdown!"); } meta::DatePartionedTableFilesSchema files; @@ -182,7 +184,7 @@ Status DBImpl::PreloadTable(const std::string &table_id) { ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, (MetricType)file.metric_type_, file.nlist_); if(engine == nullptr) { ENGINE_LOG_ERROR << "Invalid engine type"; - return Status::Error("Invalid engine type"); + return Status(DB_ERROR, "Invalid engine type"); } size += engine->PhysicalSize(); @@ -195,7 +197,7 @@ Status DBImpl::PreloadTable(const std::string &table_id) { } catch (std::exception &ex) { std::string msg = "Pre-load table encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; - return Status::Error(msg); + return Status(DB_ERROR, msg); } } } @@ -205,7 +207,7 @@ Status DBImpl::PreloadTable(const std::string &table_id) { Status DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { if (shutting_down_.load(std::memory_order_acquire)){ - return Status::Error("Milsvus server is shutdown!"); + return Status(DB_ERROR, "Milsvus server is shutdown!"); } return meta_ptr_->UpdateTableFlag(table_id, flag); @@ -213,7 +215,7 @@ Status DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) { if (shutting_down_.load(std::memory_order_acquire)){ - return Status::Error("Milsvus server is shutdown!"); + return Status(DB_ERROR, "Milsvus server is shutdown!"); } return meta_ptr_->Count(table_id, row_count); @@ -223,7 +225,7 @@ Status DBImpl::InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) { // ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache"; if (shutting_down_.load(std::memory_order_acquire)){ - return Status::Error("Milsvus server is shutdown!"); + return Status(DB_ERROR, "Milsvus server is shutdown!"); } Status status; @@ -312,7 +314,7 @@ Status DBImpl::DropIndex(const std::string& table_id) { Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors, QueryResults &results) { if (shutting_down_.load(std::memory_order_acquire)){ - return Status::Error("Milsvus server is shutdown!"); + return Status(DB_ERROR, "Milsvus server is shutdown!"); } meta::DatesT dates = {utils::GetDate()}; @@ -324,7 +326,7 @@ Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint6 Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) { if (shutting_down_.load(std::memory_order_acquire)){ - return Status::Error("Milsvus server is shutdown!"); + return Status(DB_ERROR, "Milsvus server is shutdown!"); } ENGINE_LOG_DEBUG << "Query by dates for table: " << table_id; @@ -352,7 +354,7 @@ Status DBImpl::Query(const std::string& table_id, const std::vector uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) { if (shutting_down_.load(std::memory_order_acquire)){ - return Status::Error("Milsvus server is shutdown!"); + return Status(DB_ERROR, "Milsvus server is shutdown!"); } ENGINE_LOG_DEBUG << "Query by file ids for table: " << table_id; @@ -380,7 +382,7 @@ Status DBImpl::Query(const std::string& table_id, const std::vector } if(file_id_array.empty()) { - return Status::Error("Invalid file id"); + return Status(DB_ERROR, "Invalid file id"); } cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info before query @@ -391,7 +393,7 @@ Status DBImpl::Query(const std::string& table_id, const std::vector Status DBImpl::Size(uint64_t& result) { if (shutting_down_.load(std::memory_order_acquire)){ - return Status::Error("Milsvus server is shutdown!"); + return Status(DB_ERROR, "Milsvus server is shutdown!"); } return meta_ptr_->Size(result); @@ -598,7 +600,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, std::cout << "ERROR: failed to persist merged index file: " << table_file.location_ << ", possible out of disk space" << std::endl; - return Status::Error(msg); + return Status(DB_ERROR, msg); } //step 4: update table files state @@ -706,7 +708,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { (MetricType)file.metric_type_, file.nlist_); if(to_index == nullptr) { ENGINE_LOG_ERROR << "Invalid engine type"; - return Status::Error("Invalid engine type"); + return Status(DB_ERROR, "Invalid engine type"); } try { @@ -753,7 +755,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough" << std::endl; - return Status::Error(msg); + return Status(DB_ERROR, msg); } //step 4: if table has been deleted, dont save index file @@ -779,7 +781,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { std::cout << "ERROR: failed to persist index file: " << table_file.location_ << ", possible out of disk space" << std::endl; - return Status::Error(msg); + return Status(DB_ERROR, msg); } //step 6: update meta @@ -814,7 +816,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { } catch (std::exception& ex) { std::string msg = "Build index encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; - return Status::Error(msg); + return Status(DB_ERROR, msg); } return Status::OK(); diff --git a/cpp/src/db/Status.cpp b/cpp/src/db/Status.cpp index 65524c62a9..5744f1bdbe 100644 --- a/cpp/src/db/Status.cpp +++ b/cpp/src/db/Status.cpp @@ -12,57 +12,62 @@ namespace zilliz { namespace milvus { namespace engine { -const char* Status::CopyState(const char* state) { - uint32_t size; - std::memcpy(&size, state, sizeof(size)); - char* result = new char[size+5]; - memcpy(result, state, size+5); - return result; +constexpr int CODE_WIDTH = sizeof(ErrorCode); + +Status::Status(ErrorCode code, const std::string& msg) { + //4 bytes store code + //4 bytes store message length + //the left bytes store message string + const uint32_t length = (uint32_t)msg.size(); + char* result = new char[length + sizeof(length) + CODE_WIDTH]; + std::memcpy(result, &code, CODE_WIDTH); + std::memcpy(result + CODE_WIDTH, &length, sizeof(length)); + memcpy(result + sizeof(length) + CODE_WIDTH, msg.data(), length); + + state_ = result; } -Status::Status(Code code, const std::string& msg, const std::string& msg2) { - assert(code != kOK); - const uint32_t len1 = msg.size(); - const uint32_t len2 = msg2.size(); - const uint32_t size = len1 + (len2 ? (2+len2) : 0); - char* result = new char[size+5]; - std::memcpy(result, &size, sizeof(size)); - result[4] = static_cast(code); - memcpy(result+5, msg.data(), len1); - if (len2) { - result[5 + len1] = ':'; - result[6 + len1] = ' '; - memcpy(result + 7 + len1, msg2.data(), len2); - } - state_ = result; +Status::Status() + : state_(nullptr) { + +} + +Status::~Status() { + delete[] state_; +} + +const char* Status::CopyState(const char* state) { + uint32_t length = 0; + std::memcpy(&length, state + CODE_WIDTH, sizeof(length)); + int buff_len = length + sizeof(length) + CODE_WIDTH; + char* result = new char[buff_len]; + memcpy(result, state, buff_len); + return result; } std::string Status::ToString() const { if (state_ == nullptr) return "OK"; - char tmp[30]; + char tmp[32]; const char* type; switch (code()) { - case kOK: + case DB_SUCCESS: type = "OK"; break; - case kNotFound: - type = "NotFound: "; - break; - case kError: + case DB_ERROR: type = "Error: "; break; - case kInvalidDBPath: - type = "InvalidDBPath: "; - break; - case kGroupError: - type = "GroupError: "; - break; - case kDBTransactionError: + case DB_META_TRANSACTION_FAILED: type = "DBTransactionError: "; break; - case kAlreadyExist: + case DB_NOT_FOUND: + type = "NotFound: "; + break; + case DB_ALREADY_EXIST: type = "AlreadyExist: "; break; + case DB_INVALID_PATH: + type = "InvalidPath: "; + break; default: snprintf(tmp, sizeof(tmp), "Unkown code(%d): ", static_cast(code())); @@ -71,9 +76,9 @@ std::string Status::ToString() const { } std::string result(type); - uint32_t length; - memcpy(&length, state_, sizeof(length)); - result.append(state_ + 5, length); + uint32_t length = 0; + memcpy(&length, state_ + CODE_WIDTH, sizeof(length)); + result.append(state_ + sizeof(length) + CODE_WIDTH, length); return result; } diff --git a/cpp/src/db/Status.h b/cpp/src/db/Status.h index 8d58b96e85..60733a5548 100644 --- a/cpp/src/db/Status.h +++ b/cpp/src/db/Status.h @@ -5,8 +5,9 @@ ******************************************************************************/ #pragma once -#include +#include "utils/Error.h" +#include namespace zilliz { namespace milvus { @@ -14,9 +15,9 @@ namespace engine { class Status { public: - Status() noexcept : state_(nullptr) {} - - ~Status() { delete[] state_; } + Status(ErrorCode code, const std::string &msg); + Status(); + ~Status(); Status(const Status &rhs); @@ -31,64 +32,17 @@ class Status { static Status OK() { return Status(); } - static Status - NotFound(const std::string &msg, const std::string &msg2 = "") { - return Status(kNotFound, msg, msg2); - } - static Status - Error(const std::string &msg, const std::string &msg2 = "") { - return Status(kError, msg, msg2); - } - - static Status - InvalidDBPath(const std::string &msg, const std::string &msg2 = "") { - return Status(kInvalidDBPath, msg, msg2); - } - static Status - GroupError(const std::string &msg, const std::string &msg2 = "") { - return Status(kGroupError, msg, msg2); - } - static Status - DBTransactionError(const std::string &msg, const std::string &msg2 = "") { - return Status(kDBTransactionError, msg, msg2); - } - - static Status - AlreadyExist(const std::string &msg, const std::string &msg2 = "") { - return Status(kAlreadyExist, msg, msg2); - } - - bool ok() const { return state_ == nullptr; } - - bool IsNotFound() const { return code() == kNotFound; } - bool IsError() const { return code() == kError; } - - bool IsInvalidDBPath() const { return code() == kInvalidDBPath; } - bool IsGroupError() const { return code() == kGroupError; } - bool IsDBTransactionError() const { return code() == kDBTransactionError; } - bool IsAlreadyExist() const { return code() == kAlreadyExist; } + bool ok() const { return state_ == nullptr || code() == DB_SUCCESS; } std::string ToString() const; + ErrorCode code() const { + return (state_ == nullptr) ? DB_SUCCESS : *(ErrorCode*)(state_); + } + private: const char *state_ = nullptr; - enum Code { - kOK = 0, - kNotFound, - kError, - - kInvalidDBPath, - kGroupError, - kDBTransactionError, - - kAlreadyExist, - }; - - Code code() const { - return (state_ == nullptr) ? kOK : static_cast(state_[4]); - } - Status(Code code, const std::string &msg, const std::string &msg2); static const char *CopyState(const char *s); }; // Status diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index efe238d86f..f03624451b 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -70,7 +70,7 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id auto status = server::CommonUtil::CreateDirectory(table_path); if (status != 0) { ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; - return Status::Error("Failed to create table path"); + return Status(DB_ERROR, "Failed to create table path"); } for(auto& path : options.slave_paths) { @@ -78,7 +78,7 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id status = server::CommonUtil::CreateDirectory(table_path); if (status != 0) { ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; - return Status::Error("Failed to create table path"); + return Status(DB_ERROR, "Failed to create table path"); } } @@ -110,7 +110,7 @@ Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& auto status = server::CommonUtil::CreateDirectory(parent_path); if (status != 0) { ENGINE_LOG_ERROR << "Create directory " << parent_path << " Error"; - return Status::DBTransactionError("Failed to create partition directory"); + return Status(DB_ERROR, "Failed to create partition directory"); } table_file.location_ = parent_path + "/" + table_file.file_id_; @@ -137,7 +137,7 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& tab std::string msg = "Table file doesn't exist: " + table_file.file_id_; ENGINE_LOG_ERROR << msg; - return Status::Error(msg); + return Status(DB_ERROR, msg); } Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 344d26f156..59508ebcef 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -41,7 +41,7 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2"; AutoGenParams(index_->GetType(), 0, build_cfg); auto ec = std::static_pointer_cast(index_)->Build(build_cfg); - if (ec != server::KNOWHERE_SUCCESS) { throw Exception("Build index error"); } + if (ec != KNOWHERE_SUCCESS) { throw Exception("Build index error"); } } ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index, @@ -85,8 +85,8 @@ VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) { Status ExecutionEngineImpl::AddWithIds(long n, const float *xdata, const long *xids) { auto ec = index_->Add(n, xdata, xids); - if (ec != server::KNOWHERE_SUCCESS) { - return Status::Error("Add error"); + if (ec != KNOWHERE_SUCCESS) { + return Status(DB_ERROR, "Add error"); } return Status::OK(); } @@ -117,8 +117,8 @@ size_t ExecutionEngineImpl::PhysicalSize() const { Status ExecutionEngineImpl::Serialize() { auto ec = write_index(index_, location_); - if (ec != server::KNOWHERE_SUCCESS) { - return Status::Error("Serialize: write to disk error"); + if (ec != KNOWHERE_SUCCESS) { + return Status(DB_ERROR, "Serialize: write to disk error"); } return Status::OK(); } @@ -134,15 +134,15 @@ Status ExecutionEngineImpl::Load(bool to_cache) { if(index_ == nullptr) { std::string msg = "Failed to load index from " + location_; ENGINE_LOG_ERROR << msg; - return Status::Error(msg); + return Status(DB_ERROR, msg); } else { ENGINE_LOG_DEBUG << "Disk io from: " << location_; } } catch (knowhere::KnowhereException &e) { ENGINE_LOG_ERROR << e.what(); - return Status::Error(e.what()); + return Status(DB_ERROR, e.what()); } catch (std::exception &e) { - return Status::Error(e.what()); + return Status(DB_ERROR, e.what()); } } @@ -160,7 +160,7 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) { } else { if(index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to gpu"; - return Status::Error("index is null"); + return Status(DB_ERROR, "index is null"); } try { @@ -168,9 +168,9 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) { ENGINE_LOG_DEBUG << "CPU to GPU" << device_id; } catch (knowhere::KnowhereException &e) { ENGINE_LOG_ERROR << e.what(); - return Status::Error(e.what()); + return Status(DB_ERROR, e.what()); } catch (std::exception &e) { - return Status::Error(e.what()); + return Status(DB_ERROR, e.what()); } } @@ -189,7 +189,7 @@ Status ExecutionEngineImpl::CopyToCpu() { } else { if(index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to cpu"; - return Status::Error("index is null"); + return Status(DB_ERROR, "index is null"); } try { @@ -197,9 +197,9 @@ Status ExecutionEngineImpl::CopyToCpu() { ENGINE_LOG_DEBUG << "GPU to CPU"; } catch (knowhere::KnowhereException &e) { ENGINE_LOG_ERROR << e.what(); - return Status::Error(e.what()); + return Status(DB_ERROR, e.what()); } catch (std::exception &e) { - return Status::Error(e.what()); + return Status(DB_ERROR, e.what()); } } @@ -223,7 +223,7 @@ ExecutionEnginePtr ExecutionEngineImpl::Clone() { Status ExecutionEngineImpl::Merge(const std::string &location) { if (location == location_) { - return Status::Error("Cannot Merge Self"); + return Status(DB_ERROR, "Cannot Merge Self"); } ENGINE_LOG_DEBUG << "Merge index file: " << location << " to: " << location_; @@ -235,26 +235,26 @@ Status ExecutionEngineImpl::Merge(const std::string &location) { to_merge = read_index(location); } catch (knowhere::KnowhereException &e) { ENGINE_LOG_ERROR << e.what(); - return Status::Error(e.what()); + return Status(DB_ERROR, e.what()); } catch (std::exception &e) { - return Status::Error(e.what()); + return Status(DB_ERROR, e.what()); } } if(index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to merge"; - return Status::Error("index is null"); + return Status(DB_ERROR, "index is null"); } if (auto file_index = std::dynamic_pointer_cast(to_merge)) { auto ec = index_->Add(file_index->Count(), file_index->GetRawVectors(), file_index->GetRawIds()); - if (ec != server::KNOWHERE_SUCCESS) { + if (ec != KNOWHERE_SUCCESS) { ENGINE_LOG_ERROR << "Merge: Add Error"; - return Status::Error("Merge: Add Error"); + return Status(DB_ERROR, "Merge: Add Error"); } return Status::OK(); } else { - return Status::Error("file index type is not idmap"); + return Status(DB_ERROR, "file index type is not idmap"); } } @@ -284,7 +284,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t from_index->GetRawVectors(), from_index->GetRawIds(), build_cfg); - if (ec != server::KNOWHERE_SUCCESS) { throw Exception("Build index error"); } + if (ec != KNOWHERE_SUCCESS) { throw Exception("Build index error"); } return std::make_shared(to_index, location, engine_type, metric_type_, nlist_); } @@ -297,15 +297,15 @@ Status ExecutionEngineImpl::Search(long n, long *labels) const { if(index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search"; - return Status::Error("index is null"); + return Status(DB_ERROR, "index is null"); } ENGINE_LOG_DEBUG << "Search Params: [k] " << k << " [nprobe] " << nprobe; auto cfg = Config::object{{"k", k}, {"nprobe", nprobe}}; auto ec = index_->Search(n, data, distances, labels, cfg); - if (ec != server::KNOWHERE_SUCCESS) { + if (ec != KNOWHERE_SUCCESS) { ENGINE_LOG_ERROR << "Search error"; - return Status::Error("Search: Search Error"); + return Status(DB_ERROR, "Search: Search Error"); } return Status::OK(); } diff --git a/cpp/src/db/insert/MemTable.cpp b/cpp/src/db/insert/MemTable.cpp index ff9c25e3e7..0bb67d1387 100644 --- a/cpp/src/db/insert/MemTable.cpp +++ b/cpp/src/db/insert/MemTable.cpp @@ -38,7 +38,7 @@ Status MemTable::Add(VectorSource::Ptr &source, IDNumbers &vector_ids) { if (!status.ok()) { std::string err_msg = "MemTable::Add failed: " + status.ToString(); ENGINE_LOG_ERROR << err_msg; - return Status::Error(err_msg); + return Status(DB_ERROR, err_msg); } } return Status::OK(); @@ -58,7 +58,7 @@ Status MemTable::Serialize() { if (!status.ok()) { std::string err_msg = "MemTable::Serialize failed: " + status.ToString(); ENGINE_LOG_ERROR << err_msg; - return Status::Error(err_msg); + return Status(DB_ERROR, err_msg); } std::lock_guard lock(mutex_); mem_table_file = mem_table_file_list_.erase(mem_table_file); diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index 2054a5047e..25abb8db90 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -49,7 +49,7 @@ Status MemTableFile::Add(const VectorSource::Ptr &source, IDNumbers& vector_ids) std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " + std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_; ENGINE_LOG_ERROR << err_msg; - return Status::Error(err_msg); + return Status(DB_ERROR, err_msg); } size_t single_vector_mem_size = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index dafe12872f..13c3597bd1 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -34,9 +34,15 @@ using namespace mysqlpp; namespace { -Status HandleException(const std::string &desc, std::exception &e) { - ENGINE_LOG_ERROR << desc << ": " << e.what(); - return Status::DBTransactionError(desc, e.what()); +Status HandleException(const std::string &desc, const char* what = nullptr) { + if(what == nullptr) { + ENGINE_LOG_ERROR << desc; + return Status(DB_META_TRANSACTION_FAILED, desc); + } else { + std::string msg = desc + ":" + what; + ENGINE_LOG_ERROR << msg; + return Status(DB_META_TRANSACTION_FAILED, msg); + } } } @@ -71,8 +77,9 @@ Status MySQLMetaImpl::Initialize() { if (!boost::filesystem::is_directory(options_.path)) { auto ret = boost::filesystem::create_directory(options_.path); if (!ret) { - ENGINE_LOG_ERROR << "Failed to create db directory " << options_.path; - return Status::DBTransactionError("Failed to create db directory", options_.path); + std::string msg = "Failed to create db directory " + options_.path; + ENGINE_LOG_ERROR << msg; + return Status(DB_META_TRANSACTION_FAILED, msg); } } @@ -97,7 +104,7 @@ Status MySQLMetaImpl::Initialize() { std::string dialect = pieces_match[1].str(); std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); if (dialect.find("mysql") == std::string::npos) { - return Status::Error("URI's dialect is not MySQL"); + return Status(DB_ERROR, "URI's dialect is not MySQL"); } std::string username = pieces_match[2].str(); std::string password = pieces_match[3].str(); @@ -125,13 +132,13 @@ Status MySQLMetaImpl::Initialize() { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } if (!connectionPtr->thread_aware()) { ENGINE_LOG_ERROR << "MySQL++ wasn't built with thread awareness! Can't run without it."; - return Status::Error("MySQL++ wasn't built with thread awareness! Can't run without it."); + return Status(DB_ERROR, "MySQL++ wasn't built with thread awareness! Can't run without it."); } Query InitializeQuery = connectionPtr->query(); @@ -150,7 +157,7 @@ Status MySQLMetaImpl::Initialize() { ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); if (!InitializeQuery.exec()) { - return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); + return HandleException("Initialization Error", InitializeQuery.error()); } InitializeQuery << "CREATE TABLE IF NOT EXISTS TableFiles (" << @@ -168,24 +175,22 @@ Status MySQLMetaImpl::Initialize() { ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); if (!InitializeQuery.exec()) { - return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); + return HandleException("Initialization Error", InitializeQuery.error()); } } //Scoped Connection - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR DURING INITIALIZATION" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR DURING INITIALIZATION", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR DURING INITIALIZATION", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR DURING INITIALIZATION" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR DURING INITIALIZATION", er.what()); + return HandleException("GENERAL ERROR DURING INITIALIZATION", e.what()); } catch (std::exception &e) { - return HandleException("Encounter exception during initialization", e); + return HandleException("GENERAL ERROR DURING INITIALIZATION", e.what()); } } else { ENGINE_LOG_ERROR << "Wrong URI format. URI = " << uri; - return Status::Error("Wrong URI format"); + return Status(DB_ERROR, "Wrong URI format"); } return Status::OK(); @@ -217,7 +222,7 @@ Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } @@ -232,19 +237,17 @@ Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str(); if (!dropPartitionsByDatesQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES"; - return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", - dropPartitionsByDatesQuery.error()); + return HandleException("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", dropPartitionsByDatesQuery.error()); } } //Scoped Connection - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES", er.what()); + return HandleException("GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES", e.what()); } return Status::OK(); } @@ -256,7 +259,7 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query createTableQuery = connectionPtr->query(); @@ -274,9 +277,9 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { if (res.num_rows() == 1) { int state = res[0]["state"]; if (TableSchema::TO_DELETE == state) { - return Status::Error("Table already exists and it is in delete state, please wait a second"); + return Status(DB_ERROR, "Table already exists and it is in delete state, please wait a second"); } else { - return Status::AlreadyExist("Table already exists"); + return Status(DB_ALREADY_EXIST, "Table already exists"); } } } @@ -307,23 +310,20 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { //Consume all results to avoid "Commands out of sync" error } else { - ENGINE_LOG_ERROR << "Add Table Error"; - return Status::DBTransactionError("Add Table Error", createTableQuery.error()); + return HandleException("Add Table Error", createTableQuery.error()); } } //Scoped Connection return utils::CreateTablePath(options_, table_schema.table_id_); - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN CREATING TABLE", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN ADDING TABLE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE", er.what()); + return HandleException("GENERAL ERROR WHEN CREATING TABLE", e.what()); } catch (std::exception &e) { - return HandleException("Encounter exception when create table", e); + return HandleException("GENERAL ERROR WHEN CREATING TABLE", e.what()); } } @@ -331,7 +331,7 @@ Status MySQLMetaImpl::FilesByType(const std::string &table_id, const std::vector &file_types, std::vector &file_ids) { if(file_types.empty()) { - return Status::Error("file types array is empty"); + return Status(DB_ERROR, "file types array is empty"); } try { @@ -342,7 +342,7 @@ Status MySQLMetaImpl::FilesByType(const std::string &table_id, ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } std::string types; @@ -406,14 +406,14 @@ Status MySQLMetaImpl::FilesByType(const std::string &table_id, << " index files:" << index_count << " backup files:" << backup_count; } - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN GET FILE BY TYPE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN GET FILE BY TYPE", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN GET FILE BY TYPE", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN GET FILE BY TYPE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN GET FILE BY TYPE", er.what()); + return HandleException("GENERAL ERROR WHEN GET FILE BY TYPE", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN GET FILE BY TYPE", e.what()); } return Status::OK(); @@ -427,7 +427,7 @@ Status MySQLMetaImpl::UpdateTableIndexParam(const std::string &table_id, const T ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query updateTableIndexParamQuery = connectionPtr->query(); @@ -462,24 +462,22 @@ Status MySQLMetaImpl::UpdateTableIndexParam(const std::string &table_id, const T if (!updateTableIndexParamQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE INDEX PARAM"; - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE INDEX PARAM", - updateTableIndexParamQuery.error()); + return HandleException("QUERY ERROR WHEN UPDATING TABLE INDEX PARAM", updateTableIndexParamQuery.error()); } } else { - return Status::NotFound("Table " + table_id + " not found"); + return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); } } //Scoped Connection - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE INDEX PARAM" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE INDEX PARAM", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN UPDATING TABLE INDEX PARAM", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE INDEX PARAM" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE INDEX PARAM", er.what()); + return HandleException("GENERAL ERROR WHEN UPDATING TABLE INDEX PARAM", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN UPDATING TABLE INDEX PARAM", e.what()); } return Status::OK(); @@ -493,7 +491,7 @@ Status MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query updateTableFlagQuery = connectionPtr->query(); @@ -503,23 +501,20 @@ Status MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFlag: " << updateTableFlagQuery.str(); - if (!updateTableFlagQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FLAG"; - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FLAG", - updateTableFlagQuery.error()); + return HandleException("QUERY ERROR WHEN UPDATING TABLE FLAG", updateTableFlagQuery.error()); } } //Scoped Connection - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FLAG" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FLAG", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN UPDATING TABLE FLAG", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FLAG" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FLAG", er.what()); + return HandleException("GENERAL ERROR WHEN UPDATING TABLE FLAG", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN UPDATING TABLE FLAG", e.what()); } return Status::OK(); @@ -533,7 +528,7 @@ Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query describeTableIndexQuery = connectionPtr->query(); @@ -553,19 +548,19 @@ Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex index.nlist_ = resRow["nlist"]; index.metric_type_ = resRow["metric_type"]; } else { - return Status::NotFound("Table " + table_id + " not found"); + return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); } } //Scoped Connection - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBE TABLE INDEX" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBE TABLE INDEX", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN DESCRIBE TABLE INDEX", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DESCRIBE TABLE INDEX" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBE TABLE INDEX", er.what()); + return HandleException("GENERAL ERROR WHEN DESCRIBE TABLE INDEX", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN UPDATING TABLE FLAG", e.what()); } return Status::OK(); @@ -579,7 +574,7 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query dropTableIndexQuery = connectionPtr->query(); @@ -594,9 +589,7 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); if (!dropTableIndexQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROP TABLE INDEX"; - return Status::DBTransactionError("QUERY ERROR WHEN DROP TABLE INDEX", - dropTableIndexQuery.error()); + return HandleException("QUERY ERROR WHEN DROPPING TABLE INDEX", dropTableIndexQuery.error()); } //set all backup file to raw @@ -609,9 +602,7 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); if (!dropTableIndexQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROP TABLE INDEX"; - return Status::DBTransactionError("QUERY ERROR WHEN DROP TABLE INDEX", - dropTableIndexQuery.error()); + return HandleException("QUERY ERROR WHEN DROPPING TABLE INDEX", dropTableIndexQuery.error()); } //set table index type to raw @@ -624,21 +615,19 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); if (!dropTableIndexQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROP TABLE INDEX"; - return Status::DBTransactionError("QUERY ERROR WHEN DROP TABLE INDEX", - dropTableIndexQuery.error()); + return HandleException("QUERY ERROR WHEN DROPPING TABLE INDEX", dropTableIndexQuery.error()); } } //Scoped Connection - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROP TABLE INDEX" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DROP TABLE INDEX", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN DROPPING TABLE INDEX", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DROP TABLE INDEX" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DROP TABLE INDEX", er.what()); + return HandleException("GENERAL ERROR WHEN DROPPING TABLE INDEX", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN DROPPING TABLE INDEX", e.what()); } return Status::OK(); @@ -651,7 +640,7 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } //soft delete table @@ -664,8 +653,7 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str(); if (!deleteTableQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE"; - return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); + return HandleException("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); } } //Scoped Connection @@ -674,14 +662,14 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { DeleteTableFiles(table_id); } - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN DELETING TABLE", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE", er.what()); + return HandleException("GENERAL ERROR WHEN DELETING TABLE", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN DELETING TABLE", e.what()); } return Status::OK(); @@ -694,7 +682,7 @@ Status MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } //soft delete table files @@ -709,18 +697,17 @@ Status MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); if (!deleteTableFilesQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES"; - return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableFilesQuery.error()); + return HandleException("QUERY ERROR WHEN DELETING TABLE FILES", deleteTableFilesQuery.error()); } } //Scoped Connection - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE FILES", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN DELETING TABLE FILES", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE FILES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE FILES", er.what()); + return HandleException("GENERAL ERROR WHEN DELETING TABLE FILES", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN DELETING TABLE FILES", e.what()); } return Status::OK(); @@ -734,7 +721,7 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query describeTableQuery = connectionPtr->query(); @@ -765,17 +752,17 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { table_schema.metric_type_ = resRow["metric_type"]; } else { - return Status::NotFound("Table " + table_schema.table_id_ + " not found"); + return Status(DB_NOT_FOUND, "Table " + table_schema.table_id_ + " not found"); } - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBING TABLE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING TABLE", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN DESCRIBING TABLE", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DESCRIBING TABLE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBING TABLE", er.what()); + return HandleException("GENERAL ERROR WHEN DESCRIBING TABLE", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN DESCRIBING TABLE", e.what()); } return Status::OK(); @@ -789,7 +776,7 @@ Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query hasTableQuery = connectionPtr->query(); @@ -808,14 +795,14 @@ Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { int check = res[0]["check"]; has_or_not = (check == 1); - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CHECKING IF TABLE EXISTS" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN CHECKING IF TABLE EXISTS", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN CHECKING IF TABLE EXISTS", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CHECKING IF TABLE EXISTS" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN CHECKING IF TABLE EXISTS", er.what()); + return HandleException("GENERAL ERROR WHEN CHECKING IF TABLE EXISTS", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN CHECKING IF TABLE EXISTS", e.what()); } return Status::OK(); @@ -829,7 +816,7 @@ Status MySQLMetaImpl::AllTables(std::vector &table_schema_array) { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query allTablesQuery = connectionPtr->query(); @@ -863,14 +850,14 @@ Status MySQLMetaImpl::AllTables(std::vector &table_schema_array) { table_schema_array.emplace_back(table_schema); } - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBING ALL TABLES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING ALL TABLES", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN DESCRIBING ALL TABLES", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DESCRIBING ALL TABLES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBING ALL TABLES", er.what()); + return HandleException("GENERAL ERROR WHEN DESCRIBING ALL TABLES", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN DESCRIBING ALL TABLES", e.what()); } return Status::OK(); @@ -917,7 +904,7 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query createTableFileQuery = connectionPtr->query(); @@ -934,23 +921,20 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { //Consume all results to avoid "Commands out of sync" error } else { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE FILE"; - return Status::DBTransactionError("Add file Error", createTableFileQuery.error()); + return HandleException("QUERY ERROR WHEN CREATING TABLE FILE", createTableFileQuery.error()); } } // Scoped Connection return utils::CreateTableFilePath(options_, file_schema); - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE FILE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE FILE", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN CREATING TABLE FILE", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN ADDING TABLE FILE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE FILE", er.what()); - } catch (std::exception &ex) { - return HandleException("Encounter exception when create table file", ex); + return HandleException("GENERAL ERROR WHEN CREATING TABLE FILE", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN CREATING TABLE FILE", e.what()); } } @@ -964,7 +948,7 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query filesToIndexQuery = connectionPtr->query(); @@ -1023,14 +1007,14 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { files.push_back(table_file); } - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO INDEX" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO INDEX", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", er.what()); + return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", e.what()); } return Status::OK(); @@ -1049,7 +1033,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query filesToSearchQuery = connectionPtr->query(); @@ -1137,14 +1121,14 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, files[table_file.date_].push_back(table_file); } - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO SEARCH" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO SEARCH", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", er.what()); + return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", e.what()); } return Status::OK(); @@ -1170,7 +1154,7 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query filesToMergeQuery = connectionPtr->query(); @@ -1230,14 +1214,14 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, files[table_file.date_].push_back(table_file); } - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO MERGE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO MERGE", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", er.what()); + return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", e.what()); } return Status::OK(); @@ -1263,10 +1247,9 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } - Query getTableFileQuery = connectionPtr->query(); getTableFileQuery << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on " << "FROM TableFiles " << @@ -1321,14 +1304,14 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, table_files.emplace_back(file_schema); } - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING TABLE FILES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING TABLE FILES", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN RETRIEVING TABLE FILES", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING TABLE FILES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING TABLE FILES", er.what()); + return HandleException("GENERAL ERROR WHEN RETRIEVING TABLE FILES", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN RETRIEVING TABLE FILES", e.what()); } return Status::OK(); @@ -1352,7 +1335,7 @@ Status MySQLMetaImpl::Archive() { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query archiveQuery = connectionPtr->query(); @@ -1364,17 +1347,17 @@ Status MySQLMetaImpl::Archive() { ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str(); if (!archiveQuery.exec()) { - return Status::DBTransactionError("QUERY ERROR DURING ARCHIVE", archiveQuery.error()); + return HandleException("QUERY ERROR DURING ARCHIVE", archiveQuery.error()); } - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DURING ARCHIVE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DURING ARCHIVE", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN DURING ARCHIVE", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DURING ARCHIVE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DURING ARCHIVE", er.what()); + return HandleException("GENERAL ERROR WHEN DURING ARCHIVE", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN DURING ARCHIVE", e.what()); } } if (criteria == "disk") { @@ -1398,10 +1381,9 @@ Status MySQLMetaImpl::Size(uint64_t &result) { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } - Query getSizeQuery = connectionPtr->query(); getSizeQuery << "SELECT IFNULL(SUM(file_size),0) AS sum " << "FROM TableFiles " << @@ -1418,14 +1400,14 @@ Status MySQLMetaImpl::Size(uint64_t &result) { result = res[0]["sum"]; } - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING SIZE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING SIZE", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN RETRIEVING SIZE", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING SIZE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING SIZE", er.what()); + return HandleException("GENERAL ERROR WHEN RETRIEVING SIZE", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN RETRIEVING SIZE", e.what()); } return Status::OK(); @@ -1445,7 +1427,7 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query discardFilesQuery = connectionPtr->query(); @@ -1488,21 +1470,20 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { status = discardFilesQuery.exec(); if (!status) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES"; - return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", discardFilesQuery.error()); + return HandleException("QUERY ERROR WHEN DISCARDING FILES", discardFilesQuery.error()); } } //Scoped Connection return DiscardFiles(to_discard_size); - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN DISCARDING FILES", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DISCARDING FILES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DISCARDING FILES", er.what()); + return HandleException("GENERAL ERROR WHEN DISCARDING FILES", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN DISCARDING FILES", e.what()); } } @@ -1516,7 +1497,7 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query updateTableFileQuery = connectionPtr->query(); @@ -1564,26 +1545,24 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); - if (!updateTableFileQuery.exec()) { ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE"; - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", - updateTableFileQuery.error()); + return HandleException("QUERY ERROR WHEN UPDATING TABLE FILE", updateTableFileQuery.error()); } } //Scoped Connection - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILE", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILE", er.what()); + return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILE", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILE", e.what()); } + return Status::OK(); } @@ -1592,7 +1571,7 @@ Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query updateTableFilesToIndexQuery = connectionPtr->query(); @@ -1605,19 +1584,17 @@ Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFilesToIndex: " << updateTableFilesToIndexQuery.str(); if (!updateTableFilesToIndexQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE"; - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", - updateTableFilesToIndexQuery.error()); + return HandleException("QUERY ERROR WHEN UPDATING TABLE FILE TO INDEX", updateTableFilesToIndexQuery.error()); } - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES TO INDEX" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES TO INDEX", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES TO INDEX", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILES TO INDEX" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILES TO INDEX", er.what()); + return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES TO INDEX", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES TO INDEX", e.what()); } return Status::OK(); @@ -1630,7 +1607,7 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query updateTableFilesQuery = connectionPtr->query(); @@ -1689,21 +1666,19 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); if (!updateTableFilesQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES"; - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", - updateTableFilesQuery.error()); + return HandleException("QUERY ERROR WHEN UPDATING TABLE FILES", updateTableFilesQuery.error()); } } } //Scoped Connection - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILES", er.what()); + return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES", e.what()); } return Status::OK(); @@ -1721,7 +1696,7 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query cleanUpFilesWithTTLQuery = connectionPtr->query(); @@ -1775,21 +1750,19 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); if (!cleanUpFilesWithTTLQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; - return Status::DBTransactionError("CleanUpFilesWithTTL Error", - cleanUpFilesWithTTLQuery.error()); + return HandleException("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", cleanUpFilesWithTTLQuery.error()); } } } //Scoped Connection - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); + return HandleException("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", e.what()); } //remove to_delete tables @@ -1800,7 +1773,7 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query cleanUpFilesWithTTLQuery = connectionPtr->query(); @@ -1832,21 +1805,19 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); if (!cleanUpFilesWithTTLQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; - return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", - cleanUpFilesWithTTLQuery.error()); + return HandleException("QUERY ERROR WHEN CLEANING UP TABLES WITH TTL", cleanUpFilesWithTTLQuery.error()); } } } //Scoped Connection - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); + return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); } //remove deleted table folder @@ -1858,7 +1829,7 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } for(auto& table_id : table_ids) { @@ -1876,14 +1847,14 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } } } - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", er.what()); + return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); } return Status::OK(); @@ -1894,7 +1865,7 @@ Status MySQLMetaImpl::CleanUp() { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query cleanUpQuery = connectionPtr->query(); @@ -1917,19 +1888,18 @@ Status MySQLMetaImpl::CleanUp() { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); if (!cleanUpQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; - return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); + return HandleException("QUERY ERROR WHEN CLEANING UP FILES", cleanUpQuery.error()); } } - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN CLEANING UP FILES", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES", er.what()); + return HandleException("GENERAL ERROR WHEN CLEANING UP FILES", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN CLEANING UP FILES", e.what()); } return Status::OK(); @@ -1952,7 +1922,7 @@ Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } @@ -1980,19 +1950,19 @@ Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { errorMsg << "MySQLMetaImpl::Count: " << "table dimension = " << std::to_string(table_schema.dimension_) << ", table_id = " << table_id; ENGINE_LOG_ERROR << errorMsg.str(); - return Status::Error(errorMsg.str()); + return Status(DB_ERROR, errorMsg.str()); } result /= table_schema.dimension_; result /= sizeof(float); - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING COUNT" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING COUNT", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN RETRIEVING COUNT", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING COUNT" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING COUNT", er.what()); + return HandleException("GENERAL ERROR WHEN RETRIEVING COUNT", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN RETRIEVING COUNT", e.what()); } return Status::OK(); @@ -2004,7 +1974,7 @@ Status MySQLMetaImpl::DropAll() { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to database server"); } Query dropTableQuery = connectionPtr->query(); @@ -2015,20 +1985,17 @@ Status MySQLMetaImpl::DropAll() { if (dropTableQuery.exec()) { return Status::OK(); } else { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING TABLE"; - return Status::DBTransactionError("DROP TABLE ERROR", dropTableQuery.error()); + return HandleException("QUERY ERROR WHEN DROPPING ALL", dropTableQuery.error()); } - } catch (const BadQuery &er) { + } catch (const BadQuery &e) { // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING TABLE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DROPPING TABLE", er.what()); - } catch (const Exception &er) { + return HandleException("GENERAL ERROR WHEN DROPPING ALL", e.what()); + } catch (const Exception &e) { // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DROPPING TABLE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING TABLE", er.what()); + return HandleException("GENERAL ERROR WHEN DROPPING ALL", e.what()); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR WHEN DROPPING ALL", e.what()); } - - return Status::OK(); } } // namespace meta diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index e85ab5122c..2b198b61dc 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -29,9 +29,15 @@ using namespace sqlite_orm; namespace { -Status HandleException(const std::string& desc, std::exception &e) { - ENGINE_LOG_ERROR << desc << ": " << e.what(); - return Status::DBTransactionError(desc, e.what()); +Status HandleException(const std::string &desc, const char* what = nullptr) { + if(what == nullptr) { + ENGINE_LOG_ERROR << desc; + return Status(DB_META_TRANSACTION_FAILED, desc); + } else { + std::string msg = desc + ":" + what; + ENGINE_LOG_ERROR << msg; + return Status(DB_META_TRANSACTION_FAILED, msg); + } } } @@ -97,8 +103,9 @@ Status SqliteMetaImpl::Initialize() { if (!boost::filesystem::is_directory(options_.path)) { auto ret = boost::filesystem::create_directory(options_.path); if (!ret) { - ENGINE_LOG_ERROR << "Failed to create db directory " << options_.path; - return Status::InvalidDBPath("Failed to create db directory", options_.path); + std::string msg = "Failed to create db directory " + options_.path; + ENGINE_LOG_ERROR << msg; + return Status(DB_INVALID_PATH, msg); } } @@ -141,7 +148,7 @@ Status SqliteMetaImpl::DropPartitionsByDates(const std::string &table_id, in(&TableFileSchema::date_, dates) )); } catch (std::exception &e) { - return HandleException("Encounter exception when drop partition", e); + return HandleException("Encounter exception when drop partition", e.what()); } return Status::OK(); @@ -162,10 +169,10 @@ Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) { where(c(&TableSchema::table_id_) == table_schema.table_id_)); if (table.size() == 1) { if(TableSchema::TO_DELETE == std::get<0>(table[0])) { - return Status::Error("Table already exists and it is in delete state, please wait a second"); + return Status(DB_ERROR, "Table already exists and it is in delete state, please wait a second"); } else { // Change from no error to already exist. - return Status::AlreadyExist("Table already exists"); + return Status(DB_ALREADY_EXIST, "Table already exists"); } } } @@ -176,15 +183,14 @@ Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) { try { auto id = ConnectorPtr->insert(table_schema); table_schema.id_ = id; - } catch (...) { - ENGINE_LOG_ERROR << "sqlite transaction failed"; - return Status::DBTransactionError("Add Table Error"); + } catch (std::exception &e) { + return HandleException("Encounter exception when create table", e.what()); } return utils::CreateTablePath(options_, table_schema.table_id_); } catch (std::exception &e) { - return HandleException("Encounter exception when create table", e); + return HandleException("Encounter exception when create table", e.what()); } return Status::OK(); @@ -208,7 +214,7 @@ Status SqliteMetaImpl::DeleteTable(const std::string& table_id) { )); } catch (std::exception &e) { - return HandleException("Encounter exception when delete table", e); + return HandleException("Encounter exception when delete table", e.what()); } return Status::OK(); @@ -233,7 +239,7 @@ Status SqliteMetaImpl::DeleteTableFiles(const std::string& table_id) { )); } catch (std::exception &e) { - return HandleException("Encounter exception when delete table files", e); + return HandleException("Encounter exception when delete table files", e.what()); } return Status::OK(); @@ -266,11 +272,11 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { table_schema.nlist_ = std::get<7>(groups[0]); table_schema.metric_type_ = std::get<8>(groups[0]); } else { - return Status::NotFound("Table " + table_schema.table_id_ + " not found"); + return Status(DB_NOT_FOUND, "Table " + table_schema.table_id_ + " not found"); } } catch (std::exception &e) { - return HandleException("Encounter exception when describe table", e); + return HandleException("Encounter exception when describe table", e.what()); } return Status::OK(); @@ -280,7 +286,7 @@ Status SqliteMetaImpl::FilesByType(const std::string& table_id, const std::vector& file_types, std::vector& file_ids) { if(file_types.empty()) { - return Status::Error("file types array is empty"); + return Status(DB_ERROR, "file types array is empty"); } try { @@ -330,7 +336,7 @@ Status SqliteMetaImpl::FilesByType(const std::string& table_id, } } catch (std::exception &e) { - return HandleException("Encounter exception when check non index files", e); + return HandleException("Encounter exception when check non index files", e.what()); } return Status::OK(); } @@ -366,7 +372,7 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const ConnectorPtr->update(table_schema); } else { - return Status::NotFound("Table " + table_id + " not found"); + return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); } //set all backup file to raw @@ -382,7 +388,7 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const } catch (std::exception &e) { std::string msg = "Encounter exception when update table index: table_id = " + table_id; - return HandleException(msg, e); + return HandleException(msg, e.what()); } return Status::OK(); @@ -403,7 +409,7 @@ Status SqliteMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag } catch (std::exception &e) { std::string msg = "Encounter exception when update table flag: table_id = " + table_id; - return HandleException(msg, e); + return HandleException(msg, e.what()); } return Status::OK(); @@ -424,11 +430,11 @@ Status SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableInde index.nlist_ = std::get<1>(groups[0]); index.metric_type_ = std::get<2>(groups[0]); } else { - return Status::NotFound("Table " + table_id + " not found"); + return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); } } catch (std::exception &e) { - return HandleException("Encounter exception when describe index", e); + return HandleException("Encounter exception when describe index", e.what()); } return Status::OK(); @@ -475,7 +481,7 @@ Status SqliteMetaImpl::DropTableIndex(const std::string &table_id) { )); } catch (std::exception &e) { - return HandleException("Encounter exception when delete table index files", e); + return HandleException("Encounter exception when delete table index files", e.what()); } return Status::OK(); @@ -496,7 +502,7 @@ Status SqliteMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { } } catch (std::exception &e) { - return HandleException("Encounter exception when lookup table", e); + return HandleException("Encounter exception when lookup table", e.what()); } return Status::OK(); @@ -532,7 +538,7 @@ Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { } } catch (std::exception &e) { - return HandleException("Encounter exception when lookup all tables", e); + return HandleException("Encounter exception when lookup all tables", e.what()); } return Status::OK(); @@ -571,8 +577,8 @@ Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { return utils::CreateTableFilePath(options_, file_schema); - } catch (std::exception& ex) { - return HandleException("Encounter exception when create table file", ex); + } catch (std::exception& e) { + return HandleException("Encounter exception when create table file", e.what()); } return Status::OK(); @@ -629,7 +635,7 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { } } catch (std::exception &e) { - return HandleException("Encounter exception when iterate raw files", e); + return HandleException("Encounter exception when iterate raw files", e.what()); } return Status::OK(); @@ -654,8 +660,12 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, auto match_tableid = c(&TableFileSchema::table_id_) == table_id; - std::vector file_type = {(int) TableFileSchema::RAW, (int) TableFileSchema::TO_INDEX, (int) TableFileSchema::INDEX}; - auto match_type = in(&TableFileSchema::file_type_, file_type); + std::vector file_types = { + (int) TableFileSchema::RAW, + (int) TableFileSchema::TO_INDEX, + (int) TableFileSchema::INDEX + }; + auto match_type = in(&TableFileSchema::file_type_, file_types); TableSchema table_schema; table_schema.table_id_ = table_id; @@ -710,7 +720,7 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, ENGINE_LOG_ERROR << "No file to search for table: " << table_id; } } catch (std::exception &e) { - return HandleException("Encounter exception when iterate index files", e); + return HandleException("Encounter exception when iterate index files", e.what()); } @@ -773,7 +783,7 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, files[table_file.date_].push_back(table_file); } } catch (std::exception &e) { - return HandleException("Encounter exception when iterate merge files", e); + return HandleException("Encounter exception when iterate merge files", e.what()); } return Status::OK(); @@ -824,7 +834,7 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, table_files.emplace_back(file_schema); } } catch (std::exception &e) { - return HandleException("Encounter exception when lookup table files", e); + return HandleException("Encounter exception when lookup table files", e.what()); } return Status::OK(); @@ -856,7 +866,7 @@ Status SqliteMetaImpl::Archive() { c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE )); } catch (std::exception &e) { - return HandleException("Encounter exception when update table files", e); + return HandleException("Encounter exception when update table files", e.what()); } } if (criteria == engine::ARCHIVE_CONF_DISK) { @@ -886,7 +896,7 @@ Status SqliteMetaImpl::Size(uint64_t &result) { } } catch (std::exception &e) { - return HandleException("Encounter exception when calculte db size", e); + return HandleException("Encounter exception when calculte db size", e.what()); } return Status::OK(); @@ -943,12 +953,11 @@ Status SqliteMetaImpl::DiscardFiles(long to_discard_size) { }); if (!commited) { - ENGINE_LOG_ERROR << "sqlite transaction failed"; - return Status::DBTransactionError("Update table file error"); + return HandleException("DiscardFiles error: sqlite transaction failed"); } } catch (std::exception &e) { - return HandleException("Encounter exception when discard table file", e); + return HandleException("Encounter exception when discard table file", e.what()); } return DiscardFiles(to_discard_size); @@ -976,7 +985,7 @@ Status SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { } catch (std::exception &e) { std::string msg = "Exception update table file: table_id = " + file_schema.table_id_ + " file_id = " + file_schema.file_id_; - return HandleException(msg, e); + return HandleException(msg, e.what()); } return Status::OK(); } @@ -997,7 +1006,7 @@ Status SqliteMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW )); } catch (std::exception &e) { - return HandleException("Encounter exception when update table files to to_index", e); + return HandleException("Encounter exception when update table files to to_index", e.what()); } return Status::OK(); @@ -1038,12 +1047,11 @@ Status SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { }); if (!commited) { - ENGINE_LOG_ERROR << "sqlite transaction failed"; - return Status::DBTransactionError("Update table files error"); + return HandleException("UpdateTableFiles error: sqlite transaction failed"); } } catch (std::exception &e) { - return HandleException("Encounter exception when update table files", e); + return HandleException("Encounter exception when update table files", e.what()); } return Status::OK(); } @@ -1088,12 +1096,11 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { }); if (!commited) { - ENGINE_LOG_ERROR << "sqlite transaction failed"; - return Status::DBTransactionError("Clean files error"); + return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed"); } } catch (std::exception &e) { - return HandleException("Encounter exception when clean table files", e); + return HandleException("Encounter exception when clean table files", e.what()); } //remove to_delete tables @@ -1117,12 +1124,11 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { }); if (!commited) { - ENGINE_LOG_ERROR << "sqlite transaction failed"; - return Status::DBTransactionError("Clean files error"); + return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed"); } } catch (std::exception &e) { - return HandleException("Encounter exception when clean table files", e); + return HandleException("Encounter exception when clean table files", e.what()); } //remove deleted table folder @@ -1139,7 +1145,7 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } } catch (std::exception &e) { - return HandleException("Encounter exception when delete table folder", e); + return HandleException("Encounter exception when delete table folder", e.what()); } return Status::OK(); @@ -1152,8 +1158,12 @@ Status SqliteMetaImpl::CleanUp() { //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here std::lock_guard meta_lock(meta_mutex_); - std::vector file_type = {(int) TableFileSchema::NEW, (int) TableFileSchema::NEW_INDEX, (int) TableFileSchema::NEW_MERGE}; - auto files = ConnectorPtr->select(columns(&TableFileSchema::id_), where(in(&TableFileSchema::file_type_, file_type))); + std::vector file_types = { + (int) TableFileSchema::NEW, + (int) TableFileSchema::NEW_INDEX, + (int) TableFileSchema::NEW_MERGE + }; + auto files = ConnectorPtr->select(columns(&TableFileSchema::id_), where(in(&TableFileSchema::file_type_, file_types))); auto commited = ConnectorPtr->transaction([&]() mutable { for (auto &file : files) { @@ -1164,12 +1174,11 @@ Status SqliteMetaImpl::CleanUp() { }); if (!commited) { - ENGINE_LOG_ERROR << "sqlite transaction failed"; - return Status::DBTransactionError("Clean files error"); + return HandleException("CleanUp error: sqlite transaction failed"); } } catch (std::exception &e) { - return HandleException("Encounter exception when clean table file", e); + return HandleException("Encounter exception when clean table file", e.what()); } return Status::OK(); @@ -1180,9 +1189,13 @@ Status SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) { try { server::MetricCollector metric; - std::vector file_type = {(int) TableFileSchema::RAW, (int) TableFileSchema::TO_INDEX, (int) TableFileSchema::INDEX}; + std::vector file_types = { + (int) TableFileSchema::RAW, + (int) TableFileSchema::TO_INDEX, + (int) TableFileSchema::INDEX + }; auto selected = ConnectorPtr->select(columns(&TableFileSchema::row_count_), - where(in(&TableFileSchema::file_type_, file_type) + where(in(&TableFileSchema::file_type_, file_types) and c(&TableFileSchema::table_id_) == table_id)); TableSchema table_schema; @@ -1199,7 +1212,7 @@ Status SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) { } } catch (std::exception &e) { - return HandleException("Encounter exception when calculate table file size", e); + return HandleException("Encounter exception when calculate table file size", e.what()); } return Status::OK(); } @@ -1211,7 +1224,7 @@ Status SqliteMetaImpl::DropAll() { ConnectorPtr->drop_table("Tables"); ConnectorPtr->drop_table("TableFiles"); } catch (std::exception &e) { - return HandleException("Encounter exception when drop all meta", e); + return HandleException("Encounter exception when drop all meta", e.what()); } return Status::OK(); diff --git a/cpp/src/db/scheduler/TaskDispatchQueue.cpp b/cpp/src/db/scheduler/TaskDispatchQueue.cpp index b728e925a9..e1bf7b75f5 100644 --- a/cpp/src/db/scheduler/TaskDispatchQueue.cpp +++ b/cpp/src/db/scheduler/TaskDispatchQueue.cpp @@ -53,7 +53,7 @@ TaskDispatchQueue::Front() { if (queue_.empty()) { std::string error_msg = "blocking queue empty"; SERVER_LOG_ERROR << error_msg; - throw server::ServerException(server::SERVER_BLOCKING_QUEUE_EMPTY, error_msg); + throw server::ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg); } ScheduleTaskPtr front(queue_.front()); return front; @@ -67,7 +67,7 @@ TaskDispatchQueue::Back() { if (queue_.empty()) { std::string error_msg = "blocking queue empty"; SERVER_LOG_ERROR << error_msg; - throw server::ServerException(server::SERVER_BLOCKING_QUEUE_EMPTY, error_msg); + throw server::ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg); } ScheduleTaskPtr back(queue_.back()); diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index 061189c4f6..fffd582112 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -134,7 +134,7 @@ Status SearchTask::ClusterResult(const std::vector &output_ids, std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) + " distance array size: " + std::to_string(output_distence.size()); ENGINE_LOG_ERROR << msg; - return Status::Error(msg); + return Status(DB_ERROR, msg); } result_set.clear(); @@ -249,7 +249,7 @@ Status SearchTask::TopkResult(SearchContext::ResultSet &result_src, if (result_src.size() != result_target.size()) { std::string msg = "Invalid result set size"; ENGINE_LOG_ERROR << msg; - return Status::Error(msg); + return Status(DB_ERROR, msg); } std::function ReduceWorker = [&](size_t from_index, size_t to_index) { diff --git a/cpp/src/metrics/MetricBase.h b/cpp/src/metrics/MetricBase.h index 3fb947cb10..315eaa0f88 100644 --- a/cpp/src/metrics/MetricBase.h +++ b/cpp/src/metrics/MetricBase.h @@ -21,7 +21,7 @@ class MetricsBase{ return instance; } - virtual ServerError Init() {}; + virtual ErrorCode Init() {}; virtual void AddVectorsSuccessTotalIncrement(double value = 1) {}; virtual void AddVectorsFailTotalIncrement(double value = 1) {}; diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index 5f98acfc84..d12acca87d 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -14,7 +14,7 @@ namespace zilliz { namespace milvus { namespace server { -ServerError + ErrorCode PrometheusMetrics::Init() { try { ConfigNode &configNode = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC); diff --git a/cpp/src/metrics/PrometheusMetrics.h b/cpp/src/metrics/PrometheusMetrics.h index f0fe74cfaa..70443d6331 100644 --- a/cpp/src/metrics/PrometheusMetrics.h +++ b/cpp/src/metrics/PrometheusMetrics.h @@ -39,8 +39,7 @@ class PrometheusMetrics: public MetricsBase { return instance; } - ServerError - Init(); + ErrorCode Init(); private: std::shared_ptr exposer_ptr_; diff --git a/cpp/src/scheduler/task/SearchTask.cpp b/cpp/src/scheduler/task/SearchTask.cpp index 869f22cbdc..a0028856f8 100644 --- a/cpp/src/scheduler/task/SearchTask.cpp +++ b/cpp/src/scheduler/task/SearchTask.cpp @@ -205,7 +205,7 @@ Status XSearchTask::ClusterResult(const std::vector &output_ids, std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) + " distance array size: " + std::to_string(output_distence.size()); ENGINE_LOG_ERROR << msg; - return Status::Error(msg); + return Status(DB_ERROR, msg); } result_set.clear(); @@ -320,7 +320,7 @@ Status XSearchTask::TopkResult(SearchContext::ResultSet &result_src, if (result_src.size() != result_target.size()) { std::string msg = "Invalid result set size"; ENGINE_LOG_ERROR << msg; - return Status::Error(msg); + return Status(DB_ERROR, msg); } std::function ReduceWorker = [&](size_t from_index, size_t to_index) { diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 620da3b830..305c5ba0ef 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -21,7 +21,7 @@ DBWrapper::DBWrapper() { } -ServerError DBWrapper::StartService() { +ErrorCode DBWrapper::StartService() { //db config zilliz::milvus::engine::Options opt; ConfigNode& db_config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); @@ -79,7 +79,7 @@ ServerError DBWrapper::StartService() { opt.meta.archive_conf.SetCriterias(criterial); //create db root folder - ServerError err = CommonUtil::CreateDirectory(opt.meta.path); + ErrorCode err = CommonUtil::CreateDirectory(opt.meta.path); if(err != SERVER_SUCCESS) { std::cout << "ERROR! Failed to create database root path: " << opt.meta.path << std::endl; kill(0, SIGUSR1); @@ -112,7 +112,7 @@ ServerError DBWrapper::StartService() { return SERVER_SUCCESS; } -ServerError DBWrapper::StopService() { +ErrorCode DBWrapper::StopService() { if(db_) { db_->Stop(); } diff --git a/cpp/src/server/DBWrapper.h b/cpp/src/server/DBWrapper.h index 8b25dc0d28..6d0d3f77aa 100644 --- a/cpp/src/server/DBWrapper.h +++ b/cpp/src/server/DBWrapper.h @@ -29,8 +29,8 @@ public: return GetInstance().EngineDB(); } - ServerError StartService(); - ServerError StopService(); + ErrorCode StartService(); + ErrorCode StopService(); std::shared_ptr EngineDB() { return db_; diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 2df4279130..449c2d422d 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -19,6 +19,7 @@ #include #include #include +#include "knowhere/index/vector_index/gpu_ivf.h" #include "metrics/Metrics.h" #include "DBWrapper.h" @@ -209,10 +210,10 @@ Server::Stop() { } -ServerError +ErrorCode Server::LoadConfig() { ServerConfig::GetInstance().LoadConfigFile(config_filename_); - ServerError err = ServerConfig::GetInstance().ValidateConfig(); + ErrorCode err = ServerConfig::GetInstance().ValidateConfig(); if(err != SERVER_SUCCESS){ exit(0); } @@ -232,6 +233,7 @@ Server::StopService() { grpc::GrpcMilvusServer::StopService(); DBWrapper::GetInstance().StopService(); engine::StopSchedulerService(); + knowhere::FaissGpuResourceMgr::GetInstance().Free(); // free gpu resource. } } diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index 58e676f7ff..c54f90a198 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -29,7 +29,7 @@ class Server { void Daemonize(); static void HandleSignal(int signal); - ServerError LoadConfig(); + ErrorCode LoadConfig(); void StartService(); void StopService(); diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/ServerConfig.cpp index 56d11848ee..0be48966e1 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/ServerConfig.cpp @@ -28,7 +28,7 @@ ServerConfig::GetInstance() { return config; } -ServerError +ErrorCode ServerConfig::LoadConfigFile(const std::string& config_filename) { std::string filename = config_filename; if(filename.empty()){ @@ -44,7 +44,7 @@ ServerConfig::LoadConfigFile(const std::string& config_filename) { try { ConfigMgr* mgr = const_cast(ConfigMgr::GetInstance()); - ServerError err = mgr->LoadConfigFile(filename); + ErrorCode err = mgr->LoadConfigFile(filename); if(err != 0) { std::cout << "Server failed to load config file" << std::endl; exit(1);//directly exit program if the config file is illegal @@ -58,7 +58,7 @@ ServerConfig::LoadConfigFile(const std::string& config_filename) { return SERVER_SUCCESS; } -ServerError ServerConfig::ValidateConfig() const { +ErrorCode ServerConfig::ValidateConfig() const { //server config validation ConfigNode server_config = GetConfig(CONFIG_SERVER); uint32_t gpu_index = (uint32_t)server_config.GetInt32Value(CONFIG_GPU_INDEX, 0); diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index 69ee67fa64..99c535dc04 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -68,8 +68,8 @@ class ServerConfig { public: static ServerConfig &GetInstance(); - ServerError LoadConfigFile(const std::string& config_filename); - ServerError ValidateConfig() const; + ErrorCode LoadConfigFile(const std::string& config_filename); + ErrorCode ValidateConfig() const; void PrintAll() const; ConfigNode GetConfig(const std::string& name) const; diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp index 6f1a42b641..391c6bab1f 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp @@ -16,8 +16,8 @@ namespace grpc { using namespace ::milvus; namespace { - const std::map &ErrorMap() { - static const std::map code_map = { + const std::map &ErrorMap() { + static const std::map code_map = { {SERVER_UNEXPECTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, {SERVER_UNSUPPORTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, {SERVER_NULL_POINTER, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, @@ -66,7 +66,7 @@ GrpcBaseTask::~GrpcBaseTask() { WaitToFinish(); } -ServerError GrpcBaseTask::Execute() { +ErrorCode GrpcBaseTask::Execute() { error_code_ = OnExecute(); Done(); return error_code_; @@ -77,7 +77,7 @@ void GrpcBaseTask::Done() { finish_cond_.notify_all(); } -ServerError GrpcBaseTask::SetError(ServerError error_code, const std::string &error_msg) { +ErrorCode GrpcBaseTask::SetError(ErrorCode error_code, const std::string &error_msg) { error_code_ = error_code; error_msg_ = error_msg; @@ -85,7 +85,7 @@ ServerError GrpcBaseTask::SetError(ServerError error_code, const std::string &er return error_code_; } -ServerError GrpcBaseTask::WaitToFinish() { +ErrorCode GrpcBaseTask::WaitToFinish() { std::unique_lock lock(finish_mtx_); finish_cond_.wait(lock, [this] { return done_; }); @@ -112,7 +112,7 @@ void GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Statu if (!task_ptr->IsAsync()) { task_ptr->WaitToFinish(); - ServerError err = task_ptr->ErrorCode(); + ErrorCode err = task_ptr->ErrorID(); if (err != SERVER_SUCCESS) { grpc_status->set_reason(task_ptr->ErrorMsg()); grpc_status->set_error_code(ErrorMap().at(err)); @@ -153,12 +153,12 @@ void GrpcRequestScheduler::Stop() { SERVER_LOG_INFO << "Scheduler stopped"; } -ServerError GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) { +ErrorCode GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) { if (task_ptr == nullptr) { return SERVER_NULL_POINTER; } - ServerError err = PutTaskToQueue(task_ptr); + ErrorCode err = PutTaskToQueue(task_ptr); if (err != SERVER_SUCCESS) { SERVER_LOG_ERROR << "Put task to queue failed with code: " << err; return err; @@ -185,7 +185,7 @@ void GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) { } try { - ServerError err = task->Execute(); + ErrorCode err = task->Execute(); if (err != SERVER_SUCCESS) { SERVER_LOG_ERROR << "Task failed with code: " << err; } @@ -195,7 +195,7 @@ void GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) { } } -ServerError GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) { +ErrorCode GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) { std::lock_guard lock(queue_mtx_); std::string group_name = task_ptr->TaskGroup(); diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.h b/cpp/src/server/grpc_impl/GrpcRequestScheduler.h index 96be98836c..b873904cec 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.h +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.h @@ -25,24 +25,24 @@ protected: virtual ~GrpcBaseTask(); public: - ServerError Execute(); + ErrorCode Execute(); void Done(); - ServerError WaitToFinish(); + ErrorCode WaitToFinish(); std::string TaskGroup() const { return task_group_; } - ServerError ErrorCode() const { return error_code_; } + ErrorCode ErrorID() const { return error_code_; } std::string ErrorMsg() const { return error_msg_; } bool IsAsync() const { return async_; } protected: - virtual ServerError OnExecute() = 0; + virtual ErrorCode OnExecute() = 0; - ServerError SetError(ServerError error_code, const std::string &msg); + ErrorCode SetError(ErrorCode error_code, const std::string &msg); protected: mutable std::mutex finish_mtx_; @@ -51,7 +51,7 @@ protected: std::string task_group_; bool async_; bool done_; - ServerError error_code_; + ErrorCode error_code_; std::string error_msg_; }; @@ -71,7 +71,7 @@ public: void Stop(); - ServerError ExecuteTask(const BaseTaskPtr &task_ptr); + ErrorCode ExecuteTask(const BaseTaskPtr &task_ptr); static void ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Status *grpc_status); @@ -82,7 +82,7 @@ protected: void TakeTaskToExecute(TaskQueuePtr task_queue); - ServerError PutTaskToQueue(const BaseTaskPtr &task_ptr); + ErrorCode PutTaskToQueue(const BaseTaskPtr &task_ptr); private: mutable std::mutex queue_mtx_; diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index e0529c1999..a9f5bf41b7 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -67,7 +67,7 @@ namespace { void ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range> &range_array, std::vector &dates, - ServerError &error_code, + ErrorCode &error_code, std::string &error_msg) { dates.clear(); for (auto &range : range_array) { @@ -123,13 +123,13 @@ CreateTableTask::Create(const ::milvus::grpc::TableSchema *schema) { return std::shared_ptr(new CreateTableTask(schema)); } -ServerError +ErrorCode CreateTableTask::OnExecute() { TimeRecorder rc("CreateTableTask"); try { //step 1: check arguments - ServerError res = ValidationUtil::ValidateTableName(schema_->table_name().table_name()); + ErrorCode res = ValidationUtil::ValidateTableName(schema_->table_name().table_name()); if (res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + schema_->table_name().table_name()); } @@ -160,7 +160,7 @@ CreateTableTask::OnExecute() { engine::Status stat = DBWrapper::DB()->CreateTable(table_info); if (!stat.ok()) { //table could exist - if(stat.IsAlreadyExist()) { + if(stat.code() == DB_ALREADY_EXIST) { return SetError(SERVER_INVALID_TABLE_NAME, stat.ToString()); } return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); @@ -187,13 +187,13 @@ DescribeTableTask::Create(const std::string &table_name, ::milvus::grpc::TableSc return std::shared_ptr(new DescribeTableTask(table_name, schema)); } -ServerError +ErrorCode DescribeTableTask::OnExecute() { TimeRecorder rc("DescribeTableTask"); try { //step 1: check arguments - ServerError res = ValidationUtil::ValidateTableName(table_name_); + ErrorCode res = ValidationUtil::ValidateTableName(table_name_); if (res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -235,14 +235,14 @@ CreateIndexTask::Create(const ::milvus::grpc::IndexParam *index_param) { return std::shared_ptr(new CreateIndexTask(index_param)); } -ServerError +ErrorCode CreateIndexTask::OnExecute() { try { TimeRecorder rc("CreateIndexTask"); //step 1: check arguments std::string table_name_ = index_param_->table_name().table_name(); - ServerError res = ValidationUtil::ValidateTableName(table_name_); + ErrorCode res = ValidationUtil::ValidateTableName(table_name_); if (res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -298,13 +298,13 @@ HasTableTask::Create(const std::string &table_name, bool &has_table) { return std::shared_ptr(new HasTableTask(table_name, has_table)); } -ServerError +ErrorCode HasTableTask::OnExecute() { try { TimeRecorder rc("HasTableTask"); //step 1: check arguments - ServerError res = ValidationUtil::ValidateTableName(table_name_); + ErrorCode res = ValidationUtil::ValidateTableName(table_name_); if (res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -335,13 +335,13 @@ DropTableTask::Create(const std::string &table_name) { return std::shared_ptr(new DropTableTask(table_name)); } -ServerError +ErrorCode DropTableTask::OnExecute() { try { TimeRecorder rc("DropTableTask"); //step 1: check arguments - ServerError res = ValidationUtil::ValidateTableName(table_name_); + ErrorCode res = ValidationUtil::ValidateTableName(table_name_); if (res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -351,7 +351,7 @@ DropTableTask::OnExecute() { table_info.table_id_ = table_name_; engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if (!stat.ok()) { - if (stat.IsNotFound()) { + if (stat.code() == DB_NOT_FOUND) { return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); } else { return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); @@ -387,7 +387,7 @@ ShowTablesTask::Create(::grpc::ServerWriter<::milvus::grpc::TableName> *writer) return std::shared_ptr(new ShowTablesTask(writer)); } -ServerError +ErrorCode ShowTablesTask::OnExecute() { std::vector schema_array; engine::Status stat = DBWrapper::DB()->AllTables(schema_array); @@ -424,13 +424,13 @@ InsertTask::Create(const ::milvus::grpc::InsertParam *insert_param, return std::shared_ptr(new InsertTask(insert_param, record_ids)); } -ServerError +ErrorCode InsertTask::OnExecute() { try { TimeRecorder rc("InsertVectorTask"); //step 1: check arguments - ServerError res = ValidationUtil::ValidateTableName(insert_param_->table_name()); + ErrorCode res = ValidationUtil::ValidateTableName(insert_param_->table_name()); if (res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + insert_param_->table_name()); } @@ -450,7 +450,7 @@ InsertTask::OnExecute() { table_info.table_id_ = insert_param_->table_name(); engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if (!stat.ok()) { - if (stat.IsNotFound()) { + if (stat.code() == DB_NOT_FOUND) { return SetError(SERVER_TABLE_NOT_EXIST, "Table " + insert_param_->table_name() + " not exists"); } else { @@ -489,7 +489,7 @@ InsertTask::OnExecute() { } uint64_t vec_dim = insert_param_->row_record_array(i).vector_data().size(); if (vec_dim != table_info.dimension_) { - ServerError error_code = SERVER_INVALID_VECTOR_DIMENSION; + ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; std::string error_msg = "Invalid rowrecord dimension: " + std::to_string(vec_dim) + " vs. table dimension:" + std::to_string(table_info.dimension_); @@ -569,14 +569,14 @@ SearchTask::Create(const ::milvus::grpc::SearchParam *search_vector_infos, response)); } -ServerError +ErrorCode SearchTask::OnExecute() { try { TimeRecorder rc("SearchTask"); //step 1: check table name std::string table_name_ = search_param_->table_name(); - ServerError res = ValidationUtil::ValidateTableName(table_name_); + ErrorCode res = ValidationUtil::ValidateTableName(table_name_); if (res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -586,7 +586,7 @@ SearchTask::OnExecute() { table_info.table_id_ = table_name_; engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if (!stat.ok()) { - if (stat.IsNotFound()) { + if (stat.code() == DB_NOT_FOUND) { return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); } else { return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); @@ -612,7 +612,7 @@ SearchTask::OnExecute() { //step 4: check date range, and convert to db dates std::vector dates; - ServerError error_code = SERVER_SUCCESS; + ErrorCode error_code = SERVER_SUCCESS; std::string error_msg; std::vector<::milvus::grpc::Range> range_array; @@ -642,7 +642,7 @@ SearchTask::OnExecute() { } uint64_t query_vec_dim = search_param_->query_record_array(i).vector_data().size(); if (query_vec_dim != table_info.dimension_) { - ServerError error_code = SERVER_INVALID_VECTOR_DIMENSION; + ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; std::string error_msg = "Invalid rowrecord dimension: " + std::to_string(query_vec_dim) + " vs. table dimension:" + std::to_string(table_info.dimension_); return SetError(error_code, error_msg); @@ -722,13 +722,13 @@ CountTableTask::Create(const std::string &table_name, int64_t &row_count) { return std::shared_ptr(new CountTableTask(table_name, row_count)); } -ServerError +ErrorCode CountTableTask::OnExecute() { try { TimeRecorder rc("GetTableRowCountTask"); //step 1: check arguments - ServerError res = SERVER_SUCCESS; + ErrorCode res = SERVER_SUCCESS; res = ValidationUtil::ValidateTableName(table_name_); if (res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); @@ -765,7 +765,7 @@ CmdTask::Create(const std::string &cmd, std::string &result) { return std::shared_ptr(new CmdTask(cmd, result)); } -ServerError +ErrorCode CmdTask::OnExecute() { if (cmd_ == "version") { result_ = MILVUS_VERSION; @@ -794,14 +794,14 @@ DeleteByRangeTask::Create(const ::milvus::grpc::DeleteByRangeParam *delete_by_ra return std::shared_ptr(new DeleteByRangeTask(delete_by_range_param)); } -ServerError +ErrorCode DeleteByRangeTask::OnExecute() { try { TimeRecorder rc("DeleteByRangeTask"); //step 1: check arguments std::string table_name = delete_by_range_param_->table_name(); - ServerError res = ValidationUtil::ValidateTableName(table_name); + ErrorCode res = ValidationUtil::ValidateTableName(table_name); if (res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name); } @@ -811,7 +811,7 @@ DeleteByRangeTask::OnExecute() { table_info.table_id_ = table_name; engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if (!stat.ok()) { - if (stat.IsNotFound()) { + if (stat.code(), DB_NOT_FOUND) { return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name + " not exists"); } else { return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); @@ -822,7 +822,7 @@ DeleteByRangeTask::OnExecute() { //step 3: check date range, and convert to db dates std::vector dates; - ServerError error_code = SERVER_SUCCESS; + ErrorCode error_code = SERVER_SUCCESS; std::string error_msg; std::vector<::milvus::grpc::Range> range_array; @@ -862,13 +862,13 @@ PreloadTableTask::Create(const std::string &table_name){ return std::shared_ptr(new PreloadTableTask(table_name)); } -ServerError +ErrorCode PreloadTableTask::OnExecute() { try { TimeRecorder rc("PreloadTableTask"); //step 1: check arguments - ServerError res = ValidationUtil::ValidateTableName(table_name_); + ErrorCode res = ValidationUtil::ValidateTableName(table_name_); if (res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -902,13 +902,13 @@ DescribeIndexTask::Create(const std::string &table_name, return std::shared_ptr(new DescribeIndexTask(table_name, index_param)); } -ServerError +ErrorCode DescribeIndexTask::OnExecute() { try { TimeRecorder rc("DescribeIndexTask"); //step 1: check arguments - ServerError res = ValidationUtil::ValidateTableName(table_name_); + ErrorCode res = ValidationUtil::ValidateTableName(table_name_); if (res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -944,26 +944,19 @@ DropIndexTask::Create(const std::string &table_name){ return std::shared_ptr(new DropIndexTask(table_name)); } -ServerError +ErrorCode DropIndexTask::OnExecute() { try { TimeRecorder rc("DropIndexTask"); //step 1: check arguments - ServerError res = ValidationUtil::ValidateTableName(table_name_); + ErrorCode res = ValidationUtil::ValidateTableName(table_name_); if (res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } - //step 2:check index existence - engine::TableIndex index; - engine::Status stat = DBWrapper::DB()->DescribeIndex(table_name_, index); - if (index.engine_type_ == 1) { - return SetError(SERVER_UNEXPECTED_ERROR, "index not existed"); - } - - //step 3: check table existence - stat = DBWrapper::DB()->DropIndex(table_name_); + //step 2: check table existence + auto stat = DBWrapper::DB()->DropIndex(table_name_); if (!stat.ok()) { return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.h b/cpp/src/server/grpc_impl/GrpcRequestTask.h index c91847a49c..ab43bdef70 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.h +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.h @@ -29,7 +29,7 @@ protected: explicit CreateTableTask(const ::milvus::grpc::TableSchema *request); - ServerError + ErrorCode OnExecute() override; private: @@ -45,7 +45,7 @@ public: protected: HasTableTask(const std::string &request, bool &has_table); - ServerError + ErrorCode OnExecute() override; @@ -63,7 +63,7 @@ public: protected: DescribeTableTask(const std::string &table_name, ::milvus::grpc::TableSchema *schema); - ServerError + ErrorCode OnExecute() override; @@ -82,7 +82,7 @@ protected: explicit DropTableTask(const std::string &table_name); - ServerError + ErrorCode OnExecute() override; @@ -100,7 +100,7 @@ protected: explicit CreateIndexTask(const ::milvus::grpc::IndexParam *index_Param); - ServerError + ErrorCode OnExecute() override; @@ -118,7 +118,7 @@ protected: explicit ShowTablesTask(::grpc::ServerWriter<::milvus::grpc::TableName> *writer); - ServerError + ErrorCode OnExecute() override; private: @@ -136,7 +136,7 @@ protected: InsertTask(const ::milvus::grpc::InsertParam *insert_Param, ::milvus::grpc::VectorIds *record_ids_); - ServerError + ErrorCode OnExecute() override; private: @@ -157,7 +157,7 @@ protected: const std::vector &file_id_array, ::milvus::grpc::TopKQueryResultList *response); - ServerError + ErrorCode OnExecute() override; private: @@ -175,7 +175,7 @@ public: protected: CountTableTask(const std::string &table_name, int64_t &row_count); - ServerError + ErrorCode OnExecute() override; private: @@ -192,7 +192,7 @@ public: protected: CmdTask(const std::string &cmd, std::string &result); - ServerError + ErrorCode OnExecute() override; private: @@ -209,7 +209,7 @@ public: protected: DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param); - ServerError + ErrorCode OnExecute() override; private: @@ -225,7 +225,7 @@ public: protected: PreloadTableTask(const std::string &table_name); - ServerError + ErrorCode OnExecute() override; private: @@ -243,7 +243,7 @@ protected: DescribeIndexTask(const std::string &table_name, ::milvus::grpc::IndexParam *index_param); - ServerError + ErrorCode OnExecute() override; private: @@ -260,7 +260,7 @@ public: protected: DropIndexTask(const std::string &table_name); - ServerError + ErrorCode OnExecute() override; private: diff --git a/cpp/src/utils/CommonUtil.cpp b/cpp/src/utils/CommonUtil.cpp index a376ee536f..c8fdb62e15 100644 --- a/cpp/src/utils/CommonUtil.cpp +++ b/cpp/src/utils/CommonUtil.cpp @@ -61,7 +61,7 @@ bool CommonUtil::IsDirectoryExist(const std::string &path) { return true; } -ServerError CommonUtil::CreateDirectory(const std::string &path) { +ErrorCode CommonUtil::CreateDirectory(const std::string &path) { if(path.empty()) { return SERVER_SUCCESS; } @@ -74,7 +74,7 @@ ServerError CommonUtil::CreateDirectory(const std::string &path) { fs::path fs_path(path); fs::path parent_path = fs_path.parent_path(); - ServerError err = CreateDirectory(parent_path.string()); + ErrorCode err = CreateDirectory(parent_path.string()); if(err != SERVER_SUCCESS){ return err; } @@ -122,7 +122,7 @@ namespace { } } -ServerError CommonUtil::DeleteDirectory(const std::string &path) { +ErrorCode CommonUtil::DeleteDirectory(const std::string &path) { if(path.empty()) { return SERVER_SUCCESS; } diff --git a/cpp/src/utils/CommonUtil.h b/cpp/src/utils/CommonUtil.h index 095b237584..99122142ad 100755 --- a/cpp/src/utils/CommonUtil.h +++ b/cpp/src/utils/CommonUtil.h @@ -22,8 +22,8 @@ class CommonUtil { static bool IsFileExist(const std::string &path); static uint64_t GetFileSize(const std::string &path); static bool IsDirectoryExist(const std::string &path); - static ServerError CreateDirectory(const std::string &path); - static ServerError DeleteDirectory(const std::string &path); + static ErrorCode CreateDirectory(const std::string &path); + static ErrorCode DeleteDirectory(const std::string &path); static std::string GetExePath(); diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index e62e4d3271..6142618128 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -11,82 +11,95 @@ namespace zilliz { namespace milvus { -namespace server { -using ServerError = int32_t; +using ErrorCode = int32_t; -constexpr ServerError SERVER_SUCCESS = 0; - -constexpr ServerError SERVER_ERROR_CODE_BASE = 0x30000; - -constexpr ServerError -ToGlobalServerErrorCode(const ServerError error_code) { +constexpr ErrorCode SERVER_SUCCESS = 0; +constexpr ErrorCode SERVER_ERROR_CODE_BASE = 0x30000; +constexpr ErrorCode +ToServerErrorCode(const ErrorCode error_code) { return SERVER_ERROR_CODE_BASE + error_code; } -constexpr ServerError SERVER_UNEXPECTED_ERROR = ToGlobalServerErrorCode(1); -constexpr ServerError SERVER_UNSUPPORTED_ERROR = ToGlobalServerErrorCode(2); -constexpr ServerError SERVER_NULL_POINTER = ToGlobalServerErrorCode(3); -constexpr ServerError SERVER_INVALID_ARGUMENT = ToGlobalServerErrorCode(4); -constexpr ServerError SERVER_FILE_NOT_FOUND = ToGlobalServerErrorCode(5); -constexpr ServerError SERVER_NOT_IMPLEMENT = ToGlobalServerErrorCode(6); -constexpr ServerError SERVER_BLOCKING_QUEUE_EMPTY = ToGlobalServerErrorCode(7); -constexpr ServerError SERVER_CANNOT_CREATE_FOLDER = ToGlobalServerErrorCode(8); -constexpr ServerError SERVER_CANNOT_CREATE_FILE = ToGlobalServerErrorCode(9); -constexpr ServerError SERVER_CANNOT_DELETE_FOLDER = ToGlobalServerErrorCode(10); -constexpr ServerError SERVER_CANNOT_DELETE_FILE = ToGlobalServerErrorCode(11); -constexpr ServerError SERVER_BUILD_INDEX_ERROR = ToGlobalServerErrorCode(12); +constexpr ErrorCode DB_SUCCESS = 0; +constexpr ErrorCode DB_ERROR_CODE_BASE = 0x40000; +constexpr ErrorCode +ToDbErrorCode(const ErrorCode error_code) { + return DB_ERROR_CODE_BASE + error_code; +} -constexpr ServerError SERVER_TABLE_NOT_EXIST = ToGlobalServerErrorCode(100); -constexpr ServerError SERVER_INVALID_TABLE_NAME = ToGlobalServerErrorCode(101); -constexpr ServerError SERVER_INVALID_TABLE_DIMENSION = ToGlobalServerErrorCode(102); -constexpr ServerError SERVER_INVALID_TIME_RANGE = ToGlobalServerErrorCode(103); -constexpr ServerError SERVER_INVALID_VECTOR_DIMENSION = ToGlobalServerErrorCode(104); -constexpr ServerError SERVER_INVALID_INDEX_TYPE = ToGlobalServerErrorCode(105); -constexpr ServerError SERVER_INVALID_ROWRECORD = ToGlobalServerErrorCode(106); -constexpr ServerError SERVER_INVALID_ROWRECORD_ARRAY = ToGlobalServerErrorCode(107); -constexpr ServerError SERVER_INVALID_TOPK = ToGlobalServerErrorCode(108); -constexpr ServerError SERVER_ILLEGAL_VECTOR_ID = ToGlobalServerErrorCode(109); -constexpr ServerError SERVER_ILLEGAL_SEARCH_RESULT = ToGlobalServerErrorCode(110); -constexpr ServerError SERVER_CACHE_ERROR = ToGlobalServerErrorCode(111); -constexpr ServerError SERVER_WRITE_ERROR = ToGlobalServerErrorCode(112); -constexpr ServerError SERVER_INVALID_NPROBE = ToGlobalServerErrorCode(113); -constexpr ServerError SERVER_INVALID_INDEX_NLIST = ToGlobalServerErrorCode(114); -constexpr ServerError SERVER_INVALID_INDEX_METRIC_TYPE = ToGlobalServerErrorCode(115); -constexpr ServerError SERVER_INVALID_INDEX_FILE_SIZE = ToGlobalServerErrorCode(116); +constexpr ErrorCode KNOWHERE_SUCCESS = 0; +constexpr ErrorCode KNOWHERE_ERROR_CODE_BASE = 0x50000; +constexpr ErrorCode +ToKnowhereErrorCode(const ErrorCode error_code) { + return KNOWHERE_ERROR_CODE_BASE + error_code; +} +//server error code +constexpr ErrorCode SERVER_UNEXPECTED_ERROR = ToServerErrorCode(1); +constexpr ErrorCode SERVER_UNSUPPORTED_ERROR = ToServerErrorCode(2); +constexpr ErrorCode SERVER_NULL_POINTER = ToServerErrorCode(3); +constexpr ErrorCode SERVER_INVALID_ARGUMENT = ToServerErrorCode(4); +constexpr ErrorCode SERVER_FILE_NOT_FOUND = ToServerErrorCode(5); +constexpr ErrorCode SERVER_NOT_IMPLEMENT = ToServerErrorCode(6); +constexpr ErrorCode SERVER_BLOCKING_QUEUE_EMPTY = ToServerErrorCode(7); +constexpr ErrorCode SERVER_CANNOT_CREATE_FOLDER = ToServerErrorCode(8); +constexpr ErrorCode SERVER_CANNOT_CREATE_FILE = ToServerErrorCode(9); +constexpr ErrorCode SERVER_CANNOT_DELETE_FOLDER = ToServerErrorCode(10); +constexpr ErrorCode SERVER_CANNOT_DELETE_FILE = ToServerErrorCode(11); +constexpr ErrorCode SERVER_BUILD_INDEX_ERROR = ToServerErrorCode(12); -constexpr ServerError SERVER_LICENSE_FILE_NOT_EXIST = ToGlobalServerErrorCode(500); -constexpr ServerError SERVER_LICENSE_VALIDATION_FAIL = ToGlobalServerErrorCode(501); +constexpr ErrorCode SERVER_TABLE_NOT_EXIST = ToServerErrorCode(100); +constexpr ErrorCode SERVER_INVALID_TABLE_NAME = ToServerErrorCode(101); +constexpr ErrorCode SERVER_INVALID_TABLE_DIMENSION = ToServerErrorCode(102); +constexpr ErrorCode SERVER_INVALID_TIME_RANGE = ToServerErrorCode(103); +constexpr ErrorCode SERVER_INVALID_VECTOR_DIMENSION = ToServerErrorCode(104); +constexpr ErrorCode SERVER_INVALID_INDEX_TYPE = ToServerErrorCode(105); +constexpr ErrorCode SERVER_INVALID_ROWRECORD = ToServerErrorCode(106); +constexpr ErrorCode SERVER_INVALID_ROWRECORD_ARRAY = ToServerErrorCode(107); +constexpr ErrorCode SERVER_INVALID_TOPK = ToServerErrorCode(108); +constexpr ErrorCode SERVER_ILLEGAL_VECTOR_ID = ToServerErrorCode(109); +constexpr ErrorCode SERVER_ILLEGAL_SEARCH_RESULT = ToServerErrorCode(110); +constexpr ErrorCode SERVER_CACHE_ERROR = ToServerErrorCode(111); +constexpr ErrorCode SERVER_WRITE_ERROR = ToServerErrorCode(112); +constexpr ErrorCode SERVER_INVALID_NPROBE = ToServerErrorCode(113); +constexpr ErrorCode SERVER_INVALID_INDEX_NLIST = ToServerErrorCode(114); +constexpr ErrorCode SERVER_INVALID_INDEX_METRIC_TYPE = ToServerErrorCode(115); +constexpr ErrorCode SERVER_INVALID_INDEX_FILE_SIZE = ToServerErrorCode(116); -constexpr ServerError DB_META_TRANSACTION_FAILED = ToGlobalServerErrorCode(1000); +//db error code +constexpr ErrorCode DB_META_TRANSACTION_FAILED = ToDbErrorCode(1); +constexpr ErrorCode DB_ERROR = ToDbErrorCode(2); +constexpr ErrorCode DB_NOT_FOUND = ToDbErrorCode(3); +constexpr ErrorCode DB_ALREADY_EXIST = ToDbErrorCode(4); +constexpr ErrorCode DB_INVALID_PATH = ToDbErrorCode(5); -using KnowhereError = int32_t; -constexpr KnowhereError KNOWHERE_SUCCESS = 0; -constexpr KnowhereError KNOWHERE_ERROR = ToGlobalServerErrorCode(1); -constexpr KnowhereError KNOWHERE_INVALID_ARGUMENT = ToGlobalServerErrorCode(2); -constexpr KnowhereError KNOWHERE_UNEXPECTED_ERROR = ToGlobalServerErrorCode(3); +//knowhere error code +constexpr ErrorCode KNOWHERE_ERROR = ToKnowhereErrorCode(1); +constexpr ErrorCode KNOWHERE_INVALID_ARGUMENT = ToKnowhereErrorCode(2); +constexpr ErrorCode KNOWHERE_UNEXPECTED_ERROR = ToKnowhereErrorCode(3); -class ServerException : public std::exception { -public: - ServerException(ServerError error_code, - const std::string &message = std::string()) +namespace server { + class ServerException : public std::exception { + public: + ServerException(ErrorCode error_code, + const std::string &message = std::string()) : error_code_(error_code), message_(message) {} -public: - ServerError error_code() const { - return error_code_; - } + public: + ErrorCode error_code() const { + return error_code_; + } - virtual const char *what() const noexcept { - return message_.c_str(); - } + virtual const char *what() const noexcept { + return message_.c_str(); + } -private: - ServerError error_code_; - std::string message_; -}; + private: + ErrorCode error_code_; + std::string message_; + }; +} -} // namespace server } // namespace milvus } // namespace zilliz diff --git a/cpp/src/utils/StringHelpFunctions.cpp b/cpp/src/utils/StringHelpFunctions.cpp index 03c689d82c..d45f515761 100644 --- a/cpp/src/utils/StringHelpFunctions.cpp +++ b/cpp/src/utils/StringHelpFunctions.cpp @@ -24,9 +24,9 @@ void StringHelpFunctions::TrimStringQuote(std::string &string, const std::string } } -ServerError StringHelpFunctions::SplitStringByDelimeter(const std::string &str, - const std::string &delimeter, - std::vector &result) { +ErrorCode StringHelpFunctions::SplitStringByDelimeter(const std::string &str, + const std::string &delimeter, + std::vector &result) { if(str.empty()) { return SERVER_SUCCESS; } @@ -46,10 +46,10 @@ ServerError StringHelpFunctions::SplitStringByDelimeter(const std::string &str, return SERVER_SUCCESS; } -ServerError StringHelpFunctions::SplitStringByQuote(const std::string &str, - const std::string &delimeter, - const std::string "e, - std::vector &result) { + ErrorCode StringHelpFunctions::SplitStringByQuote(const std::string &str, + const std::string &delimeter, + const std::string "e, + std::vector &result) { if (quote.empty()) { return SplitStringByDelimeter(str, delimeter, result); } diff --git a/cpp/src/utils/StringHelpFunctions.h b/cpp/src/utils/StringHelpFunctions.h index b5d8ce22b7..f9101ab1e2 100644 --- a/cpp/src/utils/StringHelpFunctions.h +++ b/cpp/src/utils/StringHelpFunctions.h @@ -29,9 +29,9 @@ public: // ,b, | b | // ,, | | // a a - static ServerError SplitStringByDelimeter(const std::string &str, - const std::string &delimeter, - std::vector &result); + static ErrorCode SplitStringByDelimeter(const std::string &str, + const std::string &delimeter, + std::vector &result); //assume the table has two columns, quote='\"', delimeter=',' // a,b a | b @@ -40,10 +40,10 @@ public: // "aa,bb" aa,bb // 55,1122\"aa,bb\",yyy,\"kkk\" 55 | 1122aa,bb | yyy | kkk // "55,1122"aa,bb",yyy,"kkk" illegal - static ServerError SplitStringByQuote(const std::string &str, - const std::string &delimeter, - const std::string "e, - std::vector &result); + static ErrorCode SplitStringByQuote(const std::string &str, + const std::string &delimeter, + const std::string "e, + std::vector &result); }; diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index 2245496903..f12d68b364 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -12,7 +12,7 @@ constexpr size_t table_name_size_limit = 255; constexpr int64_t table_dimension_limit = 16384; constexpr int32_t index_file_size_limit = 4096; //index trigger size max = 4096 MB -ServerError +ErrorCode ValidationUtil::ValidateTableName(const std::string &table_name) { // Table name shouldn't be empty. @@ -46,7 +46,7 @@ ValidationUtil::ValidateTableName(const std::string &table_name) { return SERVER_SUCCESS; } -ServerError +ErrorCode ValidationUtil::ValidateTableDimension(int64_t dimension) { if (dimension <= 0 || dimension > table_dimension_limit) { SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << table_dimension_limit; @@ -56,7 +56,7 @@ ValidationUtil::ValidateTableDimension(int64_t dimension) { } } -ServerError +ErrorCode ValidationUtil::ValidateTableIndexType(int32_t index_type) { int engine_type = (int)engine::EngineType(index_type); if(engine_type <= 0 || engine_type > (int)engine::EngineType::MAX_VALUE) { @@ -66,7 +66,7 @@ ValidationUtil::ValidateTableIndexType(int32_t index_type) { return SERVER_SUCCESS; } -ServerError +ErrorCode ValidationUtil::ValidateTableIndexNlist(int32_t nlist) { if(nlist <= 0) { return SERVER_INVALID_INDEX_NLIST; @@ -75,7 +75,7 @@ ValidationUtil::ValidateTableIndexNlist(int32_t nlist) { return SERVER_SUCCESS; } -ServerError +ErrorCode ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) { if(index_file_size <= 0 || index_file_size > index_file_size_limit) { return SERVER_INVALID_INDEX_FILE_SIZE; @@ -84,7 +84,7 @@ ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) { return SERVER_SUCCESS; } -ServerError +ErrorCode ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { if(metric_type != (int32_t)engine::MetricType::L2 && metric_type != (int32_t)engine::MetricType::IP) { return SERVER_INVALID_INDEX_METRIC_TYPE; @@ -92,7 +92,7 @@ ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { return SERVER_SUCCESS; } -ServerError +ErrorCode ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) { if (top_k <= 0 || top_k > 1024) { return SERVER_INVALID_TOPK; @@ -101,7 +101,7 @@ ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchem return SERVER_SUCCESS; } -ServerError +ErrorCode ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema) { if (nprobe <= 0 || nprobe > table_schema.nlist_) { return SERVER_INVALID_NPROBE; @@ -110,7 +110,7 @@ ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSc return SERVER_SUCCESS; } -ServerError +ErrorCode ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { int num_devices = 0; auto cuda_err = cudaGetDeviceCount(&num_devices); @@ -126,7 +126,7 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { return SERVER_SUCCESS; } -ServerError +ErrorCode ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) { cudaDeviceProp deviceProp; auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index); diff --git a/cpp/src/utils/ValidationUtil.h b/cpp/src/utils/ValidationUtil.h index d1d84f7104..78581d49c4 100644 --- a/cpp/src/utils/ValidationUtil.h +++ b/cpp/src/utils/ValidationUtil.h @@ -9,34 +9,34 @@ namespace server { class ValidationUtil { public: - static ServerError + static ErrorCode ValidateTableName(const std::string &table_name); - static ServerError + static ErrorCode ValidateTableDimension(int64_t dimension); - static ServerError + static ErrorCode ValidateTableIndexType(int32_t index_type); - static ServerError + static ErrorCode ValidateTableIndexNlist(int32_t nlist); - static ServerError + static ErrorCode ValidateTableIndexFileSize(int64_t index_file_size); - static ServerError + static ErrorCode ValidateTableIndexMetricType(int32_t metric_type); - static ServerError + static ErrorCode ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema); - static ServerError + static ErrorCode ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema); - static ServerError + static ErrorCode ValidateGpuIndex(uint32_t gpu_index); - static ServerError + static ErrorCode GetGpuMemory(uint32_t gpu_index, size_t &memory); }; diff --git a/cpp/src/wrapper/knowhere/vec_impl.cpp b/cpp/src/wrapper/knowhere/vec_impl.cpp index 0989178783..685cc8f84a 100644 --- a/cpp/src/wrapper/knowhere/vec_impl.cpp +++ b/cpp/src/wrapper/knowhere/vec_impl.cpp @@ -21,7 +21,7 @@ namespace engine { using namespace zilliz::knowhere; -server::KnowhereError VecIndexImpl::BuildAll(const long &nb, +ErrorCode VecIndexImpl::BuildAll(const long &nb, const float *xb, const long *ids, const Config &cfg, @@ -38,42 +38,45 @@ server::KnowhereError VecIndexImpl::BuildAll(const long &nb, index_->Add(dataset, cfg); } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_UNEXPECTED_ERROR; + return KNOWHERE_UNEXPECTED_ERROR; } catch (jsoncons::json_exception &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_INVALID_ARGUMENT; + return KNOWHERE_INVALID_ARGUMENT; } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_ERROR; + return KNOWHERE_ERROR; } - return server::KNOWHERE_SUCCESS; + return KNOWHERE_SUCCESS; } -server::KnowhereError VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const Config &cfg) { +ErrorCode VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const Config &cfg) { try { auto dataset = GenDatasetWithIds(nb, dim, xb, ids); index_->Add(dataset, cfg); } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_UNEXPECTED_ERROR; + return KNOWHERE_UNEXPECTED_ERROR; } catch (jsoncons::json_exception &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_INVALID_ARGUMENT; + return KNOWHERE_INVALID_ARGUMENT; } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_ERROR; + return KNOWHERE_ERROR; } - return server::KNOWHERE_SUCCESS; + return KNOWHERE_SUCCESS; } -server::KnowhereError VecIndexImpl::Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) { +ErrorCode VecIndexImpl::Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) { try { auto k = cfg["k"].as(); auto dataset = GenDataset(nq, dim, xq); - Config search_cfg; - auto res = index_->Search(dataset, cfg); + Config search_cfg = cfg; + + ParameterValidation(type, search_cfg); + + auto res = index_->Search(dataset, search_cfg); auto ids_array = res->array()[0]; auto dis_array = res->array()[1]; @@ -102,15 +105,15 @@ server::KnowhereError VecIndexImpl::Search(const long &nq, const float *xq, floa memcpy(dist, p_dist, sizeof(float) * nq * k); } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_UNEXPECTED_ERROR; + return KNOWHERE_UNEXPECTED_ERROR; } catch (jsoncons::json_exception &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_INVALID_ARGUMENT; + return KNOWHERE_INVALID_ARGUMENT; } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_ERROR; + return KNOWHERE_ERROR; } - return server::KNOWHERE_SUCCESS; + return KNOWHERE_SUCCESS; } zilliz::knowhere::BinarySet VecIndexImpl::Serialize() { @@ -118,10 +121,10 @@ zilliz::knowhere::BinarySet VecIndexImpl::Serialize() { return index_->Serialize(); } -server::KnowhereError VecIndexImpl::Load(const zilliz::knowhere::BinarySet &index_binary) { +ErrorCode VecIndexImpl::Load(const zilliz::knowhere::BinarySet &index_binary) { index_->Load(index_binary); dim = Dimension(); - return server::KNOWHERE_SUCCESS; + return KNOWHERE_SUCCESS; } int64_t VecIndexImpl::Dimension() { @@ -177,24 +180,24 @@ int64_t *BFIndex::GetRawIds() { return std::static_pointer_cast(index_)->GetRawIds(); } -server::KnowhereError BFIndex::Build(const Config &cfg) { +ErrorCode BFIndex::Build(const Config &cfg) { try { dim = cfg["dim"].as(); std::static_pointer_cast(index_)->Train(cfg); } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_UNEXPECTED_ERROR; + return KNOWHERE_UNEXPECTED_ERROR; } catch (jsoncons::json_exception &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_INVALID_ARGUMENT; + return KNOWHERE_INVALID_ARGUMENT; } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_ERROR; + return KNOWHERE_ERROR; } - return server::KNOWHERE_SUCCESS; + return KNOWHERE_SUCCESS; } -server::KnowhereError BFIndex::BuildAll(const long &nb, +ErrorCode BFIndex::BuildAll(const long &nb, const float *xb, const long *ids, const Config &cfg, @@ -208,19 +211,19 @@ server::KnowhereError BFIndex::BuildAll(const long &nb, index_->Add(dataset, cfg); } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_UNEXPECTED_ERROR; + return KNOWHERE_UNEXPECTED_ERROR; } catch (jsoncons::json_exception &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_INVALID_ARGUMENT; + return KNOWHERE_INVALID_ARGUMENT; } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_ERROR; + return KNOWHERE_ERROR; } - return server::KNOWHERE_SUCCESS; + return KNOWHERE_SUCCESS; } // TODO(linxj): add lock here. -server::KnowhereError IVFMixIndex::BuildAll(const long &nb, +ErrorCode IVFMixIndex::BuildAll(const long &nb, const float *xb, const long *ids, const Config &cfg, @@ -242,26 +245,26 @@ server::KnowhereError IVFMixIndex::BuildAll(const long &nb, type = ConvertToCpuIndexType(type); } else { WRAPPER_LOG_ERROR << "Build IVFMIXIndex Failed"; - return server::KNOWHERE_ERROR; + return KNOWHERE_ERROR; } } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_UNEXPECTED_ERROR; + return KNOWHERE_UNEXPECTED_ERROR; } catch (jsoncons::json_exception &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_INVALID_ARGUMENT; + return KNOWHERE_INVALID_ARGUMENT; } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_ERROR; + return KNOWHERE_ERROR; } - return server::KNOWHERE_SUCCESS; + return KNOWHERE_SUCCESS; } -server::KnowhereError IVFMixIndex::Load(const zilliz::knowhere::BinarySet &index_binary) { +ErrorCode IVFMixIndex::Load(const zilliz::knowhere::BinarySet &index_binary) { //index_ = std::make_shared(); index_->Load(index_binary); dim = Dimension(); - return server::KNOWHERE_SUCCESS; + return KNOWHERE_SUCCESS; } } diff --git a/cpp/src/wrapper/knowhere/vec_impl.h b/cpp/src/wrapper/knowhere/vec_impl.h index f03f299f78..e98d030a56 100644 --- a/cpp/src/wrapper/knowhere/vec_impl.h +++ b/cpp/src/wrapper/knowhere/vec_impl.h @@ -19,7 +19,7 @@ class VecIndexImpl : public VecIndex { public: explicit VecIndexImpl(std::shared_ptr index, const IndexType &type) : index_(std::move(index)), type(type) {}; - server::KnowhereError BuildAll(const long &nb, + ErrorCode BuildAll(const long &nb, const float *xb, const long *ids, const Config &cfg, @@ -30,12 +30,12 @@ class VecIndexImpl : public VecIndex { IndexType GetType() override; int64_t Dimension() override; int64_t Count() override; - server::KnowhereError Add(const long &nb, const float *xb, const long *ids, const Config &cfg) override; + ErrorCode Add(const long &nb, const float *xb, const long *ids, const Config &cfg) override; zilliz::knowhere::BinarySet Serialize() override; - server::KnowhereError Load(const zilliz::knowhere::BinarySet &index_binary) override; + ErrorCode Load(const zilliz::knowhere::BinarySet &index_binary) override; VecIndexPtr Clone() override; int64_t GetDeviceId() override; - server::KnowhereError Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) override; + ErrorCode Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) override; protected: int64_t dim = 0; @@ -48,22 +48,22 @@ class IVFMixIndex : public VecIndexImpl { explicit IVFMixIndex(std::shared_ptr index, const IndexType &type) : VecIndexImpl(std::move(index), type) {}; - server::KnowhereError BuildAll(const long &nb, + ErrorCode BuildAll(const long &nb, const float *xb, const long *ids, const Config &cfg, const long &nt, const float *xt) override; - server::KnowhereError Load(const zilliz::knowhere::BinarySet &index_binary) override; + ErrorCode Load(const zilliz::knowhere::BinarySet &index_binary) override; }; class BFIndex : public VecIndexImpl { public: explicit BFIndex(std::shared_ptr index) : VecIndexImpl(std::move(index), IndexType::FAISS_IDMAP) {}; - server::KnowhereError Build(const Config& cfg); + ErrorCode Build(const Config& cfg); float *GetRawVectors(); - server::KnowhereError BuildAll(const long &nb, + ErrorCode BuildAll(const long &nb, const float *xb, const long *ids, const Config &cfg, diff --git a/cpp/src/wrapper/knowhere/vec_index.cpp b/cpp/src/wrapper/knowhere/vec_index.cpp index 0665ffc166..e14496884d 100644 --- a/cpp/src/wrapper/knowhere/vec_index.cpp +++ b/cpp/src/wrapper/knowhere/vec_index.cpp @@ -71,7 +71,7 @@ size_t FileIOWriter::operator()(void *ptr, size_t size) { } -VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config& cfg) { +VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config &cfg) { std::shared_ptr index; auto gpu_device = cfg.get_with_default("gpu_id", 0); switch (type) { @@ -172,7 +172,7 @@ VecIndexPtr read_index(const std::string &location) { return LoadVecIndex(current_type, load_data_list); } -server::KnowhereError write_index(VecIndexPtr index, const std::string &location) { +ErrorCode write_index(VecIndexPtr index, const std::string &location) { try { auto binaryset = index->Serialize(); auto index_type = index->GetType(); @@ -192,12 +192,12 @@ server::KnowhereError write_index(VecIndexPtr index, const std::string &location } } catch (knowhere::KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_UNEXPECTED_ERROR; + return KNOWHERE_UNEXPECTED_ERROR; } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); - return server::KNOWHERE_ERROR; + return KNOWHERE_ERROR; } - return server::KNOWHERE_SUCCESS; + return KNOWHERE_SUCCESS; } @@ -235,6 +235,31 @@ void AutoGenParams(const IndexType &type, const long &size, zilliz::knowhere::Co } } +#if CUDA_VERSION > 9000 +#define GPU_MAX_NRPOBE 2048 +#else +#define GPU_MAX_NRPOBE 1024 +#endif + +void ParameterValidation(const IndexType &type, Config &cfg) { + switch (type) { + case IndexType::FAISS_IVFSQ8_GPU: + case IndexType::FAISS_IVFFLAT_GPU: + case IndexType::FAISS_IVFPQ_GPU: { + if (cfg.get_with_default("nprobe", 0) != 0) { + auto nprobe = cfg["nprobe"].as(); + if (nprobe > GPU_MAX_NRPOBE) { + WRAPPER_LOG_WARNING << "When search with GPU, nprobe shoud be no more than " << GPU_MAX_NRPOBE << ", but you passed " << nprobe + << ". Search with " << GPU_MAX_NRPOBE << " instead"; + cfg.insert_or_assign("nprobe", GPU_MAX_NRPOBE); + } + } + break; + } + default:break; + } +} + IndexType ConvertToCpuIndexType(const IndexType &type) { // TODO(linxj): add IDMAP switch (type) { diff --git a/cpp/src/wrapper/knowhere/vec_index.h b/cpp/src/wrapper/knowhere/vec_index.h index c69106159a..137429d1ad 100644 --- a/cpp/src/wrapper/knowhere/vec_index.h +++ b/cpp/src/wrapper/knowhere/vec_index.h @@ -14,6 +14,8 @@ #include "knowhere/common/config.h" #include "knowhere/common/binary_set.h" +#include "cuda.h" + namespace zilliz { namespace milvus { @@ -42,23 +44,23 @@ using VecIndexPtr = std::shared_ptr; class VecIndex { public: - virtual server::KnowhereError BuildAll(const long &nb, - const float *xb, - const long *ids, - const Config &cfg, - const long &nt = 0, - const float *xt = nullptr) = 0; + virtual ErrorCode BuildAll(const long &nb, + const float *xb, + const long *ids, + const Config &cfg, + const long &nt = 0, + const float *xt = nullptr) = 0; - virtual server::KnowhereError Add(const long &nb, - const float *xb, - const long *ids, - const Config &cfg = Config()) = 0; + virtual ErrorCode Add(const long &nb, + const float *xb, + const long *ids, + const Config &cfg = Config()) = 0; - virtual server::KnowhereError Search(const long &nq, - const float *xq, - float *dist, - long *ids, - const Config &cfg = Config()) = 0; + virtual ErrorCode Search(const long &nq, + const float *xq, + float *dist, + long *ids, + const Config &cfg = Config()) = 0; virtual VecIndexPtr CopyToGpu(const int64_t& device_id, const Config &cfg = Config()) = 0; @@ -77,10 +79,10 @@ class VecIndex { virtual zilliz::knowhere::BinarySet Serialize() = 0; - virtual server::KnowhereError Load(const zilliz::knowhere::BinarySet &index_binary) = 0; + virtual ErrorCode Load(const zilliz::knowhere::BinarySet &index_binary) = 0; }; -extern server::KnowhereError write_index(VecIndexPtr index, const std::string &location); +extern ErrorCode write_index(VecIndexPtr index, const std::string &location); extern VecIndexPtr read_index(const std::string &location); @@ -90,6 +92,8 @@ extern VecIndexPtr LoadVecIndex(const IndexType &index_type, const zilliz::knowh extern void AutoGenParams(const IndexType& type, const long& size, Config& cfg); +extern void ParameterValidation(const IndexType& type, Config& cfg); + extern IndexType ConvertToCpuIndexType(const IndexType& type); extern IndexType ConvertToGpuIndexType(const IndexType& type); diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index ff78178dfb..2cf5055091 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -38,9 +38,13 @@ set(unittest_libs cudart ) +foreach(dir ${CORE_INCLUDE_DIRS}) + include_directories(${dir}) +endforeach() + add_subdirectory(server) add_subdirectory(db) -#add_subdirectory(knowhere) +add_subdirectory(knowhere) add_subdirectory(metrics) #add_subdirectory(scheduler) #add_subdirectory(storage) \ No newline at end of file diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 64f56996d4..eae0b9bb83 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -35,8 +35,6 @@ link_directories("/usr/local/cuda/lib64") include_directories(/usr/include/mysql) -#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY) - set(db_test_src ${config_files} ${cache_srcs} @@ -69,25 +67,10 @@ set(db_libs set(knowhere_libs knowhere - SPTAGLibStatic - arrow - jemalloc_pic - faiss - openblas - lapack - tbb cudart cublas ) -if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS}) -else() - set(db_libs ${db_libs} - lapack - openblas) -endif() - target_link_libraries(db_test ${db_libs} ${knowhere_libs} ${unittest_libs}) install(TARGETS db_test DESTINATION unittest) diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 2baad61ca4..193e1ac771 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -15,6 +15,7 @@ #include "db/Utils.h" #include "db/meta/MetaConsts.h" +using namespace zilliz::milvus; using namespace zilliz::milvus::engine; TEST_F(MetaTest, TABLE_TEST) { @@ -38,7 +39,7 @@ TEST_F(MetaTest, TABLE_TEST) { table.table_id_ = table_id; status = impl_->CreateTable(table); - ASSERT_TRUE(status.IsAlreadyExist()); + ASSERT_EQ(status.code(), DB_ALREADY_EXIST); table.table_id_ = ""; status = impl_->CreateTable(table); diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index a948d1f4d9..bc853c8f23 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -41,24 +41,29 @@ TEST(DBMiscTest, STATUS_TEST) { std::string str = status.ToString(); ASSERT_FALSE(str.empty()); - status = engine::Status::Error("wrong", "mistake"); - ASSERT_TRUE(status.IsError()); + status = engine::Status(DB_ERROR, "mistake"); + ASSERT_EQ(status.code(), DB_ERROR); str = status.ToString(); ASSERT_FALSE(str.empty()); - status = engine::Status::NotFound("wrong", "mistake"); - ASSERT_TRUE(status.IsNotFound()); + status = engine::Status(DB_NOT_FOUND, "mistake"); + ASSERT_EQ(status.code(), DB_NOT_FOUND); str = status.ToString(); ASSERT_FALSE(str.empty()); - status = engine::Status::DBTransactionError("wrong", "mistake"); - ASSERT_TRUE(status.IsDBTransactionError()); + status = engine::Status(DB_ALREADY_EXIST, "mistake"); + ASSERT_EQ(status.code(), DB_ALREADY_EXIST); + str = status.ToString(); + ASSERT_FALSE(str.empty()); + + status = engine::Status(DB_META_TRANSACTION_FAILED, "mistake"); + ASSERT_EQ(status.code(), DB_META_TRANSACTION_FAILED); str = status.ToString(); ASSERT_FALSE(str.empty()); engine::Status status_copy = engine::Status::OK(); CopyStatus(status_copy, status); - ASSERT_TRUE(status.IsDBTransactionError()); + ASSERT_EQ(status.code(), DB_META_TRANSACTION_FAILED); } TEST(DBMiscTest, OPTIONS_TEST) { diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index f54f451fcf..aa0fa176f9 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -19,6 +19,7 @@ #include +using namespace zilliz::milvus; using namespace zilliz::milvus::engine; TEST_F(MySqlMetaTest, TABLE_TEST) { @@ -42,7 +43,7 @@ TEST_F(MySqlMetaTest, TABLE_TEST) { table.table_id_ = table_id; status = impl_->CreateTable(table); - ASSERT_TRUE(status.IsAlreadyExist()); + ASSERT_EQ(status.code(), DB_ALREADY_EXIST); table.table_id_ = ""; status = impl_->CreateTable(table); diff --git a/cpp/unittest/knowhere/CMakeLists.txt b/cpp/unittest/knowhere/CMakeLists.txt index 6dc99d858f..089ff992fa 100644 --- a/cpp/unittest/knowhere/CMakeLists.txt +++ b/cpp/unittest/knowhere/CMakeLists.txt @@ -8,13 +8,6 @@ set(helper set(knowhere_libs knowhere - SPTAGLibStatic - arrow - jemalloc_pic - faiss - openblas - lapack - tbb cudart cublas ) diff --git a/cpp/unittest/knowhere/knowhere_test.cpp b/cpp/unittest/knowhere/knowhere_test.cpp index 8d2524276c..2bd4bddc70 100644 --- a/cpp/unittest/knowhere/knowhere_test.cpp +++ b/cpp/unittest/knowhere/knowhere_test.cpp @@ -40,6 +40,9 @@ class KnowhereWrapperTest index_ = GetVecIndexFactory(index_type); } + void TearDown() override { + zilliz::knowhere::FaissGpuResourceMgr::GetInstance().Free(); + } void AssertResult(const std::vector &ids, const std::vector &dis) { EXPECT_EQ(ids.size(), nq * k); diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 69f0f9e94a..1f0f5bd023 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -1,13 +1,3 @@ - - -#set(CURL_LIBRARY "-lcurl") -#find_package(CURL REQUIRED) -#INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIR}) -# -#INCLUDE_DIRECTORIES(~/development/lib/usr/local/include) -#LINK_DIRECTORIES(~/development/lib/usr/local/lib) - - include_directories(../../src) @@ -72,13 +62,6 @@ add_executable(metrics_test ${count_test_src} ${require_files} ) set(knowhere_libs knowhere - SPTAGLibStatic - arrow - jemalloc_pic - faiss - openblas - lapack - tbb ) target_link_libraries(metrics_test @@ -96,12 +79,5 @@ target_link_libraries(metrics_test mysqlpp ${unittest_libs} ) -if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - target_link_libraries(metrics_test ${MKL_LIBS} ${MKL_LIBS}) -else() - target_link_libraries(metrics_test - lapack - openblas) -endif() install(TARGETS metrics_test DESTINATION unittest) \ No newline at end of file diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/metrics_test.cpp index f589cf3ace..2d31154e3c 100644 --- a/cpp/unittest/metrics/metrics_test.cpp +++ b/cpp/unittest/metrics/metrics_test.cpp @@ -127,7 +127,7 @@ TEST_F(MetricTest, Metric_Tes) { TEST_F(MetricTest, Collector_Metrics_Test){ engine::Status status = engine::Status::OK(); server::CollectInsertMetrics insert_metrics0(0, status); - status = engine::Status::Error("error"); + status = engine::Status(DB_ERROR, "error"); server::CollectInsertMetrics insert_metrics1(0, status); server::CollectQueryMetrics query_metrics(10); diff --git a/cpp/unittest/scheduler/CMakeLists.txt b/cpp/unittest/scheduler/CMakeLists.txt index 4f8b944611..feb7d6178f 100644 --- a/cpp/unittest/scheduler/CMakeLists.txt +++ b/cpp/unittest/scheduler/CMakeLists.txt @@ -33,8 +33,6 @@ link_directories("/usr/local/cuda/lib64") include_directories(/usr/include/mysql) -#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY) - set(scheduler_test_src ${unittest_srcs} ${test_srcs} @@ -69,25 +67,10 @@ set(scheduler_libs set(knowhere_libs knowhere - SPTAGLibStatic - arrow - jemalloc_pic - faiss - openblas - lapack - tbb cudart cublas ) -if (${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS}) -else () - set(db_libs ${db_libs} - lapack - openblas) -endif () - target_link_libraries(scheduler_test ${scheduler_libs} ${knowhere_libs} ${unittest_libs}) diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index cf7e62cb5e..3dfca57994 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -34,9 +34,6 @@ cuda_add_executable(server_test set(require_libs knowhere - faiss - openblas - lapack stdc++ cudart cublas @@ -51,14 +48,6 @@ set(require_libs pthread ) -if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - set(require_libs ${require_libs} ${MKL_LIBS} ${MKL_LIBS}) -else() - set(require_libs ${require_libs} - lapack - openblas) -endif() - target_link_libraries(server_test ${require_libs} ${cuda_library} diff --git a/cpp/unittest/server/cache_test.cpp b/cpp/unittest/server/cache_test.cpp index ad5e393e9a..65dc621e19 100644 --- a/cpp/unittest/server/cache_test.cpp +++ b/cpp/unittest/server/cache_test.cpp @@ -30,12 +30,12 @@ public: class MockVecIndex : public engine::VecIndex { public: - virtual server::KnowhereError BuildAll(const long &nb, - const float *xb, - const long *ids, - const engine::Config &cfg, - const long &nt = 0, - const float *xt = nullptr) { + virtual ErrorCode BuildAll(const long &nb, + const float *xb, + const long *ids, + const engine::Config &cfg, + const long &nt = 0, + const float *xt = nullptr) { } @@ -51,18 +51,18 @@ public: return engine::IndexType::INVALID; } - virtual server::KnowhereError Add(const long &nb, - const float *xb, - const long *ids, - const engine::Config &cfg = engine::Config()) { + virtual ErrorCode Add(const long &nb, + const float *xb, + const long *ids, + const engine::Config &cfg = engine::Config()) { } - virtual server::KnowhereError Search(const long &nq, - const float *xq, - float *dist, - long *ids, - const engine::Config &cfg = engine::Config()) { + virtual ErrorCode Search(const long &nq, + const float *xq, + float *dist, + long *ids, + const engine::Config &cfg = engine::Config()) { } @@ -87,7 +87,7 @@ public: return binset; } - virtual server::KnowhereError Load(const zilliz::knowhere::BinarySet &index_binary) { + virtual ErrorCode Load(const zilliz::knowhere::BinarySet &index_binary) { } diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/config_test.cpp index ba68d98979..e23f085059 100644 --- a/cpp/unittest/server/config_test.cpp +++ b/cpp/unittest/server/config_test.cpp @@ -27,14 +27,14 @@ static constexpr uint64_t GB = MB*1024; TEST(ConfigTest, CONFIG_TEST) { server::ConfigMgr* config_mgr = server::ConfigMgr::GetInstance(); - server::ServerError err = config_mgr->LoadConfigFile(""); - ASSERT_EQ(err, server::SERVER_UNEXPECTED_ERROR); + ErrorCode err = config_mgr->LoadConfigFile(""); + ASSERT_EQ(err, SERVER_UNEXPECTED_ERROR); err = config_mgr->LoadConfigFile(LOG_FILE_PATH); - ASSERT_EQ(err, server::SERVER_UNEXPECTED_ERROR); + ASSERT_EQ(err, SERVER_UNEXPECTED_ERROR); err = config_mgr->LoadConfigFile(CONFIG_FILE_PATH); - ASSERT_EQ(err, server::SERVER_SUCCESS); + ASSERT_EQ(err, SERVER_SUCCESS); config_mgr->Print(); @@ -95,11 +95,11 @@ TEST(ConfigTest, CONFIG_TEST) { TEST(ConfigTest, SERVER_CONFIG_TEST) { server::ServerConfig& config = server::ServerConfig::GetInstance(); - server::ServerError err = config.LoadConfigFile(CONFIG_FILE_PATH); - ASSERT_EQ(err, server::SERVER_SUCCESS); + ErrorCode err = config.LoadConfigFile(CONFIG_FILE_PATH); + ASSERT_EQ(err, SERVER_SUCCESS); err = server::ServerConfig::GetInstance().ValidateConfig(); - ASSERT_EQ(err, server::SERVER_SUCCESS); + ASSERT_EQ(err, SERVER_SUCCESS); server::ConfigNode node1 = config.GetConfig("server_config"); server::ConfigNode& node2 = config.GetConfig("cache_config"); @@ -125,26 +125,26 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) { server::ConfigNode& cache_config = config.GetConfig(server::CONFIG_CACHE); cache_config.SetValue(server::CACHE_FREE_PERCENT, "2.0"); err = config.ValidateConfig(); - ASSERT_NE(err, server::SERVER_SUCCESS); + ASSERT_NE(err, SERVER_SUCCESS); size_t cache_cap = 16; size_t insert_buffer_size = (total_mem - cache_cap*GB + 1*GB)/GB; db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size)); cache_config.SetValue(server::CONFIG_CPU_CACHE_CAPACITY, std::to_string(cache_cap)); err = config.ValidateConfig(); - ASSERT_NE(err, server::SERVER_SUCCESS); + ASSERT_NE(err, SERVER_SUCCESS); cache_cap = total_mem/GB + 2; cache_config.SetValue(server::CONFIG_CPU_CACHE_CAPACITY, std::to_string(cache_cap)); err = config.ValidateConfig(); - ASSERT_NE(err, server::SERVER_SUCCESS); + ASSERT_NE(err, SERVER_SUCCESS); insert_buffer_size = total_mem/GB + 2; db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size)); err = config.ValidateConfig(); - ASSERT_NE(err, server::SERVER_SUCCESS); + ASSERT_NE(err, SERVER_SUCCESS); server_config.SetValue(server::CONFIG_GPU_INDEX, "9999"); err = config.ValidateConfig(); - ASSERT_NE(err, server::SERVER_SUCCESS); + ASSERT_NE(err, SERVER_SUCCESS); } \ No newline at end of file diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index 290655f565..68e74a846f 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -25,8 +25,8 @@ static const char* LOG_FILE_PATH = "./milvus/conf/log_config.conf"; TEST(UtilTest, EXCEPTION_TEST) { std::string err_msg = "failed"; - server::ServerException ex(server::SERVER_UNEXPECTED_ERROR, err_msg); - ASSERT_EQ(ex.error_code(), server::SERVER_UNEXPECTED_ERROR); + server::ServerException ex(SERVER_UNEXPECTED_ERROR, err_msg); + ASSERT_EQ(ex.error_code(), SERVER_UNEXPECTED_ERROR); std::string msg = ex.what(); ASSERT_EQ(msg, err_msg); } @@ -44,19 +44,19 @@ TEST(UtilTest, COMMON_TEST) { std::string path1 = "/tmp/milvus_test/"; std::string path2 = path1 + "common_test_12345/"; std::string path3 = path2 + "abcdef"; - server::ServerError err = server::CommonUtil::CreateDirectory(path3); - ASSERT_EQ(err, server::SERVER_SUCCESS); + ErrorCode err = server::CommonUtil::CreateDirectory(path3); + ASSERT_EQ(err, SERVER_SUCCESS); //test again err = server::CommonUtil::CreateDirectory(path3); - ASSERT_EQ(err, server::SERVER_SUCCESS); + ASSERT_EQ(err, SERVER_SUCCESS); ASSERT_TRUE(server::CommonUtil::IsDirectoryExist(path3)); err = server::CommonUtil::DeleteDirectory(path1); - ASSERT_EQ(err, server::SERVER_SUCCESS); + ASSERT_EQ(err, SERVER_SUCCESS); //test again err = server::CommonUtil::DeleteDirectory(path1); - ASSERT_EQ(err, server::SERVER_SUCCESS); + ASSERT_EQ(err, SERVER_SUCCESS); ASSERT_FALSE(server::CommonUtil::IsDirectoryExist(path1)); ASSERT_FALSE(server::CommonUtil::IsFileExist(path1)); @@ -94,24 +94,24 @@ TEST(UtilTest, STRINGFUNCTIONS_TEST) { str = "a,b,c"; std::vector result; - server::ServerError err = server::StringHelpFunctions::SplitStringByDelimeter(str , ",", result); - ASSERT_EQ(err, server::SERVER_SUCCESS); + ErrorCode err = server::StringHelpFunctions::SplitStringByDelimeter(str , ",", result); + ASSERT_EQ(err, SERVER_SUCCESS); ASSERT_EQ(result.size(), 3UL); result.clear(); err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result); - ASSERT_EQ(err, server::SERVER_SUCCESS); + ASSERT_EQ(err, SERVER_SUCCESS); ASSERT_EQ(result.size(), 3UL); result.clear(); err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "", result); - ASSERT_EQ(err, server::SERVER_SUCCESS); + ASSERT_EQ(err, SERVER_SUCCESS); ASSERT_EQ(result.size(), 3UL); str = "55,\"aa,gg,yy\",b"; result.clear(); err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result); - ASSERT_EQ(err, server::SERVER_SUCCESS); + ASSERT_EQ(err, SERVER_SUCCESS); ASSERT_EQ(result.size(), 3UL); @@ -159,86 +159,86 @@ TEST(UtilTest, LOG_TEST) { TEST(UtilTest, VALIDATE_TABLENAME_TEST) { std::string table_name = "Normal123_"; - server:: ServerError res = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, server::SERVER_SUCCESS); + ErrorCode res = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(res, SERVER_SUCCESS); table_name = "12sds"; res = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); + ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); table_name = ""; res = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); + ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); table_name = "_asdasd"; res = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, server::SERVER_SUCCESS); + ASSERT_EQ(res, SERVER_SUCCESS); table_name = "!@#!@"; res = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); + ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); table_name = "_!@#!@"; res = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); + ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); table_name = "中文"; res = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); + ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); table_name = std::string(10000, 'a'); res = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); + ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); } TEST(UtilTest, VALIDATE_DIMENSIONTEST) { - ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(-1), server::SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(0), server::SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16385), server::SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16384), server::SERVER_SUCCESS); - ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(1), server::SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(-1), SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(0), SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16385), SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16384), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(1), SERVER_SUCCESS); } TEST(UtilTest, VALIDATE_INDEX_TEST) { - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::INVALID), server::SERVER_INVALID_INDEX_TYPE); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::INVALID), SERVER_INVALID_INDEX_TYPE); for(int i = 1; i <= (int)engine::EngineType::MAX_VALUE; i++) { - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType(i), server::SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType(i), SERVER_SUCCESS); } - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), server::SERVER_INVALID_INDEX_TYPE); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), SERVER_INVALID_INDEX_TYPE); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexNlist(0), server::SERVER_INVALID_INDEX_NLIST); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexNlist(100), server::SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexNlist(0), SERVER_INVALID_INDEX_NLIST); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexNlist(100), SERVER_SUCCESS); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexFileSize(0), server::SERVER_INVALID_INDEX_FILE_SIZE); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexFileSize(100), server::SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexFileSize(0), SERVER_INVALID_INDEX_FILE_SIZE); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexFileSize(100), SERVER_SUCCESS); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(0), server::SERVER_INVALID_INDEX_METRIC_TYPE); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(1), server::SERVER_SUCCESS); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(2), server::SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(0), SERVER_INVALID_INDEX_METRIC_TYPE); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(1), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(2), SERVER_SUCCESS); } TEST(ValidationUtilTest, ValidateTopkTest) { engine::meta::TableSchema schema; - ASSERT_EQ(server::ValidationUtil::ValidateSearchTopk(10, schema), server::SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(65536, schema), server::SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(0, schema), server::SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateSearchTopk(10, schema), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(65536, schema), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(0, schema), SERVER_SUCCESS); } TEST(ValidationUtilTest, ValidateNprobeTest) { engine::meta::TableSchema schema; schema.nlist_ = 100; - ASSERT_EQ(server::ValidationUtil::ValidateSearchNprobe(10, schema), server::SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(0, schema), server::SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(101, schema), server::SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateSearchNprobe(10, schema), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(0, schema), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(101, schema), SERVER_SUCCESS); } TEST(ValidationUtilTest, ValidateGpuTest) { - ASSERT_EQ(server::ValidationUtil::ValidateGpuIndex(0), server::SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateGpuIndex(100), server::SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateGpuIndex(0), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateGpuIndex(100), SERVER_SUCCESS); size_t memory = 0; - ASSERT_EQ(server::ValidationUtil::GetGpuMemory(0, memory), server::SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::GetGpuMemory(100, memory), server::SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::GetGpuMemory(0, memory), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::GetGpuMemory(100, memory), SERVER_SUCCESS); } TEST(UtilTest, TIMERECORDER_TEST) {