From ff79e1796e6d2af6aa91951b247805926755cc82 Mon Sep 17 00:00:00 2001 From: zhiru Date: Tue, 3 Sep 2019 10:18:55 +0800 Subject: [PATCH 1/3] cmake: integerated knowhere Former-commit-id: bd87ac66e3bc90a85a4fb6517711c4a2c36caaef --- cpp/CMakeLists.txt | 31 +- cpp/build.sh | 6 +- cpp/src/CMakeLists.txt | 55 ++- cpp/src/core/CMakeLists.txt | 24 +- ...{BuildUtils.cmake => BuildUtilsCore.cmake} | 2 +- ...eOptions.cmake => DefineOptionsCore.cmake} | 2 +- ...ges.cmake => ThirdPartyPackagesCore.cmake} | 401 +++++++++--------- cpp/src/core/src/CMakeLists.txt | 60 ++- cpp/src/core/test/CMakeLists.txt | 70 +-- cpp/src/core/test/faiss_ori/CMakeLists.txt | 12 +- cpp/src/core/test/test_nsg/CMakeLists.txt | 32 +- cpp/unittest/CMakeLists.txt | 9 + cpp/unittest/db/CMakeLists.txt | 28 +- cpp/unittest/knowhere/CMakeLists.txt | 14 +- cpp/unittest/metrics/CMakeLists.txt | 28 +- cpp/unittest/scheduler/CMakeLists.txt | 28 +- cpp/unittest/server/CMakeLists.txt | 20 +- 17 files changed, 443 insertions(+), 379 deletions(-) rename cpp/src/core/cmake/{BuildUtils.cmake => BuildUtilsCore.cmake} (99%) rename cpp/src/core/cmake/{DefineOptions.cmake => DefineOptionsCore.cmake} (98%) rename cpp/src/core/cmake/{ThirdPartyPackages.cmake => ThirdPartyPackagesCore.cmake} (78%) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 08898aacdf..f5cd5af950 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -31,6 +31,7 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release") set(BUILD_TYPE "release") else() set(BUILD_TYPE "debug") + SET(CMAKE_VERBOSE_MAKEFILE on) endif() message(STATUS "Build type = ${BUILD_TYPE}") @@ -70,21 +71,21 @@ set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR}) find_package(CUDA) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda") set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") -message("CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}") -message("CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}") +message(STATUS "CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}") +message(STATUS "CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}") set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED on) if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") - message("building milvus_engine on x86 architecture") + message(STATUS "building milvus_engine on x86 architecture") set(MILVUS_BUILD_ARCH x86_64) elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)") - message("building milvus_engine on ppc architecture") + message(STATUS "building milvus_engine on ppc architecture") set(MILVUS_BUILD_ARCH ppc64le) else() - message("unknown processor type") - message("CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}") + message(WARNING "unknown processor type") + message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}") set(MILVUS_BUILD_ARCH unknown) endif() @@ -94,6 +95,8 @@ else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp") endif() +set(ALLOW_DUPLICATE_CUSTOM_TARGETS TRUE) + set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") include(ExternalProject) @@ -104,13 +107,13 @@ include(ThirdPartyPackages) include_directories(${MILVUS_SOURCE_DIR}) link_directories(${MILVUS_BINARY_DIR}) -if (NOT DEFINED KNOWHERE_BUILD_DIR) - message(FATAL_ERROR "You must set environment variable KNOWHERE_BUILD_DIR") -endif() -message(STATUS "Build with ${KNOWHERE_BUILD_DIR}") -include_directories(${KNOWHERE_BUILD_DIR}/include) -include_directories(${KNOWHERE_BUILD_DIR}/include/SPTAG/AnnService) -link_directories(${KNOWHERE_BUILD_DIR}/lib) +#if (NOT DEFINED KNOWHERE_BUILD_DIR) +# message(FATAL_ERROR "You must set environment variable KNOWHERE_BUILD_DIR") +#endif() +#message(STATUS "Build with ${KNOWHERE_BUILD_DIR}") +#include_directories(${KNOWHERE_BUILD_DIR}/include) +#include_directories(${KNOWHERE_BUILD_DIR}/include/SPTAG/AnnService) +#link_directories(${KNOWHERE_BUILD_DIR}/lib) ## Following should be check @@ -119,7 +122,7 @@ set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) add_compile_definitions(PROFILER=${PROFILER}) -message("MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}") +message(STATUS "MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}") if (MILVUS_ENABLE_PROFILING STREQUAL "ON") ADD_DEFINITIONS(-DMILVUS_ENABLE_PROFILING) endif() diff --git a/cpp/build.sh b/cpp/build.sh index b2dba3c870..aa41b7d318 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -83,9 +83,9 @@ if [[ ! -d cmake_build ]]; then MAKE_CLEAN="ON" fi -pushd `pwd`/src/core -./build.sh ${KNOWHERE_OPTIONS} -popd +#pushd `pwd`/src/core +#./build.sh ${KNOWHERE_OPTIONS} +#popd cd cmake_build diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 7af3c54787..9f25cae8ef 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -4,6 +4,15 @@ # Proprietary and confidential. #------------------------------------------------------------------------------- +add_subdirectory(core) + +set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) +foreach(dir ${CORE_INCLUDE_DIRS}) + message("dir = ${dir}") + include_directories(${dir}) +endforeach() +#include_directories(${CMAKE_SOURCE_DIR}/src/core/thirdparty/SPTAG/AnnService) + aux_source_directory(cache cache_files) aux_source_directory(config config_files) aux_source_directory(server server_files) @@ -87,12 +96,12 @@ set(client_grpc_lib set(third_party_libs knowhere - SPTAGLibStatic - arrow - jemalloc_pic - faiss - openblas - lapack +# SPTAGLibStatic +# arrow +# jemalloc_pic +# faiss +# openblas +# lapack easyloggingpp sqlite ${client_grpc_lib} @@ -115,14 +124,14 @@ set(third_party_libs cudart ) -if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - set(third_party_libs ${third_party_libs} - ${MKL_LIBS}) -else() - set(third_party_libs ${third_party_libs} - lapack - openblas) -endif() +#if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") +# set(third_party_libs ${third_party_libs} +# ${MKL_LIBS}) +#else() +# set(third_party_libs ${third_party_libs} +# lapack +# openblas) +#endif() if (MILVUS_ENABLE_PROFILING STREQUAL "ON") set(third_party_libs ${third_party_libs} @@ -171,13 +180,13 @@ set(server_libs set(knowhere_libs knowhere - SPTAGLibStatic - arrow - jemalloc_pic - faiss - openblas - lapack - tbb +# SPTAGLibStatic +# arrow +# jemalloc_pic +# faiss +# openblas +# lapack +# tbb ) add_executable(milvus_server @@ -195,8 +204,8 @@ target_link_libraries(milvus_server ${server_libs} ${knowhere_libs} ${third_part install(TARGETS milvus_server DESTINATION bin) install(FILES - ${KNOWHERE_BUILD_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX} - ${KNOWHERE_BUILD_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}.2 + ${CMAKE_SOURCE_DIR}/src/core/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX} + ${CMAKE_SOURCE_DIR}/src/core/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}.2 ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3 ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4 diff --git a/cpp/src/core/CMakeLists.txt b/cpp/src/core/CMakeLists.txt index 1024a71e71..962fac2cf2 100644 --- a/cpp/src/core/CMakeLists.txt +++ b/cpp/src/core/CMakeLists.txt @@ -5,6 +5,7 @@ #------------------------------------------------------------------------------- cmake_minimum_required(VERSION 3.14) +message(STATUS "---------------core--------------") message(STATUS "Building using CMake version: ${CMAKE_VERSION}") set(KNOWHERE_VERSION "0.1.0") @@ -42,14 +43,14 @@ find_package(CUDA) #set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") - message("building milvus_engine on x86 architecture") + message(STATUS "building milvus_engine on x86 architecture") set(KNOWHERE_BUILD_ARCH x86_64) elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)") - message("building milvus_engine on ppc architecture") + message(STATUS "building milvus_engine on ppc architecture") set(KNOWHERE_BUILD_ARCH ppc64le) else() - message("unknown processor type") - message("CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}") + message(WARNING "unknown processor type") + message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}") set(KNOWHERE_BUILD_ARCH unknown) endif() @@ -60,14 +61,21 @@ else() endif() message(STATUS "Build type = ${BUILD_TYPE}") -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") +set(CORE_SOURCE_DIR ${PROJECT_SOURCE_DIR}) +set(CORE_BINARY_DIR ${PROJECT_BINARY_DIR}) +message(STATUS "Core source dir: ${PROJECT_SOURCE_DIR}") +message(STATUS "Core binary dir: ${PROJECT_BINARY_DIR}") +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CORE_SOURCE_DIR}/cmake") + include(ExternalProject) -include(DefineOptions) -include(BuildUtils) -include(ThirdPartyPackages) +include(DefineOptionsCore) +include(BuildUtilsCore) +include(ThirdPartyPackagesCore) add_subdirectory(src) +set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) + if(BUILD_UNIT_TEST STREQUAL "ON") add_subdirectory(test) endif() diff --git a/cpp/src/core/cmake/BuildUtils.cmake b/cpp/src/core/cmake/BuildUtilsCore.cmake similarity index 99% rename from cpp/src/core/cmake/BuildUtils.cmake rename to cpp/src/core/cmake/BuildUtilsCore.cmake index 2d778bf703..ce798c4bb0 100644 --- a/cpp/src/core/cmake/BuildUtils.cmake +++ b/cpp/src/core/cmake/BuildUtilsCore.cmake @@ -75,7 +75,7 @@ function(ExternalProject_Use_Cache project_name package_file install_path) "Extracting ${package_file} to ${install_path}" COMMAND ${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + WORKING_DIRECTORY ${CORE_BINARY_DIR} ) ExternalProject_Add_StepTargets(${project_name} extract) diff --git a/cpp/src/core/cmake/DefineOptions.cmake b/cpp/src/core/cmake/DefineOptionsCore.cmake similarity index 98% rename from cpp/src/core/cmake/DefineOptions.cmake rename to cpp/src/core/cmake/DefineOptionsCore.cmake index 589e2d9ae8..e20d3d77ab 100644 --- a/cpp/src/core/cmake/DefineOptions.cmake +++ b/cpp/src/core/cmake/DefineOptionsCore.cmake @@ -104,7 +104,7 @@ macro(config_summary) message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}") if(${CMAKE_EXPORT_COMPILE_COMMANDS}) message( - STATUS " Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json") + STATUS " Compile commands: ${CORE_BINARY_DIR}/compile_commands.json") endif() foreach(category ${KNOWHERE_OPTION_CATEGORIES}) diff --git a/cpp/src/core/cmake/ThirdPartyPackages.cmake b/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake similarity index 78% rename from cpp/src/core/cmake/ThirdPartyPackages.cmake rename to cpp/src/core/cmake/ThirdPartyPackagesCore.cmake index 945f50fb73..76ee18f4e6 100644 --- a/cpp/src/core/cmake/ThirdPartyPackages.cmake +++ b/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake @@ -18,7 +18,7 @@ set(KNOWHERE_THIRDPARTY_DEPENDENCIES ARROW - BOOST +# BOOST FAISS GTest LAPACK @@ -120,7 +120,7 @@ endif (UNIX) # ---------------------------------------------------------------------- # thirdparty directory -set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty") +set(THIRDPARTY_DIR "${CORE_SOURCE_DIR}/thirdparty") # ---------------------------------------------------------------------- # JFrog @@ -243,14 +243,14 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) set(${_LIB_NAME} "${_LIB_VERSION}") endforeach() -if(DEFINED ENV{KNOWHERE_BOOST_URL}) - set(BOOST_SOURCE_URL "$ENV{KNOWHERE_BOOST_URL}") -else() - string(REPLACE "." "_" BOOST_VERSION_UNDERSCORES ${BOOST_VERSION}) - set(BOOST_SOURCE_URL - "https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz" - ) -endif() +#if(DEFINED ENV{KNOWHERE_BOOST_URL}) +# set(BOOST_SOURCE_URL "$ENV{KNOWHERE_BOOST_URL}") +#else() +# string(REPLACE "." "_" BOOST_VERSION_UNDERSCORES ${BOOST_VERSION}) +# set(BOOST_SOURCE_URL +# "https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz" +# ) +#endif() if(DEFINED ENV{KNOWHERE_FAISS_URL}) set(FAISS_SOURCE_URL "$ENV{KNOWHERE_FAISS_URL}") @@ -258,7 +258,7 @@ else() set(FAISS_SOURCE_URL "http://192.168.1.105:6060/jinhai/faiss/-/archive/${FAISS_VERSION}/faiss-${FAISS_VERSION}.tar.gz") # set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz") # set(FAISS_SOURCE_URL "${CMAKE_SOURCE_DIR}/thirdparty/faiss-1.5.3") - message(STATUS ${FAISS_SOURCE_URL}) + message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}") endif() # set(FAISS_MD5 "a589663865a8558205533c8ac414278c") set(FAISS_MD5 "31167ecbd1903fec600dc4ac00b9be9e") @@ -296,10 +296,11 @@ set(OPENBLAS_MD5 "8a110a25b819a4b94e8a9580702b6495") # ---------------------------------------------------------------------- # ARROW +set(ARROW_PREFIX "${CORE_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp") macro(build_arrow) message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source") - set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp") +# set(ARROW_PREFIX "${CORE_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp") set(ARROW_STATIC_LIB_NAME arrow) # set(PARQUET_STATIC_LIB_NAME parquet) # set(ARROW_CUDA_STATIC_LIB_NAME arrow_cuda) @@ -361,13 +362,13 @@ macro(build_arrow) "${ARROW_STATIC_LIB}" ) - ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL}) + ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL}) else() file(DOWNLOAD ${ARROW_CACHE_URL} ${ARROW_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${ARROW_CACHE_URL} TO ${ARROW_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) - ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR}) endif() endif() else() @@ -404,7 +405,7 @@ macro(build_arrow) # INTERFACE_LINK_LIBRARIES thrift) add_dependencies(arrow arrow_ep) - set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep") + set(JEMALLOC_PREFIX "${CORE_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep") add_custom_command(TARGET arrow_ep POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_PREFIX}/lib/ @@ -413,7 +414,7 @@ macro(build_arrow) endmacro() -if(KNOWHERE_WITH_ARROW) +if(KNOWHERE_WITH_ARROW AND NOT TARGET arrow_ep) resolve_dependency(ARROW) @@ -424,188 +425,188 @@ endif() # ---------------------------------------------------------------------- # Add Boost dependencies (code adapted from Apache Kudu (incubating)) -set(Boost_USE_MULTITHREADED ON) -if(MSVC AND KNOWHERE_USE_STATIC_CRT) - set(Boost_USE_STATIC_RUNTIME ON) -endif() -set(Boost_ADDITIONAL_VERSIONS - "1.70.0" - "1.70" - "1.69.0" - "1.69" - "1.68.0" - "1.68" - "1.67.0" - "1.67" - "1.66.0" - "1.66" - "1.65.0" - "1.65" - "1.64.0" - "1.64" - "1.63.0" - "1.63" - "1.62.0" - "1.61" - "1.61.0" - "1.62" - "1.60.0" - "1.60") - -# TODO -if(KNOWHERE_BOOST_VENDORED) -# system thread serialization wserialization regex - set(BOOST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/boost_ep-prefix/src/boost_ep") - set(BOOST_LIB_DIR "${BOOST_PREFIX}/stage/lib") - set(BOOST_BUILD_LINK "static") - set(BOOST_STATIC_SYSTEM_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(BOOST_STATIC_FILESYSTEM_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(BOOST_STATIC_SERIALIZATION_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_serialization${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(BOOST_STATIC_WSERIALIZATION_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_wserialization${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(BOOST_STATIC_REGEX_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(BOOST_STATIC_THREAD_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_thread${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(BOOST_SYSTEM_LIBRARY boost_system_static) - set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static) - set(BOOST_SERIALIZATION_LIBRARY boost_serialization_static) - set(BOOST_WSERIALIZATION_LIBRARY boost_wserialization_static) - set(BOOST_REGEX_LIBRARY boost_regex_static) - set(BOOST_THREAD_LIBRARY boost_thread_static) - - if(KNOWHERE_BOOST_HEADER_ONLY) - set(BOOST_BUILD_PRODUCTS) - set(BOOST_CONFIGURE_COMMAND "") - set(BOOST_BUILD_COMMAND "") - else() - set(BOOST_BUILD_PRODUCTS ${BOOST_STATIC_SYSTEM_LIBRARY} - ${BOOST_STATIC_FILESYSTEM_LIBRARY} ${BOOST_STATIC_SERIALIZATION_LIBRARY} - ${BOOST_STATIC_WSERIALIZATION_LIBRARY} ${BOOST_STATIC_REGEX_LIBRARY} - ${BOOST_STATIC_THREAD_LIBRARY}) - set(BOOST_CONFIGURE_COMMAND "./bootstrap.sh" "--prefix=${BOOST_PREFIX}" - "--with-libraries=filesystem,serialization,wserialization,system,thread,regex") - if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") - set(BOOST_BUILD_VARIANT "debug") - else() - set(BOOST_BUILD_VARIANT "release") - endif() - set(BOOST_BUILD_COMMAND - "./b2" - "link=${BOOST_BUILD_LINK}" - "variant=${BOOST_BUILD_VARIANT}" - "cxxflags=-fPIC") - - add_thirdparty_lib(boost_system STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}") - - add_thirdparty_lib(boost_filesystem STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}") - - add_thirdparty_lib(boost_serialization STATIC_LIB "${BOOST_STATIC_SERIALIZATION_LIBRARY}") - - add_thirdparty_lib(boost_wserialization STATIC_LIB "${BOOST_STATIC_WSERIALIZATION_LIBRARY}") - - add_thirdparty_lib(boost_regex STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}") - - add_thirdparty_lib(boost_thread STATIC_LIB "${BOOST_STATIC_THREAD_LIBRARY}") - - set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY} ${BOOST_SERIALIZATION_LIBRARY} - ${BOOST_WSERIALIZATION_LIBRARY} ${BOOST_REGEX_LIBRARY} ${BOOST_THREAD_LIBRARY}) - endif() - externalproject_add(boost_ep - URL - ${BOOST_SOURCE_URL} - BUILD_BYPRODUCTS - ${BOOST_BUILD_PRODUCTS} - BUILD_IN_SOURCE - 1 - CONFIGURE_COMMAND - ${BOOST_CONFIGURE_COMMAND} - BUILD_COMMAND - ${BOOST_BUILD_COMMAND} - INSTALL_COMMAND - "" - ${EP_LOG_OPTIONS}) - set(Boost_INCLUDE_DIR "${BOOST_PREFIX}") - set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}") - add_dependencies(boost_system_static boost_ep) - add_dependencies(boost_filesystem_static boost_ep) - add_dependencies(boost_serialization_static boost_ep) - add_dependencies(boost_wserialization_static boost_ep) - add_dependencies(boost_regex_static boost_ep) - add_dependencies(boost_thread_static boost_ep) - -#else() -# if(MSVC) -# # disable autolinking in boost -# add_definitions(-DBOOST_ALL_NO_LIB) -# endif() - -# if(DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT) -# # In older versions of CMake (such as 3.2), the system paths for Boost will -# # In older versions of CMake (such as 3.2), the system paths for Boost will -# # be looked in first even if we set $BOOST_ROOT or pass -DBOOST_ROOT -# set(Boost_NO_SYSTEM_PATHS ON) -# endif() - -# if(KNOWHERE_BOOST_USE_SHARED) -# # Find shared Boost libraries. -# set(Boost_USE_STATIC_LIBS OFF) -# set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS}) -# set(BUILD_SHARED_LIBS ON) +#set(Boost_USE_MULTITHREADED ON) +#if(MSVC AND KNOWHERE_USE_STATIC_CRT) +# set(Boost_USE_STATIC_RUNTIME ON) +#endif() +#set(Boost_ADDITIONAL_VERSIONS +# "1.70.0" +# "1.70" +# "1.69.0" +# "1.69" +# "1.68.0" +# "1.68" +# "1.67.0" +# "1.67" +# "1.66.0" +# "1.66" +# "1.65.0" +# "1.65" +# "1.64.0" +# "1.64" +# "1.63.0" +# "1.63" +# "1.62.0" +# "1.61" +# "1.61.0" +# "1.62" +# "1.60.0" +# "1.60") # -# if(MSVC) -# # force all boost libraries to dynamic link -# add_definitions(-DBOOST_ALL_DYN_LINK) -# endif() +## TODO +#if(KNOWHERE_BOOST_VENDORED) +## system thread serialization wserialization regex +# set(BOOST_PREFIX "${CORE_BINARY_DIR}/boost_ep-prefix/src/boost_ep") +# set(BOOST_LIB_DIR "${BOOST_PREFIX}/stage/lib") +# set(BOOST_BUILD_LINK "static") +# set(BOOST_STATIC_SYSTEM_LIBRARY +# "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}" +# ) +# set(BOOST_STATIC_FILESYSTEM_LIBRARY +# "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}" +# ) +# set(BOOST_STATIC_SERIALIZATION_LIBRARY +# "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_serialization${CMAKE_STATIC_LIBRARY_SUFFIX}" +# ) +# set(BOOST_STATIC_WSERIALIZATION_LIBRARY +# "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_wserialization${CMAKE_STATIC_LIBRARY_SUFFIX}" +# ) +# set(BOOST_STATIC_REGEX_LIBRARY +# "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}" +# ) +# set(BOOST_STATIC_THREAD_LIBRARY +# "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_thread${CMAKE_STATIC_LIBRARY_SUFFIX}" +# ) +# set(BOOST_SYSTEM_LIBRARY boost_system_static) +# set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static) +# set(BOOST_SERIALIZATION_LIBRARY boost_serialization_static) +# set(BOOST_WSERIALIZATION_LIBRARY boost_wserialization_static) +# set(BOOST_REGEX_LIBRARY boost_regex_static) +# set(BOOST_THREAD_LIBRARY boost_thread_static) # -# if(KNOWHERE_BOOST_HEADER_ONLY) -# find_package(Boost REQUIRED) -# else() -# find_package(Boost COMPONENTS serialization system filesystem REQUIRED) -# set(BOOST_SYSTEM_LIBRARY Boost::system) -# set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) -# set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) -# set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) -# endif() -# set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP}) -# unset(BUILD_SHARED_LIBS_KEEP) +# if(KNOWHERE_BOOST_HEADER_ONLY) +# set(BOOST_BUILD_PRODUCTS) +# set(BOOST_CONFIGURE_COMMAND "") +# set(BOOST_BUILD_COMMAND "") # else() -# # Find static boost headers and libs -# # TODO Differentiate here between release and debug builds -# set(Boost_USE_STATIC_LIBS ON) -# if(KNOWHERE_BOOST_HEADER_ONLY) -# find_package(Boost REQUIRED) +# set(BOOST_BUILD_PRODUCTS ${BOOST_STATIC_SYSTEM_LIBRARY} +# ${BOOST_STATIC_FILESYSTEM_LIBRARY} ${BOOST_STATIC_SERIALIZATION_LIBRARY} +# ${BOOST_STATIC_WSERIALIZATION_LIBRARY} ${BOOST_STATIC_REGEX_LIBRARY} +# ${BOOST_STATIC_THREAD_LIBRARY}) +# set(BOOST_CONFIGURE_COMMAND "./bootstrap.sh" "--prefix=${BOOST_PREFIX}" +# "--with-libraries=filesystem,serialization,wserialization,system,thread,regex") +# if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") +# set(BOOST_BUILD_VARIANT "debug") # else() -# find_package(Boost COMPONENTS serialization system filesystem REQUIRED) -# set(BOOST_SYSTEM_LIBRARY Boost::system) -# set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) -# set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) -# set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) +# set(BOOST_BUILD_VARIANT "release") # endif() +# set(BOOST_BUILD_COMMAND +# "./b2" +# "link=${BOOST_BUILD_LINK}" +# "variant=${BOOST_BUILD_VARIANT}" +# "cxxflags=-fPIC") +# +# add_thirdparty_lib(boost_system STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}") +# +# add_thirdparty_lib(boost_filesystem STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}") +# +# add_thirdparty_lib(boost_serialization STATIC_LIB "${BOOST_STATIC_SERIALIZATION_LIBRARY}") +# +# add_thirdparty_lib(boost_wserialization STATIC_LIB "${BOOST_STATIC_WSERIALIZATION_LIBRARY}") +# +# add_thirdparty_lib(boost_regex STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}") +# +# add_thirdparty_lib(boost_thread STATIC_LIB "${BOOST_STATIC_THREAD_LIBRARY}") +# +# set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY} ${BOOST_SERIALIZATION_LIBRARY} +# ${BOOST_WSERIALIZATION_LIBRARY} ${BOOST_REGEX_LIBRARY} ${BOOST_THREAD_LIBRARY}) # endif() -endif() - -#message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR}) -#message(STATUS "Boost libraries: " ${Boost_LIBRARIES}) - -include_directories(SYSTEM ${Boost_INCLUDE_DIR}) -link_directories(SYSTEM ${BOOST_LIB_DIR}) +# externalproject_add(boost_ep +# URL +# ${BOOST_SOURCE_URL} +# BUILD_BYPRODUCTS +# ${BOOST_BUILD_PRODUCTS} +# BUILD_IN_SOURCE +# 1 +# CONFIGURE_COMMAND +# ${BOOST_CONFIGURE_COMMAND} +# BUILD_COMMAND +# ${BOOST_BUILD_COMMAND} +# INSTALL_COMMAND +# "" +# ${EP_LOG_OPTIONS}) +# set(Boost_INCLUDE_DIR "${BOOST_PREFIX}") +# set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}") +# add_dependencies(boost_system_static boost_ep) +# add_dependencies(boost_filesystem_static boost_ep) +# add_dependencies(boost_serialization_static boost_ep) +# add_dependencies(boost_wserialization_static boost_ep) +# add_dependencies(boost_regex_static boost_ep) +# add_dependencies(boost_thread_static boost_ep) +# +##else() +## if(MSVC) +## # disable autolinking in boost +## add_definitions(-DBOOST_ALL_NO_LIB) +## endif() +# +## if(DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT) +## # In older versions of CMake (such as 3.2), the system paths for Boost will +## # In older versions of CMake (such as 3.2), the system paths for Boost will +## # be looked in first even if we set $BOOST_ROOT or pass -DBOOST_ROOT +## set(Boost_NO_SYSTEM_PATHS ON) +## endif() +# +## if(KNOWHERE_BOOST_USE_SHARED) +## # Find shared Boost libraries. +## set(Boost_USE_STATIC_LIBS OFF) +## set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS}) +## set(BUILD_SHARED_LIBS ON) +## +## if(MSVC) +## # force all boost libraries to dynamic link +## add_definitions(-DBOOST_ALL_DYN_LINK) +## endif() +## +## if(KNOWHERE_BOOST_HEADER_ONLY) +## find_package(Boost REQUIRED) +## else() +## find_package(Boost COMPONENTS serialization system filesystem REQUIRED) +## set(BOOST_SYSTEM_LIBRARY Boost::system) +## set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) +## set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) +## set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) +## endif() +## set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP}) +## unset(BUILD_SHARED_LIBS_KEEP) +## else() +## # Find static boost headers and libs +## # TODO Differentiate here between release and debug builds +## set(Boost_USE_STATIC_LIBS ON) +## if(KNOWHERE_BOOST_HEADER_ONLY) +## find_package(Boost REQUIRED) +## else() +## find_package(Boost COMPONENTS serialization system filesystem REQUIRED) +## set(BOOST_SYSTEM_LIBRARY Boost::system) +## set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) +## set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) +## set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) +## endif() +## endif() +#endif() +# +##message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR}) +##message(STATUS "Boost libraries: " ${Boost_LIBRARIES}) +# +#include_directories(SYSTEM ${Boost_INCLUDE_DIR}) +#link_directories(SYSTEM ${BOOST_LIB_DIR}) # ---------------------------------------------------------------------- # OpenBLAS macro(build_openblas) message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source") - set(OPENBLAS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep") + set(OPENBLAS_PREFIX "${CORE_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep") set(OPENBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include") set(OPENBLAS_STATIC_LIB "${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}") @@ -638,13 +639,13 @@ macro(build_openblas) BUILD_BYPRODUCTS ${OPENBLAS_STATIC_LIB}) - ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL}) + ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL}) else() file(DOWNLOAD ${OPENBLAS_CACHE_URL} ${OPENBLAS_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${OPENBLAS_CACHE_URL} TO ${OPENBLAS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) - ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR}) endif() endif() else() @@ -690,7 +691,7 @@ endmacro() macro(build_lapack) message(STATUS "Building LAPACK-${LAPACK_VERSION} from source") - set(LAPACK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep") + set(LAPACK_PREFIX "${CORE_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep") set(LAPACK_INCLUDE_DIR "${LAPACK_PREFIX}/include") set(LAPACK_STATIC_LIB "${LAPACK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lapack${CMAKE_STATIC_LIBRARY_SUFFIX}") @@ -722,13 +723,13 @@ macro(build_lapack) BUILD_BYPRODUCTS ${LAPACK_STATIC_LIB}) - ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL}) + ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL}) else() file(DOWNLOAD ${LAPACK_CACHE_URL} ${LAPACK_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${LAPACK_CACHE_URL} TO ${LAPACK_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) - ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR}) endif() endif() else() @@ -779,7 +780,7 @@ macro(build_gtest) -Wno-ignored-attributes) endif() - set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") + set(GTEST_PREFIX "${CORE_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") set(GTEST_STATIC_LIB "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}") @@ -821,13 +822,13 @@ macro(build_gtest) ${GTEST_CMAKE_ARGS} ${EP_LOG_OPTIONS}) - ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL}) + ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL}) else() file(DOWNLOAD ${GTEST_CACHE_URL} ${GTEST_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${GTEST_CACHE_URL} TO ${GTEST_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) - ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR}) endif() endif() else() @@ -870,7 +871,7 @@ macro(build_gtest) endmacro() -if (KNOWHERE_BUILD_TESTS) +if (KNOWHERE_BUILD_TESTS AND NOT TARGET googletest_ep) #message(STATUS "Resolving gtest dependency") resolve_dependency(GTest) @@ -888,7 +889,7 @@ endif() macro(build_faiss) message(STATUS "Building FAISS-${FAISS_VERSION} from source") - set(FAISS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep") + set(FAISS_PREFIX "${CORE_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep") set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include") set(FAISS_STATIC_LIB "${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}") @@ -951,13 +952,13 @@ macro(build_faiss) ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) - ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL}) + ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL}) else() file(DOWNLOAD ${FAISS_CACHE_URL} ${FAISS_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${FAISS_CACHE_URL} TO ${FAISS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) - ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR}) endif() endif() else() @@ -996,7 +997,7 @@ macro(build_faiss) endmacro() -if(KNOWHERE_WITH_FAISS) +if(KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep) resolve_dependency(OpenBLAS) get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) @@ -1011,7 +1012,7 @@ if(KNOWHERE_WITH_FAISS) resolve_dependency(FAISS) get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM "${FAISS_INCLUDE_DIR}") -# include_directories(SYSTEM "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/") +# include_directories(SYSTEM "${CORE_BINARY_DIR}/faiss_ep-prefix/src/") # link_directories(SYSTEM ${FAISS_PREFIX}/) link_directories(SYSTEM ${FAISS_PREFIX}/lib/) # link_directories(SYSTEM ${FAISS_PREFIX}/gpu/) diff --git a/cpp/src/core/src/CMakeLists.txt b/cpp/src/core/src/CMakeLists.txt index 5b119f8b2b..a9ee7cd3db 100644 --- a/cpp/src/core/src/CMakeLists.txt +++ b/cpp/src/core/src/CMakeLists.txt @@ -1,16 +1,16 @@ -set(TBB_DIR ${CMAKE_SOURCE_DIR}/thirdparty/tbb) +set(TBB_DIR ${CORE_SOURCE_DIR}/thirdparty/tbb) set(TBB_LIBRARIES ${TBB_DIR}/libtbb.so) include_directories(${TBB_DIR}/include) include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64) -include_directories(${CMAKE_SOURCE_DIR}/include) -include_directories(${CMAKE_SOURCE_DIR}/thirdparty) -include_directories(${CMAKE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) -include_directories(${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) +include_directories(${CORE_SOURCE_DIR}/include) +include_directories(${CORE_SOURCE_DIR}/thirdparty) +include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) +include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) -set(SPTAG_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/SPTAG) +set(SPTAG_SOURCE_DIR ${CORE_SOURCE_DIR}/thirdparty/SPTAG) file(GLOB HDR_FILES ${SPTAG_SOURCE_DIR}/AnnService/inc/Core/*.h ${SPTAG_SOURCE_DIR}/AnnService/inc/Core/Common/*.h @@ -26,7 +26,9 @@ file(GLOB SRC_FILES #add_library(SPTAGLib SHARED ${SRC_FILES} ${HDR_FILES}) #target_link_libraries(SPTAGLib ${TBB_LIBRARIES}) -add_library(SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES}) +if(NOT TARGET SPTAGLibStatic) + add_library(SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES}) +endif() set(external_srcs knowhere/adapter/sptag.cpp @@ -58,7 +60,7 @@ set(depend_libs openblas lapack arrow - jemalloc_pic + ${ARROW_PREFIX}/lib/libjemalloc_pic.a cudart cublas gomp @@ -66,11 +68,13 @@ set(depend_libs pthread ) -add_library( - knowhere STATIC - ${external_srcs} - ${index_srcs} -) +if(NOT TARGET knowhere) + add_library( + knowhere STATIC + ${external_srcs} + ${index_srcs} + ) +endif() #target_compile_options(knowhere PUBLIC "-fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free") target_link_libraries( @@ -112,24 +116,42 @@ INSTALL(FILES ${OPENBLAS_REAL_STATIC_LIB} DESTINATION lib ) -INSTALL(FILES ${CMAKE_SOURCE_DIR}/thirdparty/tbb/libtbb.so.2 +INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so.2 # RENAME "libtbb.so.2" DESTINATION lib ) -INSTALL(FILES ${CMAKE_SOURCE_DIR}/thirdparty/tbb/libtbb.so +INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so # RENAME "libtbb.so" DESTINATION lib ) +set(CORE_INCLUDE_DIRS + ${CORE_SOURCE_DIR}/include + ${CORE_SOURCE_DIR}/thirdparty + ${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService + ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include + ${ARROW_INCLUDE_DIR} + ${FAISS_INCLUDE_DIR} + ${OPENBLAS_INCLUDE_DIR} + ${LAPACK_INCLUDE_DIR} + ${CORE_SOURCE_DIR}/thirdparty/tbb/include + ) + +set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) +#include_directories(SYSTEM ${CORE_SOURCE_DIR}/include) +#include_directories(SYSTEM ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) +#include_directories(SYSTEM ${CORE_SOURCE_DIR}/thirdparty/tbb/include) +#include_directories(SYSTEM ${SPTAG_SOURCE_DIR}/AnnService/inc) + INSTALL(DIRECTORY - ${CMAKE_SOURCE_DIR}/include/knowhere - ${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons - ${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext + ${CORE_SOURCE_DIR}/include/knowhere + ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons + ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext ${ARROW_INCLUDE_DIR}/arrow # ${ARROW_INCLUDE_DIR}/parquet ${FAISS_PREFIX}/include/faiss ${OPENBLAS_INCLUDE_DIR}/ - ${CMAKE_SOURCE_DIR}/thirdparty/tbb/include/tbb + ${CORE_SOURCE_DIR}/thirdparty/tbb/include/tbb DESTINATION include) diff --git a/cpp/src/core/test/CMakeLists.txt b/cpp/src/core/test/CMakeLists.txt index fbbf151264..d2629a4570 100644 --- a/cpp/src/core/test/CMakeLists.txt +++ b/cpp/src/core/test/CMakeLists.txt @@ -1,17 +1,19 @@ -include_directories(${CMAKE_SOURCE_DIR}/thirdparty) -include_directories(${CMAKE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) -include_directories(${CMAKE_SOURCE_DIR}/include) -include_directories(${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) +include_directories(${CORE_SOURCE_DIR}/thirdparty) +include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) +include_directories(${CORE_SOURCE_DIR}/include) +include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) include_directories(/usr/local/cuda/include) link_directories(/usr/local/cuda/lib64) -link_directories(${CMAKE_SOURCE_DIR}/thirdparty/tbb) +link_directories(${CORE_SOURCE_DIR}/thirdparty/tbb) set(unittest_libs gtest gmock gtest_main gmock_main) +message(STATUS "arrow prefix: ${ARROW_PREFIX}") +message(STATUS "libjemalloc_pic path: ${ARROW_PREFIX}/lib/libjemalloc_pic.a") set(depend_libs faiss openblas lapack - arrow jemalloc_pic + arrow "${ARROW_PREFIX}/lib/libjemalloc_pic.a" tbb ) @@ -24,32 +26,36 @@ set(basic_libs # #<<<<<<<<<< set(ivf_srcs - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp + ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp utils.cpp ) -add_executable(test_ivf test_ivf.cpp ${ivf_srcs}) +if(NOT TARGET test_ivf) + add_executable(test_ivf test_ivf.cpp ${ivf_srcs}) +endif() target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs}) #<<<<<<<<<< # #<<<<<<<<<< set(idmap_srcs - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp + ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp utils.cpp ) -add_executable(test_idmap test_idmap.cpp ${idmap_srcs}) +if(NOT TARGET test_idmap) + add_executable(test_idmap test_idmap.cpp ${idmap_srcs}) +endif() target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) #<<<<<<<<<< @@ -57,17 +63,19 @@ target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) # #<<<<<<<<<< set(kdt_srcs - ${CMAKE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp + ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp + ${CORE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp + ${CORE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp utils.cpp ) -add_executable(test_kdt test_kdt.cpp ${kdt_srcs}) +if(NOT TARGET test_kdt) + add_executable(test_kdt test_kdt.cpp ${kdt_srcs}) +endif() target_link_libraries(test_kdt SPTAGLibStatic ${depend_libs} ${unittest_libs} ${basic_libs}) diff --git a/cpp/src/core/test/faiss_ori/CMakeLists.txt b/cpp/src/core/test/faiss_ori/CMakeLists.txt index a6b20a2373..1209ec00b8 100644 --- a/cpp/src/core/test/faiss_ori/CMakeLists.txt +++ b/cpp/src/core/test/faiss_ori/CMakeLists.txt @@ -1,15 +1,15 @@ -include_directories(${CMAKE_SOURCE_DIR}/thirdparty) -include_directories(${CMAKE_SOURCE_DIR}/include) +include_directories(${CORE_SOURCE_DIR}/thirdparty) +include_directories(${CORE_SOURCE_DIR}/include) include_directories(/usr/local/cuda/include) link_directories(/usr/local/cuda/lib64) -link_directories(${CMAKE_SOURCE_DIR}/thirdparty/tbb) +link_directories(${CORE_SOURCE_DIR}/thirdparty/tbb) set(unittest_libs gtest gmock gtest_main gmock_main) set(depend_libs faiss openblas lapack - arrow jemalloc_pic + arrow ${ARROW_PREFIX}/lib/libjemalloc_pic.a tbb ) @@ -21,6 +21,8 @@ set(basic_libs # #<<<<<<<<<< -add_executable(test_gpu gpuresource_test.cpp) +if(NOT TARGET test_gpu) + add_executable(test_gpu gpuresource_test.cpp) +endif() target_link_libraries(test_gpu ${depend_libs} ${unittest_libs} ${basic_libs}) #<<<<<<<<<< diff --git a/cpp/src/core/test/test_nsg/CMakeLists.txt b/cpp/src/core/test/test_nsg/CMakeLists.txt index 0ec275d9ee..7ebd534379 100644 --- a/cpp/src/core/test/test_nsg/CMakeLists.txt +++ b/cpp/src/core/test/test_nsg/CMakeLists.txt @@ -14,33 +14,35 @@ else () endif () message(${OpenMP_CXX_FLAGS}) -include_directories(${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg) +include_directories(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg) #include_directories(/home/zilliz/opt/app/pyenv/versions/3.6.8/include/python3.6m) #include_directories(/home/zilliz/opt/app/pyenv/versions/3.6.8/envs/megasearch_testframework_dev/lib/python3.6/site-packages/numpy/core/include) -aux_source_directory(${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src) +aux_source_directory(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src) #add_library(nsg_raw SHARED ${nsg_src}) #target_link_libraries(nsg_raw # gomp) set(interface_src - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp + ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp + ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp + ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp ../utils.cpp ) -add_executable(test_nsg - test_nsg.cpp - ${interface_src} - ${nsg_src} - ) +if(NOT TARGET test_nsg) + add_executable(test_nsg + test_nsg.cpp + ${interface_src} + ${nsg_src} + ) +endif() #target_link_libraries(test_nsg # # libprofiler.so ## -ltcmalloc diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index ac666c86a9..83290ba9cf 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -6,6 +6,8 @@ link_directories( "${CMAKE_BINARY_DIR}/lib" ) +#include_directories(${CMAKE_SOURCE_DIR}/src/core/include) +#include_directories(${CMAKE_SOURCE_DIR}/src/core/thirdparty/SPTAG/AnnService) aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) @@ -38,6 +40,13 @@ set(unittest_libs cudart ) +#message(STATUS "unittest core dir: ${CMAKE_CURRENT_BINARY_DIR}/core") +#add_subdirectory(${CMAKE_SOURCE_DIR}/src/core ${CMAKE_CURRENT_BINARY_DIR}/core) +foreach(dir ${CORE_INCLUDE_DIRS}) + message("dir = ${dir}") + include_directories(${dir}) +endforeach() + add_subdirectory(server) add_subdirectory(db) add_subdirectory(knowhere) diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 564d03a828..53a99ddfeb 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -69,24 +69,24 @@ set(db_libs set(knowhere_libs knowhere - SPTAGLibStatic - arrow - jemalloc_pic - faiss - openblas - lapack - tbb +# SPTAGLibStatic +# arrow +# jemalloc_pic +# faiss +# openblas +# lapack +# tbb cudart cublas ) -if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS}) -else() - set(db_libs ${db_libs} - lapack - openblas) -endif() +#if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") +# set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS}) +#else() +# set(db_libs ${db_libs} +# lapack +# openblas) +#endif() target_link_libraries(db_test ${db_libs} ${knowhere_libs} ${unittest_libs}) diff --git a/cpp/unittest/knowhere/CMakeLists.txt b/cpp/unittest/knowhere/CMakeLists.txt index 1f02464ad9..db538001e7 100644 --- a/cpp/unittest/knowhere/CMakeLists.txt +++ b/cpp/unittest/knowhere/CMakeLists.txt @@ -8,13 +8,13 @@ set(helper set(knowhere_libs knowhere - SPTAGLibStatic - arrow - jemalloc_pic - faiss - openblas - lapack - tbb +# SPTAGLibStatic +# arrow +# jemalloc_pic +# faiss +# openblas +# lapack +# tbb cudart cublas ) diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index c066165a2c..b1760aa579 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -72,13 +72,13 @@ add_executable(metrics_test ${count_test_src} ${require_files} ) set(knowhere_libs knowhere - SPTAGLibStatic - arrow - jemalloc_pic - faiss - openblas - lapack - tbb +# SPTAGLibStatic +# arrow +# jemalloc_pic +# faiss +# openblas +# lapack +# tbb ) target_link_libraries(metrics_test @@ -96,12 +96,12 @@ target_link_libraries(metrics_test mysqlpp ${unittest_libs} ) -if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - target_link_libraries(metrics_test ${MKL_LIBS} ${MKL_LIBS}) -else() - target_link_libraries(metrics_test - lapack - openblas) -endif() +#if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") +# target_link_libraries(metrics_test ${MKL_LIBS} ${MKL_LIBS}) +#else() +# target_link_libraries(metrics_test +# lapack +# openblas) +#endif() install(TARGETS metrics_test DESTINATION bin) \ No newline at end of file diff --git a/cpp/unittest/scheduler/CMakeLists.txt b/cpp/unittest/scheduler/CMakeLists.txt index 24210cb84d..ddc56233c4 100644 --- a/cpp/unittest/scheduler/CMakeLists.txt +++ b/cpp/unittest/scheduler/CMakeLists.txt @@ -69,24 +69,24 @@ set(scheduler_libs set(knowhere_libs knowhere - SPTAGLibStatic - arrow - jemalloc_pic - faiss - openblas - lapack - tbb +# SPTAGLibStatic +# arrow +# jemalloc_pic +# faiss +# openblas +# lapack +# tbb cudart cublas ) -if (${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS}) -else () - set(db_libs ${db_libs} - lapack - openblas) -endif () +#if (${BUILD_FAISS_WITH_MKL} STREQUAL "ON") +# set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS}) +#else () +# set(db_libs ${db_libs} +# lapack +# openblas) +#endif () target_link_libraries(scheduler_test ${scheduler_libs} ${knowhere_libs} ${unittest_libs}) diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index 33c7fcdea2..6b8bf6d836 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -34,9 +34,9 @@ cuda_add_executable(server_test set(require_libs knowhere - faiss - openblas - lapack +# faiss +# openblas +# lapack stdc++ cudart cublas @@ -51,13 +51,13 @@ set(require_libs pthread ) -if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - set(require_libs ${require_libs} ${MKL_LIBS} ${MKL_LIBS}) -else() - set(require_libs ${require_libs} - lapack - openblas) -endif() +#if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") +# set(require_libs ${require_libs} ${MKL_LIBS} ${MKL_LIBS}) +#else() +# set(require_libs ${require_libs} +# lapack +# openblas) +#endif() target_link_libraries(server_test ${require_libs} From ea830ebc67b7123fe7c2ac7e2f11f71e10ea4935 Mon Sep 17 00:00:00 2001 From: zhiru Date: Fri, 6 Sep 2019 16:30:47 +0800 Subject: [PATCH 2/3] cmake: integrated knowhere Former-commit-id: 81afb501886848d478c7046f04b933e2a3f34ac0 --- cpp/CHANGELOG.md | 1 + cpp/CMakeLists.txt | 8 - cpp/src/CMakeLists.txt | 24 -- cpp/src/core/CMakeLists.txt | 2 - .../core/cmake/ThirdPartyPackagesCore.cmake | 265 +----------------- cpp/src/core/src/CMakeLists.txt | 24 -- cpp/src/core/test/CMakeLists.txt | 8 - cpp/src/core/test/faiss_ori/CMakeLists.txt | 2 - cpp/src/core/test/test_nsg/CMakeLists.txt | 13 +- cpp/unittest/CMakeLists.txt | 11 +- cpp/unittest/db/CMakeLists.txt | 17 -- cpp/unittest/knowhere/CMakeLists.txt | 7 - cpp/unittest/metrics/CMakeLists.txt | 24 -- cpp/unittest/scheduler/CMakeLists.txt | 17 -- cpp/unittest/server/CMakeLists.txt | 11 - 15 files changed, 11 insertions(+), 423 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index af32680e23..c6082b043c 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -94,6 +94,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-338 - NewAPI: refine code to support CreateIndex - MS-339 - NewAPI: refine code to support DropIndex - MS-340 - NewAPI: implement DescribeIndex +- MS-495 - cmake: integrated knowhere ## Task - MS-297 - disable mysql unit test diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index f5cd5af950..16e8041c29 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -107,14 +107,6 @@ include(ThirdPartyPackages) include_directories(${MILVUS_SOURCE_DIR}) link_directories(${MILVUS_BINARY_DIR}) -#if (NOT DEFINED KNOWHERE_BUILD_DIR) -# message(FATAL_ERROR "You must set environment variable KNOWHERE_BUILD_DIR") -#endif() -#message(STATUS "Build with ${KNOWHERE_BUILD_DIR}") -#include_directories(${KNOWHERE_BUILD_DIR}/include) -#include_directories(${KNOWHERE_BUILD_DIR}/include/SPTAG/AnnService) -#link_directories(${KNOWHERE_BUILD_DIR}/lib) - ## Following should be check set(MILVUS_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include) diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index a2ac3002b3..387693a30c 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -8,10 +8,8 @@ add_subdirectory(core) set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) foreach(dir ${CORE_INCLUDE_DIRS}) - message("dir = ${dir}") include_directories(${dir}) endforeach() -#include_directories(${CMAKE_SOURCE_DIR}/src/core/thirdparty/SPTAG/AnnService) aux_source_directory(cache cache_files) aux_source_directory(config config_files) @@ -96,12 +94,6 @@ set(client_grpc_lib set(third_party_libs knowhere -# SPTAGLibStatic -# arrow -# jemalloc_pic -# faiss -# openblas -# lapack easyloggingpp sqlite ${client_grpc_lib} @@ -124,15 +116,6 @@ set(third_party_libs cudart ) -#if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") -# set(third_party_libs ${third_party_libs} -# ${MKL_LIBS}) -#else() -# set(third_party_libs ${third_party_libs} -# lapack -# openblas) -#endif() - if (MILVUS_ENABLE_PROFILING STREQUAL "ON") set(third_party_libs ${third_party_libs} gperftools @@ -180,13 +163,6 @@ set(server_libs set(knowhere_libs knowhere -# SPTAGLibStatic -# arrow -# jemalloc_pic -# faiss -# openblas -# lapack -# tbb ) add_executable(milvus_server diff --git a/cpp/src/core/CMakeLists.txt b/cpp/src/core/CMakeLists.txt index 962fac2cf2..890c7ed4d1 100644 --- a/cpp/src/core/CMakeLists.txt +++ b/cpp/src/core/CMakeLists.txt @@ -39,8 +39,6 @@ endif() MESSAGE(STATUS "CMAKE_CXX_FLAGS" ${CMAKE_CXX_FLAGS}) find_package(CUDA) -#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda") -#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") message(STATUS "building milvus_engine on x86 architecture") diff --git a/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake b/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake index c823f108bd..fad0869021 100644 --- a/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake +++ b/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake @@ -6,7 +6,6 @@ # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an @@ -18,7 +17,6 @@ set(KNOWHERE_THIRDPARTY_DEPENDENCIES ARROW -# BOOST FAISS GTest LAPACK @@ -53,11 +51,8 @@ endmacro() macro(resolve_dependency DEPENDENCY_NAME) if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") #message(STATUS "Finding ${DEPENDENCY_NAME} package") -# find_package(${DEPENDENCY_NAME} QUIET) -# if (NOT ${DEPENDENCY_NAME}_FOUND) #message(STATUS "${DEPENDENCY_NAME} package not found") build_dependency(${DEPENDENCY_NAME}) -# endif () elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") build_dependency(${DEPENDENCY_NAME}) elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") @@ -202,14 +197,6 @@ endif() set(MAKE_BUILD_ARGS "-j8") -## Using make -j in sub-make is fragile -## see discussion https://github.com/apache/KNOWHERE/pull/2779 -#if(${CMAKE_GENERATOR} MATCHES "Makefiles") -# set(MAKE_BUILD_ARGS "") -#else() -# # limit the maximum number of jobs for ninja -# set(MAKE_BUILD_ARGS "-j4") -#endif() # ---------------------------------------------------------------------- # Find pthreads @@ -243,21 +230,10 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) set(${_LIB_NAME} "${_LIB_VERSION}") endforeach() -#if(DEFINED ENV{KNOWHERE_BOOST_URL}) -# set(BOOST_SOURCE_URL "$ENV{KNOWHERE_BOOST_URL}") -#else() -# string(REPLACE "." "_" BOOST_VERSION_UNDERSCORES ${BOOST_VERSION}) -# set(BOOST_SOURCE_URL -# "https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz" -# ) -#endif() - if(DEFINED ENV{KNOWHERE_FAISS_URL}) set(FAISS_SOURCE_URL "$ENV{KNOWHERE_FAISS_URL}") else() set(FAISS_SOURCE_URL "http://192.168.1.105:6060/jinhai/faiss/-/archive/${FAISS_VERSION}/faiss-${FAISS_VERSION}.tar.gz") -# set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz") -# set(FAISS_SOURCE_URL "${CMAKE_SOURCE_DIR}/thirdparty/faiss-1.5.3") message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}") endif() # set(FAISS_MD5 "a589663865a8558205533c8ac414278c") @@ -301,25 +277,14 @@ set(ARROW_PREFIX "${CORE_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp") macro(build_arrow) message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source") -# set(ARROW_PREFIX "${CORE_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp") set(ARROW_STATIC_LIB_NAME arrow) -# set(PARQUET_STATIC_LIB_NAME parquet) - # set(ARROW_CUDA_STATIC_LIB_NAME arrow_cuda) - set(ARROW_STATIC_LIB + set(ARROW_STATIC_LIB "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) -# set(PARQUET_STATIC_LIB -# "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${PARQUET_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" -# ) - # set(ARROW_CUDA_STATIC_LIB - # "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_CUDA_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" - # ) - set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include") + set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include") set(ARROW_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - # "-DARROW_THRIFT_URL=${THRIFT_SOURCE_URL}" - #"env ARROW_THRIFT_URL=${THRIFT_SOURCE_URL}" -DARROW_BUILD_STATIC=ON -DARROW_BUILD_SHARED=OFF -DARROW_PARQUET=OFF @@ -329,8 +294,7 @@ macro(build_arrow) -DCMAKE_BUILD_TYPE=Release -DARROW_DEPENDENCY_SOURCE=BUNDLED) #Build all arrow dependencies from source instead of calling find_package first - # set($ENV{ARROW_THRIFT_URL} ${THRIFT_SOURCE_URL}) - + if(USE_JFROG_CACHE STREQUAL "ON") execute_process(COMMAND sh -c "git ls-remote --heads --tags ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID) if(${ARROW_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+") @@ -403,8 +367,7 @@ macro(build_arrow) set_target_properties(arrow PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") - # INTERFACE_LINK_LIBRARIES thrift) - add_dependencies(arrow arrow_ep) + add_dependencies(arrow arrow_ep) set(JEMALLOC_PREFIX "${CORE_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep") @@ -423,185 +386,6 @@ if(KNOWHERE_WITH_ARROW AND NOT TARGET arrow_ep) include_directories(SYSTEM ${ARROW_INCLUDE_DIR}) endif() -# ---------------------------------------------------------------------- -# Add Boost dependencies (code adapted from Apache Kudu (incubating)) - -#set(Boost_USE_MULTITHREADED ON) -#if(MSVC AND KNOWHERE_USE_STATIC_CRT) -# set(Boost_USE_STATIC_RUNTIME ON) -#endif() -#set(Boost_ADDITIONAL_VERSIONS -# "1.70.0" -# "1.70" -# "1.69.0" -# "1.69" -# "1.68.0" -# "1.68" -# "1.67.0" -# "1.67" -# "1.66.0" -# "1.66" -# "1.65.0" -# "1.65" -# "1.64.0" -# "1.64" -# "1.63.0" -# "1.63" -# "1.62.0" -# "1.61" -# "1.61.0" -# "1.62" -# "1.60.0" -# "1.60") -# -## TODO -#if(KNOWHERE_BOOST_VENDORED) -## system thread serialization wserialization regex -# set(BOOST_PREFIX "${CORE_BINARY_DIR}/boost_ep-prefix/src/boost_ep") -# set(BOOST_LIB_DIR "${BOOST_PREFIX}/stage/lib") -# set(BOOST_BUILD_LINK "static") -# set(BOOST_STATIC_SYSTEM_LIBRARY -# "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}" -# ) -# set(BOOST_STATIC_FILESYSTEM_LIBRARY -# "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}" -# ) -# set(BOOST_STATIC_SERIALIZATION_LIBRARY -# "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_serialization${CMAKE_STATIC_LIBRARY_SUFFIX}" -# ) -# set(BOOST_STATIC_WSERIALIZATION_LIBRARY -# "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_wserialization${CMAKE_STATIC_LIBRARY_SUFFIX}" -# ) -# set(BOOST_STATIC_REGEX_LIBRARY -# "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}" -# ) -# set(BOOST_STATIC_THREAD_LIBRARY -# "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_thread${CMAKE_STATIC_LIBRARY_SUFFIX}" -# ) -# set(BOOST_SYSTEM_LIBRARY boost_system_static) -# set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static) -# set(BOOST_SERIALIZATION_LIBRARY boost_serialization_static) -# set(BOOST_WSERIALIZATION_LIBRARY boost_wserialization_static) -# set(BOOST_REGEX_LIBRARY boost_regex_static) -# set(BOOST_THREAD_LIBRARY boost_thread_static) -# -# if(KNOWHERE_BOOST_HEADER_ONLY) -# set(BOOST_BUILD_PRODUCTS) -# set(BOOST_CONFIGURE_COMMAND "") -# set(BOOST_BUILD_COMMAND "") -# else() -# set(BOOST_BUILD_PRODUCTS ${BOOST_STATIC_SYSTEM_LIBRARY} -# ${BOOST_STATIC_FILESYSTEM_LIBRARY} ${BOOST_STATIC_SERIALIZATION_LIBRARY} -# ${BOOST_STATIC_WSERIALIZATION_LIBRARY} ${BOOST_STATIC_REGEX_LIBRARY} -# ${BOOST_STATIC_THREAD_LIBRARY}) -# set(BOOST_CONFIGURE_COMMAND "./bootstrap.sh" "--prefix=${BOOST_PREFIX}" -# "--with-libraries=filesystem,serialization,wserialization,system,thread,regex") -# if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") -# set(BOOST_BUILD_VARIANT "debug") -# else() -# set(BOOST_BUILD_VARIANT "release") -# endif() -# set(BOOST_BUILD_COMMAND -# "./b2" -# "link=${BOOST_BUILD_LINK}" -# "variant=${BOOST_BUILD_VARIANT}" -# "cxxflags=-fPIC") -# -# add_thirdparty_lib(boost_system STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}") -# -# add_thirdparty_lib(boost_filesystem STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}") -# -# add_thirdparty_lib(boost_serialization STATIC_LIB "${BOOST_STATIC_SERIALIZATION_LIBRARY}") -# -# add_thirdparty_lib(boost_wserialization STATIC_LIB "${BOOST_STATIC_WSERIALIZATION_LIBRARY}") -# -# add_thirdparty_lib(boost_regex STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}") -# -# add_thirdparty_lib(boost_thread STATIC_LIB "${BOOST_STATIC_THREAD_LIBRARY}") -# -# set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY} ${BOOST_SERIALIZATION_LIBRARY} -# ${BOOST_WSERIALIZATION_LIBRARY} ${BOOST_REGEX_LIBRARY} ${BOOST_THREAD_LIBRARY}) -# endif() -# externalproject_add(boost_ep -# URL -# ${BOOST_SOURCE_URL} -# BUILD_BYPRODUCTS -# ${BOOST_BUILD_PRODUCTS} -# BUILD_IN_SOURCE -# 1 -# CONFIGURE_COMMAND -# ${BOOST_CONFIGURE_COMMAND} -# BUILD_COMMAND -# ${BOOST_BUILD_COMMAND} -# INSTALL_COMMAND -# "" -# ${EP_LOG_OPTIONS}) -# set(Boost_INCLUDE_DIR "${BOOST_PREFIX}") -# set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}") -# add_dependencies(boost_system_static boost_ep) -# add_dependencies(boost_filesystem_static boost_ep) -# add_dependencies(boost_serialization_static boost_ep) -# add_dependencies(boost_wserialization_static boost_ep) -# add_dependencies(boost_regex_static boost_ep) -# add_dependencies(boost_thread_static boost_ep) -# -##else() -## if(MSVC) -## # disable autolinking in boost -## add_definitions(-DBOOST_ALL_NO_LIB) -## endif() -# -## if(DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT) -## # In older versions of CMake (such as 3.2), the system paths for Boost will -## # In older versions of CMake (such as 3.2), the system paths for Boost will -## # be looked in first even if we set $BOOST_ROOT or pass -DBOOST_ROOT -## set(Boost_NO_SYSTEM_PATHS ON) -## endif() -# -## if(KNOWHERE_BOOST_USE_SHARED) -## # Find shared Boost libraries. -## set(Boost_USE_STATIC_LIBS OFF) -## set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS}) -## set(BUILD_SHARED_LIBS ON) -## -## if(MSVC) -## # force all boost libraries to dynamic link -## add_definitions(-DBOOST_ALL_DYN_LINK) -## endif() -## -## if(KNOWHERE_BOOST_HEADER_ONLY) -## find_package(Boost REQUIRED) -## else() -## find_package(Boost COMPONENTS serialization system filesystem REQUIRED) -## set(BOOST_SYSTEM_LIBRARY Boost::system) -## set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) -## set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) -## set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) -## endif() -## set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP}) -## unset(BUILD_SHARED_LIBS_KEEP) -## else() -## # Find static boost headers and libs -## # TODO Differentiate here between release and debug builds -## set(Boost_USE_STATIC_LIBS ON) -## if(KNOWHERE_BOOST_HEADER_ONLY) -## find_package(Boost REQUIRED) -## else() -## find_package(Boost COMPONENTS serialization system filesystem REQUIRED) -## set(BOOST_SYSTEM_LIBRARY Boost::system) -## set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) -## set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) -## set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) -## endif() -## endif() -#endif() -# -##message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR}) -##message(STATUS "Boost libraries: " ${Boost_LIBRARIES}) -# -#include_directories(SYSTEM ${Boost_INCLUDE_DIR}) -#link_directories(SYSTEM ${BOOST_LIB_DIR}) - # ---------------------------------------------------------------------- # OpenBLAS @@ -679,14 +463,6 @@ macro(build_openblas) add_dependencies(openblas openblas_ep) endmacro() -#if(KNOWHERE_WITH_OPENBLAS) -# resolve_dependency(OpenBLAS) -# -# get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) -# include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") -# link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib) -#endif() - # ---------------------------------------------------------------------- # LAPACK @@ -757,14 +533,6 @@ macro(build_lapack) add_dependencies(lapack lapack_ep) endmacro() -#if(KNOWHERE_WITH_LAPACK) -# resolve_dependency(LAPACK) -# -# get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) -# include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") -# link_directories(SYSTEM ${LAPACK_PREFIX}/lib) -#endif() - # ---------------------------------------------------------------------- # Google gtest @@ -873,7 +641,6 @@ macro(build_gtest) endmacro() if (KNOWHERE_BUILD_TESTS AND NOT TARGET googletest_ep) - #message(STATUS "Resolving gtest dependency") resolve_dependency(GTest) if(NOT GTEST_VENDORED) @@ -895,12 +662,7 @@ macro(build_faiss) set(FAISS_STATIC_LIB "${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}") - # add_custom_target(faiss_dependencies) - # add_dependencies(faiss_dependencies openblas_ep) - # add_dependencies(faiss_dependencies openblas) - # get_target_property(FAISS_OPENBLAS_LIB_DIR openblas IMPORTED_LOCATION) - # get_filename_component(FAISS_OPENBLAS_LIB "${FAISS_OPENBLAS_LIB_DIR}" DIRECTORY) - + set(FAISS_CONFIGURE_ARGS "--prefix=${FAISS_PREFIX}" "CFLAGS=${EP_C_FLAGS}" @@ -908,13 +670,7 @@ macro(build_faiss) "LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib -lopenblas -llapack" --without-python) - # if(OPENBLAS_STATIC_LIB) - # set(OPENBLAS_LIBRARY ${OPENBLAS_STATIC_LIB}) - # else() - # set(OPENBLAS_LIBRARY ${OPENBLAS_SHARED_LIB}) - # endif() - # set(FAISS_DEPENDENCIES ${FAISS_DEPENDENCIES} ${OPENBLAS_LIBRARY}) - + if(${KNOWHERE_WITH_FAISS_GPU_VERSION} STREQUAL "ON") set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}" @@ -925,9 +681,7 @@ macro(build_faiss) endif() if(USE_JFROG_CACHE STREQUAL "ON") -# Check_Last_Modify("${CMAKE_SOURCE_DIR}/thirdparty/faiss_cache_check_lists.txt" "${CMAKE_SOURCE_DIR}" FAISS_LAST_MODIFIED_COMMIT_ID) string(MD5 FAISS_COMBINE_MD5 "${FAISS_MD5}${LAPACK_MD5}${OPENBLAS_MD5}") - # string(MD5 FAISS_COMBINE_MD5 "${FAISS_LAST_MODIFIED_COMMIT_ID}${LAPACK_MD5}${OPENBLAS_MD5}") set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMBINE_MD5}.tar.gz") set(FAISS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${FAISS_CACHE_PACKAGE_NAME}") set(FAISS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${FAISS_CACHE_PACKAGE_NAME}") @@ -991,10 +745,6 @@ macro(build_faiss) INTERFACE_LINK_LIBRARIES "openblas;lapack" ) add_dependencies(faiss faiss_ep) - #add_dependencies(faiss openblas_ep) - #add_dependencies(faiss lapack_ep) - #target_link_libraries(faiss ${OPENBLAS_PREFIX}/lib) - #target_link_libraries(faiss ${LAPACK_PREFIX}/lib) endmacro() @@ -1013,8 +763,5 @@ if(KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep) resolve_dependency(FAISS) get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM "${FAISS_INCLUDE_DIR}") -# include_directories(SYSTEM "${CORE_BINARY_DIR}/faiss_ep-prefix/src/") -# link_directories(SYSTEM ${FAISS_PREFIX}/) link_directories(SYSTEM ${FAISS_PREFIX}/lib/) -# link_directories(SYSTEM ${FAISS_PREFIX}/gpu/) endif() diff --git a/cpp/src/core/src/CMakeLists.txt b/cpp/src/core/src/CMakeLists.txt index a9ee7cd3db..a51a62b44a 100644 --- a/cpp/src/core/src/CMakeLists.txt +++ b/cpp/src/core/src/CMakeLists.txt @@ -24,8 +24,6 @@ file(GLOB SRC_FILES ${SPTAG_SOURCE_DIR}/AnnService/src/Core/KDT/*.cpp ${SPTAG_SOURCE_DIR}/AnnService/src/Helper/*.cpp) -#add_library(SPTAGLib SHARED ${SRC_FILES} ${HDR_FILES}) -#target_link_libraries(SPTAGLib ${TBB_LIBRARIES}) if(NOT TARGET SPTAGLibStatic) add_library(SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES}) endif() @@ -53,7 +51,6 @@ set(index_srcs ) set(depend_libs -# libtcmalloc.a SPTAGLibStatic ${TBB_LIBRARIES} faiss @@ -75,25 +72,12 @@ if(NOT TARGET knowhere) ${index_srcs} ) endif() -#target_compile_options(knowhere PUBLIC "-fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free") target_link_libraries( knowhere ${depend_libs} ) -#add_library( -# knowhereS SHARED -# ${external_srcs} -# ${index_srcs} -#) -# -#target_link_libraries( -# knowhereS -## ${TBB_LIBRARIES} -# ${depend_libs} -#) - INSTALL(TARGETS knowhere SPTAGLibStatic @@ -102,7 +86,6 @@ INSTALL(TARGETS INSTALL(FILES ${ARROW_STATIC_LIB} -# ${PARQUET_STATIC_LIB} ${ARROW_PREFIX}/lib/libjemalloc_pic.a ${FAISS_STATIC_LIB} ${LAPACK_STATIC_LIB} @@ -117,11 +100,9 @@ INSTALL(FILES ${OPENBLAS_REAL_STATIC_LIB} ) INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so.2 -# RENAME "libtbb.so.2" DESTINATION lib ) INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so -# RENAME "libtbb.so" DESTINATION lib ) @@ -138,17 +119,12 @@ set(CORE_INCLUDE_DIRS ) set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) -#include_directories(SYSTEM ${CORE_SOURCE_DIR}/include) -#include_directories(SYSTEM ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) -#include_directories(SYSTEM ${CORE_SOURCE_DIR}/thirdparty/tbb/include) -#include_directories(SYSTEM ${SPTAG_SOURCE_DIR}/AnnService/inc) INSTALL(DIRECTORY ${CORE_SOURCE_DIR}/include/knowhere ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext ${ARROW_INCLUDE_DIR}/arrow -# ${ARROW_INCLUDE_DIR}/parquet ${FAISS_PREFIX}/include/faiss ${OPENBLAS_INCLUDE_DIR}/ ${CORE_SOURCE_DIR}/thirdparty/tbb/include/tbb diff --git a/cpp/src/core/test/CMakeLists.txt b/cpp/src/core/test/CMakeLists.txt index d2629a4570..ba3f0d65bd 100644 --- a/cpp/src/core/test/CMakeLists.txt +++ b/cpp/src/core/test/CMakeLists.txt @@ -24,7 +24,6 @@ set(basic_libs # -#<<<<<<<<<< set(ivf_srcs ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp @@ -39,10 +38,8 @@ if(NOT TARGET test_ivf) add_executable(test_ivf test_ivf.cpp ${ivf_srcs}) endif() target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs}) -#<<<<<<<<<< # -#<<<<<<<<<< set(idmap_srcs ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp @@ -57,11 +54,8 @@ if(NOT TARGET test_idmap) add_executable(test_idmap test_idmap.cpp ${idmap_srcs}) endif() target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) -#<<<<<<<<<< - # -#<<<<<<<<<< set(kdt_srcs ${CORE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp @@ -79,8 +73,6 @@ endif() target_link_libraries(test_kdt SPTAGLibStatic ${depend_libs} ${unittest_libs} ${basic_libs}) -#<<<<<<<<<< - add_subdirectory(faiss_ori) add_subdirectory(test_nsg) diff --git a/cpp/src/core/test/faiss_ori/CMakeLists.txt b/cpp/src/core/test/faiss_ori/CMakeLists.txt index 1209ec00b8..2f46d25d40 100644 --- a/cpp/src/core/test/faiss_ori/CMakeLists.txt +++ b/cpp/src/core/test/faiss_ori/CMakeLists.txt @@ -20,9 +20,7 @@ set(basic_libs # -#<<<<<<<<<< if(NOT TARGET test_gpu) add_executable(test_gpu gpuresource_test.cpp) endif() target_link_libraries(test_gpu ${depend_libs} ${unittest_libs} ${basic_libs}) -#<<<<<<<<<< diff --git a/cpp/src/core/test/test_nsg/CMakeLists.txt b/cpp/src/core/test/test_nsg/CMakeLists.txt index 7ebd534379..642c1a3d25 100644 --- a/cpp/src/core/test/test_nsg/CMakeLists.txt +++ b/cpp/src/core/test/test_nsg/CMakeLists.txt @@ -2,7 +2,6 @@ include_directories(/usr/local/include/gperftools) link_directories(/usr/local/lib) -#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free") add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO) find_package(OpenMP) @@ -15,15 +14,9 @@ endif () message(${OpenMP_CXX_FLAGS}) include_directories(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg) -#include_directories(/home/zilliz/opt/app/pyenv/versions/3.6.8/include/python3.6m) -#include_directories(/home/zilliz/opt/app/pyenv/versions/3.6.8/envs/megasearch_testframework_dev/lib/python3.6/site-packages/numpy/core/include) aux_source_directory(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src) -#add_library(nsg_raw SHARED ${nsg_src}) -#target_link_libraries(nsg_raw -# gomp) - set(interface_src ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp @@ -43,10 +36,6 @@ if(NOT TARGET test_nsg) ${nsg_src} ) endif() -#target_link_libraries(test_nsg -# # libprofiler.so -## -ltcmalloc -## gomp -# ) + target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs}) ############################## \ No newline at end of file diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index c9ce44d82d..d251dce01a 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -6,8 +6,6 @@ link_directories( "${CMAKE_BINARY_DIR}/lib" ) -#include_directories(${CMAKE_SOURCE_DIR}/src/core/include) -#include_directories(${CMAKE_SOURCE_DIR}/src/core/thirdparty/SPTAG/AnnService) aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) @@ -40,16 +38,13 @@ set(unittest_libs cudart ) -#message(STATUS "unittest core dir: ${CMAKE_CURRENT_BINARY_DIR}/core") -#add_subdirectory(${CMAKE_SOURCE_DIR}/src/core ${CMAKE_CURRENT_BINARY_DIR}/core) foreach(dir ${CORE_INCLUDE_DIRS}) - message("dir = ${dir}") include_directories(${dir}) endforeach() add_subdirectory(server) add_subdirectory(db) -#add_subdirectory(knowhere) +add_subdirectory(knowhere) add_subdirectory(metrics) -#add_subdirectory(scheduler) -#add_subdirectory(storage) \ No newline at end of file +add_subdirectory(scheduler) +add_subdirectory(storage) \ No newline at end of file diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 931c8b01a6..eae0b9bb83 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -35,8 +35,6 @@ link_directories("/usr/local/cuda/lib64") include_directories(/usr/include/mysql) -#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY) - set(db_test_src ${config_files} ${cache_srcs} @@ -69,25 +67,10 @@ set(db_libs set(knowhere_libs knowhere -# SPTAGLibStatic -# arrow -# jemalloc_pic -# faiss -# openblas -# lapack -# tbb cudart cublas ) -#if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") -# set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS}) -#else() -# set(db_libs ${db_libs} -# lapack -# openblas) -#endif() - target_link_libraries(db_test ${db_libs} ${knowhere_libs} ${unittest_libs}) install(TARGETS db_test DESTINATION unittest) diff --git a/cpp/unittest/knowhere/CMakeLists.txt b/cpp/unittest/knowhere/CMakeLists.txt index 452b748fb8..089ff992fa 100644 --- a/cpp/unittest/knowhere/CMakeLists.txt +++ b/cpp/unittest/knowhere/CMakeLists.txt @@ -8,13 +8,6 @@ set(helper set(knowhere_libs knowhere -# SPTAGLibStatic -# arrow -# jemalloc_pic -# faiss -# openblas -# lapack -# tbb cudart cublas ) diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index fcaef4c7ec..1f0f5bd023 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -1,13 +1,3 @@ - - -#set(CURL_LIBRARY "-lcurl") -#find_package(CURL REQUIRED) -#INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIR}) -# -#INCLUDE_DIRECTORIES(~/development/lib/usr/local/include) -#LINK_DIRECTORIES(~/development/lib/usr/local/lib) - - include_directories(../../src) @@ -72,13 +62,6 @@ add_executable(metrics_test ${count_test_src} ${require_files} ) set(knowhere_libs knowhere -# SPTAGLibStatic -# arrow -# jemalloc_pic -# faiss -# openblas -# lapack -# tbb ) target_link_libraries(metrics_test @@ -96,12 +79,5 @@ target_link_libraries(metrics_test mysqlpp ${unittest_libs} ) -#if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") -# target_link_libraries(metrics_test ${MKL_LIBS} ${MKL_LIBS}) -#else() -# target_link_libraries(metrics_test -# lapack -# openblas) -#endif() install(TARGETS metrics_test DESTINATION unittest) \ No newline at end of file diff --git a/cpp/unittest/scheduler/CMakeLists.txt b/cpp/unittest/scheduler/CMakeLists.txt index 16599077fe..feb7d6178f 100644 --- a/cpp/unittest/scheduler/CMakeLists.txt +++ b/cpp/unittest/scheduler/CMakeLists.txt @@ -33,8 +33,6 @@ link_directories("/usr/local/cuda/lib64") include_directories(/usr/include/mysql) -#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY) - set(scheduler_test_src ${unittest_srcs} ${test_srcs} @@ -69,25 +67,10 @@ set(scheduler_libs set(knowhere_libs knowhere -# SPTAGLibStatic -# arrow -# jemalloc_pic -# faiss -# openblas -# lapack -# tbb cudart cublas ) -#if (${BUILD_FAISS_WITH_MKL} STREQUAL "ON") -# set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS}) -#else () -# set(db_libs ${db_libs} -# lapack -# openblas) -#endif () - target_link_libraries(scheduler_test ${scheduler_libs} ${knowhere_libs} ${unittest_libs}) diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index d8b16267e8..3dfca57994 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -34,9 +34,6 @@ cuda_add_executable(server_test set(require_libs knowhere -# faiss -# openblas -# lapack stdc++ cudart cublas @@ -51,14 +48,6 @@ set(require_libs pthread ) -#if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") -# set(require_libs ${require_libs} ${MKL_LIBS} ${MKL_LIBS}) -#else() -# set(require_libs ${require_libs} -# lapack -# openblas) -#endif() - target_link_libraries(server_test ${require_libs} ${cuda_library} From 01c629336332983a6eeb2cb469e28b1e41ab09b3 Mon Sep 17 00:00:00 2001 From: zhiru Date: Fri, 6 Sep 2019 16:54:46 +0800 Subject: [PATCH 3/3] update Former-commit-id: 9a00d0a0d8c93bbc8b2b8ae744fc5729b9d03865 --- cpp/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index ea42c33c59..8d61f970b3 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -91,13 +91,13 @@ Please mark all change in change log and use the ticket from JIRA. - MS-482 - Change search stream transport to unary in grpc - MS-487 - Define metric type in CreateTable - MS-488 - Improve code format in scheduler +- MS-495 - cmake: integrated knowhere ## New Feature - MS-343 - Implement ResourceMgr - MS-338 - NewAPI: refine code to support CreateIndex - MS-339 - NewAPI: refine code to support DropIndex - MS-340 - NewAPI: implement DescribeIndex -- MS-495 - cmake: integrated knowhere ## Task - MS-297 - disable mysql unit test