milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'cmake' into 'branch-0.4.0' 

MS-495: cmake: integrated knowhere

See merge request megasearch/milvus!498

Former-commit-id: e1b45640f991398147279dbf408725d8bdbb89a7
pull/191/head
jinhai 2019-09-06 16:58:19 +08:00
parent be75b7b81a 01c6293363
commit d6c7909b24
18 changed files with 175 additions and 523 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cpp/CHANGELOG.md
View File

@ -91,6 +91,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-482 - Change search stream transport to unary in grpc - MS-482 - Change search stream transport to unary in grpc
- MS-487 - Define metric type in CreateTable - MS-487 - Define metric type in CreateTable
- MS-488 - Improve code format in scheduler - MS-488 - Improve code format in scheduler
- MS-495 - cmake: integrated knowhere
## New Feature ## New Feature
- MS-343 - Implement ResourceMgr - MS-343 - Implement ResourceMgr

25
cpp/CMakeLists.txt
View File

@ -31,6 +31,7 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release")
set(BUILD_TYPE "release") set(BUILD_TYPE "release")
else() else()
set(BUILD_TYPE "debug") set(BUILD_TYPE "debug")
SET(CMAKE_VERBOSE_MAKEFILE on)
endif() endif()
message(STATUS "Build type = ${BUILD_TYPE}") message(STATUS "Build type = ${BUILD_TYPE}")
@ -70,21 +71,21 @@ set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR})
find_package(CUDA) find_package(CUDA)
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda") set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda")
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
message("CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}") message(STATUS "CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}")
message("CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}") message(STATUS "CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}")
set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED on) set(CMAKE_CXX_STANDARD_REQUIRED on)
if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
message("building milvus_engine on x86 architecture") message(STATUS "building milvus_engine on x86 architecture")
set(MILVUS_BUILD_ARCH x86_64) set(MILVUS_BUILD_ARCH x86_64)
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)") elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)")
message("building milvus_engine on ppc architecture") message(STATUS "building milvus_engine on ppc architecture")
set(MILVUS_BUILD_ARCH ppc64le) set(MILVUS_BUILD_ARCH ppc64le)
else() else()
message("unknown processor type") message(WARNING "unknown processor type")
message("CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}") message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
set(MILVUS_BUILD_ARCH unknown) set(MILVUS_BUILD_ARCH unknown)
endif() endif()
@ -94,6 +95,8 @@ else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp")
endif() endif()
set(ALLOW_DUPLICATE_CUSTOM_TARGETS TRUE)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
include(ExternalProject) include(ExternalProject)
@ -104,14 +107,6 @@ include(ThirdPartyPackages)
include_directories(${MILVUS_SOURCE_DIR}) include_directories(${MILVUS_SOURCE_DIR})
link_directories(${MILVUS_BINARY_DIR}) link_directories(${MILVUS_BINARY_DIR})
if (NOT DEFINED KNOWHERE_BUILD_DIR)
message(FATAL_ERROR "You must set environment variable KNOWHERE_BUILD_DIR")
endif()
message(STATUS "Build with ${KNOWHERE_BUILD_DIR}")
include_directories(${KNOWHERE_BUILD_DIR}/include)
include_directories(${KNOWHERE_BUILD_DIR}/include/SPTAG/AnnService)
link_directories(${KNOWHERE_BUILD_DIR}/lib)
## Following should be check ## Following should be check
set(MILVUS_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include) set(MILVUS_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include)
@ -119,7 +114,7 @@ set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
add_compile_definitions(PROFILER=${PROFILER}) add_compile_definitions(PROFILER=${PROFILER})
message("MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}") message(STATUS "MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}")
if (MILVUS_ENABLE_PROFILING STREQUAL "ON") if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
ADD_DEFINITIONS(-DMILVUS_ENABLE_PROFILING) ADD_DEFINITIONS(-DMILVUS_ENABLE_PROFILING)
endif() endif()

6
cpp/build.sh
View File

@ -83,9 +83,9 @@ if [[ ! -d cmake_build ]]; then
MAKE_CLEAN="ON" MAKE_CLEAN="ON"
fi fi
pushd `pwd`/src/core #pushd `pwd`/src/core
./build.sh ${KNOWHERE_OPTIONS} #./build.sh ${KNOWHERE_OPTIONS}
popd #popd
cd cmake_build cd cmake_build

33
cpp/src/CMakeLists.txt
View File

@ -4,6 +4,13 @@
# Proprietary and confidential. # Proprietary and confidential.
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
add_subdirectory(core)
set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
foreach(dir ${CORE_INCLUDE_DIRS})
include_directories(${dir})
endforeach()
aux_source_directory(cache cache_files) aux_source_directory(cache cache_files)
aux_source_directory(config config_files) aux_source_directory(config config_files)
aux_source_directory(server server_files) aux_source_directory(server server_files)
@ -87,12 +94,6 @@ set(client_grpc_lib
set(third_party_libs set(third_party_libs
knowhere knowhere
SPTAGLibStatic
arrow
jemalloc_pic
faiss
openblas
lapack
easyloggingpp easyloggingpp
sqlite sqlite
${client_grpc_lib} ${client_grpc_lib}
@ -115,15 +116,6 @@ set(third_party_libs
cudart cudart
) )
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(third_party_libs ${third_party_libs}
${MKL_LIBS})
else()
set(third_party_libs ${third_party_libs}
lapack
openblas)
endif()
if (MILVUS_ENABLE_PROFILING STREQUAL "ON") if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
set(third_party_libs ${third_party_libs} set(third_party_libs ${third_party_libs}
gperftools gperftools
@ -171,13 +163,6 @@ set(server_libs
set(knowhere_libs set(knowhere_libs
knowhere knowhere
SPTAGLibStatic
arrow
jemalloc_pic
faiss
openblas
lapack
tbb
) )
add_executable(milvus_server add_executable(milvus_server
@ -195,8 +180,8 @@ target_link_libraries(milvus_server ${server_libs} ${knowhere_libs} ${third_part
install(TARGETS milvus_server DESTINATION bin) install(TARGETS milvus_server DESTINATION bin)
install(FILES install(FILES
${KNOWHERE_BUILD_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_SOURCE_DIR}/src/core/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}
${KNOWHERE_BUILD_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}.2 ${CMAKE_SOURCE_DIR}/src/core/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}.2
${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}
${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3 ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3
${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4 ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4

26
cpp/src/core/CMakeLists.txt
View File

@ -5,6 +5,7 @@
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
cmake_minimum_required(VERSION 3.14) cmake_minimum_required(VERSION 3.14)
message(STATUS "---------------core--------------")
message(STATUS "Building using CMake version: ${CMAKE_VERSION}") message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
set(KNOWHERE_VERSION "0.1.0") set(KNOWHERE_VERSION "0.1.0")
@ -38,18 +39,16 @@ endif()
MESSAGE(STATUS "CMAKE_CXX_FLAGS" ${CMAKE_CXX_FLAGS}) MESSAGE(STATUS "CMAKE_CXX_FLAGS" ${CMAKE_CXX_FLAGS})
find_package(CUDA) find_package(CUDA)
#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda")
#set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
message("building milvus_engine on x86 architecture") message(STATUS "building milvus_engine on x86 architecture")
set(KNOWHERE_BUILD_ARCH x86_64) set(KNOWHERE_BUILD_ARCH x86_64)
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)") elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)")
message("building milvus_engine on ppc architecture") message(STATUS "building milvus_engine on ppc architecture")
set(KNOWHERE_BUILD_ARCH ppc64le) set(KNOWHERE_BUILD_ARCH ppc64le)
else() else()
message("unknown processor type") message(WARNING "unknown processor type")
message("CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}") message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
set(KNOWHERE_BUILD_ARCH unknown) set(KNOWHERE_BUILD_ARCH unknown)
endif() endif()
@ -60,14 +59,21 @@ else()
endif() endif()
message(STATUS "Build type = ${BUILD_TYPE}") message(STATUS "Build type = ${BUILD_TYPE}")
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") set(CORE_SOURCE_DIR ${PROJECT_SOURCE_DIR})
set(CORE_BINARY_DIR ${PROJECT_BINARY_DIR})
message(STATUS "Core source dir: ${PROJECT_SOURCE_DIR}")
message(STATUS "Core binary dir: ${PROJECT_BINARY_DIR}")
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CORE_SOURCE_DIR}/cmake")
include(ExternalProject) include(ExternalProject)
include(DefineOptions) include(DefineOptionsCore)
include(BuildUtils) include(BuildUtilsCore)
include(ThirdPartyPackages) include(ThirdPartyPackagesCore)
add_subdirectory(src) add_subdirectory(src)
set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
if(BUILD_UNIT_TEST STREQUAL "ON") if(BUILD_UNIT_TEST STREQUAL "ON")
add_subdirectory(test) add_subdirectory(test)
endif() endif()

2
cpp/src/core/cmake/BuildUtils.cmake → cpp/src/core/cmake/BuildUtilsCore.cmake
View File

@ -75,7 +75,7 @@ function(ExternalProject_Use_Cache project_name package_file install_path)
"Extracting ${package_file} to ${install_path}" "Extracting ${package_file} to ${install_path}"
COMMAND COMMAND
${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path} ${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} WORKING_DIRECTORY ${CORE_BINARY_DIR}
) )
ExternalProject_Add_StepTargets(${project_name} extract) ExternalProject_Add_StepTargets(${project_name} extract)

2
cpp/src/core/cmake/DefineOptions.cmake → cpp/src/core/cmake/DefineOptionsCore.cmake
View File

@ -104,7 +104,7 @@ macro(config_summary)
message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}") message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}")
if(${CMAKE_EXPORT_COMPILE_COMMANDS}) if(${CMAKE_EXPORT_COMPILE_COMMANDS})
message( message(
STATUS " Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json") STATUS " Compile commands: ${CORE_BINARY_DIR}/compile_commands.json")
endif() endif()
foreach(category ${KNOWHERE_OPTION_CATEGORIES}) foreach(category ${KNOWHERE_OPTION_CATEGORIES})

306
cpp/src/core/cmake/ThirdPartyPackages.cmake → cpp/src/core/cmake/ThirdPartyPackagesCore.cmake
View File

@ -6,7 +6,6 @@
# "License"); you may not use this file except in compliance # "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at # with the License. You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, # Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an # software distributed under the License is distributed on an
@ -18,7 +17,6 @@
set(KNOWHERE_THIRDPARTY_DEPENDENCIES set(KNOWHERE_THIRDPARTY_DEPENDENCIES
ARROW ARROW
BOOST
FAISS FAISS
GTest GTest
LAPACK LAPACK
@ -53,11 +51,8 @@ endmacro()
macro(resolve_dependency DEPENDENCY_NAME) macro(resolve_dependency DEPENDENCY_NAME)
if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
#message(STATUS "Finding ${DEPENDENCY_NAME} package") #message(STATUS "Finding ${DEPENDENCY_NAME} package")
# find_package(${DEPENDENCY_NAME} QUIET)
# if (NOT ${DEPENDENCY_NAME}_FOUND)
#message(STATUS "${DEPENDENCY_NAME} package not found") #message(STATUS "${DEPENDENCY_NAME} package not found")
build_dependency(${DEPENDENCY_NAME}) build_dependency(${DEPENDENCY_NAME})
# endif ()
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED")
build_dependency(${DEPENDENCY_NAME}) build_dependency(${DEPENDENCY_NAME})
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM")
@ -120,7 +115,7 @@ endif (UNIX)
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
# thirdparty directory # thirdparty directory
set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty") set(THIRDPARTY_DIR "${CORE_SOURCE_DIR}/thirdparty")
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
# JFrog # JFrog
@ -202,14 +197,6 @@ endif()
set(MAKE_BUILD_ARGS "-j8") set(MAKE_BUILD_ARGS "-j8")
## Using make -j in sub-make is fragile
## see discussion https://github.com/apache/KNOWHERE/pull/2779
#if(${CMAKE_GENERATOR} MATCHES "Makefiles")
# set(MAKE_BUILD_ARGS "")
#else()
# # limit the maximum number of jobs for ninja
# set(MAKE_BUILD_ARGS "-j4")
#endif()
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
# Find pthreads # Find pthreads
@ -243,22 +230,11 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
set(${_LIB_NAME} "${_LIB_VERSION}") set(${_LIB_NAME} "${_LIB_VERSION}")
endforeach() endforeach()
if(DEFINED ENV{KNOWHERE_BOOST_URL})
set(BOOST_SOURCE_URL "$ENV{KNOWHERE_BOOST_URL}")
else()
string(REPLACE "." "_" BOOST_VERSION_UNDERSCORES ${BOOST_VERSION})
set(BOOST_SOURCE_URL
"https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz"
)
endif()
if(DEFINED ENV{KNOWHERE_FAISS_URL}) if(DEFINED ENV{KNOWHERE_FAISS_URL})
set(FAISS_SOURCE_URL "$ENV{KNOWHERE_FAISS_URL}") set(FAISS_SOURCE_URL "$ENV{KNOWHERE_FAISS_URL}")
else() else()
set(FAISS_SOURCE_URL "http://192.168.1.105:6060/jinhai/faiss/-/archive/${FAISS_VERSION}/faiss-${FAISS_VERSION}.tar.gz") set(FAISS_SOURCE_URL "http://192.168.1.105:6060/jinhai/faiss/-/archive/${FAISS_VERSION}/faiss-${FAISS_VERSION}.tar.gz")
# set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz") message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}")
# set(FAISS_SOURCE_URL "${CMAKE_SOURCE_DIR}/thirdparty/faiss-1.5.3")
message(STATUS ${FAISS_SOURCE_URL})
endif() endif()
# set(FAISS_MD5 "a589663865a8558205533c8ac414278c") # set(FAISS_MD5 "a589663865a8558205533c8ac414278c")
# set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c # set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c
@ -297,28 +273,18 @@ set(OPENBLAS_MD5 "8a110a25b819a4b94e8a9580702b6495")
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
# ARROW # ARROW
set(ARROW_PREFIX "${CORE_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp")
macro(build_arrow) macro(build_arrow)
message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source") message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source")
set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp")
set(ARROW_STATIC_LIB_NAME arrow) set(ARROW_STATIC_LIB_NAME arrow)
# set(PARQUET_STATIC_LIB_NAME parquet) set(ARROW_STATIC_LIB
# set(ARROW_CUDA_STATIC_LIB_NAME arrow_cuda)
set(ARROW_STATIC_LIB
"${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
) )
# set(PARQUET_STATIC_LIB set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include")
# "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${PARQUET_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
# )
# set(ARROW_CUDA_STATIC_LIB
# "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_CUDA_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
# )
set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include")
set(ARROW_CMAKE_ARGS set(ARROW_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS} ${EP_COMMON_CMAKE_ARGS}
# "-DARROW_THRIFT_URL=${THRIFT_SOURCE_URL}"
#"env ARROW_THRIFT_URL=${THRIFT_SOURCE_URL}"
-DARROW_BUILD_STATIC=ON -DARROW_BUILD_STATIC=ON
-DARROW_BUILD_SHARED=OFF -DARROW_BUILD_SHARED=OFF
-DARROW_PARQUET=OFF -DARROW_PARQUET=OFF
@ -328,8 +294,7 @@ macro(build_arrow)
-DCMAKE_BUILD_TYPE=Release -DCMAKE_BUILD_TYPE=Release
-DARROW_DEPENDENCY_SOURCE=BUNDLED) #Build all arrow dependencies from source instead of calling find_package first -DARROW_DEPENDENCY_SOURCE=BUNDLED) #Build all arrow dependencies from source instead of calling find_package first
# set($ENV{ARROW_THRIFT_URL} ${THRIFT_SOURCE_URL})
if(USE_JFROG_CACHE STREQUAL "ON") if(USE_JFROG_CACHE STREQUAL "ON")
execute_process(COMMAND sh -c "git ls-remote --heads --tags ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID) execute_process(COMMAND sh -c "git ls-remote --heads --tags ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID)
if(${ARROW_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+") if(${ARROW_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+")
@ -362,13 +327,13 @@ macro(build_arrow)
"${ARROW_STATIC_LIB}" "${ARROW_STATIC_LIB}"
) )
ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL}) ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL})
else() else()
file(DOWNLOAD ${ARROW_CACHE_URL} ${ARROW_CACHE_PACKAGE_PATH} STATUS status) file(DOWNLOAD ${ARROW_CACHE_URL} ${ARROW_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code) list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${ARROW_CACHE_URL} TO ${ARROW_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") message(STATUS "DOWNLOADING FROM ${ARROW_CACHE_URL} TO ${ARROW_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0) if (status_code EQUAL 0)
ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR})
endif() endif()
endif() endif()
else() else()
@ -402,10 +367,9 @@ macro(build_arrow)
set_target_properties(arrow set_target_properties(arrow
PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}" PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}")
# INTERFACE_LINK_LIBRARIES thrift) add_dependencies(arrow arrow_ep)
add_dependencies(arrow arrow_ep)
set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep") set(JEMALLOC_PREFIX "${CORE_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep")
add_custom_command(TARGET arrow_ep POST_BUILD add_custom_command(TARGET arrow_ep POST_BUILD
COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_PREFIX}/lib/ COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_PREFIX}/lib/
@ -414,7 +378,7 @@ macro(build_arrow)
endmacro() endmacro()
if(KNOWHERE_WITH_ARROW) if(KNOWHERE_WITH_ARROW AND NOT TARGET arrow_ep)
resolve_dependency(ARROW) resolve_dependency(ARROW)
@ -422,191 +386,12 @@ if(KNOWHERE_WITH_ARROW)
include_directories(SYSTEM ${ARROW_INCLUDE_DIR}) include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
endif() endif()
# ----------------------------------------------------------------------
# Add Boost dependencies (code adapted from Apache Kudu (incubating))
set(Boost_USE_MULTITHREADED ON)
if(MSVC AND KNOWHERE_USE_STATIC_CRT)
set(Boost_USE_STATIC_RUNTIME ON)
endif()
set(Boost_ADDITIONAL_VERSIONS
"1.70.0"
"1.70"
"1.69.0"
"1.69"
"1.68.0"
"1.68"
"1.67.0"
"1.67"
"1.66.0"
"1.66"
"1.65.0"
"1.65"
"1.64.0"
"1.64"
"1.63.0"
"1.63"
"1.62.0"
"1.61"
"1.61.0"
"1.62"
"1.60.0"
"1.60")
# TODO
if(KNOWHERE_BOOST_VENDORED)
# system thread serialization wserialization regex
set(BOOST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/boost_ep-prefix/src/boost_ep")
set(BOOST_LIB_DIR "${BOOST_PREFIX}/stage/lib")
set(BOOST_BUILD_LINK "static")
set(BOOST_STATIC_SYSTEM_LIBRARY
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(BOOST_STATIC_FILESYSTEM_LIBRARY
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(BOOST_STATIC_SERIALIZATION_LIBRARY
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_serialization${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(BOOST_STATIC_WSERIALIZATION_LIBRARY
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_wserialization${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(BOOST_STATIC_REGEX_LIBRARY
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(BOOST_STATIC_THREAD_LIBRARY
"${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_thread${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(BOOST_SYSTEM_LIBRARY boost_system_static)
set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
set(BOOST_SERIALIZATION_LIBRARY boost_serialization_static)
set(BOOST_WSERIALIZATION_LIBRARY boost_wserialization_static)
set(BOOST_REGEX_LIBRARY boost_regex_static)
set(BOOST_THREAD_LIBRARY boost_thread_static)
if(KNOWHERE_BOOST_HEADER_ONLY)
set(BOOST_BUILD_PRODUCTS)
set(BOOST_CONFIGURE_COMMAND "")
set(BOOST_BUILD_COMMAND "")
else()
set(BOOST_BUILD_PRODUCTS ${BOOST_STATIC_SYSTEM_LIBRARY}
${BOOST_STATIC_FILESYSTEM_LIBRARY} ${BOOST_STATIC_SERIALIZATION_LIBRARY}
${BOOST_STATIC_WSERIALIZATION_LIBRARY} ${BOOST_STATIC_REGEX_LIBRARY}
${BOOST_STATIC_THREAD_LIBRARY})
set(BOOST_CONFIGURE_COMMAND "./bootstrap.sh" "--prefix=${BOOST_PREFIX}"
"--with-libraries=filesystem,serialization,wserialization,system,thread,regex")
if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
set(BOOST_BUILD_VARIANT "debug")
else()
set(BOOST_BUILD_VARIANT "release")
endif()
set(BOOST_BUILD_COMMAND
"./b2"
"link=${BOOST_BUILD_LINK}"
"variant=${BOOST_BUILD_VARIANT}"
"cxxflags=-fPIC")
add_thirdparty_lib(boost_system STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}")
add_thirdparty_lib(boost_filesystem STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}")
add_thirdparty_lib(boost_serialization STATIC_LIB "${BOOST_STATIC_SERIALIZATION_LIBRARY}")
add_thirdparty_lib(boost_wserialization STATIC_LIB "${BOOST_STATIC_WSERIALIZATION_LIBRARY}")
add_thirdparty_lib(boost_regex STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}")
add_thirdparty_lib(boost_thread STATIC_LIB "${BOOST_STATIC_THREAD_LIBRARY}")
set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY} ${BOOST_SERIALIZATION_LIBRARY}
${BOOST_WSERIALIZATION_LIBRARY} ${BOOST_REGEX_LIBRARY} ${BOOST_THREAD_LIBRARY})
endif()
externalproject_add(boost_ep
URL
${BOOST_SOURCE_URL}
BUILD_BYPRODUCTS
${BOOST_BUILD_PRODUCTS}
BUILD_IN_SOURCE
1
CONFIGURE_COMMAND
${BOOST_CONFIGURE_COMMAND}
BUILD_COMMAND
${BOOST_BUILD_COMMAND}
INSTALL_COMMAND
""
${EP_LOG_OPTIONS})
set(Boost_INCLUDE_DIR "${BOOST_PREFIX}")
set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}")
add_dependencies(boost_system_static boost_ep)
add_dependencies(boost_filesystem_static boost_ep)
add_dependencies(boost_serialization_static boost_ep)
add_dependencies(boost_wserialization_static boost_ep)
add_dependencies(boost_regex_static boost_ep)
add_dependencies(boost_thread_static boost_ep)
#else()
# if(MSVC)
# # disable autolinking in boost
# add_definitions(-DBOOST_ALL_NO_LIB)
# endif()
# if(DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT)
# # In older versions of CMake (such as 3.2), the system paths for Boost will
# # In older versions of CMake (such as 3.2), the system paths for Boost will
# # be looked in first even if we set $BOOST_ROOT or pass -DBOOST_ROOT
# set(Boost_NO_SYSTEM_PATHS ON)
# endif()
# if(KNOWHERE_BOOST_USE_SHARED)
# # Find shared Boost libraries.
# set(Boost_USE_STATIC_LIBS OFF)
# set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS})
# set(BUILD_SHARED_LIBS ON)
#
# if(MSVC)
# # force all boost libraries to dynamic link
# add_definitions(-DBOOST_ALL_DYN_LINK)
# endif()
#
# if(KNOWHERE_BOOST_HEADER_ONLY)
# find_package(Boost REQUIRED)
# else()
# find_package(Boost COMPONENTS serialization system filesystem REQUIRED)
# set(BOOST_SYSTEM_LIBRARY Boost::system)
# set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem)
# set(BOOST_SERIALIZATION_LIBRARY Boost::serialization)
# set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY})
# endif()
# set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP})
# unset(BUILD_SHARED_LIBS_KEEP)
# else()
# # Find static boost headers and libs
# # TODO Differentiate here between release and debug builds
# set(Boost_USE_STATIC_LIBS ON)
# if(KNOWHERE_BOOST_HEADER_ONLY)
# find_package(Boost REQUIRED)
# else()
# find_package(Boost COMPONENTS serialization system filesystem REQUIRED)
# set(BOOST_SYSTEM_LIBRARY Boost::system)
# set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem)
# set(BOOST_SERIALIZATION_LIBRARY Boost::serialization)
# set(KNOWHERE_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY})
# endif()
# endif()
endif()
#message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR})
#message(STATUS "Boost libraries: " ${Boost_LIBRARIES})
include_directories(SYSTEM ${Boost_INCLUDE_DIR})
link_directories(SYSTEM ${BOOST_LIB_DIR})
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
# OpenBLAS # OpenBLAS
macro(build_openblas) macro(build_openblas)
message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source") message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source")
set(OPENBLAS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep") set(OPENBLAS_PREFIX "${CORE_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep")
set(OPENBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include") set(OPENBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include")
set(OPENBLAS_STATIC_LIB set(OPENBLAS_STATIC_LIB
"${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}") "${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}")
@ -639,13 +424,13 @@ macro(build_openblas)
BUILD_BYPRODUCTS BUILD_BYPRODUCTS
${OPENBLAS_STATIC_LIB}) ${OPENBLAS_STATIC_LIB})
ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL}) ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL})
else() else()
file(DOWNLOAD ${OPENBLAS_CACHE_URL} ${OPENBLAS_CACHE_PACKAGE_PATH} STATUS status) file(DOWNLOAD ${OPENBLAS_CACHE_URL} ${OPENBLAS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code) list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${OPENBLAS_CACHE_URL} TO ${OPENBLAS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") message(STATUS "DOWNLOADING FROM ${OPENBLAS_CACHE_URL} TO ${OPENBLAS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0) if (status_code EQUAL 0)
ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR})
endif() endif()
endif() endif()
else() else()
@ -678,20 +463,12 @@ macro(build_openblas)
add_dependencies(openblas openblas_ep) add_dependencies(openblas openblas_ep)
endmacro() endmacro()
#if(KNOWHERE_WITH_OPENBLAS)
# resolve_dependency(OpenBLAS)
#
# get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
# include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}")
# link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib)
#endif()
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
# LAPACK # LAPACK
macro(build_lapack) macro(build_lapack)
message(STATUS "Building LAPACK-${LAPACK_VERSION} from source") message(STATUS "Building LAPACK-${LAPACK_VERSION} from source")
set(LAPACK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep") set(LAPACK_PREFIX "${CORE_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep")
set(LAPACK_INCLUDE_DIR "${LAPACK_PREFIX}/include") set(LAPACK_INCLUDE_DIR "${LAPACK_PREFIX}/include")
set(LAPACK_STATIC_LIB set(LAPACK_STATIC_LIB
"${LAPACK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lapack${CMAKE_STATIC_LIBRARY_SUFFIX}") "${LAPACK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lapack${CMAKE_STATIC_LIBRARY_SUFFIX}")
@ -723,13 +500,13 @@ macro(build_lapack)
BUILD_BYPRODUCTS BUILD_BYPRODUCTS
${LAPACK_STATIC_LIB}) ${LAPACK_STATIC_LIB})
ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL}) ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL})
else() else()
file(DOWNLOAD ${LAPACK_CACHE_URL} ${LAPACK_CACHE_PACKAGE_PATH} STATUS status) file(DOWNLOAD ${LAPACK_CACHE_URL} ${LAPACK_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code) list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${LAPACK_CACHE_URL} TO ${LAPACK_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") message(STATUS "DOWNLOADING FROM ${LAPACK_CACHE_URL} TO ${LAPACK_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0) if (status_code EQUAL 0)
ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR})
endif() endif()
endif() endif()
else() else()
@ -756,14 +533,6 @@ macro(build_lapack)
add_dependencies(lapack lapack_ep) add_dependencies(lapack lapack_ep)
endmacro() endmacro()
#if(KNOWHERE_WITH_LAPACK)
# resolve_dependency(LAPACK)
#
# get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES)
# include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}")
# link_directories(SYSTEM ${LAPACK_PREFIX}/lib)
#endif()
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
# Google gtest # Google gtest
@ -780,7 +549,7 @@ macro(build_gtest)
-Wno-ignored-attributes) -Wno-ignored-attributes)
endif() endif()
set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") set(GTEST_PREFIX "${CORE_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
set(GTEST_STATIC_LIB set(GTEST_STATIC_LIB
"${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}") "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
@ -822,13 +591,13 @@ macro(build_gtest)
${GTEST_CMAKE_ARGS} ${GTEST_CMAKE_ARGS}
${EP_LOG_OPTIONS}) ${EP_LOG_OPTIONS})
ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL}) ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL})
else() else()
file(DOWNLOAD ${GTEST_CACHE_URL} ${GTEST_CACHE_PACKAGE_PATH} STATUS status) file(DOWNLOAD ${GTEST_CACHE_URL} ${GTEST_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code) list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${GTEST_CACHE_URL} TO ${GTEST_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") message(STATUS "DOWNLOADING FROM ${GTEST_CACHE_URL} TO ${GTEST_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0) if (status_code EQUAL 0)
ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR})
endif() endif()
endif() endif()
else() else()
@ -871,8 +640,7 @@ macro(build_gtest)
endmacro() endmacro()
if (KNOWHERE_BUILD_TESTS) if (KNOWHERE_BUILD_TESTS AND NOT TARGET googletest_ep)
#message(STATUS "Resolving gtest dependency")
resolve_dependency(GTest) resolve_dependency(GTest)
if(NOT GTEST_VENDORED) if(NOT GTEST_VENDORED)
@ -889,17 +657,12 @@ endif()
macro(build_faiss) macro(build_faiss)
message(STATUS "Building FAISS-${FAISS_VERSION} from source") message(STATUS "Building FAISS-${FAISS_VERSION} from source")
set(FAISS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep") set(FAISS_PREFIX "${CORE_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep")
set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include") set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include")
set(FAISS_STATIC_LIB set(FAISS_STATIC_LIB
"${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}") "${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}")
# add_custom_target(faiss_dependencies)
# add_dependencies(faiss_dependencies openblas_ep)
# add_dependencies(faiss_dependencies openblas)
# get_target_property(FAISS_OPENBLAS_LIB_DIR openblas IMPORTED_LOCATION)
# get_filename_component(FAISS_OPENBLAS_LIB "${FAISS_OPENBLAS_LIB_DIR}" DIRECTORY)
set(FAISS_CONFIGURE_ARGS set(FAISS_CONFIGURE_ARGS
"--prefix=${FAISS_PREFIX}" "--prefix=${FAISS_PREFIX}"
"CFLAGS=${EP_C_FLAGS}" "CFLAGS=${EP_C_FLAGS}"
@ -907,13 +670,7 @@ macro(build_faiss)
"LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib -lopenblas -llapack" "LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib -lopenblas -llapack"
--without-python) --without-python)
# if(OPENBLAS_STATIC_LIB)
# set(OPENBLAS_LIBRARY ${OPENBLAS_STATIC_LIB})
# else()
# set(OPENBLAS_LIBRARY ${OPENBLAS_SHARED_LIB})
# endif()
# set(FAISS_DEPENDENCIES ${FAISS_DEPENDENCIES} ${OPENBLAS_LIBRARY})
if(${KNOWHERE_WITH_FAISS_GPU_VERSION} STREQUAL "ON") if(${KNOWHERE_WITH_FAISS_GPU_VERSION} STREQUAL "ON")
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}" "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}"
@ -924,9 +681,7 @@ macro(build_faiss)
endif() endif()
if(USE_JFROG_CACHE STREQUAL "ON") if(USE_JFROG_CACHE STREQUAL "ON")
# Check_Last_Modify("${CMAKE_SOURCE_DIR}/thirdparty/faiss_cache_check_lists.txt" "${CMAKE_SOURCE_DIR}" FAISS_LAST_MODIFIED_COMMIT_ID)
string(MD5 FAISS_COMBINE_MD5 "${FAISS_MD5}${LAPACK_MD5}${OPENBLAS_MD5}") string(MD5 FAISS_COMBINE_MD5 "${FAISS_MD5}${LAPACK_MD5}${OPENBLAS_MD5}")
# string(MD5 FAISS_COMBINE_MD5 "${FAISS_LAST_MODIFIED_COMMIT_ID}${LAPACK_MD5}${OPENBLAS_MD5}")
set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMBINE_MD5}.tar.gz") set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMBINE_MD5}.tar.gz")
set(FAISS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${FAISS_CACHE_PACKAGE_NAME}") set(FAISS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${FAISS_CACHE_PACKAGE_NAME}")
set(FAISS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${FAISS_CACHE_PACKAGE_NAME}") set(FAISS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${FAISS_CACHE_PACKAGE_NAME}")
@ -952,13 +707,13 @@ macro(build_faiss)
ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep)
ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL}) ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${CORE_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL})
else() else()
file(DOWNLOAD ${FAISS_CACHE_URL} ${FAISS_CACHE_PACKAGE_PATH} STATUS status) file(DOWNLOAD ${FAISS_CACHE_URL} ${FAISS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code) list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${FAISS_CACHE_URL} TO ${FAISS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") message(STATUS "DOWNLOADING FROM ${FAISS_CACHE_URL} TO ${FAISS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0) if (status_code EQUAL 0)
ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CORE_BINARY_DIR})
endif() endif()
endif() endif()
else() else()
@ -990,14 +745,10 @@ macro(build_faiss)
INTERFACE_LINK_LIBRARIES "openblas;lapack" ) INTERFACE_LINK_LIBRARIES "openblas;lapack" )
add_dependencies(faiss faiss_ep) add_dependencies(faiss faiss_ep)
#add_dependencies(faiss openblas_ep)
#add_dependencies(faiss lapack_ep)
#target_link_libraries(faiss ${OPENBLAS_PREFIX}/lib)
#target_link_libraries(faiss ${LAPACK_PREFIX}/lib)
endmacro() endmacro()
if(KNOWHERE_WITH_FAISS) if(KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep)
resolve_dependency(OpenBLAS) resolve_dependency(OpenBLAS)
get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
@ -1012,8 +763,5 @@ if(KNOWHERE_WITH_FAISS)
resolve_dependency(FAISS) resolve_dependency(FAISS)
get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES) get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM "${FAISS_INCLUDE_DIR}") include_directories(SYSTEM "${FAISS_INCLUDE_DIR}")
# include_directories(SYSTEM "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/")
# link_directories(SYSTEM ${FAISS_PREFIX}/)
link_directories(SYSTEM ${FAISS_PREFIX}/lib/) link_directories(SYSTEM ${FAISS_PREFIX}/lib/)
# link_directories(SYSTEM ${FAISS_PREFIX}/gpu/)
endif() endif()

76
cpp/src/core/src/CMakeLists.txt
View File

@ -1,16 +1,16 @@
set(TBB_DIR ${CMAKE_SOURCE_DIR}/thirdparty/tbb) set(TBB_DIR ${CORE_SOURCE_DIR}/thirdparty/tbb)
set(TBB_LIBRARIES ${TBB_DIR}/libtbb.so) set(TBB_LIBRARIES ${TBB_DIR}/libtbb.so)
include_directories(${TBB_DIR}/include) include_directories(${TBB_DIR}/include)
include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64) link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64)
include_directories(${CMAKE_SOURCE_DIR}/include) include_directories(${CORE_SOURCE_DIR}/include)
include_directories(${CMAKE_SOURCE_DIR}/thirdparty) include_directories(${CORE_SOURCE_DIR}/thirdparty)
include_directories(${CMAKE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
include_directories(${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
set(SPTAG_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../thirdparty/SPTAG) set(SPTAG_SOURCE_DIR ${CORE_SOURCE_DIR}/thirdparty/SPTAG)
file(GLOB HDR_FILES file(GLOB HDR_FILES
${SPTAG_SOURCE_DIR}/AnnService/inc/Core/*.h ${SPTAG_SOURCE_DIR}/AnnService/inc/Core/*.h
${SPTAG_SOURCE_DIR}/AnnService/inc/Core/Common/*.h ${SPTAG_SOURCE_DIR}/AnnService/inc/Core/Common/*.h
@ -24,9 +24,9 @@ file(GLOB SRC_FILES
${SPTAG_SOURCE_DIR}/AnnService/src/Core/KDT/*.cpp ${SPTAG_SOURCE_DIR}/AnnService/src/Core/KDT/*.cpp
${SPTAG_SOURCE_DIR}/AnnService/src/Helper/*.cpp) ${SPTAG_SOURCE_DIR}/AnnService/src/Helper/*.cpp)
#add_library(SPTAGLib SHARED ${SRC_FILES} ${HDR_FILES}) if(NOT TARGET SPTAGLibStatic)
#target_link_libraries(SPTAGLib ${TBB_LIBRARIES}) add_library(SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES})
add_library(SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES}) endif()
set(external_srcs set(external_srcs
knowhere/adapter/sptag.cpp knowhere/adapter/sptag.cpp
@ -51,14 +51,13 @@ set(index_srcs
) )
set(depend_libs set(depend_libs
# libtcmalloc.a
SPTAGLibStatic SPTAGLibStatic
${TBB_LIBRARIES} ${TBB_LIBRARIES}
faiss faiss
openblas openblas
lapack lapack
arrow arrow
jemalloc_pic ${ARROW_PREFIX}/lib/libjemalloc_pic.a
cudart cudart
cublas cublas
gomp gomp
@ -66,30 +65,19 @@ set(depend_libs
pthread pthread
) )
add_library( if(NOT TARGET knowhere)
knowhere STATIC add_library(
${external_srcs} knowhere STATIC
${index_srcs} ${external_srcs}
) ${index_srcs}
#target_compile_options(knowhere PUBLIC "-fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free") )
endif()
target_link_libraries( target_link_libraries(
knowhere knowhere
${depend_libs} ${depend_libs}
) )
#add_library(
# knowhereS SHARED
# ${external_srcs}
# ${index_srcs}
#)
#
#target_link_libraries(
# knowhereS
## ${TBB_LIBRARIES}
# ${depend_libs}
#)
INSTALL(TARGETS INSTALL(TARGETS
knowhere knowhere
SPTAGLibStatic SPTAGLibStatic
@ -98,7 +86,6 @@ INSTALL(TARGETS
INSTALL(FILES INSTALL(FILES
${ARROW_STATIC_LIB} ${ARROW_STATIC_LIB}
# ${PARQUET_STATIC_LIB}
${ARROW_PREFIX}/lib/libjemalloc_pic.a ${ARROW_PREFIX}/lib/libjemalloc_pic.a
${FAISS_STATIC_LIB} ${FAISS_STATIC_LIB}
${LAPACK_STATIC_LIB} ${LAPACK_STATIC_LIB}
@ -112,24 +99,35 @@ INSTALL(FILES ${OPENBLAS_REAL_STATIC_LIB}
DESTINATION lib DESTINATION lib
) )
INSTALL(FILES ${CMAKE_SOURCE_DIR}/thirdparty/tbb/libtbb.so.2 INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so.2
# RENAME "libtbb.so.2"
DESTINATION lib DESTINATION lib
) )
INSTALL(FILES ${CMAKE_SOURCE_DIR}/thirdparty/tbb/libtbb.so INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so
# RENAME "libtbb.so"
DESTINATION lib DESTINATION lib
) )
set(CORE_INCLUDE_DIRS
${CORE_SOURCE_DIR}/include
${CORE_SOURCE_DIR}/thirdparty
${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService
${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include
${ARROW_INCLUDE_DIR}
${FAISS_INCLUDE_DIR}
${OPENBLAS_INCLUDE_DIR}
${LAPACK_INCLUDE_DIR}
${CORE_SOURCE_DIR}/thirdparty/tbb/include
)
set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
INSTALL(DIRECTORY INSTALL(DIRECTORY
${CMAKE_SOURCE_DIR}/include/knowhere ${CORE_SOURCE_DIR}/include/knowhere
${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons
${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext
${ARROW_INCLUDE_DIR}/arrow ${ARROW_INCLUDE_DIR}/arrow
# ${ARROW_INCLUDE_DIR}/parquet
${FAISS_PREFIX}/include/faiss ${FAISS_PREFIX}/include/faiss
${OPENBLAS_INCLUDE_DIR}/ ${OPENBLAS_INCLUDE_DIR}/
${CMAKE_SOURCE_DIR}/thirdparty/tbb/include/tbb ${CORE_SOURCE_DIR}/thirdparty/tbb/include/tbb
DESTINATION DESTINATION
include) include)

78
cpp/src/core/test/CMakeLists.txt
View File

@ -1,17 +1,19 @@
include_directories(${CMAKE_SOURCE_DIR}/thirdparty) include_directories(${CORE_SOURCE_DIR}/thirdparty)
include_directories(${CMAKE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
include_directories(${CMAKE_SOURCE_DIR}/include) include_directories(${CORE_SOURCE_DIR}/include)
include_directories(${CMAKE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
include_directories(/usr/local/cuda/include) include_directories(/usr/local/cuda/include)
link_directories(/usr/local/cuda/lib64) link_directories(/usr/local/cuda/lib64)
link_directories(${CMAKE_SOURCE_DIR}/thirdparty/tbb) link_directories(${CORE_SOURCE_DIR}/thirdparty/tbb)
set(unittest_libs set(unittest_libs
gtest gmock gtest_main gmock_main) gtest gmock gtest_main gmock_main)
message(STATUS "arrow prefix: ${ARROW_PREFIX}")
message(STATUS "libjemalloc_pic path: ${ARROW_PREFIX}/lib/libjemalloc_pic.a")
set(depend_libs set(depend_libs
faiss openblas lapack faiss openblas lapack
arrow jemalloc_pic arrow "${ARROW_PREFIX}/lib/libjemalloc_pic.a"
tbb tbb
) )
@ -22,57 +24,55 @@ set(basic_libs
#<IVF-TEST> #<IVF-TEST>
#<<<<<<<<<<
set(ivf_srcs set(ivf_srcs
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
utils.cpp utils.cpp
) )
add_executable(test_ivf test_ivf.cpp ${ivf_srcs}) if(NOT TARGET test_ivf)
add_executable(test_ivf test_ivf.cpp ${ivf_srcs})
endif()
target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs}) target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs})
#<<<<<<<<<<
#<IDMAP-TEST> #<IDMAP-TEST>
#<<<<<<<<<<
set(idmap_srcs set(idmap_srcs
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
utils.cpp utils.cpp
) )
add_executable(test_idmap test_idmap.cpp ${idmap_srcs}) if(NOT TARGET test_idmap)
add_executable(test_idmap test_idmap.cpp ${idmap_srcs})
endif()
target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs})
#<<<<<<<<<<
#<KDT-TEST> #<KDT-TEST>
#<<<<<<<<<<
set(kdt_srcs set(kdt_srcs
${CMAKE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp ${CORE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp ${CORE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
utils.cpp utils.cpp
) )
add_executable(test_kdt test_kdt.cpp ${kdt_srcs}) if(NOT TARGET test_kdt)
add_executable(test_kdt test_kdt.cpp ${kdt_srcs})
endif()
target_link_libraries(test_kdt target_link_libraries(test_kdt
SPTAGLibStatic SPTAGLibStatic
${depend_libs} ${unittest_libs} ${basic_libs}) ${depend_libs} ${unittest_libs} ${basic_libs})
#<<<<<<<<<<
add_subdirectory(faiss_ori) add_subdirectory(faiss_ori)
add_subdirectory(test_nsg) add_subdirectory(test_nsg)

14
cpp/src/core/test/faiss_ori/CMakeLists.txt
View File

@ -1,15 +1,15 @@
include_directories(${CMAKE_SOURCE_DIR}/thirdparty) include_directories(${CORE_SOURCE_DIR}/thirdparty)
include_directories(${CMAKE_SOURCE_DIR}/include) include_directories(${CORE_SOURCE_DIR}/include)
include_directories(/usr/local/cuda/include) include_directories(/usr/local/cuda/include)
link_directories(/usr/local/cuda/lib64) link_directories(/usr/local/cuda/lib64)
link_directories(${CMAKE_SOURCE_DIR}/thirdparty/tbb) link_directories(${CORE_SOURCE_DIR}/thirdparty/tbb)
set(unittest_libs set(unittest_libs
gtest gmock gtest_main gmock_main) gtest gmock gtest_main gmock_main)
set(depend_libs set(depend_libs
faiss openblas lapack faiss openblas lapack
arrow jemalloc_pic arrow ${ARROW_PREFIX}/lib/libjemalloc_pic.a
tbb tbb
) )
@ -20,7 +20,7 @@ set(basic_libs
#<GPU-TEST> #<GPU-TEST>
#<<<<<<<<<< if(NOT TARGET test_gpu)
add_executable(test_gpu gpuresource_test.cpp) add_executable(test_gpu gpuresource_test.cpp)
endif()
target_link_libraries(test_gpu ${depend_libs} ${unittest_libs} ${basic_libs}) target_link_libraries(test_gpu ${depend_libs} ${unittest_libs} ${basic_libs})
#<<<<<<<<<<

45
cpp/src/core/test/test_nsg/CMakeLists.txt
View File

@ -2,7 +2,6 @@
include_directories(/usr/local/include/gperftools) include_directories(/usr/local/include/gperftools)
link_directories(/usr/local/lib) link_directories(/usr/local/lib)
#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free")
add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO) add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO)
find_package(OpenMP) find_package(OpenMP)
@ -14,37 +13,29 @@ else ()
endif () endif ()
message(${OpenMP_CXX_FLAGS}) message(${OpenMP_CXX_FLAGS})
include_directories(${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg) include_directories(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg)
#include_directories(/home/zilliz/opt/app/pyenv/versions/3.6.8/include/python3.6m)
#include_directories(/home/zilliz/opt/app/pyenv/versions/3.6.8/envs/megasearch_testframework_dev/lib/python3.6/site-packages/numpy/core/include)
aux_source_directory(${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src) aux_source_directory(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src)
#add_library(nsg_raw SHARED ${nsg_src})
#target_link_libraries(nsg_raw
# gomp)
set(interface_src set(interface_src
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/exception.cpp ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
${CMAKE_SOURCE_DIR}/src/knowhere/common/timer.cpp ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
../utils.cpp ../utils.cpp
) )
add_executable(test_nsg if(NOT TARGET test_nsg)
test_nsg.cpp add_executable(test_nsg
${interface_src} test_nsg.cpp
${nsg_src} ${interface_src}
) ${nsg_src}
#target_link_libraries(test_nsg )
# # libprofiler.so endif()
## -ltcmalloc
## gomp
# )
target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs}) target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs})
############################## ##############################

8
cpp/unittest/CMakeLists.txt
View File

@ -38,9 +38,13 @@ set(unittest_libs
cudart cudart
) )
foreach(dir ${CORE_INCLUDE_DIRS})
include_directories(${dir})
endforeach()
add_subdirectory(server) add_subdirectory(server)
add_subdirectory(db) add_subdirectory(db)
add_subdirectory(knowhere) add_subdirectory(knowhere)
add_subdirectory(metrics) add_subdirectory(metrics)
#add_subdirectory(scheduler) add_subdirectory(scheduler)
#add_subdirectory(storage) add_subdirectory(storage)

17
cpp/unittest/db/CMakeLists.txt
View File

@ -35,8 +35,6 @@ link_directories("/usr/local/cuda/lib64")
include_directories(/usr/include/mysql) include_directories(/usr/include/mysql)
#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY)
set(db_test_src set(db_test_src
${config_files} ${config_files}
${cache_srcs} ${cache_srcs}
@ -69,25 +67,10 @@ set(db_libs
set(knowhere_libs set(knowhere_libs
knowhere knowhere
SPTAGLibStatic
arrow
jemalloc_pic
faiss
openblas
lapack
tbb
cudart cudart
cublas cublas
) )
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS})
else()
set(db_libs ${db_libs}
lapack
openblas)
endif()
target_link_libraries(db_test ${db_libs} ${knowhere_libs} ${unittest_libs}) target_link_libraries(db_test ${db_libs} ${knowhere_libs} ${unittest_libs})
install(TARGETS db_test DESTINATION unittest) install(TARGETS db_test DESTINATION unittest)

7
cpp/unittest/knowhere/CMakeLists.txt
View File

@ -8,13 +8,6 @@ set(helper
set(knowhere_libs set(knowhere_libs
knowhere knowhere
SPTAGLibStatic
arrow
jemalloc_pic
faiss
openblas
lapack
tbb
cudart cudart
cublas cublas
) )

24
cpp/unittest/metrics/CMakeLists.txt
View File

@ -1,13 +1,3 @@
#set(CURL_LIBRARY "-lcurl")
#find_package(CURL REQUIRED)
#INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIR})
#
#INCLUDE_DIRECTORIES(~/development/lib/usr/local/include)
#LINK_DIRECTORIES(~/development/lib/usr/local/lib)
include_directories(../../src) include_directories(../../src)
@ -72,13 +62,6 @@ add_executable(metrics_test ${count_test_src} ${require_files} )
set(knowhere_libs set(knowhere_libs
knowhere knowhere
SPTAGLibStatic
arrow
jemalloc_pic
faiss
openblas
lapack
tbb
) )
target_link_libraries(metrics_test target_link_libraries(metrics_test
@ -96,12 +79,5 @@ target_link_libraries(metrics_test
mysqlpp mysqlpp
${unittest_libs} ${unittest_libs}
) )
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
target_link_libraries(metrics_test ${MKL_LIBS} ${MKL_LIBS})
else()
target_link_libraries(metrics_test
lapack
openblas)
endif()
install(TARGETS metrics_test DESTINATION unittest) install(TARGETS metrics_test DESTINATION unittest)

17
cpp/unittest/scheduler/CMakeLists.txt
View File

@ -33,8 +33,6 @@ link_directories("/usr/local/cuda/lib64")
include_directories(/usr/include/mysql) include_directories(/usr/include/mysql)
#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY)
set(scheduler_test_src set(scheduler_test_src
${unittest_srcs} ${unittest_srcs}
${test_srcs} ${test_srcs}
@ -69,25 +67,10 @@ set(scheduler_libs
set(knowhere_libs set(knowhere_libs
knowhere knowhere
SPTAGLibStatic
arrow
jemalloc_pic
faiss
openblas
lapack
tbb
cudart cudart
cublas cublas
) )
if (${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS})
else ()
set(db_libs ${db_libs}
lapack
openblas)
endif ()
target_link_libraries(scheduler_test ${scheduler_libs} ${knowhere_libs} ${unittest_libs}) target_link_libraries(scheduler_test ${scheduler_libs} ${knowhere_libs} ${unittest_libs})

11
cpp/unittest/server/CMakeLists.txt
View File

@ -34,9 +34,6 @@ cuda_add_executable(server_test
set(require_libs set(require_libs
knowhere knowhere
faiss
openblas
lapack
stdc++ stdc++
cudart cudart
cublas cublas
@ -51,14 +48,6 @@ set(require_libs
pthread pthread
) )
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(require_libs ${require_libs} ${MKL_LIBS} ${MKL_LIBS})
else()
set(require_libs ${require_libs}
lapack
openblas)
endif()
target_link_libraries(server_test target_link_libraries(server_test
${require_libs} ${require_libs}
${cuda_library} ${cuda_library}