milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/branch-0.4.0' into branch-0.4.0 

Former-commit-id: b2343181af61af6f369bae7cd747ade0378d8087
pull/191/head
zhiru 2019-08-06 18:07:45 +08:00
parent 81928c91e4 9edd391fe2
commit 5b8f62c229
7 changed files with 432 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

435
cpp/cmake/ThirdPartyPackages.cmake
View File

@ -376,6 +376,7 @@ else ()
set(ROCKSDB_SOURCE_URL
"https://github.com/facebook/rocksdb/archive/${ROCKSDB_VERSION}.tar.gz")
endif()
set(ROCKSDB_MD5 "a8f2f594182e97a08629bcc66dfd3fa0")
if(DEFINED ENV{MILVUS_SNAPPY_URL})
set(SNAPPY_SOURCE_URL "$ENV{MILVUS_SNAPPY_URL}")
@ -443,6 +444,7 @@ else()
set(LIBUNWIND_SOURCE_URL
"https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz")
endif()
set(LIBUNWIND_MD5 "a04f69d66d8e16f8bf3ab72a69112cd6")
if(DEFINED ENV{MILVUS_GPERFTOOLS_URL})
set(GPERFTOOLS_SOURCE_URL "$ENV{MILVUS_GPERFTOOLS_URL}")
@ -450,6 +452,7 @@ else()
set(GPERFTOOLS_SOURCE_URL
"https://github.com/gperftools/gperftools/releases/download/gperftools-${GPERFTOOLS_VERSION}/gperftools-${GPERFTOOLS_VERSION}.tar.gz")
endif()
set(GPERFTOOLS_MD5 "c6a852a817e9160c79bdb2d3101b4601")
if(DEFINED ENV{MILVUS_GRPC_URL})
set(GRPC_SOURCE_URL "$ENV{MILVUS_GRPC_URL}")
@ -488,10 +491,9 @@ macro(build_arrow)
set(ARROW_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ARROW_CACHE_PACKAGE_NAME}")
set(ARROW_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ARROW_CACHE_PACKAGE_NAME}")
file(DOWNLOAD ${ARROW_CACHE_URL} ${ARROW_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${ARROW_CACHE_URL} TO ${ARROW_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (NOT status_code EQUAL 0)
execute_process(COMMAND wget -q --method HEAD ${ARROW_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote cache file ${ARROW_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(arrow_ep
GIT_REPOSITORY
${ARROW_SOURCE_URL}
@ -515,7 +517,12 @@ macro(build_arrow)
ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL})
else()
ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
file(DOWNLOAD ${ARROW_CACHE_URL} ${ARROW_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${ARROW_CACHE_URL} TO ${ARROW_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
message(FATAL_ERROR "The last commit ID of \"${ARROW_SOURCE_URL}\" repository don't match!")
@ -683,10 +690,9 @@ macro(build_bzip2)
set(BZIP2_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${BZIP2_CACHE_PACKAGE_NAME}")
set(BZIP2_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${BZIP2_CACHE_PACKAGE_NAME}")
file(DOWNLOAD ${BZIP2_CACHE_URL} ${BZIP2_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${BZIP2_CACHE_URL} TO ${BZIP2_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (NOT status_code EQUAL 0)
execute_process(COMMAND wget -q --method HEAD ${BZIP2_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote cache file ${BZIP2_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(bzip2_ep
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
@ -711,7 +717,12 @@ macro(build_bzip2)
ExternalProject_Create_Cache(bzip2_ep ${BZIP2_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/bzip2_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${BZIP2_CACHE_URL})
else()
ExternalProject_Use_Cache(bzip2_ep ${BZIP2_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
file(DOWNLOAD ${BZIP2_CACHE_URL} ${BZIP2_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${BZIP2_CACHE_URL} TO ${BZIP2_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(bzip2_ep ${BZIP2_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(bzip2_ep
@ -831,10 +842,9 @@ macro(build_easyloggingpp)
set(EASYLOGGINGPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${EASYLOGGINGPP_CACHE_PACKAGE_NAME}")
set(EASYLOGGINGPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${EASYLOGGINGPP_CACHE_PACKAGE_NAME}")
file(DOWNLOAD ${EASYLOGGINGPP_CACHE_URL} ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${EASYLOGGINGPP_CACHE_URL} TO ${EASYLOGGINGPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (NOT status_code EQUAL 0)
execute_process(COMMAND wget -q --method HEAD ${EASYLOGGINGPP_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote cache file ${EASYLOGGINGPP_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(easyloggingpp_ep
URL
${EASYLOGGINGPP_SOURCE_URL}
@ -849,7 +859,12 @@ macro(build_easyloggingpp)
ExternalProject_Create_Cache(easyloggingpp_ep ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/easyloggingpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${EASYLOGGINGPP_CACHE_URL})
else()
ExternalProject_Use_Cache(easyloggingpp_ep ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
file(DOWNLOAD ${EASYLOGGINGPP_CACHE_URL} ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${EASYLOGGINGPP_CACHE_URL} TO ${EASYLOGGINGPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(easyloggingpp_ep ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(easyloggingpp_ep
@ -898,10 +913,9 @@ macro(build_openblas)
set(OPENBLAS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${OPENBLAS_CACHE_PACKAGE_NAME}")
set(OPENBLAS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${OPENBLAS_CACHE_PACKAGE_NAME}")
file(DOWNLOAD ${OPENBLAS_CACHE_URL} ${OPENBLAS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${OPENBLAS_CACHE_URL} TO ${OPENBLAS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (NOT status_code EQUAL 0)
execute_process(COMMAND wget -q --method HEAD ${OPENBLAS_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${OPENBLAS_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(openblas_ep
URL
${OPENBLAS_SOURCE_URL}
@ -922,7 +936,12 @@ macro(build_openblas)
ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL})
else()
ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
file(DOWNLOAD ${OPENBLAS_CACHE_URL} ${OPENBLAS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${OPENBLAS_CACHE_URL} TO ${OPENBLAS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(openblas_ep
@ -974,10 +993,9 @@ macro(build_lapack)
set(LAPACK_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${LAPACK_CACHE_PACKAGE_NAME}")
set(LAPACK_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${LAPACK_CACHE_PACKAGE_NAME}")
file(DOWNLOAD ${LAPACK_CACHE_URL} ${LAPACK_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${LAPACK_CACHE_URL} TO ${LAPACK_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (NOT status_code EQUAL 0)
execute_process(COMMAND wget -q --method HEAD ${LAPACK_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${LAPACK_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(lapack_ep
URL
${LAPACK_SOURCE_URL}
@ -992,7 +1010,12 @@ macro(build_lapack)
ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL})
else()
ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
file(DOWNLOAD ${LAPACK_CACHE_URL} ${LAPACK_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${LAPACK_CACHE_URL} TO ${LAPACK_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(lapack_ep
@ -1086,10 +1109,9 @@ macro(build_faiss)
set(FAISS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${FAISS_CACHE_PACKAGE_NAME}")
set(FAISS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${FAISS_CACHE_PACKAGE_NAME}")
file(DOWNLOAD ${FAISS_CACHE_URL} ${FAISS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${FAISS_CACHE_URL} TO ${FAISS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (NOT status_code EQUAL 0)
execute_process(COMMAND wget -q --method HEAD ${FAISS_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${FAISS_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(faiss_ep
URL
${FAISS_SOURCE_URL}
@ -1112,7 +1134,12 @@ macro(build_faiss)
ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL})
else()
ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
file(DOWNLOAD ${FAISS_CACHE_URL} ${FAISS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${FAISS_CACHE_URL} TO ${FAISS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(faiss_ep
@ -1340,10 +1367,9 @@ macro(build_lz4)
set(LZ4_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${LZ4_CACHE_PACKAGE_NAME}")
set(LZ4_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${LZ4_CACHE_PACKAGE_NAME}")
file(DOWNLOAD ${LZ4_CACHE_URL} ${LZ4_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${LZ4_CACHE_URL} TO ${LZ4_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (NOT status_code EQUAL 0)
execute_process(COMMAND wget -q --method HEAD ${LZ4_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${LZ4_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(lz4_ep
URL
${LZ4_SOURCE_URL}
@ -1367,7 +1393,12 @@ macro(build_lz4)
ExternalProject_Create_Cache(lz4_ep ${LZ4_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LZ4_CACHE_URL})
else()
ExternalProject_Use_Cache(lz4_ep ${LZ4_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
file(DOWNLOAD ${LZ4_CACHE_URL} ${LZ4_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${LZ4_CACHE_URL} TO ${LZ4_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(lz4_ep ${LZ4_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(lz4_ep
@ -1430,10 +1461,9 @@ macro(build_mysqlpp)
set(MYSQLPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${MYSQLPP_CACHE_PACKAGE_NAME}")
set(MYSQLPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${MYSQLPP_CACHE_PACKAGE_NAME}")
file(DOWNLOAD ${MYSQLPP_CACHE_URL} ${MYSQLPP_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${MYSQLPP_CACHE_URL} TO ${MYSQLPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (NOT status_code EQUAL 0)
execute_process(COMMAND wget -q --method HEAD ${MYSQLPP_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${MYSQLPP_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(mysqlpp_ep
URL
${MYSQLPP_SOURCE_URL}
@ -1450,7 +1480,12 @@ macro(build_mysqlpp)
ExternalProject_Create_Cache(mysqlpp_ep ${MYSQLPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/mysqlpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${MYSQLPP_CACHE_URL})
else()
ExternalProject_Use_Cache(mysqlpp_ep ${MYSQLPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
file(DOWNLOAD ${MYSQLPP_CACHE_URL} ${MYSQLPP_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${MYSQLPP_CACHE_URL} TO ${MYSQLPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(mysqlpp_ep ${MYSQLPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(mysqlpp_ep
@ -1520,10 +1555,9 @@ macro(build_prometheus)
set(PROMETHEUS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${PROMETHEUS_CACHE_PACKAGE_NAME}")
set(PROMETHEUS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${PROMETHEUS_CACHE_PACKAGE_NAME}")
file(DOWNLOAD ${PROMETHEUS_CACHE_URL} ${PROMETHEUS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${PROMETHEUS_CACHE_URL} TO ${PROMETHEUS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (NOT status_code EQUAL 0)
execute_process(COMMAND wget -q --method HEAD ${PROMETHEUS_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${PROMETHEUS_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(prometheus_ep
GIT_REPOSITORY
${PROMETHEUS_SOURCE_URL}
@ -1550,7 +1584,12 @@ macro(build_prometheus)
ExternalProject_Create_Cache(prometheus_ep ${PROMETHEUS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/prometheus_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${PROMETHEUS_CACHE_URL})
else()
ExternalProject_Use_Cache(prometheus_ep ${PROMETHEUS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
file(DOWNLOAD ${PROMETHEUS_CACHE_URL} ${PROMETHEUS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${PROMETHEUS_CACHE_URL} TO ${PROMETHEUS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(prometheus_ep ${PROMETHEUS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
message(FATAL_ERROR "The last commit ID of \"${PROMETHEUS_SOURCE_URL}\" repository don't match!")
@ -1630,25 +1669,65 @@ macro(build_rocksdb)
"${ROCKSDB_PREFIX}/lib/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ROCKSDB_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
externalproject_add(rocksdb_ep
URL
${ROCKSDB_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
""
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
static_lib
"prefix=${ROCKSDB_PREFIX}"
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE}
install-static
"INSTALL_PATH=${ROCKSDB_PREFIX}/lib"
BUILD_BYPRODUCTS
"${ROCKSDB_STATIC_LIB}")
if(USE_JFROG_CACHE STREQUAL "ON")
string(MD5 ROCKSDB_COMBINE_MD5 "${ROCKSDB_LAST_COMMIT_ID}")
set(ROCKSDB_CACHE_PACKAGE_NAME "rocksdb_${ROCKSDB_MD5}.tar.gz")
set(ROCKSDB_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ROCKSDB_CACHE_PACKAGE_NAME}")
set(ROCKSDB_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ROCKSDB_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${ROCKSDB_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${ROCKSDB_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(rocksdb_ep
URL
${ROCKSDB_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
""
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
static_lib
"prefix=${ROCKSDB_PREFIX}"
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE}
install-static
"INSTALL_PATH=${ROCKSDB_PREFIX}/lib"
BUILD_BYPRODUCTS
"${ROCKSDB_STATIC_LIB}")
ExternalProject_Create_Cache(rocksdb_ep ${ROCKSDB_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/rocksdb_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ROCKSDB_CACHE_URL})
else()
file(DOWNLOAD ${ROCKSDB_CACHE_URL} ${ROCKSDB_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${ROCKSDB_CACHE_URL} TO ${ROCKSDB_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(rocksdb_ep ${ROCKSDB_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(rocksdb_ep
URL
${ROCKSDB_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
""
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
static_lib
"prefix=${ROCKSDB_PREFIX}"
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE}
install-static
"INSTALL_PATH=${ROCKSDB_PREFIX}/lib"
BUILD_BYPRODUCTS
"${ROCKSDB_STATIC_LIB}")
endif()
file(MAKE_DIRECTORY "${ROCKSDB_PREFIX}/include")
@ -1690,10 +1769,9 @@ macro(build_snappy)
set(SNAPPY_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${SNAPPY_CACHE_PACKAGE_NAME}")
set(SNAPPY_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${SNAPPY_CACHE_PACKAGE_NAME}")
file(DOWNLOAD ${SNAPPY_CACHE_URL} ${SNAPPY_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${SNAPPY_CACHE_URL} TO ${SNAPPY_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (NOT status_code EQUAL 0)
execute_process(COMMAND wget -q --method HEAD ${SNAPPY_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${SNAPPY_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(snappy_ep
${EP_LOG_OPTIONS}
BUILD_COMMAND
@ -1712,7 +1790,12 @@ macro(build_snappy)
ExternalProject_Create_Cache(snappy_ep ${SNAPPY_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/snappy_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${SNAPPY_CACHE_URL})
else()
ExternalProject_Use_Cache(snappy_ep ${SNAPPY_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
file(DOWNLOAD ${SNAPPY_CACHE_URL} ${SNAPPY_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${SNAPPY_CACHE_URL} TO ${SNAPPY_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(snappy_ep ${SNAPPY_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(snappy_ep
@ -1772,10 +1855,9 @@ macro(build_sqlite)
set(SQLITE_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${SQLITE_CACHE_PACKAGE_NAME}")
set(SQLITE_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${SQLITE_CACHE_PACKAGE_NAME}")
file(DOWNLOAD ${SQLITE_CACHE_URL} ${SQLITE_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${SQLITE_CACHE_URL} TO ${SQLITE_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (NOT status_code EQUAL 0)
execute_process(COMMAND wget -q --method HEAD ${SQLITE_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${SQLITE_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(sqlite_ep
URL
${SQLITE_SOURCE_URL}
@ -1793,7 +1875,12 @@ macro(build_sqlite)
ExternalProject_Create_Cache(sqlite_ep ${SQLITE_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/sqlite_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${SQLITE_CACHE_URL})
else()
ExternalProject_Use_Cache(sqlite_ep ${SQLITE_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
file(DOWNLOAD ${SQLITE_CACHE_URL} ${SQLITE_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${SQLITE_CACHE_URL} TO ${SQLITE_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(sqlite_ep ${SQLITE_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(sqlite_ep
@ -1907,10 +1994,9 @@ macro(build_thrift)
set(THRIFT_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${THRIFT_CACHE_PACKAGE_NAME}")
set(THRIFT_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${THRIFT_CACHE_PACKAGE_NAME}")
file(DOWNLOAD ${THRIFT_CACHE_URL} ${THRIFT_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${THRIFT_CACHE_URL} TO ${THRIFT_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (NOT status_code EQUAL 0)
execute_process(COMMAND wget -q --method HEAD ${THRIFT_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${THRIFT_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(thrift_ep
URL
${THRIFT_SOURCE_URL}
@ -1930,7 +2016,12 @@ macro(build_thrift)
ExternalProject_Create_Cache(thrift_ep ${THRIFT_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/thrift_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${THRIFT_CACHE_URL})
else()
ExternalProject_Use_Cache(thrift_ep ${THRIFT_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
file(DOWNLOAD ${THRIFT_CACHE_URL} ${THRIFT_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${THRIFT_CACHE_URL} TO ${THRIFT_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(thrift_ep ${THRIFT_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(thrift_ep
@ -1990,10 +2081,9 @@ macro(build_yamlcpp)
set(YAMLCPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${YAMLCPP_CACHE_PACKAGE_NAME}")
set(YAMLCPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${YAMLCPP_CACHE_PACKAGE_NAME}")
file(DOWNLOAD ${YAMLCPP_CACHE_URL} ${YAMLCPP_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${YAMLCPP_CACHE_URL} TO ${YAMLCPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (NOT status_code EQUAL 0)
execute_process(COMMAND wget -q --method HEAD ${YAMLCPP_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${YAMLCPP_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(yaml-cpp_ep
URL
${YAMLCPP_SOURCE_URL}
@ -2008,7 +2098,12 @@ macro(build_yamlcpp)
ExternalProject_Create_Cache(yaml-cpp_ep ${YAMLCPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/yaml-cpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${YAMLCPP_CACHE_URL})
else()
ExternalProject_Use_Cache(yaml-cpp_ep ${YAMLCPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
file(DOWNLOAD ${YAMLCPP_CACHE_URL} ${YAMLCPP_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${YAMLCPP_CACHE_URL} TO ${YAMLCPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(yaml-cpp_ep ${YAMLCPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(yaml-cpp_ep
@ -2058,10 +2153,9 @@ macro(build_zlib)
set(ZLIB_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ZLIB_CACHE_PACKAGE_NAME}")
set(ZLIB_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ZLIB_CACHE_PACKAGE_NAME}")
file(DOWNLOAD ${ZLIB_CACHE_URL} ${ZLIB_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${ZLIB_CACHE_URL} TO ${ZLIB_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (NOT status_code EQUAL 0)
execute_process(COMMAND wget -q --method HEAD ${ZLIB_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${ZLIB_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(zlib_ep
URL
${ZLIB_SOURCE_URL}
@ -2076,7 +2170,12 @@ macro(build_zlib)
ExternalProject_Create_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ZLIB_CACHE_URL})
else()
ExternalProject_Use_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
file(DOWNLOAD ${ZLIB_CACHE_URL} ${ZLIB_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${ZLIB_CACHE_URL} TO ${ZLIB_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(zlib_ep
@ -2144,10 +2243,9 @@ macro(build_zstd)
set(ZSTD_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ZSTD_CACHE_PACKAGE_NAME}")
set(ZSTD_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ZSTD_CACHE_PACKAGE_NAME}")
file(DOWNLOAD ${ZSTD_CACHE_URL} ${ZSTD_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${ZSTD_CACHE_URL} TO ${ZSTD_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (NOT status_code EQUAL 0)
execute_process(COMMAND wget -q --method HEAD ${ZSTD_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${ZSTD_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(zstd_ep
${EP_LOG_OPTIONS}
CMAKE_ARGS
@ -2166,7 +2264,12 @@ macro(build_zstd)
ExternalProject_Create_Cache(zstd_ep ${ZSTD_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/zstd_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ZSTD_CACHE_URL})
else()
ExternalProject_Use_Cache(zstd_ep ${ZSTD_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
file(DOWNLOAD ${ZSTD_CACHE_URL} ${ZSTD_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${ZSTD_CACHE_URL} TO ${ZSTD_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(zstd_ep ${ZSTD_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(zstd_ep
@ -2237,10 +2340,9 @@ macro(build_aws)
set(AWS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${AWS_CACHE_PACKAGE_NAME}")
set(AWS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${AWS_CACHE_PACKAGE_NAME}")
file(DOWNLOAD ${AWS_CACHE_URL} ${AWS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${AWS_CACHE_URL} TO ${AWS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (NOT status_code EQUAL 0)
execute_process(COMMAND wget -q --method HEAD ${AWS_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${AWS_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(aws_ep
${EP_LOG_OPTIONS}
CMAKE_ARGS
@ -2258,7 +2360,12 @@ macro(build_aws)
ExternalProject_Create_Cache(aws_ep ${AWS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/aws_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${AWS_CACHE_URL})
else()
ExternalProject_Use_Cache(aws_ep ${AWS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
file(DOWNLOAD ${AWS_CACHE_URL} ${AWS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${AWS_CACHE_URL} TO ${AWS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(aws_ep ${AWS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(aws_ep
@ -2319,21 +2426,56 @@ macro(build_libunwind)
set(LIBUNWIND_SHARED_LIB "${LIBUNWIND_PREFIX}/lib/libunwind${CMAKE_SHARED_LIBRARY_SUFFIX}")
set(LIBUNWIND_CONFIGURE_ARGS "--prefix=${LIBUNWIND_PREFIX}")
externalproject_add(libunwind_ep
URL
${LIBUNWIND_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${LIBUNWIND_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${LIBUNWIND_SHARED_LIB})
if(USE_JFROG_CACHE STREQUAL "ON")
set(LIBUNWIND_CACHE_PACKAGE_NAME "libunwind_${LIBUNWIND_MD5}.tar.gz")
set(LIBUNWIND_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${LIBUNWIND_CACHE_PACKAGE_NAME}")
set(LIBUNWIND_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${LIBUNWIND_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${LIBUNWIND_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${LIBUNWIND_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(libunwind_ep
URL
${LIBUNWIND_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${LIBUNWIND_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${LIBUNWIND_SHARED_LIB})
ExternalProject_Create_Cache(libunwind_ep ${LIBUNWIND_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/libunwind_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LIBUNWIND_CACHE_URL})
else()
file(DOWNLOAD ${LIBUNWIND_CACHE_URL} ${LIBUNWIND_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${LIBUNWIND_CACHE_URL} TO ${LIBUNWIND_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(libunwind_ep ${LIBUNWIND_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(libunwind_ep
URL
${LIBUNWIND_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${LIBUNWIND_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${LIBUNWIND_SHARED_LIB})
endif()
file(MAKE_DIRECTORY "${LIBUNWIND_INCLUDE_DIR}")
@ -2362,21 +2504,56 @@ macro(build_gperftools)
set(GPERFTOOLS_STATIC_LIB "${GPERFTOOLS_PREFIX}/lib/libprofiler${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(GPERFTOOLS_CONFIGURE_ARGS "--prefix=${GPERFTOOLS_PREFIX}")
externalproject_add(gperftools_ep
URL
${GPERFTOOLS_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${GPERFTOOLS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${GPERFTOOLS_STATIC_LIB})
if(USE_JFROG_CACHE STREQUAL "ON")
set(GPERFTOOLS_CACHE_PACKAGE_NAME "gperftools_${GPERFTOOLS_MD5}.tar.gz")
set(GPERFTOOLS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GPERFTOOLS_CACHE_PACKAGE_NAME}")
set(GPERFTOOLS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GPERFTOOLS_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${GPERFTOOLS_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${GPERFTOOLS_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(gperftools_ep
URL
${GPERFTOOLS_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${GPERFTOOLS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${GPERFTOOLS_STATIC_LIB})
ExternalProject_Create_Cache(gperftools_ep ${GPERFTOOLS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/gperftools_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GPERFTOOLS_CACHE_URL})
else()
file(DOWNLOAD ${GPERFTOOLS_CACHE_URL} ${GPERFTOOLS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${GPERFTOOLS_CACHE_URL} TO ${GPERFTOOLS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(gperftools_ep ${GPERFTOOLS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
externalproject_add(gperftools_ep
URL
${GPERFTOOLS_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${GPERFTOOLS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${GPERFTOOLS_STATIC_LIB})
endif()
file(MAKE_DIRECTORY "${GPERFTOOLS_INCLUDE_DIR}")

27
cpp/src/db/DBImpl.cpp
View File

@ -325,6 +325,9 @@ void DBImpl::StartMetricTask() {
server::Metrics::GetInstance().GPUMemoryUsageGaugeSet();
server::Metrics::GetInstance().OctetsSet();
server::Metrics::GetInstance().CPUCoreUsagePercentSet();
ENGINE_LOG_TRACE << "Metric task finished";
}
@ -562,15 +565,25 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
auto to_remove = file;
to_remove.file_type_ = meta::TableFileSchema::TO_DELETE;
meta::TableFilesSchema update_files = {to_remove, table_file};
meta_ptr_->UpdateTableFiles(update_files);
meta::TableFilesSchema update_files = {table_file, to_remove};
status = meta_ptr_->UpdateTableFiles(update_files);
if(status.ok()) {
ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size "
<< index->PhysicalSize() << " bytes"
<< " from file " << to_remove.file_id_;
ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size "
<< index->PhysicalSize() << " bytes"
<< " from file " << to_remove.file_id_;
if(options_.insert_cache_immediately_) {
index->Cache();
}
} else {
//failed to update meta, mark the new file as to_delete, don't delete old file
to_remove.file_type_ = meta::TableFileSchema::TO_INDEX;
status = meta_ptr_->UpdateTableFile(to_remove);
ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << to_remove.file_id_ << " to to_index";
if(options_.insert_cache_immediately_) {
index->Cache();
table_file.file_type_ = meta::TableFileSchema::TO_DELETE;
status = meta_ptr_->UpdateTableFile(table_file);
ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";
}
} catch (std::exception& ex) {

2
cpp/src/metrics/MetricBase.h
View File

@ -64,6 +64,8 @@ class MetricsBase{
virtual void ConnectionGaugeDecrement() {};
virtual void KeepingAliveCounterIncrement(double value = 1) {};
virtual void OctetsSet() {};
virtual void CPUCoreUsagePercentSet() {};
};

49
cpp/src/metrics/PrometheusMetrics.cpp
View File

@ -44,6 +44,8 @@ PrometheusMetrics::Init() {
void
PrometheusMetrics::CPUUsagePercentSet() {
if(!startup_) return ;
int numProcessor = server::SystemInfo::GetInstance().num_processor();
double usage_percent = server::SystemInfo::GetInstance().CPUPercent();
CPU_usage_percent_.Set(usage_percent);
}
@ -58,36 +60,30 @@ PrometheusMetrics::RAMUsagePercentSet() {
void
PrometheusMetrics::GPUPercentGaugeSet() {
if(!startup_) return;
int numDevide = server::SystemInfo::GetInstance().num_device();
std::vector<unsigned int> values = server::SystemInfo::GetInstance().GPUPercent();
if(numDevide >= 1) GPU0_percent_gauge_.Set(static_cast<double>(values[0]));
if(numDevide >= 2) GPU1_percent_gauge_.Set(static_cast<double>(values[1]));
if(numDevide >= 3) GPU2_percent_gauge_.Set(static_cast<double>(values[2]));
if(numDevide >= 4) GPU3_percent_gauge_.Set(static_cast<double>(values[3]));
if(numDevide >= 5) GPU4_percent_gauge_.Set(static_cast<double>(values[4]));
if(numDevide >= 6) GPU5_percent_gauge_.Set(static_cast<double>(values[5]));
if(numDevide >= 7) GPU6_percent_gauge_.Set(static_cast<double>(values[6]));
if(numDevide >= 8) GPU7_percent_gauge_.Set(static_cast<double>(values[7]));
int numDevice = server::SystemInfo::GetInstance().num_device();
std::vector<unsigned long long > used_total = server::SystemInfo::GetInstance().GPUMemoryTotal();
std::vector<unsigned long long > used_memory = server::SystemInfo::GetInstance().GPUMemoryUsed();
for (int i = 0; i < numDevice; i++) {
prometheus::Gauge &GPU_percent = GPU_percent_.Add({{"DeviceNum", std::to_string(i)}});
double percent = (double)used_memory[i] / (double)used_total[i];
GPU_percent.Set(percent * 100);
}
// to do
}
void PrometheusMetrics::GPUMemoryUsageGaugeSet() {
if(!startup_) return;
std::vector<unsigned long long> values = server::SystemInfo::GetInstance().GPUMemoryUsed();
constexpr unsigned long long MtoB = 1024*1024;
int numDevice = values.size();
int numDevice = server::SystemInfo::GetInstance().num_device();
if(numDevice >=1) GPU0_memory_usage_gauge_.Set(values[0]/MtoB);
if(numDevice >=2) GPU1_memory_usage_gauge_.Set(values[1]/MtoB);
if(numDevice >=3) GPU2_memory_usage_gauge_.Set(values[2]/MtoB);
if(numDevice >=4) GPU3_memory_usage_gauge_.Set(values[3]/MtoB);
if(numDevice >=5) GPU4_memory_usage_gauge_.Set(values[4]/MtoB);
if(numDevice >=6) GPU5_memory_usage_gauge_.Set(values[5]/MtoB);
if(numDevice >=7) GPU6_memory_usage_gauge_.Set(values[6]/MtoB);
if(numDevice >=8) GPU7_memory_usage_gauge_.Set(values[7]/MtoB);
for (int i = 0; i < numDevice; i++) {
prometheus::Gauge &GPU_memory = GPU_memory_usage_.Add({{"DeviceNum", std::to_string(i)}});
GPU_memory.Set(values[i] / MtoB);
}
// to do
}
void PrometheusMetrics::AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) {
// MB/s
@ -140,6 +136,17 @@ void PrometheusMetrics::OctetsSet() {
outoctets_gauge_.Set((in_and_out_octets.second-old_outoctets)/total_second);
}
void PrometheusMetrics::CPUCoreUsagePercentSet() {
if (!startup_)
return;
std::vector<double> cpu_core_percent = server::SystemInfo::GetInstance().CPUCorePercent();
for (int i = 0; i < cpu_core_percent.size(); i++) {
prometheus::Gauge &core_percent = CPU_.Add({{"CPU", std::to_string(i)}});
core_percent.Set(cpu_core_percent[i]);
}
}
}

30
cpp/src/metrics/PrometheusMetrics.h
View File

@ -12,6 +12,7 @@
#include <prometheus/registry.h>
#include <prometheus/exposer.h>
#include <iostream>
#include "server/ServerConfig.h"
#include "MetricBase.h"
@ -78,6 +79,9 @@ class PrometheusMetrics: public MetricsBase {
void QueryVectorResponseSummaryObserve(double value, int count = 1) override { if (startup_) for(int i = 0 ; i < count ; ++i) query_vector_response_summary_.Observe(value);};
void QueryVectorResponsePerSecondGaugeSet(double value) override {if (startup_) query_vector_response_per_second_gauge_.Set(value);};
void CPUUsagePercentSet() override ;
void CPUCoreUsagePercentSet() override;
void RAMUsagePercentSet() override ;
void QueryResponsePerSecondGaugeSet(double value) override {if(startup_) query_response_per_second_gauge.Set(value);};
void GPUPercentGaugeSet() override ;
@ -322,7 +326,7 @@ class PrometheusMetrics: public MetricsBase {
prometheus::Gauge &faiss_disk_load_IO_speed_gauge_ = faiss_disk_load_IO_speed_.Add({{"DB","Faiss"}});
////all from CacheMgr.cpp
////all from CacheMgr.cpp
//record cache access count
prometheus::Family<prometheus::Counter> &cache_access_ = prometheus::BuildCounter()
.Name("cache_access_total")
@ -392,7 +396,8 @@ class PrometheusMetrics: public MetricsBase {
.Name("CPU_usage_percent")
.Help("CPU usage percent by this this process")
.Register(*registry_);
prometheus::Gauge &CPU_usage_percent_ = CPU_.Add({});
prometheus::Gauge &CPU_usage_percent_ = CPU_.Add({{"CPU", "0"}});
prometheus::Family<prometheus::Gauge> &RAM_ = prometheus::BuildGauge()
.Name("RAM_usage_percent")
@ -405,33 +410,12 @@ class PrometheusMetrics: public MetricsBase {
.Name("Gpu_usage_percent")
.Help("GPU_usage_percent ")
.Register(*registry_);
prometheus::Gauge &GPU0_percent_gauge_ = GPU_percent_.Add({{"DeviceNum", "0"}});
prometheus::Gauge &GPU1_percent_gauge_ = GPU_percent_.Add({{"DeviceNum", "1"}});
prometheus::Gauge &GPU2_percent_gauge_ = GPU_percent_.Add({{"DeviceNum", "2"}});
prometheus::Gauge &GPU3_percent_gauge_ = GPU_percent_.Add({{"DeviceNum", "3"}});
prometheus::Gauge &GPU4_percent_gauge_ = GPU_percent_.Add({{"DeviceNum", "4"}});
prometheus::Gauge &GPU5_percent_gauge_ = GPU_percent_.Add({{"DeviceNum", "5"}});
prometheus::Gauge &GPU6_percent_gauge_ = GPU_percent_.Add({{"DeviceNum", "6"}});
prometheus::Gauge &GPU7_percent_gauge_ = GPU_percent_.Add({{"DeviceNum", "7"}});
// std::vector<prometheus::Gauge> GPU_percent_gauges_;
//GPU Mempry used
prometheus::Family<prometheus::Gauge> &GPU_memory_usage_ = prometheus::BuildGauge()
.Name("GPU_memory_usage_total")
.Help("GPU memory usage total ")
.Register(*registry_);
prometheus::Gauge &GPU0_memory_usage_gauge_ = GPU_memory_usage_.Add({{"DeviceNum", "0"}});
prometheus::Gauge &GPU1_memory_usage_gauge_ = GPU_memory_usage_.Add({{"DeviceNum", "1"}});
prometheus::Gauge &GPU2_memory_usage_gauge_ = GPU_memory_usage_.Add({{"DeviceNum", "2"}});
prometheus::Gauge &GPU3_memory_usage_gauge_ = GPU_memory_usage_.Add({{"DeviceNum", "3"}});
prometheus::Gauge &GPU4_memory_usage_gauge_ = GPU_memory_usage_.Add({{"DeviceNum", "4"}});
prometheus::Gauge &GPU5_memory_usage_gauge_ = GPU_memory_usage_.Add({{"DeviceNum", "5"}});
prometheus::Gauge &GPU6_memory_usage_gauge_ = GPU_memory_usage_.Add({{"DeviceNum", "6"}});
prometheus::Gauge &GPU7_memory_usage_gauge_ = GPU_memory_usage_.Add({{"DeviceNum", "7"}});
// std::vector<prometheus::Gauge> GPU_memory_usage_gauges_;
prometheus::Family<prometheus::Gauge> &query_index_type_per_second_ = prometheus::BuildGauge()
.Name("query_index_throughtout_per_microsecond")

70
cpp/src/metrics/SystemInfo.cpp
View File

@ -105,9 +105,65 @@ SystemInfo::GetProcessUsedMemory() {
double
SystemInfo::MemoryPercent() {
if (!initialized_) Init();
return GetProcessUsedMemory()*100/total_ram_;
return (double)(GetProcessUsedMemory()*100)/(double)total_ram_;
}
std::vector<double>
SystemInfo::CPUCorePercent() {
std::vector<unsigned long long> prev_work_time_array;
std::vector<unsigned long long> prev_total_time_array = getTotalCpuTime(prev_work_time_array);
usleep(100000);
std::vector<unsigned long long> cur_work_time_array;
std::vector<unsigned long long> cur_total_time_array = getTotalCpuTime(cur_work_time_array);
std::vector<double> cpu_core_percent;
for (int i = 0; i < num_processors_; i++) {
double total_cpu_time = cur_total_time_array[i] - prev_total_time_array[i];
double cpu_work_time = cur_work_time_array[i] - prev_work_time_array[i];
cpu_core_percent.push_back((cpu_work_time / total_cpu_time) * 100);
}
return cpu_core_percent;
}
std::vector<unsigned long long>
SystemInfo::getTotalCpuTime(std::vector<unsigned long long> &work_time_array)
{
std::vector<unsigned long long> total_time_array;
FILE* file = fopen("/proc/stat", "r");
if (file == NULL) {
perror("Could not open stat file");
return total_time_array;
}
unsigned long long user = 0, nice = 0, system = 0, idle = 0;
unsigned long long iowait = 0, irq = 0, softirq = 0, steal = 0, guest = 0, guestnice = 0;
for (int i = 0; i < num_processors_; i++) {
char buffer[1024];
char* ret = fgets(buffer, sizeof(buffer) - 1, file);
if (ret == NULL) {
perror("Could not read stat file");
fclose(file);
return total_time_array;
}
sscanf(buffer,
"cpu %16llu %16llu %16llu %16llu %16llu %16llu %16llu %16llu %16llu %16llu",
&user, &nice, &system, &idle, &iowait, &irq, &softirq, &steal, &guest, &guestnice);
work_time_array.push_back(user + nice + system);
total_time_array.push_back(user + nice + system + idle + iowait + irq + softirq + steal);
}
fclose(file);
return total_time_array;
}
double
SystemInfo::CPUPercent() {
if (!initialized_) Init();
@ -136,17 +192,17 @@ SystemInfo::CPUPercent() {
}
std::vector<unsigned int>
SystemInfo::GPUPercent() {
std::vector<unsigned long long>
SystemInfo::GPUMemoryTotal() {
// get GPU usage percent
if(!initialized_) Init();
std::vector<unsigned int> result;
nvmlUtilization_t utilization;
std::vector<unsigned long long > result;
nvmlMemory_t nvmlMemory;
for (int i = 0; i < num_device_; ++i) {
nvmlDevice_t device;
nvmlDeviceGetHandleByIndex(i, &device);
nvmlDeviceGetUtilizationRates(device, &utilization);
result.push_back(utilization.gpu);
nvmlDeviceGetMemoryInfo(device, &nvmlMemory);
result.push_back(nvmlMemory.total);
}
return result;
}

7
cpp/src/metrics/SystemInfo.h
View File

@ -46,6 +46,7 @@ class SystemInfo {
}
void Init();
int num_processor() const { return num_processors_;};
int num_device() const {return num_device_;};
unsigned long long get_inoctets() { return in_octets_;};
unsigned long long get_octets() { return out_octets_;};
@ -59,9 +60,13 @@ class SystemInfo {
double MemoryPercent();
double CPUPercent();
std::pair<unsigned long long , unsigned long long > Octets();
std::vector<unsigned int> GPUPercent();
std::vector<unsigned long long> GPUMemoryTotal();
std::vector<unsigned long long> GPUMemoryUsed();
std::vector<double> CPUCorePercent();
std::vector<unsigned long long> getTotalCpuTime(std::vector<unsigned long long> &workTime);
};
}