diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 94972363fa..efb810e2a5 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -15,6 +15,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-6 - Implement SDK interface part 1 - MS-21 - Implement SDK interface part 2 - MS-26 - cmake. Add thirdparty packages +- MS-31 - cmake: add prometheus ### Task diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index b6b8ad1aa0..f4e32948dd 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -70,9 +70,9 @@ endif() set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") -if (BUILD_UNIT_TEST) - option(MEGASEARCH_BUILD_TESTS "Build the megasearch test suite" ON) -endif(BUILD_UNIT_TEST) +#if (BUILD_UNIT_TEST) +# option(MEGASEARCH_BUILD_TESTS "Build the megasearch test suite" ON) +#endif(BUILD_UNIT_TEST) include(ExternalProject) include(DefineOptions) diff --git a/cpp/cmake/DefineOptions.cmake b/cpp/cmake/DefineOptions.cmake index 3171402cf1..ce2e4ae6be 100644 --- a/cpp/cmake/DefineOptions.cmake +++ b/cpp/cmake/DefineOptions.cmake @@ -53,7 +53,7 @@ define_option_string(MEGASEARCH_DEPENDENCY_SOURCE "SYSTEM") define_option(MEGASEARCH_VERBOSE_THIRDPARTY_BUILD - "Show output from ExternalProjects rather than just logging to files" OFF) + "Show output from ExternalProjects rather than just logging to files" ON) define_option(MEGASEARCH_BOOST_USE_SHARED "Rely on boost shared libraries where relevant" OFF) @@ -79,6 +79,8 @@ define_option(MEGASEARCH_WITH_LZ4 "Build with lz4 compression" ON) define_option(MEGASEARCH_WITH_OPENBLAS "Build with OpenBLAS library" ON) +define_option(MEGASEARCH_WITH_PROMETHEUS "Build with PROMETHEUS library" ON) + define_option(MEGASEARCH_WITH_ROCKSDB "Build with RocksDB library" ON) define_option(MEGASEARCH_WITH_SNAPPY "Build with Snappy compression" ON) @@ -114,9 +116,13 @@ endif() #---------------------------------------------------------------------- -#set_option_category("Test and benchmark") -# -#define_option(MEGASEARCH_BUILD_TESTS "Build the MEGASEARCH googletest unit tests" ON) +set_option_category("Test and benchmark") + +if (BUILD_UNIT_TEST) + define_option(MEGASEARCH_BUILD_TESTS "Build the MEGASEARCH googletest unit tests" ON) +else() + define_option(MEGASEARCH_BUILD_TESTS "Build the MEGASEARCH googletest unit tests" OFF) +endif(BUILD_UNIT_TEST) #---------------------------------------------------------------------- macro(config_summary) diff --git a/cpp/cmake/GoogleTest.cmake b/cpp/cmake/GoogleTest.cmake deleted file mode 100644 index 9ea62af1c4..0000000000 --- a/cpp/cmake/GoogleTest.cmake +++ /dev/null @@ -1,92 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Tries to find GTest headers and libraries. -# -# Usage of this module as follows: -# -# find_package(GTest) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# GTest_HOME - When set, this path is inspected instead of standard library -# locations as the root of the GTest installation. -# The environment variable GTEST_HOME overrides this veriable. -# -# This module defines -# GTEST_INCLUDE_DIR, directory containing headers -# GTEST_LIBS, directory containing gtest libraries -# GTEST_STATIC_LIB, path to libgtest.a -# GTEST_SHARED_LIB, path to libgtest's shared library -# GTEST_FOUND, whether gtest has been found - -if( NOT "${GTEST_HOME}" STREQUAL "") - file( TO_CMAKE_PATH "${GTEST_HOME}" _native_path ) - list( APPEND _gtest_roots ${_native_path} ) -elseif ( GTest_HOME ) - list( APPEND _gtest_roots ${GTest_HOME} ) -endif() - -# Try the parameterized roots, if they exist -if ( _gtest_roots ) - find_path( GTEST_INCLUDE_DIR NAMES gtest/gtest.h - PATHS ${_gtest_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "include" ) - find_library( GTEST_LIBRARIES NAMES gtest gtest_main - PATHS ${_gtest_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "lib" ) -else () - find_path( GTEST_INCLUDE_DIR NAMES gtest/gtest.h ) - find_library( GTEST_LIBRARIES NAMES gtest ) -endif () - - -if (GTEST_INCLUDE_DIR AND GTEST_LIBRARIES) - set(GTEST_FOUND TRUE) - get_filename_component( GTEST_LIBS ${GTEST_LIBRARIES} PATH ) - set(GTEST_LIB_NAME gtest) - set(GTEST_STATIC_LIB ${GTEST_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GTEST_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}) - set(GTEST_MAIN_STATIC_LIB ${GTEST_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GTEST_LIB_NAME}_main${CMAKE_STATIC_LIBRARY_SUFFIX}) - set(GTEST_SHARED_LIB ${GTEST_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${GTEST_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) -else () - set(GTEST_FOUND FALSE) -endif () - -if (GTEST_FOUND) - if (NOT GTest_FIND_QUIETLY) - message(STATUS "Found the GTest library: ${GTEST_LIBRARIES}") - endif () -else () - if (NOT GTest_FIND_QUIETLY) - set(GTEST_ERR_MSG "Could not find the GTest library. Looked in ") - if ( _gtest_roots ) - set(GTEST_ERR_MSG "${GTEST_ERR_MSG} in ${_gtest_roots}.") - else () - set(GTEST_ERR_MSG "${GTEST_ERR_MSG} system search paths.") - endif () - if (GTest_FIND_REQUIRED) - message(FATAL_ERROR "${GTEST_ERR_MSG}") - else (GTest_FIND_REQUIRED) - message(STATUS "${GTEST_ERR_MSG}") - endif (GTest_FIND_REQUIRED) - endif () -endif () - -mark_as_advanced( - GTEST_INCLUDE_DIR - GTEST_LIBS - GTEST_LIBRARIES - GTEST_STATIC_LIB - GTEST_SHARED_LIB -) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 121799d9eb..0b08cd358a 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -25,6 +25,7 @@ set(MEGASEARCH_THIRDPARTY_DEPENDENCIES LAPACK Lz4 OpenBLAS + Prometheus RocksDB Snappy SQLite @@ -58,6 +59,8 @@ macro(build_dependency DEPENDENCY_NAME) build_gtest() elseif ("${DEPENDENCY_NAME}" STREQUAL "OpenBLAS") build_openblas() + elseif ("${DEPENDENCY_NAME}" STREQUAL "Prometheus") + build_prometheus() elseif ("${DEPENDENCY_NAME}" STREQUAL "RocksDB") build_rocksdb() elseif ("${DEPENDENCY_NAME}" STREQUAL "Snappy") @@ -244,6 +247,13 @@ else () "https://github.com/xianyi/OpenBLAS/archive/${OPENBLAS_VERSION}.tar.gz") endif() +if (DEFINED ENV{MEGASEARCH_PROMETHEUS_URL}) + set(PROMETHEUS_SOURCE_URL "$ENV{PROMETHEUS_OPENBLAS_URL}") +else () + set(PROMETHEUS_SOURCE_URL + "https://github.com/JinHai-CN/prometheus-cpp/archive/${PROMETHEUS_VERSION}.tar.gz") +endif() + if (DEFINED ENV{MEGASEARCH_ROCKSDB_URL}) set(ROCKSDB_SOURCE_URL "$ENV{MEGASEARCH_ROCKSDB_URL}") else () @@ -796,7 +806,6 @@ macro(build_gtest) endmacro() -message(STATUS "MEGASEARCH_BUILD_TESTS: ${MEGASEARCH_BUILD_TESTS}") if (MEGASEARCH_BUILD_TESTS) #message(STATUS "Resolving gtest dependency") resolve_dependency(GTest) @@ -884,6 +893,92 @@ if(MEGASEARCH_WITH_LZ4) include_directories(SYSTEM ${LZ4_INCLUDE_DIR}) endif() +# ---------------------------------------------------------------------- +# Prometheus + +macro(build_prometheus) + message(STATUS "Building Prometheus-${PROMETHEUS_VERSION} from source") + set(PROMETHEUS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/prometheus_ep-prefix/src/prometheus_ep") + set(PROMETHEUS_STATIC_LIB_NAME prometheus-cpp) + set(PROMETHEUS_CORE_STATIC_LIB + "${PROMETHEUS_PREFIX}/core/${CMAKE_STATIC_LIBRARY_PREFIX}${PROMETHEUS_STATIC_LIB_NAME}-core${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(PROMETHEUS_PUSH_STATIC_LIB + "${PROMETHEUS_PREFIX}/push/${CMAKE_STATIC_LIBRARY_PREFIX}${PROMETHEUS_STATIC_LIB_NAME}-push${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + set(PROMETHEUS_PULL_STATIC_LIB + "${PROMETHEUS_PREFIX}/pull/${CMAKE_STATIC_LIBRARY_PREFIX}${PROMETHEUS_STATIC_LIB_NAME}-pull${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + + set(PROMETHEUS_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + -DCMAKE_INSTALL_LIBDIR=lib + -DBUILD_SHARED_LIBS=OFF + "-DCMAKE_INSTALL_PREFIX=${PROMETHEUS_PREFIX}") + + externalproject_add(prometheus_ep + URL + ${PROMETHEUS_SOURCE_URL} + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${PROMETHEUS_CMAKE_ARGS} + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + BUILD_IN_SOURCE + 1 + INSTALL_COMMAND + ${MAKE} + "DESTDIR=${PROMETHEUS_PREFIX}" + install + BUILD_BYPRODUCTS + "${PROMETHEUS_CORE_STATIC_LIB}" + "${PROMETHEUS_PUSH_STATIC_LIB}" + "${PROMETHEUS_PULL_STATIC_LIB}") + + #file(MAKE_DIRECTORY "${PROMETHEUS_PREFIX}/include") + + add_library(prometheus-cpp-push STATIC IMPORTED) + set_target_properties(prometheus-cpp-push + PROPERTIES IMPORTED_LOCATION "${PROMETHEUS_PUSH_STATIC_LIB}") +# INTERFACE_INCLUDE_DIRECTORIES +# "${PROMETHEUS_PREFIX}/push/include") + add_dependencies(prometheus-cpp-push prometheus_ep) + + add_library(prometheus-cpp-pull STATIC IMPORTED) + set_target_properties(prometheus-cpp-pull + PROPERTIES IMPORTED_LOCATION "${PROMETHEUS_PULL_STATIC_LIB}") +# INTERFACE_INCLUDE_DIRECTORIES +# "${PROMETHEUS_PREFIX}/pull/include") + add_dependencies(prometheus-cpp-pull prometheus_ep) + + add_library(prometheus-cpp-core STATIC IMPORTED) + set_target_properties(prometheus-cpp-core + PROPERTIES IMPORTED_LOCATION "${PROMETHEUS_CORE_STATIC_LIB}") +# INTERFACE_INCLUDE_DIRECTORIES +# "${PROMETHEUS_PREFIX}/core/include") + add_dependencies(prometheus-cpp-core prometheus_ep) +endmacro() + +if(MEGASEARCH_WITH_PROMETHEUS) + + resolve_dependency(Prometheus) + + # TODO: Don't use global includes but rather target_include_directories + #get_target_property(PROMETHEUS-core_INCLUDE_DIRS prometheus-core INTERFACE_INCLUDE_DIRECTORIES) + + #get_target_property(PROMETHEUS_PUSH_INCLUDE_DIRS prometheus_push INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM ${PROMETHEUS_PREFIX}/push/) + include_directories(SYSTEM ${PROMETHEUS_PREFIX}/push/include) + + #get_target_property(PROMETHEUS_PULL_INCLUDE_DIRS prometheus_pull INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM ${PROMETHEUS_PREFIX}/pull/) + include_directories(SYSTEM ${PROMETHEUS_PREFIX}/pull/include) + + link_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/) + include_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/include) +endif() + # ---------------------------------------------------------------------- # RocksDB @@ -926,6 +1021,7 @@ macro(build_rocksdb) endmacro() if(MEGASEARCH_WITH_ROCKSDB) + resolve_dependency(RocksDB) # TODO: Don't use global includes but rather target_include_directories diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 7947d07c99..75a593af26 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -79,6 +79,9 @@ set(third_party_libs faiss lapack openblas + prometheus-cpp-push + prometheus-cpp-pull + prometheus-cpp-core rocksdb boost_system_static boost_filesystem_static diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 98e5381a56..6357f5dfa9 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -31,6 +31,7 @@ GTEST_VERSION=1.8.1 LAPACK_VERSION=v3.8.0 LZ4_VERSION=v1.9.1 OPENBLAS_VERSION=v0.3.6 +PROMETHEUS_VERSION=master ROCKSDB_VERSION=v6.0.2 SNAPPY_VERSION=1.1.7 SQLITE_VERSION=3280000 @@ -53,6 +54,7 @@ DEPENDENCIES=( "MEGASEARCH_LAPACK_URL lapack-${LAPACK_VERSION}.tar.gz https://github.com/Reference-LAPACK/lapack/archive/${LAPACK_VERSION}.tar.gz "MEGASEARCH_LZ4_URL lz4-${LZ4_VERSION}.tar.gz https://github.com/lz4/lz4/archive/${LZ4_VERSION}.tar.gz" "MEGASEARCH_OPENBLAS_URL openblas-${OPENBLAS_VERSION}.tar.gz https://github.com/xianyi/OpenBLAS/archive/${OPENBLAS_VERSION}.tar.gz" + "MEGASEARCH_PROMETHEUS_URL prometheus-${PROMETHEUS_VERSION}.tar.gz https://github.com/JinHai-CN/prometheus-cpp/archive/${PROMETHEUS_VERSION}.tar.gz" "MEGASEARCH_ROCKSDB_URL rocksdb-${ROCKSDB_VERSION}.tar.gz https://github.com/facebook/rocksdb/archive/${ROCKSDB_VERSION}.tar.gz" "MEGASEARCH_SNAPPY_URL snappy-${SNAPPY_VERSION}.tar.gz https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz" "MEGASEARCH_SQLITE_URL sqlite-autoconf-${SQLITE_VERSION}.tar.gz https://www.sqlite.org/2019/sqlite-autoconf-${SQLITE_VERSION}.tar.gz"