From 958e94f6f0343f34f96a2026b3593f257e97cfe6 Mon Sep 17 00:00:00 2001 From: Enwei Jiao Date: Wed, 23 Nov 2022 10:39:11 +0800 Subject: [PATCH] Use Conan as c++ package manager (#19920) Signed-off-by: Enwei Jiao Signed-off-by: Enwei Jiao --- .env | 6 +- .github/workflows/code-checker.yaml | 35 ++++-- .github/workflows/mac.yaml | 20 ++-- .github/workflows/main.yaml | 24 +++- .gitignore | 3 +- Makefile | 8 +- build/builder.sh | 1 + build/docker/milvus/ubuntu20.04/Dockerfile | 2 +- docker-compose.yml | 13 +- internal/core/CMakeLists.txt | 36 ++++-- internal/core/build-support/run_cpplint.py | 58 ++++----- internal/core/cmake/DefineOptions.cmake | 8 -- internal/core/conanfile.txt | 32 +++++ internal/core/src/common/CMakeLists.txt | 32 ++--- internal/core/src/pb/CMakeLists.txt | 7 +- internal/core/src/pb/common.pb.h | 2 +- internal/core/src/pb/index_cgo_msg.pb.h | 2 +- internal/core/src/pb/plan.pb.h | 2 +- internal/core/src/pb/schema.pb.h | 2 +- internal/core/src/pb/segcore.pb.h | 2 +- internal/core/src/segcore/CMakeLists.txt | 6 +- internal/core/src/segcore/Reduce.cpp | 2 +- internal/core/src/segcore/segment_c.cpp | 2 +- internal/core/src/storage/CMakeLists.txt | 15 +-- internal/core/thirdparty/CMakeLists.txt | 41 ------- internal/core/thirdparty/arrow/CMakeLists.txt | 111 ------------------ .../core/thirdparty/aws_sdk/CMakeLists.txt | 77 ------------ .../google_benchmark/CMakeLists.txt | 20 ---- internal/core/thirdparty/gtest/CMakeLists.txt | 65 ---------- .../core/thirdparty/jemalloc/CMakeLists.txt | 80 ------------- .../core/thirdparty/knowhere/CMakeLists.txt | 1 + .../core/thirdparty/marisa/CMakeLists.txt | 74 ------------ .../thirdparty/opentracing/CMakeLists.txt | 68 ----------- .../core/thirdparty/protobuf/CMakeLists.txt | 47 -------- .../core/thirdparty/rapidxml/CMakeLists.txt | 20 ---- .../core/thirdparty/rocksdb/CMakeLists.txt | 81 ++----------- .../core/thirdparty/rocksdb/rocksdb-win.pc.in | 7 +- .../core/thirdparty/rocksdb/rocksdb.pc.in | 1 - .../core/thirdparty/yaml-cpp/CMakeLists.txt | 50 -------- internal/core/unittest/bench/CMakeLists.txt | 4 +- internal/core/unittest/test_c_api.cpp | 2 +- internal/core/unittest/test_string_index.cpp | 6 +- .../test_utils/indexbuilder_test_utils.h | 6 +- scripts/core_build.sh | 82 ++++++------- scripts/install_deps.sh | 61 ++++------ scripts/install_deps_msys.sh | 11 +- 46 files changed, 278 insertions(+), 957 deletions(-) create mode 100644 internal/core/conanfile.txt delete mode 100644 internal/core/thirdparty/arrow/CMakeLists.txt delete mode 100644 internal/core/thirdparty/aws_sdk/CMakeLists.txt delete mode 100644 internal/core/thirdparty/google_benchmark/CMakeLists.txt delete mode 100644 internal/core/thirdparty/gtest/CMakeLists.txt delete mode 100644 internal/core/thirdparty/jemalloc/CMakeLists.txt delete mode 100644 internal/core/thirdparty/marisa/CMakeLists.txt delete mode 100644 internal/core/thirdparty/opentracing/CMakeLists.txt delete mode 100644 internal/core/thirdparty/protobuf/CMakeLists.txt delete mode 100644 internal/core/thirdparty/rapidxml/CMakeLists.txt delete mode 100644 internal/core/thirdparty/yaml-cpp/CMakeLists.txt diff --git a/.env b/.env index fe06f4a434..5bcf24488d 100644 --- a/.env +++ b/.env @@ -1,8 +1,8 @@ IMAGE_REPO=milvusdb IMAGE_ARCH=amd64 OS_NAME=ubuntu20.04 -DATE_VERSION=20220915-18817a4 -LATEST_DATE_VERSION=20220915-18817a4 +DATE_VERSION=20221121-0a289c0 +LATEST_DATE_VERSION=20221121-0a289c0 MINIO_ADDRESS=minio:9000 PULSAR_ADDRESS=pulsar://pulsar:6650 -ETCD_ENDPOINTS=etcd:2379 \ No newline at end of file +ETCD_ENDPOINTS=etcd:2379 diff --git a/.github/workflows/code-checker.yaml b/.github/workflows/code-checker.yaml index b8558de4c2..3f13be3b97 100644 --- a/.github/workflows/code-checker.yaml +++ b/.github/workflows/code-checker.yaml @@ -32,14 +32,14 @@ jobs: fail-fast: false matrix: include: - - name: Code Checker AMD64 Ubuntu 20.04 - os: ubuntu-20.04 + - name: Code Checker AMD64 Ubuntu 20.04 + os: ubuntu-20.04 env: UBUNTU: 20.04 steps: - name: Checkout uses: actions/checkout@v2 - - name: "Generate CCache Hash" + - name: 'Generate CCache Hash' env: CORE_HASH: ${{ hashFiles( 'internal/core/**/*.cpp', 'internal/core/**/*.cc', 'internal/core/**/*.c', 'internal/core/**/*.h', 'internal/core/**/*.hpp', 'internal/core/**/CMakeLists.txt') }} run: | @@ -63,21 +63,26 @@ jobs: path: .docker/amd64-ubuntu20.04-go-mod key: ubuntu20.04-go-mod-${{ hashFiles('**/go.sum') }} restore-keys: ubuntu20.04-go-mod- + - name: Cache Conan Packages + uses: actions/cache@v3 + with: + path: .docker/amd64-ubuntu20.04-conan + key: ubuntu20.04-conan-${{ hashFiles('internal/core/conanfile.txt') }} + restore-keys: ubuntu20.04-conan- - name: Code Check env: -# CHECK_BUILDER: "1" - OS_NAME: "ubuntu20.04" + OS_NAME: 'ubuntu20.04' run: | - ./build/builder.sh /bin/bash -c "make check-proto-product && make verifiers" + ./build/builder.sh /bin/bash -c "make check-proto-product && make verifiers" centos: name: Code Checker CentOS 7 - # Run in centos docker + # Run in centos docker runs-on: ubuntu-20.04 timeout-minutes: 60 steps: - name: Checkout uses: actions/checkout@v2 - - name: "Generate CCache Hash" + - name: 'Generate CCache Hash' env: CORE_HASH: ${{ hashFiles( 'internal/core/**/*.cpp', 'internal/core/**/*.cc', 'internal/core/**/*.c', 'internal/core/**/*.h', 'internal/core/**/*.hpp', 'internal/core/**/CMakeLists.txt') }} run: | @@ -101,11 +106,17 @@ jobs: path: .docker/amd64-centos7-go-mod key: centos7-go-mod-${{ hashFiles('**/go.sum') }} restore-keys: centos7-go-mod- + - name: Cache Conan Packages + uses: actions/cache@v3 + with: + path: .docker/amd64-centos7-conan + key: centos7-conan-${{ hashFiles('internal/core/conanfile.txt') }} + restore-keys: centos7-conan- - name: Code Check env: - OS_NAME: "centos7" + OS_NAME: 'centos7' run: | - ./build/builder.sh /bin/bash -c "make install" + ./build/builder.sh /bin/bash -c "make install" mingw64: name: ${{ matrix.name }} runs-on: ${{ matrix.os }} @@ -114,8 +125,8 @@ jobs: fail-fast: false matrix: include: - - name: Code Checker MinGW - os: windows-2019 + - name: Code Checker MinGW + os: windows-2019 steps: - name: Set git to use LF run: | diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 820aa3a4fa..e4c61feddc 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -26,11 +26,11 @@ jobs: mac: name: Code Checker MacOS 12 runs-on: macos-12 - timeout-minutes: 60 + timeout-minutes: 120 steps: - name: Checkout uses: actions/checkout@v2 - - name: "Generate CCache Hash" + - name: 'Generate CCache Hash' env: CORE_HASH: ${{ hashFiles( 'internal/core/**/*.cpp', 'internal/core/**/*.cc', 'internal/core/**/*.c', 'internal/core/**/*.h', 'internal/core/**/*.hpp', 'internal/core/**/CMakeLists.txt') }} run: | @@ -40,10 +40,10 @@ jobs: uses: actions/cache@v3 with: path: | - /tmp/thirdparty + /tmp/thirdparty key: macos-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }} restore-keys: macos-thirdparty- - - name: Mac Cache Cmake Build + - name: Mac Cache Cmake Build uses: actions/cache@v3 with: path: | @@ -66,6 +66,12 @@ jobs: path: ~/go/pkg/mod key: macos-go-mod-${{ hashFiles('**/go.sum') }} restore-keys: macos-go-mod- + - name: Mac Cache Conan Packages + uses: pat-s/always-upload-cache@v3 + with: + path: ~/.conan + key: macos-conan-${{ hashFiles('internal/core/conanfile.txt') }} + restore-keys: macos-conan- - name: Code Check env: CCACHE_DIR: /var/tmp/ccache @@ -78,14 +84,14 @@ jobs: if [[ ! -d "/var/tmp/ccache" ]];then mkdir -p /var/tmp/ccache fi - brew install boost libomp ninja tbb openblas ccache + brew install boost libomp ninja tbb openblas ccache conan if [[ ! -d "/usr/local/opt/llvm" ]]; then ln -s /usr/local/opt/llvm@14 /usr/local/opt/llvm fi - make milvus && sh scripts/collect_arrow_dep.sh -f /tmp/thirdparty + make milvus - name: Upload Cmake log uses: actions/upload-artifact@v3 if: ${{ failure() }} with: name: cmake-log - path: cmake_build/CMakeFiles/*.log \ No newline at end of file + path: cmake_build/CMakeFiles/*.log diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 105a1434ba..d752fd10bd 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -36,7 +36,7 @@ jobs: UT: name: Build and test AMD64 Ubuntu ${{ matrix.ubuntu }} runs-on: ubuntu-${{ matrix.ubuntu }} - timeout-minutes: 60 + timeout-minutes: 120 strategy: fail-fast: false matrix: @@ -46,7 +46,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - - name: "Generate CCache Hash" + - name: 'Generate CCache Hash' env: CORE_HASH: ${{ hashFiles( 'internal/core/**/*.cpp', 'internal/core/**/*.cc', 'internal/core/**/*.c', 'internal/core/**/*.h', 'internal/core/**/*.hpp', 'internal/core/**/CMakeLists.txt') }} run: | @@ -70,6 +70,12 @@ jobs: path: .docker/amd64-ubuntu${{ matrix.ubuntu }}-go-mod key: ubuntu${{ matrix.ubuntu }}-go-mod-${{ hashFiles('**/go.sum') }} restore-keys: ubuntu${{ matrix.ubuntu }}-go-mod- + - name: Cache Conan Packages + uses: actions/cache@v3 + with: + path: .docker/amd64-ubuntu${{ matrix.ubuntu }}-conan + key: ubuntu${{ matrix.ubuntu }}-conan-${{ hashFiles('internal/core/conanfile.txt') }} + restore-keys: ubuntu${{ matrix.ubuntu }}-conan- - name: Start Service shell: bash run: | @@ -79,7 +85,7 @@ jobs: ./build/builder.sh /bin/bash -c "make ci-ut" - name: Get the status & restart count for containers when ut failed if: ${{ failure() }} - run: | + run: | echo "----show current status for each container---" docker ps --format "table {{.Names}}\t\t{{.Image}}\t\t{{.Status}}" for container in $(docker ps --format "table {{.Names}}" | grep -v "NAMES"); do @@ -101,7 +107,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - name: Download code coverage results + - name: Download code coverage results uses: actions/download-artifact@v3.0.1 with: name: code-coverage-result @@ -118,4 +124,12 @@ jobs: files: ./go_coverage.txt,./lcov_output.info name: ubuntu-20.04-unittests fail_ci_if_error: true - verbose: true \ No newline at end of file + - name: Retry Upload coverage to Codecov + if: "${{ failure() }} && github.repository == 'milvus-io/milvus'" + uses: codecov/codecov-action@v3.1.1 + id: retry_upload_cov + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./go_coverage.txt,./lcov_output.info + name: ubuntu-${{ matrix.ubuntu }}-unittests + fail_ci_if_error: true diff --git a/.gitignore b/.gitignore index e70f3797f3..07cced4e7c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ **/cmake-build-release/* **/cmake_build_release/* **/cmake_build/* +**/CmakeFiles/* .cache internal/core/output/* @@ -91,4 +92,4 @@ cwrapper_rocksdb_build/ internal/kv/rocksdb/cwrapper/ # local file data -**/data/* \ No newline at end of file +**/data/* diff --git a/Makefile b/Makefile index 33da871f1d..f6b45ba26f 100644 --- a/Makefile +++ b/Makefile @@ -247,11 +247,9 @@ install: milvus @echo "Installing binary to './bin'" @mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/milvus $(GOPATH)/bin/milvus @mkdir -p $(LIBRARY_PATH) -ifeq ($(OS),Darwin) - @cp -r -P $(PWD)/internal/core/output/lib/*.dylib* $(LIBRARY_PATH) -else - @cp -r -P $(PWD)/internal/core/output/lib/*.so* $(LIBRARY_PATH) -endif + -cp -r -P $(PWD)/internal/core/output/lib/*.dylib* $(LIBRARY_PATH) 2>/dev/null + -cp -r -P $(PWD)/internal/core/output/lib/*.so* $(LIBRARY_PATH) 2>/dev/null + -cp -r -P $(PWD)/internal/core/output/lib64/*.so* $(LIBRARY_PATH) 2>/dev/null @echo "Installation successful." clean: diff --git a/build/builder.sh b/build/builder.sh index cbf5ff53d0..29d955b999 100755 --- a/build/builder.sh +++ b/build/builder.sh @@ -36,6 +36,7 @@ mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/amd64-${OS_NAME}-ccache" mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/amd64-${OS_NAME}-go-mod" mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/thirdparty" mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/amd64-${OS_NAME}-vscode-extensions" +mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/amd64-${OS_NAME}-conan" chmod -R 777 "${DOCKER_VOLUME_DIRECTORY:-.docker}" docker-compose pull --ignore-pull-failures builder diff --git a/build/docker/milvus/ubuntu20.04/Dockerfile b/build/docker/milvus/ubuntu20.04/Dockerfile index c067799e01..f5c2d63fd6 100644 --- a/build/docker/milvus/ubuntu20.04/Dockerfile +++ b/build/docker/milvus/ubuntu20.04/Dockerfile @@ -20,4 +20,4 @@ COPY ./lib/ /milvus/lib/ ENV PATH=/milvus/bin:$PATH ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib -ENV LD_PRELOAD=/milvus/lib/libjemalloc.so \ No newline at end of file +ENV LD_PRELOAD=/milvus/lib/libjemalloc.so diff --git a/docker-compose.yml b/docker-compose.yml index b66c9d3126..28ab931d61 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,11 +34,12 @@ services: - ${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-go-mod:/go/pkg/mod:delegated - ${DOCKER_VOLUME_DIRECTORY:-.docker}/thirdparty:/tmp/thirdparty:delegated - ${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-vscode-extensions:/home/milvus/.vscode-server/extensions:delegated - working_dir: "/go/src/github.com/milvus-io/milvus" + - ${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-conan:/home/milvus/.conan:delegated + working_dir: '/go/src/github.com/milvus-io/milvus' depends_on: - - etcd - - minio - - pulsar + - etcd + - minio + - pulsar # Command command: &builder-command > /bin/bash -c " @@ -53,7 +54,7 @@ services: - ETCD_QUOTA_BACKEND_BYTES=4294967296 - ETCD_SNAPSHOT_COUNT=50000 healthcheck: - test: ["CMD", "/opt/bitnami/scripts/etcd/healthcheck.sh"] + test: ['CMD', '/opt/bitnami/scripts/etcd/healthcheck.sh'] interval: 30s timeout: 20s retries: 3 @@ -69,7 +70,7 @@ services: MINIO_SECRET_KEY: minioadmin command: minio server /minio_data healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + test: ['CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live'] interval: 30s timeout: 20s retries: 3 diff --git a/internal/core/CMakeLists.txt b/internal/core/CMakeLists.txt index c3d87fc7f5..e4e469e1d9 100644 --- a/internal/core/CMakeLists.txt +++ b/internal/core/CMakeLists.txt @@ -93,6 +93,12 @@ set( CMAKE_INSTALL_RPATH "/usr/lib" "${CMAKE_INSTALL_PREFIX}/lib" "${CMAKE_INSTA set( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE ) # **************************** Dependencies **************************** +include( ${CMAKE_BINARY_DIR}/conan/conanbuildinfo.cmake ) +set( CONAN_DISABLE_CHECK_COMPILER ON ) +conan_basic_setup( KEEP_RPATHS ) + +message( STATUS "cmake prefix path ${CMAKE_PREFIX_PATH}") + include( CTest ) include( BuildUtils ) include( DefineOptions ) @@ -110,13 +116,13 @@ include( ThirdPartyPackages ) message( STATUS "Building Milvus CPU version" ) if (LINUX OR MSYS) -append_flags( CMAKE_CXX_FLAGS - FLAGS - "-fPIC" - "-DELPP_THREAD_SAFE" - "-fopenmp" - "-Werror" - ) + append_flags( CMAKE_CXX_FLAGS + FLAGS + "-fPIC" + "-DELPP_THREAD_SAFE" + "-fopenmp" + "-Werror" + ) endif () if ( APPLE ) @@ -174,6 +180,17 @@ add_custom_target(lint find_program( CPPLINT_BIN NAMES cpplint cpplint.py HINTS ${BUILD_SUPPORT_DIR} ) message( STATUS "Found cpplint executable at ${CPPLINT_BIN}" ) +# Use the first Python installation on PATH, not the newest one +set(Python3_FIND_STRATEGY "LOCATION") +# On Windows, use registry last, not first +set(Python3_FIND_REGISTRY "LAST") +# On macOS, use framework last, not first +set(Python3_FIND_FRAMEWORK "LAST") +find_package(Python3) +set(PYTHON_EXECUTABLE ${Python3_EXECUTABLE}) + +message( STATUS "Found Python at ${PYTHON_EXECUTABLE}") + # # "make clang-format" and "make check-clang-format" targets # @@ -276,3 +293,8 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/common/ DESTINATION include/common FILES_MATCHING PATTERN "*_c.h" ) + +install(DIRECTORY ${CMAKE_BINARY_DIR}/lib/ + DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR} +) + diff --git a/internal/core/build-support/run_cpplint.py b/internal/core/build-support/run_cpplint.py index 98c4170858..ec5de543ff 100755 --- a/internal/core/build-support/run_cpplint.py +++ b/internal/core/build-support/run_cpplint.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -31,7 +31,7 @@ from functools import partial # * readability/casting is disabled as it aggressively warns about functions # with names like "int32", so "int32(x)", where int32 is a function name, # warns with -_filters = ''' +_filters = """ -whitespace/comments -readability/casting -readability/todo @@ -40,7 +40,7 @@ _filters = ''' -build/c++11 -runtime/references -build/include_order -'''.split() +""".split() def _get_chunk_key(filenames): @@ -57,19 +57,25 @@ def _check_some_files(completed_processes, filenames): if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Runs cpplint on all of the source files.") - parser.add_argument("--cpplint_binary", - required=True, - help="Path to the cpplint binary") - parser.add_argument("--exclude_globs", - help="Filename containing globs for files " - "that should be excluded from the checks") - parser.add_argument("--source_dir", - required=True, - help="Root directory of the source code") - parser.add_argument("--quiet", default=False, - action="store_true", - help="If specified, only print errors") + description="Runs cpplint on all of the source files." + ) + parser.add_argument( + "--cpplint_binary", required=True, help="Path to the cpplint binary" + ) + parser.add_argument( + "--exclude_globs", + help="Filename containing globs for files " + "that should be excluded from the checks", + ) + parser.add_argument( + "--source_dir", required=True, help="Root directory of the source code" + ) + parser.add_argument( + "--quiet", + default=False, + action="store_true", + help="If specified, only print errors", + ) arguments = parser.parse_args() exclude_globs = [] @@ -83,20 +89,18 @@ if __name__ == "__main__": cmd = [ arguments.cpplint_binary, - '--verbose=2', - '--linelength=120', - '--filter=' + ','.join(_filters) + "--verbose=2", + "--linelength=120", + "--filter=" + ",".join(_filters), ] - if (arguments.cpplint_binary.endswith('.py') and - platform.system() == 'Windows'): + if arguments.cpplint_binary.endswith(".py") and platform.system() == "Windows": # Windows doesn't support executable scripts; execute with # sys.executable cmd.insert(0, sys.executable) if arguments.quiet: - cmd.append('--quiet') + cmd.append("--quiet") else: - print("\n".join(map(lambda x: "Linting {}".format(x), - linted_filenames))) + print("\n".join(map(lambda x: "Linting {}".format(x), linted_filenames))) # lint files in chunks: each invocation of cpplint will process 16 files chunks = lintutils.chunk(linted_filenames, 16) @@ -107,8 +111,7 @@ if __name__ == "__main__": # record completed processes (keyed by the first filename in the input # chunk) for lookup in _check_some_files completed_processes = { - _get_chunk_key(filenames): result - for filenames, result in zip(chunks, results) + _get_chunk_key(filenames): result for filenames, result in zip(chunks, results) } checker = partial(_check_some_files, completed_processes) pool = mp.Pool() @@ -118,7 +121,7 @@ if __name__ == "__main__": for problem_files, stdout in pool.imap(checker, chunks): if problem_files: if isinstance(stdout, bytes): - stdout = stdout.decode('utf8') + stdout = stdout.decode("utf8") print(stdout, file=sys.stderr) error = True except Exception: @@ -129,4 +132,3 @@ if __name__ == "__main__": pool.join() sys.exit(1 if error else 0) - diff --git a/internal/core/cmake/DefineOptions.cmake b/internal/core/cmake/DefineOptions.cmake index a356e433a4..0b57ee97ef 100644 --- a/internal/core/cmake/DefineOptions.cmake +++ b/internal/core/cmake/DefineOptions.cmake @@ -62,14 +62,6 @@ define_option(MILVUS_USE_CCACHE "Use ccache when compiling (if available)" ON) define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD "Show output from ExternalProjects rather than just logging to files" ON) -define_option(MILVUS_WITH_EASYLOGGINGPP "Build with Easylogging++ library" ON) - -define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON) - -define_option(MILVUS_WITH_OPENTRACING "Build with Opentracing" ON) - -define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON) - #---------------------------------------------------------------------- set_option_category("Test and benchmark") diff --git a/internal/core/conanfile.txt b/internal/core/conanfile.txt new file mode 100644 index 0000000000..b83265d1b0 --- /dev/null +++ b/internal/core/conanfile.txt @@ -0,0 +1,32 @@ +[requires] +rocksdb/6.29.5 +boost/1.80.0 +onetbb/2021.3.0 +zstd/1.5.2 +arrow/8.0.1 +openssl/1.1.1q +aws-sdk-cpp/1.9.234 +benchmark/1.7.0 +gtest/1.8.1 +protobuf/3.9.1 +rapidxml/1.13 +yaml-cpp/0.7.0 +marisa/0.2.6 +zlib/1.2.13 + +[generators] +cmake + +[options] +rocksdb:shared=True +arrow:parquet=True +arrow:compute=True +arrow:with_zstd=True +aws-sdk-cpp:text-to-speech=False +aws-sdk-cpp:transfer=False +gtest:build_gmock=False + +[imports] +lib, *.dylib -> ../lib +lib, *.dll -> ../lib +lib, *.so* -> ../lib diff --git a/internal/core/src/common/CMakeLists.txt b/internal/core/src/common/CMakeLists.txt index 3b315c5cf4..6f79200d2e 100644 --- a/internal/core/src/common/CMakeLists.txt +++ b/internal/core/src/common/CMakeLists.txt @@ -22,27 +22,15 @@ set(COMMON_SRC add_library(milvus_common SHARED ${COMMON_SRC}) -if ( MSYS ) - target_link_libraries(milvus_common - milvus_config - milvus_utils - milvus_log - yaml-cpp - boost_bitset_ext - arrow - parquet - ) -else() - target_link_libraries(milvus_common - milvus_config - milvus_utils - milvus_log - yaml-cpp - boost_bitset_ext - arrow - parquet - arrow_bundled - ) -endif() +target_link_libraries(milvus_common + milvus_config + milvus_utils + milvus_log + yaml-cpp + boost_bitset_ext + # arrow + # parquet + ${CONAN_LIBS} + ) install(TARGETS milvus_common DESTINATION "${CMAKE_INSTALL_LIBDIR}") diff --git a/internal/core/src/pb/CMakeLists.txt b/internal/core/src/pb/CMakeLists.txt index fc9191834c..3c5ca9f922 100644 --- a/internal/core/src/pb/CMakeLists.txt +++ b/internal/core/src/pb/CMakeLists.txt @@ -14,7 +14,6 @@ set( proto_file_names common.proto ) -set( PROTO_GOOGLE_PATH "${MILVUS_SOURCE_DIR}/../../cmake_build/thirdparty/protobuf/protobuf-src/src/" ) set( PROTO_PATH "${MILVUS_SOURCE_DIR}/../../cmake_build/thirdparty/milvus-proto/proto/") FOREACH(proto_file ${proto_file_names}) @@ -25,11 +24,9 @@ FOREACH(proto_file ${proto_file_names}) OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/${proto_src_file_name}.pb.h" "${CMAKE_CURRENT_SOURCE_DIR}/${proto_src_file_name}.pb.cc" COMMAND protobuf::protoc --proto_path=${PROTO_PATH}/ - --proto_path=${PROTO_GOOGLE_PATH}/ --cpp_out=${CMAKE_CURRENT_SOURCE_DIR}/ ${proto_file} DEPENDS ${PROTO_PATH}/${proto_file} ) - ENDFOREACH(proto_file) set( proto_file_names @@ -39,7 +36,6 @@ set( proto_file_names ) set( PROTO_PATH "${MILVUS_SOURCE_DIR}/../proto/") -set( PROTO_GOOGLE_PATH "${MILVUS_SOURCE_DIR}/../../cmake_build/thirdparty/protobuf/protobuf-src/src/" ) set( PROTO_API_PATH "${MILVUS_SOURCE_DIR}/../../cmake_build/thirdparty/milvus-proto/proto/" ) FOREACH(proto_file ${proto_file_names}) @@ -50,7 +46,6 @@ FOREACH(proto_file ${proto_file_names}) OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/${proto_src_file_name}.pb.h" "${CMAKE_CURRENT_SOURCE_DIR}/${proto_src_file_name}.pb.cc" COMMAND protobuf::protoc --proto_path=${PROTO_PATH}/ - --proto_path=${PROTO_GOOGLE_PATH}/ --proto_path=${PROTO_API_PATH}/ --cpp_out=${CMAKE_CURRENT_SOURCE_DIR}/ ${proto_file} @@ -61,4 +56,4 @@ ENDFOREACH(proto_file) add_library(milvus_proto STATIC ${milvus_proto_srcs}) message(STATUS "milvus proto sources: " ${milvus_proto_srcs}) -target_link_libraries( milvus_proto PUBLIC protobuf::libprotobuf) +target_link_libraries( milvus_proto PUBLIC ${CONAN_LIBS} ) diff --git a/internal/core/src/pb/common.pb.h b/internal/core/src/pb/common.pb.h index dc49771a43..ff1327bc74 100644 --- a/internal/core/src/pb/common.pb.h +++ b/internal/core/src/pb/common.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3009000 < PROTOBUF_MIN_PROTOC_VERSION +#if 3009001 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/internal/core/src/pb/index_cgo_msg.pb.h b/internal/core/src/pb/index_cgo_msg.pb.h index 406bb03fef..54c8fc3802 100644 --- a/internal/core/src/pb/index_cgo_msg.pb.h +++ b/internal/core/src/pb/index_cgo_msg.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3009000 < PROTOBUF_MIN_PROTOC_VERSION +#if 3009001 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/internal/core/src/pb/plan.pb.h b/internal/core/src/pb/plan.pb.h index 6b60fba6c5..7796f8e7f3 100644 --- a/internal/core/src/pb/plan.pb.h +++ b/internal/core/src/pb/plan.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3009000 < PROTOBUF_MIN_PROTOC_VERSION +#if 3009001 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/internal/core/src/pb/schema.pb.h b/internal/core/src/pb/schema.pb.h index 349875d8d3..da18f7113c 100644 --- a/internal/core/src/pb/schema.pb.h +++ b/internal/core/src/pb/schema.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3009000 < PROTOBUF_MIN_PROTOC_VERSION +#if 3009001 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/internal/core/src/pb/segcore.pb.h b/internal/core/src/pb/segcore.pb.h index 3c39eec6c5..ca2cd31aca 100644 --- a/internal/core/src/pb/segcore.pb.h +++ b/internal/core/src/pb/segcore.pb.h @@ -13,7 +13,7 @@ #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3009000 < PROTOBUF_MIN_PROTOC_VERSION +#if 3009001 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/internal/core/src/segcore/CMakeLists.txt b/internal/core/src/segcore/CMakeLists.txt index 1b58ab37b1..901af5d738 100644 --- a/internal/core/src/segcore/CMakeLists.txt +++ b/internal/core/src/segcore/CMakeLists.txt @@ -40,16 +40,16 @@ add_library(milvus_segcore SHARED ${SEGCORE_FILES}) find_library(TBB NAMES tbb) set(PLATFORM_LIBS dl) + if (MSYS) -set(PLATFORM_LIBS ) -endif () + set(PLATFORM_LIBS ) +endif() target_link_libraries(milvus_segcore milvus_query ${PLATFORM_LIBS} ${TBB} ${OpenMP_CXX_FLAGS} -# gperftools ) install(TARGETS milvus_segcore DESTINATION "${CMAKE_INSTALL_LIBDIR}") diff --git a/internal/core/src/segcore/Reduce.cpp b/internal/core/src/segcore/Reduce.cpp index ecc52b9dce..b5b2aa4e69 100644 --- a/internal/core/src/segcore/Reduce.cpp +++ b/internal/core/src/segcore/Reduce.cpp @@ -335,7 +335,7 @@ ReduceHelper::GetSearchResultDataSlice(int slice_index) { } // SearchResultData to blob - auto size = search_result_data->ByteSize(); + auto size = search_result_data->ByteSizeLong(); auto buffer = std::vector(size); search_result_data->SerializePartialToArray(buffer.data(), size); diff --git a/internal/core/src/segcore/segment_c.cpp b/internal/core/src/segcore/segment_c.cpp index c43804991d..aae9bd913a 100644 --- a/internal/core/src/segcore/segment_c.cpp +++ b/internal/core/src/segcore/segment_c.cpp @@ -95,7 +95,7 @@ Retrieve(CSegmentInterface c_segment, CRetrievePlan c_plan, uint64_t timestamp, auto plan = (const milvus::query::RetrievePlan*)c_plan; auto retrieve_result = segment->Retrieve(plan, timestamp); - auto size = retrieve_result->ByteSize(); + auto size = retrieve_result->ByteSizeLong(); void* buffer = malloc(size); retrieve_result->SerializePartialToArray(buffer, size); diff --git a/internal/core/src/storage/CMakeLists.txt b/internal/core/src/storage/CMakeLists.txt index 502b642d97..49aa3407d7 100644 --- a/internal/core/src/storage/CMakeLists.txt +++ b/internal/core/src/storage/CMakeLists.txt @@ -48,14 +48,11 @@ add_library(milvus_storage SHARED ${STORAGE_FILES}) find_package(Boost REQUIRED COMPONENTS filesystem) -if(BUILD_DISK_ANN STREQUAL "ON") - target_link_libraries(milvus_storage PUBLIC milvus_common Boost::filesystem aws-cpp-sdk-s3 pthread) -else() - target_link_libraries(milvus_storage PUBLIC milvus_common Boost::filesystem pthread) -endif() - -if(NOT CMAKE_INSTALL_PREFIX) - set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}) -endif() +target_link_libraries(milvus_storage PUBLIC + milvus_common + Boost::filesystem + pthread + ${CONAN_LIBS} + ) install(TARGETS milvus_storage DESTINATION "${CMAKE_INSTALL_LIBDIR}") diff --git a/internal/core/thirdparty/CMakeLists.txt b/internal/core/thirdparty/CMakeLists.txt index 31fe6cfcdd..8757e4a196 100644 --- a/internal/core/thirdparty/CMakeLists.txt +++ b/internal/core/thirdparty/CMakeLists.txt @@ -40,47 +40,6 @@ find_package( Threads REQUIRED ) add_subdirectory( knowhere ) -# ****************************** Thirdparty googletest *************************************** -if ( MILVUS_BUILD_TESTS) - add_subdirectory( gtest ) - add_subdirectory( google_benchmark ) -endif() - -if ( MILVUS_BUILD_TESTS AND LINUX ) - add_subdirectory( profiler ) -endif() - - -# ****************************** Thirdparty yaml *************************************** -add_subdirectory( yaml-cpp ) - -# ****************************** Thirdparty opentracing *************************************** -if ( MILVUS_WITH_OPENTRACING ) - add_subdirectory( opentracing ) - -endif() -add_subdirectory( protobuf ) add_subdirectory( boost_ext ) -add_subdirectory( arrow ) add_subdirectory( rocksdb ) -# ******************************* Thirdparty aws sdk ******************************** -if ( LINUX AND BUILD_DISK_ANN STREQUAL "ON" ) - add_subdirectory( aws_sdk ) -endif() - -# ******************************* Thirdparty marisa ******************************** -# TODO: support win. -if ( LINUX OR APPLE) - add_subdirectory( marisa ) -endif() - -# ******************************* Thirdparty jemalloc ******************************** -if ( LINUX ) - add_subdirectory( jemalloc ) -endif() - -# ******************************* Thirdparty rapidxml ******************************** -#if ( LINUX ) -# add_subdirectory( rapidxml ) -#endif() diff --git a/internal/core/thirdparty/arrow/CMakeLists.txt b/internal/core/thirdparty/arrow/CMakeLists.txt deleted file mode 100644 index be59d89150..0000000000 --- a/internal/core/thirdparty/arrow/CMakeLists.txt +++ /dev/null @@ -1,111 +0,0 @@ -# Licensed to the LF AI & Data foundation under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set( ARROW_VERSION "8.0.0" ) -set( ARROW_SOURCE_URL - "https://github.com/apache/arrow/archive/apache-arrow-${ARROW_VERSION}.tar.gz") -set( ARROW_MD5 "b1d285d2b90feceb61155baf9a21f81c" ) - -# These three components is required by arrow -find_package(Boost REQUIRED COMPONENTS regex system filesystem) -message( STATUS "Find Boost: include dirs-${Boost_INCLUDE_DIRS}, version-${Boost_VERSION}") - -macro( build_arrow ) - message( STATUS "Building ARROW-${ARROW_VERSION} from source" ) - - set( ARROW_CMAKE_ARGS - "-DARROW_WITH_LZ4=OFF" - "-DARROW_WITH_ZSTD=ON" - "-Dzstd_SOURCE=BUNDLED" - "-DARROW_WITH_BROTLI=OFF" - "-DARROW_WITH_SNAPPY=OFF" - "-DARROW_WITH_ZLIB=OFF" - "-DARROW_BUILD_STATIC=ON" - "-DARROW_BUILD_SHARED=OFF" - "-DARROW_BOOST_USE_SHARED=OFF" - "-DARROW_BUILD_TESTS=OFF" - "-DARROW_TEST_MEMCHECK=OFF" - "-DARROW_BUILD_BENCHMARKS=OFF" - "-DARROW_CUDA=OFF" - "-DARROW_PYTHON=OFF" - "-DARROW_WITH_RE2=OFF" - "-DARROW_BUILD_UTILITIES=OFF" - "-DARROW_PARQUET=ON" - "-DPARQUET_BUILD_SHARED=OFF" - "-DThrift_SOURCE=BUNDLED" - "-Dutf8proc_SOURCE=BUNDLED" - "-DARROW_S3=OFF" - "-DCMAKE_VERBOSE_MAKEFILE=ON" - "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" - "-DCMAKE_INCLUDE_PATH=${Boost_INCLUDE_DIRS}" - ) - if ( WIN32 ) - set( ARROW_CMAKE_ARGS ${ARROW_CMAKE_ARGS} "-DARROW_JEMALLOC=OFF" ) - else () - set( ARROW_CMAKE_ARGS ${ARROW_CMAKE_ARGS} "-DARROW_JEMALLOC=ON" ) - endif () - - ExternalProject_Add( - arrow-ep - PREFIX ${CMAKE_BINARY_DIR}/3rdparty_download/arrow-subbuild - BINARY_DIR arrow-bin - DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} - INSTALL_DIR ${CMAKE_INSTALL_PREFIX} - SOURCE_SUBDIR "cpp" - URL ${ARROW_SOURCE_URL} - URL_MD5 ${ARROW_MD5} - CMAKE_ARGS ${ARROW_CMAKE_ARGS} - ${EP_LOG_OPTIONS} - ) - - ExternalProject_Get_Property( arrow-ep INSTALL_DIR ) - if( NOT IS_DIRECTORY ${INSTALL_DIR}/include ) - file( MAKE_DIRECTORY "${INSTALL_DIR}/include" ) - endif() - - add_library( arrow_bundled STATIC IMPORTED ) - set_target_properties( arrow_bundled - PROPERTIES - IMPORTED_GLOBAL TRUE - IMPORTED_LOCATION ${INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libarrow_bundled_dependencies.a - INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include ) - add_dependencies(arrow_bundled arrow-ep ) - - add_library( arrow STATIC IMPORTED ) - set_target_properties( arrow - PROPERTIES - IMPORTED_GLOBAL TRUE - IMPORTED_LOCATION ${INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libarrow.a - INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include ) - add_dependencies(arrow arrow_bundled) - target_link_libraries(arrow INTERFACE arrow_bundled) - - - add_library( parquet STATIC IMPORTED ) - set_target_properties( parquet - PROPERTIES - IMPORTED_GLOBAL TRUE - IMPORTED_LOCATION ${INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libparquet.a - INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include ) - add_dependencies( parquet arrow-ep ) - target_link_libraries( parquet INTERFACE arrow ) -endmacro() - -if (MSYS) - message(STATUS "Using arrow form system") -else (MSYS) - build_arrow() -endif (MSYS) diff --git a/internal/core/thirdparty/aws_sdk/CMakeLists.txt b/internal/core/thirdparty/aws_sdk/CMakeLists.txt deleted file mode 100644 index 1206e78eb3..0000000000 --- a/internal/core/thirdparty/aws_sdk/CMakeLists.txt +++ /dev/null @@ -1,77 +0,0 @@ -# ------------------------------------------------------------------------------- -# Copyright (C) 2019-2020 Zilliz. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under the License. -# ------------------------------------------------------------------------------- - -set(AWS_SDK_VERSION "1.8.186") - -macro(build_aws_sdk_s3) - message(STATUS "Building AWS-SDK-${AWS_SDK_VERSION} from source") - - set(AWS_SDK_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) - - set(AWS_SDK_BUILD_COMMAND make) - set(AWS_SDK_INSTALL_COMMAND make install) - - set(AWS_SDK_S3_CMAKE_ARGS - "-DCMAKE_BUILD_TYPE=Release" - "-DBUILD_ONLY=s3" - "-DENABLE_TESTING=OFF" - "-DAUTORUN_UNIT_TESTS=OFF" - "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}") - - ExternalProject_Add(aws_sdk_s3_ep - - GIT_REPOSITORY https://github.com/aws/aws-sdk-cpp.git - GIT_TAG ${AWS_SDK_VERSION} - - # BINARY_DIR aws-s3-bin - PREFIX ${CMAKE_BINARY_DIR}/3rdparty_download/aws-sdk-subbuild - BUILD_IN_SOURCE 1 - #PATCH_COMMAND sh prefetch_crt_dependency.sh - LIST_SEPARATOR "|" - BUILD_COMMAND ${AWS_SDK_BUILD_COMMAND} - INSTALL_COMMAND ${AWS_SDK_INSTALL_COMMAND} - CMAKE_ARGS ${AWS_SDK_S3_CMAKE_ARGS} - ) - - add_library(aws-cpp-sdk-core SHARED IMPORTED) - set_target_properties(aws-cpp-sdk-core - PROPERTIES - IMPORTED_GLOBAL TRUE - IMPORTED_LOCATION ${AWS_SDK_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/${CMAKE_SHARED_LIBRARY_PREFIX}aws-cpp-sdk-core${CMAKE_SHARED_LIBRARY_SUFFIX} - INTERFACE_INCLUDE_DIRECTORIES ${AWS_SDK_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}) - add_dependencies(aws-cpp-sdk-core aws_sdk_s3_ep) - - add_library(aws-cpp-sdk-s3 SHARED IMPORTED) - set_target_properties(aws-cpp-sdk-s3 - PROPERTIES - IMPORTED_GLOBAL TRUE - IMPORTED_LOCATION ${AWS_SDK_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/${CMAKE_SHARED_LIBRARY_PREFIX}aws-cpp-sdk-s3${CMAKE_SHARED_LIBRARY_SUFFIX} - INTERFACE_INCLUDE_DIRECTORIES ${AWS_SDK_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}) - add_dependencies(aws-cpp-sdk-s3 aws_sdk_s3_ep) - - # add_library(aws-cpp-sdk-sts SHARED IMPORTED) - # set_target_properties(aws-cpp-sdk-sts - # PROPERTIES - # IMPORTED_GLOBAL TRUE - # IMPORTED_LOCATION ${AWS_SDK_INSTALL_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}aws-cpp-sdk-sts${CMAKE_SHARED_LIBRARY_SUFFIX} - # INTERFACE_INCLUDE_DIRECTORIES ${AWS_SDK_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}) - # add_dependencies(aws-cpp-sdk-sts aws_sdk_s3_ep) - get_target_property(S3_IMPORTED_LOCATION aws-cpp-sdk-s3 IMPORTED_LOCATION) - get_target_property(S3_INTERFACE_INCLUDE_DIRECTORIES aws-cpp-sdk-s3 INTERFACE_INCLUDE_DIRECTORIES) - message("AWS_SDK_INSTALL_PREFIX: ${AWS_SDK_INSTALL_PREFIX}") - message("CMAKE_INSTALL_LIBDIR: ${CMAKE_INSTALL_LIBDIR}") - message("S3_IMPORTED_LOCATION: ${S3_IMPORTED_LOCATION}") - message("S3_INTERFACE_INCLUDE_DIRECTORIES: ${S3_INTERFACE_INCLUDE_DIRECTORIES}") -endmacro() - -build_aws_sdk_s3() diff --git a/internal/core/thirdparty/google_benchmark/CMakeLists.txt b/internal/core/thirdparty/google_benchmark/CMakeLists.txt deleted file mode 100644 index 9a419d7e2f..0000000000 --- a/internal/core/thirdparty/google_benchmark/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -FetchContent_Declare(google_benchmark - URL https://github.com/google/benchmark/archive/v1.5.2.tar.gz - URL_MD5 084b34aceaeac11a6607d35220ca2efa - DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} - SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/google_benchmark - BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/google_benchmark - ) - -FetchContent_GetProperties( google_benchmark ) -if ( NOT google_benchmark_POPULATED ) - - FetchContent_Populate( google_benchmark ) - - # Adding the following targets: - # benchmark::benchmark_main - message("gb=${google_benchmark_SOURCE_DIR}") - add_subdirectory( ${google_benchmark_SOURCE_DIR} - ${google_benchmark_BINARY_DIR} - EXCLUDE_FROM_ALL ) -endif() diff --git a/internal/core/thirdparty/gtest/CMakeLists.txt b/internal/core/thirdparty/gtest/CMakeLists.txt deleted file mode 100644 index d1728b0b83..0000000000 --- a/internal/core/thirdparty/gtest/CMakeLists.txt +++ /dev/null @@ -1,65 +0,0 @@ -#------------------------------------------------------------------------------- -# Copyright (C) 2019-2020 Zilliz. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under the License. -#------------------------------------------------------------------------------- - -if ( DEFINED ENV{MILVUS_GTEST_URL} ) - set( GTEST_SOURCE_URL "$ENV{MILVUS_GTEST_URL}" ) -else() - set( GTEST_SOURCE_URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.zip" ) -endif() - -message( STATUS "Building gtest-${GTEST_VERSION} from source" ) - -set( CMAKE_POLICY_DEFAULT_CMP0022 NEW ) # for googletest only - -FetchContent_Declare( - googletest - URL ${GTEST_SOURCE_URL} - URL_MD5 "ad6868782b5952b7476a7c1c72d5a714" - SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/googletest-src - BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/googletest-build - DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} ) - -FetchContent_GetProperties( googletest ) -if ( NOT googletest_POPULATED ) - - FetchContent_Populate( googletest ) - - # Adding the following targets: - # gtest, gtest_main, gmock, gmock_main - add_subdirectory( ${googletest_SOURCE_DIR} - ${googletest_BINARY_DIR} - EXCLUDE_FROM_ALL ) -endif() -# include(GoogleTest) - -# **************************************************************** -# Create ALIAS Target -# **************************************************************** -# if (NOT TARGET GTest:gtest) -# add_library( GTest::gtest ALIAS gtest ) -# endif() -# if (NOT TARGET GTest:main) -# add_library( GTest::main ALIAS gtest_main ) -# endif() -# if (NOT TARGET GMock:gmock) -# target_link_libraries( gmock INTERFACE GTest::gtest ) -# add_library( GMock::gmock ALIAS gmock ) -# endif() -# if (NOT TARGET GMock:main) -# target_link_libraries( gmock_main INTERFACE GTest::gtest ) -# add_library( GMock::main ALIAS gmock_main ) -# endif() - - -get_property( var DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/googletest-src" PROPERTY COMPILE_OPTIONS ) -message( STATUS "gtest compile options: ${var}" ) diff --git a/internal/core/thirdparty/jemalloc/CMakeLists.txt b/internal/core/thirdparty/jemalloc/CMakeLists.txt deleted file mode 100644 index 73d369adb1..0000000000 --- a/internal/core/thirdparty/jemalloc/CMakeLists.txt +++ /dev/null @@ -1,80 +0,0 @@ -#------------------------------------------------------------------------------- -# Copyright (C) 2019-2020 Zilliz. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under the License. -#------------------------------------------------------------------------------- - -if (DEFINED ENV{MILVUS_JEMALLOC_URL}) - set(JEMALLOC_SOURCE_URL "$ENV{MILVUS_JEMALLOC_URL}") -else () - set(JEMALLOC_SOURCE_URL - "https://github.com/jemalloc/jemalloc/releases/download/${MILVUS_JEMALLOC_BUILD_VERSION}/jemalloc-${MILVUS_JEMALLOC_BUILD_VERSION}.tar.bz2") -endif () - -# ---------------------------------------------------------------------- -# jemalloc - Unix-only high-performance allocator -message(STATUS "Building (vendored) jemalloc from source") -# We only use a vendored jemalloc as we want to control its version. -# Also our build of jemalloc is specially prefixed so that it will not -# conflict with the default allocator as well as other jemalloc -# installations. -# find_package(jemalloc) - -set(JEMALLOC_PREFIX "${CMAKE_INSTALL_PREFIX}") -set(JEMALLOC_LIB_DIR "${JEMALLOC_PREFIX}/lib") -set(JEMALLOC_STATIC_LIB "${JEMALLOC_LIB_DIR}/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}") -set(JEMALLOC_CONFIGURE_COMMAND ./configure "AR=${CMAKE_AR}" "CC=${CMAKE_C_COMPILER}") - -message("CMAKE_OSX_SYSROOT: ${CMAKE_OSX_SYSROOT}") -if (CMAKE_OSX_SYSROOT) - list(APPEND JEMALLOC_CONFIGURE_COMMAND "SDKROOT=${CMAKE_OSX_SYSROOT}") -endif () - -if (DEFINED MILVUS_JEMALLOC_LG_PAGE) - # Used for arm64 manylinux wheels in order to make the wheel work on both - # 4k and 64k page arm64 systems. - list(APPEND JEMALLOC_CONFIGURE_COMMAND "--with-lg-page=${MILVUS_JEMALLOC_LG_PAGE}") -endif () - -list(APPEND - JEMALLOC_CONFIGURE_COMMAND - "--prefix=${JEMALLOC_PREFIX}" - "--libdir=${JEMALLOC_LIB_DIR}") -if (CMAKE_BUILD_TYPE EQUAL "DEBUG") - # Enable jemalloc debug checks when Milvus itself has debugging enabled - list(APPEND JEMALLOC_CONFIGURE_COMMAND "--enable-debug") -endif () -set(JEMALLOC_BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS}) -if (CMAKE_OSX_SYSROOT) - list(APPEND JEMALLOC_BUILD_COMMAND "SDKROOT=${CMAKE_OSX_SYSROOT}") -endif () - -ExternalProject_Add( - jemalloc_ep - PREFIX ${CMAKE_BINARY_DIR}/3rdparty_download/jemalloc-subbuild - URL ${JEMALLOC_SOURCE_URL} - PATCH_COMMAND touch doc/jemalloc.3 doc/jemalloc.html - CONFIGURE_COMMAND ${JEMALLOC_CONFIGURE_COMMAND} - BUILD_IN_SOURCE 1 - BUILD_COMMAND ${JEMALLOC_BUILD_COMMAND} - BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" - INSTALL_COMMAND ${MAKE} install) - -add_library(jemalloc SHARED IMPORTED) -set_target_properties(jemalloc - PROPERTIES INTERFACE_LINK_LIBRARIES Threads::Threads - IMPORTED_LOCATION "${JEMALLOC_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libjemalloc.so" - INTERFACE_INCLUDE_DIRECTORIES "${JEMALLOC_PREFIX}/jemalloc/include") -add_dependencies(jemalloc jemalloc_ep) - -get_target_property(JEMALLOC_IMPORTED_LOCATION jemalloc IMPORTED_LOCATION) -get_target_property(JEMALLOC_INTERFACE_INCLUDE_DIRECTORIES jemalloc INTERFACE_INCLUDE_DIRECTORIES) -message("JEMALLOC_IMPORTED_LOCATION: ${JEMALLOC_IMPORTED_LOCATION}") -message("JEMALLOC_INTERFACE_INCLUDE_DIRECTORIES: ${JEMALLOC_INTERFACE_INCLUDE_DIRECTORIES}") diff --git a/internal/core/thirdparty/knowhere/CMakeLists.txt b/internal/core/thirdparty/knowhere/CMakeLists.txt index 3395d22795..467f1489a9 100644 --- a/internal/core/thirdparty/knowhere/CMakeLists.txt +++ b/internal/core/thirdparty/knowhere/CMakeLists.txt @@ -30,6 +30,7 @@ macro(build_knowhere) ${EP_COMMON_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${KNOWHERE_INSTALL_PREFIX} + -DCMAKE_PREFIX_PATH=${CONAN_BOOST_ROOT} ) if ( BUILD_DISK_ANN STREQUAL "ON" ) diff --git a/internal/core/thirdparty/marisa/CMakeLists.txt b/internal/core/thirdparty/marisa/CMakeLists.txt deleted file mode 100644 index 3bea443c0c..0000000000 --- a/internal/core/thirdparty/marisa/CMakeLists.txt +++ /dev/null @@ -1,74 +0,0 @@ -set( MARISA_VERSION "v0.2.6") -set( MARISA_MD5 "695cecf504ced27ac13aa33d97d69dd0") - -if ( DEFINED ENV{MILVUS_MARISA_URL} ) - set( MARISA_SOURCE_URL "$ENV{MILVUS_MARISA_URL}" ) -else () - set( MARISA_SOURCE_URL - "https://github.com/s-yata/marisa-trie/archive/refs/tags/${MARISA_VERSION}.tar.gz" ) -endif () - -macro(build_marisa) - message(STATUS "Building marisa-${MARISA_VERSION} from source") - - set (MARISA_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) - set (MARISA_DIR "${CMAKE_CURRENT_BINARY_DIR}/src") - if ( LINUX ) - set (MARISA_CONFIGURE_COMMAND cd ${MARISA_DIR} && libtoolize && autoreconf -i && ./configure --prefix=${MARISA_INSTALL_PREFIX}) - else() - set (MARISA_CONFIGURE_COMMAND brew install automake libtool && cd ${MARISA_DIR} && glibtoolize && autoreconf -i && ./configure --prefix=${MARISA_INSTALL_PREFIX}) - endif() - - set (MARISA_BUILD_COMMAND make -j) - set (MARISA_INSTALL_COMMAND make install) - - message(${MARISA_DIR}) - - externalproject_add(marisa_ep - URL ${MARISA_SOURCE_URL} - URL_MD5 ${MARISA_MD5} - SOURCE_DIR ${MARISA_DIR} - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND ${MARISA_CONFIGURE_COMMAND} - BUILD_COMMAND ${MARISA_BUILD_COMMAND} - INSTALL_COMMAND ${MARISA_INSTALL_COMMAND} - ) - - if( NOT IS_DIRECTORY ${MARISA_INSTALL_PREFIX}/include ) - file( MAKE_DIRECTORY "${MARISA_INSTALL_PREFIX}/include" ) - endif() - - add_library(marisa SHARED IMPORTED) - set_target_properties( marisa - PROPERTIES - IMPORTED_GLOBAL TRUE - IMPORTED_LOCATION ${MARISA_INSTALL_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}marisa${CMAKE_SHARED_LIBRARY_SUFFIX} - INTERFACE_INCLUDE_DIRECTORIES ${MARISA_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}) - - get_target_property(MARISA_IMPORTED_LOCATION marisa IMPORTED_LOCATION) - get_target_property(MARISA_INTERFACE_INCLUDE_DIRECTORIES marisa INTERFACE_INCLUDE_DIRECTORIES) - message("MARISA_INSTALL_PREFIX: ${MARISA_INSTALL_PREFIX}") - message("CMAKE_INSTALL_LIBDIR: ${CMAKE_INSTALL_LIBDIR}") - message("MARISA_IMPORTED_LOCATION: ${MARISA_IMPORTED_LOCATION}") - message("MARISA_INTERFACE_INCLUDE_DIRECTORIES: ${MARISA_INTERFACE_INCLUDE_DIRECTORIES}") - - add_dependencies(marisa marisa_ep) -endmacro() - -set(MARISA_SOURCE "AUTO") -if (MARISA_SOURCE STREQUAL "AUTO") - find_package(marisa) - message(STATUS "marisa libraries: ${MARISA_LIBRARIES}") - message(STATUS "marisa found: ${MARISA_FOUND}") - - if (MARISA_FOUND) - add_library(marisa) - else() - build_marisa() - endif() -elseif (MARISA_SOURCE STREQUAL "BUNDLED") - build_marisa() -elseif (MARISA_SOURCE STREQUAL "SYSTEM") - find_package(marisa) - add_library(marisa) -endif () diff --git a/internal/core/thirdparty/opentracing/CMakeLists.txt b/internal/core/thirdparty/opentracing/CMakeLists.txt deleted file mode 100644 index 056bc1b86c..0000000000 --- a/internal/core/thirdparty/opentracing/CMakeLists.txt +++ /dev/null @@ -1,68 +0,0 @@ -#------------------------------------------------------------------------------- -# Copyright (C) 2019-2020 Zilliz. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under the License. -#------------------------------------------------------------------------------- - -if ( DEFINED ENV{MILVUS_OPENTRACING_URL} ) - set(OPENTRACING_SOURCE_URL "$ENV{MILVUS_OPENTRACING_URL}") -else () - set(OPENTRACING_SOURCE_URL - "https://github.com/opentracing/opentracing-cpp/archive/${OPENTRACING_VERSION}.tar.gz" ) -endif () - -message(STATUS "Building OPENTRACING-${OPENTRACING_VERSION} from source") - -FetchContent_Declare( - opentracing - URL ${OPENTRACING_SOURCE_URL} - URL_MD5 "e598ba4b81ae8e1ceed8cd8bbf86f2fd" - DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} - SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/opentracing-src - BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/opentracing-build - ) - -set( BUILD_SHARED_LIBS CACHE BOOL OFF FORCE) -set( ENABLE_LINTING CACHE BOOL OFF FORCE) -set( BUILD_TESTING CACHE BOOL OFF FORCE ) -FetchContent_GetProperties( opentracing ) -if ( NOT opentracing_POPULATED ) - - FetchContent_Populate( opentracing ) - - # Adding the following targets: - # opentracing-static - # opentracing_mocktracer-static - add_subdirectory( ${opentracing_SOURCE_DIR} - ${opentracing_BINARY_DIR} - EXCLUDE_FROM_ALL ) - - # Opentracing-cpp CMakeLists.txt file didn't give a - # correct interface directories - target_include_directories( opentracing-static - INTERFACE $ - $ - $ ) - target_include_directories( opentracing_mocktracer-static - INTERFACE $ ) - # Adding the following ALIAS Targets: - # opentracing::opentracing - # opentracing::mocktracer - if ( NOT TARGET opentracing::opentracing ) - add_library( opentracing::opentracing ALIAS opentracing-static ) - endif() - if ( NOT TARGET opentracing::mocktracer ) - add_library( opentracing::mocktracer ALIAS opentracing_mocktracer-static ) - endif() - -endif() - -get_property( var DIRECTORY "${opentracing_SOURCE_DIR}" PROPERTY COMPILE_OPTIONS ) -message( STATUS "opentracing compile options: ${var}" ) diff --git a/internal/core/thirdparty/protobuf/CMakeLists.txt b/internal/core/thirdparty/protobuf/CMakeLists.txt deleted file mode 100644 index 6b2f967a37..0000000000 --- a/internal/core/thirdparty/protobuf/CMakeLists.txt +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the LF AI & Data foundation under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set( PROTOBUF_SOURCE_URL - "https://github.com/protocolbuffers/protobuf/releases/download/v3.9.0/protobuf-cpp-3.9.0.zip" ) - -message( STATUS "Building protobuf-cpp-3.9.0 from source" ) - -FetchContent_Declare( protobuf - URL ${PROTOBUF_SOURCE_URL} - URL_MD5 "9562b27cc6ac5ebd087f201f1310c885" - DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} - SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf-src - BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf-build -) - -set( protobuf_BUILD_TESTS CACHE BOOL OFF FORCE ) -set( protobuf_WITH_ZLIB CACHE BOOL OFF FORCE ) -if ( NOT protobuf_POPULATED ) - FetchContent_Populate( protobuf ) - - # Adding the following targets: - # protobuf::libprotobuf - static target - # protobuf::protoc - executable target - add_subdirectory( ${protobuf_SOURCE_DIR}/cmake - ${protobuf_BINARY_DIR} - EXCLUDE_FROM_ALL ) -endif() - -get_target_property( PROTOBUF_INCLUDE_DIR protobuf::libprotobuf INTERFACE_INCLUDE_DIRECTORIES ) -message( STATUS "protobuf interface include dir" ${PROTOBUF_INCLUDE_DIR} ) - -get_target_property( PROTOC_INCLUDE_DIR protobuf::protoc INCLUDE_DIRECTORIES ) -message( STATUS "protoc interface include dir" ${PROTOC_INCLUDE_DIR} ) diff --git a/internal/core/thirdparty/rapidxml/CMakeLists.txt b/internal/core/thirdparty/rapidxml/CMakeLists.txt deleted file mode 100644 index 2d956d27cb..0000000000 --- a/internal/core/thirdparty/rapidxml/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the LF AI & Data foundation under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -message("Download rapidxml to ${CMAKE_INSTALL_PREFIX}/include/rapidxml.hpp") -file(DOWNLOAD - https://raw.githubusercontent.com/dwd/rapidxml/master/rapidxml.hpp - ${CMAKE_INSTALL_PREFIX}/include/rapidxml/rapidxml.hpp) \ No newline at end of file diff --git a/internal/core/thirdparty/rocksdb/CMakeLists.txt b/internal/core/thirdparty/rocksdb/CMakeLists.txt index 8bed87a7ba..ca1295b79c 100644 --- a/internal/core/thirdparty/rocksdb/CMakeLists.txt +++ b/internal/core/thirdparty/rocksdb/CMakeLists.txt @@ -1,75 +1,8 @@ -# Licensed to the LF AI & Data foundation under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set( ROCKSDB_VERSION "6.27.3" ) -set( ROCKSDB_SOURCE_URL - "https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz") - -macro( build_rocksdb ) - message( STATUS "Building ROCKSDB-${ROCKSDB_VERSION} from source" ) - - set ( ROCKSDB_MD5 "e4a0625f0cec82060e62c81b787a1124" ) - set( FPIC_ARG "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" ) - set( ROCKSDB_CMAKE_ARGS - "-DWITH_GFLAGS=OFF" - "-DROCKSDB_BUILD_SHARED=OFF" - "-DWITH_TESTS=OFF" - "-DWITH_BENCHMARK_TOOLS=OFF" - "-DWITH_CORE_TOOLS=OFF" - "-DWITH_TOOLS=OFF" - "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}" - "-DROCKSDB_INSTALL_ON_WINDOWS=ON" - "-DFAIL_ON_WARNINGS=OFF" - ${FPIC_ARG} - #This is used to solve 'illegal instruction' problem in some machine - "-DPORTABLE=ON" - ) - - ExternalProject_Add( - rocksdb-ep - PREFIX ${CMAKE_BINARY_DIR}/3rdparty_download/rocksdb-subbuild - BINARY_DIR rocksdb-bin - DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} - INSTALL_DIR ${CMAKE_INSTALL_PREFIX} - URL ${ROCKSDB_SOURCE_URL} - URL_MD5 ${ROCKSDB_MD5} - CMAKE_ARGS ${ROCKSDB_CMAKE_ARGS} - ) - - ExternalProject_Get_Property( rocksdb-ep INSTALL_DIR ) - ExternalProject_Get_Property( rocksdb-ep BINARY_DIR ) - - if( NOT IS_DIRECTORY ${INSTALL_DIR}/include ) - file( MAKE_DIRECTORY "${INSTALL_DIR}/include" ) - endif() - - milvus_add_pkg_config("rocksdb") - add_library( rocksdb STATIC IMPORTED ) - set_target_properties( rocksdb - PROPERTIES - IMPORTED_GLOBAL TRUE - IMPORTED_LOCATION ${INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/librocksdb.a - INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include ) - -endmacro() - if (MSYS) - message(STATUS "Using rocksdb form system") - configure_file(rocksdb-win.pc.in "${CMAKE_CURRENT_BINARY_DIR}/rocksdb.pc" @ONLY) - install(FILES "${CMAKE_CURRENT_BINARY_DIR}/rocksdb.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") -else (MSYS) - build_rocksdb() -endif (MSYS) + configure_file(rocksdb-win.pc.in "${CMAKE_CURRENT_BINARY_DIR}/rocksdb.pc" @ONLY) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/rocksdb.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") +else() + milvus_add_pkg_config("rocksdb") +endif() + +install(DIRECTORY "${CONAN_INCLUDE_DIRS_ROCKSDB}/rocksdb/" DESTINATION "${CMAKE_INSTALL_PREFIX}/include/rocksdb/") diff --git a/internal/core/thirdparty/rocksdb/rocksdb-win.pc.in b/internal/core/thirdparty/rocksdb/rocksdb-win.pc.in index b0082aeea6..60923fe9c9 100644 --- a/internal/core/thirdparty/rocksdb/rocksdb-win.pc.in +++ b/internal/core/thirdparty/rocksdb/rocksdb-win.pc.in @@ -1,10 +1,9 @@ -libdir=/mingw64/lib -includedir=/mingw64/include - +libdir=@CMAKE_INSTALL_FULL_LIBDIR@ +includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ Name: Rocksdb Description: Rocksdb Version: @ROCKSDB_VERSION@ Libs: -L${libdir} -lrocksdb -Libs.private: -lz -lbz2 +Libs.private: -lz -lbz2 │ Cflags: -I${includedir} diff --git a/internal/core/thirdparty/rocksdb/rocksdb.pc.in b/internal/core/thirdparty/rocksdb/rocksdb.pc.in index 1c55c9a8f9..4df5e7ffd3 100644 --- a/internal/core/thirdparty/rocksdb/rocksdb.pc.in +++ b/internal/core/thirdparty/rocksdb/rocksdb.pc.in @@ -1,6 +1,5 @@ libdir=@CMAKE_INSTALL_FULL_LIBDIR@ includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ - Name: Rocksdb Description: Rocksdb Version: @ROCKSDB_VERSION@ diff --git a/internal/core/thirdparty/yaml-cpp/CMakeLists.txt b/internal/core/thirdparty/yaml-cpp/CMakeLists.txt deleted file mode 100644 index dcd83e63c1..0000000000 --- a/internal/core/thirdparty/yaml-cpp/CMakeLists.txt +++ /dev/null @@ -1,50 +0,0 @@ -#------------------------------------------------------------------------------- -# Copyright (C) 2019-2020 Zilliz. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under the License. -#------------------------------------------------------------------------------- - -if ( DEFINED ENV{MILVUS_YAMLCPP_URL} ) - set( YAMLCPP_SOURCE_URL "$ENV{MILVUS_YAMLCPP_URL}" ) -else() - set( YAMLCPP_SOURCE_URL - "https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-${YAMLCPP_VERSION}.tar.gz" ) -endif() - -message( STATUS "Building yaml-cpp-${YAMLCPP_VERSION} from source" ) -FetchContent_Declare( - yaml-cpp - URL ${YAMLCPP_SOURCE_URL} - URL_MD5 "b45bf1089a382e81f6b661062c10d0c2" - DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} - SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/yaml-src - BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/yaml-build - ) - -set( YAML_CPP_BUILD_TESTS CACHE BOOL OFF FORCE ) -set( YAML_CPP_BUILD_TOOLS CACHE BOOL OFF FORCE ) -FetchContent_GetProperties( yaml-cpp ) -if ( NOT yaml-cpp_POPULATED ) - - FetchContent_Populate( yaml-cpp ) - - # Adding the following targets: - # yaml-cpp::yaml-cpp, yaml-cpp - add_subdirectory( ${yaml-cpp_SOURCE_DIR} - ${yaml-cpp_BINARY_DIR} - EXCLUDE_FROM_ALL ) - -endif() - -get_target_property( YAML_CPP_INCLUDE_DIR yaml-cpp INTERFACE_INCLUDE_DIRECTORIES ) -message( STATUS ${YAML_CPP_INCLUDE_DIR} ) - -get_property( var DIRECTORY "${yaml-cpp_SOURCE_DIR}" PROPERTY COMPILE_OPTIONS ) -message( STATUS "yaml compile options: ${var}" ) \ No newline at end of file diff --git a/internal/core/unittest/bench/CMakeLists.txt b/internal/core/unittest/bench/CMakeLists.txt index defdc125a2..b59c27926d 100644 --- a/internal/core/unittest/bench/CMakeLists.txt +++ b/internal/core/unittest/bench/CMakeLists.txt @@ -28,7 +28,7 @@ target_link_libraries(all_bench pthread ) -target_link_libraries(all_bench benchmark::benchmark_main) +target_link_libraries(all_bench benchmark_main) add_executable(indexbuilder_bench ${indexbuilder_bench_srcs}) target_link_libraries(indexbuilder_bench @@ -41,4 +41,4 @@ target_link_libraries(indexbuilder_bench milvus_utils ) -target_link_libraries(indexbuilder_bench benchmark::benchmark_main) +target_link_libraries(indexbuilder_bench benchmark_main) diff --git a/internal/core/unittest/test_c_api.cpp b/internal/core/unittest/test_c_api.cpp index 786b79a3db..b064bc3514 100644 --- a/internal/core/unittest/test_c_api.cpp +++ b/internal/core/unittest/test_c_api.cpp @@ -275,7 +275,7 @@ TEST(CApiTest, CPlan) { template std::vector serialize(const Message* msg) { - auto l = msg->ByteSize(); + auto l = msg->ByteSizeLong(); std::vector ret(l); auto ok = msg->SerializeToArray(ret.data(), l); assert(ok); diff --git a/internal/core/unittest/test_string_index.cpp b/internal/core/unittest/test_string_index.cpp index 9f9183ed3f..786173159e 100644 --- a/internal/core/unittest/test_string_index.cpp +++ b/internal/core/unittest/test_string_index.cpp @@ -264,9 +264,9 @@ TEST_F(StringIndexMarisaTest, BaseIndexCodec) { strings[i] = std::to_string(std::rand() % 10); } *str_arr.mutable_data() = {strings.begin(), strings.end()}; - auto data = new char[str_arr.ByteSize()]; - str_arr.SerializeToArray(data, str_arr.ByteSize()); - index->BuildWithRawData(str_arr.ByteSize(), data); + auto data = new char[str_arr.ByteSizeLong()]; + str_arr.SerializeToArray(data, str_arr.ByteSizeLong()); + index->BuildWithRawData(str_arr.ByteSizeLong(), data); std::vector invalid_strings = {std::to_string(nb)}; auto copy_index = milvus::index::CreateStringIndexMarisa(); diff --git a/internal/core/unittest/test_utils/indexbuilder_test_utils.h b/internal/core/unittest/test_utils/indexbuilder_test_utils.h index 994b8a6bb7..14be09e374 100644 --- a/internal/core/unittest/test_utils/indexbuilder_test_utils.h +++ b/internal/core/unittest/test_utils/indexbuilder_test_utils.h @@ -474,9 +474,9 @@ PrintMapParams(const std::vector& tps) { // memory generated by this function should be freed by the caller. auto GenDsFromPB(const google::protobuf::Message& msg) { - auto data = new char[msg.ByteSize()]; - msg.SerializeToArray(data, msg.ByteSize()); - return knowhere::GenDataset(msg.ByteSize(), 8, data); + auto data = new char[msg.ByteSizeLong()]; + msg.SerializeToArray(data, msg.ByteSizeLong()); + return knowhere::GenDataset(msg.ByteSizeLong(), 8, data); } template diff --git a/scripts/core_build.sh b/scripts/core_build.sh index 3185a6bb61..9e834b2f44 100755 --- a/scripts/core_build.sh +++ b/scripts/core_build.sh @@ -37,11 +37,11 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located done -SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" +ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" -CPP_SRC_DIR="${SCRIPTS_DIR}/../internal/core" +CPP_SRC_DIR="${ROOT_DIR}/internal/core" -BUILD_OUTPUT_DIR="${SCRIPTS_DIR}/../cmake_build" +BUILD_OUTPUT_DIR="${ROOT_DIR}/cmake_build" BUILD_TYPE="Release" BUILD_UNITTEST="OFF" INSTALL_PREFIX="${CPP_SRC_DIR}/output" @@ -180,17 +180,42 @@ fi unameOut="$(uname -s)" case "${unameOut}" in - Darwin*) - llvm_prefix="$(brew --prefix llvm)" - export CLANG_TOOLS_PATH="${llvm_prefix}/bin" - export CC="${llvm_prefix}/bin/clang" - export CXX="${llvm_prefix}/bin/clang++" - export LDFLAGS="-L${llvm_prefix}/lib -L/usr/local/opt/libomp/lib" - export CXXFLAGS="-I${llvm_prefix}/include -I/usr/local/include -I/usr/local/opt/libomp/include" - ;; - *) echo "==System:${unameOut}"; -esac + Darwin*) + llvm_prefix="$(brew --prefix llvm)" + export CLANG_TOOLS_PATH="${llvm_prefix}/bin" + export CC="${llvm_prefix}/bin/clang" + export CXX="${llvm_prefix}/bin/clang++" + export LDFLAGS="-L${llvm_prefix}/lib -L/usr/local/opt/libomp/lib" + export CXXFLAGS="-I${llvm_prefix}/include -I/usr/local/include -I/usr/local/opt/libomp/include" + conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler=clang -s compiler.libcxx=libc++ + ;; + Linux*) + if [[ `gcc -v 2>&1 | sed -n 's/.*\(--with-default-libstdcxx-abi\)=\(\w*\).*/\2/p'` == "gcc4" ]]; then + conan install ${CPP_SRC_DIR} --install-folder conan --build=missing + else + conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.libcxx=libstdc++11 + fi + ;; + *) + cat << EOF >> msys2_profile +[tool_requires] +mingw-w64/8.1 +[settings] +os_build=Windows +os=Windows +arch=x86_64 +arch_build=x86_64 +compiler=gcc +compiler.version=8.4 +compiler.exception=seh +compiler.libcxx=libstdc++11 +compiler.threads=posix +build_type=Release +EOF + conan install ${CPP_SRC_DIR} --install-folder conan --build=missing --profile msys2_profile + ;; +esac CMAKE_CMD="cmake \ ${CMAKE_EXTRA_ARGS} \ @@ -213,29 +238,6 @@ ${CPP_SRC_DIR}" echo ${CMAKE_CMD} ${CMAKE_CMD} -G "${CMAKE_GENERATOR}" - -# enable offline build of arrow dependency if files exist. -arrowDepKeys=( -"ARROW_JEMALLOC_URL" -"ARROW_THRIFT_URL" -"ARROW_UTF8PROC_URL" -"ARROW_XSIMD_URL" -"ARROW_ZSTD_URL" -) -arrowDepValues=( -"jemalloc-5.2.1.tar.bz2" -"thrift-0.13.0.tar.gz" -"utf8proc-v2.7.0.tar.gz" -"xsimd-7d1778c3b38d63db7cec7145d939f40bc5d859d1.tar.gz" -"zstd-v1.5.1.tar.gz" -) -for i in "${!arrowDepValues[@]}"; do - if test -f "${CUSTOM_THIRDPARTY_PATH}/${arrowDepValues[$i]}"; then - echo "${arrowDepValues[$i]} exists." - export ${arrowDepKeys[$i]}=${CUSTOM_THIRDPARTY_PATH}/${arrowDepValues[$i]} - fi -done - set if [[ ${RUN_CPPLINT} == "ON" ]]; then @@ -254,14 +256,6 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then exit 1 fi echo "clang-format check passed!" - - # clang-tidy check - # make check-clang-tidy || true - # if [ $? -ne 0 ]; then - # echo "ERROR! clang-tidy check failed" - # exit 1 - # fi - # echo "clang-tidy check passed!" else # compile and build make -j ${jobs} install || exit 1 diff --git a/scripts/install_deps.sh b/scripts/install_deps.sh index 5d58bb3f55..1720846d60 100755 --- a/scripts/install_deps.sh +++ b/scripts/install_deps.sh @@ -18,48 +18,39 @@ function install_linux_deps() { if [[ -x "$(command -v apt)" ]]; then - # for Ubuntu 18.04 - sudo apt install -y g++ gcc make lcov libtool m4 autoconf automake ccache libssl-dev zlib1g-dev libboost-regex-dev \ - libboost-program-options-dev libboost-system-dev libboost-filesystem-dev \ - libboost-serialization-dev python3-dev libboost-python-dev libcurl4-openssl-dev gfortran libtbb-dev libzstd-dev libaio-dev \ - uuid-dev libpulse-dev + # for Ubuntu 18.04 & 20.04 + sudo apt install -y wget curl ca-certificates gnupg2 \ + g++ gcc gfortran git make ccache libssl-dev zlib1g-dev unzip \ + clang-format-10 clang-tidy-10 lcov libtool m4 autoconf automake python3 python3-pip \ + pkg-config uuid-dev libaio-dev libgoogle-perftools-dev + + sudo pip3 install conan elif [[ -x "$(command -v yum)" ]]; then - # for CentOS 7 - sudo yum install -y epel-release centos-release-scl-rh && \ - sudo yum install -y git make lcov libtool m4 autoconf automake ccache openssl-devel zlib-devel libzstd-devel \ - libcurl-devel python3-devel \ - devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran \ - llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra libuuid-devel pulseaudio-libs-devel + # for CentOS devtoolset-7 + sudo yum install -y epel-release centos-release-scl-rh + sudo yum install -y wget curl which \ + git make automake python3-devel \ + devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran \ + llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra \ + libaio libuuid-devel unzip \ + ccache lcov libtool m4 autoconf automake - echo "source scl_source enable devtoolset-7" | sudo tee -a /etc/profile.d/devtoolset-7.sh - echo "source scl_source enable llvm-toolset-7.0" | sudo tee -a /etc/profile.d/llvm-toolset-7.sh - echo "export CLANG_TOOLS_PATH=/opt/rh/llvm-toolset-7.0/root/usr/bin" | sudo tee -a /etc/profile.d/llvm-toolset-7.sh - source "/etc/profile.d/llvm-toolset-7.sh" - - # Install tbb - git clone https://github.com/wjakob/tbb.git && \ - cd tbb/build && \ - cmake .. && make -j && \ - sudo make install && \ - cd ../../ && rm -rf tbb/ - - # Install boost - wget -q https://boostorg.jfrog.io/artifactory/main/release/1.65.1/source/boost_1_65_1.tar.gz && \ - tar zxf boost_1_65_1.tar.gz && cd boost_1_65_1 && \ - ./bootstrap.sh --prefix=/usr/local --with-toolset=gcc --without-libraries=python && \ - sudo ./b2 -j2 --prefix=/usr/local --without-python toolset=gcc install && \ - cd ../ && rm -rf ./boost_1_65_1* + sudo pip3 install conan + echo "source scl_source enable devtoolset-7" | sudo tee -a /etc/profile.d/devtoolset-7.sh + echo "source scl_source enable llvm-toolset-7.0" | sudo tee -a /etc/profile.d/llvm-toolset-7.sh + echo "export CLANG_TOOLS_PATH=/opt/rh/llvm-toolset-7.0/root/usr/bin" | sudo tee -a /etc/profile.d/llvm-toolset-7.sh + source "/etc/profile.d/llvm-toolset-7.sh" else - echo "Error Install Dependencies ..." - exit 1 + echo "Error Install Dependencies ..." + exit 1 fi + # install cmake + wget -qO- "https://cmake.org/files/v3.24/cmake-3.24.0-linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local } function install_mac_deps() { - sudo xcode-select --install > /dev/null 2>&1 - brew install boost libomp ninja tbb cmake llvm ccache zstd - brew uninstall grep - brew install grep + sudo xcode-select --install > /dev/null 2>&1 + brew install conan libomp ninja cmake llvm ccache grep export PATH="/usr/local/opt/grep/libexec/gnubin:$PATH" brew update && brew upgrade && brew cleanup diff --git a/scripts/install_deps_msys.sh b/scripts/install_deps_msys.sh index a08296010c..ba1cfc9c0e 100644 --- a/scripts/install_deps_msys.sh +++ b/scripts/install_deps_msys.sh @@ -15,18 +15,15 @@ pacmanInstall() mingw-w64-x86_64-make \ mingw-w64-x86_64-ccache \ mingw-w64-x86_64-cmake \ - mingw-w64-x86_64-boost \ - mingw-w64-x86_64-intel-tbb \ mingw-w64-x86_64-openblas \ mingw-w64-x86_64-clang \ mingw-w64-x86_64-clang-tools-extra \ mingw-w64-x86_64-python2 \ + mingw-w64-x86_64-python-pip \ mingw-w64-x86_64-diffutils \ - mingw-w64-x86_64-arrow \ - mingw-w64-x86_64-go \ - mingw-w64-x86_64-zstd - pacman -U --noconfirm \ - https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-rocksdb-6.26.1-1-any.pkg.tar.zst + mingw-w64-x86_64-go + + pip3 install conan } updateKey()