From d10873a9abfe14acb2dcca71633cd00d90dff32c Mon Sep 17 00:00:00 2001 From: XuanYang-cn <51370125+XuanYang-cn@users.noreply.github.com> Date: Sat, 25 Jul 2020 14:30:01 +0800 Subject: [PATCH] Rearrange CMlists dir db (#3005) * delete repeated parent-scoup set INDEX_INCLUDE_DIRS Signed-off-by: yangxuan * -ERearrange CMLists.txt for dir db Signed-off-by: yangxuan * Rearrange CMkListx.txt for dir db Signed-off-by: yangxuan * fix index include dirs not find problem Signed-off-by: yangxuan * fix a bug Signed-off-by: yangxuan Co-authored-by: yangxuan --- core/CMakeLists.txt | 5 + core/src/CMakeLists.txt | 167 ++------------------------------ core/src/db/CMakeLists.txt | 151 +++++++++++++++++++++++++++++ core/src/tracing/CMakeLists.txt | 2 +- 4 files changed, 167 insertions(+), 158 deletions(-) create mode 100644 core/src/db/CMakeLists.txt diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 9117013422..4a29e302a8 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -63,6 +63,8 @@ if (MILVUS_GPU_VERSION) add_compile_definitions("MILVUS_GPU_VERSION") enable_language(CUDA) find_package(CUDA 10 REQUIRED) + include_directories(${CUDA_INCLUDE_DIRS}) + link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda") else () message(STATUS "Building Milvus CPU version") @@ -85,8 +87,10 @@ else () endif () # **************************** Source files **************************** + add_subdirectory(src) +# Unittest lib if (BUILD_UNIT_TEST STREQUAL "ON") if (BUILD_COVERAGE STREQUAL "ON") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") @@ -95,6 +99,7 @@ if (BUILD_UNIT_TEST STREQUAL "ON") add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/unittest) endif () + add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean) # **************************** Install **************************** diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index 0732c66f3f..9e5cc5e451 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -18,35 +18,17 @@ include_directories(${MILVUS_THIRDPARTY_SRC}) include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status) include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus) -add_subdirectory(config) -add_subdirectory(tracing) set(FOUND_OPENBLAS "unknown") add_subdirectory(index) - set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE) foreach (dir ${INDEX_INCLUDE_DIRS}) include_directories(${dir}) endforeach () -aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files) aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files) aux_source_directory(${MILVUS_ENGINE_SRC}/metrics/prometheus metrics_prometheus_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/attr db_attr_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/merge db_merge_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/wal db_wal_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/snapshot db_snapshot_files) - -aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_main_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta/backend db_meta_backend_files) -set(db_meta_files - ${db_meta_main_files} - ${db_meta_backend_files} - ) set(grpc_service_files ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.grpc.pb.cc @@ -55,8 +37,6 @@ set(grpc_service_files ${MILVUS_ENGINE_SRC}/grpc/gen-status/status.pb.cc ) -add_subdirectory(query) -add_subdirectory(search) # Now no use of context aux_source_directory(${MILVUS_ENGINE_SRC}/context context_files) @@ -78,15 +58,6 @@ set(scheduler_files ${scheduler_task_files} ) -aux_source_directory(${MILVUS_THIRDPARTY_SRC}/easyloggingpp thirdparty_easyloggingpp_files) -aux_source_directory(${MILVUS_THIRDPARTY_SRC}/nlohmann thirdparty_nlohmann_files) -aux_source_directory(${MILVUS_THIRDPARTY_SRC}/dablooms thirdparty_dablooms_files) -set(thirdparty_files - ${thirdparty_easyloggingpp_files} - ${thirdparty_nlohmann_files} - ${thirdparty_dablooms_files} - ) - aux_source_directory(${MILVUS_ENGINE_SRC}/server server_service_files) aux_source_directory(${MILVUS_ENGINE_SRC}/server/init server_init_files) aux_source_directory(${MILVUS_ENGINE_SRC}/server/delivery/request delivery_request_files) @@ -127,54 +98,6 @@ set(web_server_files ${web_impl_files} ) -aux_source_directory(${MILVUS_ENGINE_SRC}/storage storage_main_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/storage/disk storage_disk_files) -#aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 storage_s3_files) -set(storage_files - ${storage_main_files} - ${storage_disk_files} -# ${storage_s3_files} - ) - -aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files) - -aux_source_directory(${MILVUS_ENGINE_SRC}/index/archive wrapper_files) - -aux_source_directory(${MILVUS_ENGINE_SRC}/codecs codecs_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/codecs/default codecs_default_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/codecs/snapshot codecs_snapshot_files) - -aux_source_directory(${MILVUS_ENGINE_SRC}/segment segment_files) - -set(engine_files - ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp - ${cache_files} - ${db_main_files} - ${db_attr_files} - ${db_engine_files} - ${db_insert_files} - ${db_meta_files} - ${db_merge_files} - ${db_wal_files} - ${db_snapshot_files} - ${storage_files} - ${thirdparty_files} - ${utils_files} - ${wrapper_files} - ${codecs_files} - ${codecs_default_files} - ${codecs_snapshot_files} - ${segment_files} - ) - -set(grpc_lib - grpcpp_channelz - grpc++ - grpc - grpc_protobuf - grpc_protoc - ) - set(prometheus_lib prometheus-cpp-push prometheus-cpp-pull @@ -182,85 +105,24 @@ set(prometheus_lib curl ) -set(boost_lib - libboost_system.a - libboost_filesystem.a - libboost_serialization.a - ) - set(s3_client_lib aws-cpp-sdk-s3 aws-cpp-sdk-core ) -set(third_party_libs - sqlite - ${grpc_lib} - yaml-cpp - mysqlpp - zlib - fiu - ${boost_lib} - ) -if (MILVUS_GPU_VERSION) - include_directories(${CUDA_INCLUDE_DIRS}) - link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") - set(cuda_lib - ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so - cudart - cublas - ) - set(third_party_libs ${third_party_libs} - ${cuda_lib} - ) - aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/gpu wrapper_gpu_files) - set(engine_files ${engine_files} - ${wrapper_gpu_files} - ) -endif () +add_subdirectory(config) +add_subdirectory(tracing) +add_subdirectory(query) +add_subdirectory(search) +add_subdirectory(db) # target milvus_engine -# cannot be enabled together with ENABLE_CPU_PROFILING -if (ENABLE_MEM_PROFILING) - set(third_party_libs ${third_party_libs} - tcmalloc - ) -endif () +# **************************** Get&Print Include Directories **************************** +get_property(dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES ) -if (ENABLE_CPU_PROFILING) - set(third_party_libs ${third_party_libs} - gperftools - libunwind - ) -endif () - -if (MILVUS_WITH_PROMETHEUS) - set(third_party_libs ${third_party_libs} - ${prometheus_lib} - ) -endif () - -if (MILVUS_WITH_AWS) - set(third_party_libs ${third_party_libs} - ${s3_client_lib} - curl - crypto - ) -endif () - -set(engine_libs - pthread - libgomp.a - libgfortran.a - dl - ) - -if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") - set(engine_libs - ${engine_libs} - libquadmath.a - ) -endif () +foreach (dir ${dirs}) + message(STATUS "Current Include DIRS: "${dir}) +endforeach () if (MILVUS_WITH_PROMETHEUS) add_library(metrics STATIC ${metrics_files} ${metrics_prometheus_files}) @@ -281,15 +143,6 @@ endif () target_link_libraries(metrics ${metrics_lib}) - -add_library(milvus_engine STATIC ${engine_files}) -target_link_libraries(milvus_engine - knowhere - ${third_party_libs} - ${engine_libs} - metrics - ) - set(server_libs milvus_engine config diff --git a/core/src/db/CMakeLists.txt b/core/src/db/CMakeLists.txt new file mode 100644 index 0000000000..cfb7ef6470 --- /dev/null +++ b/core/src/db/CMakeLists.txt @@ -0,0 +1,151 @@ +aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/attr db_attr_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/merge db_merge_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/wal db_wal_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/snapshot db_snapshot_files) + +aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_main_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta/backend db_meta_backend_files) +set(db_meta_files + ${db_meta_main_files} + ${db_meta_backend_files} + ) + +aux_source_directory(${MILVUS_ENGINE_SRC}/storage storage_main_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/storage/disk storage_disk_files) +#aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 storage_s3_files) +set(storage_files + ${storage_main_files} + ${storage_disk_files} +# ${storage_s3_files} + ) +aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/index/archive wrapper_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/codecs codecs_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/codecs/default codecs_default_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/codecs/snapshot codecs_snapshot_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/segment segment_files) + + +aux_source_directory(${MILVUS_THIRDPARTY_SRC}/easyloggingpp thirdparty_easyloggingpp_files) +aux_source_directory(${MILVUS_THIRDPARTY_SRC}/nlohmann thirdparty_nlohmann_files) +aux_source_directory(${MILVUS_THIRDPARTY_SRC}/dablooms thirdparty_dablooms_files) + +set(thirdparty_files + ${thirdparty_easyloggingpp_files} + ${thirdparty_nlohmann_files} + ${thirdparty_dablooms_files} + ) + +set(engine_files + ${CMAKE_CURRENT_SOURCE_DIR}/../main.cpp + ${cache_files} + ${db_main_files} + ${db_attr_files} + ${db_engine_files} + ${db_insert_files} + ${db_meta_files} + ${db_merge_files} + ${db_wal_files} + ${db_snapshot_files} + ${storage_files} + ${thirdparty_files} + ${utils_files} + ${wrapper_files} + ${codecs_files} + ${codecs_default_files} + ${codecs_snapshot_files} + ${segment_files} + ) + +set(engine_libs + pthread + libgomp.a + libgfortran.a + ) + +set(grpc_lib + grpcpp_channelz + grpc++ + grpc + grpc_protobuf + grpc_protoc + ) + +set(boost_lib + libboost_system.a + libboost_filesystem.a + libboost_serialization.a + ) + +set(third_party_libs + sqlite + ${grpc_lib} + yaml-cpp + mysqlpp + zlib + fiu + ${boost_lib} + ) + +if (MILVUS_GPU_VERSION) + set(cuda_lib + ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so + cudart + cublas + ) + set(third_party_libs ${third_party_libs} + ${cuda_lib} + ) + + aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/gpu wrapper_gpu_files) + set(engine_files ${engine_files} + ${wrapper_gpu_files} + ) +endif () + +# cannot be enabled together with ENABLE_CPU_PROFILING +if (ENABLE_MEM_PROFILING) + set(third_party_libs ${third_party_libs} + tcmalloc + ) +endif () + +if (ENABLE_CPU_PROFILING) + set(third_party_libs ${third_party_libs} + gperftools + libunwind + ) +endif () + +if (MILVUS_WITH_PROMETHEUS) + set(third_party_libs ${third_party_libs} + ${prometheus_lib} + ) +endif () + +if (MILVS_WITH_AWS) + set(third_party_libs ${third_party_libs} + ${s3_client_lib} + curl + crypto + ) +endif () + +if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") + set(engine_libs + ${engine_libs} + libquadmath.a + ) +endif () + +add_library(milvus_engine STATIC ${engine_files}) +target_link_libraries(milvus_engine + PUBLIC + knowhere + ${third_party_libs} + ${engine_libs} + ) diff --git a/core/src/tracing/CMakeLists.txt b/core/src/tracing/CMakeLists.txt index c9106f357a..8c950a4234 100644 --- a/core/src/tracing/CMakeLists.txt +++ b/core/src/tracing/CMakeLists.txt @@ -5,8 +5,8 @@ add_library(tracing STATIC ${tracing_files} ${thirdparty_files}) set(tracing_lib opentracing opentracing_mocktracer - ${grpc_lib} pthread z + dl ) target_link_libraries(tracing ${tracing_lib})