From dc707bb8a39854a78bf0e34983b4e454e82d73c7 Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Fri, 3 Apr 2020 16:35:48 +0800 Subject: [PATCH] Caiyd 1858 disable s3 build (#1860) * #1858 disable s3 build Signed-off-by: yudong.cai * update changelog Signed-off-by: yudong.cai * disable s3 in unittest Signed-off-by: yudong.cai --- CHANGELOG.md | 1 + core/cmake/DefineOptions.cmake | 2 +- core/src/CMakeLists.txt | 15 ++++++++++----- core/src/config/Config.cpp | 2 +- core/src/db/Utils.cpp | 24 ++++++++++++------------ core/src/server/Server.cpp | 14 +++++++------- core/unittest/CMakeLists.txt | 29 +++++++++++++++++++---------- 7 files changed, 51 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a13c046b2..0286dea862 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ Please mark all change in change log and use the issue from GitHub ## Improvement - \#1627 Move read/write index APIs into codec - \#1784 Add Substructure and Superstructure in http module +- \#1858 Disable S3 build ## Task diff --git a/core/cmake/DefineOptions.cmake b/core/cmake/DefineOptions.cmake index d5eab6a6a0..e9d5ab1340 100644 --- a/core/cmake/DefineOptions.cmake +++ b/core/cmake/DefineOptions.cmake @@ -87,7 +87,7 @@ define_option(MILVUS_WITH_OPENTRACING "Build with Opentracing" ON) define_option(MILVUS_WITH_FIU "Build with fiu" OFF) -define_option(MILVUS_WITH_AWS "Build with aws" ON) +define_option(MILVUS_WITH_AWS "Build with aws" OFF) define_option(MILVUS_WITH_OATPP "Build with oatpp" ON) diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index da650b0673..0d6b6e71a1 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -109,11 +109,11 @@ set(web_server_files aux_source_directory(${MILVUS_ENGINE_SRC}/storage storage_main_files) aux_source_directory(${MILVUS_ENGINE_SRC}/storage/disk storage_disk_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 storage_s3_files) +#aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 storage_s3_files) set(storage_files ${storage_main_files} ${storage_disk_files} - ${storage_s3_files} +# ${storage_s3_files} ) aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files) @@ -184,9 +184,6 @@ set(third_party_libs zlib fiu ${boost_lib} - ${s3_client_lib} - curl - crypto ) if (MILVUS_GPU_VERSION) @@ -219,6 +216,14 @@ if (MILVUS_WITH_PROMETHEUS) ) endif () +if (MILVUS_WITH_AWS) + set(third_party_libs ${third_party_libs} + ${s3_client_lib} + curl + crypto + ) +endif () + set(engine_libs pthread libgomp.a diff --git a/core/src/config/Config.cpp b/core/src/config/Config.cpp index eab0d70410..fd30835387 100644 --- a/core/src/config/Config.cpp +++ b/core/src/config/Config.cpp @@ -154,7 +154,7 @@ Config::ValidateConfig() { bool storage_s3_enable; CONFIG_CHECK(GetStorageConfigS3Enable(storage_s3_enable)); - std::cout << "S3 " << (storage_s3_enable ? "ENABLED !" : "DISABLED !") << std::endl; + // std::cout << "S3 " << (storage_s3_enable ? "ENABLED !" : "DISABLED !") << std::endl; std::string storage_s3_address; CONFIG_CHECK(GetStorageConfigS3Address(storage_s3_address)); diff --git a/core/src/db/Utils.cpp b/core/src/db/Utils.cpp index f714813884..acb8307d3f 100644 --- a/core/src/db/Utils.cpp +++ b/core/src/db/Utils.cpp @@ -20,7 +20,7 @@ #include #include "config/Config.h" -#include "storage/s3/S3ClientWrapper.h" +//#include "storage/s3/S3ClientWrapper.h" #include "utils/CommonUtil.h" #include "utils/Log.h" @@ -117,19 +117,19 @@ DeleteTablePath(const DBMetaOptions& options, const std::string& collection_id, } } - bool s3_enable = false; - server::Config& config = server::Config::GetInstance(); - config.GetStorageConfigS3Enable(s3_enable); + // bool s3_enable = false; + // server::Config& config = server::Config::GetInstance(); + // config.GetStorageConfigS3Enable(s3_enable); - if (s3_enable) { - std::string table_path = options.path_ + TABLES_FOLDER + collection_id; + // if (s3_enable) { + // std::string table_path = options.path_ + TABLES_FOLDER + collection_id; - auto& storage_inst = milvus::storage::S3ClientWrapper::GetInstance(); - Status stat = storage_inst.DeleteObjects(table_path); - if (!stat.ok()) { - return stat; - } - } + // auto& storage_inst = milvus::storage::S3ClientWrapper::GetInstance(); + // Status stat = storage_inst.DeleteObjects(table_path); + // if (!stat.ok()) { + // return stat; + // } + // } return Status::OK(); } diff --git a/core/src/server/Server.cpp b/core/src/server/Server.cpp index 339d470e24..6036fa0333 100644 --- a/core/src/server/Server.cpp +++ b/core/src/server/Server.cpp @@ -23,7 +23,7 @@ #include "server/grpc_impl/GrpcServer.h" #include "server/web_impl/WebServer.h" #include "src/version.h" -#include "storage/s3/S3ClientWrapper.h" +//#include "storage/s3/S3ClientWrapper.h" #include "tracing/TracerUtil.h" #include "utils/Log.h" #include "utils/LogUtil.h" @@ -272,11 +272,11 @@ Server::StartService() { grpc::GrpcServer::GetInstance().Start(); web::WebServer::GetInstance().Start(); - stat = storage::S3ClientWrapper::GetInstance().StartService(); - if (!stat.ok()) { - SERVER_LOG_ERROR << "S3Client start service fail: " << stat.message(); - goto FAIL; - } + // stat = storage::S3ClientWrapper::GetInstance().StartService(); + // if (!stat.ok()) { + // SERVER_LOG_ERROR << "S3Client start service fail: " << stat.message(); + // goto FAIL; + // } return Status::OK(); FAIL: @@ -286,7 +286,7 @@ FAIL: void Server::StopService() { - storage::S3ClientWrapper::GetInstance().StopService(); + // storage::S3ClientWrapper::GetInstance().StopService(); web::WebServer::GetInstance().Stop(); grpc::GrpcServer::GetInstance().Stop(); DBWrapper::GetInstance().StopService(); diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index dc1119ab11..85855a11df 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -101,11 +101,11 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/tracing tracing_files) aux_source_directory(${MILVUS_ENGINE_SRC}/storage storage_main_files) aux_source_directory(${MILVUS_ENGINE_SRC}/storage/disk storage_disk_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 storage_s3_files) +#aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 storage_s3_files) set(storage_files ${storage_main_files} ${storage_disk_files} - ${storage_s3_files} +# ${storage_s3_files} ) aux_source_directory(${MILVUS_ENGINE_SRC}/codecs codecs_files) @@ -163,21 +163,30 @@ set(unittest_libs opentracing opentracing_mocktracer fiu + ) + +if (MILVUS_WITH_AWS) + set(s3_client_lib aws-cpp-sdk-s3 aws-cpp-sdk-core + ) + set(unittest_libs ${unittest_libs} + ${s3_client_lib} curl crypto ) +endif () + if (MILVUS_WITH_PROMETHEUS) set(prometheus_lib - prometheus-cpp-push - prometheus-cpp-pull - prometheus-cpp-core - curl - ) + prometheus-cpp-push + prometheus-cpp-pull + prometheus-cpp-core + ) set(unittest_libs ${unittest_libs} - ${prometheus_lib} - ) + ${prometheus_lib} + curl + ) endif () set(unittest_libs ${unittest_libs} dl @@ -199,4 +208,4 @@ add_subdirectory(metrics) add_subdirectory(scheduler) add_subdirectory(server) add_subdirectory(thirdparty) -add_subdirectory(storage) +#add_subdirectory(storage)