milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support aliyun oss storage (#16002) 

- Support aliyun oss with oss-cpp-sdk
- Add parameter for using s3 with https scheme

Signed-off-by: Ji Bin <matrixji@live.com>
pull/16152/head
Ji Bin 2022-03-11 14:34:00 +08:00 committed by GitHub
parent 9193addd8c
commit de8050d597
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 1152 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/cmake/DefineOptions.cmake
View File

@ -91,6 +91,8 @@ define_option(MILVUS_WITH_FIU "Build with fiu" OFF)
define_option(MILVUS_WITH_AWS "Build with aws" ON)
define_option(MILVUS_WITH_OSS "Build with oss" ON)
define_option(MILVUS_WITH_OATPP "Build with oatpp" ON)
#----------------------------------------------------------------------

66
core/cmake/ThirdPartyPackages.cmake
View File

@ -23,6 +23,7 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES
Opentracing
fiu
AWS
OSS
oatpp
armadillo
apu)
@ -63,6 +64,8 @@ macro(build_dependency DEPENDENCY_NAME)
build_oatpp()
elseif("${DEPENDENCY_NAME}" STREQUAL "AWS")
build_aws()
elseif("${DEPENDENCY_NAME}" STREQUAL "OSS")
build_oss()
elseif("${DEPENDENCY_NAME}" STREQUAL "armadillo")
build_armadillo()
elseif("${DEPENDENCY_NAME}" STREQUAL "apu")
@ -336,6 +339,12 @@ else ()
set(AWS_SOURCE_URL "https://github.com/aws/aws-sdk-cpp/archive/${AWS_VERSION}.tar.gz")
endif ()
if (DEFINED ENV{MILVUS_OSS_URL})
set(OSS_SOURCE_URL "$ENV{MILVUS_OSS_URL}")
else ()
set(OSS_SOURCE_URL "https://github.com/aliyun/aliyun-oss-cpp-sdk/archive/${OSS_VERSION}.tar.gz")
endif ()
if (DEFINED ENV{MILVUS_ARMADILLO_URL})
set(ARMADILLO_SOURCE_URL "$ENV{MILVUS_ARMADILLO_URL}")
else ()
@ -1115,6 +1124,63 @@ if(MILVUS_WITH_AWS)
endif()
# ----------------------------------------------------------------------
# OSS
macro(build_oss)
message(STATUS "Building aliyun-oss-sdk-${OSS_VERSION} from source")
set(OSS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/oss_ep-prefix/src/oss_ep")
set(OSS_CMAKE_ARGS
${EP_COMMON_TOOLCHAIN}
"-DCMAKE_INSTALL_PREFIX=${OSS_PREFIX}"
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_LIBDIR=lib)
set(OSS_CPP_SDK_STATIC_LIB
"${OSS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}alibabacloud-oss-cpp-sdk${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(OSS_INCLUDE_DIR "${OSS_PREFIX}/include")
set(OSS_CMAKE_ARGS
${OSS_CMAKE_ARGS}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_C_FLAGS=${EP_C_FLAGS}
-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS})
externalproject_add(oss_ep
${EP_LOG_OPTIONS}
CMAKE_ARGS
${OSS_CMAKE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
INSTALL_DIR
${OSS_PREFIX}
URL
${OSS_SOURCE_URL}
BUILD_BYPRODUCTS
"${OSS_CPP_SDK_STATIC_LIB}")
file(MAKE_DIRECTORY "${OSS_INCLUDE_DIR}")
add_library(oss-cpp-sdk STATIC IMPORTED)
set_target_properties(oss-cpp-sdk
PROPERTIES
IMPORTED_LOCATION "${OSS_CPP_SDK_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${OSS_INCLUDE_DIR}"
)
add_dependencies(oss-cpp-sdk oss_ep)
endmacro()
if(MILVUS_WITH_OSS)
resolve_dependency(OSS)
link_directories(SYSTEM ${OSS_PREFIX}/lib)
get_target_property(OSS_CPP_SDK_INCLUDE_DIR oss-cpp-sdk INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM ${OSS_CPP_SDK_INCLUDE_DIR})
endif()
# ----------------------------------------------------------------------
# armadillo

16
core/conf/demo/server_config.yaml
View File

@ -66,6 +66,8 @@ network:
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_enabled | If using s3 storage backend. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_use_https | If using s3 using https scheme. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_address | The s3 server address, support domain/hostname/ipaddress | String | 127.0.0.1 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_port | The s3 server port. | Integer | 80 |
@ -83,6 +85,20 @@ network:
# | like ceph-radosgw, minio, when deploy on aws s3 service, | | |
# | this should be set correctly, e.g "ap-east-1", "us-east-1" | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_enabled | If using oss storage backend. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_endpoint | The oss endpoint. | String | see description |
# | default: https://oss-cn-hangzhou.aliyuncs.com | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_access_key | The access key for accessing oss service. | String | oss_access_key |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_secret_key | The secrey key for accessing oss service. | String | oss_secret_key |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_bucket | The oss bucket name for store milvus's data. | String | oss_bucket |
# | Note: please using differnet bucket for different milvus | | |
# | cluster. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
storage:
path: /var/lib/milvus
auto_flush_interval: 1

17
core/src/CMakeLists.txt
View File

@ -121,11 +121,15 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/storage/disk storage_disk_files)
if(MILVUS_WITH_AWS)
aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 storage_s3_files)
endif()
if(MILVUS_WITH_OSS)
aux_source_directory(${MILVUS_ENGINE_SRC}/storage/oss storage_oss_files)
endif()
set(storage_files
${storage_main_files}
${storage_disk_files}
${storage_s3_files}
${storage_oss_files}
)
aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files)
@ -189,6 +193,10 @@ set(s3_client_lib
aws-cpp-sdk-core
)
set(oss_sdk_lib
alibabacloud-oss-cpp-sdk
)
set(third_party_libs
sqlite
${grpc_lib}
@ -245,6 +253,15 @@ if (MILVUS_WITH_AWS)
)
endif ()
if (MILVUS_WITH_OSS)
add_compile_definitions(MILVUS_WITH_OSS)
set(third_party_libs ${third_party_libs}
${oss_sdk_lib}
curl
crypto
)
endif ()
set(engine_libs
pthread
libgomp.a

181
core/src/config/Config.cpp
View File

@ -88,6 +88,8 @@ const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_DEFAULT = "10";
#ifdef MILVUS_WITH_AWS
const char* CONFIG_STORAGE_S3_ENABLE = "s3_enabled";
const char* CONFIG_STORAGE_S3_ENABLE_DEFAULT = "false";
const char* CONFIG_STORAGE_S3_USE_HTTPS = "s3_use_https";
const char* CONFIG_STORAGE_S3_USE_HTTPS_DEFAULT = "false";
const char* CONFIG_STORAGE_S3_ADDRESS = "s3_address";
const char* CONFIG_STORAGE_S3_ADDRESS_DEFAULT = "127.0.0.1";
const char* CONFIG_STORAGE_S3_PORT = "s3_port";
@ -101,6 +103,18 @@ const char* CONFIG_STORAGE_S3_BUCKET_DEFAULT = "";
const char* CONFIG_STORAGE_S3_REGION = "s3_region";
const char* CONFIG_STORAGE_S3_REGION_DEFAULT = "";
#endif
#ifdef MILVUS_WITH_OSS
const char* CONFIG_STORAGE_OSS_ENABLE = "oss_enabled";
const char* CONFIG_STORAGE_OSS_ENABLE_DEFAULT = "false";
const char* CONFIG_STORAGE_OSS_ENDPOINT = "oss_endpoint";
const char* CONFIG_STORAGE_OSS_ENDPOINT_DEFAULT = "https://oss-cn-hangzhou.aliyuncs.com";
const char* CONFIG_STORAGE_OSS_ACCESS_KEY = "oss_access_key";
const char* CONFIG_STORAGE_OSS_ACCESS_KEY_DEFAULT = "";
const char* CONFIG_STORAGE_OSS_SECRET_KEY = "oss_secret_key";
const char* CONFIG_STORAGE_OSS_SECRET_KEY_DEFAULT = "";
const char* CONFIG_STORAGE_OSS_BUCKET = "oss_bucket";
const char* CONFIG_STORAGE_OSS_BUCKET_DEFAULT = "";
#endif
const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MIN = 0;
const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MAX = 3600;
@ -367,6 +381,9 @@ Config::ValidateConfig() {
STATUS_CHECK(GetStorageConfigS3Enable(storage_s3_enable));
// std::cout << "S3 " << (storage_s3_enable ? "ENABLED !" : "DISABLED !") << std::endl;
bool storage_s3_use_https;
STATUS_CHECK(GetStorageConfigS3UseHttps(storage_s3_use_https));
std::string storage_s3_address;
STATUS_CHECK(GetStorageConfigS3Address(storage_s3_address));
@ -386,6 +403,23 @@ Config::ValidateConfig() {
STATUS_CHECK(GetStorageConfigS3Region(storage_s3_region));
#endif
#ifdef MILVUS_WITH_OSS
bool storage_oss_enable;
STATUS_CHECK(GetStorageConfigOSSEnable(storage_oss_enable));
std::string storage_oss_endpoint;
STATUS_CHECK(GetStorageConfigOSSEndpoint(storage_oss_endpoint));
std::string storage_oss_access_key;
STATUS_CHECK(GetStorageConfigOSSAccessKey(storage_oss_access_key));
std::string storage_oss_secret_key;
STATUS_CHECK(GetStorageConfigOSSSecretKey(storage_oss_secret_key));
std::string storage_oss_bucket;
STATUS_CHECK(GetStorageConfigOSSBucket(storage_oss_bucket));
#endif
/* metric config */
bool metric_enable_monitor;
STATUS_CHECK(GetMetricConfigEnableMonitor(metric_enable_monitor));
@ -532,6 +566,7 @@ Config::ResetDefaultConfig() {
STATUS_CHECK(SetStorageConfigFileCleanupTimeout(CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_DEFAULT));
#ifdef MILVUS_WITH_AWS
STATUS_CHECK(SetStorageConfigS3Enable(CONFIG_STORAGE_S3_ENABLE_DEFAULT));
STATUS_CHECK(SetStorageConfigS3UseHttps(CONFIG_STORAGE_S3_USE_HTTPS_DEFAULT));
STATUS_CHECK(SetStorageConfigS3Address(CONFIG_STORAGE_S3_ADDRESS_DEFAULT));
STATUS_CHECK(SetStorageConfigS3Port(CONFIG_STORAGE_S3_PORT_DEFAULT));
STATUS_CHECK(SetStorageConfigS3AccessKey(CONFIG_STORAGE_S3_ACCESS_KEY_DEFAULT));
@ -539,6 +574,14 @@ Config::ResetDefaultConfig() {
STATUS_CHECK(SetStorageConfigS3Bucket(CONFIG_STORAGE_S3_BUCKET_DEFAULT));
#endif
#ifdef MILVUS_WITH_OSS
STATUS_CHECK(SetStorageConfigOSSEnable(CONFIG_STORAGE_OSS_ENABLE_DEFAULT));
STATUS_CHECK(SetStorageConfigOSSEndpoint(CONFIG_STORAGE_OSS_ENDPOINT_DEFAULT));
STATUS_CHECK(SetStorageConfigOSSAccessKey(CONFIG_STORAGE_OSS_ACCESS_KEY_DEFAULT));
STATUS_CHECK(SetStorageConfigOSSSecretKey(CONFIG_STORAGE_OSS_SECRET_KEY_DEFAULT));
STATUS_CHECK(SetStorageConfigOSSBucket(CONFIG_STORAGE_OSS_BUCKET_DEFAULT));
#endif
/* metric config */
STATUS_CHECK(SetMetricConfigEnableMonitor(CONFIG_METRIC_ENABLE_MONITOR_DEFAULT));
STATUS_CHECK(SetMetricConfigAddress(CONFIG_METRIC_ADDRESS_DEFAULT));
@ -1327,6 +1370,16 @@ Config::CheckStorageConfigS3Enable(const std::string& value) {
return Status::OK();
}
Status
Config::CheckStorageConfigS3UseHttps(const std::string& value) {
if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
std::string msg =
"Invalid storage config: " + value + ". Possible reason: storage_config.s3_use_https is not a boolean.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
Status
Config::CheckStorageConfigS3Address(const std::string& value) {
if (!ValidationUtil::ValidateHostname(value).ok()) {
@ -1387,6 +1440,52 @@ Config::CheckStorageConfigS3Region(const std::string& /* unused */) {
}
#endif
#ifdef MILVUS_WITH_OSS
Status
Config::CheckStorageConfigOSSEnable(const std::string& value) {
if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
std::string msg =
"Invalid storage config: " + value + ". Possible reason: storage_config.oss_enable is not a boolean.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
Status
Config::CheckStorageConfigOSSEndpoint(const std::string& value) {
if (value.empty()) {
return Status(SERVER_INVALID_ARGUMENT, "storage_config.oss_endpoint is empty.");
}
return Status::OK();
}
Status
Config::CheckStorageConfigOSSAccessKey(const std::string& value) {
if (value.empty()) {
return Status(SERVER_INVALID_ARGUMENT, "storage_config.oss_access_key is empty.");
}
return Status::OK();
}
Status
Config::CheckStorageConfigOSSSecretKey(const std::string& value) {
if (value.empty()) {
return Status(SERVER_INVALID_ARGUMENT, "storage_config.oss_secret_key is empty.");
}
return Status::OK();
}
Status
Config::CheckStorageConfigOSSBucket(const std::string& value) {
if (value.empty()) {
return Status(SERVER_INVALID_ARGUMENT, "storage_config.oss_bucket is empty.");
}
return Status::OK();
}
#endif
/* metric config */
Status
Config::CheckMetricConfigEnableMonitor(const std::string& value) {
@ -2411,6 +2510,14 @@ Config::GetStorageConfigS3Enable(bool& value) {
return Status::OK();
}
Status
Config::GetStorageConfigS3UseHttps(bool& value) {
std::string str = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_USE_HTTPS, CONFIG_STORAGE_S3_USE_HTTPS_DEFAULT);
STATUS_CHECK(CheckStorageConfigS3UseHttps(str));
STATUS_CHECK(StringHelpFunctions::ConvertToBoolean(str, value));
return Status::OK();
}
Status
Config::GetStorageConfigS3Address(std::string& value) {
value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_ADDRESS, CONFIG_STORAGE_S3_ADDRESS_DEFAULT);
@ -2448,6 +2555,41 @@ Config::GetStorageConfigS3Region(std::string& value) {
}
#endif
#ifdef MILVUS_WITH_OSS
Status
Config::GetStorageConfigOSSEnable(bool& value) {
std::string str = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_OSS_ENABLE, CONFIG_STORAGE_OSS_ENABLE_DEFAULT);
STATUS_CHECK(CheckStorageConfigOSSEnable(str));
STATUS_CHECK(StringHelpFunctions::ConvertToBoolean(str, value));
return Status::OK();
}
Status
Config::GetStorageConfigOSSEndpoint(std::string& value) {
value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_OSS_ENDPOINT, CONFIG_STORAGE_OSS_ENDPOINT_DEFAULT);
return CheckStorageConfigOSSEndpoint(value);
}
Status
Config::GetStorageConfigOSSAccessKey(std::string& value) {
value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_OSS_ACCESS_KEY, CONFIG_STORAGE_OSS_ACCESS_KEY_DEFAULT);
return Status::OK();
}
Status
Config::GetStorageConfigOSSSecretKey(std::string& value) {
value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_OSS_SECRET_KEY, CONFIG_STORAGE_OSS_SECRET_KEY_DEFAULT);
return Status::OK();
}
Status
Config::GetStorageConfigOSSBucket(std::string& value) {
value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_OSS_BUCKET, CONFIG_STORAGE_OSS_BUCKET_DEFAULT);
return Status::OK();
}
#endif
/* metric config */
Status
Config::GetMetricConfigEnableMonitor(bool& value) {
@ -2950,6 +3092,12 @@ Config::SetStorageConfigS3Enable(const std::string& value) {
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_S3_ENABLE, value);
}
Status
Config::SetStorageConfigS3UseHttps(const std::string& value) {
STATUS_CHECK(CheckStorageConfigS3UseHttps(value));
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_S3_USE_HTTPS, value);
}
Status
Config::SetStorageConfigS3Address(const std::string& value) {
STATUS_CHECK(CheckStorageConfigS3Address(value));
@ -2987,6 +3135,39 @@ Config::SetStorageConfigS3Region(const std::string& value) {
}
#endif
#ifdef MILVUS_WITH_OSS
Status
Config::SetStorageConfigOSSEnable(const std::string& value) {
STATUS_CHECK(CheckStorageConfigOSSEnable(value));
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_OSS_ENABLE, value);
}
Status
Config::SetStorageConfigOSSEndpoint(const std::string& value) {
STATUS_CHECK(CheckStorageConfigOSSEndpoint(value));
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_OSS_ENDPOINT, value);
}
Status
Config::SetStorageConfigOSSAccessKey(const std::string& value) {
STATUS_CHECK(CheckStorageConfigOSSAccessKey(value));
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_OSS_ACCESS_KEY, value);
}
Status
Config::SetStorageConfigOSSSecretKey(const std::string& value) {
STATUS_CHECK(CheckStorageConfigOSSSecretKey(value));
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_OSS_SECRET_KEY, value);
}
Status
Config::SetStorageConfigOSSBucket(const std::string& value) {
STATUS_CHECK(CheckStorageConfigOSSBucket(value));
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_OSS_BUCKET, value);
}
#endif
/* metric config */
Status
Config::SetMetricConfigEnableMonitor(const std::string& value) {

50
core/src/config/Config.h
View File

@ -272,6 +272,8 @@ class Config {
Status
CheckStorageConfigS3Enable(const std::string& value);
Status
CheckStorageConfigS3UseHttps(const std::string& value);
Status
CheckStorageConfigS3Address(const std::string& value);
Status
CheckStorageConfigS3Port(const std::string& value);
@ -285,6 +287,19 @@ class Config {
CheckStorageConfigS3Region(const std::string& value);
#endif
#ifdef MILVUS_WITH_OSS
Status
CheckStorageConfigOSSEnable(const std::string& value);
Status
CheckStorageConfigOSSEndpoint(const std::string& value);
Status
CheckStorageConfigOSSAccessKey(const std::string& value);
Status
CheckStorageConfigOSSSecretKey(const std::string& value);
Status
CheckStorageConfigOSSBucket(const std::string& value);
#endif
/* metric config */
Status
CheckMetricConfigEnableMonitor(const std::string& value);
@ -434,6 +449,9 @@ class Config {
Status
GetStorageConfigS3Enable(bool& value);
Status
GetStorageConfigS3UseHttps(bool& value);
Status
GetStorageConfigS3Address(std::string& value);
@ -453,6 +471,23 @@ class Config {
GetStorageConfigS3Region(std::string& value);
#endif
#ifdef MILVUS_WITH_OSS
Status
GetStorageConfigOSSEnable(bool& value);
Status
GetStorageConfigOSSEndpoint(std::string& value);
Status
GetStorageConfigOSSAccessKey(std::string& value);
Status
GetStorageConfigOSSSecretKey(std::string& value);
Status
GetStorageConfigOSSBucket(std::string& value);
#endif
/* metric config */
Status
GetMetricConfigEnableMonitor(bool& value);
@ -595,6 +630,8 @@ class Config {
Status
SetStorageConfigS3Enable(const std::string& value);
Status
SetStorageConfigS3UseHttps(const std::string& value);
Status
SetStorageConfigS3Address(const std::string& value);
Status
SetStorageConfigS3Port(const std::string& value);
@ -608,6 +645,19 @@ class Config {
SetStorageConfigS3Region(const std::string& value);
#endif
#ifdef MILVUS_WITH_OSS
Status
SetStorageConfigOSSEnable(const std::string& value);
Status
SetStorageConfigOSSEndpoint(const std::string& value);
Status
SetStorageConfigOSSAccessKey(const std::string& value);
Status
SetStorageConfigOSSSecretKey(const std::string& value);
Status
SetStorageConfigOSSBucket(const std::string& value);
#endif
/* metric config */
Status
SetMetricConfigEnableMonitor(const std::string& value);

55
core/src/db/Utils.cpp
View File

@ -24,6 +24,9 @@
#ifdef MILVUS_WITH_AWS
#include "storage/s3/S3ClientWrapper.h"
#endif
#ifdef MILVUS_WITH_OSS
#include "storage/oss/OSSClientWrapper.h"
#endif
#include <map>
#include "utils/CommonUtil.h"
@ -124,9 +127,12 @@ DeleteCollectionPath(const DBMetaOptions& options, const std::string& collection
}
}
#if defined(MILVUS_WITH_AWS) || defined(MILVUS_WITH_OSS)
server::Config& config = server::Config::GetInstance();
#endif
#ifdef MILVUS_WITH_AWS
bool s3_enable = false;
server::Config& config = server::Config::GetInstance();
config.GetStorageConfigS3Enable(s3_enable);
if (s3_enable) {
@ -137,6 +143,18 @@ DeleteCollectionPath(const DBMetaOptions& options, const std::string& collection
}
#endif
#ifdef MILVUS_WITH_OSS
bool oss_enable = false;
config.GetStorageConfigOSSEnable(oss_enable);
if (oss_enable) {
std::string table_path = options.path_ + TABLES_FOLDER + collection_id;
auto& storage_inst = milvus::storage::OSSClientWrapper::GetInstance();
return storage_inst.DeleteObjects(table_path);
}
#endif
return Status::OK();
}
@ -167,9 +185,13 @@ Status
DeleteCollectionFilePath(const DBMetaOptions& options, meta::SegmentSchema& table_file) {
utils::GetCollectionFilePath(options, table_file);
boost::filesystem::remove(table_file.location_);
#if 0
bool s3_enable = false;
#if defined(MILVUS_WITH_AWS) || defined(MILVUS_WITH_OSS)
server::Config& config = server::Config::GetInstance();
#endif
#if MILVUS_WITH_AWS
bool s3_enable = false;
config.GetStorageConfigS3Enable(s3_enable);
if (s3_enable) {
@ -177,6 +199,17 @@ DeleteCollectionFilePath(const DBMetaOptions& options, meta::SegmentSchema& tabl
return storage_inst.DeleteObjects(table_file.location_);
}
#endif
#if MILVUS_WITH_OSS
bool oss_enable = false;
config.GetStorageConfigOSSEnable(s3_enable);
if (oss_enable) {
auto& storage_inst = milvus::storage::OSSClientWrapper::GetInstance();
return storage_inst.DeleteObjects(table_file.location_);
}
#endif
return Status::OK();
}
@ -186,9 +219,13 @@ DeleteSegment(const DBMetaOptions& options, meta::SegmentSchema& table_file) {
std::string segment_dir;
GetParentPath(table_file.location_, segment_dir);
boost::filesystem::remove_all(segment_dir);
#if defined(MILVUS_WITH_AWS) || defined(MILVUS_WITH_OSS)
server::Config& config = server::Config::GetInstance();
#endif
#ifdef MILVUS_WITH_AWS
bool s3_enable = false;
server::Config& config = server::Config::GetInstance();
config.GetStorageConfigS3Enable(s3_enable);
if (s3_enable) {
@ -196,6 +233,16 @@ DeleteSegment(const DBMetaOptions& options, meta::SegmentSchema& table_file) {
return storage_inst.DeleteObjects(segment_dir);
}
#endif
#ifdef MILVUS_WITH_OSS
bool oss_enable = false;
config.GetStorageConfigOSSEnable(oss_enable);
if (oss_enable) {
auto& storage_inst = milvus::storage::OSSClientWrapper::GetInstance();
return storage_inst.DeleteObjects(segment_dir);
}
#endif
return Status::OK();
}

33
core/src/storage/FSHandler.h
View File

@ -32,6 +32,12 @@
#include "storage/s3/S3IOWriter.h"
#include "storage/s3/S3Operation.h"
#endif
#ifdef MILVUS_WITH_OSS
#include "storage/oss/OSSClientWrapper.h"
#include "storage/oss/OSSIOReader.h"
#include "storage/oss/OSSIOWriter.h"
#include "storage/oss/OSSOperation.h"
#endif
#include "config/Config.h"
namespace milvus {
@ -54,8 +60,12 @@ createFsHandler(const std::string& directory) {
storage::IOReaderPtr reader_ptr{nullptr};
storage::IOWriterPtr writer_ptr{nullptr};
storage::OperationPtr operation_ptr{nullptr};
#if defined(MILVUS_WITH_AWS) || defined(MILVUS_WITH_OSS)
server::Config& config = server::Config::GetInstance();
#endif
#ifdef MILVUS_WITH_AWS
auto& config = milvus::server::Config::GetInstance();
bool enableS3 = false;
config.GetStorageConfigS3Enable(enableS3);
if (enableS3) {
@ -63,16 +73,25 @@ createFsHandler(const std::string& directory) {
reader_ptr = std::make_shared<storage::S3IOReader>();
writer_ptr = std::make_shared<storage::S3IOWriter>();
operation_ptr = std::make_shared<storage::S3Operation>(directory);
} else {
reader_ptr = std::make_shared<storage::DiskIOReader>();
writer_ptr = std::make_shared<storage::DiskIOWriter>();
operation_ptr = std::make_shared<storage::DiskOperation>(directory);
return std::make_shared<storage::FSHandler>(reader_ptr, writer_ptr, operation_ptr);
}
#else
#endif
#ifdef MILVUS_WITH_OSS
bool enableOSS = false;
config.GetStorageConfigOSSEnable(enableOSS);
if (enableOSS) {
OSSClientWrapper::GetInstance();
reader_ptr = std::make_shared<storage::OSSIOReader>();
writer_ptr = std::make_shared<storage::OSSIOWriter>();
operation_ptr = std::make_shared<storage::OSSOperation>(directory);
return std::make_shared<storage::FSHandler>(reader_ptr, writer_ptr, operation_ptr);
}
#endif
reader_ptr = std::make_shared<storage::DiskIOReader>();
writer_ptr = std::make_shared<storage::DiskIOWriter>();
operation_ptr = std::make_shared<storage::DiskOperation>(directory);
#endif
return std::make_shared<storage::FSHandler>(reader_ptr, writer_ptr, operation_ptr);
}

244
core/src/storage/oss/OSSClientWrapper.cpp Normal file
View File

@ -0,0 +1,244 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "storage/oss/OSSClientWrapper.h"
#include <alibabacloud/oss/OssClient.h>
#include <fstream>
#include <iostream>
#include <memory>
#include <utility>
#include "config/Config.h"
#include "utils/Error.h"
#include "utils/Log.h"
namespace milvus {
namespace storage {
Status
OSSClientWrapper::StartService() {
server::Config& config = server::Config::GetInstance();
AlibabaCloud::OSS::InitializeSdk();
AlibabaCloud::OSS::ClientConfiguration conf;
config.GetStorageConfigOSSEndpoint(oss_endpoint_);
config.GetStorageConfigOSSAccessKey(oss_access_key_);
config.GetStorageConfigOSSSecretKey(oss_secret_key_);
config.GetStorageConfigOSSBucket(oss_bucket_);
client_ptr_ = std::make_shared<AlibabaCloud::OSS::OssClient>(oss_endpoint_, oss_access_key_, oss_secret_key_, conf);
auto status = CreateBucket();
if (!status.ok()) {
return status;
}
return Status::OK();
}
void
OSSClientWrapper::StopService() {
client_ptr_ = nullptr;
AlibabaCloud::OSS::ShutdownSdk();
}
Status
OSSClientWrapper::CreateBucket() {
AlibabaCloud::OSS::CreateBucketRequest request(oss_bucket_, AlibabaCloud::OSS::StorageClass::IA,
AlibabaCloud::OSS::CannedAccessControlList::Private);
const auto outcome = client_ptr_->CreateBucket(request);
if (!outcome.isSuccess()) {
const auto& err = outcome.error();
if (err.Code() != "BucketAlreadyExists") {
LOG_STORAGE_ERROR_ << "ERROR: CreateBucket: " << err.Code() << ": " << err.Message();
return Status(SERVER_UNEXPECTED_ERROR, err.Message());
}
}
LOG_STORAGE_DEBUG_ << "CreateBucket '" << oss_bucket_ << "' successfully!";
return Status::OK();
}
Status
OSSClientWrapper::DeleteBucket() {
const auto outcome = client_ptr_->DeleteBucket(oss_bucket_);
if (!outcome.isSuccess()) {
const auto& err = outcome.error();
LOG_STORAGE_WARNING_ << "ERROR: DeleteBucket: " << err.Code() << ": " << err.Message();
return Status(SERVER_UNEXPECTED_ERROR, err.Message());
}
LOG_STORAGE_DEBUG_ << "DeleteBucket '" << oss_bucket_ << "' successfully!";
return Status::OK();
}
Status
OSSClientWrapper::PutObjectFile(const std::string& object_name, const std::string& file_path) {
struct stat buffer;
if (stat(file_path.c_str(), &buffer) != 0) {
std::string str = "File '" + file_path + "' not exist!";
LOG_STORAGE_WARNING_ << "ERROR: " << str;
return Status(SERVER_UNEXPECTED_ERROR, str);
}
std::shared_ptr<std::iostream> content = std::make_shared<std::fstream>(file_path, std::ios::in | std::ios::binary);
AlibabaCloud::OSS::PutObjectRequest request(oss_bucket_, normalize_object_name(object_name), content);
const auto outcome = client_ptr_->PutObject(request);
if (!outcome.isSuccess()) {
const auto& err = outcome.error();
LOG_STORAGE_WARNING_ << "ERROR: PutObject: " << object_name << "," << err.Code() << ": " << err.Message();
return Status(SERVER_UNEXPECTED_ERROR, err.Message());
}
LOG_STORAGE_DEBUG_ << "PutObjectFile '" << object_name << "' successfully!";
return Status::OK();
}
Status
OSSClientWrapper::PutObjectStr(const std::string& object_name, const std::string& content) {
std::shared_ptr<std::iostream> content_stream = std::make_shared<std::stringstream>();
*content_stream << content;
AlibabaCloud::OSS::PutObjectRequest request(oss_bucket_, normalize_object_name(object_name), content_stream);
const auto outcome = client_ptr_->PutObject(request);
if (!outcome.isSuccess()) {
const auto& err = outcome.error();
LOG_STORAGE_WARNING_ << "ERROR: PutObjectStr: " << err.Code() << ": " << err.Message();
return Status(SERVER_UNEXPECTED_ERROR, err.Message());
}
LOG_STORAGE_DEBUG_ << "PutObjectStr '" << object_name << "' successfully!";
return Status::OK();
}
Status
OSSClientWrapper::GetObjectFile(const std::string& object_name, const std::string& file_path) {
AlibabaCloud::OSS::GetObjectRequest request(oss_bucket_, normalize_object_name(object_name));
request.setResponseStreamFactory([=]() {
return std::make_shared<std::fstream>(
file_path, std::ios_base::out | std::ios_base::in | std::ios_base::trunc | std::ios_base::binary);
});
const auto outcome = client_ptr_->GetObject(request);
if (!outcome.isSuccess()) {
const auto& err = outcome.error();
LOG_STORAGE_WARNING_ << "ERROR: GetObjectFile: " << object_name << ", " << err.Code() << ": " << err.Message();
return Status(SERVER_UNEXPECTED_ERROR, err.Message());
}
LOG_STORAGE_DEBUG_ << "GetObjectFile '" << object_name << "' successfully!";
return Status::OK();
}
Status
OSSClientWrapper::GetObjectStr(const std::string& object_name, std::string& content) {
AlibabaCloud::OSS::GetObjectRequest request(oss_bucket_, normalize_object_name(object_name));
const auto outcome = client_ptr_->GetObject(request);
if (!outcome.isSuccess()) {
const auto& err = outcome.error();
LOG_STORAGE_WARNING_ << "ERROR: GetObjectStr: " << err.Code() << ": " << err.Message();
return Status(SERVER_UNEXPECTED_ERROR, err.Message());
}
auto content_length = outcome.result().Metadata().ContentLength();
LOG_STORAGE_DEBUG_ << "GetObjectStr"
<< " success, Content-Length:" << content_length << std::endl;
auto& stream = outcome.result().Content();
content.resize(content_length);
stream->read(content.data(), content_length);
return Status::OK();
}
Status
OSSClientWrapper::ListObjects(std::vector<std::string>& object_list, const std::string& prefix) {
AlibabaCloud::OSS::ListObjectsRequest request(oss_bucket_);
request.setPrefix(normalize_object_name(prefix) + '/');
request.setDelimiter("/");
request.setMaxKeys(1000);
std::string next_marker = "";
bool is_truncated = false;
do {
request.setMarker(next_marker);
const auto outcome = client_ptr_->ListObjects(request);
if (!outcome.isSuccess()) {
const auto& err = outcome.error();
LOG_STORAGE_WARNING_ << "ERROR: ListObjects: " << err.Code() << ": " << err.Message();
return Status(SERVER_UNEXPECTED_ERROR, err.Message());
}
for (const auto& object : outcome.result().ObjectSummarys()) {
object_list.emplace_back(object.Key());
}
next_marker = outcome.result().NextMarker();
is_truncated = outcome.result().IsTruncated();
} while (is_truncated);
if (prefix.empty()) {
LOG_STORAGE_DEBUG_ << "ListObjects '" << oss_bucket_ << "' successfully!";
} else {
LOG_STORAGE_DEBUG_ << "ListObjects '" << oss_bucket_ << ":" << prefix << "' successfully!";
}
for (const auto& path : object_list) {
LOG_STORAGE_DEBUG_ << " object: '" << path;
}
return Status::OK();
}
Status
OSSClientWrapper::DeleteObject(const std::string& object_name) {
AlibabaCloud::OSS::DeleteObjectRequest request(oss_bucket_, normalize_object_name(object_name));
const auto outcome = client_ptr_->DeleteObject(request);
if (!outcome.isSuccess()) {
const auto& err = outcome.error();
LOG_STORAGE_WARNING_ << "ERROR: DeleteObject: " << err.Code() << ": " << err.Message();
return Status(SERVER_UNEXPECTED_ERROR, err.Message());
}
LOG_STORAGE_DEBUG_ << "DeleteObject '" << object_name << "' successfully!";
return Status::OK();
}
Status
OSSClientWrapper::DeleteObjects(const std::string& prefix) {
std::vector<std::string> object_list;
ListObjects(object_list, prefix);
for (const auto& object : object_list) {
auto status = DeleteObject(object);
if (!status.ok()) {
return status;
}
}
return Status::OK();
}
std::string
OSSClientWrapper::normalize_object_name(const std::string& object_key) {
std::string object_name = object_key;
if (object_name.empty()) {
return object_name;
}
if (object_name[0] == '/') {
object_name = object_name.substr(1);
}
if (object_name[object_name.size() - 1] == '/') {
object_name = object_name.substr(0, object_name.size() - 1);
}
return object_name;
}
} // namespace storage
} // namespace milvus

83
core/src/storage/oss/OSSClientWrapper.h Normal file
View File

@ -0,0 +1,83 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <alibabacloud/oss/OssClient.h>
#include <memory>
#include <string>
#include <vector>
#include "utils/Status.h"
namespace milvus {
namespace storage {
class OSSClientWrapper {
public:
static OSSClientWrapper&
GetInstance() {
static OSSClientWrapper wrapper;
return wrapper;
}
OSSClientWrapper() {
StartService();
}
~OSSClientWrapper() {
StopService();
}
Status
StartService();
void
StopService();
Status
CreateBucket();
Status
DeleteBucket();
Status
PutObjectFile(const std::string& object_key, const std::string& file_path);
Status
PutObjectStr(const std::string& object_key, const std::string& content);
Status
GetObjectFile(const std::string& object_key, const std::string& file_path);
Status
GetObjectStr(const std::string& object_key, std::string& content);
Status
ListObjects(std::vector<std::string>& object_list, const std::string& prefix = "");
Status
DeleteObject(const std::string& object_key);
Status
DeleteObjects(const std::string& prefix);
private:
std::string
normalize_object_name(const std::string& object_key);
std::shared_ptr<AlibabaCloud::OSS::OssClient> client_ptr_;
std::string oss_endpoint_;
std::string oss_access_key_;
std::string oss_secret_key_;
std::string oss_bucket_;
};
} // namespace storage
} // namespace milvus

53
core/src/storage/oss/OSSIOReader.cpp Normal file
View File

@ -0,0 +1,53 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "storage/oss/OSSIOReader.h"
#include <cstring>
#include "OSSClientWrapper.h"
namespace milvus {
namespace storage {
bool
OSSIOReader::open(const std::string& name) {
name_ = name;
pos_ = 0;
return (OSSClientWrapper::GetInstance().GetObjectStr(name_, buffer_).ok());
}
void
OSSIOReader::read(void* ptr, int64_t size) {
memcpy(ptr, buffer_.data() + pos_, size);
pos_ += size;
}
void
OSSIOReader::seekg(int64_t pos) {
pos_ = pos;
}
int64_t
OSSIOReader::length() {
return buffer_.length();
}
void
OSSIOReader::close() {
}
} // namespace storage
} // namespace milvus

65
core/src/storage/oss/OSSIOReader.h Normal file
View File

@ -0,0 +1,65 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include <string>
#include "storage/IOReader.h"
namespace milvus {
namespace storage {
class OSSIOReader : public IOReader {
public:
OSSIOReader() = default;
~OSSIOReader() = default;
// No copy and move
OSSIOReader(const OSSIOReader&) = delete;
OSSIOReader(OSSIOReader&&) = delete;
OSSIOReader&
operator=(const OSSIOReader&) = delete;
OSSIOReader&
operator=(OSSIOReader&&) = delete;
bool
open(const std::string& name) override;
void
read(void* ptr, int64_t size) override;
void
seekg(int64_t pos) override;
int64_t
length() override;
void
close() override;
public:
std::string name_;
std::string buffer_;
int64_t pos_;
};
using OSSIOReaderPtr = std::shared_ptr<OSSIOReader>;
} // namespace storage
} // namespace milvus

49
core/src/storage/oss/OSSIOWriter.cpp Normal file
View File

@ -0,0 +1,49 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "storage/oss/OSSIOWriter.h"
#include "OSSClientWrapper.h"
namespace milvus {
namespace storage {
bool
OSSIOWriter::open(const std::string& name) {
name_ = name;
len_ = 0;
buffer_ = "";
return true;
}
void
OSSIOWriter::write(void* ptr, int64_t size) {
buffer_ += std::string(reinterpret_cast<char*>(ptr), size);
len_ += size;
}
int64_t
OSSIOWriter::length() {
return len_;
}
void
OSSIOWriter::close() {
OSSClientWrapper::GetInstance().PutObjectStr(name_, buffer_);
}
} // namespace storage
} // namespace milvus

61
core/src/storage/oss/OSSIOWriter.h Normal file
View File

@ -0,0 +1,61 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include <string>
#include "storage/IOWriter.h"
namespace milvus {
namespace storage {
class OSSIOWriter : public IOWriter {
public:
OSSIOWriter() = default;
~OSSIOWriter() = default;
OSSIOWriter(const OSSIOWriter&) = delete;
OSSIOWriter(OSSIOWriter&&) = delete;
OSSIOWriter&
operator=(const OSSIOWriter&) = delete;
OSSIOWriter&
operator=(OSSIOWriter&&) = delete;
bool
open(const std::string& name) override;
void
write(void* ptr, int64_t size) override;
int64_t
length() override;
void
close() override;
public:
std::string name_;
int64_t len_;
std::string buffer_;
};
using OSSIOWriterPtr = std::shared_ptr<OSSIOWriter>;
} // namespace storage
} // namespace milvus

62
core/src/storage/oss/OSSOperation.cpp Normal file
View File

@ -0,0 +1,62 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "storage/oss/OSSOperation.h"
#include "OSSClientWrapper.h"
namespace milvus {
namespace storage {
OSSOperation::OSSOperation(const std::string& dir_path) : dir_path_(dir_path), local_operation_(dir_path) {
CreateDirectory();
}
void
OSSOperation::CreateDirectory() {
local_operation_.CreateDirectory();
}
const std::string&
OSSOperation::GetDirectory() const {
return dir_path_;
}
void
OSSOperation::ListDirectory(std::vector<std::string>& file_paths) {
OSSClientWrapper::GetInstance().ListObjects(file_paths, dir_path_);
}
bool
OSSOperation::DeleteFile(const std::string& file_path) {
(void)local_operation_.DeleteFile(file_path);
return OSSClientWrapper::GetInstance().DeleteObject(file_path).ok();
}
bool
OSSOperation::CacheGet(const std::string& file_path) {
return OSSClientWrapper::GetInstance().GetObjectFile(file_path, file_path).ok();
}
bool
OSSOperation::CachePut(const std::string& file_path) {
// TODO: try introducing LRU
return OSSClientWrapper::GetInstance().PutObjectFile(file_path, file_path).ok();
}
} // namespace storage
} // namespace milvus

58
core/src/storage/oss/OSSOperation.h Normal file
View File

@ -0,0 +1,58 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include <string>
#include <vector>
#include "storage/Operation.h"
#include "storage/disk/DiskOperation.h"
namespace milvus {
namespace storage {
class OSSOperation : public Operation {
public:
explicit OSSOperation(const std::string& dir_path);
void
CreateDirectory() final;
const std::string&
GetDirectory() const final;
void
ListDirectory(std::vector<std::string>& file_paths) final;
bool
DeleteFile(const std::string& file_path) final;
bool
CacheGet(const std::string& file_path) final;
bool
CachePut(const std::string& file_path) final;
private:
const std::string dir_path_;
DiskOperation local_operation_;
};
} // namespace storage
} // namespace milvus

8
core/src/storage/s3/S3ClientWrapper.cpp
View File

@ -38,7 +38,9 @@ Status
S3ClientWrapper::StartService() {
server::Config& config = server::Config::GetInstance();
bool s3_enable = false;
auto s3_use_https = false;
config.GetStorageConfigS3Enable(s3_enable);
config.GetStorageConfigS3UseHttps(s3_use_https);
fiu_do_on("S3ClientWrapper.StartService.s3_disable", s3_enable = false);
if (!s3_enable) {
LOG_STORAGE_INFO_ << "S3 not enabled!";
@ -56,7 +58,11 @@ S3ClientWrapper::StartService() {
Aws::Client::ClientConfiguration cfg;
cfg.endpointOverride = s3_address_ + ":" + s3_port_;
cfg.scheme = Aws::Http::Scheme::HTTP;
if (s3_use_https) {
cfg.scheme = Aws::Http::Scheme::HTTPS;
} else {
cfg.scheme = Aws::Http::Scheme::HTTP;
}
cfg.verifySSL = false;
if (!s3_region_.empty()) {
cfg.region = s3_region_.c_str();

1
core/thirdparty/versions.txt vendored
View File

@ -13,5 +13,6 @@ OPENTRACING_VERSION=v1.5.1
FIU_VERSION=1.00
OATPP_VERSION=1.0.1
AWS_VERSION=1.7.250
OSS_VERSION=1.9.0
# vim: set filetype=sh:

15
shards/all_in_one/ro_server.yml
View File

@ -66,6 +66,8 @@ network:
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_enabled | If using s3 storage backend. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_use_https | If using s3 using https scheme. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_address | The s3 server address, support domain/hostname/ipaddress | String | 127.0.0.1 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_port | The s3 server port. | Integer | 80 |
@ -83,6 +85,19 @@ network:
# | like ceph-radosgw, minio, when deploy on aws s3 service, | | |
# | this should be set correctly, e.g "ap-east-1", "us-east-1" | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_enabled | If using oss storage backend. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_endpoint | The oss endpoint. | String | see description |
# | default: https://oss-cn-hangzhou.aliyuncs.com | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_access_key | The access key for accessing oss service. | String | oss_access_key |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_secret_key | The secrey key for accessing oss service. | String | oss_secret_key |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_bucket | The oss bucket name for store milvus's data. | String | oss_bucket |
# | Note: please using differnet bucket for different milvus | | |
# | cluster. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
storage:
path: /var/lib/milvus
auto_flush_interval: 1

15
shards/all_in_one/wr_server.yml
View File

@ -66,6 +66,8 @@ network:
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_enabled | If using s3 storage backend. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_use_https | If using s3 using https scheme. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_address | The s3 server address, support domain/hostname/ipaddress | String | 127.0.0.1 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_port | The s3 server port. | Integer | 80 |
@ -83,6 +85,19 @@ network:
# | like ceph-radosgw, minio, when deploy on aws s3 service, | | |
# | this should be set correctly, e.g "ap-east-1", "us-east-1" | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_enabled | If using oss storage backend. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_endpoint | The oss endpoint. | String | see description |
# | default: https://oss-cn-hangzhou.aliyuncs.com | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_access_key | The access key for accessing oss service. | String | oss_access_key |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_secret_key | The secrey key for accessing oss service. | String | oss_secret_key |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_bucket | The oss bucket name for store milvus's data. | String | oss_bucket |
# | Note: please using differnet bucket for different milvus | | |
# | cluster. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
storage:
path: /var/lib/milvus
auto_flush_interval: 1

15
shards/all_in_one_with_mysql/ro_server.yml
View File

@ -66,6 +66,8 @@ network:
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_enabled | If using s3 storage backend. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_use_https | If using s3 using https scheme. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_address | The s3 server address, support domain/hostname/ipaddress | String | 127.0.0.1 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_port | The s3 server port. | Integer | 80 |
@ -83,6 +85,19 @@ network:
# | like ceph-radosgw, minio, when deploy on aws s3 service, | | |
# | this should be set correctly, e.g "ap-east-1", "us-east-1" | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_enabled | If using oss storage backend. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_endpoint | The oss endpoint. | String | see description |
# | default: https://oss-cn-hangzhou.aliyuncs.com | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_access_key | The access key for accessing oss service. | String | oss_access_key |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_secret_key | The secrey key for accessing oss service. | String | oss_secret_key |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_bucket | The oss bucket name for store milvus's data. | String | oss_bucket |
# | Note: please using differnet bucket for different milvus | | |
# | cluster. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
storage:
path: /var/lib/milvus
auto_flush_interval: 1

15
shards/all_in_one_with_mysql/wr_server.yml
View File

@ -70,6 +70,8 @@ network:
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_enabled | If using s3 storage backend. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_use_https | If using s3 using https scheme. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_address | The s3 server address, support domain/hostname/ipaddress | String | 127.0.0.1 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# s3_port | The s3 server port. | Integer | 80 |
@ -87,6 +89,19 @@ network:
# | like ceph-radosgw, minio, when deploy on aws s3 service, | | |
# | this should be set correctly, e.g "ap-east-1", "us-east-1" | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_enabled | If using oss storage backend. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_endpoint | The oss endpoint. | String | see description |
# | default: https://oss-cn-hangzhou.aliyuncs.com | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_access_key | The access key for accessing oss service. | String | oss_access_key |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_secret_key | The secrey key for accessing oss service. | String | oss_secret_key |
#----------------------+------------------------------------------------------------+------------+-----------------+
# oss_bucket | The oss bucket name for store milvus's data. | String | oss_bucket |
# | Note: please using differnet bucket for different milvus | | |
# | cluster. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
storage:
path: /var/lib/milvus
auto_flush_interval: 1