mirror of https://github.com/milvus-io/milvus.git
feat: Support tencent cloud object storage for milvus (#30163)
issue: #30162 Signed-off-by: Cai Zhang <cai.zhang@zilliz.com>pull/30211/head
cai.zhang
GitHub
parent
6a73860815
commit
6cf2f09b60
1
go.mod
1
go.mod
|
|
@ -191,6 +191,7 @@ require (
|
|||
github.com/streamnative/pulsarctl v0.5.0 // indirect
|
||||
github.com/stretchr/objx v0.5.0 // indirect
|
||||
github.com/subosito/gotenv v1.2.0 // indirect
|
||||
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.841 // indirect
|
||||
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
|
|
|
|||
8
go.sum
8
go.sum
|
|
@ -287,6 +287,7 @@ github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2C
|
|||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
|
||||
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
|
||||
github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
|
|
@ -511,6 +512,7 @@ github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYb
|
|||
github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE=
|
||||
github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro=
|
||||
github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8=
|
||||
github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d/go.mod h1:JJNrCn9otv/2QP4D7SMJBgaleKpOf66PnW6F5WGNRIc=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
|
|
@ -842,6 +844,12 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
|
|||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
|
||||
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go v1.0.800/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go v1.0.841/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go v3.0.233+incompatible h1:q+D/Y9jla3afgsIihtyhwyl0c2W+eRWNM9ohVwPiiPw=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go v3.0.233+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.841 h1:SJJR4tLnr0V17uEVS+arAmR1yl8n6dObBZs77SAmXZE=
|
||||
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.841/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0=
|
||||
github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
|
||||
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4=
|
||||
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM=
|
||||
|
|
|
|||
|
|
@ -56,7 +56,9 @@ set(STORAGE_FILES
|
|||
LocalChunkManager.cpp
|
||||
DiskFileManagerImpl.cpp
|
||||
ThreadPools.cpp
|
||||
ChunkCache.cpp)
|
||||
ChunkCache.cpp
|
||||
TencentCloudCredentialsProvider.cpp
|
||||
TencentCloudSTSClient.cpp)
|
||||
|
||||
add_library(milvus_storage SHARED ${STORAGE_FILES})
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,9 @@
|
|||
|
||||
#include "storage/MinioChunkManager.h"
|
||||
#include "storage/AliyunSTSClient.h"
|
||||
#include "storage/TencentCloudSTSClient.h"
|
||||
#include "storage/AliyunCredentialsProvider.h"
|
||||
#include "storage/TencentCloudCredentialsProvider.h"
|
||||
#include "common/Consts.h"
|
||||
#include "common/EasyAssert.h"
|
||||
#include "log/Log.h"
|
||||
|
|
@ -188,4 +190,47 @@ AliyunChunkManager::AliyunChunkManager(const StorageConfig& storage_config) {
|
|||
storage_config.useSSL);
|
||||
}
|
||||
|
||||
TencentCloudChunkManager::TencentCloudChunkManager(
|
||||
const StorageConfig& storage_config) {
|
||||
default_bucket_name_ = storage_config.bucket_name;
|
||||
remote_root_path_ = storage_config.root_path;
|
||||
|
||||
InitSDKAPIDefault(storage_config.log_level);
|
||||
|
||||
Aws::Client::ClientConfiguration config = generateConfig(storage_config);
|
||||
|
||||
StorageConfig mutable_config = storage_config;
|
||||
mutable_config.useVirtualHost = true;
|
||||
if (storage_config.useIAM) {
|
||||
auto tencent_cloud_provider = Aws::MakeShared<
|
||||
Aws::Auth::TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider>(
|
||||
"TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider");
|
||||
auto tencent_cloud_credentials =
|
||||
tencent_cloud_provider->GetAWSCredentials();
|
||||
AssertInfo(!tencent_cloud_credentials.GetAWSAccessKeyId().empty(),
|
||||
"if use iam, access key id should not be empty");
|
||||
AssertInfo(!tencent_cloud_credentials.GetAWSSecretKey().empty(),
|
||||
"if use iam, secret key should not be empty");
|
||||
AssertInfo(!tencent_cloud_credentials.GetSessionToken().empty(),
|
||||
"if use iam, token should not be empty");
|
||||
client_ = std::make_shared<Aws::S3::S3Client>(
|
||||
tencent_cloud_provider,
|
||||
config,
|
||||
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
|
||||
mutable_config.useVirtualHost);
|
||||
} else {
|
||||
BuildAccessKeyClient(mutable_config, config);
|
||||
}
|
||||
|
||||
PreCheck(storage_config);
|
||||
|
||||
LOG_INFO(
|
||||
"init TencentCloudChunkManager with "
|
||||
"parameter[endpoint={}][bucket_name={}][root_path={}][use_secure={}]",
|
||||
storage_config.address,
|
||||
storage_config.bucket_name,
|
||||
storage_config.root_path,
|
||||
storage_config.useSSL);
|
||||
}
|
||||
|
||||
} // namespace milvus::storage
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@
|
|||
|
||||
#include "storage/AliyunSTSClient.h"
|
||||
#include "storage/AliyunCredentialsProvider.h"
|
||||
#include "storage/TencentCloudSTSClient.h"
|
||||
#include "storage/TencentCloudCredentialsProvider.h"
|
||||
#include "storage/prometheus_client.h"
|
||||
#include "common/EasyAssert.h"
|
||||
#include "log/Log.h"
|
||||
|
|
|
|||
|
|
@ -45,7 +45,11 @@
|
|||
|
||||
namespace milvus::storage {
|
||||
|
||||
enum class RemoteStorageType { S3 = 0, GOOGLE_CLOUD = 1, ALIYUN_CLOUD = 2 };
|
||||
enum class RemoteStorageType {
|
||||
S3 = 0,
|
||||
GOOGLE_CLOUD = 1,
|
||||
ALIYUN_CLOUD = 2,
|
||||
};
|
||||
|
||||
template <typename... Args>
|
||||
|
||||
|
|
@ -258,6 +262,15 @@ class AliyunChunkManager : public MinioChunkManager {
|
|||
}
|
||||
};
|
||||
|
||||
class TencentCloudChunkManager : public MinioChunkManager {
|
||||
public:
|
||||
explicit TencentCloudChunkManager(const StorageConfig& storage_config);
|
||||
virtual std::string
|
||||
GetName() const {
|
||||
return "TencentCloudChunkManager";
|
||||
}
|
||||
};
|
||||
|
||||
using MinioChunkManagerPtr = std::unique_ptr<MinioChunkManager>;
|
||||
|
||||
static const char* GOOGLE_CLIENT_FACTORY_ALLOCATION_TAG =
|
||||
|
|
|
|||
|
|
@ -0,0 +1,185 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License
|
||||
|
||||
/**
|
||||
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
* SPDX-License-Identifier: Apache-2.0.
|
||||
*/
|
||||
|
||||
#include <fstream>
|
||||
#include <aws/core/config/AWSProfileConfigLoader.h>
|
||||
#include <aws/core/platform/Environment.h>
|
||||
#include <aws/core/utils/logging/LogMacros.h>
|
||||
#include <aws/core/client/SpecifiedRetryableErrorsRetryStrategy.h>
|
||||
#include <aws/core/utils/UUID.h>
|
||||
#include "TencentCloudCredentialsProvider.h"
|
||||
|
||||
static const char STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG[] =
|
||||
"TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider";
|
||||
static const int STS_CREDENTIAL_PROVIDER_EXPIRATION_GRACE_PERIOD =
|
||||
7200; // tencent cloud support 7200s.
|
||||
|
||||
namespace Aws {
|
||||
namespace Auth {
|
||||
TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider::
|
||||
TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider()
|
||||
: m_initialized(false) {
|
||||
m_region = Aws::Environment::GetEnv("TKE_REGION");
|
||||
m_roleArn = Aws::Environment::GetEnv("TKE_ROLE_ARN");
|
||||
m_tokenFile = Aws::Environment::GetEnv("TKE_WEB_IDENTITY_TOKEN_FILE");
|
||||
m_providerId = Aws::Environment::GetEnv("TKE_PROVIDER_ID");
|
||||
auto currentTimePoint = std::chrono::high_resolution_clock::now();
|
||||
auto nanoseconds = std::chrono::time_point_cast<std::chrono::nanoseconds>(
|
||||
currentTimePoint);
|
||||
auto timestamp = nanoseconds.time_since_epoch().count();
|
||||
m_sessionName = "tencentcloud-cpp-sdk-" + std::to_string(timestamp / 1000);
|
||||
|
||||
if (m_roleArn.empty() || m_tokenFile.empty() || m_region.empty()) {
|
||||
auto profile = Aws::Config::GetCachedConfigProfile(
|
||||
Aws::Auth::GetConfigProfileName());
|
||||
m_roleArn = profile.GetRoleArn();
|
||||
m_tokenFile = profile.GetValue("web_identity_token_file");
|
||||
m_sessionName = profile.GetValue("role_session_name");
|
||||
}
|
||||
|
||||
if (m_tokenFile.empty()) {
|
||||
AWS_LOGSTREAM_WARN(STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG,
|
||||
"Token file must be specified to use STS AssumeRole "
|
||||
"web identity creds provider.");
|
||||
return; // No need to do further constructing
|
||||
} else {
|
||||
AWS_LOGSTREAM_DEBUG(STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG,
|
||||
"Resolved token_file from profile_config or "
|
||||
"environment variable to be "
|
||||
<< m_tokenFile);
|
||||
}
|
||||
|
||||
if (m_roleArn.empty()) {
|
||||
AWS_LOGSTREAM_WARN(STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG,
|
||||
"RoleArn must be specified to use STS AssumeRole "
|
||||
"web identity creds provider.");
|
||||
return; // No need to do further constructing
|
||||
} else {
|
||||
AWS_LOGSTREAM_DEBUG(STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG,
|
||||
"Resolved role_arn from profile_config or "
|
||||
"environment variable to be "
|
||||
<< m_roleArn);
|
||||
}
|
||||
|
||||
if (m_region.empty()) {
|
||||
AWS_LOGSTREAM_WARN(STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG,
|
||||
"Region must be specified to use STS AssumeRole "
|
||||
"web identity creds provider.");
|
||||
return; // No need to do further constructing
|
||||
} else {
|
||||
AWS_LOGSTREAM_DEBUG(STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG,
|
||||
"Resolved region from profile_config or "
|
||||
"environment variable to be "
|
||||
<< m_region);
|
||||
}
|
||||
|
||||
if (m_sessionName.empty()) {
|
||||
m_sessionName = Aws::Utils::UUID::RandomUUID();
|
||||
} else {
|
||||
AWS_LOGSTREAM_DEBUG(STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG,
|
||||
"Resolved session_name from profile_config or "
|
||||
"environment variable to be "
|
||||
<< m_sessionName);
|
||||
}
|
||||
|
||||
Aws::Client::ClientConfiguration config;
|
||||
config.scheme = Aws::Http::Scheme::HTTPS;
|
||||
config.region = m_region;
|
||||
|
||||
Aws::Vector<Aws::String> retryableErrors;
|
||||
retryableErrors.push_back("IDPCommunicationError");
|
||||
retryableErrors.push_back("InvalidIdentityToken");
|
||||
|
||||
config.retryStrategy =
|
||||
Aws::MakeShared<Aws::Client::SpecifiedRetryableErrorsRetryStrategy>(
|
||||
STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG,
|
||||
retryableErrors,
|
||||
3 /*maxRetries*/);
|
||||
|
||||
m_client = Aws::MakeUnique<Aws::Internal::TencentCloudSTSCredentialsClient>(
|
||||
STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG, config);
|
||||
m_initialized = true;
|
||||
AWS_LOGSTREAM_INFO(
|
||||
STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG,
|
||||
"Creating STS AssumeRole with web identity creds provider.");
|
||||
}
|
||||
|
||||
Aws::Auth::AWSCredentials
|
||||
TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider::GetAWSCredentials() {
|
||||
// A valid client means required information like role arn and token file were constructed correctly.
|
||||
// We can use this provider to load creds, otherwise, we can just return empty creds.
|
||||
if (!m_initialized) {
|
||||
return Aws::Auth::AWSCredentials();
|
||||
}
|
||||
RefreshIfExpired();
|
||||
Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock);
|
||||
return m_credentials;
|
||||
}
|
||||
|
||||
void
|
||||
TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider::Reload() {
|
||||
AWS_LOGSTREAM_INFO(
|
||||
STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG,
|
||||
"Credentials have expired, attempting to renew from STS.");
|
||||
|
||||
Aws::IFStream tokenFile(m_tokenFile.c_str());
|
||||
if (tokenFile) {
|
||||
Aws::String token((std::istreambuf_iterator<char>(tokenFile)),
|
||||
std::istreambuf_iterator<char>());
|
||||
if (!token.empty() && token.back() == '\n') {
|
||||
token.pop_back();
|
||||
}
|
||||
m_token = token;
|
||||
} else {
|
||||
AWS_LOGSTREAM_ERROR(STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG,
|
||||
"Can't open token file: " << m_tokenFile);
|
||||
return;
|
||||
}
|
||||
Aws::Internal::TencentCloudSTSCredentialsClient::
|
||||
STSAssumeRoleWithWebIdentityRequest request{
|
||||
m_region, m_providerId, m_token, m_roleArn, m_sessionName};
|
||||
|
||||
auto result = m_client->GetAssumeRoleWithWebIdentityCredentials(request);
|
||||
AWS_LOGSTREAM_TRACE(
|
||||
STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG,
|
||||
"Successfully retrieved credentials with AWS_ACCESS_KEY: "
|
||||
<< result.creds.GetAWSAccessKeyId());
|
||||
m_credentials = result.creds;
|
||||
}
|
||||
|
||||
bool
|
||||
TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider::ExpiresSoon() const {
|
||||
return (
|
||||
(m_credentials.GetExpiration() - Aws::Utils::DateTime::Now()).count() <
|
||||
STS_CREDENTIAL_PROVIDER_EXPIRATION_GRACE_PERIOD);
|
||||
}
|
||||
|
||||
void
|
||||
TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider::RefreshIfExpired() {
|
||||
Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock);
|
||||
if (!m_credentials.IsEmpty() && !ExpiresSoon()) {
|
||||
return;
|
||||
}
|
||||
|
||||
guard.UpgradeToWriterLock();
|
||||
if (!m_credentials.IsExpiredOrEmpty() && !ExpiresSoon()) {
|
||||
return;
|
||||
}
|
||||
|
||||
Reload();
|
||||
}
|
||||
} // namespace Auth
|
||||
}; // namespace Aws
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License
|
||||
|
||||
/**
|
||||
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
* SPDX-License-Identifier: Apache-2.0.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <aws/core/Core_EXPORTS.h>
|
||||
#include <aws/core/utils/DateTime.h>
|
||||
#include <aws/core/utils/memory/stl/AWSString.h>
|
||||
#include <aws/core/internal/AWSHttpResourceClient.h>
|
||||
#include <aws/core/auth/AWSCredentialsProvider.h>
|
||||
#include "TencentCloudSTSClient.h"
|
||||
|
||||
namespace Aws {
|
||||
namespace Auth {
|
||||
/**
|
||||
* To support retrieving credentials of STS AssumeRole with web identity.
|
||||
* Note that STS accepts request with protocol of queryxml. Calling GetAWSCredentials() will trigger (if expired)
|
||||
* a query request using AWSHttpResourceClient under the hood.
|
||||
*/
|
||||
class AWS_CORE_API TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider
|
||||
: public AWSCredentialsProvider {
|
||||
public:
|
||||
TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider();
|
||||
|
||||
/**
|
||||
* Retrieves the credentials if found, otherwise returns empty credential set.
|
||||
*/
|
||||
AWSCredentials
|
||||
GetAWSCredentials() override;
|
||||
|
||||
protected:
|
||||
void
|
||||
Reload() override;
|
||||
|
||||
private:
|
||||
void
|
||||
RefreshIfExpired();
|
||||
Aws::String
|
||||
CalculateQueryString() const;
|
||||
|
||||
Aws::UniquePtr<Aws::Internal::TencentCloudSTSCredentialsClient> m_client;
|
||||
Aws::Auth::AWSCredentials m_credentials;
|
||||
Aws::String m_region;
|
||||
Aws::String m_roleArn;
|
||||
Aws::String m_tokenFile;
|
||||
Aws::String m_sessionName;
|
||||
Aws::String m_providerId;
|
||||
Aws::String m_token;
|
||||
bool m_initialized;
|
||||
bool
|
||||
ExpiresSoon() const;
|
||||
};
|
||||
} // namespace Auth
|
||||
} // namespace Aws
|
||||
|
|
@ -0,0 +1,150 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License
|
||||
|
||||
/**
|
||||
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
* SPDX-License-Identifier: Apache-2.0.
|
||||
*/
|
||||
|
||||
#include <mutex>
|
||||
#include <sstream>
|
||||
#include <aws/core/internal/AWSHttpResourceClient.h>
|
||||
#include <aws/core/client/DefaultRetryStrategy.h>
|
||||
#include <aws/core/http/HttpClient.h>
|
||||
#include <aws/core/http/HttpClientFactory.h>
|
||||
#include <aws/core/http/HttpResponse.h>
|
||||
#include <aws/core/utils/logging/LogMacros.h>
|
||||
#include <aws/core/utils/StringUtils.h>
|
||||
#include <aws/core/platform/Environment.h>
|
||||
#include <aws/core/client/AWSError.h>
|
||||
#include "TencentCloudSTSClient.h"
|
||||
|
||||
namespace Aws {
|
||||
namespace Http {
|
||||
class HttpClient;
|
||||
class HttpRequest;
|
||||
enum class HttpResponseCode;
|
||||
} // namespace Http
|
||||
|
||||
namespace Client {
|
||||
Aws::String
|
||||
ComputeUserAgentString();
|
||||
}
|
||||
|
||||
namespace Internal {
|
||||
|
||||
static const char STS_RESOURCE_CLIENT_LOG_TAG[] =
|
||||
"TencentCloudSTSResourceClient"; // [tencent cloud]
|
||||
|
||||
TencentCloudSTSCredentialsClient::TencentCloudSTSCredentialsClient(
|
||||
const Aws::Client::ClientConfiguration& clientConfiguration)
|
||||
: AWSHttpResourceClient(clientConfiguration, STS_RESOURCE_CLIENT_LOG_TAG) {
|
||||
SetErrorMarshaller(Aws::MakeUnique<Aws::Client::XmlErrorMarshaller>(
|
||||
STS_RESOURCE_CLIENT_LOG_TAG));
|
||||
|
||||
// [tencent cloud]
|
||||
m_endpoint = "https://sts.tencentcloudapi.com";
|
||||
|
||||
AWS_LOGSTREAM_INFO(
|
||||
STS_RESOURCE_CLIENT_LOG_TAG,
|
||||
"Creating STS ResourceClient with endpoint: " << m_endpoint);
|
||||
}
|
||||
|
||||
TencentCloudSTSCredentialsClient::STSAssumeRoleWithWebIdentityResult
|
||||
TencentCloudSTSCredentialsClient::GetAssumeRoleWithWebIdentityCredentials(
|
||||
const STSAssumeRoleWithWebIdentityRequest& request) {
|
||||
// Calculate query string
|
||||
Aws::StringStream ss;
|
||||
// curl -X POST "https://sts.tencentcloudapi.com"
|
||||
// -d "{\"ProviderId\": $ProviderId, \"WebIdentityToken\": $WebIdentityToken,\"RoleArn\":$RoleArn,\"RoleSessionName\":$RoleSessionName,\"DurationSeconds\":7200}"
|
||||
// -H "Authorization: SKIP"
|
||||
// -H "Content-Type: application/json; charset=utf-8"
|
||||
// -H "Host: sts.tencentcloudapi.com"
|
||||
// -H "X-TC-Action: AssumeRoleWithWebIdentity"
|
||||
// -H "X-TC-Timestamp: $timestamp"
|
||||
// -H "X-TC-Version: 2018-08-13"
|
||||
// -H "X-TC-Region: $region"
|
||||
// -H "X-TC-Token: $token"
|
||||
|
||||
ss << R"({"ProviderId": ")" << request.providerId
|
||||
<< R"(", "WebIdentityToken": ")" << request.webIdentityToken
|
||||
<< R"(", "RoleArn": ")" << request.roleArn
|
||||
<< R"(", "RoleSessionName": ")" << request.roleSessionName << R"("})";
|
||||
|
||||
std::shared_ptr<Aws::Http::HttpRequest> httpRequest(
|
||||
Aws::Http::CreateHttpRequest(
|
||||
m_endpoint,
|
||||
Aws::Http::HttpMethod::HTTP_POST,
|
||||
Aws::Utils::Stream::DefaultResponseStreamFactoryMethod));
|
||||
|
||||
httpRequest->SetUserAgent(Aws::Client::ComputeUserAgentString());
|
||||
httpRequest->SetHeaderValue("Authorization", "SKIP");
|
||||
httpRequest->SetHeaderValue("Host", "sts.tencentcloudapi.com");
|
||||
httpRequest->SetHeaderValue("X-TC-Action", "AssumeRoleWithWebIdentity");
|
||||
httpRequest->SetHeaderValue(
|
||||
"X-TC-Timestamp",
|
||||
std::to_string(Aws::Utils::DateTime::Now().Seconds()));
|
||||
httpRequest->SetHeaderValue("X-TC-Version", "2018-08-13");
|
||||
httpRequest->SetHeaderValue("X-TC-Region", request.region);
|
||||
httpRequest->SetHeaderValue("X-TC-Token", "");
|
||||
|
||||
std::shared_ptr<Aws::IOStream> body =
|
||||
Aws::MakeShared<Aws::StringStream>("STS_RESOURCE_CLIENT_LOG_TAG");
|
||||
*body << ss.str();
|
||||
|
||||
httpRequest->AddContentBody(body);
|
||||
body->seekg(0, body->end);
|
||||
auto streamSize = body->tellg();
|
||||
body->seekg(0, body->beg);
|
||||
Aws::StringStream contentLength;
|
||||
contentLength << streamSize;
|
||||
httpRequest->SetContentLength(contentLength.str());
|
||||
// httpRequest->SetContentType("application/x-www-form-urlencoded");
|
||||
httpRequest->SetContentType("application/json; charset=utf-8");
|
||||
|
||||
auto headers = httpRequest->GetHeaders();
|
||||
Aws::String credentialsStr =
|
||||
GetResourceWithAWSWebServiceResult(httpRequest).GetPayload();
|
||||
|
||||
// Parse credentials
|
||||
STSAssumeRoleWithWebIdentityResult result;
|
||||
if (credentialsStr.empty()) {
|
||||
AWS_LOGSTREAM_WARN(STS_RESOURCE_CLIENT_LOG_TAG,
|
||||
"Get an empty credential from sts");
|
||||
return result;
|
||||
}
|
||||
|
||||
auto json = Utils::Json::JsonView(credentialsStr);
|
||||
auto rootNode = json.GetObject("Response");
|
||||
if (rootNode.IsNull()) {
|
||||
AWS_LOGSTREAM_WARN(STS_RESOURCE_CLIENT_LOG_TAG,
|
||||
"Get Response from credential result failed");
|
||||
return result;
|
||||
}
|
||||
|
||||
auto credentialsNode = rootNode.GetObject("Credentials");
|
||||
if (credentialsNode.IsNull()) {
|
||||
AWS_LOGSTREAM_WARN(STS_RESOURCE_CLIENT_LOG_TAG,
|
||||
"Get Credentials from Response failed");
|
||||
return result;
|
||||
}
|
||||
result.creds.SetAWSAccessKeyId(credentialsNode.GetString("TmpSecretId"));
|
||||
result.creds.SetAWSSecretKey(credentialsNode.GetString("TmpSecretKey"));
|
||||
result.creds.SetSessionToken(credentialsNode.GetString("Token"));
|
||||
result.creds.SetExpiration(Aws::Utils::DateTime(
|
||||
Aws::Utils::StringUtils::Trim(rootNode.GetString("Expiration").c_str())
|
||||
.c_str(),
|
||||
Aws::Utils::DateFormat::ISO_8601));
|
||||
|
||||
return result;
|
||||
}
|
||||
} // namespace Internal
|
||||
} // namespace Aws
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License
|
||||
|
||||
/**
|
||||
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
* SPDX-License-Identifier: Apache-2.0.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <aws/core/Core_EXPORTS.h>
|
||||
#include <aws/core/utils/memory/AWSMemory.h>
|
||||
#include <aws/core/utils/memory/stl/AWSString.h>
|
||||
#include <aws/core/client/ClientConfiguration.h>
|
||||
#include <aws/core/client/AWSErrorMarshaller.h>
|
||||
#include <aws/core/auth/AWSCredentials.h>
|
||||
#include <aws/core/AmazonWebServiceResult.h>
|
||||
#include <aws/core/utils/DateTime.h>
|
||||
#include <aws/core/internal/AWSHttpResourceClient.h>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
|
||||
namespace Aws {
|
||||
namespace Http {
|
||||
class HttpClient;
|
||||
class HttpRequest;
|
||||
enum class HttpResponseCode;
|
||||
} // namespace Http
|
||||
|
||||
namespace Internal {
|
||||
/**
|
||||
* To support retrieving credentials from STS.
|
||||
* Note that STS accepts request with protocol of queryxml. Calling GetResource() will trigger
|
||||
* a query request using AWSHttpResourceClient under the hood.
|
||||
*/
|
||||
class AWS_CORE_API TencentCloudSTSCredentialsClient
|
||||
: public AWSHttpResourceClient {
|
||||
public:
|
||||
/**
|
||||
* Initializes the provider to retrieve credentials from STS when it expires.
|
||||
*/
|
||||
explicit TencentCloudSTSCredentialsClient(
|
||||
const Client::ClientConfiguration& clientConfiguration);
|
||||
|
||||
TencentCloudSTSCredentialsClient&
|
||||
operator=(TencentCloudSTSCredentialsClient& rhs) = delete;
|
||||
TencentCloudSTSCredentialsClient(
|
||||
const TencentCloudSTSCredentialsClient& rhs) = delete;
|
||||
TencentCloudSTSCredentialsClient&
|
||||
operator=(TencentCloudSTSCredentialsClient&& rhs) = delete;
|
||||
TencentCloudSTSCredentialsClient(
|
||||
const TencentCloudSTSCredentialsClient&& rhs) = delete;
|
||||
|
||||
// If you want to make an AssumeRoleWithWebIdentity call to sts. use these classes to pass data to and get info from
|
||||
// TencentCloudSTSCredentialsClient client. If you want to make an AssumeRole call to sts, define the request/result
|
||||
// members class/struct like this.
|
||||
struct STSAssumeRoleWithWebIdentityRequest {
|
||||
Aws::String region;
|
||||
Aws::String providerId;
|
||||
Aws::String webIdentityToken;
|
||||
Aws::String roleArn;
|
||||
Aws::String roleSessionName;
|
||||
};
|
||||
|
||||
struct STSAssumeRoleWithWebIdentityResult {
|
||||
Aws::Auth::AWSCredentials creds;
|
||||
};
|
||||
|
||||
STSAssumeRoleWithWebIdentityResult
|
||||
GetAssumeRoleWithWebIdentityCredentials(
|
||||
const STSAssumeRoleWithWebIdentityRequest& request);
|
||||
|
||||
private:
|
||||
Aws::String m_endpoint;
|
||||
};
|
||||
} // namespace Internal
|
||||
} // namespace Aws
|
||||
|
|
@ -54,13 +54,15 @@ enum class CloudProviderType : int8_t {
|
|||
GCP = 2,
|
||||
ALIYUN = 3,
|
||||
AZURE = 4,
|
||||
TENCENTCLOUD = 5,
|
||||
};
|
||||
|
||||
std::map<std::string, CloudProviderType> CloudProviderType_Map = {
|
||||
{"aws", CloudProviderType::AWS},
|
||||
{"gcp", CloudProviderType::GCP},
|
||||
{"aliyun", CloudProviderType::ALIYUN},
|
||||
{"azure", CloudProviderType::AZURE}};
|
||||
{"azure", CloudProviderType::AZURE},
|
||||
{"tencent", CloudProviderType::TENCENTCLOUD}};
|
||||
|
||||
std::map<std::string, int> ReadAheadPolicy_Map = {
|
||||
{"normal", MADV_NORMAL},
|
||||
|
|
@ -670,6 +672,10 @@ CreateChunkManager(const StorageConfig& storage_config) {
|
|||
case CloudProviderType::ALIYUN: {
|
||||
return std::make_shared<AliyunChunkManager>(storage_config);
|
||||
}
|
||||
case CloudProviderType::TENCENTCLOUD: {
|
||||
return std::make_shared<TencentCloudChunkManager>(
|
||||
storage_config);
|
||||
}
|
||||
#ifdef AZURE_BUILD_DIR
|
||||
case CloudProviderType::AZURE: {
|
||||
return std::make_shared<AzureChunkManager>(storage_config);
|
||||
|
|
|
|||
|
|
@ -84,7 +84,7 @@ if (DEFINED AZURE_BUILD_DIR)
|
|||
set(MILVUS_TEST_FILES
|
||||
${MILVUS_TEST_FILES}
|
||||
test_azure_chunk_manager.cpp
|
||||
#need update aws-sdk-cpp, see more from https://github.com/aws/aws-sdk-cpp/issues/2119
|
||||
#need update aws-sdk-cpp, see more from https://github.com/aws/aws-sdk-cpp/issues/2119
|
||||
#test_remote_chunk_manager.cpp
|
||||
)
|
||||
include_directories("${AZURE_BUILD_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/include")
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ import (
|
|||
|
||||
"github.com/milvus-io/milvus/internal/storage/aliyun"
|
||||
"github.com/milvus-io/milvus/internal/storage/gcp"
|
||||
"github.com/milvus-io/milvus/internal/storage/tencent"
|
||||
"github.com/milvus-io/milvus/pkg/log"
|
||||
"github.com/milvus-io/milvus/pkg/util/retry"
|
||||
)
|
||||
|
|
@ -62,6 +63,12 @@ func newMinioClient(ctx context.Context, c *config) (*minio.Client, error) {
|
|||
if !c.useIAM {
|
||||
creds = credentials.NewStaticV2(c.accessKeyID, c.secretAccessKeyID, "")
|
||||
}
|
||||
case CloudProviderTencent:
|
||||
newMinioFn = tencent.NewMinioClient
|
||||
if !c.useIAM {
|
||||
creds = credentials.NewStaticV4(c.accessKeyID, c.secretAccessKeyID, "")
|
||||
}
|
||||
|
||||
default: // aws, minio
|
||||
matchedDefault = true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,11 +38,11 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
CloudProviderGCP = "gcp"
|
||||
CloudProviderAWS = "aws"
|
||||
CloudProviderAliyun = "aliyun"
|
||||
|
||||
CloudProviderAzure = "azure"
|
||||
CloudProviderGCP = "gcp"
|
||||
CloudProviderAWS = "aws"
|
||||
CloudProviderAliyun = "aliyun"
|
||||
CloudProviderAzure = "azure"
|
||||
CloudProviderTencent = "tencent"
|
||||
)
|
||||
|
||||
type ObjectStorage interface {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,85 @@
|
|||
package tencent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/cockroachdb/errors"
|
||||
"github.com/minio/minio-go/v7"
|
||||
minioCred "github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
|
||||
)
|
||||
|
||||
// NewMinioClient returns a minio.Client which is compatible for tencent OSS
|
||||
func NewMinioClient(address string, opts *minio.Options) (*minio.Client, error) {
|
||||
if opts == nil {
|
||||
opts = &minio.Options{}
|
||||
}
|
||||
if opts.Creds == nil {
|
||||
credProvider, err := NewCredentialProvider()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to create credential provider")
|
||||
}
|
||||
opts.Creds = minioCred.New(credProvider)
|
||||
}
|
||||
if address == "" {
|
||||
address = fmt.Sprintf("cos.%s.myqcloud.com", opts.Region)
|
||||
opts.Secure = true
|
||||
}
|
||||
return minio.New(address, opts)
|
||||
}
|
||||
|
||||
// Credential is defined to mock tencent credential.Credentials
|
||||
//
|
||||
//go:generate mockery --name=Credential --with-expecter
|
||||
type Credential interface {
|
||||
common.CredentialIface
|
||||
}
|
||||
|
||||
// CredentialProvider implements "github.com/minio/minio-go/v7/pkg/credentials".Provider
|
||||
// also implements transport
|
||||
type CredentialProvider struct {
|
||||
// tencentCreds doesn't provide a way to get the expired time, so we use the cache to check if it's expired
|
||||
// when tencentCreds.GetSecretId is different from the cache, we know it's expired
|
||||
akCache string
|
||||
tencentCreds Credential
|
||||
}
|
||||
|
||||
func NewCredentialProvider() (minioCred.Provider, error) {
|
||||
provider, err := common.DefaultTkeOIDCRoleArnProvider()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to create tencent credential provider")
|
||||
}
|
||||
|
||||
cred, err := provider.GetCredential()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to get tencent credential")
|
||||
}
|
||||
return &CredentialProvider{tencentCreds: cred}, nil
|
||||
}
|
||||
|
||||
// Retrieve returns nil if it successfully retrieved the value.
|
||||
// Error is returned if the value were not obtainable, or empty.
|
||||
// according to the caller minioCred.Credentials.Get(),
|
||||
// it already has a lock, so we don't need to worry about concurrency
|
||||
func (c *CredentialProvider) Retrieve() (minioCred.Value, error) {
|
||||
ret := minioCred.Value{}
|
||||
ak := c.tencentCreds.GetSecretId()
|
||||
ret.AccessKeyID = ak
|
||||
c.akCache = ak
|
||||
|
||||
sk := c.tencentCreds.GetSecretKey()
|
||||
ret.SecretAccessKey = sk
|
||||
|
||||
securityToken := c.tencentCreds.GetToken()
|
||||
ret.SessionToken = securityToken
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// IsExpired returns if the credentials are no longer valid, and need
|
||||
// to be retrieved.
|
||||
// according to the caller minioCred.Credentials.IsExpired(),
|
||||
// it already has a lock, so we don't need to worry about concurrency
|
||||
func (c CredentialProvider) IsExpired() bool {
|
||||
ak := c.tencentCreds.GetSecretId()
|
||||
return ak != c.akCache
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
package tencent
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func Test_NewMinioClient(t *testing.T) {
|
||||
t.Run("ak sk ok", func(t *testing.T) {
|
||||
minioCli, err := NewMinioClient("xxx.cos.ap-beijing.myqcloud.com", &minio.Options{
|
||||
Creds: credentials.NewStaticV2("ak", "sk", ""),
|
||||
Secure: true,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "https", minioCli.EndpointURL().Scheme)
|
||||
})
|
||||
|
||||
t.Run("iam failed", func(t *testing.T) {
|
||||
_, err := NewMinioClient("", nil)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
Loading…
Reference in New Issue