mirror of https://github.com/milvus-io/milvus.git
Support aliyun oss as object storage with ak or IAM (#22376)
Signed-off-by: shaoyue.chen <shaoyue.chen@zilliz.com>pull/22557/head
shaoyue
GitHub
parent
9e17e2660e
commit
32581e6452
|
|
@ -75,11 +75,14 @@ minio:
|
|||
# For more information, refer to
|
||||
# aws: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html
|
||||
# gcp: https://cloud.google.com/storage/docs/access-control/iam
|
||||
# aliyun (ack): https://www.alibabacloud.com/help/en/container-service-for-kubernetes/latest/use-rrsa-to-enforce-access-control
|
||||
# aliyun (ecs): https://www.alibabacloud.com/help/en/elastic-compute-service/latest/attach-an-instance-ram-role
|
||||
useIAM: false
|
||||
# Cloud Provider of S3. Supports: "aws", "gcp".
|
||||
# Cloud Provider of S3. Supports: "aws", "gcp", "aliyun".
|
||||
# You can use "aws" for other cloud provider supports S3 API with signature v4, e.g.: minio
|
||||
# You can use "gcp" for other cloud provider supports S3 API with signature v2
|
||||
# When useIAM enabled, only "aws" & "gcp" is supported for now
|
||||
# You can use "aliyun" for other cloud provider uses virtual host style bucket
|
||||
# When useIAM enabled, only "aws", "gcp", "aliyun" is supported for now
|
||||
cloudProvider: aws
|
||||
# Custom endpoint for fetch IAM role credentials. when useIAM is true & cloudProvider is "aws".
|
||||
# Leave it empty if you want to use AWS default endpoint
|
||||
|
|
|
|||
10
go.mod
10
go.mod
|
|
@ -10,7 +10,7 @@ require (
|
|||
github.com/antonmedv/expr v1.8.9
|
||||
github.com/apache/arrow/go/v8 v8.0.0-20220322092137-778b1772fd20
|
||||
github.com/apache/pulsar-client-go v0.6.1-0.20210728062540-29414db801a7
|
||||
github.com/apache/thrift v0.15.0
|
||||
github.com/apache/thrift v0.15.0 // indirect
|
||||
github.com/benesch/cgosymbolizer v0.0.0-20190515212042-bec6fe6e597b
|
||||
github.com/bits-and-blooms/bloom/v3 v3.0.1
|
||||
github.com/casbin/casbin/v2 v2.44.2
|
||||
|
|
@ -172,21 +172,24 @@ require (
|
|||
google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29 // indirect
|
||||
gopkg.in/ini.v1 v1.62.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
sigs.k8s.io/yaml v1.2.0 // indirect
|
||||
)
|
||||
|
||||
require github.com/ianlancetaylor/cgosymbolizer v0.0.0-20221217025313-27d3c9f66b6a // indirect
|
||||
|
||||
require (
|
||||
github.com/aliyun/credentials-go v1.2.6
|
||||
github.com/cockroachdb/errors v1.9.1
|
||||
github.com/golang/mock v1.5.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/uber/jaeger-client-go v2.30.0+incompatible
|
||||
go.opentelemetry.io/otel/exporters/jaeger v1.11.2
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.11.2
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68 // indirect
|
||||
github.com/alibabacloud-go/tea v1.1.8 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.2.0 // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect
|
||||
github.com/cockroachdb/redact v1.1.3 // indirect
|
||||
|
|
@ -199,7 +202,6 @@ require (
|
|||
github.com/kr/pretty v0.3.0 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/olekukonko/tablewriter v0.0.5 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/rogpeppe/go-internal v1.8.1 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2 // indirect
|
||||
|
|
|
|||
18
go.sum
18
go.sum
|
|
@ -75,6 +75,12 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
|
|||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
|
||||
github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68 h1:NqugFkGxx1TXSh/pBcU00Y6bljgDPaFdh5MUSeJ7e50=
|
||||
github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68/go.mod h1:6pb/Qy8c+lqua8cFpEy7g39NRRqOWc3rOwAy8m5Y2BY=
|
||||
github.com/alibabacloud-go/tea v1.1.8 h1:vFF0707fqjGiQTxrtMnIXRjOCvQXf49CuDVRtTopmwU=
|
||||
github.com/alibabacloud-go/tea v1.1.8/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4=
|
||||
github.com/aliyun/credentials-go v1.2.6 h1:dSMxpj4uXZj0MYOsEyljlssHzfdHw/M84iQ5QKF0Uxg=
|
||||
github.com/aliyun/credentials-go v1.2.6/go.mod h1:/KowD1cfGSLrLsH28Jr8W+xwoId0ywIy5lNzDz6O1vw=
|
||||
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
|
||||
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
|
|
@ -386,8 +392,9 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
|||
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
|
||||
|
|
@ -562,10 +569,6 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le
|
|||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
|
||||
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
|
||||
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230129073344-87a125853a0b h1:HoJ3J70COnaR3WQTA4gN70DkiaMRPkyLI6yXrPqpFiU=
|
||||
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230129073344-87a125853a0b/go.mod h1:148qnlmZ0Fdm1Fq+Mj/OW2uDoEP25g3mjh0vMGtkgmk=
|
||||
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230302072344-f5dca5d8857b h1:O8ueZJ150EZ78naAjeTkqtoNpI2Pw7YtG3Qg19EMmX0=
|
||||
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230302072344-f5dca5d8857b/go.mod h1:148qnlmZ0Fdm1Fq+Mj/OW2uDoEP25g3mjh0vMGtkgmk=
|
||||
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230303054144-16f081962572 h1:QOimLfT1VwjH9jUkq2SN3uT/WGfgcD4pYBoFOGUMwjM=
|
||||
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230303054144-16f081962572/go.mod h1:148qnlmZ0Fdm1Fq+Mj/OW2uDoEP25g3mjh0vMGtkgmk=
|
||||
github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A=
|
||||
|
|
@ -722,8 +725,9 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf
|
|||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
|
||||
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0=
|
||||
github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
|
||||
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
|
||||
|
|
@ -1362,6 +1366,7 @@ gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183/go.mod h1:FvqrFXt+jCsyQibeRv
|
|||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U=
|
||||
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/errgo.v1 v1.0.0/go.mod h1:CxwszS/Xz1C49Ucd2i6Zil5UToP1EmyrFhKaMVbg1mk=
|
||||
|
|
@ -1372,6 +1377,7 @@ gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/R
|
|||
gopkg.in/httprequest.v1 v1.2.1/go.mod h1:x2Otw96yda5+8+6ZeWwHIJTFkEHWP/qP8pJOzqEtWPM=
|
||||
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU=
|
||||
gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,96 @@
|
|||
package aliyun
|
||||
|
||||
import (
|
||||
"github.com/aliyun/credentials-go/credentials" // >= v1.2.6
|
||||
"github.com/cockroachdb/errors"
|
||||
"github.com/minio/minio-go/v7"
|
||||
minioCred "github.com/minio/minio-go/v7/pkg/credentials"
|
||||
|
||||
"github.com/milvus-io/milvus/internal/log"
|
||||
)
|
||||
|
||||
const OSSDefaultAddress = "oss.aliyuncs.com"
|
||||
|
||||
// NewMinioClient returns a minio.Client which is compatible for aliyun OSS
|
||||
func NewMinioClient(address string, opts *minio.Options) (*minio.Client, error) {
|
||||
if opts == nil {
|
||||
opts = &minio.Options{}
|
||||
}
|
||||
if opts.Creds == nil {
|
||||
credProvider, err := NewCredentialProvider()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to create credential provider")
|
||||
}
|
||||
opts.Creds = minioCred.New(credProvider)
|
||||
}
|
||||
if address == "" {
|
||||
address = OSSDefaultAddress
|
||||
opts.Secure = true
|
||||
}
|
||||
return minio.New(address, opts)
|
||||
}
|
||||
|
||||
// Credential is defined to mock aliyun credential.Credentials
|
||||
//go:generate mockery --name=Credential --with-expecter
|
||||
type Credential interface {
|
||||
credentials.Credential
|
||||
}
|
||||
|
||||
// CredentialProvider implements "github.com/minio/minio-go/v7/pkg/credentials".Provider
|
||||
// also implements transport
|
||||
type CredentialProvider struct {
|
||||
// aliyunCreds doesn't provide a way to get the expire time, so we use the cache to check if it's expired
|
||||
// when aliyunCreds.GetAccessKeyId is different from the cache, we know it's expired
|
||||
akCache string
|
||||
aliyunCreds Credential
|
||||
}
|
||||
|
||||
func NewCredentialProvider() (minioCred.Provider, error) {
|
||||
aliyunCreds, err := credentials.NewCredential(nil)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to create aliyun credential")
|
||||
}
|
||||
// backend, err := minio.DefaultTransport(true)
|
||||
// if err != nil {
|
||||
// return nil, errors.Wrap(err, "failed to create default transport")
|
||||
// }
|
||||
return &CredentialProvider{aliyunCreds: aliyunCreds}, nil
|
||||
}
|
||||
|
||||
// Retrieve returns nil if it successfully retrieved the value.
|
||||
// Error is returned if the value were not obtainable, or empty.
|
||||
// according to the caller minioCred.Credentials.Get(),
|
||||
// it already has a lock, so we don't need to worry about concurrency
|
||||
func (c *CredentialProvider) Retrieve() (minioCred.Value, error) {
|
||||
ret := minioCred.Value{}
|
||||
ak, err := c.aliyunCreds.GetAccessKeyId()
|
||||
if err != nil {
|
||||
return ret, errors.Wrap(err, "failed to get access key id from aliyun credential")
|
||||
}
|
||||
ret.AccessKeyID = *ak
|
||||
sk, err := c.aliyunCreds.GetAccessKeySecret()
|
||||
if err != nil {
|
||||
return minioCred.Value{}, errors.Wrap(err, "failed to get access key secret from aliyun credential")
|
||||
}
|
||||
securityToken, err := c.aliyunCreds.GetSecurityToken()
|
||||
if err != nil {
|
||||
return minioCred.Value{}, errors.Wrap(err, "failed to get security token from aliyun credential")
|
||||
}
|
||||
ret.SecretAccessKey = *sk
|
||||
c.akCache = *ak
|
||||
ret.SessionToken = *securityToken
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// IsExpired returns if the credentials are no longer valid, and need
|
||||
// to be retrieved.
|
||||
// according to the caller minioCred.Credentials.IsExpired(),
|
||||
// it already has a lock, so we don't need to worry about concurrency
|
||||
func (c CredentialProvider) IsExpired() bool {
|
||||
ak, err := c.aliyunCreds.GetAccessKeyId()
|
||||
if err != nil {
|
||||
log.Warn("failed to get access key id from aliyun credential, assume it's expired")
|
||||
return true
|
||||
}
|
||||
return *ak != c.akCache
|
||||
}
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
package aliyun
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/cockroachdb/errors"
|
||||
"github.com/milvus-io/milvus/internal/storage/aliyun/mocks"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestNewMinioClient(t *testing.T) {
|
||||
t.Run("ak sk ok", func(t *testing.T) {
|
||||
minioCli, err := NewMinioClient(OSSDefaultAddress+":443", &minio.Options{
|
||||
Creds: credentials.NewStaticV2("ak", "sk", ""),
|
||||
Secure: true,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, OSSDefaultAddress+":443", minioCli.EndpointURL().Host)
|
||||
assert.Equal(t, "https", minioCli.EndpointURL().Scheme)
|
||||
})
|
||||
|
||||
t.Run("iam failed", func(t *testing.T) {
|
||||
_, err := NewMinioClient("", nil)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("iam ok", func(t *testing.T) {
|
||||
os.Setenv("ALIBABA_CLOUD_ROLE_ARN", "roleArn")
|
||||
os.Setenv("ALIBABA_CLOUD_OIDC_PROVIDER_ARN", "oidcProviderArn")
|
||||
os.Setenv("ALIBABA_CLOUD_OIDC_TOKEN_FILE", "oidcTokenFilePath")
|
||||
minioCli, err := NewMinioClient("", nil)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, OSSDefaultAddress, minioCli.EndpointURL().Host)
|
||||
assert.Equal(t, "https", minioCli.EndpointURL().Scheme)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCredentialProvider_Retrieve(t *testing.T) {
|
||||
c := new(CredentialProvider)
|
||||
mockAliyunCreds := mocks.NewCredential(t)
|
||||
c.aliyunCreds = mockAliyunCreds
|
||||
|
||||
ak := "ak"
|
||||
sk := "sk"
|
||||
token := "token"
|
||||
errMock := errors.Errorf("mock")
|
||||
|
||||
t.Run("get ak or sk or token failed", func(t *testing.T) {
|
||||
mockAliyunCreds.EXPECT().GetAccessKeyId().Return(nil, errMock).Times(1)
|
||||
_, err := c.Retrieve()
|
||||
assert.Error(t, err)
|
||||
|
||||
mockAliyunCreds.EXPECT().GetAccessKeyId().Return(&ak, nil).Times(1)
|
||||
mockAliyunCreds.EXPECT().GetAccessKeySecret().Return(nil, errMock).Times(1)
|
||||
_, err = c.Retrieve()
|
||||
assert.Error(t, err)
|
||||
|
||||
mockAliyunCreds.EXPECT().GetAccessKeyId().Return(&ak, nil).Times(1)
|
||||
mockAliyunCreds.EXPECT().GetAccessKeySecret().Return(&sk, nil).Times(1)
|
||||
mockAliyunCreds.EXPECT().GetSecurityToken().Return(nil, errMock).Times(1)
|
||||
_, err = c.Retrieve()
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("ok", func(t *testing.T) {
|
||||
mockAliyunCreds.EXPECT().GetAccessKeyId().Return(&ak, nil).Times(1)
|
||||
mockAliyunCreds.EXPECT().GetAccessKeySecret().Return(&sk, nil).Times(1)
|
||||
mockAliyunCreds.EXPECT().GetSecurityToken().Return(&token, nil).Times(1)
|
||||
ret, err := c.Retrieve()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, ak, ret.AccessKeyID)
|
||||
assert.Equal(t, sk, ret.SecretAccessKey)
|
||||
assert.Equal(t, token, ret.SessionToken)
|
||||
assert.Equal(t, c.akCache, ret.AccessKeyID)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCredentialProvider_IsExpired(t *testing.T) {
|
||||
c := new(CredentialProvider)
|
||||
mockAliyunCreds := mocks.NewCredential(t)
|
||||
c.aliyunCreds = mockAliyunCreds
|
||||
|
||||
ak := "ak"
|
||||
errMock := errors.Errorf("mock")
|
||||
t.Run("expired", func(t *testing.T) {
|
||||
mockAliyunCreds.EXPECT().GetAccessKeyId().Return(&ak, nil).Times(1)
|
||||
assert.True(t, c.IsExpired())
|
||||
})
|
||||
|
||||
t.Run("not expired", func(t *testing.T) {
|
||||
c.akCache = ak
|
||||
mockAliyunCreds.EXPECT().GetAccessKeyId().Return(&ak, nil).Times(1)
|
||||
assert.False(t, c.IsExpired())
|
||||
})
|
||||
|
||||
t.Run("get failed, assume expired", func(t *testing.T) {
|
||||
mockAliyunCreds.EXPECT().GetAccessKeyId().Return(nil, errMock).Times(1)
|
||||
assert.True(t, c.IsExpired())
|
||||
})
|
||||
}
|
||||
|
|
@ -0,0 +1,244 @@
|
|||
// Code generated by mockery v2.16.0. DO NOT EDIT.
|
||||
|
||||
package mocks
|
||||
|
||||
import mock "github.com/stretchr/testify/mock"
|
||||
|
||||
// Credential is an autogenerated mock type for the Credential type
|
||||
type Credential struct {
|
||||
mock.Mock
|
||||
}
|
||||
|
||||
type Credential_Expecter struct {
|
||||
mock *mock.Mock
|
||||
}
|
||||
|
||||
func (_m *Credential) EXPECT() *Credential_Expecter {
|
||||
return &Credential_Expecter{mock: &_m.Mock}
|
||||
}
|
||||
|
||||
// GetAccessKeyId provides a mock function with given fields:
|
||||
func (_m *Credential) GetAccessKeyId() (*string, error) {
|
||||
ret := _m.Called()
|
||||
|
||||
var r0 *string
|
||||
if rf, ok := ret.Get(0).(func() *string); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*string)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func() error); ok {
|
||||
r1 = rf()
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// Credential_GetAccessKeyId_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetAccessKeyId'
|
||||
type Credential_GetAccessKeyId_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// GetAccessKeyId is a helper method to define mock.On call
|
||||
func (_e *Credential_Expecter) GetAccessKeyId() *Credential_GetAccessKeyId_Call {
|
||||
return &Credential_GetAccessKeyId_Call{Call: _e.mock.On("GetAccessKeyId")}
|
||||
}
|
||||
|
||||
func (_c *Credential_GetAccessKeyId_Call) Run(run func()) *Credential_GetAccessKeyId_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Credential_GetAccessKeyId_Call) Return(_a0 *string, _a1 error) *Credential_GetAccessKeyId_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
// GetAccessKeySecret provides a mock function with given fields:
|
||||
func (_m *Credential) GetAccessKeySecret() (*string, error) {
|
||||
ret := _m.Called()
|
||||
|
||||
var r0 *string
|
||||
if rf, ok := ret.Get(0).(func() *string); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*string)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func() error); ok {
|
||||
r1 = rf()
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// Credential_GetAccessKeySecret_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetAccessKeySecret'
|
||||
type Credential_GetAccessKeySecret_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// GetAccessKeySecret is a helper method to define mock.On call
|
||||
func (_e *Credential_Expecter) GetAccessKeySecret() *Credential_GetAccessKeySecret_Call {
|
||||
return &Credential_GetAccessKeySecret_Call{Call: _e.mock.On("GetAccessKeySecret")}
|
||||
}
|
||||
|
||||
func (_c *Credential_GetAccessKeySecret_Call) Run(run func()) *Credential_GetAccessKeySecret_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Credential_GetAccessKeySecret_Call) Return(_a0 *string, _a1 error) *Credential_GetAccessKeySecret_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
// GetBearerToken provides a mock function with given fields:
|
||||
func (_m *Credential) GetBearerToken() *string {
|
||||
ret := _m.Called()
|
||||
|
||||
var r0 *string
|
||||
if rf, ok := ret.Get(0).(func() *string); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*string)
|
||||
}
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// Credential_GetBearerToken_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetBearerToken'
|
||||
type Credential_GetBearerToken_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// GetBearerToken is a helper method to define mock.On call
|
||||
func (_e *Credential_Expecter) GetBearerToken() *Credential_GetBearerToken_Call {
|
||||
return &Credential_GetBearerToken_Call{Call: _e.mock.On("GetBearerToken")}
|
||||
}
|
||||
|
||||
func (_c *Credential_GetBearerToken_Call) Run(run func()) *Credential_GetBearerToken_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Credential_GetBearerToken_Call) Return(_a0 *string) *Credential_GetBearerToken_Call {
|
||||
_c.Call.Return(_a0)
|
||||
return _c
|
||||
}
|
||||
|
||||
// GetSecurityToken provides a mock function with given fields:
|
||||
func (_m *Credential) GetSecurityToken() (*string, error) {
|
||||
ret := _m.Called()
|
||||
|
||||
var r0 *string
|
||||
if rf, ok := ret.Get(0).(func() *string); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*string)
|
||||
}
|
||||
}
|
||||
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(1).(func() error); ok {
|
||||
r1 = rf()
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// Credential_GetSecurityToken_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSecurityToken'
|
||||
type Credential_GetSecurityToken_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// GetSecurityToken is a helper method to define mock.On call
|
||||
func (_e *Credential_Expecter) GetSecurityToken() *Credential_GetSecurityToken_Call {
|
||||
return &Credential_GetSecurityToken_Call{Call: _e.mock.On("GetSecurityToken")}
|
||||
}
|
||||
|
||||
func (_c *Credential_GetSecurityToken_Call) Run(run func()) *Credential_GetSecurityToken_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Credential_GetSecurityToken_Call) Return(_a0 *string, _a1 error) *Credential_GetSecurityToken_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
// GetType provides a mock function with given fields:
|
||||
func (_m *Credential) GetType() *string {
|
||||
ret := _m.Called()
|
||||
|
||||
var r0 *string
|
||||
if rf, ok := ret.Get(0).(func() *string); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*string)
|
||||
}
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// Credential_GetType_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetType'
|
||||
type Credential_GetType_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// GetType is a helper method to define mock.On call
|
||||
func (_e *Credential_Expecter) GetType() *Credential_GetType_Call {
|
||||
return &Credential_GetType_Call{Call: _e.mock.On("GetType")}
|
||||
}
|
||||
|
||||
func (_c *Credential_GetType_Call) Run(run func()) *Credential_GetType_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run()
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *Credential_GetType_Call) Return(_a0 *string) *Credential_GetType_Call {
|
||||
_c.Call.Return(_a0)
|
||||
return _c
|
||||
}
|
||||
|
||||
type mockConstructorTestingTNewCredential interface {
|
||||
mock.TestingT
|
||||
Cleanup(func())
|
||||
}
|
||||
|
||||
// NewCredential creates a new instance of Credential. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
func NewCredential(t mockConstructorTestingTNewCredential) *Credential {
|
||||
mock := &Credential{}
|
||||
mock.Mock.Test(t)
|
||||
|
||||
t.Cleanup(func() { mock.AssertExpectations(t) })
|
||||
|
||||
return mock
|
||||
}
|
||||
|
|
@ -28,6 +28,7 @@ import (
|
|||
"github.com/cockroachdb/errors"
|
||||
|
||||
"github.com/milvus-io/milvus/internal/log"
|
||||
"github.com/milvus-io/milvus/internal/storage/aliyun"
|
||||
"github.com/milvus-io/milvus/internal/storage/gcp"
|
||||
"github.com/milvus-io/milvus/internal/util/merr"
|
||||
"github.com/milvus-io/milvus/internal/util/retry"
|
||||
|
|
@ -42,8 +43,9 @@ var (
|
|||
)
|
||||
|
||||
const (
|
||||
CloudProviderGCP = "gcp"
|
||||
CloudProviderAWS = "aws"
|
||||
CloudProviderGCP = "gcp"
|
||||
CloudProviderAWS = "aws"
|
||||
CloudProviderAliyun = "aliyun"
|
||||
)
|
||||
|
||||
func WrapErrNoSuchKey(key string) error {
|
||||
|
|
@ -77,8 +79,17 @@ func NewMinioChunkManager(ctx context.Context, opts ...Option) (*MinioChunkManag
|
|||
func newMinioChunkManagerWithConfig(ctx context.Context, c *config) (*MinioChunkManager, error) {
|
||||
var creds *credentials.Credentials
|
||||
var newMinioFn = minio.New
|
||||
var bucketLookupType = minio.BucketLookupAuto
|
||||
|
||||
switch c.cloudProvider {
|
||||
case CloudProviderAliyun:
|
||||
// auto doesn't work for aliyun, so we set to dns deliberately
|
||||
bucketLookupType = minio.BucketLookupDNS
|
||||
if c.useIAM {
|
||||
newMinioFn = aliyun.NewMinioClient
|
||||
} else {
|
||||
creds = credentials.NewStaticV4(c.accessKeyID, c.secretAccessKeyID, "")
|
||||
}
|
||||
case CloudProviderGCP:
|
||||
newMinioFn = gcp.NewMinioClient
|
||||
if !c.useIAM {
|
||||
|
|
@ -92,8 +103,9 @@ func newMinioChunkManagerWithConfig(ctx context.Context, c *config) (*MinioChunk
|
|||
}
|
||||
}
|
||||
minioOpts := &minio.Options{
|
||||
Creds: creds,
|
||||
Secure: c.useSSL,
|
||||
BucketLookup: bucketLookupType,
|
||||
Creds: creds,
|
||||
Secure: c.useSSL,
|
||||
}
|
||||
minIOClient, err := newMinioFn(c.address, minioOpts)
|
||||
// options nil or invalid formatted endpoint, don't need to retry
|
||||
|
|
|
|||
|
|
@ -721,10 +721,12 @@ func (p *MinioConfig) Init(base *BaseTable) {
|
|||
Key: "minio.useIAM",
|
||||
DefaultValue: DefaultMinioUseIAM,
|
||||
Version: "2.0.0",
|
||||
Doc: `Whether to ` + "useIAM" + ` role to access S3/GCS instead of access/secret keys
|
||||
Doc: `Whether to useIAM role to access S3/GCS instead of access/secret keys
|
||||
For more information, refer to
|
||||
aws: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html
|
||||
gcp: https://cloud.google.com/storage/docs/access-control/iam`,
|
||||
gcp: https://cloud.google.com/storage/docs/access-control/iam
|
||||
aliyun (ack): https://www.alibabacloud.com/help/en/container-service-for-kubernetes/latest/use-rrsa-to-enforce-access-control
|
||||
aliyun (ecs): https://www.alibabacloud.com/help/en/elastic-compute-service/latest/attach-an-instance-ram-role`,
|
||||
Export: true,
|
||||
}
|
||||
p.UseIAM.Init(base.mgr)
|
||||
|
|
@ -733,10 +735,11 @@ gcp: https://cloud.google.com/storage/docs/access-control/iam`,
|
|||
Key: "minio.cloudProvider",
|
||||
DefaultValue: DefaultMinioCloudProvider,
|
||||
Version: "2.2.0",
|
||||
Doc: `Cloud Provider of S3. Supports: "aws", "gcp".
|
||||
Doc: `Cloud Provider of S3. Supports: "aws", "gcp", "aliyun".
|
||||
You can use "aws" for other cloud provider supports S3 API with signature v4, e.g.: minio
|
||||
You can use "gcp" for other cloud provider supports S3 API with signature v2
|
||||
When ` + "useIAM" + ` enabled, only "aws" & "gcp" is supported for now`,
|
||||
You can use "aliyun" for other cloud provider uses virtual host style bucket
|
||||
When useIAM enabled, only "aws", "gcp", "aliyun" is supported for now`,
|
||||
Export: true,
|
||||
}
|
||||
p.CloudProvider.Init(base.mgr)
|
||||
|
|
|
|||
Loading…
Reference in New Issue