milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support the apikey authentication (#27723) 

Signed-off-by: SimFG <bang.fu@zilliz.com>
pull/27752/head
SimFG 2023-10-17 21:00:11 +08:00 committed by GitHub
parent 2f201c25e2
commit 84d05b939b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 121 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
go.mod
View File

@ -24,7 +24,7 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/klauspost/compress v1.16.5
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231011053327-5f3a9bd32b37
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231017024957-5e5a27fd4875
github.com/minio/minio-go/v7 v7.0.56
github.com/prometheus/client_golang v1.14.0
github.com/prometheus/client_model v0.3.0
@ -58,6 +58,8 @@ require (
stathat.com/c/consistent v1.0.0
)
require github.com/milvus-io/milvus/pkg v0.0.0-20230607023836-1593278f9d9c
require (
cloud.google.com/go/compute v1.19.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
@ -138,7 +140,6 @@ require (
github.com/mattn/go-colorable v0.1.11 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/milvus-io/milvus/pkg v0.0.0-20230607023836-1593278f9d9c // indirect
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
github.com/minio/highwayhash v1.0.2 // indirect

11
go.sum
View File

@ -281,7 +281,6 @@ github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2C
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
@ -508,7 +507,6 @@ github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYb
github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE=
github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro=
github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8=
github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d/go.mod h1:JJNrCn9otv/2QP4D7SMJBgaleKpOf66PnW6F5WGNRIc=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@ -547,7 +545,6 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76 h1:IVlcvV0CjvfBYYod5ePe89l+3LBAl//6n9kJ9Vr2i0k=
github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76/go.mod h1:Iu9BHUvTh8/KpbuSoKx/CaJEdJvFxSverxIy7I+nq7s=
github.com/linkedin/goavro v2.1.0+incompatible/go.mod h1:bBCwI2eGYpUI/4820s67MElg9tdeLbINjLjiM2xZFYM=
github.com/linkedin/goavro/v2 v2.9.8/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
github.com/linkedin/goavro/v2 v2.10.0/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
@ -585,12 +582,8 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0-dev.1 h1:x6vhrVyK3wEuXIDHt0uk2l/UFPa/RRGWk1nkjgN5jkI=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0-dev.1/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.1-0.20230907032509-23756009c643 h1:3MXEYckliGnyepZeLDrhn+speelsoRKU1IwD8JrxXMo=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.1-0.20230907032509-23756009c643/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231011053327-5f3a9bd32b37 h1:iwLDdLHL9EGDQWRVZHocjcyAzYTRQEN0NatW6DvzKeY=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231011053327-5f3a9bd32b37/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231017024957-5e5a27fd4875 h1:7OPJn0sOeueXNnreWup0GR7ZlXEURpcKklzplXM9kDg=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231017024957-5e5a27fd4875/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A=
github.com/milvus-io/pulsar-client-go v0.6.10/go.mod h1:lQqCkgwDF8YFYjKA+zOheTk1tev2B+bKj5j7+nm8M1w=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=

1
internal/distributed/proxy/service.go
View File

@ -119,6 +119,7 @@ func authenticate(c *gin.Context) {
if !proxy.Params.CommonCfg.AuthorizationEnabled.GetAsBool() {
return
}
// TODO fubang
username, password, ok := httpserver.ParseUsernamePassword(c)
if ok {
if proxy.PasswordVerify(c, username, password) {

57
internal/proxy/authentication_interceptor.go
View File

@ -15,19 +15,7 @@ import (
"github.com/milvus-io/milvus/pkg/util/merr"
)
func parseMD(authorization []string) (username, password string) {
if len(authorization) < 1 {
log.Warn("key not found in header")
return
}
// token format: base64<username:password>
// token := strings.TrimPrefix(authorization[0], "Bearer ")
token := authorization[0]
rawToken, err := crypto.Base64Decode(token)
if err != nil {
log.Warn("fail to decode the token", zap.Error(err))
return
}
func parseMD(rawToken string) (username, password string) {
secrets := strings.SplitN(rawToken, util.CredentialSeperator, 2)
if len(secrets) < 2 {
log.Warn("invalid token format, length of secrets less than 2")
@ -68,12 +56,45 @@ func AuthenticationInterceptor(ctx context.Context) (context.Context, error) {
// 2. if rpc call from sdk
if Params.CommonCfg.AuthorizationEnabled.GetAsBool() {
if !validSourceID(ctx, md[strings.ToLower(util.HeaderSourceID)]) {
username, password := parseMD(md[strings.ToLower(util.HeaderAuthorize)])
if !passwordVerify(ctx, username, password, globalMetaCache) {
msg := fmt.Sprintf("username: %s, password: %s", username, password)
return nil, merr.WrapErrParameterInvalid("vaild username and password", msg, "auth check failure, please check username and password are correct")
authStrArr := md[strings.ToLower(util.HeaderAuthorize)]
if len(authStrArr) < 1 {
log.Warn("key not found in header")
return nil, merr.WrapErrParameterInvalidMsg("missing authorization in header")
}
// token format: base64<username:password>
// token := strings.TrimPrefix(authorization[0], "Bearer ")
token := authStrArr[0]
rawToken, err := crypto.Base64Decode(token)
if err != nil {
log.Warn("fail to decode the token", zap.Error(err))
return nil, merr.WrapErrParameterInvalidMsg("invalid token format")
}
if !strings.Contains(rawToken, util.CredentialSeperator) {
// apikey authentication
if hoo == nil {
return nil, merr.WrapErrServiceInternal("internal: Milvus Proxy is not ready yet. please wait")
}
user, err := hoo.VerifyAPIKey(rawToken)
if err != nil {
log.Warn("fail to verify apikey", zap.String("api_key", rawToken), zap.Error(err))
return nil, merr.WrapErrParameterInvalidMsg("invalid apikey: [%s]", rawToken)
}
metrics.UserRPCCounter.WithLabelValues(user).Inc()
userToken := fmt.Sprintf("%s%s%s", user, util.CredentialSeperator, "___")
md[strings.ToLower(util.HeaderAuthorize)] = []string{crypto.Base64Encode(userToken)}
ctx = metadata.NewIncomingContext(ctx, md)
} else {
// username+password authentication
username, password := parseMD(rawToken)
if !passwordVerify(ctx, username, password, globalMetaCache) {
msg := fmt.Sprintf("username: %s, password: %s", username, password)
return nil, merr.WrapErrParameterInvalid("vaild username and password", msg, "auth check failure, please check username and password are correct")
}
metrics.UserRPCCounter.WithLabelValues(username).Inc()
}
metrics.UserRPCCounter.WithLabelValues(username).Inc()
}
}
return ctx, nil

68
internal/proxy/authentication_interceptor_test.go
View File

@ -2,8 +2,10 @@ package proxy
import (
"context"
"strings"
"testing"
"github.com/cockroachdb/errors"
"github.com/stretchr/testify/assert"
"google.golang.org/grpc/metadata"
@ -16,7 +18,12 @@ import (
// validAuth validates the authentication
func TestValidAuth(t *testing.T) {
validAuth := func(ctx context.Context, authorization []string) bool {
username, password := parseMD(authorization)
if len(authorization) < 1 {
return false
}
token := authorization[0]
rawToken, _ := crypto.Base64Decode(token)
username, password := parseMD(rawToken)
if username == "" || password == "" {
return false
}
@ -84,4 +91,63 @@ func TestAuthenticationInterceptor(t *testing.T) {
ctx = metadata.NewIncomingContext(ctx, md)
_, err = AuthenticationInterceptor(ctx)
assert.NoError(t, err)
{
// wrong authorization style
md = metadata.Pairs(util.HeaderAuthorize, "123456")
ctx = metadata.NewIncomingContext(ctx, md)
_, err = AuthenticationInterceptor(ctx)
assert.Error(t, err)
}
{
// invalid user
md = metadata.Pairs(util.HeaderAuthorize, crypto.Base64Encode("mockUser2:mockPass"))
ctx = metadata.NewIncomingContext(ctx, md)
_, err = AuthenticationInterceptor(ctx)
assert.Error(t, err)
}
{
// default hook
md = metadata.Pairs(util.HeaderAuthorize, crypto.Base64Encode("mockapikey"))
ctx = metadata.NewIncomingContext(ctx, md)
_, err = AuthenticationInterceptor(ctx)
assert.Error(t, err)
}
{
// verify apikey error
hoo = mockAPIHook{mockErr: errors.New("err")}
md = metadata.Pairs(util.HeaderAuthorize, crypto.Base64Encode("mockapikey"))
ctx = metadata.NewIncomingContext(ctx, md)
_, err = AuthenticationInterceptor(ctx)
assert.Error(t, err)
}
{
hoo = mockAPIHook{mockErr: nil, apiUser: "mockUser"}
md = metadata.Pairs(util.HeaderAuthorize, crypto.Base64Encode("mockapikey"))
ctx = metadata.NewIncomingContext(ctx, md)
authCtx, err := AuthenticationInterceptor(ctx)
assert.NoError(t, err)
md, ok := metadata.FromIncomingContext(authCtx)
assert.True(t, ok)
authStrArr := md[strings.ToLower(util.HeaderAuthorize)]
token := authStrArr[0]
rawToken, err := crypto.Base64Decode(token)
assert.NoError(t, err)
user, _ := parseMD(rawToken)
assert.Equal(t, "mockUser", user)
}
}
type mockAPIHook struct {
defaultHook
mockErr error
apiUser string
}
func (m mockAPIHook) VerifyAPIKey(apiKey string) (string, error) {
return m.apiUser, m.mockErr
}

5
internal/proxy/hook_interceptor.go
View File

@ -7,6 +7,7 @@ import (
"strconv"
"strings"
"github.com/cockroachdb/errors"
"go.uber.org/zap"
"google.golang.org/grpc"
@ -19,6 +20,10 @@ import (
type defaultHook struct{}
func (d defaultHook) VerifyAPIKey(key string) (string, error) {
return "", errors.New("default hook, can't verify api key")
}
func (d defaultHook) Init(params map[string]string) error {
return nil
}

4
internal/proxy/hook_interceptor_test.go
View File

@ -142,6 +142,10 @@ func TestHookInterceptor(t *testing.T) {
func TestDefaultHook(t *testing.T) {
d := defaultHook{}
assert.NoError(t, d.Init(nil))
{
_, err := d.VerifyAPIKey("key")
assert.Error(t, err)
}
assert.NotPanics(t, func() {
d.Release()
})