mirror of https://github.com/milvus-io/milvus.git
129 lines
4.0 KiB
Go
129 lines
4.0 KiB
Go
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package msgstream
|
|
|
|
import (
|
|
"context"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
|
|
"github.com/milvus-io/milvus/pkg/log"
|
|
"github.com/milvus-io/milvus/pkg/mq/common"
|
|
"github.com/milvus-io/milvus/pkg/util/typeutil"
|
|
)
|
|
|
|
// UniqueID is an alias for short
|
|
type UniqueID = typeutil.UniqueID
|
|
|
|
// Timestamp is an alias for short
|
|
type Timestamp = typeutil.Timestamp
|
|
|
|
// IntPrimaryKey is an alias for short
|
|
type IntPrimaryKey = typeutil.IntPrimaryKey
|
|
|
|
// MsgPosition is an alias for short
|
|
type MsgPosition = msgpb.MsgPosition
|
|
|
|
// MessageID is an alias for short
|
|
type MessageID = common.MessageID
|
|
|
|
// MsgPack represents a batch of msg in msgstream
|
|
type MsgPack struct {
|
|
BeginTs Timestamp
|
|
EndTs Timestamp
|
|
Msgs []TsMsg
|
|
StartPositions []*MsgPosition
|
|
EndPositions []*MsgPosition
|
|
}
|
|
|
|
// RepackFunc is a function type which used to repack message after hash by primary key
|
|
type RepackFunc func(msgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, error)
|
|
|
|
// MsgStream is an interface that can be used to produce and consume message on message queue
|
|
type MsgStream interface {
|
|
Close()
|
|
|
|
AsProducer(ctx context.Context, channels []string)
|
|
Produce(context.Context, *MsgPack) error
|
|
SetRepackFunc(repackFunc RepackFunc)
|
|
GetProduceChannels() []string
|
|
Broadcast(context.Context, *MsgPack) (map[string][]MessageID, error)
|
|
|
|
AsConsumer(ctx context.Context, channels []string, subName string, position common.SubscriptionInitialPosition) error
|
|
Chan() <-chan *MsgPack
|
|
// Seek consume message from the specified position
|
|
// includeCurrentMsg indicates whether to consume the current message, and in the milvus system, it should be always false
|
|
Seek(ctx context.Context, msgPositions []*MsgPosition, includeCurrentMsg bool) error
|
|
|
|
GetLatestMsgID(channel string) (MessageID, error)
|
|
CheckTopicValid(channel string) error
|
|
|
|
ForceEnableProduce(can bool)
|
|
}
|
|
|
|
type ReplicateConfig struct {
|
|
ReplicateID string
|
|
CheckFunc CheckReplicateMsgFunc
|
|
}
|
|
|
|
type CheckReplicateMsgFunc func(*ReplicateMsg) bool
|
|
|
|
func GetReplicateConfig(replicateID, dbName, colName string) *ReplicateConfig {
|
|
if replicateID == "" {
|
|
return nil
|
|
}
|
|
replicateConfig := &ReplicateConfig{
|
|
ReplicateID: replicateID,
|
|
CheckFunc: func(msg *ReplicateMsg) bool {
|
|
if !msg.GetIsEnd() {
|
|
return false
|
|
}
|
|
log.Info("check replicate msg",
|
|
zap.String("replicateID", replicateID),
|
|
zap.String("dbName", dbName),
|
|
zap.String("colName", colName),
|
|
zap.Any("msg", msg))
|
|
if msg.GetIsCluster() {
|
|
return true
|
|
}
|
|
return msg.GetDatabase() == dbName && (msg.GetCollection() == colName || msg.GetCollection() == "")
|
|
},
|
|
}
|
|
return replicateConfig
|
|
}
|
|
|
|
func GetReplicateID(msg TsMsg) string {
|
|
msgBase, ok := msg.(interface{ GetBase() *commonpb.MsgBase })
|
|
if !ok {
|
|
log.Warn("fail to get msg base, please check it", zap.Any("type", msg.Type()))
|
|
return ""
|
|
}
|
|
return msgBase.GetBase().GetReplicateInfo().GetReplicateID()
|
|
}
|
|
|
|
func MatchReplicateID(msg TsMsg, replicateID string) bool {
|
|
return GetReplicateID(msg) == replicateID
|
|
}
|
|
|
|
type Factory interface {
|
|
NewMsgStream(ctx context.Context) (MsgStream, error)
|
|
NewTtMsgStream(ctx context.Context) (MsgStream, error)
|
|
NewMsgStreamDisposer(ctx context.Context) func([]string, string) error
|
|
}
|