milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

MEP-30:Support basic coordinators active-standby mechanism #16901 (#19166) 

Signed-off-by: wayblink <anyang.wang@zilliz.com>

Signed-off-by: wayblink <anyang.wang@zilliz.com>
pull/19568/head
wayblink 2022-09-29 18:35:02 +08:00 committed by GitHub
parent 51eb12a034
commit 54e2c079c4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 662 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
configs/milvus.yaml
View File

@ -120,6 +120,7 @@ rocksmq:
rootCoord:
address: localhost
port: 53100
enableActiveStandby: false # Enable active-standby
dmlChannelNum: 256 # The number of dml channels created at system startup
maxPartitionNum: 4096 # Maximum number of partitions in a collection
@ -170,8 +171,7 @@ queryCoord:
loadTimeoutSeconds: 600
checkHandoffInterval: 5000
taskMergeCap: 8
enableActiveStandby: false # Enable active-standby
# Related configuration of queryNode, used to run hybrid search between vector and scalar data.
queryNode:
@ -217,6 +217,7 @@ queryNode:
indexCoord:
address: localhost
port: 31000
enableActiveStandby: false # Enable active-standby
minSegmentNumRowsToEnableIndex: 1024 # It's a threshold. When the segment num rows is less than this value, the segment will not be indexed
@ -242,6 +243,7 @@ dataCoord:
port: 13333
enableCompaction: true # Enable data segment compaction
enableGarbageCollection: true
enableActiveStandby: false # Enable active-standby
segment:
maxSize: 512 # Maximum size of a segment in MB
@ -358,6 +360,10 @@ common:
# 0 is close, 1 is one-way authentication, 2 is two-way authentication.
tlsMode: 0
session:
ttl: 60 # ttl value when session granting a lease to register service
retryTimes: 30 # retry times when session sending etcd requests
# QuotaConfig, configurations of Milvus quota and limits.
# By default, we enable:
# 1. TT protection;

32
internal/datacoord/server.go
View File

@ -87,6 +87,8 @@ const (
ServerStateInitializing ServerState = 1
// ServerStateHealthy state stands for healthy `Server` instance
ServerStateHealthy ServerState = 2
// ServerStateStandby state stands for standby `Server` instance
ServerStateStandby ServerState = 3
)
type dataNodeCreatorFunc func(ctx context.Context, addr string) (types.DataNode, error)
@ -134,6 +136,9 @@ type Server struct {
//icEventCh <-chan *sessionutil.SessionEvent
qcEventCh <-chan *sessionutil.SessionEvent
enableActiveStandBy bool
activateFunc func()
dataNodeCreator dataNodeCreatorFunc
rootCoordClientCreator rootCoordCreatorFunc
indexCoord types.IndexCoord
@ -201,8 +206,8 @@ func CreateServer(ctx context.Context, factory dependency.Factory, opts ...Optio
dataNodeCreator: defaultDataNodeCreatorFunc,
rootCoordClientCreator: defaultRootCoordCreatorFunc,
helper: defaultServerHelper(),
metricsCacheManager: metricsinfo.NewMetricsCacheManager(),
metricsCacheManager: metricsinfo.NewMetricsCacheManager(),
enableActiveStandBy: Params.DataCoordCfg.EnableActiveStandby,
}
for _, opt := range opts {
@ -227,6 +232,9 @@ func (s *Server) QuitSignal() <-chan struct{} {
// Register registers data service at etcd
func (s *Server) Register() error {
s.session.Register()
if s.enableActiveStandBy {
s.session.ProcessActiveStandBy(s.activateFunc)
}
go s.session.LivenessCheck(s.serverLoopCtx, func() {
logutil.Logger(s.ctx).Error("disconnected from etcd and exited", zap.Int64("serverID", s.session.ServerID))
if err := s.Stop(); err != nil {
@ -248,6 +256,7 @@ func (s *Server) initSession() error {
return errors.New("failed to initialize session")
}
s.session.Init(typeutil.DataCoordRole, Params.DataCoordCfg.Address, true, true)
s.session.SetEnableActiveStandBy(s.enableActiveStandBy)
Params.DataCoordCfg.SetNodeID(s.session.ServerID)
Params.SetLogger(Params.DataCoordCfg.GetNodeID())
return nil
@ -310,11 +319,24 @@ func (s *Server) Start() error {
s.compactionHandler.start()
s.compactionTrigger.start()
s.startServerLoop()
if s.enableActiveStandBy {
s.activateFunc = func() {
// todo complete the activateFunc
log.Info("datacoord switch from standby to active, activating")
s.startServerLoop()
atomic.StoreInt64(&s.isServing, ServerStateHealthy)
logutil.Logger(s.ctx).Debug("startup success")
}
atomic.StoreInt64(&s.isServing, ServerStateStandby)
logutil.Logger(s.ctx).Debug("DataCoord enter standby mode successfully")
} else {
s.startServerLoop()
atomic.StoreInt64(&s.isServing, ServerStateHealthy)
logutil.Logger(s.ctx).Debug("DataCoord startup successfully")
}
Params.DataCoordCfg.CreatedTime = time.Now()
Params.DataCoordCfg.UpdatedTime = time.Now()
atomic.StoreInt64(&s.isServing, ServerStateHealthy)
logutil.Logger(s.ctx).Debug("startup success")
// DataCoord (re)starts successfully and starts to collection segment stats
// data from all DataNode.

55
internal/datacoord/server_test.go
View File

@ -3209,3 +3209,58 @@ func Test_initGarbageCollection(t *testing.T) {
assert.Contains(t, err.Error(), "too many colons in address")
})
}
func testDataCoordBase(t *testing.T, opts ...Option) *Server {
var err error
Params.CommonCfg.DataCoordTimeTick = Params.CommonCfg.DataCoordTimeTick + strconv.Itoa(rand.Int())
factory := dependency.NewDefaultFactory(true)
etcdCli, err := etcd.GetEtcdClient(&Params.EtcdCfg)
assert.Nil(t, err)
sessKey := path.Join(Params.EtcdCfg.MetaRootPath, sessionutil.DefaultServiceRoot)
_, err = etcdCli.Delete(context.Background(), sessKey, clientv3.WithPrefix())
assert.Nil(t, err)
svr := CreateServer(context.TODO(), factory, opts...)
svr.SetEtcdClient(etcdCli)
svr.dataNodeCreator = func(ctx context.Context, addr string) (types.DataNode, error) {
return newMockDataNodeClient(0, nil)
}
svr.rootCoordClientCreator = func(ctx context.Context, metaRootPath string, etcdCli *clientv3.Client) (types.RootCoord, error) {
return newMockRootCoordService(), nil
}
err = svr.Init()
assert.Nil(t, err)
err = svr.Start()
assert.Nil(t, err)
err = svr.Register()
assert.Nil(t, err)
resp, err := svr.GetComponentStates(context.Background())
assert.Nil(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
assert.Equal(t, internalpb.StateCode_Healthy, resp.GetState().GetStateCode())
// stop channal watch state watcher in tests
if svr.channelManager != nil && svr.channelManager.stopChecker != nil {
svr.channelManager.stopChecker()
}
return svr
}
func TestDataCoord_DisableActiveStandby(t *testing.T) {
Params.Init()
Params.DataCoordCfg.EnableActiveStandby = false
svr := testDataCoordBase(t)
defer closeTestServer(t, svr)
}
// make sure the main functions work well when EnableActiveStandby=true
func TestDataCoord_EnableActiveStandby(t *testing.T) {
Params.Init()
Params.DataCoordCfg.EnableActiveStandby = true
svr := testDataCoordBase(t)
defer closeTestServer(t, svr)
}

30
internal/indexcoord/index_coord.go
View File

@ -101,6 +101,9 @@ type IndexCoord struct {
dataCoordClient types.DataCoord
rootCoordClient types.RootCoord
enableActiveStandBy bool
activateFunc func()
// Add callback functions at different stages
startCallbacks []func()
closeCallbacks []func()
@ -114,10 +117,11 @@ func NewIndexCoord(ctx context.Context, factory dependency.Factory) (*IndexCoord
rand.Seed(time.Now().UnixNano())
ctx1, cancel := context.WithCancel(ctx)
i := &IndexCoord{
loopCtx: ctx1,
loopCancel: cancel,
reqTimeoutInterval: time.Second * 10,
factory: factory,
loopCtx: ctx1,
loopCancel: cancel,
reqTimeoutInterval: time.Second * 10,
factory: factory,
enableActiveStandBy: Params.IndexCoordCfg.EnableActiveStandby,
}
i.UpdateStateCode(internalpb.StateCode_Abnormal)
return i, nil
@ -126,6 +130,9 @@ func NewIndexCoord(ctx context.Context, factory dependency.Factory) (*IndexCoord
// Register register IndexCoord role at etcd.
func (i *IndexCoord) Register() error {
i.session.Register()
if i.enableActiveStandBy {
i.session.ProcessActiveStandBy(i.activateFunc)
}
go i.session.LivenessCheck(i.loopCtx, func() {
log.Error("Index Coord disconnected from etcd, process will exit", zap.Int64("Server Id", i.session.ServerID))
if err := i.Stop(); err != nil {
@ -147,6 +154,7 @@ func (i *IndexCoord) initSession() error {
return errors.New("failed to initialize session")
}
i.session.Init(typeutil.IndexCoordRole, Params.IndexCoordCfg.Address, true, true)
i.session.SetEnableActiveStandBy(i.enableActiveStandBy)
Params.SetLogger(i.session.ServerID)
i.serverID = i.session.ServerID
return nil
@ -281,8 +289,18 @@ func (i *IndexCoord) Start() error {
Params.IndexCoordCfg.CreatedTime = time.Now()
Params.IndexCoordCfg.UpdatedTime = time.Now()
i.UpdateStateCode(internalpb.StateCode_Healthy)
log.Debug("IndexCoord start successfully", zap.Any("State", i.stateCode.Load()))
if i.enableActiveStandBy {
i.activateFunc = func() {
log.Info("IndexCoord switch from standby to active, reload the KV")
i.metaTable.reloadFromKV()
i.UpdateStateCode(internalpb.StateCode_Healthy)
}
i.UpdateStateCode(internalpb.StateCode_StandBy)
log.Info("IndexCoord start successfully", zap.Any("state", i.stateCode.Load()))
} else {
i.UpdateStateCode(internalpb.StateCode_Healthy)
log.Info("IndexCoord start successfully", zap.Any("state", i.stateCode.Load()))
}
return startErr
}

16
internal/indexcoord/index_coord_test.go
View File

@ -42,9 +42,8 @@ import (
"github.com/milvus-io/milvus/internal/util/sessionutil"
)
func TestIndexCoord(t *testing.T) {
func testIndexCoord(t *testing.T) {
ctx := context.Background()
Params.InitOnce()
Params.EtcdCfg.MetaRootPath = "indexcoord-ut"
// first start an IndexNode
@ -298,6 +297,19 @@ func TestIndexCoord(t *testing.T) {
assert.NoError(t, err)
}
func TestIndexCoord_DisableActiveStandby(t *testing.T) {
Params.InitOnce()
Params.IndexCoordCfg.EnableActiveStandby = false
testIndexCoord(t)
}
// make sure the main functions work well when EnableActiveStandby=true
func TestIndexCoord_EnableActiveStandby(t *testing.T) {
Params.InitOnce()
Params.IndexCoordCfg.EnableActiveStandby = true
testIndexCoord(t)
}
func TestIndexCoord_GetComponentStates(t *testing.T) {
ic := &IndexCoord{}
ic.stateCode.Store(internalpb.StateCode_Healthy)

1
internal/proto/internal.proto
View File

@ -9,6 +9,7 @@ enum StateCode {
Initializing = 0;
Healthy = 1;
Abnormal = 2;
StandBy = 3;
}
message ComponentInfo {

300
internal/proto/internalpb/internal.pb.go
View File

@ -28,18 +28,21 @@ const (
StateCode_Initializing StateCode = 0
StateCode_Healthy StateCode = 1
StateCode_Abnormal StateCode = 2
StateCode_StandBy StateCode = 3
)
var StateCode_name = map[int32]string{
0: "Initializing",
1: "Healthy",
2: "Abnormal",
3: "StandBy",
}
var StateCode_value = map[string]int32{
"Initializing": 0,
"Healthy": 1,
"Abnormal": 2,
"StandBy": 3,
}
func (x StateCode) String() string {
@ -2836,152 +2839,153 @@ func init() {
func init() { proto.RegisterFile("internal.proto", fileDescriptor_41f4a519b878ee3b) }
var fileDescriptor_41f4a519b878ee3b = []byte{
// 2347 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x59, 0xcd, 0x6f, 0x1c, 0x49,
0x15, 0xdf, 0x9e, 0x9e, 0xf1, 0xcc, 0xbc, 0x19, 0x8f, 0xdb, 0x15, 0x67, 0x77, 0xe2, 0x24, 0x1b,
0xa7, 0x59, 0xc0, 0x24, 0x6c, 0x12, 0xbc, 0xbb, 0xc9, 0x0a, 0x10, 0x21, 0xf6, 0x64, 0x83, 0x15,
0x3b, 0x38, 0xed, 0x10, 0x09, 0x2e, 0xad, 0x9a, 0xe9, 0xf2, 0x4c, 0x93, 0xee, 0xae, 0x4e, 0x55,
0xb5, 0x9d, 0xc9, 0x89, 0x03, 0x27, 0x56, 0x70, 0xe3, 0x82, 0x04, 0x67, 0x84, 0x84, 0xc4, 0x6d,
0x6f, 0x20, 0x71, 0xe2, 0xc4, 0x89, 0x0b, 0xff, 0x0a, 0xe2, 0x80, 0xea, 0xa3, 0x7b, 0x3e, 0x3c,
0x76, 0x6c, 0x47, 0xbb, 0x1b, 0xa4, 0xbd, 0x75, 0xbd, 0xf7, 0xea, 0xf5, 0xab, 0xf7, 0x7e, 0xef,
0xd5, 0x7b, 0xdd, 0xd0, 0x0a, 0x13, 0x41, 0x58, 0x82, 0xa3, 0x1b, 0x29, 0xa3, 0x82, 0xa2, 0xf3,
0x71, 0x18, 0xed, 0x67, 0x5c, 0xaf, 0x6e, 0xe4, 0xcc, 0xe5, 0x66, 0x8f, 0xc6, 0x31, 0x4d, 0x34,
0x79, 0xb9, 0xc9, 0x7b, 0x03, 0x12, 0x63, 0xbd, 0x72, 0xff, 0x6a, 0xc1, 0xfc, 0x06, 0x8d, 0x53,
0x9a, 0x90, 0x44, 0x6c, 0x26, 0x7b, 0x14, 0xbd, 0x0d, 0x73, 0x09, 0x0d, 0xc8, 0x66, 0xa7, 0x6d,
0xad, 0x58, 0xab, 0xb6, 0x67, 0x56, 0x08, 0x41, 0x99, 0xd1, 0x88, 0xb4, 0x4b, 0x2b, 0xd6, 0x6a,
0xdd, 0x53, 0xcf, 0xe8, 0x2e, 0x00, 0x17, 0x58, 0x10, 0xbf, 0x47, 0x03, 0xd2, 0xb6, 0x57, 0xac,
0xd5, 0xd6, 0xda, 0xca, 0x8d, 0x99, 0x56, 0xdc, 0xd8, 0x95, 0x82, 0x1b, 0x34, 0x20, 0x5e, 0x9d,
0xe7, 0x8f, 0xe8, 0x87, 0x00, 0xe4, 0x85, 0x60, 0xd8, 0x0f, 0x93, 0x3d, 0xda, 0x2e, 0xaf, 0xd8,
0xab, 0x8d, 0xb5, 0xab, 0x93, 0x0a, 0x8c, 0xf1, 0x0f, 0xc9, 0xf0, 0x29, 0x8e, 0x32, 0xb2, 0x83,
0x43, 0xe6, 0xd5, 0xd5, 0x26, 0x69, 0xae, 0xfb, 0x6f, 0x0b, 0x16, 0x8a, 0x03, 0xa8, 0x77, 0x70,
0xf4, 0x5d, 0xa8, 0xa8, 0x57, 0xa8, 0x13, 0x34, 0xd6, 0xde, 0x3b, 0xc2, 0xa2, 0x89, 0x73, 0x7b,
0x7a, 0x0b, 0xfa, 0x09, 0x9c, 0xe3, 0x59, 0xb7, 0x97, 0xb3, 0x7c, 0x45, 0xe5, 0xed, 0x92, 0x32,
0xed, 0x64, 0x9a, 0xd0, 0xb8, 0x02, 0x63, 0xd2, 0x07, 0x30, 0x27, 0x35, 0x65, 0x5c, 0x79, 0xa9,
0xb1, 0x76, 0x71, 0xe6, 0x21, 0x77, 0x95, 0x88, 0x67, 0x44, 0xdd, 0x8b, 0x70, 0xe1, 0x01, 0x11,
0x53, 0xa7, 0xf3, 0xc8, 0xf3, 0x8c, 0x70, 0x61, 0x98, 0x4f, 0xc2, 0x98, 0x3c, 0x09, 0x7b, 0xcf,
0x36, 0x06, 0x38, 0x49, 0x48, 0x94, 0x33, 0x2f, 0xc3, 0xc5, 0x07, 0x44, 0x6d, 0x08, 0xb9, 0x08,
0x7b, 0x7c, 0x8a, 0x7d, 0x1e, 0xce, 0x3d, 0x20, 0xa2, 0x13, 0x4c, 0x91, 0x9f, 0x42, 0xed, 0x91,
0x0c, 0xb6, 0x84, 0xc1, 0x6d, 0xa8, 0xe2, 0x20, 0x60, 0x84, 0x73, 0xe3, 0xc5, 0x4b, 0x33, 0x2d,
0xbe, 0xa7, 0x65, 0xbc, 0x5c, 0x78, 0x16, 0x4c, 0xdc, 0x9f, 0x03, 0x6c, 0x26, 0xa1, 0xd8, 0xc1,
0x0c, 0xc7, 0xfc, 0x48, 0x80, 0x75, 0xa0, 0xc9, 0x05, 0x66, 0xc2, 0x4f, 0x95, 0x9c, 0x71, 0xf9,
0x09, 0xd0, 0xd0, 0x50, 0xdb, 0xb4, 0x76, 0xf7, 0xa7, 0x00, 0xbb, 0x82, 0x85, 0x49, 0x7f, 0x2b,
0xe4, 0x42, 0xbe, 0x6b, 0x5f, 0xca, 0xc9, 0x43, 0xd8, 0xab, 0x75, 0xcf, 0xac, 0xc6, 0xc2, 0x51,
0x3a, 0x79, 0x38, 0xee, 0x42, 0x23, 0x77, 0xf7, 0x36, 0xef, 0xa3, 0x5b, 0x50, 0xee, 0x62, 0x4e,
0x8e, 0x75, 0xcf, 0x36, 0xef, 0xaf, 0x63, 0x4e, 0x3c, 0x25, 0xe9, 0xfe, 0xb9, 0x04, 0x4b, 0x13,
0x61, 0x31, 0x8e, 0x3f, 0xbd, 0x2a, 0xe9, 0xe6, 0xa0, 0xbb, 0xd9, 0x51, 0xe6, 0xdb, 0x9e, 0x7a,
0x46, 0x2e, 0x34, 0x7b, 0x34, 0x8a, 0x48, 0x4f, 0x84, 0x34, 0xd9, 0xec, 0x28, 0xa4, 0xd9, 0xde,
0x04, 0x4d, 0xca, 0xa4, 0x98, 0x89, 0x50, 0x2f, 0xb9, 0x4a, 0x39, 0xdb, 0x9b, 0xa0, 0xa1, 0x6f,
0x81, 0x23, 0x18, 0xde, 0x27, 0x91, 0x2f, 0xc2, 0x98, 0x70, 0x81, 0xe3, 0xb4, 0x5d, 0x59, 0xb1,
0x56, 0xcb, 0xde, 0x82, 0xa6, 0x3f, 0xc9, 0xc9, 0xe8, 0x26, 0x9c, 0xeb, 0x67, 0x98, 0xe1, 0x44,
0x10, 0x32, 0x26, 0x3d, 0xa7, 0xa4, 0x51, 0xc1, 0x1a, 0x6d, 0xb8, 0x0e, 0x8b, 0x52, 0x8c, 0x66,
0x62, 0x4c, 0xbc, 0xaa, 0xc4, 0x1d, 0xc3, 0x28, 0x84, 0xdd, 0xcf, 0x2c, 0x38, 0x3f, 0xe5, 0x2f,
0x9e, 0xd2, 0x84, 0x93, 0x33, 0x38, 0xec, 0x2c, 0x11, 0x47, 0x77, 0x74, 0x21, 0x91, 0x49, 0x7b,
0x42, 0x2c, 0x6a, 0x79, 0xf7, 0x57, 0x36, 0xbc, 0xb3, 0xc1, 0x88, 0x2a, 0x73, 0xb9, 0xf7, 0xcf,
0x1e, 0xec, 0x77, 0xa0, 0x1a, 0x74, 0xfd, 0x04, 0xc7, 0x79, 0x5a, 0xcd, 0x05, 0xdd, 0x47, 0x38,
0x26, 0xe8, 0x1b, 0xd0, 0x1a, 0x45, 0x57, 0x52, 0x54, 0xcc, 0xeb, 0xde, 0x14, 0x15, 0xbd, 0x07,
0xf3, 0x45, 0x84, 0x95, 0x58, 0x59, 0x89, 0x4d, 0x12, 0x0b, 0x4c, 0x55, 0x8e, 0xc1, 0xd4, 0xdc,
0x0c, 0x4c, 0xad, 0x40, 0x63, 0x0c, 0x3f, 0x2a, 0x9a, 0xb6, 0x37, 0x4e, 0x92, 0x69, 0xa8, 0x6f,
0x9d, 0x76, 0x6d, 0xc5, 0x5a, 0x6d, 0x7a, 0x66, 0x85, 0x6e, 0xc1, 0xb9, 0xfd, 0x90, 0x89, 0x0c,
0x47, 0xa6, 0x12, 0x49, 0x3b, 0x78, 0xbb, 0xae, 0x72, 0x75, 0x16, 0x0b, 0xad, 0xc1, 0x52, 0x3a,
0x18, 0xf2, 0xb0, 0x37, 0xb5, 0x05, 0xd4, 0x96, 0x99, 0x3c, 0xf7, 0xef, 0x16, 0x9c, 0xef, 0x30,
0x9a, 0xbe, 0x11, 0xa1, 0xc8, 0x9d, 0x5c, 0x3e, 0xc6, 0xc9, 0x95, 0xc3, 0x4e, 0x76, 0x7f, 0x5d,
0x82, 0xb7, 0x35, 0xa2, 0x76, 0x72, 0xc7, 0x7e, 0x0e, 0xa7, 0xf8, 0x26, 0x2c, 0x8c, 0xde, 0xaa,
0x05, 0x66, 0x1f, 0xe3, 0xeb, 0xd0, 0x2a, 0x02, 0xac, 0xe5, 0xbe, 0x58, 0x48, 0xb9, 0x9f, 0x96,
0x60, 0x49, 0x06, 0xf5, 0x2b, 0x6f, 0x48, 0x6f, 0xfc, 0xc1, 0x02, 0xa4, 0xd1, 0x71, 0x2f, 0x0a,
0x31, 0xff, 0x32, 0x7d, 0xb1, 0x04, 0x15, 0x2c, 0x6d, 0x30, 0x2e, 0xd0, 0x0b, 0x97, 0x83, 0x23,
0xa3, 0xf5, 0x79, 0x59, 0x57, 0xbc, 0xd4, 0x1e, 0x7f, 0xe9, 0xef, 0x2d, 0x58, 0xbc, 0x17, 0x09,
0xc2, 0xde, 0x50, 0xa7, 0xfc, 0xad, 0x94, 0x47, 0x6d, 0x33, 0x09, 0xc8, 0x8b, 0x2f, 0xd3, 0xc0,
0xcb, 0x00, 0x7b, 0x21, 0x89, 0x82, 0x71, 0xf4, 0xd6, 0x15, 0xe5, 0xb5, 0x90, 0xdb, 0x86, 0xaa,
0x52, 0x52, 0xa0, 0x36, 0x5f, 0xca, 0x6e, 0x4f, 0x77, 0xfe, 0xa6, 0xdb, 0xab, 0x9d, 0xb8, 0xdb,
0x53, 0xdb, 0x4c, 0xb7, 0xf7, 0xcf, 0x32, 0xcc, 0x6f, 0x26, 0x9c, 0x30, 0x71, 0x76, 0xe7, 0x5d,
0x82, 0x3a, 0x1f, 0x60, 0xa6, 0x0e, 0x6a, 0xdc, 0x37, 0x22, 0x8c, 0xbb, 0xd6, 0x7e, 0x95, 0x6b,
0xcb, 0x27, 0x2c, 0x0e, 0x95, 0xe3, 0x8a, 0xc3, 0xdc, 0x31, 0x2e, 0xae, 0xbe, 0xba, 0x38, 0xd4,
0x0e, 0xdf, 0xbe, 0xf2, 0x80, 0xa4, 0x1f, 0xcb, 0xf1, 0xa4, 0xd3, 0xae, 0x2b, 0xfe, 0x88, 0x80,
0xde, 0x05, 0x28, 0x3a, 0x31, 0x7d, 0x8f, 0x96, 0xbd, 0x31, 0x8a, 0xbc, 0xbb, 0x19, 0x3d, 0x90,
0xbd, 0x62, 0x43, 0xf5, 0x8a, 0x66, 0x85, 0x3e, 0x84, 0x1a, 0xa3, 0x07, 0x7e, 0x80, 0x05, 0x6e,
0x37, 0x55, 0xf0, 0x2e, 0xcc, 0x74, 0xf6, 0x7a, 0x44, 0xbb, 0x5e, 0x95, 0xd1, 0x83, 0x0e, 0x16,
0x18, 0xdd, 0x85, 0x86, 0x42, 0x00, 0xd7, 0x1b, 0xe7, 0xd5, 0xc6, 0x77, 0x27, 0x37, 0x9a, 0x01,
0xf5, 0x13, 0x29, 0x27, 0x37, 0x79, 0x1a, 0x9a, 0x5c, 0x29, 0xb8, 0x00, 0xb5, 0x24, 0x8b, 0x7d,
0x46, 0x0f, 0x78, 0xbb, 0xa5, 0xfa, 0xc6, 0x6a, 0x92, 0xc5, 0x1e, 0x3d, 0xe0, 0x68, 0x1d, 0xaa,
0xfb, 0x84, 0xf1, 0x90, 0x26, 0xed, 0x05, 0x35, 0x8a, 0xae, 0x1e, 0x31, 0xae, 0x69, 0xc4, 0x48,
0x75, 0x4f, 0xb5, 0xbc, 0x97, 0x6f, 0x74, 0xff, 0x55, 0x86, 0xf9, 0x5d, 0x82, 0x59, 0x6f, 0x70,
0x76, 0x40, 0x2d, 0x41, 0x85, 0x91, 0xe7, 0x45, 0x73, 0xae, 0x17, 0x45, 0x7c, 0xed, 0x63, 0xe2,
0x5b, 0x3e, 0x41, 0xc7, 0x5e, 0x99, 0xd1, 0xb1, 0x3b, 0x60, 0x07, 0x3c, 0x52, 0xd0, 0xa9, 0x7b,
0xf2, 0x51, 0xf6, 0xd9, 0x69, 0x84, 0x7b, 0x64, 0x40, 0xa3, 0x80, 0x30, 0xbf, 0xcf, 0x68, 0xa6,
0xfb, 0xec, 0xa6, 0xe7, 0x8c, 0x31, 0x1e, 0x48, 0x3a, 0xba, 0x03, 0xb5, 0x80, 0x47, 0xbe, 0x18,
0xa6, 0x44, 0xe1, 0xa7, 0x75, 0xc4, 0x31, 0x3b, 0x3c, 0x7a, 0x32, 0x4c, 0x89, 0x57, 0x0d, 0xf4,
0x03, 0xba, 0x05, 0x4b, 0x9c, 0xb0, 0x10, 0x47, 0xe1, 0x4b, 0x12, 0xf8, 0xe4, 0x45, 0xca, 0xfc,
0x34, 0xc2, 0x89, 0x02, 0x59, 0xd3, 0x43, 0x23, 0xde, 0xfd, 0x17, 0x29, 0xdb, 0x89, 0x70, 0x82,
0x56, 0xc1, 0xa1, 0x99, 0x48, 0x33, 0xe1, 0x1b, 0x18, 0x84, 0x81, 0xc2, 0x9c, 0xed, 0xb5, 0x34,
0x5d, 0x45, 0x9d, 0x6f, 0x06, 0x33, 0xa7, 0x90, 0xc6, 0xa9, 0xa6, 0x90, 0xe6, 0xe9, 0xa6, 0x90,
0xf9, 0xd9, 0x53, 0x08, 0x6a, 0x41, 0x29, 0x79, 0xae, 0xb0, 0x66, 0x7b, 0xa5, 0xe4, 0xb9, 0x0c,
0xa4, 0xa0, 0xe9, 0x33, 0x85, 0x31, 0xdb, 0x53, 0xcf, 0x32, 0x89, 0x62, 0x22, 0x58, 0xd8, 0x93,
0x6e, 0x69, 0x3b, 0x2a, 0x0e, 0x63, 0x14, 0xf7, 0xbf, 0xf6, 0x08, 0x56, 0x3c, 0x8b, 0x04, 0xff,
0xa2, 0x26, 0x98, 0x02, 0x8b, 0xf6, 0x38, 0x16, 0xaf, 0x40, 0x43, 0x1b, 0xa7, 0x63, 0x5e, 0x9e,
0xb6, 0x57, 0x0a, 0xc8, 0x2c, 0x7b, 0x9e, 0x11, 0x16, 0x12, 0x6e, 0xca, 0x3e, 0x24, 0x59, 0xfc,
0x58, 0x53, 0xd0, 0x39, 0xa8, 0x08, 0x9a, 0xfa, 0xcf, 0xf2, 0x72, 0x25, 0x68, 0xfa, 0x10, 0x7d,
0x1f, 0x96, 0x39, 0xc1, 0x11, 0x09, 0xfc, 0xa2, 0xbc, 0x70, 0x9f, 0xab, 0x63, 0x93, 0xa0, 0x5d,
0x55, 0x61, 0x6e, 0x6b, 0x89, 0xdd, 0x42, 0x60, 0xd7, 0xf0, 0x65, 0x14, 0x7b, 0xba, 0x6d, 0x9f,
0xd8, 0x56, 0x53, 0x9d, 0x3d, 0x1a, 0xb1, 0x8a, 0x0d, 0x1f, 0x43, 0xbb, 0x1f, 0xd1, 0x2e, 0x8e,
0xfc, 0x43, 0x6f, 0x55, 0x23, 0x84, 0xed, 0xbd, 0xad, 0xf9, 0xbb, 0x53, 0xaf, 0x94, 0xc7, 0xe3,
0x51, 0xd8, 0x23, 0x81, 0xdf, 0x8d, 0x68, 0xb7, 0x0d, 0x0a, 0xae, 0xa0, 0x49, 0xb2, 0x5e, 0x49,
0x98, 0x1a, 0x01, 0xe9, 0x86, 0x1e, 0xcd, 0x12, 0xa1, 0xc0, 0x67, 0x7b, 0x2d, 0x4d, 0x7f, 0x94,
0xc5, 0x1b, 0x92, 0x8a, 0xbe, 0x06, 0xf3, 0x46, 0x92, 0xee, 0xed, 0x71, 0x22, 0x14, 0xea, 0x6c,
0xaf, 0xa9, 0x89, 0x3f, 0x56, 0x34, 0xf7, 0x2f, 0x36, 0x2c, 0x78, 0xd2, 0xbb, 0x64, 0x9f, 0xfc,
0x3f, 0xd5, 0x95, 0xa3, 0xf2, 0x7b, 0xee, 0x54, 0xf9, 0x5d, 0x3d, 0x71, 0x7e, 0xd7, 0x4e, 0x95,
0xdf, 0xf5, 0xd3, 0xe5, 0x37, 0x1c, 0x91, 0xdf, 0x4b, 0x50, 0x89, 0xc2, 0x38, 0xcc, 0x03, 0xac,
0x17, 0xee, 0x1f, 0x27, 0x42, 0xf6, 0x06, 0xe4, 0xec, 0x35, 0xb0, 0xc3, 0x40, 0x37, 0x90, 0x8d,
0xb5, 0xf6, 0xcc, 0x1b, 0x73, 0xb3, 0xc3, 0x3d, 0x29, 0x34, 0x7d, 0xcb, 0x56, 0x4e, 0x7d, 0xcb,
0xfe, 0x00, 0x2e, 0x1e, 0xce, 0x64, 0x66, 0xdc, 0x11, 0xb4, 0xe7, 0x54, 0x44, 0x2f, 0x4c, 0xa7,
0x72, 0xee, 0xaf, 0x00, 0x7d, 0x07, 0x96, 0xc6, 0x72, 0x79, 0xb4, 0xb1, 0xaa, 0x27, 0xfb, 0x11,
0x6f, 0xb4, 0xe5, 0xb8, 0x6c, 0xae, 0x1d, 0x97, 0xcd, 0xee, 0x3f, 0x6c, 0x98, 0xef, 0x90, 0x88,
0x08, 0xf2, 0x55, 0x13, 0x78, 0x64, 0x13, 0xf8, 0x6d, 0x40, 0x61, 0x22, 0x6e, 0x7f, 0xe8, 0xa7,
0x2c, 0x8c, 0x31, 0x1b, 0xfa, 0xcf, 0xc8, 0x30, 0x2f, 0x93, 0x8e, 0xe2, 0xec, 0x68, 0xc6, 0x43,
0x32, 0xe4, 0xaf, 0x6c, 0x0a, 0xc7, 0xbb, 0x30, 0x9d, 0x36, 0x45, 0x17, 0xf6, 0x3d, 0x68, 0x4e,
0xbc, 0xa2, 0xf9, 0x0a, 0xc0, 0x36, 0xd2, 0xd1, 0x7b, 0xdd, 0xff, 0x58, 0x50, 0xdf, 0xa2, 0x38,
0x50, 0xf3, 0xd0, 0x19, 0xc3, 0x58, 0xb4, 0xba, 0xa5, 0xe9, 0x56, 0xf7, 0x12, 0x8c, 0x46, 0x1a,
0x13, 0xc8, 0xb1, 0x19, 0x67, 0x6c, 0x56, 0x29, 0x4f, 0xce, 0x2a, 0x57, 0xa0, 0x11, 0x4a, 0x83,
0xfc, 0x14, 0x8b, 0x81, 0xae, 0x94, 0x75, 0x0f, 0x14, 0x69, 0x47, 0x52, 0xe4, 0x30, 0x93, 0x0b,
0xa8, 0x61, 0x66, 0xee, 0xc4, 0xc3, 0x8c, 0x51, 0xa2, 0x86, 0x99, 0x5f, 0x5a, 0x00, 0xea, 0xe0,
0xb2, 0x1e, 0x1c, 0x56, 0x6a, 0x9d, 0x45, 0xa9, 0x2c, 0xe1, 0x2a, 0x52, 0x24, 0xc2, 0x62, 0x94,
0x54, 0xdc, 0x38, 0x07, 0xc9, 0xa8, 0x69, 0x96, 0x49, 0x28, 0xee, 0xfe, 0xc6, 0x02, 0x50, 0x55,
0x41, 0x9b, 0x31, 0x0d, 0x3f, 0xeb, 0xf8, 0x31, 0xaf, 0x34, 0xe9, 0xba, 0xf5, 0xdc, 0x75, 0xc7,
0x7c, 0x47, 0x1d, 0xeb, 0xcb, 0xf3, 0xc3, 0x1b, 0xef, 0xaa, 0x67, 0xf7, 0xb7, 0x16, 0x34, 0x8d,
0x75, 0xda, 0xa4, 0x89, 0x28, 0x5b, 0xd3, 0x51, 0x56, 0xcd, 0x4d, 0x4c, 0xd9, 0xd0, 0xe7, 0xe1,
0x4b, 0x62, 0x0c, 0x02, 0x4d, 0xda, 0x0d, 0x5f, 0x92, 0x09, 0xf0, 0xda, 0x93, 0xe0, 0xbd, 0x0e,
0x8b, 0x8c, 0xf4, 0x48, 0x22, 0xa2, 0xa1, 0x1f, 0xd3, 0x20, 0xdc, 0x0b, 0x49, 0xa0, 0xd0, 0x50,
0xf3, 0x9c, 0x9c, 0xb1, 0x6d, 0xe8, 0xee, 0x2f, 0x2c, 0x68, 0x6c, 0xf3, 0xfe, 0x0e, 0xe5, 0x2a,
0xc9, 0xd0, 0x55, 0x68, 0x9a, 0xc2, 0xa6, 0x33, 0xdc, 0x52, 0x08, 0x6b, 0xf4, 0x46, 0xdf, 0x22,
0x65, 0x69, 0x8f, 0x79, 0xdf, 0xb8, 0xa9, 0xe9, 0xe9, 0x05, 0x5a, 0x86, 0x5a, 0xcc, 0xfb, 0xaa,
0x17, 0x37, 0xb0, 0x2c, 0xd6, 0xf2, 0xac, 0xa3, 0x2b, 0xac, 0xac, 0xae, 0xb0, 0x11, 0xc1, 0xfd,
0xcc, 0x02, 0x64, 0xbe, 0x75, 0xbe, 0xd6, 0xaf, 0x09, 0x15, 0xe5, 0xf1, 0xef, 0xa9, 0x25, 0x85,
0xf1, 0x09, 0xda, 0x54, 0x51, 0xb0, 0x0f, 0x15, 0x85, 0xeb, 0xb0, 0x18, 0x90, 0x3d, 0x9c, 0x45,
0xe3, 0xb7, 0xae, 0x36, 0xd9, 0x31, 0x8c, 0x89, 0x6f, 0xfb, 0xad, 0x0d, 0x46, 0x02, 0x92, 0x88,
0x10, 0x47, 0xea, 0x97, 0xd3, 0x32, 0xd4, 0x32, 0x2e, 0x91, 0x50, 0xf8, 0xae, 0x58, 0xa3, 0xf7,
0x01, 0x91, 0xa4, 0xc7, 0x86, 0xa9, 0x04, 0x71, 0x8a, 0x39, 0x3f, 0xa0, 0x2c, 0x30, 0x85, 0x7a,
0xb1, 0xe0, 0xec, 0x18, 0x86, 0x1c, 0x5a, 0x05, 0x49, 0x70, 0x22, 0xf2, 0x7a, 0xad, 0x57, 0x32,
0xf4, 0x21, 0xf7, 0x79, 0x96, 0x12, 0x66, 0xc2, 0x5a, 0x0d, 0xf9, 0xae, 0x5c, 0xca, 0x52, 0xce,
0x07, 0x78, 0xed, 0xa3, 0xdb, 0x23, 0xf5, 0xba, 0x44, 0xb7, 0x34, 0x39, 0xd7, 0xed, 0xde, 0x87,
0xc5, 0xad, 0x90, 0x8b, 0x1d, 0x1a, 0x85, 0xbd, 0xe1, 0x99, 0x6f, 0x1c, 0xf7, 0x53, 0x0b, 0xd0,
0xb8, 0x1e, 0xf3, 0x67, 0x63, 0xd4, 0x31, 0x58, 0x27, 0xef, 0x18, 0xae, 0x42, 0x33, 0x55, 0x6a,
0xd4, 0x7f, 0xd4, 0x3c, 0x7a, 0x0d, 0x4d, 0x93, 0xbe, 0xe5, 0xe8, 0x32, 0x80, 0x74, 0xa6, 0xcf,
0x68, 0x44, 0x74, 0xf0, 0xea, 0x5e, 0x5d, 0x52, 0x3c, 0x49, 0x70, 0xfb, 0x70, 0x61, 0x77, 0x40,
0x0f, 0x36, 0x68, 0xb2, 0x17, 0xf6, 0x33, 0x86, 0x25, 0xa0, 0x5f, 0xe3, 0x8b, 0x59, 0x1b, 0xaa,
0x29, 0x16, 0x32, 0xad, 0x4d, 0x8c, 0xf2, 0xa5, 0xfb, 0x3b, 0x0b, 0x96, 0x67, 0xbd, 0xe9, 0x75,
0x8e, 0xff, 0x00, 0xe6, 0x7b, 0x5a, 0x9d, 0xd6, 0x76, 0xf2, 0x5f, 0x87, 0x93, 0xfb, 0xdc, 0xfb,
0x50, 0xf6, 0xb0, 0x20, 0xe8, 0x26, 0x94, 0x98, 0x50, 0x16, 0xb4, 0xd6, 0xae, 0x1c, 0x51, 0xac,
0xa4, 0xa0, 0x9a, 0x86, 0x4b, 0x4c, 0xa0, 0x26, 0x58, 0x4c, 0x9d, 0xd4, 0xf2, 0x2c, 0x76, 0xed,
0x63, 0xa8, 0x17, 0x7f, 0xbb, 0x91, 0x03, 0xcd, 0xcd, 0x24, 0x14, 0xaa, 0x51, 0x0e, 0x93, 0xbe,
0xf3, 0x16, 0x6a, 0x40, 0xf5, 0x47, 0x04, 0x47, 0x62, 0x30, 0x74, 0x2c, 0xd4, 0x84, 0xda, 0xbd,
0x6e, 0x42, 0x59, 0x8c, 0x23, 0xa7, 0x74, 0x6d, 0x0d, 0x16, 0x0f, 0x7d, 0x9c, 0x90, 0x22, 0x1e,
0x3d, 0x90, 0xde, 0x0d, 0x9c, 0xb7, 0xd0, 0x02, 0x34, 0x36, 0x68, 0x94, 0xc5, 0x89, 0x26, 0x58,
0xd7, 0xfe, 0x64, 0x41, 0x2d, 0x37, 0x06, 0x2d, 0xc2, 0x7c, 0xa7, 0xb3, 0x35, 0xfa, 0xd3, 0xe1,
0xbc, 0x25, 0x0d, 0xe8, 0x74, 0xb6, 0x8a, 0xef, 0xe4, 0xfa, 0x9d, 0x9d, 0xce, 0x96, 0xaa, 0xb6,
0x4e, 0xc9, 0xac, 0x3e, 0x89, 0x32, 0x3e, 0x70, 0xec, 0x42, 0x41, 0x9c, 0x62, 0xad, 0xa0, 0x8c,
0xe6, 0xa1, 0xde, 0xd9, 0xde, 0xd2, 0x76, 0x39, 0x15, 0xb3, 0xd4, 0x0d, 0x97, 0x33, 0x27, 0xed,
0xe9, 0x6c, 0x6f, 0xad, 0x67, 0xd1, 0x33, 0x79, 0x71, 0x3b, 0x55, 0xc5, 0x7f, 0xbc, 0xa5, 0xa7,
0x34, 0xa7, 0xa6, 0xd4, 0x3f, 0xde, 0x92, 0x73, 0xe3, 0xd0, 0xa9, 0xaf, 0xdf, 0xf9, 0xd9, 0x47,
0xfd, 0x50, 0x0c, 0xb2, 0xae, 0x0c, 0xc7, 0x4d, 0xed, 0xd9, 0xf7, 0x43, 0x6a, 0x9e, 0x6e, 0xe6,
0xde, 0xbd, 0xa9, 0x9c, 0x5d, 0x2c, 0xd3, 0x6e, 0x77, 0x4e, 0x51, 0x3e, 0xf8, 0x5f, 0x00, 0x00,
0x00, 0xff, 0xff, 0x31, 0x5e, 0x70, 0xe6, 0xf4, 0x20, 0x00, 0x00,
// 2356 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x59, 0xcf, 0x6f, 0x1c, 0x49,
0xf5, 0xdf, 0x9e, 0x9e, 0xf1, 0xcc, 0xbc, 0x19, 0x8f, 0xdb, 0x15, 0x27, 0x3b, 0x71, 0x92, 0x8d,
0xd3, 0xdf, 0xfd, 0x82, 0x49, 0xd8, 0x24, 0x78, 0x77, 0x13, 0x04, 0x88, 0x10, 0x7b, 0xb2, 0xc1,
0x8a, 0x1d, 0x9c, 0x76, 0x88, 0x04, 0x97, 0x56, 0xcd, 0x74, 0x79, 0xa6, 0x49, 0x77, 0x57, 0xa7,
0xaa, 0xda, 0xce, 0xe4, 0xc4, 0x81, 0x13, 0x2b, 0xb8, 0x71, 0x41, 0x82, 0x33, 0x42, 0x42, 0xe2,
0xb6, 0x37, 0x90, 0x38, 0x71, 0xe2, 0xc4, 0x85, 0x7f, 0x05, 0x71, 0x40, 0xf5, 0xa3, 0x7b, 0x7e,
0x78, 0xec, 0xd8, 0x8e, 0x76, 0x37, 0x48, 0x7b, 0xeb, 0x7a, 0xef, 0xd5, 0xeb, 0x57, 0xef, 0x7d,
0xde, 0xab, 0xf7, 0xba, 0xa1, 0x15, 0x26, 0x82, 0xb0, 0x04, 0x47, 0x37, 0x53, 0x46, 0x05, 0x45,
0xe7, 0xe3, 0x30, 0xda, 0xcf, 0xb8, 0x5e, 0xdd, 0xcc, 0x99, 0xcb, 0xcd, 0x1e, 0x8d, 0x63, 0x9a,
0x68, 0xf2, 0x72, 0x93, 0xf7, 0x06, 0x24, 0xc6, 0x7a, 0xe5, 0xfe, 0xc5, 0x82, 0xf9, 0x0d, 0x1a,
0xa7, 0x34, 0x21, 0x89, 0xd8, 0x4c, 0xf6, 0x28, 0xba, 0x00, 0x73, 0x09, 0x0d, 0xc8, 0x66, 0xa7,
0x6d, 0xad, 0x58, 0xab, 0xb6, 0x67, 0x56, 0x08, 0x41, 0x99, 0xd1, 0x88, 0xb4, 0x4b, 0x2b, 0xd6,
0x6a, 0xdd, 0x53, 0xcf, 0xe8, 0x1e, 0x00, 0x17, 0x58, 0x10, 0xbf, 0x47, 0x03, 0xd2, 0xb6, 0x57,
0xac, 0xd5, 0xd6, 0xda, 0xca, 0xcd, 0x99, 0x56, 0xdc, 0xdc, 0x95, 0x82, 0x1b, 0x34, 0x20, 0x5e,
0x9d, 0xe7, 0x8f, 0xe8, 0x07, 0x00, 0xe4, 0xa5, 0x60, 0xd8, 0x0f, 0x93, 0x3d, 0xda, 0x2e, 0xaf,
0xd8, 0xab, 0x8d, 0xb5, 0x6b, 0x93, 0x0a, 0x8c, 0xf1, 0x8f, 0xc8, 0xf0, 0x19, 0x8e, 0x32, 0xb2,
0x83, 0x43, 0xe6, 0xd5, 0xd5, 0x26, 0x69, 0xae, 0xfb, 0x2f, 0x0b, 0x16, 0x8a, 0x03, 0xa8, 0x77,
0x70, 0xf4, 0x1d, 0xa8, 0xa8, 0x57, 0xa8, 0x13, 0x34, 0xd6, 0xde, 0x3f, 0xc2, 0xa2, 0x89, 0x73,
0x7b, 0x7a, 0x0b, 0xfa, 0x31, 0x9c, 0xe3, 0x59, 0xb7, 0x97, 0xb3, 0x7c, 0x45, 0xe5, 0xed, 0x92,
0x32, 0xed, 0x64, 0x9a, 0xd0, 0xb8, 0x02, 0x63, 0xd2, 0x87, 0x30, 0x27, 0x35, 0x65, 0x5c, 0x79,
0xa9, 0xb1, 0x76, 0x69, 0xe6, 0x21, 0x77, 0x95, 0x88, 0x67, 0x44, 0xdd, 0x4b, 0x70, 0xf1, 0x21,
0x11, 0x53, 0xa7, 0xf3, 0xc8, 0x8b, 0x8c, 0x70, 0x61, 0x98, 0x4f, 0xc3, 0x98, 0x3c, 0x0d, 0x7b,
0xcf, 0x37, 0x06, 0x38, 0x49, 0x48, 0x94, 0x33, 0xaf, 0xc0, 0xa5, 0x87, 0x44, 0x6d, 0x08, 0xb9,
0x08, 0x7b, 0x7c, 0x8a, 0x7d, 0x1e, 0xce, 0x3d, 0x24, 0xa2, 0x13, 0x4c, 0x91, 0x9f, 0x41, 0xed,
0xb1, 0x0c, 0xb6, 0x84, 0xc1, 0x1d, 0xa8, 0xe2, 0x20, 0x60, 0x84, 0x73, 0xe3, 0xc5, 0xcb, 0x33,
0x2d, 0xbe, 0xaf, 0x65, 0xbc, 0x5c, 0x78, 0x16, 0x4c, 0xdc, 0x9f, 0x01, 0x6c, 0x26, 0xa1, 0xd8,
0xc1, 0x0c, 0xc7, 0xfc, 0x48, 0x80, 0x75, 0xa0, 0xc9, 0x05, 0x66, 0xc2, 0x4f, 0x95, 0x9c, 0x71,
0xf9, 0x09, 0xd0, 0xd0, 0x50, 0xdb, 0xb4, 0x76, 0xf7, 0x27, 0x00, 0xbb, 0x82, 0x85, 0x49, 0x7f,
0x2b, 0xe4, 0x42, 0xbe, 0x6b, 0x5f, 0xca, 0xc9, 0x43, 0xd8, 0xab, 0x75, 0xcf, 0xac, 0xc6, 0xc2,
0x51, 0x3a, 0x79, 0x38, 0xee, 0x41, 0x23, 0x77, 0xf7, 0x36, 0xef, 0xa3, 0xdb, 0x50, 0xee, 0x62,
0x4e, 0x8e, 0x75, 0xcf, 0x36, 0xef, 0xaf, 0x63, 0x4e, 0x3c, 0x25, 0xe9, 0xfe, 0xa9, 0x04, 0x4b,
0x13, 0x61, 0x31, 0x8e, 0x3f, 0xbd, 0x2a, 0xe9, 0xe6, 0xa0, 0xbb, 0xd9, 0x51, 0xe6, 0xdb, 0x9e,
0x7a, 0x46, 0x2e, 0x34, 0x7b, 0x34, 0x8a, 0x48, 0x4f, 0x84, 0x34, 0xd9, 0xec, 0x28, 0xa4, 0xd9,
0xde, 0x04, 0x4d, 0xca, 0xa4, 0x98, 0x89, 0x50, 0x2f, 0xb9, 0x4a, 0x39, 0xdb, 0x9b, 0xa0, 0xa1,
0x6f, 0x80, 0x23, 0x18, 0xde, 0x27, 0x91, 0x2f, 0xc2, 0x98, 0x70, 0x81, 0xe3, 0xb4, 0x5d, 0x59,
0xb1, 0x56, 0xcb, 0xde, 0x82, 0xa6, 0x3f, 0xcd, 0xc9, 0xe8, 0x16, 0x9c, 0xeb, 0x67, 0x98, 0xe1,
0x44, 0x10, 0x32, 0x26, 0x3d, 0xa7, 0xa4, 0x51, 0xc1, 0x1a, 0x6d, 0xb8, 0x01, 0x8b, 0x52, 0x8c,
0x66, 0x62, 0x4c, 0xbc, 0xaa, 0xc4, 0x1d, 0xc3, 0x28, 0x84, 0xdd, 0xcf, 0x2c, 0x38, 0x3f, 0xe5,
0x2f, 0x9e, 0xd2, 0x84, 0x93, 0x33, 0x38, 0xec, 0x2c, 0x11, 0x47, 0x77, 0x75, 0x21, 0x91, 0x49,
0x7b, 0x42, 0x2c, 0x6a, 0x79, 0xf7, 0x97, 0x36, 0xbc, 0xbb, 0xc1, 0x88, 0x2a, 0x73, 0xb9, 0xf7,
0xcf, 0x1e, 0xec, 0x77, 0xa1, 0x1a, 0x74, 0xfd, 0x04, 0xc7, 0x79, 0x5a, 0xcd, 0x05, 0xdd, 0xc7,
0x38, 0x26, 0xe8, 0x6b, 0xd0, 0x1a, 0x45, 0x57, 0x52, 0x54, 0xcc, 0xeb, 0xde, 0x14, 0x15, 0xbd,
0x0f, 0xf3, 0x45, 0x84, 0x95, 0x58, 0x59, 0x89, 0x4d, 0x12, 0x0b, 0x4c, 0x55, 0x8e, 0xc1, 0xd4,
0xdc, 0x0c, 0x4c, 0xad, 0x40, 0x63, 0x0c, 0x3f, 0x2a, 0x9a, 0xb6, 0x37, 0x4e, 0x92, 0x69, 0xa8,
0x6f, 0x9d, 0x76, 0x6d, 0xc5, 0x5a, 0x6d, 0x7a, 0x66, 0x85, 0x6e, 0xc3, 0xb9, 0xfd, 0x90, 0x89,
0x0c, 0x47, 0xa6, 0x12, 0x49, 0x3b, 0x78, 0xbb, 0xae, 0x72, 0x75, 0x16, 0x0b, 0xad, 0xc1, 0x52,
0x3a, 0x18, 0xf2, 0xb0, 0x37, 0xb5, 0x05, 0xd4, 0x96, 0x99, 0x3c, 0xf7, 0x6f, 0x16, 0x9c, 0xef,
0x30, 0x9a, 0xbe, 0x15, 0xa1, 0xc8, 0x9d, 0x5c, 0x3e, 0xc6, 0xc9, 0x95, 0xc3, 0x4e, 0x76, 0x7f,
0x55, 0x82, 0x0b, 0x1a, 0x51, 0x3b, 0xb9, 0x63, 0x3f, 0x87, 0x53, 0x7c, 0x1d, 0x16, 0x46, 0x6f,
0xd5, 0x02, 0xb3, 0x8f, 0xf1, 0xff, 0xd0, 0x2a, 0x02, 0xac, 0xe5, 0xbe, 0x58, 0x48, 0xb9, 0x9f,
0x96, 0x60, 0x49, 0x06, 0xf5, 0x2b, 0x6f, 0x48, 0x6f, 0xfc, 0xde, 0x02, 0xa4, 0xd1, 0x71, 0x3f,
0x0a, 0x31, 0xff, 0x32, 0x7d, 0xb1, 0x04, 0x15, 0x2c, 0x6d, 0x30, 0x2e, 0xd0, 0x0b, 0x97, 0x83,
0x23, 0xa3, 0xf5, 0x79, 0x59, 0x57, 0xbc, 0xd4, 0x1e, 0x7f, 0xe9, 0xef, 0x2c, 0x58, 0xbc, 0x1f,
0x09, 0xc2, 0xde, 0x52, 0xa7, 0xfc, 0xb5, 0x94, 0x47, 0x6d, 0x33, 0x09, 0xc8, 0xcb, 0x2f, 0xd3,
0xc0, 0x2b, 0x00, 0x7b, 0x21, 0x89, 0x82, 0x71, 0xf4, 0xd6, 0x15, 0xe5, 0x8d, 0x90, 0xdb, 0x86,
0xaa, 0x52, 0x52, 0xa0, 0x36, 0x5f, 0xca, 0x6e, 0x4f, 0x77, 0xfe, 0xa6, 0xdb, 0xab, 0x9d, 0xb8,
0xdb, 0x53, 0xdb, 0x4c, 0xb7, 0xf7, 0x8f, 0x32, 0xcc, 0x6f, 0x26, 0x9c, 0x30, 0x71, 0x76, 0xe7,
0x5d, 0x86, 0x3a, 0x1f, 0x60, 0xa6, 0x0e, 0x6a, 0xdc, 0x37, 0x22, 0x8c, 0xbb, 0xd6, 0x7e, 0x9d,
0x6b, 0xcb, 0x27, 0x2c, 0x0e, 0x95, 0xe3, 0x8a, 0xc3, 0xdc, 0x31, 0x2e, 0xae, 0xbe, 0xbe, 0x38,
0xd4, 0x0e, 0xdf, 0xbe, 0xf2, 0x80, 0xa4, 0x1f, 0xcb, 0xf1, 0xa4, 0xd3, 0xae, 0x2b, 0xfe, 0x88,
0x80, 0xde, 0x03, 0x28, 0x3a, 0x31, 0x7d, 0x8f, 0x96, 0xbd, 0x31, 0x8a, 0xbc, 0xbb, 0x19, 0x3d,
0x90, 0xbd, 0x62, 0x43, 0xf5, 0x8a, 0x66, 0x85, 0x3e, 0x82, 0x1a, 0xa3, 0x07, 0x7e, 0x80, 0x05,
0x6e, 0x37, 0x55, 0xf0, 0x2e, 0xce, 0x74, 0xf6, 0x7a, 0x44, 0xbb, 0x5e, 0x95, 0xd1, 0x83, 0x0e,
0x16, 0x18, 0xdd, 0x83, 0x86, 0x42, 0x00, 0xd7, 0x1b, 0xe7, 0xd5, 0xc6, 0xf7, 0x26, 0x37, 0x9a,
0x01, 0xf5, 0x13, 0x29, 0x27, 0x37, 0x79, 0x1a, 0x9a, 0x5c, 0x29, 0xb8, 0x08, 0xb5, 0x24, 0x8b,
0x7d, 0x46, 0x0f, 0x78, 0xbb, 0xa5, 0xfa, 0xc6, 0x6a, 0x92, 0xc5, 0x1e, 0x3d, 0xe0, 0x68, 0x1d,
0xaa, 0xfb, 0x84, 0xf1, 0x90, 0x26, 0xed, 0x05, 0x35, 0x8a, 0xae, 0x1e, 0x31, 0xae, 0x69, 0xc4,
0x48, 0x75, 0xcf, 0xb4, 0xbc, 0x97, 0x6f, 0x74, 0xff, 0x59, 0x86, 0xf9, 0x5d, 0x82, 0x59, 0x6f,
0x70, 0x76, 0x40, 0x2d, 0x41, 0x85, 0x91, 0x17, 0x45, 0x73, 0xae, 0x17, 0x45, 0x7c, 0xed, 0x63,
0xe2, 0x5b, 0x3e, 0x41, 0xc7, 0x5e, 0x99, 0xd1, 0xb1, 0x3b, 0x60, 0x07, 0x3c, 0x52, 0xd0, 0xa9,
0x7b, 0xf2, 0x51, 0xf6, 0xd9, 0x69, 0x84, 0x7b, 0x64, 0x40, 0xa3, 0x80, 0x30, 0xbf, 0xcf, 0x68,
0xa6, 0xfb, 0xec, 0xa6, 0xe7, 0x8c, 0x31, 0x1e, 0x4a, 0x3a, 0xba, 0x0b, 0xb5, 0x80, 0x47, 0xbe,
0x18, 0xa6, 0x44, 0xe1, 0xa7, 0x75, 0xc4, 0x31, 0x3b, 0x3c, 0x7a, 0x3a, 0x4c, 0x89, 0x57, 0x0d,
0xf4, 0x03, 0xba, 0x0d, 0x4b, 0x9c, 0xb0, 0x10, 0x47, 0xe1, 0x2b, 0x12, 0xf8, 0xe4, 0x65, 0xca,
0xfc, 0x34, 0xc2, 0x89, 0x02, 0x59, 0xd3, 0x43, 0x23, 0xde, 0x83, 0x97, 0x29, 0xdb, 0x89, 0x70,
0x82, 0x56, 0xc1, 0xa1, 0x99, 0x48, 0x33, 0xe1, 0x1b, 0x18, 0x84, 0x81, 0xc2, 0x9c, 0xed, 0xb5,
0x34, 0x5d, 0x45, 0x9d, 0x6f, 0x06, 0x33, 0xa7, 0x90, 0xc6, 0xa9, 0xa6, 0x90, 0xe6, 0xe9, 0xa6,
0x90, 0xf9, 0xd9, 0x53, 0x08, 0x6a, 0x41, 0x29, 0x79, 0xa1, 0xb0, 0x66, 0x7b, 0xa5, 0xe4, 0x85,
0x0c, 0xa4, 0xa0, 0xe9, 0x73, 0x85, 0x31, 0xdb, 0x53, 0xcf, 0x32, 0x89, 0x62, 0x22, 0x58, 0xd8,
0x93, 0x6e, 0x69, 0x3b, 0x2a, 0x0e, 0x63, 0x14, 0xf7, 0x3f, 0xf6, 0x08, 0x56, 0x3c, 0x8b, 0x04,
0xff, 0xa2, 0x26, 0x98, 0x02, 0x8b, 0xf6, 0x38, 0x16, 0xaf, 0x42, 0x43, 0x1b, 0xa7, 0x63, 0x5e,
0x9e, 0xb6, 0x57, 0x0a, 0xc8, 0x2c, 0x7b, 0x91, 0x11, 0x16, 0x12, 0x6e, 0xca, 0x3e, 0x24, 0x59,
0xfc, 0x44, 0x53, 0xd0, 0x39, 0xa8, 0x08, 0x9a, 0xfa, 0xcf, 0xf3, 0x72, 0x25, 0x68, 0xfa, 0x08,
0x7d, 0x0f, 0x96, 0x39, 0xc1, 0x11, 0x09, 0xfc, 0xa2, 0xbc, 0x70, 0x9f, 0xab, 0x63, 0x93, 0xa0,
0x5d, 0x55, 0x61, 0x6e, 0x6b, 0x89, 0xdd, 0x42, 0x60, 0xd7, 0xf0, 0x65, 0x14, 0x7b, 0xba, 0x6d,
0x9f, 0xd8, 0x56, 0x53, 0x9d, 0x3d, 0x1a, 0xb1, 0x8a, 0x0d, 0xdf, 0x86, 0x76, 0x3f, 0xa2, 0x5d,
0x1c, 0xf9, 0x87, 0xde, 0xaa, 0x46, 0x08, 0xdb, 0xbb, 0xa0, 0xf9, 0xbb, 0x53, 0xaf, 0x94, 0xc7,
0xe3, 0x51, 0xd8, 0x23, 0x81, 0xdf, 0x8d, 0x68, 0xb7, 0x0d, 0x0a, 0xae, 0xa0, 0x49, 0xb2, 0x5e,
0x49, 0x98, 0x1a, 0x01, 0xe9, 0x86, 0x1e, 0xcd, 0x12, 0xa1, 0xc0, 0x67, 0x7b, 0x2d, 0x4d, 0x7f,
0x9c, 0xc5, 0x1b, 0x92, 0x8a, 0xfe, 0x0f, 0xe6, 0x8d, 0x24, 0xdd, 0xdb, 0xe3, 0x44, 0x28, 0xd4,
0xd9, 0x5e, 0x53, 0x13, 0x7f, 0xa4, 0x68, 0xee, 0x9f, 0x6d, 0x58, 0xf0, 0xa4, 0x77, 0xc9, 0x3e,
0xf9, 0x5f, 0xaa, 0x2b, 0x47, 0xe5, 0xf7, 0xdc, 0xa9, 0xf2, 0xbb, 0x7a, 0xe2, 0xfc, 0xae, 0x9d,
0x2a, 0xbf, 0xeb, 0xa7, 0xcb, 0x6f, 0x38, 0x22, 0xbf, 0x97, 0xa0, 0x12, 0x85, 0x71, 0x98, 0x07,
0x58, 0x2f, 0xdc, 0x3f, 0x4c, 0x84, 0xec, 0x2d, 0xc8, 0xd9, 0xeb, 0x60, 0x87, 0x81, 0x6e, 0x20,
0x1b, 0x6b, 0xed, 0x99, 0x37, 0xe6, 0x66, 0x87, 0x7b, 0x52, 0x68, 0xfa, 0x96, 0xad, 0x9c, 0xfa,
0x96, 0xfd, 0x3e, 0x5c, 0x3a, 0x9c, 0xc9, 0xcc, 0xb8, 0x23, 0x68, 0xcf, 0xa9, 0x88, 0x5e, 0x9c,
0x4e, 0xe5, 0xdc, 0x5f, 0x01, 0xfa, 0x16, 0x2c, 0x8d, 0xe5, 0xf2, 0x68, 0x63, 0x55, 0x4f, 0xf6,
0x23, 0xde, 0x68, 0xcb, 0x71, 0xd9, 0x5c, 0x3b, 0x2e, 0x9b, 0xdd, 0xbf, 0xdb, 0x30, 0xdf, 0x21,
0x11, 0x11, 0xe4, 0xab, 0x26, 0xf0, 0xc8, 0x26, 0xf0, 0x9b, 0x80, 0xc2, 0x44, 0xdc, 0xf9, 0xc8,
0x4f, 0x59, 0x18, 0x63, 0x36, 0xf4, 0x9f, 0x93, 0x61, 0x5e, 0x26, 0x1d, 0xc5, 0xd9, 0xd1, 0x8c,
0x47, 0x64, 0xc8, 0x5f, 0xdb, 0x14, 0x8e, 0x77, 0x61, 0x3a, 0x6d, 0x8a, 0x2e, 0xec, 0xbb, 0xd0,
0x9c, 0x78, 0x45, 0xf3, 0x35, 0x80, 0x6d, 0xa4, 0xa3, 0xf7, 0xba, 0xff, 0xb6, 0xa0, 0xbe, 0x45,
0x71, 0xa0, 0xe6, 0xa1, 0x33, 0x86, 0xb1, 0x68, 0x75, 0x4b, 0xd3, 0xad, 0xee, 0x65, 0x18, 0x8d,
0x34, 0x26, 0x90, 0x63, 0x33, 0xce, 0xd8, 0xac, 0x52, 0x9e, 0x9c, 0x55, 0xae, 0x42, 0x23, 0x94,
0x06, 0xf9, 0x29, 0x16, 0x03, 0x5d, 0x29, 0xeb, 0x1e, 0x28, 0xd2, 0x8e, 0xa4, 0xc8, 0x61, 0x26,
0x17, 0x50, 0xc3, 0xcc, 0xdc, 0x89, 0x87, 0x19, 0xa3, 0x44, 0x0d, 0x33, 0xbf, 0xb0, 0x00, 0xd4,
0xc1, 0x65, 0x3d, 0x38, 0xac, 0xd4, 0x3a, 0x8b, 0x52, 0x59, 0xc2, 0x55, 0xa4, 0x48, 0x84, 0xc5,
0x28, 0xa9, 0xb8, 0x71, 0x0e, 0x92, 0x51, 0xd3, 0x2c, 0x93, 0x50, 0xdc, 0xfd, 0xb5, 0x05, 0xa0,
0xaa, 0x82, 0x36, 0x63, 0x1a, 0x7e, 0xd6, 0xf1, 0x63, 0x5e, 0x69, 0xd2, 0x75, 0xeb, 0xb9, 0xeb,
0x8e, 0xf9, 0x8e, 0x3a, 0xd6, 0x97, 0xe7, 0x87, 0x37, 0xde, 0x55, 0xcf, 0xee, 0x6f, 0x2c, 0x68,
0x1a, 0xeb, 0xb4, 0x49, 0x13, 0x51, 0xb6, 0xa6, 0xa3, 0xac, 0x9a, 0x9b, 0x98, 0xb2, 0xa1, 0xcf,
0xc3, 0x57, 0xc4, 0x18, 0x04, 0x9a, 0xb4, 0x1b, 0xbe, 0x22, 0x13, 0xe0, 0xb5, 0x27, 0xc1, 0x7b,
0x03, 0x16, 0x19, 0xe9, 0x91, 0x44, 0x44, 0x43, 0x3f, 0xa6, 0x41, 0xb8, 0x17, 0x92, 0x40, 0xa1,
0xa1, 0xe6, 0x39, 0x39, 0x63, 0xdb, 0xd0, 0xdd, 0x9f, 0x5b, 0xd0, 0xd8, 0xe6, 0xfd, 0x1d, 0xca,
0x55, 0x92, 0xa1, 0x6b, 0xd0, 0x34, 0x85, 0x4d, 0x67, 0xb8, 0xa5, 0x10, 0xd6, 0xe8, 0x8d, 0xbe,
0x45, 0xca, 0xd2, 0x1e, 0xf3, 0xbe, 0x71, 0x53, 0xd3, 0xd3, 0x0b, 0xb4, 0x0c, 0xb5, 0x98, 0xf7,
0x55, 0x2f, 0x6e, 0x60, 0x59, 0xac, 0xe5, 0x59, 0x47, 0x57, 0x58, 0x59, 0x5d, 0x61, 0x23, 0x82,
0xfb, 0x99, 0x05, 0xc8, 0x7c, 0xeb, 0x7c, 0xa3, 0x5f, 0x13, 0x2a, 0xca, 0xe3, 0xdf, 0x53, 0x4b,
0x0a, 0xe3, 0x13, 0xb4, 0xa9, 0xa2, 0x60, 0x1f, 0x2a, 0x0a, 0x37, 0x60, 0x31, 0x20, 0x7b, 0x38,
0x8b, 0xc6, 0x6f, 0x5d, 0x6d, 0xb2, 0x63, 0x18, 0x13, 0xdf, 0xf6, 0x5b, 0x1b, 0x8c, 0x04, 0x24,
0x11, 0x21, 0x8e, 0xd4, 0x2f, 0xa7, 0x65, 0xa8, 0x65, 0x5c, 0x22, 0xa1, 0xf0, 0x5d, 0xb1, 0x46,
0x1f, 0x00, 0x22, 0x49, 0x8f, 0x0d, 0x53, 0x09, 0xe2, 0x14, 0x73, 0x7e, 0x40, 0x59, 0x60, 0x0a,
0xf5, 0x62, 0xc1, 0xd9, 0x31, 0x0c, 0x39, 0xb4, 0x0a, 0x92, 0xe0, 0x44, 0xe4, 0xf5, 0x5a, 0xaf,
0x64, 0xe8, 0x43, 0xee, 0xf3, 0x2c, 0x25, 0xcc, 0x84, 0xb5, 0x1a, 0xf2, 0x5d, 0xb9, 0x94, 0xa5,
0x9c, 0x0f, 0xf0, 0xda, 0xc7, 0x77, 0x46, 0xea, 0x75, 0x89, 0x6e, 0x69, 0x72, 0xae, 0xdb, 0x7d,
0x00, 0x8b, 0x5b, 0x21, 0x17, 0x3b, 0x34, 0x0a, 0x7b, 0xc3, 0x33, 0xdf, 0x38, 0xee, 0xa7, 0x16,
0xa0, 0x71, 0x3d, 0xe6, 0xcf, 0xc6, 0xa8, 0x63, 0xb0, 0x4e, 0xde, 0x31, 0x5c, 0x83, 0x66, 0xaa,
0xd4, 0xa8, 0xff, 0xa8, 0x79, 0xf4, 0x1a, 0x9a, 0x26, 0x7d, 0xcb, 0xd1, 0x15, 0x00, 0xe9, 0x4c,
0x9f, 0xd1, 0x88, 0xe8, 0xe0, 0xd5, 0xbd, 0xba, 0xa4, 0x78, 0x92, 0xe0, 0xf6, 0xe1, 0xe2, 0xee,
0x80, 0x1e, 0x6c, 0xd0, 0x64, 0x2f, 0xec, 0x67, 0x0c, 0x4b, 0x40, 0xbf, 0xc1, 0x17, 0xb3, 0x36,
0x54, 0x53, 0x2c, 0x64, 0x5a, 0x9b, 0x18, 0xe5, 0x4b, 0xf7, 0xb7, 0x16, 0x2c, 0xcf, 0x7a, 0xd3,
0x9b, 0x1c, 0xff, 0x21, 0xcc, 0xf7, 0xb4, 0x3a, 0xad, 0xed, 0xe4, 0xbf, 0x0e, 0x27, 0xf7, 0xb9,
0x0f, 0xa0, 0xec, 0x61, 0x41, 0xd0, 0x2d, 0x28, 0x31, 0xa1, 0x2c, 0x68, 0xad, 0x5d, 0x3d, 0xa2,
0x58, 0x49, 0x41, 0x35, 0x0d, 0x97, 0x98, 0x40, 0x4d, 0xb0, 0x98, 0x3a, 0xa9, 0xe5, 0x59, 0xec,
0xfa, 0x03, 0xa8, 0x17, 0x7f, 0xbb, 0x91, 0x03, 0xcd, 0xcd, 0x24, 0x14, 0xaa, 0x51, 0x0e, 0x93,
0xbe, 0xf3, 0x0e, 0x6a, 0x40, 0xf5, 0x87, 0x04, 0x47, 0x62, 0x30, 0x74, 0x2c, 0xd4, 0x84, 0xda,
0xfd, 0x6e, 0x42, 0x59, 0x8c, 0x23, 0xa7, 0x24, 0x59, 0xbb, 0x02, 0x27, 0xc1, 0xfa, 0xd0, 0xb1,
0xaf, 0xaf, 0xc1, 0xe2, 0xa1, 0x2f, 0x15, 0x52, 0xde, 0xa3, 0x07, 0xd2, 0xd5, 0x81, 0xf3, 0x0e,
0x5a, 0x80, 0xc6, 0x06, 0x8d, 0xb2, 0x38, 0xd1, 0x04, 0xeb, 0xfa, 0x1f, 0x2d, 0xa8, 0xe5, 0x96,
0xa1, 0x45, 0x98, 0xef, 0x74, 0xb6, 0x46, 0xbf, 0x3d, 0x9c, 0x77, 0xa4, 0x35, 0x9d, 0xce, 0x56,
0xf1, 0xd1, 0x5c, 0x1b, 0xd0, 0xe9, 0x6c, 0xa9, 0xd2, 0xeb, 0x94, 0xcc, 0xea, 0x93, 0x28, 0xe3,
0x03, 0xc7, 0x2e, 0x14, 0xc4, 0x29, 0xd6, 0x0a, 0xca, 0x68, 0x1e, 0xea, 0x9d, 0xed, 0x2d, 0x6d,
0x97, 0x53, 0x31, 0x4b, 0xdd, 0x7d, 0x39, 0x73, 0xd2, 0x9e, 0xce, 0xf6, 0xd6, 0x7a, 0x16, 0x3d,
0x97, 0xb7, 0xb8, 0x53, 0x55, 0xfc, 0x27, 0x5b, 0x7a, 0x64, 0x73, 0x6a, 0x4a, 0xfd, 0x93, 0x2d,
0x39, 0x44, 0x0e, 0x9d, 0xfa, 0xfa, 0xdd, 0x9f, 0x7e, 0xdc, 0x0f, 0xc5, 0x20, 0xeb, 0xca, 0xd8,
0xdc, 0xd2, 0x6e, 0xfe, 0x20, 0xa4, 0xe6, 0xe9, 0x56, 0xee, 0xea, 0x5b, 0xca, 0xf3, 0xc5, 0x32,
0xed, 0x76, 0xe7, 0x14, 0xe5, 0xc3, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x82, 0x36, 0x36, 0xda,
0x01, 0x21, 0x00, 0x00,
}

33
internal/querycoord/query_coord.go
View File

@ -94,7 +94,9 @@ type QueryCoord struct {
offlineNodesChan chan UniqueID
offlineNodes map[UniqueID]struct{}
stateCode atomic.Value
stateCode atomic.Value
enableActiveStandBy bool
activateFunc func()
factory dependency.Factory
chunkManager storage.ChunkManager
@ -104,6 +106,9 @@ type QueryCoord struct {
// Register register query service at etcd
func (qc *QueryCoord) Register() error {
qc.session.Register()
if qc.enableActiveStandBy {
qc.session.ProcessActiveStandBy(qc.activateFunc)
}
go qc.session.LivenessCheck(qc.loopCtx, func() {
log.Error("Query Coord disconnected from etcd, process will exit", zap.Int64("Server Id", qc.session.ServerID))
if err := qc.Stop(); err != nil {
@ -125,6 +130,7 @@ func (qc *QueryCoord) initSession() error {
return fmt.Errorf("session is nil, the etcd client connection may have failed")
}
qc.session.Init(typeutil.QueryCoordRole, Params.QueryCoordCfg.Address, true, true)
qc.session.SetEnableActiveStandBy(qc.enableActiveStandBy)
Params.QueryCoordCfg.SetNodeID(qc.session.ServerID)
Params.SetLogger(qc.session.ServerID)
return nil
@ -258,7 +264,17 @@ func (qc *QueryCoord) Start() error {
go qc.loadBalanceSegmentLoop()
}
qc.UpdateStateCode(internalpb.StateCode_Healthy)
if qc.enableActiveStandBy {
qc.activateFunc = func() {
// todo to complete
log.Info("querycoord switch from standby to active, activating")
qc.meta.reloadFromKV()
qc.UpdateStateCode(internalpb.StateCode_Healthy)
}
qc.UpdateStateCode(internalpb.StateCode_StandBy)
} else {
qc.UpdateStateCode(internalpb.StateCode_Healthy)
}
return nil
}
@ -303,12 +319,13 @@ func NewQueryCoord(ctx context.Context, factory dependency.Factory) (*QueryCoord
rand.Seed(time.Now().UnixNano())
ctx1, cancel := context.WithCancel(ctx)
service := &QueryCoord{
loopCtx: ctx1,
loopCancel: cancel,
factory: factory,
newNodeFn: newQueryNode,
offlineNodesChan: make(chan UniqueID, 256),
offlineNodes: make(map[UniqueID]struct{}, 256),
loopCtx: ctx1,
loopCancel: cancel,
factory: factory,
newNodeFn: newQueryNode,
offlineNodesChan: make(chan UniqueID, 256),
offlineNodes: make(map[UniqueID]struct{}, 256),
enableActiveStandBy: Params.QueryCoordCfg.EnableActiveStandby,
}
service.UpdateStateCode(internalpb.StateCode_Abnormal)

28
internal/querycoord/query_coord_test.go
View File

@ -40,6 +40,7 @@ import (
"github.com/milvus-io/milvus/api/milvuspb"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/util"
"github.com/milvus-io/milvus/internal/util/dependency"
@ -120,6 +121,33 @@ func startQueryCoord(ctx context.Context) (*QueryCoord, error) {
return coord, nil
}
func TestQueryCoord_DisableActiveStandby(t *testing.T) {
Params.Init()
Params.QueryCoordCfg.EnableActiveStandby = false
ctx := context.Background()
queryCoord, err := startQueryCoord(ctx)
assert.Nil(t, err)
resp, err := queryCoord.GetComponentStates(ctx)
assert.Nil(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
assert.Equal(t, internalpb.StateCode_Healthy, resp.GetState().GetStateCode())
defer queryCoord.Stop()
}
// make sure the main functions work well when EnableActiveStandby=true
func TestQueryCoord_EnableActiveStandby(t *testing.T) {
Params.Init()
Params.QueryCoordCfg.EnableActiveStandby = true
ctx := context.Background()
queryCoord, err := startQueryCoord(ctx)
assert.Nil(t, err)
resp, err := queryCoord.GetComponentStates(ctx)
assert.Nil(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
assert.Equal(t, internalpb.StateCode_Healthy, resp.GetState().GetStateCode())
defer queryCoord.Stop()
}
func createDefaultPartition(ctx context.Context, queryCoord *QueryCoord) error {
_, err := queryCoord.rootCoordClient.CreatePartition(ctx, nil)
return err

26
internal/rootcoord/root_coord.go
View File

@ -126,6 +126,9 @@ type Core struct {
factory dependency.Factory
importManager *importManager
enableActiveStandBy bool
activateFunc func()
}
// --------------------- function --------------------------
@ -135,9 +138,10 @@ func NewCore(c context.Context, factory dependency.Factory) (*Core, error) {
ctx, cancel := context.WithCancel(c)
rand.Seed(time.Now().UnixNano())
core := &Core{
ctx: ctx,
cancel: cancel,
factory: factory,
ctx: ctx,
cancel: cancel,
factory: factory,
enableActiveStandBy: Params.RootCoordCfg.EnableActiveStandby,
}
core.UpdateStateCode(internalpb.StateCode_Abnormal)
return core, nil
@ -259,6 +263,13 @@ func (c *Core) SetQueryCoord(s types.QueryCoord) error {
// Register register rootcoord at etcd
func (c *Core) Register() error {
c.session.Register()
if c.enableActiveStandBy {
c.session.ProcessActiveStandBy(c.activateFunc)
} else {
c.UpdateStateCode(internalpb.StateCode_Healthy)
log.Debug("RootCoord start successfully ", zap.String("State Code", internalpb.StateCode_Healthy.String()))
}
log.Info("RootCoord Register Finished")
go c.session.LivenessCheck(c.ctx, func() {
log.Error("Root Coord disconnected from etcd, process will exit", zap.Int64("Server Id", c.session.ServerID))
if err := c.Stop(); err != nil {
@ -287,6 +298,7 @@ func (c *Core) initSession() error {
return fmt.Errorf("session is nil, the etcd client connection may have failed")
}
c.session.Init(typeutil.RootCoordRole, Params.RootCoordCfg.Address, true, true)
c.session.SetEnableActiveStandBy(c.enableActiveStandBy)
Params.SetLogger(c.session.ServerID)
return nil
}
@ -596,6 +608,14 @@ func (c *Core) startInternal() error {
Params.RootCoordCfg.CreatedTime = time.Now()
Params.RootCoordCfg.UpdatedTime = time.Now()
if c.enableActiveStandBy {
c.activateFunc = func() {
// todo to complete
log.Info("rootcoord switch from standby to active, activating")
c.UpdateStateCode(internalpb.StateCode_Healthy)
}
c.UpdateStateCode(internalpb.StateCode_StandBy)
}
return nil
}

46
internal/rootcoord/root_coord_test.go
View File

@ -3,6 +3,7 @@ package rootcoord
import (
"context"
"errors"
"fmt"
"math/rand"
"sync"
"testing"
@ -21,10 +22,13 @@ import (
"github.com/milvus-io/milvus/internal/proto/proxypb"
"github.com/milvus-io/milvus/internal/proto/rootcoordpb"
mockrootcoord "github.com/milvus-io/milvus/internal/rootcoord/mocks"
"github.com/milvus-io/milvus/internal/util/dependency"
"github.com/milvus-io/milvus/internal/util/etcd"
"github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/milvus-io/milvus/internal/util/metricsinfo"
"github.com/milvus-io/milvus/internal/util/sessionutil"
"github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
@ -1309,3 +1313,45 @@ func TestCore_startTimeTickLoop(t *testing.T) {
cancel()
c.wg.Wait()
}
// make sure the main functions work well when EnableActiveStandby=true
func TestRootcoord_EnableActiveStandby(t *testing.T) {
Params.Init()
Params.RootCoordCfg.EnableActiveStandby = true
randVal := rand.Int()
Params.CommonCfg.RootCoordTimeTick = fmt.Sprintf("rootcoord-time-tick-%d", randVal)
Params.CommonCfg.RootCoordStatistics = fmt.Sprintf("rootcoord-statistics-%d", randVal)
Params.EtcdCfg.MetaRootPath = fmt.Sprintf("/%d/%s", randVal, Params.EtcdCfg.MetaRootPath)
Params.EtcdCfg.KvRootPath = fmt.Sprintf("/%d/%s", randVal, Params.EtcdCfg.KvRootPath)
Params.CommonCfg.RootCoordSubName = fmt.Sprintf("subname-%d", randVal)
Params.CommonCfg.RootCoordDml = fmt.Sprintf("rootcoord-dml-test-%d", randVal)
Params.CommonCfg.RootCoordDelta = fmt.Sprintf("rootcoord-delta-test-%d", randVal)
ctx := context.Background()
coreFactory := dependency.NewDefaultFactory(true)
etcdCli, err := etcd.GetEtcdClient(&Params.EtcdCfg)
assert.NoError(t, err)
defer etcdCli.Close()
core, err := NewCore(ctx, coreFactory)
core.etcdCli = etcdCli
assert.NoError(t, err)
err = core.Init()
assert.NoError(t, err)
err = core.Start()
assert.NoError(t, err)
core.session.TriggerKill = false
err = core.Register()
assert.NoError(t, err)
resp, err := core.DescribeCollection(ctx, &milvuspb.DescribeCollectionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_DescribeCollection,
MsgID: 0,
Timestamp: 0,
SourceID: Params.ProxyCfg.GetNodeID(),
},
CollectionName: "unexist"})
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetStatus().GetErrorCode())
err = core.Stop()
assert.NoError(t, err)
}

39
internal/util/paramtable/component_param.go
View File

@ -34,6 +34,9 @@ const (
// DefaultIndexSliceSize defines the default slice size of index file when serializing.
DefaultIndexSliceSize = 16
DefaultGracefulTime = 5000 //ms
DefaultSessionTTL = 60 //s
DefaultSessionRetryTimes = 30
)
// ComponentParam is used to quickly and easily access all components' configurations.
@ -138,6 +141,9 @@ type commonConfig struct {
AuthorizationEnabled bool
ClusterName string
SessionTTL int64
SessionRetryTimes int64
}
func (p *commonConfig) init(base *BaseTable) {
@ -177,6 +183,9 @@ func (p *commonConfig) init(base *BaseTable) {
p.initEnableAuthorization()
p.initClusterName()
p.initSessionTTL()
p.initSessionRetryTimes()
}
func (p *commonConfig) initClusterPrefix() {
@ -384,6 +393,14 @@ func (p *commonConfig) initClusterName() {
p.ClusterName = p.Base.LoadWithDefault("common.cluster.name", "")
}
func (p *commonConfig) initSessionTTL() {
p.SessionTTL = p.Base.ParseInt64WithDefault("common.session.ttl", 60)
}
func (p *commonConfig) initSessionRetryTimes() {
p.SessionRetryTimes = p.Base.ParseInt64WithDefault("common.session.retryTimes", 30)
}
///////////////////////////////////////////////////////////////////////////////
// --- rootcoord ---
type rootCoordConfig struct {
@ -403,6 +420,8 @@ type rootCoordConfig struct {
CreatedTime time.Time
UpdatedTime time.Time
EnableActiveStandby bool
}
func (p *rootCoordConfig) init(base *BaseTable) {
@ -413,6 +432,7 @@ func (p *rootCoordConfig) init(base *BaseTable) {
p.ImportTaskExpiration = p.Base.ParseFloatWithDefault("rootCoord.importTaskExpiration", 15*60)
p.ImportTaskRetention = p.Base.ParseFloatWithDefault("rootCoord.importTaskRetention", 24*60*60)
p.ImportTaskSubPath = "importtask"
p.EnableActiveStandby = p.Base.ParseBool("rootCoord.enableActiveStandby", false)
}
///////////////////////////////////////////////////////////////////////////////
@ -625,6 +645,7 @@ type queryCoordConfig struct {
DistPullInterval time.Duration
LoadTimeoutSeconds time.Duration
CheckHandoffInterval time.Duration
EnableActiveStandby bool
}
func (p *queryCoordConfig) init(base *BaseTable) {
@ -650,6 +671,7 @@ func (p *queryCoordConfig) init(base *BaseTable) {
p.initDistPullInterval()
p.initLoadTimeoutSeconds()
p.initCheckHandoffInterval()
p.initEnableActiveStandby()
}
func (p *queryCoordConfig) initTaskRetryNum() {
@ -711,6 +733,10 @@ func (p *queryCoordConfig) initMemoryUsageMaxDifferencePercentage() {
p.MemoryUsageMaxDifferencePercentage = float64(diffPercentage) / 100
}
func (p *queryCoordConfig) initEnableActiveStandby() {
p.EnableActiveStandby = p.Base.ParseBool("queryCoord.enableActiveStandby", false)
}
func (p *queryCoordConfig) initCheckInterval() {
interval := p.Base.LoadWithDefault("queryCoord.checkInterval", "1000")
checkInterval, err := strconv.ParseInt(interval, 10, 64)
@ -1070,6 +1096,7 @@ type dataCoordConfig struct {
GCInterval time.Duration
GCMissingTolerance time.Duration
GCDropTolerance time.Duration
EnableActiveStandby bool
}
func (p *dataCoordConfig) init(base *BaseTable) {
@ -1102,6 +1129,7 @@ func (p *dataCoordConfig) init(base *BaseTable) {
p.initGCInterval()
p.initGCMissingTolerance()
p.initGCDropTolerance()
p.initEnableActiveStandby()
}
func (p *dataCoordConfig) initSegmentMaxSize() {
@ -1223,6 +1251,10 @@ func (p *dataCoordConfig) GetEnableAutoCompaction() bool {
return false
}
func (p *dataCoordConfig) initEnableActiveStandby() {
p.EnableActiveStandby = p.Base.ParseBool("dataCoord.enableActiveStandby", false)
}
func (p *dataCoordConfig) SetNodeID(id UniqueID) {
p.NodeID.Store(id)
}
@ -1336,6 +1368,8 @@ type indexCoordConfig struct {
CreatedTime time.Time
UpdatedTime time.Time
EnableActiveStandby bool
}
func (p *indexCoordConfig) init(base *BaseTable) {
@ -1347,6 +1381,7 @@ func (p *indexCoordConfig) init(base *BaseTable) {
p.initIndexNodeAddress()
p.initWithCredential()
p.initIndexNodeID()
p.initEnableActiveStandby()
}
func (p *indexCoordConfig) initMinSegmentNumRowsToEnableIndex() {
@ -1373,6 +1408,10 @@ func (p *indexCoordConfig) initIndexNodeID() {
p.IndexNodeID = p.Base.ParseInt64WithDefault("indexCoord.bindIndexNodeMode.nodeID", 0)
}
func (p *indexCoordConfig) initEnableActiveStandby() {
p.EnableActiveStandby = p.Base.ParseBool("indexCoord.enableActiveStandby", false)
}
///////////////////////////////////////////////////////////////////////////////
// --- indexnode ---
type indexNodeConfig struct {

19
internal/util/paramtable/component_param_test.go
View File

@ -114,6 +114,11 @@ func TestComponentParam(t *testing.T) {
assert.Equal(t, Params.DataNodeSubName, "by-dev-dataNode")
t.Logf("datanode subname = %s", Params.DataNodeSubName)
assert.Equal(t, Params.SessionTTL, int64(DefaultSessionTTL))
t.Logf("default session TTL time = %d", Params.SessionTTL)
assert.Equal(t, Params.SessionRetryTimes, int64(DefaultSessionRetryTimes))
t.Logf("default session retry times = %d", Params.SessionRetryTimes)
})
t.Run("test rootCoordConfig", func(t *testing.T) {
@ -125,6 +130,8 @@ func TestComponentParam(t *testing.T) {
t.Logf("master MinSegmentSizeToEnableIndex = %d", Params.MinSegmentSizeToEnableIndex)
assert.NotEqual(t, Params.ImportTaskExpiration, 0)
t.Logf("master ImportTaskRetention = %f", Params.ImportTaskRetention)
assert.Equal(t, Params.EnableActiveStandby, false)
t.Logf("rootCoord EnableActiveStandby = %t", Params.EnableActiveStandby)
Params.CreatedTime = time.Now()
Params.UpdatedTime = time.Now()
@ -214,6 +221,12 @@ func TestComponentParam(t *testing.T) {
})
})
t.Run("test queryCoordConfig", func(t *testing.T) {
Params := CParams.QueryCoordCfg
assert.Equal(t, Params.EnableActiveStandby, false)
t.Logf("queryCoord EnableActiveStandby = %t", Params.EnableActiveStandby)
})
t.Run("test queryNodeConfig", func(t *testing.T) {
Params := CParams.QueryNodeCfg
@ -275,8 +288,9 @@ func TestComponentParam(t *testing.T) {
t.Run("test dataCoordConfig", func(t *testing.T) {
Params := CParams.DataCoordCfg
assert.Equal(t, 24*60*60*time.Second, Params.SegmentMaxLifetime)
assert.True(t, Params.EnableGarbageCollection)
assert.Equal(t, Params.EnableActiveStandby, false)
t.Logf("dataCoord EnableActiveStandby = %t", Params.EnableActiveStandby)
})
t.Run("test dataNodeConfig", func(t *testing.T) {
@ -323,6 +337,9 @@ func TestComponentParam(t *testing.T) {
assert.Equal(t, "localhost:22930", Params.IndexNodeAddress)
assert.False(t, Params.WithCredential)
assert.Equal(t, int64(0), Params.IndexNodeID)
assert.Equal(t, Params.EnableActiveStandby, false)
t.Logf("indexCoord EnableActiveStandby = %t", Params.EnableActiveStandby)
})
t.Run("test indexNodeConfig", func(t *testing.T) {

141
internal/util/sessionutil/session_util.go
View File

@ -13,6 +13,7 @@ import (
"github.com/blang/semver/v4"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/retry"
"go.etcd.io/etcd/api/v3/mvccpb"
v3rpc "go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
@ -25,12 +26,10 @@ const (
DefaultServiceRoot = "session/"
// DefaultIDKey default id key for Session
DefaultIDKey = "id"
// DefaultRetryTimes default retry times when registerService or getServerByID
DefaultRetryTimes = 30
// DefaultTTL default ttl value when granting a lease
DefaultTTL = 60
)
var GlobalParams paramtable.ComponentParam
// SessionEventType session event type
type SessionEventType int
@ -71,6 +70,10 @@ type Session struct {
metaRoot string
registered atomic.Value
isStandby atomic.Value
enableActiveStandBy bool
activeKey string
}
// UnmarshalJSON unmarshal bytes to Session.
@ -171,6 +174,7 @@ func (s *Session) Init(serverName, address string, exclusive bool, triggerKill b
panic(err)
}
s.ServerID = serverID
GlobalParams.InitOnce()
}
// String makes Session struct able to be logged by zap
@ -252,10 +256,18 @@ func (s *Session) getServerIDWithKey(key string) (int64, error) {
// Exclusive means whether this service can exist two at the same time, if so,
// it is false. Otherwise, set it to true.
func (s *Session) registerService() (<-chan *clientv3.LeaseKeepAliveResponse, error) {
if s.enableActiveStandBy {
s.updateStandby(true)
}
key := s.ServerName
if !s.Exclusive || s.enableActiveStandBy {
key = fmt.Sprintf("%s-%d", key, s.ServerID)
}
completeKey := path.Join(s.metaRoot, DefaultServiceRoot, key)
var ch <-chan *clientv3.LeaseKeepAliveResponse
log.Debug("service begin to register to etcd", zap.String("serverName", s.ServerName), zap.Int64("ServerID", s.ServerID))
registerFn := func() error {
resp, err := s.etcdCli.Grant(s.ctx, DefaultTTL)
resp, err := s.etcdCli.Grant(s.ctx, GlobalParams.CommonCfg.SessionTTL)
if err != nil {
log.Error("register service", zap.Error(err))
return err
@ -267,16 +279,12 @@ func (s *Session) registerService() (<-chan *clientv3.LeaseKeepAliveResponse, er
return err
}
key := s.ServerName
if !s.Exclusive {
key = fmt.Sprintf("%s-%d", key, s.ServerID)
}
txnResp, err := s.etcdCli.Txn(s.ctx).If(
clientv3.Compare(
clientv3.Version(path.Join(s.metaRoot, DefaultServiceRoot, key)),
clientv3.Version(completeKey),
"=",
0)).
Then(clientv3.OpPut(path.Join(s.metaRoot, DefaultServiceRoot, key), string(sessionJSON), clientv3.WithLease(resp.ID))).Commit()
Then(clientv3.OpPut(completeKey, string(sessionJSON), clientv3.WithLease(resp.ID))).Commit()
if err != nil {
log.Warn("compare and swap error, maybe the key has already been registered", zap.Error(err))
@ -286,6 +294,7 @@ func (s *Session) registerService() (<-chan *clientv3.LeaseKeepAliveResponse, er
if !txnResp.Succeeded {
return fmt.Errorf("function CompareAndSwap error for compare is false for key: %s", key)
}
log.Debug("put session key into etcd", zap.String("key", completeKey), zap.String("value", string(sessionJSON)))
keepAliveCtx, keepAliveCancel := context.WithCancel(context.Background())
s.keepAliveCancel = keepAliveCancel
@ -297,7 +306,7 @@ func (s *Session) registerService() (<-chan *clientv3.LeaseKeepAliveResponse, er
log.Info("Service registered successfully", zap.String("ServerName", s.ServerName), zap.Int64("serverID", s.ServerID))
return nil
}
err := retry.Do(s.ctx, registerFn, retry.Attempts(DefaultRetryTimes))
err := retry.Do(s.ctx, registerFn, retry.Attempts(uint(GlobalParams.CommonCfg.SessionRetryTimes)))
if err != nil {
return nil, err
}
@ -445,9 +454,9 @@ func (s *Session) WatchServices(prefix string, revision int64, rewatch Rewatch)
return w.eventCh
}
// WatchServicesWithVersionRange watches the service's up and down in etcd, and sends event toeventChannel.
// WatchServicesWithVersionRange watches the service's up and down in etcd, and sends event to event Channel.
// Acts like WatchServices but with extra version range check.
// prefix is a parameter to know which service to watch and can be obtained intypeutil.type.go.
// prefix is a parameter to know which service to watch and can be obtained in type util.type.go.
// revision is a etcd reversion to prevent missing key events and can be obtained in GetSessions.
// If a server up, an event will be add to channel with eventType SessionAddType.
// If a server down, an event will be add to channel with eventType SessionDelType.
@ -598,3 +607,107 @@ func (s *Session) Registered() bool {
}
return b
}
func (s *Session) SetEnableActiveStandBy(enable bool) {
s.enableActiveStandBy = enable
}
func (s *Session) updateStandby(b bool) {
s.isStandby.Store(b)
}
// ProcessActiveStandBy is used by coordinators to do active-standby mechanism.
// coordinator enabled active-standby will first call Register and then call ProcessActiveStandBy.
// steps:
// 1, Enter STANDBY mode
// 2, Try to register to active key.
// 3, If 2. return true, this service becomes ACTIVE. Exit STANDBY mode.
// 4, If 2. return false, which means an ACTIVE service already exist.
// Start watching the active key. Whenever active key disappears, STANDBY node will go backup to 2.
// activateFunc is the function to re-active the service.
func (s *Session) ProcessActiveStandBy(activateFunc func()) error {
s.activeKey = path.Join(s.metaRoot, DefaultServiceRoot, s.ServerName)
// try to register to the active_key.
// return
// 1. doRegistered: if registered the active_key by this session or by other session
// 2. revision: revision of the active_key
// 3. err: etcd error, should retry
registerActiveFn := func() (bool, int64, error) {
log.Info(fmt.Sprintf("try to register as ACTIVE %v service...", s.ServerName))
sessionJSON, err := json.Marshal(s)
if err != nil {
log.Error("json marshal error", zap.Error(err))
return false, -1, err
}
txnResp, err := s.etcdCli.Txn(s.ctx).If(
clientv3.Compare(
clientv3.Version(s.activeKey),
"=",
0)).
Then(clientv3.OpPut(s.activeKey, string(sessionJSON), clientv3.WithLease(*s.leaseID))).Commit()
if err != nil {
log.Error("register active key to etcd failed", zap.Error(err))
return false, -1, err
}
doRegistered := txnResp.Succeeded
if doRegistered {
log.Info(fmt.Sprintf("register ACTIVE %s", s.ServerName))
} else {
log.Info(fmt.Sprintf("ACTIVE %s has already been registered", s.ServerName))
}
revision := txnResp.Header.GetRevision()
return doRegistered, revision, nil
}
s.updateStandby(true)
log.Info(fmt.Sprintf("serverName: %v enter STANDBY mode", s.ServerName))
go func() {
for s.isStandby.Load().(bool) {
log.Debug(fmt.Sprintf("serverName: %v is in STANDBY ...", s.ServerName))
time.Sleep(10 * time.Second)
}
}()
for {
registered, revision, err := registerActiveFn()
if err != nil {
time.Sleep(100 * time.Millisecond)
continue
}
if registered {
break
}
log.Info(fmt.Sprintf("%s start to watch ACTIVE key", s.ServerName))
ctx, cancel := context.WithCancel(s.ctx)
watchChan := s.etcdCli.Watch(ctx, s.activeKey, clientv3.WithPrefix(), clientv3.WithPrevKV(), clientv3.WithRev(revision))
select {
case <-ctx.Done():
cancel()
case wresp, ok := <-watchChan:
if !ok {
cancel()
}
if wresp.Err() != nil {
cancel()
}
for _, event := range wresp.Events {
switch event.Type {
case mvccpb.PUT:
log.Debug("watch the ACTIVE key", zap.Any("ADD", event.Kv))
case mvccpb.DELETE:
log.Debug("watch the ACTIVE key", zap.Any("DELETE", event.Kv))
cancel()
}
}
}
cancel()
log.Info(fmt.Sprintf("stop watching ACTIVE key %v", s.activeKey))
}
s.updateStandby(false)
log.Info(fmt.Sprintf("serverName: %v quit STANDBY mode, this node will become ACTIVE", s.ServerName))
if activateFunc != nil {
activateFunc()
}
return nil
}

75
internal/util/sessionutil/session_util_test.go
View File

@ -610,3 +610,78 @@ func (suite *SessionWithVersionSuite) TestWatchServicesWithVersionRange() {
func TestSessionWithVersionRange(t *testing.T) {
suite.Run(t, new(SessionWithVersionSuite))
}
func TestSessionProcessActiveStandBy(t *testing.T) {
// initial etcd
Params.Init()
endpoints, err := Params.Load("_EtcdEndpoints")
if err != nil {
panic(err)
}
metaRoot := fmt.Sprintf("%d/%s1", rand.Int(), DefaultServiceRoot)
etcdEndpoints := strings.Split(endpoints, ",")
etcdCli, err := etcd.GetRemoteEtcdClient(etcdEndpoints)
require.NoError(t, err)
etcdKV := etcdkv.NewEtcdKV(etcdCli, metaRoot)
err = etcdKV.RemoveWithPrefix("")
assert.NoError(t, err)
defer etcdKV.Close()
defer etcdKV.RemoveWithPrefix("")
var wg sync.WaitGroup
ch := make(chan bool)
signal := make(chan struct{}, 1)
flag := false
// register session 1, will be active
ctx1 := context.Background()
s1 := NewSession(ctx1, metaRoot, etcdCli)
s1.Init("inittest", "testAddr", true, true)
s1.SetEnableActiveStandBy(true)
s1.Register()
wg.Add(1)
s1.liveCh = ch
s1.ProcessActiveStandBy(func() {
log.Debug("Session 1 become active")
wg.Done()
})
go s1.LivenessCheck(ctx1, func() {
flag = true
signal <- struct{}{}
s1.keepAliveCancel()
// directly delete the primary key to make this UT fast,
// or the session2 has to wait for session1 release after ttl(60s)
etcdCli.Delete(ctx1, s1.activeKey)
})
assert.False(t, s1.isStandby.Load().(bool))
// register session 2, will be standby
ctx2 := context.Background()
s2 := NewSession(ctx2, metaRoot, etcdCli)
s2.Init("inittest", "testAddr", true, true)
s2.SetEnableActiveStandBy(true)
s2.Register()
wg.Add(1)
go s2.ProcessActiveStandBy(func() {
log.Debug("Session 2 become active")
wg.Done()
})
assert.True(t, s2.isStandby.Load().(bool))
// sleep 20 seconds, the standby service will start watching primary key
time.Sleep(time.Second * 20)
//assert.True(t, s2.watchingPrimaryKeyLock)
// stop session 1, session 2 will take over primary service
log.Debug("Stop session 1, session 2 will take over primary service")
assert.False(t, flag)
ch <- true
assert.False(t, flag)
close(ch)
<-signal
assert.True(t, flag)
wg.Wait()
assert.False(t, s2.isStandby.Load().(bool))
}