milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor GrpcConfig (#21142) 

Signed-off-by: Enwei Jiao <enwei.jiao@zilliz.com>

Signed-off-by: Enwei Jiao <enwei.jiao@zilliz.com>
pull/21281/head
Enwei Jiao 2022-12-16 15:59:23 +08:00 committed by GitHub
parent 63cd4132a6
commit 166e9f0da5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
52 changed files with 1174 additions and 2090 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
cmd/milvus/run.go
View File

@ -12,7 +12,6 @@ import (
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/util/hardware"
"github.com/milvus-io/milvus/internal/util/metricsinfo"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
@ -93,17 +92,6 @@ func (c *run) execute(args []string, flags *flag.FlagSet) {
os.Exit(-1)
}
// Setup logger in advance for standalone and embedded Milvus.
// Any log from this point on is under control.
if c.serverType == typeutil.StandaloneRole || c.serverType == typeutil.EmbeddedRole {
var params paramtable.BaseTable
if c.serverType == typeutil.EmbeddedRole {
params.GlobalInitWithYaml("embedded-milvus.yaml")
} else {
params.Init()
}
}
runtimeDir := createRuntimeDir(c.serverType)
filename := getPidFileName(c.serverType, c.svrAlias)

1
cmd/roles/roles.go
View File

@ -183,6 +183,7 @@ func (mr *MilvusRoles) Run(local bool, alias string) {
if err := os.Setenv(metricsinfo.DeployModeEnvKey, metricsinfo.StandaloneDeployMode); err != nil {
log.Error("Failed to set deploy mode: ", zap.Error(err))
}
paramtable.Init()
params := paramtable.Get()

79
cmd/tools/config/main.go Normal file
View File

@ -0,0 +1,79 @@
package main
import (
"encoding/csv"
"os"
"reflect"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/typeutil"
"go.uber.org/zap"
)
func collect(data *[][]string, val *reflect.Value) {
if val.Kind() != reflect.Struct {
return
}
for j := 0; j < val.NumField(); j++ {
subVal := val.Field(j)
tag := val.Type().Field(j).Tag
log.Debug("subVal", zap.Any("subVal", subVal),
zap.String("name", val.Type().Field(j).Name),
zap.Any("tag", val.Type().Field(j).Tag),
zap.Any("type", val.Type().Field(j).Type),
)
t := val.Type().Field(j).Type.String()
if t == "paramtable.ParamItem" {
item := subVal.Interface().(paramtable.ParamItem)
refreshable := tag.Get("refreshable")
if refreshable == "" {
refreshable = "undefined"
}
*data = append(*data, []string{item.Key, item.GetValue(), item.Version, refreshable})
} else if t == "paramtable.ParamGroup" {
item := subVal.Interface().(paramtable.ParamGroup)
refreshable := tag.Get("refreshable")
if refreshable == "" {
refreshable = "undefined"
}
*data = append(*data, []string{item.KeyPrefix, "", item.Version, refreshable})
} else {
collect(data, &subVal)
}
}
}
func main() {
params := paramtable.ComponentParam{}
params.Init()
f, err := os.Create("configs.csv")
defer f.Close()
if err != nil {
log.Error("create file failed", zap.Error(err))
os.Exit(-1)
}
w := csv.NewWriter(f)
w.Write([]string{"key", "default", "version", "refreshable"})
val := reflect.ValueOf(params)
data := make([][]string, 0)
keySet := typeutil.NewSet[string]()
for i := 0; i < val.NumField(); i++ {
valueField := val.Field(i)
// typeField := val.Type().Field(i)
collect(&data, &valueField)
}
result := make([][]string, 0)
for _, d := range data {
if keySet.Contain(d[0]) {
continue
}
keySet.Insert(d[0])
result = append(result, d)
}
w.WriteAll(result)
w.Flush()
}

379
configs/config.go
View File

@ -1,379 +0,0 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// This file is not used for now.
package configs
import (
"fmt"
"strings"
"sync/atomic"
"github.com/BurntSushi/toml"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/util/metricsinfo"
"go.uber.org/zap"
)
const MinToleranceTime = 3600
type Config struct {
Etcd Etcd `toml:"etcd" json:"etcd"`
Minio Minio `toml:"minio" json:"minio"`
Pulsar Pulsar `toml:"pulsar" json:"pulsar"`
RocksMq RocksMq `toml:"rocksmq" json:"rocksmq"`
Grpc Grpc `toml:"grpc" json:"grpc"`
FlowGraph FlowGraph `toml:"flowgraph" json:"flowgrph"`
RootCoord RootCoord `toml:"rootcoord" json:"rootcoord"`
Proxy Proxy `toml:"proxy" json:"proxy"`
QueryCoord QueryCoord `toml:"querycoord" json:"querycoord"`
QueryNode QueryNode `toml:"querynode" json:"querynode"`
IndexCoord IndexCoord `toml:"indexcoord" json:"indexcoord"`
IndexNode IndexNode `toml:"indexnode" json:"indexnode"`
DataCoord DataCoord `toml:"datacoord" json:"datacoord"`
DataNode DataNode `toml:"datanode" json:"datanode"`
TimeTickInterval uint64 `toml:"timetick-interval" json:"timetick-interval"`
TimeTickBufferSize uint32 `toml:"timetick-buffer-size" json:"timetick-buffer-size"`
NameLengthLimit uint32 `toml:"name-length-limit" json:"name-length-limit"`
FieldCountLimit uint32 `toml:"field-count-limit" json:"field-count-limit"`
DimensionLimit uint32 `toml:"dimension-limit" json:"dimension-limit"`
ShardCountLimit uint32 `toml:"shard-count-limit" json:"shard-count-limit"`
DMLChannelCount uint32 `toml:"dml-channel-count" json:"dml-channel-count"`
PartitionCountLimit uint32 `toml:"partition-count-limit" json:"partition-count-limit"`
EnableIndexMinSegmentSize uint32 `toml:"enable-index-min-segment-size" json:"enable-index-min-segment-size"`
SIMDType string `toml:"simd-type" json:"simd-type"`
Compaction Compaction `toml:"compaction" json:"compaction"`
Log Log `toml:"log" json:"log"`
LocalStorage LocalStorage `toml:"localstorage" json:"localstorage"`
SkipQueryChannelRecover bool `toml:"skip-query-channel-recover" json:"skip-query-channel-recover"`
Metrics Metrics `toml:"" json:"metrics"`
GarbageCollector GarbageCollector `toml:"gc" json:"gc"`
}
type Etcd struct {
Endpoints []string `toml:"endpoints" json:"endpoints"`
UseEmbed bool `toml:"use-embed" json:"use-embed"`
ConfigPath string `toml:"config-path" json:"config-path"`
DataPath string `toml:"data-path" json:"data-path"`
}
type Minio struct {
Address string `toml:"address" json:"address"`
Port int `toml:"port" json:"port"`
AccessKeyID string `toml:"access-key-id" json:"access-key-id"`
SecretAccessKey string `toml:"secret-access-key" json:"secret-access-key"`
UseSSL bool `toml:"use-ssl" json:"use-ssl"`
BucketName string `toml:"bucket-name" json:"bucket-name"`
RootPath string `toml:"root-path" json:"root-path"`
}
type Pulsar struct {
Address string `toml:"address" json:"address"`
Port int `toml:"port" json:"port"`
MaxMessageSize uint64 `toml:"max-message-size" json:"max-message-size"`
}
type RocksMq struct {
Path string `toml:"path" json:"path"`
PageSize uint32 `toml:"page-size" json:"page-size"`
RetentionDuration uint32 `toml:"retention-duration" json:"retention-duration"`
RetentionSize uint32 `toml:"retention-size" json:"retention-size"`
}
type Grpc struct {
ServerMaxReceiveSize uint64 `toml:"server-max-receive-size" json:"server-max-receive-size"`
ServerMaxSendSize uint64 `toml:"server-max-send-size" json:"server-max-send-size"`
ClientMaxReceiveSize uint64 `toml:"client-max-receive-size" json:"client-max-receive-size"`
ClientMaxSendSize uint64 `toml:"client-max-send-size" json:"client-max-send-size"`
}
type RootCoord struct {
Address string `toml:"address" json:"address"`
Port int `toml:"port" json:"port"`
}
type Proxy struct {
Port int `toml:"port" json:"port"`
MaxTaskCount uint32 `toml:"max-task-count" json:"max-task-count"`
}
type QueryCoord struct {
Address string `toml:"address" json:"address"`
Port int `toml:"port" json:"port"`
AutoHandOff bool `toml:"auto-handoff" json:"auto-handoff"`
AutoBalance bool `toml:"auto-balance" json:"auto-balance"`
BalanceInterval uint32 `toml:"balance-interval" json:"balance-interval"`
MemoryUsageLimitRatio uint32 `toml:"memory-usage-limit-ratio" json:"memory-usage-limit-ratio"`
AutoBalanceMemoryUsageGapRatio uint32 `toml:"auto-balance-memory-usage-gap-ratio" json:"auto-balance-memory-usage-gap-ratio"`
}
type FlowGraph struct {
QueueLengthLimit uint32 `toml:"queue-length-limit" json:"queue-length-limit"`
ParallelismLimit uint32 `toml:"parallelism-limit" json:"parallelism-limit"`
}
type QueryNode struct {
Port int `toml:"port" json:"port"`
GracefulTime uint32 `toml:"graceful-time" json:"graceful-time"`
StatsPublishInterval uint32 `toml:"stats-publish-interval" json:"stats-publish-interval"`
SegcoreChunkRows uint32 `toml:"segcore-chunk-rows" json:"segcore-chunk-rows"`
}
type IndexCoord struct {
Address string `toml:"address" json:"address"`
Port int `toml:"port" json:"port"`
}
type IndexNode struct {
Port int `toml:"port" json:"port"`
}
type DataCoord struct {
Address string `toml:"address" json:"address"`
Port int `toml:"port" json:"port"`
}
type GarbageCollector struct {
Interval uint32 `toml:"interval" json:"interval"`
MissedFileTolerance uint32 `toml:"missed-files-tolerance" json:"missed-files-tolerance"`
DroppedFileTolerance uint32 `toml:"dropped-files-tolerance" json:"dropped-files-tolerance"`
}
type Compaction struct {
EnableCompaction bool `toml:"enable-compaction" json:"enable-compaction"`
}
type DataNode struct {
Port int `toml:"port" json:"port"`
InsertBufferSizeLimit uint32 `toml:"insert-buffer-size-limit" json:"insert-buffer-size-limit"`
}
type LocalStorage struct {
Enable bool `toml:"enable" json:"enable"`
Path string `toml:"path" json:"path"`
}
type Log struct {
Level string `toml:"level" json:"level"`
Format string `toml:"format" json:"format"`
File LogFileCfg `toml:"file" json:"file"`
}
type LogFileCfg struct {
RootPath string `toml:"root-path" json:"root-path"`
MaxSize uint32 `toml:"max-size" json:"max-size"`
MaxAge uint32 `toml:"max-age" json:"max-age"`
MaxBackups uint32 `toml:"max-backups" json:"max-backups"`
}
type Metrics struct {
GitCommit string `toml:"" json:"git-commit-key"`
DeployMode string `toml:"" json:"deploy-mode"`
GitBuildTags string `toml:"" json:"git-build-tags"`
BuildTime string `toml:"" json:"build-time"`
GoVersion string `toml:"" json:"go-version"`
}
var defaultCfg = Config{
Etcd: Etcd{
Endpoints: []string{"localhost:2379"},
UseEmbed: false,
},
Minio: Minio{
Address: "localhost",
Port: 9000,
AccessKeyID: "minioadmin",
SecretAccessKey: "minioadmin",
UseSSL: false,
BucketName: "a-bucket",
RootPath: "files",
},
Pulsar: Pulsar{
Address: "localhost",
Port: 6650,
MaxMessageSize: 5242880,
},
RocksMq: RocksMq{
Path: "/var/lib/milvus/rdb_data",
PageSize: 2147483648,
RetentionDuration: 10080,
RetentionSize: 8192,
},
Grpc: Grpc{
ServerMaxReceiveSize: 2147483647,
ServerMaxSendSize: 2147483647,
ClientMaxReceiveSize: 104857600,
ClientMaxSendSize: 104857600,
},
FlowGraph: FlowGraph{
QueueLengthLimit: 1024,
ParallelismLimit: 1024,
},
RootCoord: RootCoord{
Address: "localhost",
Port: 53100,
},
Proxy: Proxy{
Port: 53100,
MaxTaskCount: 1024,
},
QueryCoord: QueryCoord{
Address: "localhost",
Port: 19531,
AutoHandOff: true,
AutoBalance: true,
BalanceInterval: 60,
MemoryUsageLimitRatio: 90,
AutoBalanceMemoryUsageGapRatio: 30,
},
QueryNode: QueryNode{
Port: 21123,
GracefulTime: 0,
StatsPublishInterval: 1000,
SegcoreChunkRows: 32768,
},
IndexCoord: IndexCoord{
Address: "localhost",
Port: 31000,
},
IndexNode: IndexNode{
Port: 21121,
},
DataCoord: DataCoord{
Address: "localhost",
Port: 13333,
},
DataNode: DataNode{
Port: 21124,
InsertBufferSizeLimit: 16777216,
},
TimeTickInterval: 200,
TimeTickBufferSize: 512,
NameLengthLimit: 255,
FieldCountLimit: 256,
DimensionLimit: 32768,
ShardCountLimit: 256,
DMLChannelCount: 256,
PartitionCountLimit: 4096,
EnableIndexMinSegmentSize: 1024,
SIMDType: "auto",
Compaction: Compaction{
EnableCompaction: true,
},
Log: Log{
Level: "debug",
Format: "text",
File: LogFileCfg{
RootPath: "",
MaxSize: 300,
MaxAge: 10,
MaxBackups: 20,
},
},
LocalStorage: LocalStorage{
Enable: true,
Path: "/var/lib/milvus/data/",
},
SkipQueryChannelRecover: false,
Metrics: Metrics{
GitCommit: "",
DeployMode: "CLUSTER",
GitBuildTags: "",
GoVersion: "",
BuildTime: "",
},
GarbageCollector: GarbageCollector{
Interval: 3600,
MissedFileTolerance: 86400,
DroppedFileTolerance: 86400,
},
}
var globalConfig atomic.Value
func init() {
cfg := defaultCfg
SetGlobalConfig(&cfg)
}
func SetGlobalConfig(cfg *Config) {
globalConfig.Store(cfg)
}
func GetGlobalConfig() *Config {
return globalConfig.Load().(*Config)
}
func InitializeConfig(path string, enforceEnvCfg func(c *Config), enforceCmdCfg func(c *Config)) {
cfg := GetGlobalConfig()
if path != "" {
err := cfg.Load(path)
if err != nil {
if _, ok := err.(*ErrUndecodedConfig); ok {
log.Warn(err.Error())
} else {
log.Fatal("failed to load config", zap.String("filePath", path), zap.Error(err))
}
}
}
if enforceEnvCfg != nil {
enforceEnvCfg(cfg)
}
if enforceCmdCfg != nil {
enforceCmdCfg(cfg)
}
if err := cfg.Validate(); err != nil {
log.Fatal("config is not valid, please check the config file, environment variables and command options", zap.Error(err))
}
}
func (c *Config) Load(path string) error {
meta, err := toml.DecodeFile(path, c)
undecodedKeys := meta.Undecoded()
if len(undecodedKeys) > 0 && err == nil {
undecoded := make([]string, 0, len(undecodedKeys))
for _, k := range undecodedKeys {
undecoded = append(undecoded, k.String())
}
return &ErrUndecodedConfig{ConfigFile: path, Undecoded: undecoded}
}
return err
}
func (c *Config) Validate() error {
if c.Etcd.UseEmbed && c.Metrics.DeployMode != metricsinfo.StandaloneDeployMode {
return fmt.Errorf("can not use embed etcd in mode %s", c.Metrics.DeployMode)
}
if c.GarbageCollector.MissedFileTolerance < MinToleranceTime {
return fmt.Errorf("gc: missed file tolerance time can not be less than %d", MinToleranceTime)
}
if c.GarbageCollector.DroppedFileTolerance < MinToleranceTime {
return fmt.Errorf("gc: dropped file tolerance time can not be less than %d", MinToleranceTime)
}
return nil
}
type ErrUndecodedConfig struct {
ConfigFile string
Undecoded []string
}
func (e *ErrUndecodedConfig) Error() string {
return fmt.Sprintf("config file %s contains invalid configuration options: %s", e.ConfigFile, strings.Join(e.Undecoded, ","))
}

200
configs/config_test.go
View File

@ -1,200 +0,0 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// This file is not used for now.
package configs
import (
"os"
"path/filepath"
"testing"
"github.com/milvus-io/milvus/internal/util/metricsinfo"
"github.com/stretchr/testify/assert"
)
func TestLoadConfig(t *testing.T) {
tmpDir := t.TempDir()
cases := []struct {
name string
file string
content string
expectErr error
expectCfg Config
}{
{
"test load config",
"milvus.test.config.1.toml",
`timetick-interval = 200
timetick-buffer-size = 512
name-length-limit = 255
field-count-limit = 256
dimension-limit = 32768
shard-count-limit = 256
dml-channel-count = 256
partition-count-limit = 4096
enable-index-min-segment-size = 1024
simd-type = "auto"
skip-query-channel-recover = false
[etcd]
endpoints = ["localhost:2379"]
use-embed = false`,
nil,
Config{
TimeTickInterval: 200,
TimeTickBufferSize: 512,
NameLengthLimit: 255,
FieldCountLimit: 256,
DimensionLimit: 32768,
ShardCountLimit: 256,
DMLChannelCount: 256,
PartitionCountLimit: 4096,
EnableIndexMinSegmentSize: 1024,
SIMDType: "auto",
SkipQueryChannelRecover: false,
Etcd: Etcd{
Endpoints: []string{"localhost:2379"},
UseEmbed: false,
},
},
},
{
"test load undefied key",
"milvus.test.config.2.toml",
"undefied_key=200",
&ErrUndecodedConfig{filepath.Join(tmpDir, "milvus.test.config.2.toml"), []string{"undefied_key"}},
Config{},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
filePath := filepath.Join(tmpDir, c.file)
f, err := os.Create(filePath)
assert.Nil(t, err)
defer f.Close()
_, err = f.WriteString(c.content)
assert.Nil(t, err)
cfg := Config{}
err = cfg.Load(filePath)
assert.EqualValues(t, c.expectErr, err)
assert.EqualValues(t, c.expectCfg, cfg)
})
}
}
func TestValidate(t *testing.T) {
cases := []struct {
name string
cfg *Config
expectErr bool
}{
{
"test validate success",
&Config{
Etcd: Etcd{
UseEmbed: true,
},
Metrics: Metrics{
DeployMode: metricsinfo.StandaloneDeployMode,
},
GarbageCollector: GarbageCollector{
MissedFileTolerance: MinToleranceTime,
DroppedFileTolerance: MinToleranceTime,
},
},
false,
},
{
"test use embed etcd with cluster",
&Config{
Etcd: Etcd{
UseEmbed: true,
},
Metrics: Metrics{
DeployMode: metricsinfo.ClusterDeployMode,
},
GarbageCollector: GarbageCollector{
MissedFileTolerance: MinToleranceTime,
DroppedFileTolerance: MinToleranceTime,
},
},
true,
},
{
"test use little tolerance time",
&Config{
Etcd: Etcd{
UseEmbed: true,
},
Metrics: Metrics{
DeployMode: metricsinfo.StandaloneDeployMode,
},
GarbageCollector: GarbageCollector{
MissedFileTolerance: MinToleranceTime - 1,
DroppedFileTolerance: MinToleranceTime - 1,
},
},
true,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
assert.Equal(t, c.expectErr, c.cfg.Validate() != nil)
})
}
}
func TestEnforce(t *testing.T) {
cases := []struct {
name string
enforceEnv func(c *Config)
enforceCmd func(c *Config)
checkFunc func(t *testing.T, c *Config)
}{
{"test no enforce func", nil, nil, func(t *testing.T, c *Config) {}},
{
"test enforce cmd func",
func(c *Config) {
c.TimeTickInterval = 1024
},
nil,
func(t *testing.T, c *Config) {
assert.EqualValues(t, 1024, c.TimeTickInterval)
},
},
{
"test enforceCmd override enforceEnv",
func(c *Config) {
c.TimeTickInterval = 512
},
func(c *Config) {
c.TimeTickInterval = 1024
},
func(t *testing.T, c *Config) {
assert.EqualValues(t, 1024, c.TimeTickInterval)
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
InitializeConfig("", c.enforceEnv, c.enforceCmd)
cfg := GetGlobalConfig()
c.checkFunc(t, cfg)
})
}
}

133
configs/milvus.example.toml
View File

@ -1,133 +0,0 @@
# Licensed to the LF AI & Data foundation under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This file is not used for now.
timetick-interval = 200
timetick-buffer-size = 512
name-length-limit = 255
field-count-limit = 256
dimension-limit = 32768
shard-count-limit = 256
dml-channel-count = 256
partition-count-limit = 4096
enable-index-min-segment-size = 1024
simd-type = "auto"
skip-query-channel-recover = false
[etcd]
endpoints = ["localhost:2379"]
use-embed = false
[minio]
address = "localhost"
port = 9000
access-key-id = "minioadmin"
secret-access-key = "minioadmin"
use-ssl = false
bucket-name = "a-bucket"
root-path = "files"
[pulsar]
address = "localhost"
port = 6650
max-message-size = 5242880
[rocksmq]
path = "/var/lib/milvus/rdb_data"
page-size = 2147483648
retention-duration = 10080
retention-size = 8192
[grpc]
server-max-receive-size = 2147483647
server-max-send-size = 2147483647
client-max-receive-size = 104857600
client-max-send-size = 104857600
[rootcoord]
address = "localhost"
port = 53100
[proxy]
port = 19530
max-task-count = 1024
[querycoord]
address = "localhost"
port = 19531
auto-handoff = true
auto-balance = true
balance-interval = 60
memory-usage-limit-ratio = 90
auto-balance-memory-usage-gap-ratio = 30
[flowgraph]
queue-length-limit = 1024
parallelism-limit = 1024
[querynode]
port = 21123
graceful-time = 0
stats-publish-interval = 1000
segcore-chunk-rows = 32768
[indexcoord]
address = "localhost"
port = 31000
[indexnode]
port = 21121
[datacoord]
address = "localhost"
port = 13333
[gc]
interval = 3600
missed-files-tolerance = 86400
dropped-files-tolerance = 86400
[compaction]
enable-compaction = true
[datanode]
port = 21124
insert-buffer-size-limit = 16777216
[localstorage]
enable = true
path = "/var/lib/milvus/data/"
[log]
level = "debug"
format = "text"
[log.file]
root-path = ""
max-size = 300
max-age = 10
max-backups = 20

2
internal/datanode/channel_meta.go
View File

@ -543,7 +543,7 @@ func (c *ChannelMeta) mergeFlushedSegments(seg *Segment, planID UniqueID, compac
return fmt.Errorf("mismatch collection, ID=%d", seg.collectionID)
}
compactedFrom = lo.Filter[int64](compactedFrom, func(segID int64, _ int) bool {
compactedFrom = lo.Filter(compactedFrom, func(segID int64, _ int) bool {
// which means the segment is the `flushed` state
has := c.hasSegment(segID, true) && !c.hasSegment(segID, false)
if !has {

25
internal/distributed/datacoord/client/client.go
View File

@ -19,6 +19,7 @@ package grpcdatacoordclient
import (
"context"
"fmt"
"time"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
@ -37,9 +38,6 @@ import (
"google.golang.org/grpc"
)
// ClientParams is the parameters of client singleton
var ClientParams paramtable.GrpcClientConfig
var Params *paramtable.ComponentParam = paramtable.Get()
var _ types.DataCoord = (*Client)(nil)
@ -59,19 +57,20 @@ func NewClient(ctx context.Context, metaRoot string, etcdCli *clientv3.Client) (
log.Debug("DataCoordClient NewClient failed", zap.Error(err))
return nil, err
}
ClientParams.InitOnce(typeutil.DataCoordRole)
clientParams := &Params.DataCoordGrpcClientCfg
client := &Client{
grpcClient: &grpcclient.ClientBase[datapb.DataCoordClient]{
ClientMaxRecvSize: ClientParams.ClientMaxRecvSize,
ClientMaxSendSize: ClientParams.ClientMaxSendSize,
DialTimeout: ClientParams.DialTimeout,
KeepAliveTime: ClientParams.KeepAliveTime,
KeepAliveTimeout: ClientParams.KeepAliveTimeout,
ClientMaxRecvSize: clientParams.ClientMaxRecvSize.GetAsInt(),
ClientMaxSendSize: clientParams.ClientMaxSendSize.GetAsInt(),
DialTimeout: clientParams.DialTimeout.GetAsDuration(time.Millisecond),
KeepAliveTime: clientParams.KeepAliveTime.GetAsDuration(time.Millisecond),
KeepAliveTimeout: clientParams.KeepAliveTimeout.GetAsDuration(time.Millisecond),
RetryServiceNameConfig: "milvus.proto.data.DataCoord",
MaxAttempts: ClientParams.MaxAttempts,
InitialBackoff: ClientParams.InitialBackoff,
MaxBackoff: ClientParams.MaxBackoff,
BackoffMultiplier: ClientParams.BackoffMultiplier,
MaxAttempts: clientParams.MaxAttempts.GetAsInt(),
InitialBackoff: float32(clientParams.InitialBackoff.GetAsFloat()),
MaxBackoff: float32(clientParams.MaxBackoff.GetAsFloat()),
BackoffMultiplier: float32(clientParams.BackoffMultiplier.GetAsFloat()),
},
sess: sess,
}

35
internal/distributed/datacoord/service.go
View File

@ -19,7 +19,6 @@ package grpcdatacoord
import (
"context"
"fmt"
"io"
"net"
"strconv"
@ -47,12 +46,8 @@ import (
"github.com/milvus-io/milvus/internal/util/logutil"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/trace"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
// Params is the parameters for DataCoord grpc server
var Params paramtable.GrpcServerConfig
// Server is the grpc server of datacoord
type Server struct {
ctx context.Context
@ -83,31 +78,32 @@ func NewServer(ctx context.Context, factory dependency.Factory, opts ...datacoor
}
func (s *Server) init() error {
Params.InitOnce(typeutil.DataCoordRole)
etcdConfig := &paramtable.Get().EtcdCfg
Params := &paramtable.Get().DataCoordGrpcServerCfg
closer := trace.InitTracing("datacoord")
s.closer = closer
etcdCli, err := etcd.GetEtcdClient(
Params.EtcdCfg.UseEmbedEtcd.GetAsBool(),
Params.EtcdCfg.EtcdUseSSL.GetAsBool(),
Params.EtcdCfg.Endpoints.GetAsStrings(),
Params.EtcdCfg.EtcdTLSCert.GetValue(),
Params.EtcdCfg.EtcdTLSKey.GetValue(),
Params.EtcdCfg.EtcdTLSCACert.GetValue(),
Params.EtcdCfg.EtcdTLSMinVersion.GetValue())
etcdConfig.UseEmbedEtcd.GetAsBool(),
etcdConfig.EtcdUseSSL.GetAsBool(),
etcdConfig.Endpoints.GetAsStrings(),
etcdConfig.EtcdTLSCert.GetValue(),
etcdConfig.EtcdTLSKey.GetValue(),
etcdConfig.EtcdTLSCACert.GetValue(),
etcdConfig.EtcdTLSMinVersion.GetValue())
if err != nil {
log.Debug("DataCoord connect to etcd failed", zap.Error(err))
return err
}
s.etcdCli = etcdCli
s.dataCoord.SetEtcdClient(etcdCli)
s.dataCoord.SetAddress(fmt.Sprintf("%s:%d", Params.IP, Params.Port))
s.dataCoord.SetAddress(Params.GetAddress())
if s.indexCoord == nil {
var err error
log.Debug("create IndexCoord client for DataCoord")
s.indexCoord, err = icc.NewClient(s.ctx, Params.EtcdCfg.MetaRootPath.GetValue(), etcdCli)
s.indexCoord, err = icc.NewClient(s.ctx, etcdConfig.MetaRootPath.GetValue(), etcdCli)
if err != nil {
log.Warn("failed to create IndexCoord client for DataCoord", zap.Error(err))
return err
@ -137,8 +133,9 @@ func (s *Server) init() error {
}
func (s *Server) startGrpc() error {
Params := &paramtable.Get().DataCoordGrpcServerCfg
s.wg.Add(1)
go s.startGrpcLoop(Params.Port)
go s.startGrpcLoop(Params.Port.GetAsInt())
// wait for grpc server loop start
err := <-s.grpcErrChan
return err
@ -148,6 +145,7 @@ func (s *Server) startGrpcLoop(grpcPort int) {
defer logutil.LogPanic()
defer s.wg.Done()
Params := &paramtable.Get().DataCoordGrpcServerCfg
log.Debug("network port", zap.Int("port", grpcPort))
lis, err := net.Listen("tcp", ":"+strconv.Itoa(grpcPort))
if err != nil {
@ -173,8 +171,8 @@ func (s *Server) startGrpcLoop(grpcPort int) {
s.grpcServer = grpc.NewServer(
grpc.KeepaliveEnforcementPolicy(kaep),
grpc.KeepaliveParams(kasp),
grpc.MaxRecvMsgSize(Params.ServerMaxRecvSize),
grpc.MaxSendMsgSize(Params.ServerMaxSendSize),
grpc.MaxRecvMsgSize(Params.ServerMaxRecvSize.GetAsInt()),
grpc.MaxSendMsgSize(Params.ServerMaxSendSize.GetAsInt()),
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
ot.UnaryServerInterceptor(opts...),
logutil.UnaryTraceLoggerInterceptor)),
@ -205,6 +203,7 @@ func (s *Server) start() error {
// Stop stops the DataCoord server gracefully.
// Need to call the GracefulStop interface of grpc server and call the stop method of the inner DataCoord object.
func (s *Server) Stop() error {
Params := &paramtable.Get().DataCoordGrpcServerCfg
log.Debug("Datacoord stop", zap.String("Address", Params.GetAddress()))
var err error
if s.closer != nil {

23
internal/distributed/datanode/client/client.go
View File

@ -19,6 +19,7 @@ package grpcdatanodeclient
import (
"context"
"fmt"
"time"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
@ -32,8 +33,6 @@ import (
"google.golang.org/grpc"
)
var ClientParams paramtable.GrpcClientConfig
var Params *paramtable.ComponentParam = paramtable.Get()
// Client is the grpc client for DataNode
@ -47,20 +46,20 @@ func NewClient(ctx context.Context, addr string) (*Client, error) {
if addr == "" {
return nil, fmt.Errorf("address is empty")
}
ClientParams.InitOnce(typeutil.DataNodeRole)
clientParams := &Params.DataNodeGrpcClientCfg
client := &Client{
addr: addr,
grpcClient: &grpcclient.ClientBase[datapb.DataNodeClient]{
ClientMaxRecvSize: ClientParams.ClientMaxRecvSize,
ClientMaxSendSize: ClientParams.ClientMaxSendSize,
DialTimeout: ClientParams.DialTimeout,
KeepAliveTime: ClientParams.KeepAliveTime,
KeepAliveTimeout: ClientParams.KeepAliveTimeout,
ClientMaxRecvSize: clientParams.ClientMaxRecvSize.GetAsInt(),
ClientMaxSendSize: clientParams.ClientMaxSendSize.GetAsInt(),
DialTimeout: clientParams.DialTimeout.GetAsDuration(time.Millisecond),
KeepAliveTime: clientParams.KeepAliveTime.GetAsDuration(time.Millisecond),
KeepAliveTimeout: clientParams.KeepAliveTimeout.GetAsDuration(time.Millisecond),
RetryServiceNameConfig: "milvus.proto.data.DataNode",
MaxAttempts: ClientParams.MaxAttempts,
InitialBackoff: ClientParams.InitialBackoff,
MaxBackoff: ClientParams.MaxBackoff,
BackoffMultiplier: ClientParams.BackoffMultiplier,
MaxAttempts: clientParams.MaxAttempts.GetAsInt(),
InitialBackoff: float32(clientParams.InitialBackoff.GetAsFloat()),
MaxBackoff: float32(clientParams.MaxBackoff.GetAsFloat()),
BackoffMultiplier: float32(clientParams.BackoffMultiplier.GetAsFloat()),
},
}
client.grpcClient.SetRole(typeutil.DataNodeRole)

42
internal/distributed/datanode/service.go
View File

@ -49,12 +49,8 @@ import (
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/retry"
"github.com/milvus-io/milvus/internal/util/trace"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
// Params contains parameters for datanode grpc server.
var Params paramtable.GrpcServerConfig
type Server struct {
datanode types.DataNodeComponent
wg sync.WaitGroup
@ -96,8 +92,9 @@ func NewServer(ctx context.Context, factory dependency.Factory) (*Server, error)
}
func (s *Server) startGrpc() error {
Params := &paramtable.Get().DataNodeGrpcServerCfg
s.wg.Add(1)
go s.startGrpcLoop(Params.Port)
go s.startGrpcLoop(Params.Port.GetAsInt())
// wait for grpc server loop start
err := <-s.grpcErrChan
return err
@ -106,6 +103,7 @@ func (s *Server) startGrpc() error {
// startGrpcLoop starts the grep loop of datanode component.
func (s *Server) startGrpcLoop(grpcPort int) {
defer s.wg.Done()
Params := &paramtable.Get().DataNodeGrpcServerCfg
var kaep = keepalive.EnforcementPolicy{
MinTime: 5 * time.Second, // If a client pings more than once every 5 seconds, terminate the connection
PermitWithoutStream: true, // Allow pings even when there are no active streams
@ -134,8 +132,8 @@ func (s *Server) startGrpcLoop(grpcPort int) {
s.grpcServer = grpc.NewServer(
grpc.KeepaliveEnforcementPolicy(kaep),
grpc.KeepaliveParams(kasp),
grpc.MaxRecvMsgSize(Params.ServerMaxRecvSize),
grpc.MaxSendMsgSize(Params.ServerMaxSendSize),
grpc.MaxRecvMsgSize(Params.ServerMaxRecvSize.GetAsInt()),
grpc.MaxSendMsgSize(Params.ServerMaxSendSize.GetAsInt()),
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
ot.UnaryServerInterceptor(opts...),
logutil.UnaryTraceLoggerInterceptor)),
@ -184,6 +182,7 @@ func (s *Server) Run() error {
// Stop stops Datanode's grpc service.
func (s *Server) Stop() error {
Params := &paramtable.Get().DataNodeGrpcServerCfg
log.Debug("Datanode stop", zap.String("Address", Params.GetAddress()))
if s.closer != nil {
if err := s.closer.Close(); err != nil {
@ -223,31 +222,32 @@ func (s *Server) Stop() error {
// init initializes Datanode's grpc service.
func (s *Server) init() error {
etcdConfig := &paramtable.Get().EtcdCfg
Params := &paramtable.Get().DataNodeGrpcServerCfg
ctx := context.Background()
Params.InitOnce(typeutil.DataNodeRole)
if !funcutil.CheckPortAvailable(Params.Port) {
Params.Port = funcutil.GetAvailablePort()
log.Warn("DataNode found available port during init", zap.Int("port", Params.Port))
if !funcutil.CheckPortAvailable(Params.Port.GetAsInt()) {
paramtable.Get().Save(Params.Port.Key, fmt.Sprintf("%d", funcutil.GetAvailablePort()))
log.Warn("DataNode found available port during init", zap.Int("port", Params.Port.GetAsInt()))
}
etcdCli, err := etcd.GetEtcdClient(
Params.EtcdCfg.UseEmbedEtcd.GetAsBool(),
Params.EtcdCfg.EtcdUseSSL.GetAsBool(),
Params.EtcdCfg.Endpoints.GetAsStrings(),
Params.EtcdCfg.EtcdTLSCert.GetValue(),
Params.EtcdCfg.EtcdTLSKey.GetValue(),
Params.EtcdCfg.EtcdTLSCACert.GetValue(),
Params.EtcdCfg.EtcdTLSMinVersion.GetValue())
etcdConfig.UseEmbedEtcd.GetAsBool(),
etcdConfig.EtcdUseSSL.GetAsBool(),
etcdConfig.Endpoints.GetAsStrings(),
etcdConfig.EtcdTLSCert.GetValue(),
etcdConfig.EtcdTLSKey.GetValue(),
etcdConfig.EtcdTLSCACert.GetValue(),
etcdConfig.EtcdTLSMinVersion.GetValue())
if err != nil {
log.Error("failed to connect to etcd", zap.Error(err))
return err
}
s.etcdCli = etcdCli
s.SetEtcdClient(s.etcdCli)
s.datanode.SetAddress(fmt.Sprintf("%s:%d", Params.IP, Params.Port))
closer := trace.InitTracing(fmt.Sprintf("DataNode IP: %s, port: %d", Params.IP, Params.Port))
s.datanode.SetAddress(Params.GetAddress())
closer := trace.InitTracing(fmt.Sprintf("DataNode IP: %s, port: %d", Params.IP, Params.Port.GetAsInt()))
s.closer = closer
log.Info("DataNode address", zap.String("address", Params.IP+":"+strconv.Itoa(Params.Port)))
log.Info("DataNode address", zap.String("address", Params.IP+":"+strconv.Itoa(Params.Port.GetAsInt())))
err = s.startGrpc()
if err != nil {

23
internal/distributed/indexcoord/client/client.go
View File

@ -19,6 +19,7 @@ package grpcindexcoordclient
import (
"context"
"fmt"
"time"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
@ -37,8 +38,6 @@ import (
"github.com/milvus-io/milvus/internal/util/typeutil"
)
var ClientParams paramtable.GrpcClientConfig
var Params *paramtable.ComponentParam = paramtable.Get()
// Client is the grpc client of IndexCoord.
@ -55,19 +54,19 @@ func NewClient(ctx context.Context, metaRoot string, etcdCli *clientv3.Client) (
log.Debug("IndexCoordClient NewClient failed", zap.Error(err))
return nil, err
}
ClientParams.InitOnce(typeutil.IndexCoordRole)
ClientParams := &Params.IndexCoordGrpcClientCfg
client := &Client{
grpcClient: &grpcclient.ClientBase[indexpb.IndexCoordClient]{
ClientMaxRecvSize: ClientParams.ClientMaxRecvSize,
ClientMaxSendSize: ClientParams.ClientMaxSendSize,
DialTimeout: ClientParams.DialTimeout,
KeepAliveTime: ClientParams.KeepAliveTime,
KeepAliveTimeout: ClientParams.KeepAliveTimeout,
ClientMaxRecvSize: ClientParams.ClientMaxRecvSize.GetAsInt(),
ClientMaxSendSize: ClientParams.ClientMaxSendSize.GetAsInt(),
DialTimeout: ClientParams.DialTimeout.GetAsDuration(time.Millisecond),
KeepAliveTime: ClientParams.KeepAliveTime.GetAsDuration(time.Millisecond),
KeepAliveTimeout: ClientParams.KeepAliveTimeout.GetAsDuration(time.Millisecond),
RetryServiceNameConfig: "milvus.proto.index.IndexCoord",
MaxAttempts: ClientParams.MaxAttempts,
InitialBackoff: ClientParams.InitialBackoff,
MaxBackoff: ClientParams.MaxBackoff,
BackoffMultiplier: ClientParams.BackoffMultiplier,
MaxAttempts: ClientParams.MaxAttempts.GetAsInt(),
InitialBackoff: float32(ClientParams.InitialBackoff.GetAsFloat()),
MaxBackoff: float32(ClientParams.MaxBackoff.GetAsFloat()),
BackoffMultiplier: float32(ClientParams.BackoffMultiplier.GetAsFloat()),
},
sess: sess,
}

1
internal/distributed/indexcoord/client/client_test.go
View File

@ -38,7 +38,6 @@ import (
func TestIndexCoordClient(t *testing.T) {
paramtable.Init()
ClientParams.InitOnce(typeutil.IndexCoordRole)
ctx := context.Background()
factory := dependency.NewDefaultFactory(true)
server, err := grpcindexcoord.NewServer(ctx, factory)

31
internal/distributed/indexcoord/service.go
View File

@ -51,9 +51,6 @@ import (
"github.com/milvus-io/milvus/internal/util/typeutil"
)
// Params contains parameters for indexcoord grpc server.
var Params paramtable.GrpcServerConfig
// UniqueID is an alias of int64, is used as a unique identifier for the request.
type UniqueID = typeutil.UniqueID
@ -92,29 +89,30 @@ func (s *Server) Run() error {
// init initializes IndexCoord's grpc service.
func (s *Server) init() error {
Params.InitOnce(typeutil.IndexCoordRole)
etcdConfig := &paramtable.Get().EtcdCfg
Params := &paramtable.Get().IndexCoordGrpcServerCfg
closer := trace.InitTracing("IndexCoord")
s.closer = closer
etcdCli, err := etcd.GetEtcdClient(
Params.EtcdCfg.UseEmbedEtcd.GetAsBool(),
Params.EtcdCfg.EtcdUseSSL.GetAsBool(),
Params.EtcdCfg.Endpoints.GetAsStrings(),
Params.EtcdCfg.EtcdTLSCert.GetValue(),
Params.EtcdCfg.EtcdTLSKey.GetValue(),
Params.EtcdCfg.EtcdTLSCACert.GetValue(),
Params.EtcdCfg.EtcdTLSMinVersion.GetValue())
etcdConfig.UseEmbedEtcd.GetAsBool(),
etcdConfig.EtcdUseSSL.GetAsBool(),
etcdConfig.Endpoints.GetAsStrings(),
etcdConfig.EtcdTLSCert.GetValue(),
etcdConfig.EtcdTLSKey.GetValue(),
etcdConfig.EtcdTLSCACert.GetValue(),
etcdConfig.EtcdTLSMinVersion.GetValue())
if err != nil {
log.Debug("IndexCoord connect to etcd failed", zap.Error(err))
return err
}
s.etcdCli = etcdCli
s.indexcoord.SetEtcdClient(s.etcdCli)
s.indexcoord.SetAddress(fmt.Sprintf("%s:%d", Params.IP, Params.Port))
s.indexcoord.SetAddress(fmt.Sprintf("%s:%d", Params.IP, Params.Port.GetAsInt()))
s.loopWg.Add(1)
go s.startGrpcLoop(Params.Port)
go s.startGrpcLoop(Params.Port.GetAsInt())
// wait for grpc IndexCoord loop start
if err := <-s.grpcErrChan; err != nil {
log.Error("IndexCoord", zap.Any("init error", err))
@ -200,6 +198,7 @@ func (s *Server) start() error {
// Stop stops IndexCoord's grpc service.
func (s *Server) Stop() error {
Params := &paramtable.Get().DataCoordGrpcServerCfg
log.Debug("Indexcoord stop", zap.String("Address", Params.GetAddress()))
if s.closer != nil {
if err := s.closer.Close(); err != nil {
@ -301,8 +300,8 @@ func (s *Server) CheckHealth(ctx context.Context, request *milvuspb.CheckHealthR
// startGrpcLoop starts the grep loop of IndexCoord component.
func (s *Server) startGrpcLoop(grpcPort int) {
defer s.loopWg.Done()
Params := &paramtable.Get().DataCoordGrpcServerCfg
var kaep = keepalive.EnforcementPolicy{
MinTime: 5 * time.Second, // If a client pings more than once every 5 seconds, terminate the connection
PermitWithoutStream: true, // Allow pings even when there are no active streams
@ -328,8 +327,8 @@ func (s *Server) startGrpcLoop(grpcPort int) {
s.grpcServer = grpc.NewServer(
grpc.KeepaliveEnforcementPolicy(kaep),
grpc.KeepaliveParams(kasp),
grpc.MaxRecvMsgSize(Params.ServerMaxRecvSize),
grpc.MaxSendMsgSize(Params.ServerMaxSendSize),
grpc.MaxRecvMsgSize(Params.ServerMaxRecvSize.GetAsInt()),
grpc.MaxSendMsgSize(Params.ServerMaxSendSize.GetAsInt()),
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
ot.UnaryServerInterceptor(opts...),
logutil.UnaryTraceLoggerInterceptor)),

1
internal/distributed/indexcoord/service_test.go
View File

@ -39,7 +39,6 @@ func TestIndexCoordinateServer(t *testing.T) {
server, err := NewServer(ctx, factory)
assert.NoError(t, err)
assert.NotNil(t, server)
Params.InitOnce(typeutil.IndexCoordRole)
indexCoordClient := indexcoord.NewIndexCoordMock()
err = server.SetClient(indexCoordClient)

23
internal/distributed/indexnode/client/client.go
View File

@ -19,6 +19,7 @@ package grpcindexnodeclient
import (
"context"
"fmt"
"time"
"google.golang.org/grpc"
@ -33,8 +34,6 @@ import (
"github.com/milvus-io/milvus/internal/util/typeutil"
)
var ClientParams paramtable.GrpcClientConfig
var Params *paramtable.ComponentParam = paramtable.Get()
// Client is the grpc client of IndexNode.
@ -48,20 +47,20 @@ func NewClient(ctx context.Context, addr string, encryption bool) (*Client, erro
if addr == "" {
return nil, fmt.Errorf("address is empty")
}
ClientParams.InitOnce(typeutil.IndexNodeRole)
clientParams := &Params.IndexNodeGrpcClientCfg
client := &Client{
addr: addr,
grpcClient: &grpcclient.ClientBase[indexpb.IndexNodeClient]{
ClientMaxRecvSize: ClientParams.ClientMaxRecvSize,
ClientMaxSendSize: ClientParams.ClientMaxSendSize,
DialTimeout: ClientParams.DialTimeout,
KeepAliveTime: ClientParams.KeepAliveTime,
KeepAliveTimeout: ClientParams.KeepAliveTimeout,
ClientMaxRecvSize: clientParams.ClientMaxRecvSize.GetAsInt(),
ClientMaxSendSize: clientParams.ClientMaxSendSize.GetAsInt(),
DialTimeout: clientParams.DialTimeout.GetAsDuration(time.Millisecond),
KeepAliveTime: clientParams.KeepAliveTime.GetAsDuration(time.Millisecond),
KeepAliveTimeout: clientParams.KeepAliveTimeout.GetAsDuration(time.Millisecond),
RetryServiceNameConfig: "milvus.proto.index.IndexNode",
MaxAttempts: ClientParams.MaxAttempts,
InitialBackoff: ClientParams.InitialBackoff,
MaxBackoff: ClientParams.MaxBackoff,
BackoffMultiplier: ClientParams.BackoffMultiplier,
MaxAttempts: clientParams.MaxAttempts.GetAsInt(),
InitialBackoff: float32(clientParams.InitialBackoff.GetAsFloat()),
MaxBackoff: float32(clientParams.MaxBackoff.GetAsFloat()),
BackoffMultiplier: float32(clientParams.BackoffMultiplier.GetAsFloat()),
},
}
client.grpcClient.SetRole(typeutil.IndexNodeRole)

6
internal/distributed/indexnode/client/client_test.go
View File

@ -34,14 +34,10 @@ import (
"github.com/milvus-io/milvus/internal/util/metricsinfo"
"github.com/milvus-io/milvus/internal/util/mock"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
var ParamsGlobal paramtable.ComponentParam
func Test_NewClient(t *testing.T) {
paramtable.Init()
ClientParams.InitOnce(typeutil.IndexNodeRole)
ctx := context.Background()
client, err := NewClient(ctx, "", false)
assert.Nil(t, client)
@ -126,6 +122,7 @@ func Test_NewClient(t *testing.T) {
}
func TestIndexNodeClient(t *testing.T) {
paramtable.Init()
ctx := context.Background()
factory := dependency.NewDefaultFactory(true)
@ -134,7 +131,6 @@ func TestIndexNodeClient(t *testing.T) {
assert.NotNil(t, ins)
inm := indexnode.NewIndexNodeMock()
ParamsGlobal.InitOnce()
etcdCli, err := etcd.GetEtcdClient(
Params.EtcdCfg.UseEmbedEtcd.GetAsBool(),
Params.EtcdCfg.EtcdUseSSL.GetAsBool(),

36
internal/distributed/indexnode/service.go
View File

@ -45,11 +45,8 @@ import (
"github.com/milvus-io/milvus/internal/util/logutil"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/trace"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
var Params paramtable.GrpcServerConfig
// Server is the grpc wrapper of IndexNode.
type Server struct {
indexnode types.IndexNodeComponent
@ -82,6 +79,7 @@ func (s *Server) Run() error {
func (s *Server) startGrpcLoop(grpcPort int) {
defer s.loopWg.Done()
Params := &paramtable.Get().IndexNodeGrpcServerCfg
log.Debug("IndexNode", zap.String("network address", Params.GetAddress()), zap.Int("network port: ", grpcPort))
lis, err := net.Listen("tcp", ":"+strconv.Itoa(grpcPort))
if err != nil {
@ -107,8 +105,8 @@ func (s *Server) startGrpcLoop(grpcPort int) {
s.grpcServer = grpc.NewServer(
grpc.KeepaliveEnforcementPolicy(kaep),
grpc.KeepaliveParams(kasp),
grpc.MaxRecvMsgSize(Params.ServerMaxRecvSize),
grpc.MaxSendMsgSize(Params.ServerMaxSendSize),
grpc.MaxRecvMsgSize(Params.ServerMaxRecvSize.GetAsInt()),
grpc.MaxSendMsgSize(Params.ServerMaxSendSize.GetAsInt()),
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
ot.UnaryServerInterceptor(opts...),
logutil.UnaryTraceLoggerInterceptor)),
@ -124,11 +122,12 @@ func (s *Server) startGrpcLoop(grpcPort int) {
// init initializes IndexNode's grpc service.
func (s *Server) init() error {
etcdConfig := &paramtable.Get().EtcdCfg
Params := &paramtable.Get().IndexNodeGrpcServerCfg
var err error
Params.InitOnce(typeutil.IndexNodeRole)
if !funcutil.CheckPortAvailable(Params.Port) {
Params.Port = funcutil.GetAvailablePort()
log.Warn("IndexNode get available port when init", zap.Int("Port", Params.Port))
if !funcutil.CheckPortAvailable(Params.Port.GetAsInt()) {
paramtable.Get().Save(Params.Port.Key, fmt.Sprintf("%d", funcutil.GetAvailablePort()))
log.Warn("IndexNode get available port when init", zap.Int("Port", Params.Port.GetAsInt()))
}
closer := trace.InitTracing(fmt.Sprintf("IndexNode-%d", paramtable.GetNodeID()))
@ -144,7 +143,7 @@ func (s *Server) init() error {
}()
s.loopWg.Add(1)
go s.startGrpcLoop(Params.Port)
go s.startGrpcLoop(Params.Port.GetAsInt())
// wait for grpc server loop start
err = <-s.grpcErrChan
if err != nil {
@ -153,20 +152,20 @@ func (s *Server) init() error {
}
etcdCli, err := etcd.GetEtcdClient(
Params.EtcdCfg.UseEmbedEtcd.GetAsBool(),
Params.EtcdCfg.EtcdUseSSL.GetAsBool(),
Params.EtcdCfg.Endpoints.GetAsStrings(),
Params.EtcdCfg.EtcdTLSCert.GetValue(),
Params.EtcdCfg.EtcdTLSKey.GetValue(),
Params.EtcdCfg.EtcdTLSCACert.GetValue(),
Params.EtcdCfg.EtcdTLSMinVersion.GetValue())
etcdConfig.UseEmbedEtcd.GetAsBool(),
etcdConfig.EtcdUseSSL.GetAsBool(),
etcdConfig.Endpoints.GetAsStrings(),
etcdConfig.EtcdTLSCert.GetValue(),
etcdConfig.EtcdTLSKey.GetValue(),
etcdConfig.EtcdTLSCACert.GetValue(),
etcdConfig.EtcdTLSMinVersion.GetValue())
if err != nil {
log.Debug("IndexNode connect to etcd failed", zap.Error(err))
return err
}
s.etcdCli = etcdCli
s.indexnode.SetEtcdClient(etcdCli)
s.indexnode.SetAddress(fmt.Sprintf("%s:%d", Params.IP, Params.Port))
s.indexnode.SetAddress(Params.GetAddress())
err = s.indexnode.Init()
if err != nil {
log.Error("IndexNode Init failed", zap.Error(err))
@ -193,6 +192,7 @@ func (s *Server) start() error {
// Stop stops IndexNode's grpc service.
func (s *Server) Stop() error {
Params := &paramtable.Get().IndexNodeGrpcServerCfg
log.Debug("IndexNode stop", zap.String("Address", Params.GetAddress()))
if s.closer != nil {
if err := s.closer.Close(); err != nil {

23
internal/distributed/proxy/client/client.go
View File

@ -19,6 +19,7 @@ package grpcproxyclient
import (
"context"
"fmt"
"time"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
@ -32,8 +33,6 @@ import (
"google.golang.org/grpc"
)
var ClientParams paramtable.GrpcClientConfig
var Params *paramtable.ComponentParam = paramtable.Get()
// Client is the grpc client for Proxy
@ -47,20 +46,20 @@ func NewClient(ctx context.Context, addr string) (*Client, error) {
if addr == "" {
return nil, fmt.Errorf("address is empty")
}
ClientParams.InitOnce(typeutil.ProxyRole)
clientParams := &Params.ProxyGrpcClientCfg
client := &Client{
addr: addr,
grpcClient: &grpcclient.ClientBase[proxypb.ProxyClient]{
ClientMaxRecvSize: ClientParams.ClientMaxRecvSize,
ClientMaxSendSize: ClientParams.ClientMaxSendSize,
DialTimeout: ClientParams.DialTimeout,
KeepAliveTime: ClientParams.KeepAliveTime,
KeepAliveTimeout: ClientParams.KeepAliveTimeout,
ClientMaxRecvSize: clientParams.ClientMaxRecvSize.GetAsInt(),
ClientMaxSendSize: clientParams.ClientMaxSendSize.GetAsInt(),
DialTimeout: clientParams.DialTimeout.GetAsDuration(time.Millisecond),
KeepAliveTime: clientParams.KeepAliveTime.GetAsDuration(time.Millisecond),
KeepAliveTimeout: clientParams.KeepAliveTimeout.GetAsDuration(time.Millisecond),
RetryServiceNameConfig: "milvus.proto.proxy.Proxy",
MaxAttempts: ClientParams.MaxAttempts,
InitialBackoff: ClientParams.InitialBackoff,
MaxBackoff: ClientParams.MaxBackoff,
BackoffMultiplier: ClientParams.BackoffMultiplier,
MaxAttempts: clientParams.MaxAttempts.GetAsInt(),
InitialBackoff: float32(clientParams.InitialBackoff.GetAsFloat()),
MaxBackoff: float32(clientParams.MaxBackoff.GetAsFloat()),
BackoffMultiplier: float32(clientParams.BackoffMultiplier.GetAsFloat()),
},
}
client.grpcClient.SetRole(typeutil.ProxyRole)

62
internal/distributed/proxy/service.go
View File

@ -55,7 +55,6 @@ import (
"github.com/milvus-io/milvus/internal/util/logutil"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/trace"
"github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/opentracing/opentracing-go"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
@ -67,9 +66,6 @@ import (
"google.golang.org/grpc/status"
)
var Params paramtable.GrpcServerConfig
var HTTPParams paramtable.HTTPConfig
var (
errMissingMetadata = status.Errorf(codes.InvalidArgument, "missing metadata")
errInvalidToken = status.Errorf(codes.Unauthenticated, "invalid token")
@ -121,7 +117,7 @@ func (s *Server) registerHTTPServer() {
gin.DefaultWriter = io.Discard
gin.DefaultErrorWriter = io.Discard
}
if !HTTPParams.DebugMode {
if !proxy.Params.HTTPCfg.DebugMode.GetAsBool() {
gin.SetMode(gin.ReleaseMode)
}
ginHandler := gin.Default()
@ -142,6 +138,7 @@ func (s *Server) startExternalRPCServer(grpcExternalPort int, errChan chan error
func (s *Server) startExternalGrpc(grpcPort int, errChan chan error) {
defer s.wg.Done()
Params := &paramtable.Get().ProxyGrpcServerCfg
var kaep = keepalive.EnforcementPolicy{
MinTime: 5 * time.Second, // If a client pings more than once every 5 seconds, terminate the connection
PermitWithoutStream: true, // Allow pings even when there are no active streams
@ -173,8 +170,8 @@ func (s *Server) startExternalGrpc(grpcPort int, errChan chan error) {
grpcOpts := []grpc.ServerOption{
grpc.KeepaliveEnforcementPolicy(kaep),
grpc.KeepaliveParams(kasp),
grpc.MaxRecvMsgSize(Params.ServerMaxRecvSize),
grpc.MaxSendMsgSize(Params.ServerMaxSendSize),
grpc.MaxRecvMsgSize(Params.ServerMaxRecvSize.GetAsInt()),
grpc.MaxSendMsgSize(Params.ServerMaxSendSize.GetAsInt()),
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
ot.UnaryServerInterceptor(opts...),
grpc_auth.UnaryServerInterceptor(proxy.AuthenticationInterceptor),
@ -186,16 +183,16 @@ func (s *Server) startExternalGrpc(grpcPort int, errChan chan error) {
)),
}
if Params.TLSMode == 1 {
creds, err := credentials.NewServerTLSFromFile(Params.ServerPemPath, Params.ServerKeyPath)
if Params.TLSMode.GetAsInt() == 1 {
creds, err := credentials.NewServerTLSFromFile(Params.ServerPemPath.GetValue(), Params.ServerKeyPath.GetValue())
if err != nil {
log.Warn("proxy can't create creds", zap.Error(err))
errChan <- err
return
}
grpcOpts = append(grpcOpts, grpc.Creds(creds))
} else if Params.TLSMode == 2 {
cert, err := tls.LoadX509KeyPair(Params.ServerPemPath, Params.ServerKeyPath)
} else if Params.TLSMode.GetAsInt() == 2 {
cert, err := tls.LoadX509KeyPair(Params.ServerPemPath.GetValue(), Params.ServerKeyPath.GetValue())
if err != nil {
log.Warn("proxy cant load x509 key pair", zap.Error(err))
errChan <- err
@ -203,7 +200,7 @@ func (s *Server) startExternalGrpc(grpcPort int, errChan chan error) {
}
certPool := x509.NewCertPool()
rootBuf, err := ioutil.ReadFile(Params.CaPemPath)
rootBuf, err := ioutil.ReadFile(Params.CaPemPath.GetValue())
if err != nil {
log.Warn("failed read ca pem", zap.Error(err))
errChan <- err
@ -241,6 +238,7 @@ func (s *Server) startExternalGrpc(grpcPort int, errChan chan error) {
func (s *Server) startInternalGrpc(grpcPort int, errChan chan error) {
defer s.wg.Done()
Params := &paramtable.Get().ProxyGrpcServerCfg
var kaep = keepalive.EnforcementPolicy{
MinTime: 5 * time.Second, // If a client pings more than once every 5 seconds, terminate the connection
PermitWithoutStream: true, // Allow pings even when there are no active streams
@ -264,8 +262,8 @@ func (s *Server) startInternalGrpc(grpcPort int, errChan chan error) {
s.grpcInternalServer = grpc.NewServer(
grpc.KeepaliveEnforcementPolicy(kaep),
grpc.KeepaliveParams(kasp),
grpc.MaxRecvMsgSize(Params.ServerMaxRecvSize),
grpc.MaxSendMsgSize(Params.ServerMaxSendSize),
grpc.MaxRecvMsgSize(Params.ServerMaxRecvSize.GetAsInt()),
grpc.MaxSendMsgSize(Params.ServerMaxSendSize.GetAsInt()),
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
ot.UnaryServerInterceptor(opts...),
logutil.UnaryTraceLoggerInterceptor,
@ -306,56 +304,57 @@ func (s *Server) Run() error {
}
func (s *Server) init() error {
Params.InitOnce(typeutil.ProxyRole)
etcdConfig := &paramtable.Get().EtcdCfg
Params := &paramtable.Get().ProxyGrpcServerCfg
log.Debug("Proxy init service's parameter table done")
HTTPParams.InitOnce()
HTTPParams := paramtable.Get().HTTPCfg
log.Debug("Proxy init http server's parameter table done")
if !funcutil.CheckPortAvailable(Params.Port) {
Params.Port = funcutil.GetAvailablePort()
log.Warn("Proxy get available port when init", zap.Int("Port", Params.Port))
if !funcutil.CheckPortAvailable(Params.Port.GetAsInt()) {
paramtable.Get().Save(Params.Port.Key, fmt.Sprintf("%d", funcutil.GetAvailablePort()))
log.Warn("Proxy get available port when init", zap.Int("Port", Params.Port.GetAsInt()))
}
log.Debug("init Proxy's parameter table done", zap.String("internal address", Params.GetInternalAddress()), zap.String("external address", Params.GetAddress()))
serviceName := fmt.Sprintf("Proxy ip: %s, port: %d", Params.IP, Params.Port)
serviceName := fmt.Sprintf("Proxy ip: %s, port: %d", Params.IP, Params.Port.GetAsInt())
closer := trace.InitTracing(serviceName)
s.closer = closer
log.Debug("init Proxy's tracer done", zap.String("service name", serviceName))
etcdCli, err := etcd.GetEtcdClient(
Params.EtcdCfg.UseEmbedEtcd.GetAsBool(),
Params.EtcdCfg.EtcdUseSSL.GetAsBool(),
Params.EtcdCfg.Endpoints.GetAsStrings(),
Params.EtcdCfg.EtcdTLSCert.GetValue(),
Params.EtcdCfg.EtcdTLSKey.GetValue(),
Params.EtcdCfg.EtcdTLSCACert.GetValue(),
Params.EtcdCfg.EtcdTLSMinVersion.GetValue())
etcdConfig.UseEmbedEtcd.GetAsBool(),
etcdConfig.EtcdUseSSL.GetAsBool(),
etcdConfig.Endpoints.GetAsStrings(),
etcdConfig.EtcdTLSCert.GetValue(),
etcdConfig.EtcdTLSKey.GetValue(),
etcdConfig.EtcdTLSCACert.GetValue(),
etcdConfig.EtcdTLSMinVersion.GetValue())
if err != nil {
log.Debug("Proxy connect to etcd failed", zap.Error(err))
return err
}
s.etcdCli = etcdCli
s.proxy.SetEtcdClient(s.etcdCli)
s.proxy.SetAddress(fmt.Sprintf("%s:%d", Params.IP, Params.InternalPort))
s.proxy.SetAddress(Params.GetInternalAddress())
errChan := make(chan error, 1)
{
s.startInternalRPCServer(Params.InternalPort, errChan)
s.startInternalRPCServer(Params.InternalPort.GetAsInt(), errChan)
if err := <-errChan; err != nil {
log.Error("failed to create internal rpc server", zap.Error(err))
return err
}
}
{
s.startExternalRPCServer(Params.Port, errChan)
s.startExternalRPCServer(Params.Port.GetAsInt(), errChan)
if err := <-errChan; err != nil {
log.Error("failed to create external rpc server", zap.Error(err))
return err
}
}
if HTTPParams.Enabled {
if HTTPParams.Enabled.GetAsBool() {
registerHTTPHandlerOnce.Do(func() {
log.Info("register http server of proxy")
s.registerHTTPServer()
@ -509,6 +508,7 @@ func (s *Server) start() error {
// Stop stop the Proxy Server
func (s *Server) Stop() error {
Params := &paramtable.Get().ProxyGrpcServerCfg
log.Debug("Proxy stop", zap.String("internal address", Params.GetInternalAddress()), zap.String("external address", Params.GetInternalAddress()))
var err error
if s.closer != nil {

62
internal/distributed/proxy/service_test.go
View File

@ -54,6 +54,12 @@ import (
"google.golang.org/grpc/health/grpc_health_v1"
)
func TestMain(m *testing.M) {
paramtable.Init()
code := m.Run()
os.Exit(code)
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
type MockBase struct {
mock.Mock
@ -984,6 +990,7 @@ func withCredential(clientPemPath, clientKeyPath, clientCaPath string) (credenti
// waitForGrpcReady block until service available or panic after times out.
func waitForGrpcReady(opt *WaitOption) {
Params := &paramtable.Get().ProxyGrpcServerCfg
ticker := time.NewTicker(opt.Duration)
ch := make(chan error, 1)
@ -995,7 +1002,7 @@ func waitForGrpcReady(opt *WaitOption) {
if opt.TLSMode == 1 || opt.TLSMode == 2 {
var creds credentials.TransportCredentials
if opt.TLSMode == 1 {
creds, err = credentials.NewClientTLSFromFile(Params.ServerPemPath, "localhost")
creds, err = credentials.NewClientTLSFromFile(Params.ServerPemPath.GetValue(), "localhost")
} else {
creds, err = withCredential(opt.ClientPemPath, opt.ClientKeyPath, opt.CaPath)
}
@ -1034,8 +1041,9 @@ var clientKeyPath = "../../../configs/cert/client.key"
// waitForServerReady wait for internal grpc service and external service to be ready, according to the params.
func waitForServerReady() {
waitForGrpcReady(newWaitOption(waitDuration, Params.InternalPort, 0, "", "", ""))
waitForGrpcReady(newWaitOption(waitDuration, Params.Port, Params.TLSMode, clientPemPath, clientKeyPath, Params.CaPemPath))
Params := &paramtable.Get().ProxyGrpcServerCfg
waitForGrpcReady(newWaitOption(waitDuration, Params.InternalPort.GetAsInt(), 0, "", "", ""))
waitForGrpcReady(newWaitOption(waitDuration, Params.Port.GetAsInt(), Params.TLSMode.GetAsInt(), clientPemPath, clientKeyPath, Params.CaPemPath.GetValue()))
}
func runAndWaitForServerReady(server *Server) error {
@ -1502,8 +1510,7 @@ func Test_NewServer_HTTPServer_Enabled(t *testing.T) {
server.queryCoordClient = &MockQueryCoord{}
server.dataCoordClient = &MockDataCoord{}
HTTPParams.InitOnce()
HTTPParams.Enabled = true
paramtable.Get().Save(proxy.Params.HTTPCfg.Enabled.Key, "true")
err = runAndWaitForServerReady(server)
assert.Nil(t, err)
@ -1536,13 +1543,13 @@ func getServer(t *testing.T) *Server {
func Test_NewServer_TLS_TwoWay(t *testing.T) {
server := getServer(t)
Params := &paramtable.Get().ProxyGrpcServerCfg
Params.InitOnce("proxy")
Params.TLSMode = 2
Params.ServerPemPath = "../../../configs/cert/server.pem"
Params.ServerKeyPath = "../../../configs/cert/server.key"
Params.CaPemPath = "../../../configs/cert/ca.pem"
HTTPParams.Enabled = false
paramtable.Get().Save(Params.TLSMode.Key, "2")
paramtable.Get().Save(Params.ServerPemPath.Key, "../../../configs/cert/server.pem")
paramtable.Get().Save(Params.ServerKeyPath.Key, "../../../configs/cert/server.key")
paramtable.Get().Save(Params.CaPemPath.Key, "../../../configs/cert/ca.pem")
paramtable.Get().Save(proxy.Params.HTTPCfg.Enabled.Key, "false")
err := runAndWaitForServerReady(server)
assert.Nil(t, err)
@ -1553,12 +1560,12 @@ func Test_NewServer_TLS_TwoWay(t *testing.T) {
func Test_NewServer_TLS_OneWay(t *testing.T) {
server := getServer(t)
Params := &paramtable.Get().ProxyGrpcServerCfg
Params.InitOnce("proxy")
Params.TLSMode = 1
Params.ServerPemPath = "../../../configs/cert/server.pem"
Params.ServerKeyPath = "../../../configs/cert/server.key"
HTTPParams.Enabled = false
paramtable.Get().Save(Params.TLSMode.Key, "1")
paramtable.Get().Save(Params.ServerPemPath.Key, "../../../configs/cert/server.pem")
paramtable.Get().Save(Params.ServerKeyPath.Key, "../../../configs/cert/server.key")
paramtable.Get().Save(proxy.Params.HTTPCfg.Enabled.Key, "false")
err := runAndWaitForServerReady(server)
assert.Nil(t, err)
@ -1569,31 +1576,30 @@ func Test_NewServer_TLS_OneWay(t *testing.T) {
func Test_NewServer_TLS_FileNotExisted(t *testing.T) {
server := getServer(t)
Params := &paramtable.Get().ProxyGrpcServerCfg
Params.InitOnce("proxy")
Params.TLSMode = 1
Params.ServerPemPath = "../not/existed/server.pem"
Params.ServerKeyPath = "../../../configs/cert/server.key"
HTTPParams.Enabled = false
paramtable.Get().Save(Params.TLSMode.Key, "1")
paramtable.Get().Save(Params.ServerPemPath.Key, "../not/existed/server.pem")
paramtable.Get().Save(Params.ServerKeyPath.Key, "../../../configs/cert/server.key")
paramtable.Get().Save(proxy.Params.HTTPCfg.Enabled.Key, "false")
err := runAndWaitForServerReady(server)
assert.NotNil(t, err)
server.Stop()
Params.TLSMode = 2
Params.ServerPemPath = "../not/existed/server.pem"
Params.CaPemPath = "../../../configs/cert/ca.pem"
paramtable.Get().Save(Params.TLSMode.Key, "2")
paramtable.Get().Save(Params.ServerPemPath.Key, "../not/existed/server.pem")
paramtable.Get().Save(Params.CaPemPath.Key, "../../../configs/cert/ca.pem")
err = runAndWaitForServerReady(server)
assert.NotNil(t, err)
server.Stop()
Params.ServerPemPath = "../../../configs/cert/server.pem"
Params.CaPemPath = "../not/existed/ca.pem"
paramtable.Get().Save(Params.ServerPemPath.Key, "../../../configs/cert/server.pem")
paramtable.Get().Save(Params.CaPemPath.Key, "../not/existed/ca.pem")
err = runAndWaitForServerReady(server)
assert.NotNil(t, err)
server.Stop()
Params.ServerPemPath = "../../../configs/cert/server.pem"
Params.CaPemPath = "service.go"
paramtable.Get().Save(Params.CaPemPath.Key, "service.go")
err = runAndWaitForServerReady(server)
assert.NotNil(t, err)
server.Stop()

23
internal/distributed/querycoord/client/client.go
View File

@ -19,6 +19,7 @@ package grpcquerycoordclient
import (
"context"
"fmt"
"time"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
@ -36,8 +37,6 @@ import (
"google.golang.org/grpc"
)
var ClientParams paramtable.GrpcClientConfig
var Params *paramtable.ComponentParam = paramtable.Get()
// Client is the grpc client of QueryCoord.
@ -54,19 +53,19 @@ func NewClient(ctx context.Context, metaRoot string, etcdCli *clientv3.Client) (
log.Debug("QueryCoordClient NewClient failed", zap.Error(err))
return nil, err
}
ClientParams.InitOnce(typeutil.QueryCoordRole)
clientParams := &Params.QueryCoordGrpcClientCfg
client := &Client{
grpcClient: &grpcclient.ClientBase[querypb.QueryCoordClient]{
ClientMaxRecvSize: ClientParams.ClientMaxRecvSize,
ClientMaxSendSize: ClientParams.ClientMaxSendSize,
DialTimeout: ClientParams.DialTimeout,
KeepAliveTime: ClientParams.KeepAliveTime,
KeepAliveTimeout: ClientParams.KeepAliveTimeout,
ClientMaxRecvSize: clientParams.ClientMaxRecvSize.GetAsInt(),
ClientMaxSendSize: clientParams.ClientMaxSendSize.GetAsInt(),
DialTimeout: clientParams.DialTimeout.GetAsDuration(time.Millisecond),
KeepAliveTime: clientParams.KeepAliveTime.GetAsDuration(time.Millisecond),
KeepAliveTimeout: clientParams.KeepAliveTimeout.GetAsDuration(time.Millisecond),
RetryServiceNameConfig: "milvus.proto.query.QueryCoord",
MaxAttempts: ClientParams.MaxAttempts,
InitialBackoff: ClientParams.InitialBackoff,
MaxBackoff: ClientParams.MaxBackoff,
BackoffMultiplier: ClientParams.BackoffMultiplier,
MaxAttempts: clientParams.MaxAttempts.GetAsInt(),
InitialBackoff: float32(clientParams.InitialBackoff.GetAsFloat()),
MaxBackoff: float32(clientParams.MaxBackoff.GetAsFloat()),
BackoffMultiplier: float32(clientParams.BackoffMultiplier.GetAsFloat()),
},
sess: sess,
}

32
internal/distributed/querycoord/service.go
View File

@ -18,7 +18,6 @@ package grpcquerycoord
import (
"context"
"fmt"
"io"
"net"
"strconv"
@ -48,11 +47,8 @@ import (
"github.com/milvus-io/milvus/internal/util/logutil"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/trace"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
var Params paramtable.GrpcServerConfig
// Server is the grpc server of QueryCoord.
type Server struct {
wg sync.WaitGroup
@ -110,29 +106,30 @@ func (s *Server) Run() error {
// init initializes QueryCoord's grpc service.
func (s *Server) init() error {
Params.InitOnce(typeutil.QueryCoordRole)
etcdConfig := &paramtable.Get().EtcdCfg
Params := &paramtable.Get().QueryCoordGrpcServerCfg
closer := trace.InitTracing("querycoord")
s.closer = closer
etcdCli, err := etcd.GetEtcdClient(
Params.EtcdCfg.UseEmbedEtcd.GetAsBool(),
Params.EtcdCfg.EtcdUseSSL.GetAsBool(),
Params.EtcdCfg.Endpoints.GetAsStrings(),
Params.EtcdCfg.EtcdTLSCert.GetValue(),
Params.EtcdCfg.EtcdTLSKey.GetValue(),
Params.EtcdCfg.EtcdTLSCACert.GetValue(),
Params.EtcdCfg.EtcdTLSMinVersion.GetValue())
etcdConfig.UseEmbedEtcd.GetAsBool(),
etcdConfig.EtcdUseSSL.GetAsBool(),
etcdConfig.Endpoints.GetAsStrings(),
etcdConfig.EtcdTLSCert.GetValue(),
etcdConfig.EtcdTLSKey.GetValue(),
etcdConfig.EtcdTLSCACert.GetValue(),
etcdConfig.EtcdTLSMinVersion.GetValue())
if err != nil {
log.Debug("QueryCoord connect to etcd failed", zap.Error(err))
return err
}
s.etcdCli = etcdCli
s.SetEtcdClient(etcdCli)
s.queryCoord.SetAddress(fmt.Sprintf("%s:%d", Params.IP, Params.Port))
s.queryCoord.SetAddress(Params.GetAddress())
s.wg.Add(1)
go s.startGrpcLoop(Params.Port)
go s.startGrpcLoop(Params.Port.GetAsInt())
// wait for grpc server loop start
err = <-s.grpcErrChan
if err != nil {
@ -238,8 +235,8 @@ func (s *Server) init() error {
}
func (s *Server) startGrpcLoop(grpcPort int) {
defer s.wg.Done()
Params := &paramtable.Get().QueryCoordGrpcServerCfg
var kaep = keepalive.EnforcementPolicy{
MinTime: 5 * time.Second, // If a client pings more than once every 5 seconds, terminate the connection
PermitWithoutStream: true, // Allow pings even when there are no active streams
@ -264,8 +261,8 @@ func (s *Server) startGrpcLoop(grpcPort int) {
s.grpcServer = grpc.NewServer(
grpc.KeepaliveEnforcementPolicy(kaep),
grpc.KeepaliveParams(kasp),
grpc.MaxRecvMsgSize(Params.ServerMaxRecvSize),
grpc.MaxSendMsgSize(Params.ServerMaxSendSize),
grpc.MaxRecvMsgSize(Params.ServerMaxRecvSize.GetAsInt()),
grpc.MaxSendMsgSize(Params.ServerMaxSendSize.GetAsInt()),
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
ot.UnaryServerInterceptor(opts...),
logutil.UnaryTraceLoggerInterceptor)),
@ -291,6 +288,7 @@ func (s *Server) start() error {
// Stop stops QueryCoord's grpc service.
func (s *Server) Stop() error {
Params := &paramtable.Get().QueryCoordGrpcServerCfg
log.Debug("QueryCoord stop", zap.String("Address", Params.GetAddress()))
if s.closer != nil {
if err := s.closer.Close(); err != nil {

9
internal/distributed/querycoord/service_test.go
View File

@ -19,10 +19,12 @@ package grpcquerycoord
import (
"context"
"errors"
"os"
"testing"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
@ -261,6 +263,13 @@ func (m *MockIndexCoord) GetComponentStates(ctx context.Context) (*milvuspb.Comp
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
func TestMain(m *testing.M) {
paramtable.Init()
code := m.Run()
os.Exit(code)
}
func Test_NewServer(t *testing.T) {
ctx := context.Background()
server, err := NewServer(ctx, nil)

23
internal/distributed/querynode/client/client.go
View File

@ -19,6 +19,7 @@ package grpcquerynodeclient
import (
"context"
"fmt"
"time"
"google.golang.org/grpc"
@ -33,8 +34,6 @@ import (
"github.com/milvus-io/milvus/internal/util/typeutil"
)
var ClientParams paramtable.GrpcClientConfig
var Params *paramtable.ComponentParam = paramtable.Get()
// Client is the grpc client of QueryNode.
@ -48,20 +47,20 @@ func NewClient(ctx context.Context, addr string) (*Client, error) {
if addr == "" {
return nil, fmt.Errorf("addr is empty")
}
ClientParams.InitOnce(typeutil.QueryNodeRole)
clientParams := &Params.QueryNodeGrpcClientCfg
client := &Client{
addr: addr,
grpcClient: &grpcclient.ClientBase[querypb.QueryNodeClient]{
ClientMaxRecvSize: ClientParams.ClientMaxRecvSize,
ClientMaxSendSize: ClientParams.ClientMaxSendSize,
DialTimeout: ClientParams.DialTimeout,
KeepAliveTime: ClientParams.KeepAliveTime,
KeepAliveTimeout: ClientParams.KeepAliveTimeout,
ClientMaxRecvSize: clientParams.ClientMaxRecvSize.GetAsInt(),
ClientMaxSendSize: clientParams.ClientMaxSendSize.GetAsInt(),
DialTimeout: clientParams.DialTimeout.GetAsDuration(time.Millisecond),
KeepAliveTime: clientParams.KeepAliveTime.GetAsDuration(time.Millisecond),
KeepAliveTimeout: clientParams.KeepAliveTimeout.GetAsDuration(time.Millisecond),
RetryServiceNameConfig: "milvus.proto.query.QueryNode",
MaxAttempts: ClientParams.MaxAttempts,
InitialBackoff: ClientParams.InitialBackoff,
MaxBackoff: ClientParams.MaxBackoff,
BackoffMultiplier: ClientParams.BackoffMultiplier,
MaxAttempts: clientParams.MaxAttempts.GetAsInt(),
InitialBackoff: float32(clientParams.InitialBackoff.GetAsFloat()),
MaxBackoff: float32(clientParams.MaxBackoff.GetAsFloat()),
BackoffMultiplier: float32(clientParams.BackoffMultiplier.GetAsFloat()),
},
}
client.grpcClient.SetRole(typeutil.QueryNodeRole)

4
internal/distributed/querynode/client/client_test.go
View File

@ -24,14 +24,12 @@ import (
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/util/mock"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/stretchr/testify/assert"
"google.golang.org/grpc"
)
func Test_NewClient(t *testing.T) {
paramtable.Init()
ClientParams.InitOnce(typeutil.QueryNodeRole)
ctx := context.Background()
client, err := NewClient(ctx, "")
@ -42,8 +40,6 @@ func Test_NewClient(t *testing.T) {
assert.Nil(t, err)
assert.NotNil(t, client)
ClientParams.InitOnce(typeutil.QueryNodeRole)
err = client.Start()
assert.Nil(t, err)

39
internal/distributed/querynode/service.go
View File

@ -49,8 +49,6 @@ import (
"github.com/milvus-io/milvus/internal/util/typeutil"
)
var Params paramtable.GrpcServerConfig
// UniqueID is an alias for type typeutil.UniqueID, used as a unique identifier for the request.
type UniqueID = typeutil.UniqueID
@ -88,36 +86,37 @@ func NewServer(ctx context.Context, factory dependency.Factory) (*Server, error)
// init initializes QueryNode's grpc service.
func (s *Server) init() error {
Params.InitOnce(typeutil.QueryNodeRole)
etcdConfig := &paramtable.Get().EtcdCfg
Params := &paramtable.Get().QueryNodeGrpcServerCfg
if !funcutil.CheckPortAvailable(Params.Port) {
Params.Port = funcutil.GetAvailablePort()
log.Warn("QueryNode get available port when init", zap.Int("Port", Params.Port))
if !funcutil.CheckPortAvailable(Params.Port.GetAsInt()) {
paramtable.Get().Save(Params.Port.Key, fmt.Sprintf("%d", funcutil.GetAvailablePort()))
log.Warn("QueryNode get available port when init", zap.Int("Port", Params.Port.GetAsInt()))
}
closer := trace.InitTracing(fmt.Sprintf("query_node ip: %s, port: %d", Params.IP, Params.Port))
closer := trace.InitTracing(fmt.Sprintf("query_node ip: %s, port: %d", Params.IP, Params.Port.GetAsInt()))
s.closer = closer
log.Debug("QueryNode", zap.Int("port", Params.Port))
log.Debug("QueryNode", zap.Int("port", Params.Port.GetAsInt()))
etcdCli, err := etcd.GetEtcdClient(
Params.EtcdCfg.UseEmbedEtcd.GetAsBool(),
Params.EtcdCfg.EtcdUseSSL.GetAsBool(),
Params.EtcdCfg.Endpoints.GetAsStrings(),
Params.EtcdCfg.EtcdTLSCert.GetValue(),
Params.EtcdCfg.EtcdTLSKey.GetValue(),
Params.EtcdCfg.EtcdTLSCACert.GetValue(),
Params.EtcdCfg.EtcdTLSMinVersion.GetValue())
etcdConfig.UseEmbedEtcd.GetAsBool(),
etcdConfig.EtcdUseSSL.GetAsBool(),
etcdConfig.Endpoints.GetAsStrings(),
etcdConfig.EtcdTLSCert.GetValue(),
etcdConfig.EtcdTLSKey.GetValue(),
etcdConfig.EtcdTLSCACert.GetValue(),
etcdConfig.EtcdTLSMinVersion.GetValue())
if err != nil {
log.Debug("QueryNode connect to etcd failed", zap.Error(err))
return err
}
s.etcdCli = etcdCli
s.SetEtcdClient(etcdCli)
s.querynode.SetAddress(fmt.Sprintf("%s:%d", Params.IP, Params.Port))
s.querynode.SetAddress(Params.GetAddress())
log.Debug("QueryNode connect to etcd successfully")
s.wg.Add(1)
go s.startGrpcLoop(Params.Port)
go s.startGrpcLoop(Params.Port.GetAsInt())
// wait for grpc server loop start
err = <-s.grpcErrChan
if err != nil {
@ -150,6 +149,7 @@ func (s *Server) start() error {
// startGrpcLoop starts the grpc loop of QueryNode component.
func (s *Server) startGrpcLoop(grpcPort int) {
defer s.wg.Done()
Params := &paramtable.Get().QueryNodeGrpcServerCfg
var kaep = keepalive.EnforcementPolicy{
MinTime: 5 * time.Second, // If a client pings more than once every 5 seconds, terminate the connection
PermitWithoutStream: true, // Allow pings even when there are no active streams
@ -182,8 +182,8 @@ func (s *Server) startGrpcLoop(grpcPort int) {
s.grpcServer = grpc.NewServer(
grpc.KeepaliveEnforcementPolicy(kaep),
grpc.KeepaliveParams(kasp),
grpc.MaxRecvMsgSize(Params.ServerMaxRecvSize),
grpc.MaxSendMsgSize(Params.ServerMaxSendSize),
grpc.MaxRecvMsgSize(Params.ServerMaxRecvSize.GetAsInt()),
grpc.MaxSendMsgSize(Params.ServerMaxSendSize.GetAsInt()),
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
ot.UnaryServerInterceptor(opts...),
logutil.UnaryTraceLoggerInterceptor)),
@ -220,6 +220,7 @@ func (s *Server) Run() error {
// Stop stops QueryNode's grpc service.
func (s *Server) Stop() error {
Params := &paramtable.Get().QueryNodeGrpcServerCfg
log.Debug("QueryNode stop", zap.String("Address", Params.GetAddress()))
err := s.querynode.Stop()
if err != nil {

23
internal/distributed/rootcoord/client/client.go
View File

@ -19,6 +19,7 @@ package grpcrootcoordclient
import (
"context"
"fmt"
"time"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
@ -37,8 +38,6 @@ import (
"google.golang.org/grpc"
)
var ClientParams paramtable.GrpcClientConfig
var Params *paramtable.ComponentParam = paramtable.Get()
// Client grpc client
@ -59,19 +58,19 @@ func NewClient(ctx context.Context, metaRoot string, etcdCli *clientv3.Client) (
log.Debug("QueryCoordClient NewClient failed", zap.Error(err))
return nil, err
}
ClientParams.InitOnce(typeutil.RootCoordRole)
clientParams := &Params.RootCoordGrpcClientCfg
client := &Client{
grpcClient: &grpcclient.ClientBase[rootcoordpb.RootCoordClient]{
ClientMaxRecvSize: ClientParams.ClientMaxRecvSize,
ClientMaxSendSize: ClientParams.ClientMaxSendSize,
DialTimeout: ClientParams.DialTimeout,
KeepAliveTime: ClientParams.KeepAliveTime,
KeepAliveTimeout: ClientParams.KeepAliveTimeout,
ClientMaxRecvSize: clientParams.ClientMaxRecvSize.GetAsInt(),
ClientMaxSendSize: clientParams.ClientMaxSendSize.GetAsInt(),
DialTimeout: clientParams.DialTimeout.GetAsDuration(time.Millisecond),
KeepAliveTime: clientParams.KeepAliveTime.GetAsDuration(time.Millisecond),
KeepAliveTimeout: clientParams.KeepAliveTimeout.GetAsDuration(time.Millisecond),
RetryServiceNameConfig: "milvus.proto.rootcoord.RootCoord",
MaxAttempts: ClientParams.MaxAttempts,
InitialBackoff: ClientParams.InitialBackoff,
MaxBackoff: ClientParams.MaxBackoff,
BackoffMultiplier: ClientParams.BackoffMultiplier,
MaxAttempts: clientParams.MaxAttempts.GetAsInt(),
InitialBackoff: float32(clientParams.InitialBackoff.GetAsFloat()),
MaxBackoff: float32(clientParams.MaxBackoff.GetAsFloat()),
BackoffMultiplier: float32(clientParams.BackoffMultiplier.GetAsFloat()),
},
sess: sess,
}

31
internal/distributed/rootcoord/service.go
View File

@ -18,7 +18,6 @@ package grpcrootcoord
import (
"context"
"fmt"
"io"
"net"
"strconv"
@ -46,15 +45,12 @@ import (
"github.com/milvus-io/milvus/internal/util/logutil"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/trace"
"github.com/milvus-io/milvus/internal/util/typeutil"
dcc "github.com/milvus-io/milvus/internal/distributed/datacoord/client"
icc "github.com/milvus-io/milvus/internal/distributed/indexcoord/client"
qcc "github.com/milvus-io/milvus/internal/distributed/querycoord/client"
)
var Params paramtable.GrpcServerConfig
// Server grpc wrapper
type Server struct {
rootCoord types.RootCoordComponent
@ -153,30 +149,31 @@ func (s *Server) Run() error {
}
func (s *Server) init() error {
Params.InitOnce(typeutil.RootCoordRole)
etcdConfig := &paramtable.Get().EtcdCfg
Params := &paramtable.Get().RootCoordGrpcServerCfg
log.Debug("init params done..")
closer := trace.InitTracing("root_coord")
s.closer = closer
etcdCli, err := etcd.GetEtcdClient(
Params.EtcdCfg.UseEmbedEtcd.GetAsBool(),
Params.EtcdCfg.EtcdUseSSL.GetAsBool(),
Params.EtcdCfg.Endpoints.GetAsStrings(),
Params.EtcdCfg.EtcdTLSCert.GetValue(),
Params.EtcdCfg.EtcdTLSKey.GetValue(),
Params.EtcdCfg.EtcdTLSCACert.GetValue(),
Params.EtcdCfg.EtcdTLSMinVersion.GetValue())
etcdConfig.UseEmbedEtcd.GetAsBool(),
etcdConfig.EtcdUseSSL.GetAsBool(),
etcdConfig.Endpoints.GetAsStrings(),
etcdConfig.EtcdTLSCert.GetValue(),
etcdConfig.EtcdTLSKey.GetValue(),
etcdConfig.EtcdTLSCACert.GetValue(),
etcdConfig.EtcdTLSMinVersion.GetValue())
if err != nil {
log.Debug("RootCoord connect to etcd failed", zap.Error(err))
return err
}
s.etcdCli = etcdCli
s.rootCoord.SetEtcdClient(s.etcdCli)
s.rootCoord.SetAddress(fmt.Sprintf("%s:%d", Params.IP, Params.Port))
s.rootCoord.SetAddress(Params.GetAddress())
log.Debug("etcd connect done ...")
err = s.startGrpc(Params.Port)
err = s.startGrpc(Params.Port.GetAsInt())
if err != nil {
return err
}
@ -223,6 +220,7 @@ func (s *Server) startGrpc(port int) error {
func (s *Server) startGrpcLoop(port int) {
defer s.wg.Done()
Params := &paramtable.Get().RootCoordGrpcServerCfg
var kaep = keepalive.EnforcementPolicy{
MinTime: 5 * time.Second, // If a client pings more than once every 5 seconds, terminate the connection
PermitWithoutStream: true, // Allow pings even when there are no active streams
@ -247,8 +245,8 @@ func (s *Server) startGrpcLoop(port int) {
s.grpcServer = grpc.NewServer(
grpc.KeepaliveEnforcementPolicy(kaep),
grpc.KeepaliveParams(kasp),
grpc.MaxRecvMsgSize(Params.ServerMaxRecvSize),
grpc.MaxSendMsgSize(Params.ServerMaxSendSize),
grpc.MaxRecvMsgSize(Params.ServerMaxRecvSize.GetAsInt()),
grpc.MaxSendMsgSize(Params.ServerMaxSendSize.GetAsInt()),
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
ot.UnaryServerInterceptor(opts...),
logutil.UnaryTraceLoggerInterceptor)),
@ -278,6 +276,7 @@ func (s *Server) start() error {
}
func (s *Server) Stop() error {
Params := &paramtable.Get().RootCoordGrpcServerCfg
log.Debug("Rootcoord stop", zap.String("Address", Params.GetAddress()))
if s.closer != nil {
if err := s.closer.Close(); err != nil {

22
internal/distributed/rootcoord/service_test.go
View File

@ -37,7 +37,6 @@ import (
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/retry"
"github.com/milvus-io/milvus/internal/util/sessionutil"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
type proxyMock struct {
@ -160,8 +159,8 @@ func TestRun(t *testing.T) {
cancel: cancel,
grpcErrChan: make(chan error),
}
Params.InitOnce(typeutil.RootCoordRole)
Params.Port = 1000000
rcServerConfig := &paramtable.Get().RootCoordGrpcServerCfg
paramtable.Get().Save(rcServerConfig.Port.Key, "1000000")
err := svr.Run()
assert.NotNil(t, err)
assert.EqualError(t, err, "listen tcp: address 1000000: invalid port")
@ -176,7 +175,8 @@ func TestRun(t *testing.T) {
return &mockQuery{}
}
Params.Port = rand.Int()%100 + 10000
paramtable.Get().Save(rcServerConfig.Port.Key, fmt.Sprintf("%d", rand.Int()%100+10000))
etcdConfig := &paramtable.Get().EtcdCfg
rand.Seed(time.Now().UnixNano())
randVal := rand.Int()
@ -185,13 +185,13 @@ func TestRun(t *testing.T) {
rootcoord.Params.Init()
etcdCli, err := etcd.GetEtcdClient(
Params.EtcdCfg.UseEmbedEtcd.GetAsBool(),
Params.EtcdCfg.EtcdUseSSL.GetAsBool(),
Params.EtcdCfg.Endpoints.GetAsStrings(),
Params.EtcdCfg.EtcdTLSCert.GetValue(),
Params.EtcdCfg.EtcdTLSKey.GetValue(),
Params.EtcdCfg.EtcdTLSCACert.GetValue(),
Params.EtcdCfg.EtcdTLSMinVersion.GetValue())
etcdConfig.UseEmbedEtcd.GetAsBool(),
etcdConfig.EtcdUseSSL.GetAsBool(),
etcdConfig.Endpoints.GetAsStrings(),
etcdConfig.EtcdTLSCert.GetValue(),
etcdConfig.EtcdTLSKey.GetValue(),
etcdConfig.EtcdTLSCACert.GetValue(),
etcdConfig.EtcdTLSMinVersion.GetValue())
assert.Nil(t, err)
sessKey := path.Join(rootcoord.Params.EtcdCfg.MetaRootPath.GetValue(), sessionutil.DefaultServiceRoot)
_, err = etcdCli.Delete(ctx, sessKey, clientv3.WithPrefix())

2
internal/mq/mqimpl/rocksmq/client/test_helper.go
View File

@ -48,7 +48,7 @@ func newMockClient() *client {
func newRocksMQ(t *testing.T, rmqPath string) server.RocksMQ {
rocksdbPath := rmqPath
var params paramtable.BaseTable
params.Init()
params.Init(0)
rmq, err := server.NewRocksMQ(params, rocksdbPath, nil)
assert.NoError(t, err)
return rmq

4
internal/mq/mqimpl/rocksmq/server/global_rmq.go
View File

@ -42,7 +42,7 @@ var params paramtable.BaseTable
// InitRmq is deprecate implementation of global rocksmq. will be removed later
func InitRmq(rocksdbName string, idAllocator allocator.Interface) error {
var err error
params.Init()
params.Init(0)
Rmq, err = NewRocksMQ(params, rocksdbName, idAllocator)
return err
}
@ -51,7 +51,7 @@ func InitRmq(rocksdbName string, idAllocator allocator.Interface) error {
func InitRocksMQ(path string) error {
var finalErr error
once.Do(func() {
params.Init()
params.Init(0)
log.Debug("initializing global rmq", zap.String("path", path))
var fi os.FileInfo
fi, finalErr = os.Stat(path)

38
internal/mq/mqimpl/rocksmq/server/rocksmq_impl_test.go
View File

@ -164,7 +164,7 @@ func TestRocksmq_RegisterConsumer(t *testing.T) {
defer os.RemoveAll(rocksdbPath)
var params paramtable.BaseTable
params.Init()
params.Init(0)
rmq, err := NewRocksMQ(params, rocksdbPath, idAllocator)
assert.NoError(t, err)
defer rmq.Close()
@ -229,7 +229,7 @@ func TestRocksmq_Basic(t *testing.T) {
defer os.RemoveAll(rocksdbPath + kvSuffix)
defer os.RemoveAll(rocksdbPath)
var params paramtable.BaseTable
params.Init()
params.Init(0)
rmq, err := NewRocksMQ(params, rocksdbPath, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -313,7 +313,7 @@ func TestRocksmq_MultiConsumer(t *testing.T) {
defer os.RemoveAll(rocksdbPath + kvSuffix)
defer os.RemoveAll(rocksdbPath)
var params paramtable.BaseTable
params.Init()
params.Init(0)
atomic.StoreInt64(&RocksmqPageSize, 10)
rmq, err := NewRocksMQ(params, rocksdbPath, idAllocator)
@ -367,7 +367,7 @@ func TestRocksmq_Dummy(t *testing.T) {
defer os.RemoveAll(rocksdbPath + kvSuffix)
defer os.RemoveAll(rocksdbPath)
var params paramtable.BaseTable
params.Init()
params.Init(0)
rmq, err := NewRocksMQ(params, rocksdbPath, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -438,7 +438,7 @@ func TestRocksmq_Seek(t *testing.T) {
defer os.RemoveAll(rocksdbPath)
var params paramtable.BaseTable
params.Init()
params.Init(0)
rmq, err := NewRocksMQ(params, rocksdbPath, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -506,7 +506,7 @@ func TestRocksmq_Loop(t *testing.T) {
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init()
params.Init(0)
rmq, err := NewRocksMQ(params, name, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -578,7 +578,7 @@ func TestRocksmq_Goroutines(t *testing.T) {
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init()
params.Init(0)
rmq, err := NewRocksMQ(params, name, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -657,7 +657,7 @@ func TestRocksmq_Throughout(t *testing.T) {
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init()
params.Init(0)
rmq, err := NewRocksMQ(params, name, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -722,7 +722,7 @@ func TestRocksmq_MultiChan(t *testing.T) {
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init()
params.Init(0)
rmq, err := NewRocksMQ(params, name, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -776,7 +776,7 @@ func TestRocksmq_CopyData(t *testing.T) {
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init()
params.Init(0)
rmq, err := NewRocksMQ(params, name, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -844,7 +844,7 @@ func TestRocksmq_SeekToLatest(t *testing.T) {
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init()
params.Init(0)
rmq, err := NewRocksMQ(params, name, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -937,7 +937,7 @@ func TestRocksmq_GetLatestMsg(t *testing.T) {
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init()
params.Init(0)
rmq, err := NewRocksMQ(params, name, idAllocator)
assert.Nil(t, err)
@ -1013,7 +1013,7 @@ func TestRocksmq_Close(t *testing.T) {
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init()
params.Init(0)
rmq, err := NewRocksMQ(params, name, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -1047,7 +1047,7 @@ func TestRocksmq_SeekWithNoConsumerError(t *testing.T) {
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init()
params.Init(0)
rmq, err := NewRocksMQ(params, name, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -1074,7 +1074,7 @@ func TestRocksmq_SeekTopicNotExistError(t *testing.T) {
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init()
params.Init(0)
rmq, err := NewRocksMQ(params, name, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -1098,7 +1098,7 @@ func TestRocksmq_SeekTopicMutexError(t *testing.T) {
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init()
params.Init(0)
rmq, err := NewRocksMQ(params, name, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -1123,7 +1123,7 @@ func TestRocksmq_moveConsumePosError(t *testing.T) {
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init()
params.Init(0)
rmq, err := NewRocksMQ(params, name, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -1147,7 +1147,7 @@ func TestRocksmq_updateAckedInfoErr(t *testing.T) {
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init()
params.Init(0)
atomic.StoreInt64(&RocksmqPageSize, 10)
rmq, err := NewRocksMQ(params, name, idAllocator)
assert.Nil(t, err)
@ -1200,7 +1200,7 @@ func TestRocksmq_Info(t *testing.T) {
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init()
params.Init(0)
atomic.StoreInt64(&RocksmqPageSize, 10)
rmq, err := NewRocksMQ(params, name, idAllocator)
assert.Nil(t, err)

12
internal/mq/mqimpl/rocksmq/server/rocksmq_retention_test.go
View File

@ -45,7 +45,7 @@ func TestRmqRetention_Basic(t *testing.T) {
defer os.RemoveAll(metaPath)
var params paramtable.BaseTable
params.Init()
params.Init(0)
checkTimeInterval := 2
atomic.StoreInt64(&RocksmqPageSize, 10)
@ -141,7 +141,7 @@ func TestRmqRetention_NotConsumed(t *testing.T) {
defer os.RemoveAll(metaPath)
var params paramtable.BaseTable
params.Init()
params.Init(0)
atomic.StoreInt64(&RocksmqPageSize, 10)
atomic.StoreInt64(&TickerTimeInSeconds, 2)
rmq, err := NewRocksMQ(params, rocksdbPath, nil)
@ -251,7 +251,7 @@ func TestRmqRetention_MultipleTopic(t *testing.T) {
var params paramtable.BaseTable
atomic.StoreInt64(&RocksmqPageSize, 10)
atomic.StoreInt64(&TickerTimeInSeconds, 1)
params.Init()
params.Init(0)
rmq, err := NewRocksMQ(params, rocksdbPath, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -415,7 +415,7 @@ func TestRetentionInfo_InitRetentionInfo(t *testing.T) {
defer os.RemoveAll(metaPath)
var params paramtable.BaseTable
params.Init()
params.Init(0)
rmq, err := NewRocksMQ(params, rocksdbPath, idAllocator)
assert.Nil(t, err)
assert.NotNil(t, rmq)
@ -470,7 +470,7 @@ func TestRmqRetention_PageTimeExpire(t *testing.T) {
os.RemoveAll(metaPath)
var params paramtable.BaseTable
params.Init()
params.Init(0)
atomic.StoreInt64(&RocksmqPageSize, 10)
atomic.StoreInt64(&TickerTimeInSeconds, 1)
rmq, err := NewRocksMQ(params, rocksdbPath, idAllocator)
@ -594,7 +594,7 @@ func TestRmqRetention_PageSizeExpire(t *testing.T) {
os.RemoveAll(metaPath)
var params paramtable.BaseTable
params.Init()
params.Init(0)
atomic.StoreInt64(&RocksmqPageSize, 10)
atomic.StoreInt64(&TickerTimeInSeconds, 1)
rmq, err := NewRocksMQ(params, rocksdbPath, idAllocator)

2
internal/mq/msgstream/mqwrapper/kafka/kafka_client_test.go
View File

@ -25,7 +25,7 @@ import (
var Params paramtable.BaseTable
func TestMain(m *testing.M) {
Params.Init()
Params.Init(0)
mockCluster, err := kafka.NewMockCluster(1)
defer mockCluster.Close()
if err != nil {

2
internal/mq/msgstream/mqwrapper/pulsar/pulsar_client_test.go
View File

@ -47,7 +47,7 @@ const (
var Params paramtable.BaseTable
func TestMain(m *testing.M) {
Params.Init()
Params.Init(0)
exitCode := m.Run()
os.Exit(exitCode)
}

15
internal/proxy/proxy_rpc_test.go
View File

@ -2,7 +2,6 @@ package proxy
import (
"context"
"fmt"
"os"
"strings"
"sync"
@ -33,13 +32,14 @@ func TestProxyRpcLimit(t *testing.T) {
localMsg := true
factory := dependency.NewDefaultFactory(localMsg)
base := paramtable.BaseTable{}
base.Init(0)
var p paramtable.GrpcServerConfig
assert.NoError(t, err)
p.InitOnce(typeutil.ProxyRole)
p.Save("proxy.grpc.serverMaxRecvSize", "1")
p.Init(typeutil.ProxyRole, &base)
base.Save("proxy.grpc.serverMaxRecvSize", "1")
p.InitServerMaxRecvSize()
assert.Equal(t, p.ServerMaxRecvSize, 1)
assert.Equal(t, p.ServerMaxRecvSize.GetAsInt(), 1)
log.Info("Initialize parameter table of Proxy")
proxy, err := NewProxy(ctx, factory)
@ -52,7 +52,7 @@ func TestProxyRpcLimit(t *testing.T) {
go testServer.startGrpc(ctx, &wg, &p)
assert.NoError(t, testServer.waitForGrpcReady())
defer testServer.grpcServer.Stop()
client, err := grpcproxyclient.NewClient(ctx, "localhost:"+fmt.Sprint(p.Port))
client, err := grpcproxyclient.NewClient(ctx, "localhost:"+p.Port.GetValue())
assert.NoError(t, err)
proxy.stateCode.Store(commonpb.StateCode_Healthy)
@ -69,7 +69,4 @@ func TestProxyRpcLimit(t *testing.T) {
// should be limited because of the rpc limit
assert.Error(t, err)
assert.True(t, strings.Contains(err.Error(), "ResourceExhausted"))
p.Remove("proxy.grpc.serverMaxRecvSize")
p.Init(typeutil.ProxyRole)
}

16
internal/proxy/proxy_test.go
View File

@ -343,14 +343,14 @@ func (s *proxyTestServer) startGrpc(ctx context.Context, wg *sync.WaitGroup, p *
Timeout: 10 * time.Second, // Wait 10 second for the ping ack before assuming the connection is dead
}
log.Debug("Proxy server listen on tcp", zap.Int("port", p.Port))
lis, err := net.Listen("tcp", ":"+strconv.Itoa(p.Port))
log.Debug("Proxy server listen on tcp", zap.Int("port", p.Port.GetAsInt()))
lis, err := net.Listen("tcp", ":"+p.Port.GetValue())
if err != nil {
log.Warn("Proxy server failed to listen on", zap.Error(err), zap.Int("port", p.Port))
log.Warn("Proxy server failed to listen on", zap.Error(err), zap.Int("port", p.Port.GetAsInt()))
s.ch <- err
return
}
log.Debug("Proxy server already listen on tcp", zap.Int("port", p.Port))
log.Debug("Proxy server already listen on tcp", zap.Int("port", p.Port.GetAsInt()))
ctx, cancel := context.WithCancel(ctx)
defer cancel()
@ -362,8 +362,8 @@ func (s *proxyTestServer) startGrpc(ctx context.Context, wg *sync.WaitGroup, p *
s.grpcServer = grpc.NewServer(
grpc.KeepaliveEnforcementPolicy(kaep),
grpc.KeepaliveParams(kasp),
grpc.MaxRecvMsgSize(p.ServerMaxRecvSize),
grpc.MaxSendMsgSize(p.ServerMaxSendSize),
grpc.MaxRecvMsgSize(p.ServerMaxRecvSize.GetAsInt()),
grpc.MaxSendMsgSize(p.ServerMaxSendSize.GetAsInt()),
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
ot.UnaryServerInterceptor(opts...),
RateLimitInterceptor(multiLimiter),
@ -511,8 +511,10 @@ func TestProxy(t *testing.T) {
testServer := newProxyTestServer(proxy)
wg.Add(1)
base := paramtable.BaseTable{}
base.Init(0)
var p paramtable.GrpcServerConfig
p.InitOnce(typeutil.ProxyRole)
p.Init(typeutil.ProxyRole, &base)
testServer.Proxy.SetAddress(p.GetAddress())
go testServer.startGrpc(ctx, &wg, &p)

2
internal/storage/minio_chunk_manager_test.go
View File

@ -84,7 +84,7 @@ func TestMinIOCMFail(t *testing.T) {
}
func TestMinIOCM(t *testing.T) {
Params.Init()
Params.Init(0)
testBucket, err := Params.Load("minio.bucketName")
require.NoError(t, err)

2
internal/storage/vector_chunk_manager_test.go
View File

@ -143,7 +143,7 @@ var Params paramtable.BaseTable
var localPath = "/tmp/milvus_test/chunkmanager/"
func TestMain(m *testing.M) {
Params.Init()
Params.Init(0)
exitCode := m.Run()
err := os.RemoveAll(localPath)
if err != nil {

2
internal/util/grpcclient/client.go
View File

@ -293,7 +293,7 @@ func (c *ClientBase[T]) ReCall(ctx context.Context, caller func(client T) (any,
}
traceErr := fmt.Errorf("err: %w\n, %s", err, trace.StackTrace())
log.Warn(c.GetRole()+" ClientBase ReCall grpc first call get error ", zap.Error(traceErr))
log.Warn("ClientBase ReCall grpc first call get error ", zap.String("role", c.GetRole()), zap.Error(traceErr))
if !funcutil.CheckCtxValid(ctx) {
return generic.Zero[T](), ctx.Err()

12
internal/util/paramtable/autoindex_param.go
View File

@ -23,13 +23,13 @@ import (
///////////////////////////////////////////////////////////////////////////////
// --- common ---
type autoIndexConfig struct {
Enable ParamItem
Enable ParamItem `refreshable:"true"`
IndexParams ParamItem
ExtraParams ParamItem
SearchParamsYamlStr ParamItem
IndexType ParamItem
AutoIndexTypeName ParamItem
IndexParams ParamItem `refreshable:"true"`
ExtraParams ParamItem `refreshable:"true"`
SearchParamsYamlStr ParamItem `refreshable:"true"`
IndexType ParamItem `refreshable:"true"`
AutoIndexTypeName ParamItem `refreshable:"true"`
}
func (p *autoIndexConfig) init(base *BaseTable) {

172
internal/util/paramtable/base_table.go
View File

@ -19,7 +19,6 @@ import (
"runtime"
"strconv"
"strings"
"sync"
"time"
config "github.com/milvus-io/milvus/internal/config"
@ -63,28 +62,14 @@ func globalConfigPrefixs() []string {
var defaultYaml = DefaultMilvusYaml
// Base abstracts BaseTable
// TODO: it's never used, consider to substitute BaseTable or to remove it
type Base interface {
Load(key string) (string, error)
LoadRange(key, endKey string, limit int) ([]string, []string, error)
LoadYaml(fileName string) error
Remove(key string) error
Save(key, value string) error
Init()
}
// BaseTable the basics of paramtable
type BaseTable struct {
once sync.Once
mgr *config.Manager
// params *memkv.MemoryKV
mgr *config.Manager
configDir string
YamlFile string
Log log.Config
YamlFile string
}
// NewBaseTableFromYamlOnly only used in migration tool.
@ -98,19 +83,9 @@ func NewBaseTableFromYamlOnly(yaml string) *BaseTable {
return gp
}
// GlobalInitWithYaml initializes the param table with the given yaml.
// We will update the global DefaultYaml variable directly, once and for all.
// GlobalInitWithYaml shall be called at the very beginning before initiating the base table.
// GlobalInitWithYaml should be called only in standalone and embedded Milvus.
func (gp *BaseTable) GlobalInitWithYaml(yaml string) {
gp.once.Do(func() {
defaultYaml = yaml
gp.Init()
})
}
// Init initializes the param table.
func (gp *BaseTable) Init() {
// if refreshInterval greater than 0 will auto refresh config from source
func (gp *BaseTable) Init(refreshInterval int) {
formatter := func(key string) string {
ret := strings.ToLower(key)
ret = strings.TrimPrefix(ret, "milvus.")
@ -120,35 +95,32 @@ func (gp *BaseTable) Init() {
return ret
}
if gp.YamlFile == "" {
gp.YamlFile = defaultYaml
gp.YamlFile = DefaultMilvusYaml
}
gp.initConfigsFromLocal(formatter)
gp.initConfigsFromRemote(formatter)
gp.InitLogCfg()
}
func (gp *BaseTable) initConfigsFromLocal(formatter func(key string) string) {
var err error
gp.mgr, err = config.Init(config.WithEnvSource(formatter))
if err != nil {
return
}
gp.initConfigsFromLocal(refreshInterval)
gp.initConfigsFromRemote(refreshInterval)
gp.InitLogCfg()
}
func (gp *BaseTable) initConfigsFromLocal(refreshInterval int) {
gp.configDir = gp.initConfPath()
configFilePath := gp.configDir + "/" + gp.YamlFile
gp.mgr, err = config.Init(
config.WithEnvSource(formatter),
config.WithFilesSource(&config.FileInfo{
Filepath: configFilePath,
RefreshInterval: 10 * time.Second,
}))
err := gp.mgr.AddSource(config.NewFileSource(&config.FileInfo{
Filepath: configFilePath,
RefreshInterval: time.Duration(refreshInterval) * time.Second,
}))
if err != nil {
log.Warn("init baseTable with file failed", zap.String("configFile", configFilePath), zap.Error(err))
return
}
}
func (gp *BaseTable) initConfigsFromRemote(formatter func(key string) string) {
func (gp *BaseTable) initConfigsFromRemote(refreshInterval int) {
etcdConfig := EtcdConfig{}
etcdConfig.Init(gp)
etcdConfig.Endpoints.PanicIfEmpty = false
@ -168,7 +140,7 @@ func (gp *BaseTable) initConfigsFromRemote(formatter func(key string) string) {
CaCertFile: etcdConfig.EtcdTLSCACert.GetValue(),
MinVersion: etcdConfig.EtcdTLSMinVersion.GetValue(),
KeyPrefix: etcdConfig.RootPath.GetValue(),
RefreshInterval: 10 * time.Second,
RefreshInterval: time.Duration(refreshInterval) * time.Second,
}
s, err := config.NewEtcdSource(info)
@ -211,18 +183,6 @@ func (gp *BaseTable) Load(key string) (string, error) {
return gp.mgr.GetConfig(key)
}
// LoadWithPriority loads an object with multiple @keys, return the first successful value.
// If all keys not exist, return error.
// This is to be compatible with old configuration file.
func (gp *BaseTable) LoadWithPriority(keys []string) (string, error) {
for _, key := range keys {
if str, err := gp.mgr.GetConfig(key); err == nil {
return str, nil
}
}
return "", fmt.Errorf("invalid keys: %v", keys)
}
// LoadWithDefault loads an object with @key. If the object does not exist, @defaultValue will be returned.
func (gp *BaseTable) LoadWithDefault(key, defaultValue string) string {
str, err := gp.mgr.GetConfig(key)
@ -232,19 +192,6 @@ func (gp *BaseTable) LoadWithDefault(key, defaultValue string) string {
return str
}
// LoadWithDefault2 loads an object with multiple @keys, return the first successful value.
// If all keys not exist, return @defaultValue.
// This is to be compatible with old configuration file.
func (gp *BaseTable) LoadWithDefault2(keys []string, defaultValue string) string {
for _, key := range keys {
str, err := gp.mgr.GetConfig(key)
if err == nil {
return str
}
}
return defaultValue
}
func (gp *BaseTable) Get(key string) string {
value, err := gp.mgr.GetConfig(key)
if err != nil {
@ -293,18 +240,6 @@ func (gp *BaseTable) ParseBool(key string, defaultValue bool) bool {
return value
}
func (gp *BaseTable) ParseFloat(key string) float64 {
valueStr, err := gp.Load(key)
if err != nil {
panic(err)
}
value, err := strconv.ParseFloat(valueStr, 64)
if err != nil {
panic(err)
}
return value
}
func (gp *BaseTable) ParseFloatWithDefault(key string, defaultValue float64) float64 {
valueStr := gp.LoadWithDefault(key, fmt.Sprintf("%f", defaultValue))
value, err := strconv.ParseFloat(valueStr, 64)
@ -314,18 +249,6 @@ func (gp *BaseTable) ParseFloatWithDefault(key string, defaultValue float64) flo
return value
}
func (gp *BaseTable) ParseInt64(key string) int64 {
valueStr, err := gp.Load(key)
if err != nil {
panic(err)
}
value, err := strconv.ParseInt(valueStr, 10, 64)
if err != nil {
panic(err)
}
return value
}
func (gp *BaseTable) ParseInt64WithDefault(key string, defaultValue int64) int64 {
valueStr := gp.LoadWithDefault(key, strconv.FormatInt(defaultValue, 10))
value, err := strconv.ParseInt(valueStr, 10, 64)
@ -335,18 +258,6 @@ func (gp *BaseTable) ParseInt64WithDefault(key string, defaultValue int64) int64
return value
}
func (gp *BaseTable) ParseInt32(key string) int32 {
valueStr, err := gp.Load(key)
if err != nil {
panic(err)
}
value, err := strconv.ParseInt(valueStr, 10, 32)
if err != nil {
panic(err)
}
return int32(value)
}
func (gp *BaseTable) ParseInt32WithDefault(key string, defaultValue int32) int32 {
valueStr := gp.LoadWithDefault(key, strconv.FormatInt(int64(defaultValue), 10))
value, err := strconv.ParseInt(valueStr, 10, 32)
@ -356,18 +267,6 @@ func (gp *BaseTable) ParseInt32WithDefault(key string, defaultValue int32) int32
return int32(value)
}
func (gp *BaseTable) ParseInt(key string) int {
valueStr, err := gp.Load(key)
if err != nil {
panic(err)
}
value, err := strconv.Atoi(valueStr)
if err != nil {
panic(err)
}
return value
}
func (gp *BaseTable) ParseIntWithDefault(key string, defaultValue int) int {
valueStr := gp.LoadWithDefault(key, strconv.FormatInt(int64(defaultValue), 10))
value, err := strconv.Atoi(valueStr)
@ -377,45 +276,6 @@ func (gp *BaseTable) ParseIntWithDefault(key string, defaultValue int) int {
return value
}
// package methods
// ConvertRangeToIntRange converts a range of strings to a range of ints.
func ConvertRangeToIntRange(rangeStr, sep string) []int {
items := strings.Split(rangeStr, sep)
if len(items) != 2 {
panic("Illegal range ")
}
startStr := items[0]
endStr := items[1]
start, err := strconv.Atoi(startStr)
if err != nil {
panic(err)
}
end, err := strconv.Atoi(endStr)
if err != nil {
panic(err)
}
if start < 0 || end < 0 {
panic("Illegal range value")
}
if start > end {
panic("Illegal range value, start > end")
}
return []int{start, end}
}
// ConvertRangeToIntSlice convert given @rangeStr & @sep to a slice of ints.
func ConvertRangeToIntSlice(rangeStr, sep string) []int {
rangeSlice := ConvertRangeToIntRange(rangeStr, sep)
start, end := rangeSlice[0], rangeSlice[1]
var ret []int
for i := start; i < end; i++ {
ret = append(ret, i)
}
return ret
}
// InitLogCfg init log of the base table
func (gp *BaseTable) InitLogCfg() {
gp.Log = log.Config{}

58
internal/util/paramtable/base_table_test.go
View File

@ -22,7 +22,7 @@ import (
var baseParams = BaseTable{}
func TestMain(m *testing.M) {
baseParams.Init()
baseParams.Init(0)
code := m.Run()
os.Exit(code)
}
@ -109,7 +109,7 @@ func TestBaseTable_Get(t *testing.T) {
func TestBaseTable_Pulsar(t *testing.T) {
//test PULSAR ADDRESS
t.Setenv("PULSAR_ADDRESS", "pulsar://localhost:6650")
baseParams.Init()
baseParams.Init(0)
address := baseParams.Get("pulsar.address")
assert.Equal(t, "pulsar://localhost:6650", address)
@ -151,7 +151,7 @@ func TestBaseTable_Env(t *testing.T) {
t.Setenv("milvus.test", "test")
t.Setenv("milvus.test.test2", "test2")
baseParams.Init()
baseParams.Init(0)
result, _ := baseParams.Load("test")
assert.Equal(t, result, "test")
@ -160,7 +160,7 @@ func TestBaseTable_Env(t *testing.T) {
t.Setenv("milvus.invalid", "xxx=test")
baseParams.Init()
baseParams.Init(0)
result, _ = baseParams.Load("invalid")
assert.Equal(t, result, "xxx=test")
}
@ -175,18 +175,6 @@ func TestBaseTable_Parse(t *testing.T) {
assert.Panics(t, func() { baseParams.ParseBool("key", false) })
})
t.Run("ParseFloat", func(t *testing.T) {
assert.Nil(t, baseParams.Save("key", "0"))
assert.Equal(t, float64(0), baseParams.ParseFloat("key"))
assert.Nil(t, baseParams.Save("key", "3.14"))
assert.Equal(t, float64(3.14), baseParams.ParseFloat("key"))
assert.Panics(t, func() { baseParams.ParseFloat("not_exist_key") })
assert.Nil(t, baseParams.Save("key", "abc"))
assert.Panics(t, func() { baseParams.ParseFloat("key") })
})
t.Run("ParseFloatWithDefault", func(t *testing.T) {
baseParams.Remove("key")
assert.Equal(t, float64(0.0), baseParams.ParseFloatWithDefault("key", 0.0))
@ -196,18 +184,6 @@ func TestBaseTable_Parse(t *testing.T) {
assert.Equal(t, float64(2.0), baseParams.ParseFloatWithDefault("key", 3.14))
})
t.Run("ParseInt32", func(t *testing.T) {
assert.Nil(t, baseParams.Save("key", "0"))
assert.Equal(t, int32(0), baseParams.ParseInt32("key"))
assert.Nil(t, baseParams.Save("key", "314"))
assert.Equal(t, int32(314), baseParams.ParseInt32("key"))
assert.Panics(t, func() { baseParams.ParseInt32("not_exist_key") })
assert.Nil(t, baseParams.Save("key", "abc"))
assert.Panics(t, func() { baseParams.ParseInt32("key") })
})
t.Run("ParseInt32WithDefault", func(t *testing.T) {
baseParams.Remove("key")
assert.Equal(t, int32(1), baseParams.ParseInt32WithDefault("key", 1))
@ -215,18 +191,6 @@ func TestBaseTable_Parse(t *testing.T) {
assert.Equal(t, int32(2), baseParams.ParseInt32WithDefault("key", 1))
})
t.Run("ParseInt64", func(t *testing.T) {
assert.Nil(t, baseParams.Save("key", "0"))
assert.Equal(t, int64(0), baseParams.ParseInt64("key"))
assert.Nil(t, baseParams.Save("key", "314"))
assert.Equal(t, int64(314), baseParams.ParseInt64("key"))
assert.Panics(t, func() { baseParams.ParseInt64("not_exist_key") })
assert.Nil(t, baseParams.Save("key", "abc"))
assert.Panics(t, func() { baseParams.ParseInt64("key") })
})
t.Run("ParseInt64WithDefault", func(t *testing.T) {
baseParams.Remove("key")
assert.Equal(t, int64(1), baseParams.ParseInt64WithDefault("key", 1))
@ -242,20 +206,6 @@ func TestBaseTable_Parse(t *testing.T) {
})
}
func Test_ConvertRangeToIntSlice(t *testing.T) {
t.Run("ConvertRangeToIntSlice", func(t *testing.T) {
slice := ConvertRangeToIntSlice("0,10", ",")
assert.Equal(t, 10, len(slice))
assert.Panics(t, func() { ConvertRangeToIntSlice("0", ",") })
assert.Panics(t, func() { ConvertRangeToIntSlice("0, 10", ",") })
assert.Panics(t, func() { ConvertRangeToIntSlice("abc,10", ",") })
assert.Panics(t, func() { ConvertRangeToIntSlice("0,abc", ",") })
assert.Panics(t, func() { ConvertRangeToIntSlice("-1,9", ",") })
assert.Panics(t, func() { ConvertRangeToIntSlice("9,0", ",") })
})
}
func TestNewBaseTableFromYamlOnly(t *testing.T) {
var yaml string
var gp *BaseTable

394
internal/util/paramtable/component_param.go
View File

@ -20,6 +20,7 @@ import (
"sync"
"time"
"github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/shirou/gopsutil/v3/disk"
)
@ -62,7 +63,26 @@ type ComponentParam struct {
DataNodeCfg dataNodeConfig
IndexCoordCfg indexCoordConfig
IndexNodeCfg indexNodeConfig
HTTPCfg httpConfig
HookCfg hookConfig
RootCoordGrpcServerCfg GrpcServerConfig
ProxyGrpcServerCfg GrpcServerConfig
QueryCoordGrpcServerCfg GrpcServerConfig
QueryNodeGrpcServerCfg GrpcServerConfig
DataCoordGrpcServerCfg GrpcServerConfig
DataNodeGrpcServerCfg GrpcServerConfig
IndexCoordGrpcServerCfg GrpcServerConfig
IndexNodeGrpcServerCfg GrpcServerConfig
RootCoordGrpcClientCfg GrpcClientConfig
ProxyGrpcClientCfg GrpcClientConfig
QueryCoordGrpcClientCfg GrpcClientConfig
QueryNodeGrpcClientCfg GrpcClientConfig
DataCoordGrpcClientCfg GrpcClientConfig
DataNodeGrpcClientCfg GrpcClientConfig
IndexCoordGrpcClientCfg GrpcClientConfig
IndexNodeGrpcClientCfg GrpcClientConfig
}
// InitOnce initialize once
@ -88,7 +108,26 @@ func (p *ComponentParam) Init() {
p.DataNodeCfg.init(&p.BaseTable)
p.IndexCoordCfg.init(&p.BaseTable)
p.IndexNodeCfg.init(&p.BaseTable)
p.HTTPCfg.init(&p.BaseTable)
p.HookCfg.init()
p.RootCoordGrpcServerCfg.Init(typeutil.RootCoordRole, &p.BaseTable)
p.ProxyGrpcServerCfg.Init(typeutil.ProxyRole, &p.BaseTable)
p.QueryCoordGrpcServerCfg.Init(typeutil.QueryCoordRole, &p.BaseTable)
p.QueryNodeGrpcServerCfg.Init(typeutil.QueryNodeRole, &p.BaseTable)
p.DataCoordGrpcServerCfg.Init(typeutil.DataCoordRole, &p.BaseTable)
p.DataNodeGrpcServerCfg.Init(typeutil.DataNodeRole, &p.BaseTable)
p.IndexCoordGrpcServerCfg.Init(typeutil.IndexCoordRole, &p.BaseTable)
p.IndexNodeGrpcServerCfg.Init(typeutil.IndexNodeRole, &p.BaseTable)
p.RootCoordGrpcClientCfg.Init(typeutil.RootCoordRole, &p.BaseTable)
p.ProxyGrpcClientCfg.Init(typeutil.ProxyRole, &p.BaseTable)
p.QueryCoordGrpcClientCfg.Init(typeutil.QueryCoordRole, &p.BaseTable)
p.QueryNodeGrpcClientCfg.Init(typeutil.QueryNodeRole, &p.BaseTable)
p.DataCoordGrpcClientCfg.Init(typeutil.DataCoordRole, &p.BaseTable)
p.DataNodeGrpcClientCfg.Init(typeutil.DataNodeRole, &p.BaseTable)
p.IndexCoordGrpcClientCfg.Init(typeutil.IndexCoordRole, &p.BaseTable)
p.IndexNodeGrpcClientCfg.Init(typeutil.IndexNodeRole, &p.BaseTable)
}
func (p *ComponentParam) RocksmqEnable() bool {
@ -106,61 +145,65 @@ func (p *ComponentParam) KafkaEnable() bool {
// /////////////////////////////////////////////////////////////////////////////
// --- common ---
type commonConfig struct {
ClusterPrefix ParamItem
ClusterPrefix ParamItem `refreshable:"true"`
ProxySubName ParamItem
// Deprecated: do not use it anymore
ProxySubName ParamItem `refreshable:"true"`
RootCoordTimeTick ParamItem
RootCoordStatistics ParamItem
RootCoordDml ParamItem
RootCoordDelta ParamItem
RootCoordSubName ParamItem
RootCoordTimeTick ParamItem `refreshable:"true"`
RootCoordStatistics ParamItem `refreshable:"true"`
RootCoordDml ParamItem `refreshable:"false"`
RootCoordDelta ParamItem `refreshable:"false"`
// Deprecated: do not use it anymore
RootCoordSubName ParamItem `refreshable:"true"`
QueryCoordSearch ParamItem
QueryCoordSearchResult ParamItem
QueryCoordTimeTick ParamItem
QueryNodeSubName ParamItem
// Deprecated: only used in metrics as ID
QueryCoordSearch ParamItem `refreshable:"true"`
// Deprecated: only used in metrics as ID
QueryCoordSearchResult ParamItem `refreshable:"true"`
QueryCoordTimeTick ParamItem `refreshable:"true"`
QueryNodeSubName ParamItem `refreshable:"false"`
DataCoordStatistic ParamItem
DataCoordTimeTick ParamItem
DataCoordSegmentInfo ParamItem
DataCoordSubName ParamItem
DataCoordWatchSubPath ParamItem
DataNodeSubName ParamItem
// Deprecated: do not use it anymore
DataCoordStatistic ParamItem `refreshable:"true"`
DataCoordTimeTick ParamItem `refreshable:"false"`
DataCoordSegmentInfo ParamItem `refreshable:"true"`
DataCoordSubName ParamItem `refreshable:"false"`
DataCoordWatchSubPath ParamItem `refreshable:"false"`
DataNodeSubName ParamItem `refreshable:"false"`
DefaultPartitionName ParamItem
DefaultIndexName ParamItem
RetentionDuration ParamItem
EntityExpirationTTL ParamItem
DefaultPartitionName ParamItem `refreshable:"true"`
DefaultIndexName ParamItem `refreshable:"true"`
RetentionDuration ParamItem `refreshable:"true"`
EntityExpirationTTL ParamItem `refreshable:"true"`
IndexSliceSize ParamItem
ThreadCoreCoefficient ParamItem
MaxDegree ParamItem
SearchListSize ParamItem
PQCodeBudgetGBRatio ParamItem
BuildNumThreadsRatio ParamItem
SearchCacheBudgetGBRatio ParamItem
LoadNumThreadRatio ParamItem
BeamWidthRatio ParamItem
GracefulTime ParamItem
GracefulStopTimeout ParamItem // unit: s
IndexSliceSize ParamItem `refreshable:"false"`
ThreadCoreCoefficient ParamItem `refreshable:"false"`
MaxDegree ParamItem `refreshable:"true"`
SearchListSize ParamItem `refreshable:"true"`
PQCodeBudgetGBRatio ParamItem `refreshable:"true"`
BuildNumThreadsRatio ParamItem `refreshable:"true"`
SearchCacheBudgetGBRatio ParamItem `refreshable:"true"`
LoadNumThreadRatio ParamItem `refreshable:"true"`
BeamWidthRatio ParamItem `refreshable:"true"`
GracefulTime ParamItem `refreshable:"true"`
GracefulStopTimeout ParamItem `refreshable:"true"`
// Search limit, which applies on:
// maximum # of results to return (topK), and
// maximum # of search requests (nq).
// Check https://milvus.io/docs/limitations.md for more details.
TopKLimit ParamItem
TopKLimit ParamItem `refreshable:"true"`
StorageType ParamItem `refreshable:"false"`
SimdType ParamItem `refreshable:"false"`
StorageType ParamItem
SimdType ParamItem
AuthorizationEnabled ParamItem `refreshable:"false"`
SuperUsers ParamItem `refreshable:"true"`
AuthorizationEnabled ParamItem
SuperUsers ParamItem
ClusterName ParamItem `refreshable:"false"`
ClusterName ParamItem
SessionTTL ParamItem
SessionRetryTimes ParamItem
SessionTTL ParamItem `refreshable:"false"`
SessionRetryTimes ParamItem `refreshable:"false"`
}
func (p *commonConfig) init(base *BaseTable) {
@ -210,6 +253,7 @@ func (p *commonConfig) init(base *BaseTable) {
FallbackKeys: []string{"msgChannel.chanNamePrefix.rootCoordDml"},
PanicIfEmpty: true,
Formatter: chanNamePrefix,
Doc: "It is not refreshable currently",
}
p.RootCoordDml.Init(base.mgr)
@ -219,6 +263,7 @@ func (p *commonConfig) init(base *BaseTable) {
FallbackKeys: []string{"msgChannel.chanNamePrefix.rootCoordDelta"},
PanicIfEmpty: true,
Formatter: chanNamePrefix,
Doc: "It is not refreshable currently",
}
p.RootCoordDelta.Init(base.mgr)
@ -228,6 +273,7 @@ func (p *commonConfig) init(base *BaseTable) {
FallbackKeys: []string{"msgChannel.subNamePrefix.rootCoordSubNamePrefix"},
PanicIfEmpty: true,
Formatter: chanNamePrefix,
Doc: "It is deprecated",
}
p.RootCoordSubName.Init(base.mgr)
@ -237,6 +283,7 @@ func (p *commonConfig) init(base *BaseTable) {
FallbackKeys: []string{"msgChannel.chanNamePrefix.search"},
PanicIfEmpty: true,
Formatter: chanNamePrefix,
Doc: "It is deprecated",
}
p.QueryCoordSearch.Init(base.mgr)
@ -246,6 +293,7 @@ func (p *commonConfig) init(base *BaseTable) {
FallbackKeys: []string{"msgChannel.chanNamePrefix.searchResult"},
PanicIfEmpty: true,
Formatter: chanNamePrefix,
Doc: "It is deprecated",
}
p.QueryCoordSearchResult.Init(base.mgr)
@ -498,15 +546,13 @@ func (p *commonConfig) init(base *BaseTable) {
// /////////////////////////////////////////////////////////////////////////////
// --- rootcoord ---
type rootCoordConfig struct {
DmlChannelNum ParamItem
MaxPartitionNum ParamItem
MinSegmentSizeToEnableIndex ParamItem
ImportTaskExpiration ParamItem
ImportTaskRetention ParamItem
ImportTaskSubPath ParamItem
// CreatedTime ParamItem
// UpdatedTime ParamItem
EnableActiveStandby ParamItem
DmlChannelNum ParamItem `refreshable:"false"`
MaxPartitionNum ParamItem `refreshable:"true"`
MinSegmentSizeToEnableIndex ParamItem `refreshable:"true"`
ImportTaskExpiration ParamItem `refreshable:"true"`
ImportTaskRetention ParamItem `refreshable:"true"`
ImportTaskSubPath ParamItem `refreshable:"true"`
EnableActiveStandby ParamItem `refreshable:"false"`
}
func (p *rootCoordConfig) init(base *BaseTable) {
@ -565,48 +611,43 @@ func (p *rootCoordConfig) init(base *BaseTable) {
// --- proxy ---
type AccessLogConfig struct {
// if use access log
Enable ParamItem
Enable ParamItem `refreshable:"false"`
// if upload sealed access log file to minio
MinioEnable ParamItem
MinioEnable ParamItem `refreshable:"false"`
// Log path
LocalPath ParamItem
LocalPath ParamItem `refreshable:"false"`
// Log filename, leave empty to disable file log.
Filename ParamItem
Filename ParamItem `refreshable:"false"`
// Max size for a single file, in MB.
MaxSize ParamItem
MaxSize ParamItem `refreshable:"false"`
// Max time for single access log file in seconds
RotatedTime ParamItem
RotatedTime ParamItem `refreshable:"false"`
// Maximum number of old log files to retain.
MaxBackups ParamItem
MaxBackups ParamItem `refreshable:"false"`
//File path in minIO
RemotePath ParamItem
RemotePath ParamItem `refreshable:"false"`
//Max time for log file in minIO, in hours
RemoteMaxTime ParamItem
RemoteMaxTime ParamItem `refreshable:"false"`
}
type proxyConfig struct {
// Alias string
SoPath ParamItem
SoPath ParamItem `refreshable:"false"`
TimeTickInterval ParamItem
MsgStreamTimeTickBufSize ParamItem
MaxNameLength ParamItem
MaxUsernameLength ParamItem
MinPasswordLength ParamItem
MaxPasswordLength ParamItem
MaxFieldNum ParamItem
MaxShardNum ParamItem
MaxDimension ParamItem
GinLogging ParamItem
MaxUserNum ParamItem
MaxRoleNum ParamItem
TimeTickInterval ParamItem `refreshable:"false"`
MsgStreamTimeTickBufSize ParamItem `refreshable:"true"`
MaxNameLength ParamItem `refreshable:"true"`
MaxUsernameLength ParamItem `refreshable:"true"`
MinPasswordLength ParamItem `refreshable:"true"`
MaxPasswordLength ParamItem `refreshable:"true"`
MaxFieldNum ParamItem `refreshable:"true"`
MaxShardNum ParamItem `refreshable:"true"`
MaxDimension ParamItem `refreshable:"true"`
GinLogging ParamItem `refreshable:"false"`
MaxUserNum ParamItem `refreshable:"true"`
MaxRoleNum ParamItem `refreshable:"true"`
MaxTaskNum ParamItem `refreshable:"false"`
AccessLog AccessLogConfig
// required from QueryCoord
SearchResultChannelNames ParamItem
RetrieveResultChannelNames ParamItem
MaxTaskNum ParamItem
}
func (p *proxyConfig) init(base *BaseTable) {
@ -785,31 +826,34 @@ func (p *proxyConfig) init(base *BaseTable) {
// /////////////////////////////////////////////////////////////////////////////
// --- querycoord ---
type queryCoordConfig struct {
//---- Task ---
RetryNum ParamItem
RetryInterval ParamItem
TaskMergeCap ParamItem
TaskExecutionCap ParamItem
//Deprecated: Since 2.2.0
RetryNum ParamItem `refreshable:"true"`
//Deprecated: Since 2.2.0
RetryInterval ParamItem `refreshable:"true"`
TaskMergeCap ParamItem `refreshable:"false"`
TaskExecutionCap ParamItem `refreshable:"true"`
//---- Handoff ---
AutoHandoff ParamItem
//Deprecated: Since 2.2.2
AutoHandoff ParamItem `refreshable:"true"`
//---- Balance ---
AutoBalance ParamItem
OverloadedMemoryThresholdPercentage ParamItem
BalanceIntervalSeconds ParamItem
MemoryUsageMaxDifferencePercentage ParamItem
CheckInterval ParamItem
ChannelTaskTimeout ParamItem
SegmentTaskTimeout ParamItem
DistPullInterval ParamItem
HeartbeatAvailableInterval ParamItem
LoadTimeoutSeconds ParamItem
CheckHandoffInterval ParamItem
EnableActiveStandby ParamItem
AutoBalance ParamItem `refreshable:"true"`
OverloadedMemoryThresholdPercentage ParamItem `refreshable:"true"`
BalanceIntervalSeconds ParamItem `refreshable:"true"`
MemoryUsageMaxDifferencePercentage ParamItem `refreshable:"true"`
CheckInterval ParamItem `refreshable:"true"`
ChannelTaskTimeout ParamItem `refreshable:"true"`
SegmentTaskTimeout ParamItem `refreshable:"true"`
DistPullInterval ParamItem `refreshable:"false"`
HeartbeatAvailableInterval ParamItem `refreshable:"true"`
LoadTimeoutSeconds ParamItem `refreshable:"true"`
//Deprecated: Since 2.2.2, QueryCoord do not use HandOff logic anymore
CheckHandoffInterval ParamItem `refreshable:"true"`
EnableActiveStandby ParamItem `refreshable:"false"`
NextTargetSurviveTime ParamItem
UpdateNextTargetInterval ParamItem
NextTargetSurviveTime ParamItem `refreshable:"true"`
UpdateNextTargetInterval ParamItem `refreshable:"false"`
}
func (p *queryCoordConfig) init(base *BaseTable) {
@ -930,6 +974,14 @@ func (p *queryCoordConfig) init(base *BaseTable) {
}
p.HeartbeatAvailableInterval.Init(base.mgr)
p.CheckHandoffInterval = ParamItem{
Key: "queryCoord.checkHandoffInterval",
DefaultValue: "5000",
Version: "2.2.0",
PanicIfEmpty: true,
}
p.CheckHandoffInterval.Init(base.mgr)
p.EnableActiveStandby = ParamItem{
Key: "queryCoord.enableActiveStandby",
Version: "2.2.0",
@ -957,47 +1009,43 @@ func (p *queryCoordConfig) init(base *BaseTable) {
// /////////////////////////////////////////////////////////////////////////////
// --- querynode ---
type queryNodeConfig struct {
FlowGraphMaxQueueLength ParamItem
FlowGraphMaxParallelism ParamItem
FlowGraphMaxQueueLength ParamItem `refreshable:"false"`
FlowGraphMaxParallelism ParamItem `refreshable:"false"`
// stats
StatsPublishInterval ParamItem
SliceIndex ParamItem
//Deprecated: Never used
StatsPublishInterval ParamItem `refreshable:"true"`
// segcore
ChunkRows ParamItem
SmallIndexNlist ParamItem
SmallIndexNProbe ParamItem
CreatedTime ParamItem
UpdatedTime ParamItem
ChunkRows ParamItem `refreshable:"false"`
SmallIndexNlist ParamItem `refreshable:"false"`
SmallIndexNProbe ParamItem `refreshable:"false"`
// memory limit
LoadMemoryUsageFactor ParamItem
OverloadedMemoryThresholdPercentage ParamItem
LoadMemoryUsageFactor ParamItem `refreshable:"true"`
OverloadedMemoryThresholdPercentage ParamItem `refreshable:"false"`
// enable disk
EnableDisk ParamItem
DiskCapacityLimit ParamItem
MaxDiskUsagePercentage ParamItem
EnableDisk ParamItem `refreshable:"true"`
DiskCapacityLimit ParamItem `refreshable:"true"`
MaxDiskUsagePercentage ParamItem `refreshable:"true"`
// cache limit
CacheEnabled ParamItem
CacheMemoryLimit ParamItem
CacheEnabled ParamItem `refreshable:"false"`
CacheMemoryLimit ParamItem `refreshable:"false"`
GroupEnabled ParamItem
MaxReceiveChanSize ParamItem
MaxUnsolvedQueueSize ParamItem
MaxReadConcurrency ParamItem
MaxGroupNQ ParamItem
TopKMergeRatio ParamItem
CPURatio ParamItem
GroupEnabled ParamItem `refreshable:"true"`
MaxReceiveChanSize ParamItem `refreshable:"false"`
MaxUnsolvedQueueSize ParamItem `refreshable:"true"`
MaxReadConcurrency ParamItem `refreshable:"true"`
MaxGroupNQ ParamItem `refreshable:"true"`
TopKMergeRatio ParamItem `refreshable:"true"`
CPURatio ParamItem `refreshable:"true"`
GCHelperEnabled ParamItem
MinimumGOGCConfig ParamItem
MaximumGOGCConfig ParamItem
GracefulStopTimeout ParamItem
GCHelperEnabled ParamItem `refreshable:"false"`
MinimumGOGCConfig ParamItem `refreshable:"false"`
MaximumGOGCConfig ParamItem `refreshable:"false"`
GracefulStopTimeout ParamItem `refreshable:"false"`
}
func (p *queryNodeConfig) init(base *BaseTable) {
@ -1240,40 +1288,40 @@ func (p *queryNodeConfig) init(base *BaseTable) {
type dataCoordConfig struct {
// --- CHANNEL ---
MaxWatchDuration ParamItem
MaxWatchDuration ParamItem `refreshable:"false"`
// --- SEGMENTS ---
SegmentMaxSize ParamItem
DiskSegmentMaxSize ParamItem
SegmentSealProportion ParamItem
SegAssignmentExpiration ParamItem
SegmentMaxLifetime ParamItem
SegmentMaxIdleTime ParamItem
SegmentMinSizeFromIdleToSealed ParamItem
SegmentMaxSize ParamItem `refreshable:"false"`
DiskSegmentMaxSize ParamItem `refreshable:"true"`
SegmentSealProportion ParamItem `refreshable:"false"`
SegAssignmentExpiration ParamItem `refreshable:"true"`
SegmentMaxLifetime ParamItem `refreshable:"false"`
SegmentMaxIdleTime ParamItem `refreshable:"false"`
SegmentMinSizeFromIdleToSealed ParamItem `refreshable:"false"`
// compaction
EnableCompaction ParamItem
EnableAutoCompaction ParamItem
EnableCompaction ParamItem `refreshable:"false"`
EnableAutoCompaction ParamItem `refreshable:"true"`
MinSegmentToMerge ParamItem
MaxSegmentToMerge ParamItem
SegmentSmallProportion ParamItem
SegmentCompactableProportion ParamItem
SegmentExpansionRate ParamItem
CompactionTimeoutInSeconds ParamItem
CompactionCheckIntervalInSeconds ParamItem
SingleCompactionRatioThreshold ParamItem
SingleCompactionDeltaLogMaxSize ParamItem
SingleCompactionExpiredLogMaxSize ParamItem
SingleCompactionDeltalogMaxNum ParamItem
GlobalCompactionInterval ParamItem
MinSegmentToMerge ParamItem `refreshable:"true"`
MaxSegmentToMerge ParamItem `refreshable:"true"`
SegmentSmallProportion ParamItem `refreshable:"true"`
SegmentCompactableProportion ParamItem `refreshable:"true"`
SegmentExpansionRate ParamItem `refreshable:"true"`
CompactionTimeoutInSeconds ParamItem `refreshable:"true"`
CompactionCheckIntervalInSeconds ParamItem `refreshable:"false"`
SingleCompactionRatioThreshold ParamItem `refreshable:"true"`
SingleCompactionDeltaLogMaxSize ParamItem `refreshable:"true"`
SingleCompactionExpiredLogMaxSize ParamItem `refreshable:"true"`
SingleCompactionDeltalogMaxNum ParamItem `refreshable:"true"`
GlobalCompactionInterval ParamItem `refreshable:"false"`
// Garbage Collection
EnableGarbageCollection ParamItem
GCInterval ParamItem
GCMissingTolerance ParamItem
GCDropTolerance ParamItem
EnableActiveStandby ParamItem
EnableGarbageCollection ParamItem `refreshable:"false"`
GCInterval ParamItem `refreshable:"false"`
GCMissingTolerance ParamItem `refreshable:"false"`
GCDropTolerance ParamItem `refreshable:"false"`
EnableActiveStandby ParamItem `refreshable:"false"`
}
func (p *dataCoordConfig) init(base *BaseTable) {
@ -1471,17 +1519,17 @@ func (p *dataCoordConfig) init(base *BaseTable) {
// /////////////////////////////////////////////////////////////////////////////
// --- datanode ---
type dataNodeConfig struct {
FlowGraphMaxQueueLength ParamItem
FlowGraphMaxParallelism ParamItem
FlowGraphMaxQueueLength ParamItem `refreshable:"false"`
FlowGraphMaxParallelism ParamItem `refreshable:"false"`
// segment
FlushInsertBufferSize ParamItem
FlushDeleteBufferBytes ParamItem
BinLogMaxSize ParamItem
SyncPeriod ParamItem
FlushInsertBufferSize ParamItem `refreshable:"true"`
FlushDeleteBufferBytes ParamItem `refreshable:"true"`
BinLogMaxSize ParamItem `refreshable:"true"`
SyncPeriod ParamItem `refreshable:"true"`
// io concurrency to fetch stats logs
IOConcurrency ParamItem
IOConcurrency ParamItem `refreshable:"false"`
}
func (p *dataNodeConfig) init(base *BaseTable) {
@ -1541,16 +1589,16 @@ func (p *dataNodeConfig) init(base *BaseTable) {
// /////////////////////////////////////////////////////////////////////////////
// --- indexcoord ---
type indexCoordConfig struct {
BindIndexNodeMode ParamItem
IndexNodeAddress ParamItem
WithCredential ParamItem
IndexNodeID ParamItem
BindIndexNodeMode ParamItem `refreshable:"false"`
IndexNodeAddress ParamItem `refreshable:"false"`
WithCredential ParamItem `refreshable:"false"`
IndexNodeID ParamItem `refreshable:"false"`
MinSegmentNumRowsToEnableIndex ParamItem
MinSegmentNumRowsToEnableIndex ParamItem `refreshable:"true"`
GCInterval ParamItem
GCInterval ParamItem `refreshable:"false"`
EnableActiveStandby ParamItem
EnableActiveStandby ParamItem `refreshable:"false"`
}
func (p *indexCoordConfig) init(base *BaseTable) {
@ -1607,13 +1655,13 @@ func (p *indexCoordConfig) init(base *BaseTable) {
// /////////////////////////////////////////////////////////////////////////////
// --- indexnode ---
type indexNodeConfig struct {
BuildParallel ParamItem
BuildParallel ParamItem `refreshable:"false"`
// enable disk
EnableDisk ParamItem
DiskCapacityLimit ParamItem
MaxDiskUsagePercentage ParamItem
EnableDisk ParamItem `refreshable:"false"`
DiskCapacityLimit ParamItem `refreshable:"true"`
MaxDiskUsagePercentage ParamItem `refreshable:"true"`
GracefulStopTimeout ParamItem
GracefulStopTimeout ParamItem `refreshable:"false"`
}
func (p *indexNodeConfig) init(base *BaseTable) {

606
internal/util/paramtable/grpc_param.go
View File

@ -14,8 +14,6 @@ package paramtable
import (
"fmt"
"strconv"
"sync"
"time"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/util/funcutil"
@ -39,15 +37,15 @@ const (
DefaultLogLevel = "WARNING"
// Grpc Timeout related configs
DefaultDialTimeout = 5000 * time.Millisecond
DefaultKeepAliveTime = 10000 * time.Millisecond
DefaultKeepAliveTimeout = 20000 * time.Millisecond
DefaultDialTimeout = 5000
DefaultKeepAliveTime = 10000
DefaultKeepAliveTimeout = 20000
// Grpc retry policy
DefaultMaxAttempts = 5
DefaultInitialBackoff float32 = 1.0
DefaultMaxBackoff float32 = 60.0
DefaultBackoffMultiplier float32 = 2.0
DefaultInitialBackoff float64 = 1.0
DefaultMaxBackoff float64 = 60.0
DefaultBackoffMultiplier float64 = 2.0
ProxyInternalPort = 19529
ProxyExternalPort = 19530
@ -56,400 +54,328 @@ const (
///////////////////////////////////////////////////////////////////////////////
// --- grpc ---
type grpcConfig struct {
ServiceParam
once sync.Once
Domain string
IP string
TLSMode int
Port int
InternalPort int
ServerPemPath string
ServerKeyPath string
CaPemPath string
Domain string `refreshable:"false"`
IP string `refreshable:"false"`
TLSMode ParamItem `refreshable:"false"`
Port ParamItem `refreshable:"false"`
InternalPort ParamItem `refreshable:"false"`
ServerPemPath ParamItem `refreshable:"false"`
ServerKeyPath ParamItem `refreshable:"false"`
CaPemPath ParamItem `refreshable:"false"`
}
func (p *grpcConfig) init(domain string) {
p.ServiceParam.Init()
func (p *grpcConfig) init(domain string, base *BaseTable) {
p.Domain = domain
p.LoadFromEnv()
p.LoadFromArgs()
p.initPort()
p.initTLSPath()
}
// LoadFromEnv is used to initialize configuration items from env.
func (p *grpcConfig) LoadFromEnv() {
p.IP = funcutil.GetLocalIP()
}
// LoadFromArgs is used to initialize configuration items from args.
func (p *grpcConfig) LoadFromArgs() {
p.Port = ParamItem{
Key: p.Domain + ".port",
Version: "2.0.0",
DefaultValue: strconv.FormatInt(ProxyExternalPort, 10),
}
p.Port.Init(base.mgr)
}
p.InternalPort = ParamItem{
Key: p.Domain + ".internalPort",
Version: "2.0.0",
DefaultValue: strconv.FormatInt(ProxyInternalPort, 10),
}
p.InternalPort.Init(base.mgr)
func (p *grpcConfig) initPort() {
p.Port = p.ParseIntWithDefault(p.Domain+".port", ProxyExternalPort)
p.InternalPort = p.ParseIntWithDefault(p.Domain+".internalPort", ProxyInternalPort)
}
p.TLSMode = ParamItem{
Key: "common.security.tlsMode",
Version: "2.0.0",
DefaultValue: "0",
}
p.TLSMode.Init(base.mgr)
func (p *grpcConfig) initTLSPath() {
p.TLSMode = p.ParseIntWithDefault("common.security.tlsMode", 0)
p.ServerPemPath = p.Get("tls.serverPemPath")
p.ServerKeyPath = p.Get("tls.serverKeyPath")
p.CaPemPath = p.Get("tls.caPemPath")
p.ServerPemPath = ParamItem{
Key: "tls.serverPemPath",
Version: "2.0.0",
}
p.ServerPemPath.Init(base.mgr)
p.ServerKeyPath = ParamItem{
Key: "tls.serverKeyPath",
Version: "2.0.0",
}
p.ServerKeyPath.Init(base.mgr)
p.CaPemPath = ParamItem{
Key: "tls.caPemPath",
Version: "2.0.0",
}
p.CaPemPath.Init(base.mgr)
}
// GetAddress return grpc address
func (p *grpcConfig) GetAddress() string {
return p.IP + ":" + strconv.Itoa(p.Port)
return p.IP + ":" + p.Port.GetValue()
}
func (p *grpcConfig) GetInternalAddress() string {
return p.IP + ":" + strconv.Itoa(p.InternalPort)
return p.IP + ":" + p.InternalPort.GetValue()
}
// GrpcServerConfig is configuration for grpc server.
type GrpcServerConfig struct {
grpcConfig
ServerMaxSendSize int
ServerMaxRecvSize int
ServerMaxSendSize ParamItem `refreshable:"false"`
ServerMaxRecvSize ParamItem `refreshable:"false"`
}
// InitOnce initialize grpc server config once
func (p *GrpcServerConfig) InitOnce(domain string) {
p.once.Do(func() {
p.Init(domain)
})
}
func (p *GrpcServerConfig) Init(domain string, base *BaseTable) {
p.grpcConfig.init(domain, base)
func (p *GrpcServerConfig) Init(domain string) {
p.grpcConfig.init(domain)
p.InitServerMaxSendSize()
p.InitServerMaxRecvSize()
}
func (p *GrpcServerConfig) InitServerMaxSendSize() {
var err error
valueStr, err := p.LoadWithPriority([]string{p.Domain + ".grpc.serverMaxSendSize", "grpc.serverMaxSendSize"})
if err != nil {
p.ServerMaxSendSize = DefaultServerMaxSendSize
maxSendSize := strconv.FormatInt(DefaultServerMaxSendSize, 10)
p.ServerMaxSendSize = ParamItem{
Key: p.Domain + ".grpc.serverMaxSendSize",
DefaultValue: maxSendSize,
FallbackKeys: []string{"grpc.serverMaxSendSize"},
Formatter: func(v string) string {
if v == "" {
return maxSendSize
}
_, err := strconv.Atoi(v)
if err != nil {
log.Warn("Failed to parse grpc.serverMaxSendSize, set to default",
zap.String("role", p.Domain), zap.String("grpc.serverMaxSendSize", v),
zap.Error(err))
return maxSendSize
}
return v
},
}
p.ServerMaxSendSize.Init(base.mgr)
value, err := strconv.Atoi(valueStr)
if err != nil {
log.Warn("Failed to parse grpc.serverMaxSendSize, set to default",
zap.String("role", p.Domain), zap.String("grpc.serverMaxSendSize", valueStr),
zap.Error(err))
p.ServerMaxSendSize = DefaultServerMaxSendSize
} else {
p.ServerMaxSendSize = value
maxRecvSize := strconv.FormatInt(DefaultServerMaxRecvSize, 10)
p.ServerMaxRecvSize = ParamItem{
Key: p.Domain + ".grpc.serverMaxRecvSize",
DefaultValue: maxRecvSize,
FallbackKeys: []string{"grpc.serverMaxRecvSize"},
Formatter: func(v string) string {
if v == "" {
return maxRecvSize
}
_, err := strconv.Atoi(v)
if err != nil {
log.Warn("Failed to parse grpc.serverMaxRecvSize, set to default",
zap.String("role", p.Domain), zap.String("grpc.serverMaxRecvSize", v),
zap.Error(err))
return maxRecvSize
}
return v
},
}
log.Debug("initServerMaxSendSize",
zap.String("role", p.Domain), zap.Int("grpc.serverMaxSendSize", p.ServerMaxSendSize))
}
func (p *GrpcServerConfig) InitServerMaxRecvSize() {
var err error
valueStr, err := p.LoadWithPriority([]string{p.Domain + ".grpc.serverMaxRecvSize", "grpc.serverMaxRecvSize"})
if err != nil {
p.ServerMaxRecvSize = DefaultServerMaxRecvSize
}
value, err := strconv.Atoi(valueStr)
if err != nil {
log.Warn("Failed to parse grpc.serverMaxRecvSize, set to default",
zap.String("role", p.Domain), zap.String("grpc.serverMaxRecvSize", valueStr),
zap.Error(err))
p.ServerMaxRecvSize = DefaultServerMaxRecvSize
} else {
p.ServerMaxRecvSize = value
}
log.Debug("initServerMaxRecvSize",
zap.String("role", p.Domain), zap.Int("grpc.serverMaxRecvSize", p.ServerMaxRecvSize))
p.ServerMaxRecvSize.Init(base.mgr)
}
// GrpcClientConfig is configuration for grpc client.
type GrpcClientConfig struct {
grpcConfig
ClientMaxSendSize int
ClientMaxRecvSize int
ClientMaxSendSize ParamItem `refreshable:"false"`
ClientMaxRecvSize ParamItem `refreshable:"false"`
DialTimeout time.Duration
KeepAliveTime time.Duration
KeepAliveTimeout time.Duration
DialTimeout ParamItem `refreshable:"false"`
KeepAliveTime ParamItem `refreshable:"false"`
KeepAliveTimeout ParamItem `refreshable:"false"`
MaxAttempts int
InitialBackoff float32
MaxBackoff float32
BackoffMultiplier float32
MaxAttempts ParamItem `refreshable:"false"`
InitialBackoff ParamItem `refreshable:"false"`
MaxBackoff ParamItem `refreshable:"false"`
BackoffMultiplier ParamItem `refreshable:"false"`
}
// InitOnce initialize grpc client config once
func (p *GrpcClientConfig) InitOnce(domain string) {
p.once.Do(func() {
p.init(domain)
})
}
func (p *GrpcClientConfig) Init(domain string, base *BaseTable) {
p.grpcConfig.init(domain, base)
func (p *GrpcClientConfig) init(domain string) {
p.grpcConfig.init(domain)
p.initClientMaxSendSize()
p.initClientMaxRecvSize()
p.initDialTimeout()
p.initKeepAliveTimeout()
p.initKeepAliveTime()
p.initMaxAttempts()
p.initInitialBackoff()
p.initMaxBackoff()
p.initBackoffMultiplier()
}
func (p *GrpcClientConfig) ParseConfig(funcDesc string, key string, backKey string, parseValue func(string) (interface{}, error), applyValue func(interface{}, error)) {
var err error
valueStr, err := p.Load(key)
if err != nil && backKey != "" {
valueStr, err = p.Load(backKey)
maxSendSize := strconv.FormatInt(DefaultClientMaxSendSize, 10)
p.ClientMaxSendSize = ParamItem{
Key: p.Domain + ".grpc.clientMaxSendSize",
DefaultValue: maxSendSize,
FallbackKeys: []string{"grpc.clientMaxSendSize"},
Formatter: func(v string) string {
if v == "" {
return maxSendSize
}
_, err := strconv.Atoi(v)
if err != nil {
log.Warn("Failed to parse grpc.clientMaxSendSize, set to default",
zap.String("role", p.Domain), zap.String("grpc.clientMaxSendSize", v),
zap.Error(err))
return maxSendSize
}
return v
},
}
if err != nil {
log.Warn(fmt.Sprintf("Failed to load %s, set to default", key), zap.String("role", p.Domain), zap.Error(err))
applyValue(nil, err)
} else {
value, err := parseValue(valueStr)
if err != nil {
log.Warn(fmt.Sprintf("Failed to parse %s, set to default", key),
zap.String("role", p.Domain), zap.String(key, valueStr), zap.Error(err))
applyValue(nil, err)
} else {
applyValue(value, nil)
}
p.ClientMaxSendSize.Init(base.mgr)
maxRecvSize := strconv.FormatInt(DefaultClientMaxRecvSize, 10)
p.ClientMaxRecvSize = ParamItem{
Key: p.Domain + ".grpc.clientMaxRecvSize",
DefaultValue: maxRecvSize,
FallbackKeys: []string{"grpc.clientMaxRecvSize"},
Formatter: func(v string) string {
if v == "" {
return maxRecvSize
}
_, err := strconv.Atoi(v)
if err != nil {
log.Warn("Failed to parse grpc.clientMaxRecvSize, set to default",
zap.String("role", p.Domain), zap.String("grpc.clientMaxRecvSize", v),
zap.Error(err))
return maxRecvSize
}
return v
},
}
p.ClientMaxRecvSize.Init(base.mgr)
log.Debug(funcDesc, zap.String("role", p.Domain), zap.Int(key, p.ClientMaxSendSize))
}
func (p *GrpcClientConfig) initClientMaxSendSize() {
funcDesc := "Init client max send size"
key := "grpc.clientMaxSendSize"
p.ParseConfig(funcDesc, key, fmt.Sprintf("%s.%s", p.Domain, key),
func(s string) (interface{}, error) {
return strconv.Atoi(s)
},
func(i interface{}, err error) {
dialTimeout := strconv.FormatInt(DefaultDialTimeout, 10)
p.DialTimeout = ParamItem{
Key: "grpc.client.dialTimeout",
Version: "2.0.0",
Formatter: func(v string) string {
if v == "" {
return dialTimeout
}
_, err := strconv.Atoi(v)
if err != nil {
p.ClientMaxSendSize = DefaultClientMaxSendSize
return
log.Warn("Failed to convert int when parsing grpc.client.dialTimeout, set to default",
zap.String("role", p.Domain),
zap.String("grpc.client.dialTimeout", v))
return dialTimeout
}
v, ok := i.(int)
if !ok {
log.Warn(fmt.Sprintf("Failed to convert int when parsing %s, set to default", key),
zap.String("role", p.Domain), zap.Any(key, i))
p.ClientMaxSendSize = DefaultClientMaxSendSize
return
}
p.ClientMaxSendSize = v
})
}
func (p *GrpcClientConfig) initClientMaxRecvSize() {
funcDesc := "Init client max recv size"
key := "grpc.clientMaxRecvSize"
p.ParseConfig(funcDesc, key, fmt.Sprintf("%s.%s", p.Domain, key),
func(s string) (interface{}, error) {
return strconv.Atoi(s)
return v
},
func(i interface{}, err error) {
if err != nil {
p.ClientMaxRecvSize = DefaultClientMaxRecvSize
return
}
v, ok := i.(int)
if !ok {
log.Warn(fmt.Sprintf("Failed to convert int when parsing %s, set to default", key),
zap.String("role", p.Domain), zap.Any(key, i))
p.ClientMaxRecvSize = DefaultClientMaxRecvSize
return
}
p.ClientMaxRecvSize = v
})
}
}
p.DialTimeout.Init(base.mgr)
func (p *GrpcClientConfig) initDialTimeout() {
funcDesc := "Init dial timeout"
key := "grpc.client.dialTimeout"
p.ParseConfig(funcDesc, key, "",
func(s string) (interface{}, error) {
return strconv.Atoi(s)
},
func(i interface{}, err error) {
keepAliveTimeout := strconv.FormatInt(DefaultKeepAliveTimeout, 10)
p.KeepAliveTimeout = ParamItem{
Key: "grpc.client.keepAliveTimeout",
Version: "2.0.0",
Formatter: func(v string) string {
if v == "" {
return keepAliveTimeout
}
_, err := strconv.Atoi(v)
if err != nil {
p.DialTimeout = DefaultDialTimeout
return
log.Warn("Failed to convert int when parsing grpc.client.keepAliveTimeout, set to default",
zap.String("role", p.Domain),
zap.String("grpc.client.keepAliveTimeout", v))
return keepAliveTimeout
}
v, ok := i.(int)
if !ok {
log.Warn(fmt.Sprintf("Failed to convert int when parsing %s, set to default", key),
zap.String("role", p.Domain), zap.Any(key, i))
p.DialTimeout = DefaultDialTimeout
return
}
p.DialTimeout = time.Duration(v) * time.Millisecond
})
}
return v
},
}
p.KeepAliveTimeout.Init(base.mgr)
func (p *GrpcClientConfig) initKeepAliveTime() {
funcDesc := "Init keep alive time"
key := "grpc.client.keepAliveTime"
p.ParseConfig(funcDesc, key, "",
func(s string) (interface{}, error) {
return strconv.Atoi(s)
},
func(i interface{}, err error) {
keepAliveTime := strconv.FormatInt(DefaultKeepAliveTime, 10)
p.KeepAliveTime = ParamItem{
Key: "grpc.client.keepAliveTime",
Version: "2.0.0",
Formatter: func(v string) string {
if v == "" {
return keepAliveTime
}
_, err := strconv.Atoi(v)
if err != nil {
p.KeepAliveTime = DefaultKeepAliveTime
return
log.Warn("Failed to convert int when parsing grpc.client.keepAliveTime, set to default",
zap.String("role", p.Domain),
zap.String("grpc.client.keepAliveTime", v))
return keepAliveTime
}
v, ok := i.(int)
if !ok {
log.Warn(fmt.Sprintf("Failed to convert int when parsing %s, set to default", key),
zap.String("role", p.Domain), zap.Any(key, i))
p.KeepAliveTime = DefaultKeepAliveTime
return
}
p.KeepAliveTime = time.Duration(v) * time.Millisecond
})
}
return v
},
}
p.KeepAliveTime.Init(base.mgr)
func (p *GrpcClientConfig) initKeepAliveTimeout() {
funcDesc := "Init keep alive timeout"
key := "grpc.client.keepAliveTimeout"
p.ParseConfig(funcDesc, key, "",
func(s string) (interface{}, error) {
return strconv.Atoi(s)
},
func(i interface{}, err error) {
maxAttempts := strconv.FormatInt(DefaultMaxAttempts, 10)
p.MaxAttempts = ParamItem{
Key: "grpc.client.maxMaxAttempts",
Version: "2.0.0",
Formatter: func(v string) string {
if v == "" {
return maxAttempts
}
iv, err := strconv.Atoi(v)
if err != nil {
p.KeepAliveTimeout = DefaultKeepAliveTimeout
return
log.Warn("Failed to convert int when parsing grpc.client.maxMaxAttempts, set to default",
zap.String("role", p.Domain),
zap.String("grpc.client.maxMaxAttempts", v))
return maxAttempts
}
v, ok := i.(int)
if !ok {
log.Warn(fmt.Sprintf("Failed to convert int when parsing %s, set to default", key),
zap.String("role", p.Domain), zap.Any(key, i))
p.KeepAliveTimeout = DefaultKeepAliveTimeout
return
if iv < 2 || iv > 5 {
log.Warn("The value of %s should be greater than 1 and less than 6, set to default",
zap.String("role", p.Domain),
zap.String("grpc.client.maxMaxAttempts", v))
return maxAttempts
}
p.KeepAliveTimeout = time.Duration(v) * time.Millisecond
})
}
return v
},
}
p.MaxAttempts.Init(base.mgr)
func (p *GrpcClientConfig) initMaxAttempts() {
funcDesc := "Init max attempts"
key := "grpc.client.maxMaxAttempts"
p.ParseConfig(funcDesc, key, "",
func(s string) (interface{}, error) {
return strconv.Atoi(s)
},
func(i interface{}, err error) {
initialBackoff := fmt.Sprintf("%f", DefaultInitialBackoff)
p.InitialBackoff = ParamItem{
Key: "grpc.client.initialBackoff",
Version: "2.0.0",
Formatter: func(v string) string {
if v == "" {
return initialBackoff
}
_, err := strconv.Atoi(v)
if err != nil {
p.MaxAttempts = DefaultMaxAttempts
return
log.Warn("Failed to convert int when parsing grpc.client.initialBackoff, set to default",
zap.String("role", p.Domain),
zap.String("grpc.client.initialBackoff", v))
return initialBackoff
}
// This field is required and must be greater than 1.
// Any value greater than 5 will be treated as if it were 5.
// See: https://github.com/grpc/grpc-proto/blob/master/grpc/service_config/service_config.proto#L138
v, ok := i.(int)
if !ok {
log.Warn(fmt.Sprintf("Failed to convert int when parsing %s, set to default", key),
zap.String("role", p.Domain), zap.Any(key, i))
p.MaxAttempts = DefaultMaxAttempts
return
}
if v < 2 || v > 5 {
log.Warn(fmt.Sprintf("The value of %s should be greater than 1 and less than 6, set to default", key),
zap.String("role", p.Domain), zap.Any(key, i))
p.MaxAttempts = DefaultMaxAttempts
return
}
p.MaxAttempts = v
})
}
return v
},
}
p.InitialBackoff.Init(base.mgr)
func (p *GrpcClientConfig) initInitialBackoff() {
funcDesc := "Init initial back off"
key := "grpc.client.initialBackOff"
p.ParseConfig(funcDesc, key, "",
func(s string) (interface{}, error) {
return strconv.ParseFloat(s, 32)
},
func(i interface{}, err error) {
maxBackoff := fmt.Sprintf("%f", DefaultMaxBackoff)
p.MaxBackoff = ParamItem{
Key: "grpc.client.maxBackoff",
Version: "2.0.0",
Formatter: func(v string) string {
if v == "" {
return maxBackoff
}
_, err := strconv.ParseFloat(v, 64)
if err != nil {
p.InitialBackoff = DefaultInitialBackoff
return
log.Warn("Failed to convert int when parsing grpc.client.maxBackoff, set to default",
zap.String("role", p.Domain),
zap.String("grpc.client.maxBackoff", v))
return maxBackoff
}
v, ok := i.(float64)
if !ok {
log.Warn(fmt.Sprintf("Failed to convert float64 when parsing %s, set to default", key),
zap.String("role", p.Domain), zap.Any(key, i))
p.InitialBackoff = DefaultInitialBackoff
return
}
p.InitialBackoff = float32(v)
})
}
return v
},
}
p.MaxBackoff.Init(base.mgr)
func (p *GrpcClientConfig) initMaxBackoff() {
funcDesc := "Init max back off"
key := "grpc.client.maxBackoff"
p.ParseConfig(funcDesc, key, "",
func(s string) (interface{}, error) {
return strconv.ParseFloat(s, 32)
},
func(i interface{}, err error) {
backoffMultiplier := fmt.Sprintf("%f", DefaultBackoffMultiplier)
p.BackoffMultiplier = ParamItem{
Key: "grpc.client.backoffMultiplier",
Version: "2.0.0",
Formatter: func(v string) string {
if v == "" {
return backoffMultiplier
}
_, err := strconv.ParseFloat(v, 64)
if err != nil {
p.MaxBackoff = DefaultMaxBackoff
return
log.Warn("Failed to convert int when parsing grpc.client.backoffMultiplier, set to default",
zap.String("role", p.Domain),
zap.String("grpc.client.backoffMultiplier", v))
return backoffMultiplier
}
v, ok := i.(float64)
if !ok {
log.Warn(fmt.Sprintf("Failed to convert float64 when parsing %s, set to default", key),
zap.String("role", p.Domain), zap.Any(key, i))
p.MaxBackoff = DefaultMaxBackoff
return
}
p.MaxBackoff = float32(v)
})
}
func (p *GrpcClientConfig) initBackoffMultiplier() {
funcDesc := "Init back off multiplier"
key := "grpc.client.backoffMultiplier"
p.ParseConfig(funcDesc, key, "",
func(s string) (interface{}, error) {
return strconv.ParseFloat(s, 32)
return v
},
func(i interface{}, err error) {
if err != nil {
p.BackoffMultiplier = DefaultBackoffMultiplier
return
}
v, ok := i.(float64)
if !ok {
log.Warn(fmt.Sprintf("Failed to convert float64 when parsing %s, set to default", key),
zap.String("role", p.Domain), zap.Any(key, i))
p.BackoffMultiplier = DefaultBackoffMultiplier
return
}
p.BackoffMultiplier = float32(v)
})
}
p.BackoffMultiplier.Init(base.mgr)
}

225
internal/util/paramtable/grpc_param_test.go
View File

@ -21,166 +21,133 @@ import (
func TestGrpcServerParams(t *testing.T) {
role := typeutil.DataNodeRole
var Params GrpcServerConfig
Params.InitOnce(role)
base := BaseTable{}
base.Init(0)
var serverConfig GrpcServerConfig
serverConfig.Init(role, &base)
assert.Equal(t, Params.Domain, role)
t.Logf("Domain = %s", Params.Domain)
assert.Equal(t, serverConfig.Domain, role)
t.Logf("Domain = %s", serverConfig.Domain)
assert.NotEqual(t, Params.IP, "")
t.Logf("IP = %s", Params.IP)
assert.NotEqual(t, serverConfig.IP, "")
t.Logf("IP = %s", serverConfig.IP)
assert.NotZero(t, Params.Port)
t.Logf("Port = %d", Params.Port)
assert.NotZero(t, serverConfig.Port.GetValue())
t.Logf("Port = %d", serverConfig.Port.GetAsInt())
t.Logf("Address = %s", Params.GetAddress())
t.Logf("Address = %s", serverConfig.GetAddress())
assert.NotZero(t, Params.ServerMaxRecvSize)
t.Logf("ServerMaxRecvSize = %d", Params.ServerMaxRecvSize)
assert.NotZero(t, serverConfig.ServerMaxRecvSize.GetAsInt())
t.Logf("ServerMaxRecvSize = %d", serverConfig.ServerMaxRecvSize.GetAsInt())
Params.Remove(role + ".grpc.serverMaxRecvSize")
Params.InitServerMaxRecvSize()
assert.Equal(t, Params.ServerMaxRecvSize, DefaultServerMaxRecvSize)
base.Remove(role + ".grpc.serverMaxRecvSize")
assert.Equal(t, serverConfig.ServerMaxRecvSize.GetAsInt(), DefaultServerMaxRecvSize)
Params.Remove("grpc.serverMaxRecvSize")
Params.InitServerMaxRecvSize()
assert.Equal(t, Params.ServerMaxRecvSize, DefaultServerMaxRecvSize)
base.Remove("grpc.serverMaxRecvSize")
assert.Equal(t, serverConfig.ServerMaxRecvSize.GetAsInt(), DefaultServerMaxRecvSize)
Params.Save("grpc.serverMaxRecvSize", "a")
Params.InitServerMaxRecvSize()
assert.Equal(t, Params.ServerMaxSendSize, DefaultServerMaxRecvSize)
base.Save("grpc.serverMaxRecvSize", "a")
assert.Equal(t, serverConfig.ServerMaxSendSize.GetAsInt(), DefaultServerMaxRecvSize)
assert.NotZero(t, Params.ServerMaxSendSize)
t.Logf("ServerMaxSendSize = %d", Params.ServerMaxSendSize)
assert.NotZero(t, serverConfig.ServerMaxSendSize)
t.Logf("ServerMaxSendSize = %d", serverConfig.ServerMaxSendSize.GetAsInt())
Params.Remove(role + ".grpc.serverMaxSendSize")
Params.InitServerMaxSendSize()
assert.Equal(t, Params.ServerMaxSendSize, DefaultServerMaxSendSize)
base.Remove(role + ".grpc.serverMaxSendSize")
assert.Equal(t, serverConfig.ServerMaxSendSize.GetAsInt(), DefaultServerMaxSendSize)
Params.Remove("grpc.serverMaxSendSize")
Params.InitServerMaxRecvSize()
assert.Equal(t, Params.ServerMaxSendSize, DefaultServerMaxSendSize)
base.Remove("grpc.serverMaxSendSize")
assert.Equal(t, serverConfig.ServerMaxSendSize.GetAsInt(), DefaultServerMaxSendSize)
Params.Save("grpc.serverMaxSendSize", "a")
Params.InitServerMaxRecvSize()
assert.Equal(t, Params.ServerMaxSendSize, DefaultServerMaxSendSize)
base.Save("grpc.serverMaxSendSize", "a")
assert.Equal(t, serverConfig.ServerMaxSendSize.GetAsInt(), DefaultServerMaxSendSize)
}
func TestGrpcClientParams(t *testing.T) {
role := typeutil.DataNodeRole
var Params GrpcClientConfig
Params.InitOnce(role)
base := BaseTable{}
base.Init(0)
var clientConfig GrpcClientConfig
clientConfig.Init(role, &base)
assert.Equal(t, Params.Domain, role)
t.Logf("Domain = %s", Params.Domain)
assert.Equal(t, clientConfig.Domain, role)
t.Logf("Domain = %s", clientConfig.Domain)
assert.NotEqual(t, Params.IP, "")
t.Logf("IP = %s", Params.IP)
assert.NotEqual(t, clientConfig.IP, "")
t.Logf("IP = %s", clientConfig.IP)
assert.NotZero(t, Params.Port)
t.Logf("Port = %d", Params.Port)
assert.NotZero(t, clientConfig.Port.GetAsInt())
t.Logf("Port = %d", clientConfig.Port.GetAsInt())
t.Logf("Address = %s", Params.GetAddress())
t.Logf("Address = %s", clientConfig.GetAddress())
assert.NotZero(t, Params.ClientMaxRecvSize)
t.Logf("ClientMaxRecvSize = %d", Params.ClientMaxRecvSize)
assert.NotZero(t, clientConfig.ClientMaxRecvSize.GetAsInt())
t.Logf("ClientMaxRecvSize = %d", clientConfig.ClientMaxRecvSize.GetAsInt())
Params.Remove("grpc.clientMaxRecvSize")
Params.Save(role+".grpc.clientMaxRecvSize", "1000")
Params.initClientMaxRecvSize()
assert.Equal(t, Params.ClientMaxRecvSize, 1000)
base.Remove("grpc.clientMaxRecvSize")
base.Save(role+".grpc.clientMaxRecvSize", "1000")
assert.Equal(t, clientConfig.ClientMaxRecvSize.GetAsInt(), 1000)
Params.Remove(role + ".grpc.clientMaxRecvSize")
Params.initClientMaxRecvSize()
assert.Equal(t, Params.ClientMaxRecvSize, DefaultClientMaxRecvSize)
base.Remove(role + ".grpc.clientMaxRecvSize")
assert.Equal(t, clientConfig.ClientMaxRecvSize.GetAsInt(), DefaultClientMaxRecvSize)
assert.NotZero(t, Params.ClientMaxSendSize)
t.Logf("ClientMaxSendSize = %d", Params.ClientMaxSendSize)
assert.NotZero(t, clientConfig.ClientMaxSendSize.GetAsInt())
t.Logf("ClientMaxSendSize = %d", clientConfig.ClientMaxSendSize.GetAsInt())
Params.Remove("grpc.clientMaxSendSize")
Params.Save(role+".grpc.clientMaxSendSize", "2000")
Params.initClientMaxSendSize()
assert.Equal(t, Params.ClientMaxSendSize, 2000)
base.Remove("grpc.clientMaxSendSize")
base.Save(role+".grpc.clientMaxSendSize", "2000")
assert.Equal(t, clientConfig.ClientMaxSendSize.GetAsInt(), 2000)
Params.Remove(role + ".grpc.clientMaxSendSize")
Params.initClientMaxSendSize()
assert.Equal(t, Params.ClientMaxSendSize, DefaultClientMaxSendSize)
base.Remove(role + ".grpc.clientMaxSendSize")
assert.Equal(t, clientConfig.ClientMaxSendSize.GetAsInt(), DefaultClientMaxSendSize)
Params.initDialTimeout()
assert.Equal(t, Params.DialTimeout, DefaultDialTimeout)
Params.Save("grpc.client.dialTimeout", "aaa")
Params.initDialTimeout()
assert.Equal(t, Params.DialTimeout, DefaultDialTimeout)
Params.Save("grpc.client.dialTimeout", "100")
Params.initDialTimeout()
assert.Equal(t, Params.DialTimeout, 100*time.Millisecond)
assert.Equal(t, clientConfig.DialTimeout.GetAsInt(), DefaultDialTimeout)
base.Save("grpc.client.dialTimeout", "aaa")
assert.Equal(t, clientConfig.DialTimeout.GetAsInt(), DefaultDialTimeout)
base.Save("grpc.client.dialTimeout", "100")
assert.Equal(t, clientConfig.DialTimeout.GetAsDuration(time.Millisecond), 100*time.Millisecond)
Params.initKeepAliveTime()
assert.Equal(t, Params.KeepAliveTime, DefaultKeepAliveTime)
Params.Save("grpc.client.keepAliveTime", "a")
Params.initKeepAliveTime()
assert.Equal(t, Params.KeepAliveTime, DefaultKeepAliveTime)
Params.Save("grpc.client.keepAliveTime", "200")
Params.initKeepAliveTime()
assert.Equal(t, Params.KeepAliveTime, 200*time.Millisecond)
assert.Equal(t, clientConfig.KeepAliveTime.GetAsInt(), DefaultKeepAliveTime)
base.Save("grpc.client.keepAliveTime", "a")
assert.Equal(t, clientConfig.KeepAliveTime.GetAsInt(), DefaultKeepAliveTime)
base.Save("grpc.client.keepAliveTime", "200")
assert.Equal(t, clientConfig.KeepAliveTime.GetAsDuration(time.Millisecond), 200*time.Millisecond)
Params.initKeepAliveTimeout()
assert.Equal(t, Params.KeepAliveTimeout, DefaultKeepAliveTimeout)
Params.Save("grpc.client.keepAliveTimeout", "a")
Params.initKeepAliveTimeout()
assert.Equal(t, Params.KeepAliveTimeout, DefaultKeepAliveTimeout)
Params.Save("grpc.client.keepAliveTimeout", "500")
Params.initKeepAliveTimeout()
assert.Equal(t, Params.KeepAliveTimeout, 500*time.Millisecond)
assert.Equal(t, clientConfig.KeepAliveTimeout.GetAsInt(), DefaultKeepAliveTimeout)
base.Save("grpc.client.keepAliveTimeout", "a")
assert.Equal(t, clientConfig.KeepAliveTimeout.GetAsInt(), DefaultKeepAliveTimeout)
base.Save("grpc.client.keepAliveTimeout", "500")
assert.Equal(t, clientConfig.KeepAliveTimeout.GetAsDuration(time.Millisecond), 500*time.Millisecond)
Params.initMaxAttempts()
assert.Equal(t, Params.MaxAttempts, DefaultMaxAttempts)
Params.Save("grpc.client.maxMaxAttempts", "a")
Params.initMaxAttempts()
assert.Equal(t, Params.MaxAttempts, DefaultMaxAttempts)
Params.Save("grpc.client.maxMaxAttempts", "1")
Params.initMaxAttempts()
assert.Equal(t, Params.MaxAttempts, DefaultMaxAttempts)
Params.Save("grpc.client.maxMaxAttempts", "10")
Params.initMaxAttempts()
assert.Equal(t, Params.MaxAttempts, DefaultMaxAttempts)
Params.Save("grpc.client.maxMaxAttempts", "4")
Params.initMaxAttempts()
assert.Equal(t, Params.MaxAttempts, 4)
assert.Equal(t, clientConfig.MaxAttempts.GetAsInt(), DefaultMaxAttempts)
base.Save("grpc.client.maxMaxAttempts", "a")
assert.Equal(t, clientConfig.MaxAttempts.GetAsInt(), DefaultMaxAttempts)
base.Save("grpc.client.maxMaxAttempts", "1")
assert.Equal(t, clientConfig.MaxAttempts.GetAsInt(), DefaultMaxAttempts)
base.Save("grpc.client.maxMaxAttempts", "10")
assert.Equal(t, clientConfig.MaxAttempts.GetAsInt(), DefaultMaxAttempts)
base.Save("grpc.client.maxMaxAttempts", "4")
assert.Equal(t, clientConfig.MaxAttempts.GetAsInt(), 4)
Params.initInitialBackoff()
assert.Equal(t, Params.InitialBackoff, DefaultInitialBackoff)
Params.Save("grpc.client.initialBackOff", "a")
Params.initInitialBackoff()
assert.Equal(t, Params.InitialBackoff, DefaultInitialBackoff)
Params.Save("grpc.client.initialBackOff", "2.0")
Params.initInitialBackoff()
assert.Equal(t, Params.InitialBackoff, float32(2.0))
base.Save("grpc.client.initialBackOff", "a")
base.Save("grpc.client.initialBackOff", "2.0")
Params.initMaxBackoff()
assert.Equal(t, Params.MaxBackoff, DefaultMaxBackoff)
Params.Save("grpc.client.maxBackOff", "a")
Params.initMaxBackoff()
assert.Equal(t, Params.MaxBackoff, DefaultMaxBackoff)
Params.Save("grpc.client.maxBackOff", "50.0")
Params.initMaxBackoff()
assert.Equal(t, Params.MaxBackoff, float32(50.0))
assert.Equal(t, clientConfig.MaxBackoff.GetAsFloat(), DefaultMaxBackoff)
base.Save("grpc.client.maxBackOff", "a")
assert.Equal(t, clientConfig.MaxBackoff.GetAsFloat(), DefaultMaxBackoff)
base.Save("grpc.client.maxBackOff", "50.0")
assert.Equal(t, clientConfig.MaxBackoff.GetAsFloat(), 50.0)
Params.initBackoffMultiplier()
assert.Equal(t, Params.BackoffMultiplier, DefaultBackoffMultiplier)
Params.Save("grpc.client.backoffMultiplier", "a")
Params.initBackoffMultiplier()
assert.Equal(t, Params.BackoffMultiplier, DefaultBackoffMultiplier)
Params.Save("grpc.client.backoffMultiplier", "3.0")
Params.initBackoffMultiplier()
assert.Equal(t, Params.BackoffMultiplier, float32(3.0))
assert.Equal(t, clientConfig.BackoffMultiplier.GetAsFloat(), DefaultBackoffMultiplier)
base.Save("grpc.client.backoffMultiplier", "a")
assert.Equal(t, clientConfig.BackoffMultiplier.GetAsFloat(), DefaultBackoffMultiplier)
base.Save("grpc.client.backoffMultiplier", "3.0")
assert.Equal(t, clientConfig.BackoffMultiplier.GetAsFloat(), 3.0)
Params.Save("common.security.tlsMode", "1")
Params.Save("tls.serverPemPath", "/pem")
Params.Save("tls.serverKeyPath", "/key")
Params.Save("tls.caPemPath", "/ca")
Params.initTLSPath()
assert.Equal(t, Params.TLSMode, 1)
assert.Equal(t, Params.ServerPemPath, "/pem")
assert.Equal(t, Params.ServerKeyPath, "/key")
assert.Equal(t, Params.CaPemPath, "/ca")
base.Save("common.security.tlsMode", "1")
base.Save("tls.serverPemPath", "/pem")
base.Save("tls.serverKeyPath", "/key")
base.Save("tls.caPemPath", "/ca")
assert.Equal(t, clientConfig.TLSMode.GetAsInt(), 1)
assert.Equal(t, clientConfig.ServerPemPath.GetValue(), "/pem")
assert.Equal(t, clientConfig.ServerKeyPath.GetValue(), "/key")
assert.Equal(t, clientConfig.CaPemPath.GetValue(), "/ca")
}

6
internal/util/paramtable/hook_config.go
View File

@ -3,13 +3,13 @@ package paramtable
const hookYamlFile = "hook.yaml"
type hookConfig struct {
SoPath ParamItem
SoConfig ParamGroup
SoPath ParamItem `refreshable:"false"`
SoConfig ParamGroup `refreshable:"false"`
}
func (h *hookConfig) init() {
base := &BaseTable{YamlFile: hookYamlFile}
base.Init()
base.Init(0)
h.SoPath = ParamItem{
Key: "soPath",

45
internal/util/paramtable/http_param.go
View File

@ -1,35 +1,22 @@
package paramtable
import (
"sync"
)
type HTTPConfig struct {
BaseTable
once sync.Once
Enabled bool
DebugMode bool
type httpConfig struct {
Enabled ParamItem `refreshable:"false"`
DebugMode ParamItem `refreshable:"false"`
}
// InitOnce initialize HTTPConfig
func (p *HTTPConfig) InitOnce() {
p.once.Do(func() {
p.init()
})
}
func (p *httpConfig) init(base *BaseTable) {
p.Enabled = ParamItem{
Key: "proxy.http.enabled",
DefaultValue: "true",
Version: "2.1.0",
}
p.Enabled.Init(base.mgr)
func (p *HTTPConfig) init() {
p.BaseTable.Init()
p.initHTTPEnabled()
p.initHTTPDebugMode()
}
func (p *HTTPConfig) initHTTPEnabled() {
p.Enabled = p.ParseBool("proxy.http.enabled", true)
}
func (p *HTTPConfig) initHTTPDebugMode() {
p.DebugMode = p.ParseBool("proxy.http.debug_mode", false)
p.DebugMode = ParamItem{
Key: "proxy.http.debug_mode",
DefaultValue: "false",
Version: "2.1.0",
}
p.DebugMode.Init(base.mgr)
}

9
internal/util/paramtable/http_param_test.go
View File

@ -7,8 +7,9 @@ import (
)
func TestHTTPConfig_Init(t *testing.T) {
cf := new(HTTPConfig)
cf.InitOnce()
assert.Equal(t, cf.Enabled, true)
assert.Equal(t, cf.DebugMode, false)
params := ComponentParam{}
params.InitOnce()
cf := params.HTTPCfg
assert.Equal(t, cf.Enabled.GetAsBool(), true)
assert.Equal(t, cf.DebugMode.GetAsBool(), false)
}

84
internal/util/paramtable/quota_param.go
View File

@ -43,60 +43,62 @@ const (
// quotaConfig is configuration for quota and limitations.
type quotaConfig struct {
QuotaAndLimitsEnabled ParamItem
QuotaCenterCollectInterval ParamItem
QuotaAndLimitsEnabled ParamItem `refreshable:"false"`
QuotaCenterCollectInterval ParamItem `refreshable:"false"`
// ddl
DDLLimitEnabled ParamItem
DDLCollectionRate ParamItem
DDLPartitionRate ParamItem
DDLLimitEnabled ParamItem `refreshable:"true"`
DDLCollectionRate ParamItem `refreshable:"false"`
DDLPartitionRate ParamItem `refreshable:"false"`
IndexLimitEnabled ParamItem
MaxIndexRate ParamItem
FlushLimitEnabled ParamItem
MaxFlushRate ParamItem
CompactionLimitEnabled ParamItem
MaxCompactionRate ParamItem
IndexLimitEnabled ParamItem `refreshable:"true"`
MaxIndexRate ParamItem `refreshable:"false"`
FlushLimitEnabled ParamItem `refreshable:"true"`
MaxFlushRate ParamItem `refreshable:"false"`
CompactionLimitEnabled ParamItem `refreshable:"true"`
MaxCompactionRate ParamItem `refreshable:"false"`
// dml
DMLLimitEnabled ParamItem
DMLMaxInsertRate ParamItem
DMLMinInsertRate ParamItem
DMLMaxDeleteRate ParamItem
DMLMinDeleteRate ParamItem
DMLMaxBulkLoadRate ParamItem
DMLMinBulkLoadRate ParamItem
DMLLimitEnabled ParamItem `refreshable:"true"`
DMLMaxInsertRate ParamItem `refreshable:"false"`
DMLMinInsertRate ParamItem `refreshable:"false"`
DMLMaxDeleteRate ParamItem `refreshable:"false"`
DMLMinDeleteRate ParamItem `refreshable:"false"`
DMLMaxBulkLoadRate ParamItem `refreshable:"false"`
DMLMinBulkLoadRate ParamItem `refreshable:"false"`
// dql
DQLLimitEnabled ParamItem
DQLMaxSearchRate ParamItem
DQLMinSearchRate ParamItem
DQLMaxQueryRate ParamItem
DQLMinQueryRate ParamItem
DQLLimitEnabled ParamItem `refreshable:"true"`
DQLMaxSearchRate ParamItem `refreshable:"false"`
DQLMinSearchRate ParamItem `refreshable:"false"`
DQLMaxQueryRate ParamItem `refreshable:"false"`
DQLMinQueryRate ParamItem `refreshable:"false"`
// limits
MaxCollectionNum ParamItem
MaxCollectionNum ParamItem `refreshable:"true"`
// limit writing
ForceDenyWriting ParamItem
TtProtectionEnabled ParamItem
MaxTimeTickDelay ParamItem
MemProtectionEnabled ParamItem
DataNodeMemoryLowWaterLevel ParamItem
DataNodeMemoryHighWaterLevel ParamItem
QueryNodeMemoryLowWaterLevel ParamItem
QueryNodeMemoryHighWaterLevel ParamItem
DiskProtectionEnabled ParamItem
DiskQuota ParamItem
ForceDenyWriting ParamItem `refreshable:"true"`
TtProtectionEnabled ParamItem `refreshable:"true"`
MaxTimeTickDelay ParamItem `refreshable:"true"`
MemProtectionEnabled ParamItem `refreshable:"true"`
DataNodeMemoryLowWaterLevel ParamItem `refreshable:"true"`
DataNodeMemoryHighWaterLevel ParamItem `refreshable:"true"`
QueryNodeMemoryLowWaterLevel ParamItem `refreshable:"true"`
QueryNodeMemoryHighWaterLevel ParamItem `refreshable:"true"`
DiskProtectionEnabled ParamItem `refreshable:"true"`
DiskQuota ParamItem `refreshable:"true"`
// limit reading
ForceDenyReading ParamItem
QueueProtectionEnabled ParamItem
NQInQueueThreshold ParamItem
QueueLatencyThreshold ParamItem
ResultProtectionEnabled ParamItem
MaxReadResultRate ParamItem
CoolOffSpeed ParamItem
ForceDenyReading ParamItem `refreshable:"true"`
QueueProtectionEnabled ParamItem `refreshable:"true"`
NQInQueueThreshold ParamItem `refreshable:"true"`
QueueLatencyThreshold ParamItem `refreshable:"true"`
ResultProtectionEnabled ParamItem `refreshable:"true"`
MaxReadResultRate ParamItem `refreshable:"true"`
CoolOffSpeed ParamItem `refreshable:"true"`
}
func (p *quotaConfig) init(base *BaseTable) {

151
internal/util/paramtable/service_param.go
View File

@ -54,15 +54,12 @@ type ServiceParam struct {
}
func (p *ServiceParam) Init() {
p.BaseTable.Init()
p.BaseTable.Init(10)
p.LocalStorageCfg.Init(&p.BaseTable)
p.MetaStoreCfg.Init(&p.BaseTable)
p.EtcdCfg.Init(&p.BaseTable)
if p.MetaStoreCfg.MetaStoreType.GetValue() == util.MetaStoreTypeMysql {
log.Debug("Mysql protocol is used as meta store")
p.DBCfg.Init(&p.BaseTable)
}
p.DBCfg.Init(&p.BaseTable)
p.PulsarCfg.Init(&p.BaseTable)
p.KafkaCfg.Init(&p.BaseTable)
p.RocksmqCfg.Init(&p.BaseTable)
@ -73,24 +70,24 @@ func (p *ServiceParam) Init() {
// --- etcd ---
type EtcdConfig struct {
// --- ETCD ---
Endpoints ParamItem
RootPath ParamItem
MetaSubPath ParamItem
KvSubPath ParamItem
MetaRootPath CompositeParamItem
KvRootPath CompositeParamItem
EtcdLogLevel ParamItem
EtcdLogPath ParamItem
EtcdUseSSL ParamItem
EtcdTLSCert ParamItem
EtcdTLSKey ParamItem
EtcdTLSCACert ParamItem
EtcdTLSMinVersion ParamItem
Endpoints ParamItem `refreshable:"false"`
RootPath ParamItem `refreshable:"false"`
MetaSubPath ParamItem `refreshable:"false"`
KvSubPath ParamItem `refreshable:"false"`
MetaRootPath CompositeParamItem `refreshable:"false"`
KvRootPath CompositeParamItem `refreshable:"false"`
EtcdLogLevel ParamItem `refreshable:"false"`
EtcdLogPath ParamItem `refreshable:"false"`
EtcdUseSSL ParamItem `refreshable:"false"`
EtcdTLSCert ParamItem `refreshable:"false"`
EtcdTLSKey ParamItem `refreshable:"false"`
EtcdTLSCACert ParamItem `refreshable:"false"`
EtcdTLSMinVersion ParamItem `refreshable:"false"`
// --- Embed ETCD ---
UseEmbedEtcd ParamItem
ConfigPath ParamItem
DataDir ParamItem
UseEmbedEtcd ParamItem `refreshable:"false"`
ConfigPath ParamItem `refreshable:"false"`
DataDir ParamItem `refreshable:"false"`
}
func (p *EtcdConfig) Init(base *BaseTable) {
@ -112,27 +109,25 @@ func (p *EtcdConfig) Init(base *BaseTable) {
panic("embedded etcd can not be used under distributed mode")
}
if p.UseEmbedEtcd.GetAsBool() {
p.ConfigPath = ParamItem{
Key: "etcd.config.path",
Version: "2.1.0",
}
p.ConfigPath.Init(base.mgr)
p.DataDir = ParamItem{
Key: "etcd.data.dir",
DefaultValue: "default.etcd",
Version: "2.1.0",
}
p.DataDir.Init(base.mgr)
} else {
p.Endpoints = ParamItem{
Key: "etcd.endpoints",
Version: "2.0.0",
PanicIfEmpty: true,
}
p.Endpoints.Init(base.mgr)
p.ConfigPath = ParamItem{
Key: "etcd.config.path",
Version: "2.1.0",
}
p.ConfigPath.Init(base.mgr)
p.DataDir = ParamItem{
Key: "etcd.data.dir",
DefaultValue: "default.etcd",
Version: "2.1.0",
}
p.DataDir.Init(base.mgr)
p.Endpoints = ParamItem{
Key: "etcd.endpoints",
Version: "2.0.0",
PanicIfEmpty: true,
}
p.Endpoints.Init(base.mgr)
p.RootPath = ParamItem{
Key: "etcd.rootPath",
@ -217,7 +212,7 @@ func (p *EtcdConfig) Init(base *BaseTable) {
}
type LocalStorageConfig struct {
Path ParamItem
Path ParamItem `refreshable:"false"`
}
func (p *LocalStorageConfig) Init(base *BaseTable) {
@ -230,7 +225,7 @@ func (p *LocalStorageConfig) Init(base *BaseTable) {
}
type MetaStoreConfig struct {
MetaStoreType ParamItem
MetaStoreType ParamItem `refreshable:"false"`
}
func (p *MetaStoreConfig) Init(base *BaseTable) {
@ -245,14 +240,14 @@ func (p *MetaStoreConfig) Init(base *BaseTable) {
// /////////////////////////////////////////////////////////////////////////////
// --- meta db ---
type MetaDBConfig struct {
Username ParamItem
Password ParamItem
Address ParamItem
Port ParamItem
DBName ParamItem
MaxOpenConns ParamItem
MaxIdleConns ParamItem
LogLevel ParamItem
Username ParamItem `refreshable:"false"`
Password ParamItem `refreshable:"false"`
Address ParamItem `refreshable:"false"`
Port ParamItem `refreshable:"false"`
DBName ParamItem `refreshable:"false"`
MaxOpenConns ParamItem `refreshable:"false"`
MaxIdleConns ParamItem `refreshable:"false"`
LogLevel ParamItem `refreshable:"false"`
}
func (p *MetaDBConfig) Init(base *BaseTable) {
@ -316,19 +311,19 @@ func (p *MetaDBConfig) Init(base *BaseTable) {
// /////////////////////////////////////////////////////////////////////////////
// --- pulsar ---
type PulsarConfig struct {
Address ParamItem
Port ParamItem
WebAddress ParamItem
WebPort ParamItem
MaxMessageSize ParamItem
Address ParamItem `refreshable:"false"`
Port ParamItem `refreshable:"false"`
WebAddress ParamItem `refreshable:"false"`
WebPort ParamItem `refreshable:"false"`
MaxMessageSize ParamItem `refreshable:"true"`
// support auth
AuthPlugin ParamItem
AuthParams ParamItem
AuthPlugin ParamItem `refreshable:"false"`
AuthParams ParamItem `refreshable:"false"`
// support tenant
Tenant ParamItem
Namespace ParamItem
Tenant ParamItem `refreshable:"false"`
Namespace ParamItem `refreshable:"false"`
}
func (p *PulsarConfig) Init(base *BaseTable) {
@ -432,13 +427,13 @@ func (p *PulsarConfig) Init(base *BaseTable) {
// --- kafka ---
type KafkaConfig struct {
Address ParamItem
SaslUsername ParamItem
SaslPassword ParamItem
SaslMechanisms ParamItem
SecurityProtocol ParamItem
ConsumerExtraConfig ParamGroup
ProducerExtraConfig ParamGroup
Address ParamItem `refreshable:"false"`
SaslUsername ParamItem `refreshable:"false"`
SaslPassword ParamItem `refreshable:"false"`
SaslMechanisms ParamItem `refreshable:"false"`
SecurityProtocol ParamItem `refreshable:"false"`
ConsumerExtraConfig ParamGroup `refreshable:"false"`
ProducerExtraConfig ParamGroup `refreshable:"false"`
}
func (k *KafkaConfig) Init(base *BaseTable) {
@ -494,7 +489,7 @@ func (k *KafkaConfig) Init(base *BaseTable) {
// /////////////////////////////////////////////////////////////////////////////
// --- rocksmq ---
type RocksmqConfig struct {
Path ParamItem
Path ParamItem `refreshable:"false"`
}
func (r *RocksmqConfig) Init(base *BaseTable) {
@ -509,16 +504,16 @@ func (r *RocksmqConfig) Init(base *BaseTable) {
// /////////////////////////////////////////////////////////////////////////////
// --- minio ---
type MinioConfig struct {
Address ParamItem
Port ParamItem
AccessKeyID ParamItem
SecretAccessKey ParamItem
UseSSL ParamItem
BucketName ParamItem
RootPath ParamItem
UseIAM ParamItem
CloudProvider ParamItem
IAMEndpoint ParamItem
Address ParamItem `refreshable:"false"`
Port ParamItem `refreshable:"false"`
AccessKeyID ParamItem `refreshable:"false"`
SecretAccessKey ParamItem `refreshable:"false"`
UseSSL ParamItem `refreshable:"false"`
BucketName ParamItem `refreshable:"false"`
RootPath ParamItem `refreshable:"false"`
UseIAM ParamItem `refreshable:"false"`
CloudProvider ParamItem `refreshable:"false"`
IAMEndpoint ParamItem `refreshable:"false"`
}
func (p *MinioConfig) Init(base *BaseTable) {