milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add new config file(not applied) (#15891) 

Signed-off-by: sunby <bingyi.sun@zilliz.com>

Co-authored-by: sunby <bingyi.sun@zilliz.com>
pull/16083/head
Bingyi Sun 2022-03-18 15:53:23 +08:00 committed by GitHub
parent 787d4f8bfd
commit ca45e0bd00
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 715 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

379
configs/config.go Normal file
View File

@ -0,0 +1,379 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// This file is not used for now.
package configs
import (
"fmt"
"strings"
"sync/atomic"
"github.com/BurntSushi/toml"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/util/metricsinfo"
"go.uber.org/zap"
)
const MinToleranceTime = 3600
type Config struct {
Etcd Etcd `toml:"etcd" json:"etcd"`
Minio Minio `toml:"minio" json:"minio"`
Pulsar Pulsar `toml:"pulsar" json:"pulsar"`
RocksMq RocksMq `toml:"rocksmq" json:"rocksmq"`
Grpc Grpc `toml:"grpc" json:"grpc"`
FlowGraph FlowGraph `toml:"flowgraph" json:"flowgrph"`
RootCoord RootCoord `toml:"rootcoord" json:"rootcoord"`
Proxy Proxy `toml:"proxy" json:"proxy"`
QueryCoord QueryCoord `toml:"querycoord" json:"querycoord"`
QueryNode QueryNode `toml:"querynode" json:"querynode"`
IndexCoord IndexCoord `toml:"indexcoord" json:"indexcoord"`
IndexNode IndexNode `toml:"indexnode" json:"indexnode"`
DataCoord DataCoord `toml:"datacoord" json:"datacoord"`
DataNode DataNode `toml:"datanode" json:"datanode"`
TimeTickInterval uint64 `toml:"timetick-interval" json:"timetick-interval"`
TimeTickBufferSize uint32 `toml:"timetick-buffer-size" json:"timetick-buffer-size"`
NameLengthLimit uint32 `toml:"name-length-limit" json:"name-length-limit"`
FieldCountLimit uint32 `toml:"field-count-limit" json:"field-count-limit"`
DimensionLimit uint32 `toml:"dimension-limit" json:"dimension-limit"`
ShardCountLimit uint32 `toml:"shard-count-limit" json:"shard-count-limit"`
DMLChannelCount uint32 `toml:"dml-channel-count" json:"dml-channel-count"`
PartitionCountLimit uint32 `toml:"partition-count-limit" json:"partition-count-limit"`
EnableIndexMinSegmentSize uint32 `toml:"enable-index-min-segment-size" json:"enable-index-min-segment-size"`
SIMDType string `toml:"simd-type" json:"simd-type"`
Compaction Compaction `toml:"compaction" json:"compaction"`
Log Log `toml:"log" json:"log"`
LocalStorage LocalStorage `toml:"localstorage" json:"localstorage"`
SkipQueryChannelRecover bool `toml:"skip-query-channel-recover" json:"skip-query-channel-recover"`
Metrics Metrics `toml:"" json:"metrics"`
GarbageCollector GarbageCollector `toml:"gc" json:"gc"`
}
type Etcd struct {
Endpoints []string `toml:"endpoints" json:"endpoints"`
UseEmbed bool `toml:"use-embed" json:"use-embed"`
ConfigPath string `toml:"config-path" json:"config-path"`
DataPath string `toml:"data-path" json:"data-path"`
}
type Minio struct {
Address string `toml:"address" json:"address"`
Port int `toml:"port" json:"port"`
AccessKeyID string `toml:"access-key-id" json:"access-key-id"`
SecretAccessKey string `toml:"secret-access-key" json:"secret-access-key"`
UseSSL bool `toml:"use-ssl" json:"use-ssl"`
BucketName string `toml:"bucket-name" json:"bucket-name"`
RootPath string `toml:"root-path" json:"root-path"`
}
type Pulsar struct {
Address string `toml:"address" json:"address"`
Port int `toml:"port" json:"port"`
MaxMessageSize uint64 `toml:"max-message-size" json:"max-message-size"`
}
type RocksMq struct {
Path string `toml:"path" json:"path"`
PageSize uint32 `toml:"page-size" json:"page-size"`
RetentionDuration uint32 `toml:"retention-duration" json:"retention-duration"`
RetentionSize uint32 `toml:"retention-size" json:"retention-size"`
}
type Grpc struct {
ServerMaxReceiveSize uint64 `toml:"server-max-receive-size" json:"server-max-receive-size"`
ServerMaxSendSize uint64 `toml:"server-max-send-size" json:"server-max-send-size"`
ClientMaxReceiveSize uint64 `toml:"client-max-receive-size" json:"client-max-receive-size"`
ClientMaxSendSize uint64 `toml:"client-max-send-size" json:"client-max-send-size"`
}
type RootCoord struct {
Address string `toml:"address" json:"address"`
Port int `toml:"port" json:"port"`
}
type Proxy struct {
Port int `toml:"port" json:"port"`
MaxTaskCount uint32 `toml:"max-task-count" json:"max-task-count"`
}
type QueryCoord struct {
Address string `toml:"address" json:"address"`
Port int `toml:"port" json:"port"`
AutoHandOff bool `toml:"auto-handoff" json:"auto-handoff"`
AutoBalance bool `toml:"auto-balance" json:"auto-balance"`
BalanceInterval uint32 `toml:"balance-interval" json:"balance-interval"`
MemoryUsageLimitRatio uint32 `toml:"memory-usage-limit-ratio" json:"memory-usage-limit-ratio"`
AutoBalanceMemoryUsageGapRatio uint32 `toml:"auto-balance-memory-usage-gap-ratio" json:"auto-balance-memory-usage-gap-ratio"`
}
type FlowGraph struct {
QueueLengthLimit uint32 `toml:"queue-length-limit" json:"queue-length-limit"`
ParallelismLimit uint32 `toml:"parallelism-limit" json:"parallelism-limit"`
}
type QueryNode struct {
Port int `toml:"port" json:"port"`
GracefulTime uint32 `toml:"graceful-time" json:"graceful-time"`
StatsPublishInterval uint32 `toml:"stats-publish-interval" json:"stats-publish-interval"`
SegcoreChunkRows uint32 `toml:"segcore-chunk-rows" json:"segcore-chunk-rows"`
}
type IndexCoord struct {
Address string `toml:"address" json:"address"`
Port int `toml:"port" json:"port"`
}
type IndexNode struct {
Port int `toml:"port" json:"port"`
}
type DataCoord struct {
Address string `toml:"address" json:"address"`
Port int `toml:"port" json:"port"`
}
type GarbageCollector struct {
Interval uint32 `toml:"interval" json:"interval"`
MissedFileTolerance uint32 `toml:"missed-files-tolerance" json:"missed-files-tolerance"`
DroppedFileTolerance uint32 `toml:"dropped-files-tolerance" json:"dropped-files-tolerance"`
}
type Compaction struct {
EnableCompaction bool `toml:"enable-compaction" json:"enable-compaction"`
}
type DataNode struct {
Port int `toml:"port" json:"port"`
InsertBufferSizeLimit uint32 `toml:"insert-buffer-size-limit" json:"insert-buffer-size-limit"`
}
type LocalStorage struct {
Enable bool `toml:"enable" json:"enable"`
Path string `toml:"path" json:"path"`
}
type Log struct {
Level string `toml:"level" json:"level"`
Format string `toml:"format" json:"format"`
File LogFileCfg `toml:"file" json:"file"`
}
type LogFileCfg struct {
RootPath string `toml:"root-path" json:"root-path"`
MaxSize uint32 `toml:"max-size" json:"max-size"`
MaxAge uint32 `toml:"max-age" json:"max-age"`
MaxBackups uint32 `toml:"max-backups" json:"max-backups"`
}
type Metrics struct {
GitCommit string `toml:"" json:"git-commit-key"`
DeployMode string `toml:"" json:"deploy-mode"`
GitBuildTags string `toml:"" json:"git-build-tags"`
BuildTime string `toml:"" json:"build-time"`
GoVersion string `toml:"" json:"go-version"`
}
var defaultCfg = Config{
Etcd: Etcd{
Endpoints: []string{"localhost:2379"},
UseEmbed: false,
},
Minio: Minio{
Address: "localhost",
Port: 9000,
AccessKeyID: "minioadmin",
SecretAccessKey: "minioadmin",
UseSSL: false,
BucketName: "a-bucket",
RootPath: "files",
},
Pulsar: Pulsar{
Address: "localhost",
Port: 6650,
MaxMessageSize: 5242880,
},
RocksMq: RocksMq{
Path: "/var/lib/milvus/rdb_data",
PageSize: 2147483648,
RetentionDuration: 10080,
RetentionSize: 8192,
},
Grpc: Grpc{
ServerMaxReceiveSize: 2147483647,
ServerMaxSendSize: 2147483647,
ClientMaxReceiveSize: 104857600,
ClientMaxSendSize: 104857600,
},
FlowGraph: FlowGraph{
QueueLengthLimit: 1024,
ParallelismLimit: 1024,
},
RootCoord: RootCoord{
Address: "localhost",
Port: 53100,
},
Proxy: Proxy{
Port: 53100,
MaxTaskCount: 1024,
},
QueryCoord: QueryCoord{
Address: "localhost",
Port: 19531,
AutoHandOff: true,
AutoBalance: true,
BalanceInterval: 60,
MemoryUsageLimitRatio: 90,
AutoBalanceMemoryUsageGapRatio: 30,
},
QueryNode: QueryNode{
Port: 21123,
GracefulTime: 0,
StatsPublishInterval: 1000,
SegcoreChunkRows: 32768,
},
IndexCoord: IndexCoord{
Address: "localhost",
Port: 31000,
},
IndexNode: IndexNode{
Port: 21121,
},
DataCoord: DataCoord{
Address: "localhost",
Port: 13333,
},
DataNode: DataNode{
Port: 21124,
InsertBufferSizeLimit: 16777216,
},
TimeTickInterval: 200,
TimeTickBufferSize: 512,
NameLengthLimit: 255,
FieldCountLimit: 256,
DimensionLimit: 32768,
ShardCountLimit: 256,
DMLChannelCount: 256,
PartitionCountLimit: 4096,
EnableIndexMinSegmentSize: 1024,
SIMDType: "auto",
Compaction: Compaction{
EnableCompaction: true,
},
Log: Log{
Level: "debug",
Format: "text",
File: LogFileCfg{
RootPath: "",
MaxSize: 300,
MaxAge: 10,
MaxBackups: 20,
},
},
LocalStorage: LocalStorage{
Enable: true,
Path: "/var/lib/milvus/data/",
},
SkipQueryChannelRecover: false,
Metrics: Metrics{
GitCommit: "",
DeployMode: "CLUSTER",
GitBuildTags: "",
GoVersion: "",
BuildTime: "",
},
GarbageCollector: GarbageCollector{
Interval: 3600,
MissedFileTolerance: 86400,
DroppedFileTolerance: 86400,
},
}
var globalConfig atomic.Value
func init() {
cfg := defaultCfg
SetGlobalConfig(&cfg)
}
func SetGlobalConfig(cfg *Config) {
globalConfig.Store(cfg)
}
func GetGlobalConfig() *Config {
return globalConfig.Load().(*Config)
}
func InitializeConfig(path string, enforceEnvCfg func(c *Config), enforceCmdCfg func(c *Config)) {
cfg := GetGlobalConfig()
if path != "" {
err := cfg.Load(path)
if err != nil {
if _, ok := err.(*ErrUndecodedConfig); ok {
log.Warn(err.Error())
} else {
log.Fatal("failed to load config", zap.String("filePath", path), zap.Error(err))
}
}
}
if enforceEnvCfg != nil {
enforceEnvCfg(cfg)
}
if enforceCmdCfg != nil {
enforceCmdCfg(cfg)
}
if err := cfg.Validate(); err != nil {
log.Fatal("config is not valid, please check the config file, environment variables and command options", zap.Error(err))
}
}
func (c *Config) Load(path string) error {
meta, err := toml.DecodeFile(path, c)
undecodedKeys := meta.Undecoded()
if len(undecodedKeys) > 0 && err == nil {
undecoded := make([]string, 0, len(undecodedKeys))
for _, k := range undecodedKeys {
undecoded = append(undecoded, k.String())
}
return &ErrUndecodedConfig{ConfigFile: path, Undecoded: undecoded}
}
return err
}
func (c *Config) Validate() error {
if c.Etcd.UseEmbed && c.Metrics.DeployMode != metricsinfo.StandaloneDeployMode {
return fmt.Errorf("can not use embed etcd in mode %s", c.Metrics.DeployMode)
}
if c.GarbageCollector.MissedFileTolerance < MinToleranceTime {
return fmt.Errorf("gc: missed file tolerance time can not be less than %d", MinToleranceTime)
}
if c.GarbageCollector.DroppedFileTolerance < MinToleranceTime {
return fmt.Errorf("gc: dropped file tolerance time can not be less than %d", MinToleranceTime)
}
return nil
}
type ErrUndecodedConfig struct {
ConfigFile string
Undecoded []string
}
func (e *ErrUndecodedConfig) Error() string {
return fmt.Sprintf("config file %s contains invalid configuration options: %s", e.ConfigFile, strings.Join(e.Undecoded, ","))
}

200
configs/config_test.go Normal file
View File

@ -0,0 +1,200 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// This file is not used for now.
package configs
import (
"os"
"path/filepath"
"testing"
"github.com/milvus-io/milvus/internal/util/metricsinfo"
"github.com/stretchr/testify/assert"
)
func TestLoadConfig(t *testing.T) {
tmpDir := t.TempDir()
cases := []struct {
name string
file string
content string
expectErr error
expectCfg Config
}{
{
"test load config",
"milvus.test.config.1.toml",
`timetick-interval = 200
timetick-buffer-size = 512
name-length-limit = 255
field-count-limit = 256
dimension-limit = 32768
shard-count-limit = 256
dml-channel-count = 256
partition-count-limit = 4096
enable-index-min-segment-size = 1024
simd-type = "auto"
skip-query-channel-recover = false
[etcd]
endpoints = ["localhost:2379"]
use-embed = false`,
nil,
Config{
TimeTickInterval: 200,
TimeTickBufferSize: 512,
NameLengthLimit: 255,
FieldCountLimit: 256,
DimensionLimit: 32768,
ShardCountLimit: 256,
DMLChannelCount: 256,
PartitionCountLimit: 4096,
EnableIndexMinSegmentSize: 1024,
SIMDType: "auto",
SkipQueryChannelRecover: false,
Etcd: Etcd{
Endpoints: []string{"localhost:2379"},
UseEmbed: false,
},
},
},
{
"test load undefied key",
"milvus.test.config.2.toml",
"undefied_key=200",
&ErrUndecodedConfig{filepath.Join(tmpDir, "milvus.test.config.2.toml"), []string{"undefied_key"}},
Config{},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
filePath := filepath.Join(tmpDir, c.file)
f, err := os.Create(filePath)
assert.Nil(t, err)
defer f.Close()
_, err = f.WriteString(c.content)
assert.Nil(t, err)
cfg := Config{}
err = cfg.Load(filePath)
assert.EqualValues(t, c.expectErr, err)
assert.EqualValues(t, c.expectCfg, cfg)
})
}
}
func TestValidate(t *testing.T) {
cases := []struct {
name string
cfg *Config
expectErr bool
}{
{
"test validate success",
&Config{
Etcd: Etcd{
UseEmbed: true,
},
Metrics: Metrics{
DeployMode: metricsinfo.StandaloneDeployMode,
},
GarbageCollector: GarbageCollector{
MissedFileTolerance: MinToleranceTime,
DroppedFileTolerance: MinToleranceTime,
},
},
false,
},
{
"test use embed etcd with cluster",
&Config{
Etcd: Etcd{
UseEmbed: true,
},
Metrics: Metrics{
DeployMode: metricsinfo.ClusterDeployMode,
},
GarbageCollector: GarbageCollector{
MissedFileTolerance: MinToleranceTime,
DroppedFileTolerance: MinToleranceTime,
},
},
true,
},
{
"test use little tolerance time",
&Config{
Etcd: Etcd{
UseEmbed: true,
},
Metrics: Metrics{
DeployMode: metricsinfo.StandaloneDeployMode,
},
GarbageCollector: GarbageCollector{
MissedFileTolerance: MinToleranceTime - 1,
DroppedFileTolerance: MinToleranceTime - 1,
},
},
true,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
assert.Equal(t, c.expectErr, c.cfg.Validate() != nil)
})
}
}
func TestEnforce(t *testing.T) {
cases := []struct {
name string
enforceEnv func(c *Config)
enforceCmd func(c *Config)
checkFunc func(t *testing.T, c *Config)
}{
{"test no enforce func", nil, nil, func(t *testing.T, c *Config) {}},
{
"test enforce cmd func",
func(c *Config) {
c.TimeTickInterval = 1024
},
nil,
func(t *testing.T, c *Config) {
assert.EqualValues(t, 1024, c.TimeTickInterval)
},
},
{
"test enforceCmd override enforceEnv",
func(c *Config) {
c.TimeTickInterval = 512
},
func(c *Config) {
c.TimeTickInterval = 1024
},
func(t *testing.T, c *Config) {
assert.EqualValues(t, 1024, c.TimeTickInterval)
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
InitializeConfig("", c.enforceEnv, c.enforceCmd)
cfg := GetGlobalConfig()
c.checkFunc(t, cfg)
})
}
}

133
configs/milvus.example.toml Normal file
View File

@ -0,0 +1,133 @@
# Licensed to the LF AI & Data foundation under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This file is not used for now.
timetick-interval = 200
timetick-buffer-size = 512
name-length-limit = 255
field-count-limit = 256
dimension-limit = 32768
shard-count-limit = 256
dml-channel-count = 256
partition-count-limit = 4096
enable-index-min-segment-size = 1024
simd-type = "auto"
skip-query-channel-recover = false
[etcd]
endpoints = ["localhost:2379"]
use-embed = false
[minio]
address = "localhost"
port = 9000
access-key-id = "minioadmin"
secret-access-key = "minioadmin"
use-ssl = false
bucket-name = "a-bucket"
root-path = "files"
[pulsar]
address = "localhost"
port = 6650
max-message-size = 5242880
[rocksmq]
path = "/var/lib/milvus/rdb_data"
page-size = 2147483648
retention-duration = 10080
retention-size = 8192
[grpc]
server-max-receive-size = 2147483647
server-max-send-size = 2147483647
client-max-receive-size = 104857600
client-max-send-size = 104857600
[rootcoord]
address = "localhost"
port = 53100
[proxy]
port = 19530
max-task-count = 1024
[querycoord]
address = "localhost"
port = 19531
auto-handoff = true
auto-balance = true
balance-interval = 60
memory-usage-limit-ratio = 90
auto-balance-memory-usage-gap-ratio = 30
[flowgraph]
queue-length-limit = 1024
parallelism-limit = 1024
[querynode]
port = 21123
graceful-time = 0
stats-publish-interval = 1000
segcore-chunk-rows = 32768
[indexcoord]
address = "localhost"
port = 31000
[indexnode]
port = 21121
[datacoord]
address = "localhost"
port = 13333
[gc]
interval = 3600
missed-files-tolerance = 86400
dropped-files-tolerance = 86400
[compaction]
enable-compaction = true
[datanode]
port = 21124
insert-buffer-size-limit = 16777216
[localstorage]
enable = true
path = "/var/lib/milvus/data/"
[log]
level = "debug"
format = "text"
[log.file]
root-path = ""
max-size = 300
max-age = 10
max-backups = 20

1
go.mod
View File

@ -3,6 +3,7 @@ module github.com/milvus-io/milvus
go 1.16
require (
github.com/BurntSushi/toml v1.0.0 // indirect
github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect
github.com/StackExchange/wmi v1.2.1 // indirect
github.com/antonmedv/expr v1.8.9

2
go.sum
View File

@ -47,6 +47,8 @@ github.com/AthenZ/athenz v1.10.39 h1:mtwHTF/v62ewY2Z5KWhuZgVXftBej1/Tn80zx4DcawY
github.com/AthenZ/athenz v1.10.39/go.mod h1:3Tg8HLsiQZp81BJY58JBeU2BR6B/H4/0MQGfCwhHNEA=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU=
github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/DataDog/zstd v1.4.6-0.20210211175136-c6db21d202f4 h1:++HGU87uq9UsSTlFeiOV9uZR3NpYkndUXeYyLv2DTc8=