milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add interface of writenode 

Signed-off-by: become-nice <995581097@qq.com>
pull/4973/head^2
become-nice 2020-08-24 11:58:51 +08:00 committed by yefu.chen
parent fc5f5c7b5a
commit b8f012d8f0
202 changed files with 91308 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
.gitignore vendored Normal file
View File

@ -0,0 +1,17 @@
#proxy
proxy/cmake_build
proxy/thirdparty/grpc-src
proxy/thirdparty/grpc-build
.idea/
.idea/*
# Compiled source
*.a
*.so
*.so.*
*.o
*.lo
*.tar.gz
*.log
.coverage
*.pyc

0
cmd/master.go Normal file
View File

0
cmd/reader.go Normal file
View File

340
cmd/storage/benchmark.go Normal file
View File

@ -0,0 +1,340 @@
package main
import (
"context"
"crypto/md5"
"flag"
"fmt"
"github.com/pivotal-golang/bytefmt"
"log"
"math/rand"
"os"
minio "storage/internal/minio"
tikv "storage/internal/tikv"
"storage/pkg/types"
"sync"
"sync/atomic"
"time"
)
// Global variables
var durationSecs, threads, loops, numVersion, batchOpSize int
var valueSize uint64
var valueData []byte
var batchValueData [][]byte
var counter, totalKeyCount, keyNum int32
var endTime, setFinish, getFinish, deleteFinish time.Time
var totalKeys [][]byte
var logFileName = "benchmark.log"
var logFile *os.File
var store types.Store
var wg sync.WaitGroup
func runSet() {
for time.Now().Before(endTime) {
num := atomic.AddInt32(&keyNum, 1)
key := []byte(fmt.Sprint("key", num))
for ver := 1; ver <= numVersion; ver++ {
atomic.AddInt32(&counter, 1)
err := store.Set(context.Background(), key, valueData, uint64(ver))
if err != nil {
log.Fatalf("Error setting key %s, %s", key, err.Error())
//atomic.AddInt32(&setCount, -1)
}
}
}
// Remember last done time
setFinish = time.Now()
wg.Done()
}
func runBatchSet() {
for time.Now().Before(endTime) {
num := atomic.AddInt32(&keyNum, int32(batchOpSize))
keys := make([][]byte, batchOpSize)
for n := batchOpSize; n > 0; n-- {
keys[n-1] = []byte(fmt.Sprint("key", num-int32(n)))
}
for ver := 1; ver <= numVersion; ver++ {
atomic.AddInt32(&counter, 1)
err := store.BatchSet(context.Background(), keys, batchValueData, uint64(numVersion))
if err != nil {
log.Fatalf("Error setting batch keys %s %s", keys, err.Error())
//atomic.AddInt32(&batchSetCount, -1)
}
}
}
setFinish = time.Now()
wg.Done()
}
func runGet() {
for time.Now().Before(endTime) {
num := atomic.AddInt32(&counter, 1)
//num := atomic.AddInt32(&keyNum, 1)
//key := []byte(fmt.Sprint("key", num))
num = num % totalKeyCount
key := totalKeys[num]
_, err := store.Get(context.Background(), key, uint64(numVersion))
if err != nil {
log.Fatalf("Error getting key %s, %s", key, err.Error())
//atomic.AddInt32(&getCount, -1)
}
}
// Remember last done time
getFinish = time.Now()
wg.Done()
}
func runBatchGet() {
for time.Now().Before(endTime) {
num := atomic.AddInt32(&keyNum, int32(batchOpSize))
//keys := make([][]byte, batchOpSize)
//for n := batchOpSize; n > 0; n-- {
// keys[n-1] = []byte(fmt.Sprint("key", num-int32(n)))
//}
end := num % totalKeyCount
if end < int32(batchOpSize) {
end = int32(batchOpSize)
}
start := end - int32(batchOpSize)
keys := totalKeys[start:end]
atomic.AddInt32(&counter, 1)
_, err := store.BatchGet(context.Background(), keys, uint64(numVersion))
if err != nil {
log.Fatalf("Error getting key %s, %s", keys, err.Error())
//atomic.AddInt32(&batchGetCount, -1)
}
}
// Remember last done time
getFinish = time.Now()
wg.Done()
}
func runDelete() {
for time.Now().Before(endTime) {
num := atomic.AddInt32(&counter, 1)
//num := atomic.AddInt32(&keyNum, 1)
//key := []byte(fmt.Sprint("key", num))
num = num % totalKeyCount
key := totalKeys[num]
err := store.Delete(context.Background(), key, uint64(numVersion))
if err != nil {
log.Fatalf("Error getting key %s, %s", key, err.Error())
//atomic.AddInt32(&deleteCount, -1)
}
}
// Remember last done time
deleteFinish = time.Now()
wg.Done()
}
func runBatchDelete() {
for time.Now().Before(endTime) {
num := atomic.AddInt32(&keyNum, int32(batchOpSize))
//keys := make([][]byte, batchOpSize)
//for n := batchOpSize; n > 0; n-- {
// keys[n-1] = []byte(fmt.Sprint("key", num-int32(n)))
//}
end := num % totalKeyCount
if end < int32(batchOpSize) {
end = int32(batchOpSize)
}
start := end - int32(batchOpSize)
keys := totalKeys[start:end]
atomic.AddInt32(&counter, 1)
err := store.BatchDelete(context.Background(), keys, uint64(numVersion))
if err != nil {
log.Fatalf("Error getting key %s, %s", keys, err.Error())
//atomic.AddInt32(&batchDeleteCount, -1)
}
}
// Remember last done time
getFinish = time.Now()
wg.Done()
}
func main() {
// Parse command line
myflag := flag.NewFlagSet("myflag", flag.ExitOnError)
myflag.IntVar(&durationSecs, "d", 5, "Duration of each test in seconds")
myflag.IntVar(&threads, "t", 1, "Number of threads to run")
myflag.IntVar(&loops, "l", 1, "Number of times to repeat test")
var sizeArg string
var storeType string
myflag.StringVar(&sizeArg, "z", "1K", "Size of objects in bytes with postfix K, M, and G")
myflag.StringVar(&storeType, "s", "tikv", "Storage type, tikv or minio")
myflag.IntVar(&numVersion, "v", 1, "Max versions for each key")
myflag.IntVar(&batchOpSize, "b", 100, "Batch operation kv pair number")
if err := myflag.Parse(os.Args[1:]); err != nil {
os.Exit(1)
}
// Check the arguments
var err error
if valueSize, err = bytefmt.ToBytes(sizeArg); err != nil {
log.Fatalf("Invalid -z argument for object size: %v", err)
}
switch storeType {
case "tikv":
//var (
// pdAddr = []string{"127.0.0.1:2379"}
// conf = config.Default()
//)
store, err = tikv.NewTikvStore(context.Background())
if err != nil {
log.Fatalf("Error when creating storage " + err.Error())
}
case "minio":
store, err = minio.NewMinioStore(context.Background())
if err != nil {
log.Fatalf("Error when creating storage " + err.Error())
}
default:
log.Fatalf("Not supported storage type")
}
logFile, err = os.OpenFile(logFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0777)
if err != nil {
log.Fatalf("Prepare log file error, " + err.Error())
}
// Echo the parameters
log.Printf("Benchmark log will write to file %s\n", logFile.Name())
fmt.Fprint(logFile, fmt.Sprintf("Parameters: duration=%d, threads=%d, loops=%d, valueSize=%s, batchSize=%d, versions=%d\n", durationSecs, threads, loops, sizeArg, batchOpSize, numVersion))
// Init test data
valueData = make([]byte, valueSize)
rand.Read(valueData)
hasher := md5.New()
hasher.Write(valueData)
batchValueData = make([][]byte, batchOpSize)
for i := range batchValueData {
batchValueData[i] = make([]byte, valueSize)
rand.Read(batchValueData[i])
hasher := md5.New()
hasher.Write(batchValueData[i])
}
// Loop running the tests
for loop := 1; loop <= loops; loop++ {
// reset counters
counter = 0
keyNum = 0
totalKeyCount = 0
totalKeys = nil
// Run the set case
startTime := time.Now()
endTime = startTime.Add(time.Second * time.Duration(durationSecs))
for n := 1; n <= threads; n++ {
wg.Add(1)
go runSet()
}
wg.Wait()
setTime := setFinish.Sub(startTime).Seconds()
bps := float64(uint64(counter)*valueSize) / setTime
fmt.Fprint(logFile, fmt.Sprintf("Loop %d: PUT time %.1f secs, kv pairs = %d, speed = %sB/sec, %.1f operations/sec, %.1f kv/sec.\n",
loop, setTime, counter, bytefmt.ByteSize(uint64(bps)), float64(counter)/setTime, float64(counter)/setTime))
// Run the batchSet case
// key seq start from setCount
counter = 0
startTime = time.Now()
endTime = startTime.Add(time.Second * time.Duration(durationSecs))
for n := 1; n <= threads; n++ {
wg.Add(1)
go runBatchSet()
}
wg.Wait()
setTime = setFinish.Sub(startTime).Seconds()
bps = float64(uint64(counter)*valueSize*uint64(batchOpSize)) / setTime
fmt.Fprint(logFile, fmt.Sprintf("Loop %d: BATCH PUT time %.1f secs, batchs = %d, kv pairs = %d, speed = %sB/sec, %.1f operations/sec, %.1f kv/sec.\n",
loop, setTime, counter, counter*int32(batchOpSize), bytefmt.ByteSize(uint64(bps)), float64(counter)/setTime, float64(counter * int32(batchOpSize))/setTime))
// Record all test keys
totalKeyCount = keyNum
totalKeys = make([][]byte, totalKeyCount)
for i := int32(0); i < totalKeyCount; i++ {
totalKeys[i] = []byte(fmt.Sprint("key", i))
}
// Run the get case
counter = 0
startTime = time.Now()
endTime = startTime.Add(time.Second * time.Duration(durationSecs))
for n := 1; n <= threads; n++ {
wg.Add(1)
go runGet()
}
wg.Wait()
getTime := getFinish.Sub(startTime).Seconds()
bps = float64(uint64(counter)*valueSize) / getTime
fmt.Fprint(logFile, fmt.Sprintf("Loop %d: GET time %.1f secs, kv pairs = %d, speed = %sB/sec, %.1f operations/sec, %.1f kv/sec.\n",
loop, getTime, counter, bytefmt.ByteSize(uint64(bps)), float64(counter)/getTime, float64(counter)/getTime))
// Run the batchGet case
counter = 0
startTime = time.Now()
endTime = startTime.Add(time.Second * time.Duration(durationSecs))
for n := 1; n <= threads; n++ {
wg.Add(1)
go runBatchGet()
}
wg.Wait()
getTime = getFinish.Sub(startTime).Seconds()
bps = float64(uint64(counter)*valueSize*uint64(batchOpSize)) / getTime
fmt.Fprint(logFile, fmt.Sprintf("Loop %d: BATCH GET time %.1f secs, batchs = %d, kv pairs = %d, speed = %sB/sec, %.1f operations/sec, %.1f kv/sec.\n",
loop, getTime, counter, counter*int32(batchOpSize), bytefmt.ByteSize(uint64(bps)), float64(counter)/getTime, float64(counter * int32(batchOpSize))/getTime))
// Run the delete case
counter = 0
startTime = time.Now()
endTime = startTime.Add(time.Second * time.Duration(durationSecs))
for n := 1; n <= threads; n++ {
wg.Add(1)
go runDelete()
}
wg.Wait()
deleteTime := deleteFinish.Sub(startTime).Seconds()
bps = float64(uint64(counter)*valueSize) / deleteTime
fmt.Fprint(logFile, fmt.Sprintf("Loop %d: Delete time %.1f secs, kv pairs = %d, %.1f operations/sec, %.1f kv/sec.\n",
loop, deleteTime, counter, float64(counter)/deleteTime, float64(counter)/deleteTime))
// Run the batchDelete case
counter = 0
startTime = time.Now()
endTime = startTime.Add(time.Second * time.Duration(durationSecs))
for n := 1; n <= threads; n++ {
wg.Add(1)
go runBatchDelete()
}
wg.Wait()
deleteTime = setFinish.Sub(startTime).Seconds()
bps = float64(uint64(counter)*valueSize*uint64(batchOpSize)) / setTime
fmt.Fprint(logFile, fmt.Sprintf("Loop %d: BATCH DELETE time %.1f secs, batchs = %d, kv pairs = %d, %.1f operations/sec, %.1f kv/sec.\n",
loop, setTime, counter, counter*int32(batchOpSize), float64(counter)/setTime, float64(counter * int32(batchOpSize))/setTime))
// Print line mark
lineMark := "\n"
fmt.Fprint(logFile, lineMark)
// Clear test data
err = store.BatchDelete(context.Background(), totalKeys, uint64(numVersion))
if err != nil {
log.Print("Clean test data error " + err.Error())
}
}
log.Print("Benchmark test done.")
}

69
cmd/storage/tikv_example.go Normal file
View File

@ -0,0 +1,69 @@
package main
import (
"context"
"fmt"
"math"
"storage/pkg"
. "storage/pkg/types"
)
func main() {
// Create a tikv based storage
var store Store
var err error
ctx := context.Background()
store, err = storage.NewStore(ctx, TIKVDriver)
if err != nil {
panic(err.Error())
}
// Set some key-value pair with different timestamp
key := Key("key")
store.Set(ctx, key, Value("value_1"), 1)
store.Set(ctx, key, Value("value_2"), 2)
store.Set(ctx, key, Value("value_3"), 3)
store.Set(ctx, key, Value("value_4"), 4)
search := func(key Key, timestamp uint64) {
v, err := store.Get(ctx, key, timestamp)
if err != nil {
panic(err.Error())
}
fmt.Printf("Get result for key: %s, version:%d, value:%s \n", key, timestamp, v)
}
search(key, 0)
search(key, 3)
search(key, 10)
// Batch set key-value pairs with same timestamp
keys := []Key{Key("key"), Key("key1")}
values := []Value{Value("value_5"), Value("value1_5")}
store.BatchSet(ctx, keys, values, 5)
batchSearch := func(keys []Key, timestamp uint64) {
vs, err := store.BatchGet(ctx, keys, timestamp)
if err != nil {
panic(err.Error())
}
for i, v := range vs {
fmt.Printf("Get result for key: %s, version:%d, value:%s \n", keys[i], timestamp, v)
}
}
// Batch get keys
keys = []Key{Key("key"), Key("key1")}
batchSearch(keys, 5)
//Delete outdated key-value pairs for a key
store.Set(ctx, key, Value("value_6"), 6)
store.Set(ctx, key, Value("value_7"), 7)
err = store.Delete(ctx, key, 5)
search(key, 5)
// use BatchDelete all keys
keys = []Key{Key("key"), Key("key1")}
store.BatchDelete(ctx, keys , math.MaxUint64)
batchSearch(keys, math.MaxUint64)
}

0
cmd/writer.go Normal file
View File

0
conf/conf.toml Normal file
View File

188
conf/server_config.template Normal file
View File

@ -0,0 +1,188 @@
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
version: 0.5
#----------------------+------------------------------------------------------------+------------+-----------------+
# Cluster Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | If running with Mishards, set true, otherwise false. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# role | Milvus deployment role: rw / ro | Role | rw |
#----------------------+------------------------------------------------------------+------------+-----------------+
cluster:
enable: false
role: rw
#----------------------+------------------------------------------------------------+------------+-----------------+
# General Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# timezone | Use UTC-x or UTC+x to specify a time zone. | Timezone | UTC+8 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# meta_uri | URI for metadata storage, using SQLite (for single server | URI | sqlite://:@:/ |
# | Milvus) or MySQL (for distributed cluster Milvus). | | |
# | Format: dialect://username:password@host:port/database | | |
# | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | |
# | 'mysql', replace other texts with real values. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
general:
timezone: UTC+8
meta_uri: sqlite://:@:/
#----------------------+------------------------------------------------------------+------------+-----------------+
# Network Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# bind.address | IP address that Milvus server monitors. | IP | 0.0.0.0 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# bind.port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# http.enable | Enable HTTP server or not. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# http.port | Port that Milvus HTTP server monitors. | Integer | 19121 |
# | Port range (1024, 65535) | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
network:
bind.address: 0.0.0.0
bind.port: 19530
http.enable: true
http.port: 19121
#----------------------+------------------------------------------------------------+------------+-----------------+
# Storage Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# path | Path used to save meta data, vector data and index data. | Path | /var/lib/milvus |
#----------------------+------------------------------------------------------------+------------+-----------------+
# auto_flush_interval | The interval, in seconds, at which Milvus automatically | Integer | 1 (s) |
# | flushes data to disk. | | |
# | 0 means disable the regular flush. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
storage:
path: @MILVUS_DB_PATH@
auto_flush_interval: 1
#----------------------+------------------------------------------------------------+------------+-----------------+
# WAL Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | Whether to enable write-ahead logging (WAL) in Milvus. | Boolean | true |
# | If WAL is enabled, Milvus writes all data changes to log | | |
# | files in advance before implementing data changes. WAL | | |
# | ensures the atomicity and durability for Milvus operations.| | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# recovery_error_ignore| Whether to ignore logs with errors that happens during WAL | Boolean | false |
# | recovery. If true, when Milvus restarts for recovery and | | |
# | there are errors in WAL log files, log files with errors | | |
# | are ignored. If false, Milvus does not restart when there | | |
# | are errors in WAL log files. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# buffer_size | Sum total of the read buffer and the write buffer in Bytes.| String | 256MB |
# | buffer_size must be in range [64MB, 4096MB]. | | |
# | If the value you specified is out of range, Milvus | | |
# | automatically uses the boundary value closest to the | | |
# | specified value. It is recommended you set buffer_size to | | |
# | a value greater than the inserted data size of a single | | |
# | insert operation for better performance. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# path | Location of WAL log files. | String | |
#----------------------+------------------------------------------------------------+------------+-----------------+
wal:
enable: true
recovery_error_ignore: false
buffer_size: 256MB
path: @MILVUS_DB_PATH@/wal
#----------------------+------------------------------------------------------------+------------+-----------------+
# Cache Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cache_size | The size of CPU memory used for caching data for faster | String | 4GB |
# | query. The sum of 'cache_size' and 'insert_buffer_size' | | |
# | must be less than system memory size. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# insert_buffer_size | Buffer size used for data insertion. | String | 1GB |
# | The sum of 'insert_buffer_size' and 'cache_size' | | |
# | must be less than system memory size. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# preload_collection | A comma-separated list of collection names that need to | StringList | |
# | be pre-loaded when Milvus server starts up. | | |
# | '*' means preload all existing tables (single-quote or | | |
# | double-quote required). | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
cache:
cache_size: 4GB
insert_buffer_size: 1GB
preload_collection:
#----------------------+------------------------------------------------------------+------------+-----------------+
# GPU Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | Use GPU devices or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# cache_size | The size of GPU memory per card used for cache. | String | 1GB |
#----------------------+------------------------------------------------------------+------------+-----------------+
# gpu_search_threshold | A Milvus performance tuning parameter. This value will be | Integer | 1000 |
# | compared with 'nq' to decide if the search computation will| | |
# | be executed on GPUs only. | | |
# | If nq >= gpu_search_threshold, the search computation will | | |
# | be executed on GPUs only; | | |
# | if nq < gpu_search_threshold, the search computation will | | |
# | be executed on both CPUs and GPUs. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# search_devices | The list of GPU devices used for search computation. | DeviceList | gpu0 |
# | Must be in format gpux. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# build_index_devices | The list of GPU devices used for index building. | DeviceList | gpu0 |
# | Must be in format gpux. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
gpu:
enable: @GPU_ENABLE@
cache_size: 1GB
gpu_search_threshold: 1000
search_devices:
- gpu0
build_index_devices:
- gpu0
#----------------------+------------------------------------------------------------+------------+-----------------+
# Logs Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# level | Log level in Milvus. Must be one of debug, info, warning, | String | debug |
# | error, fatal | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# trace.enable | Whether to enable trace level logging in Milvus. | Boolean | true |
#----------------------+------------------------------------------------------------+------------+-----------------+
# path | Absolute path to the folder holding the log files. | String | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# max_log_file_size | The maximum size of each log file, size range | String | 1024MB |
# | [512MB, 4096MB]. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# log_rotate_num | The maximum number of log files that Milvus keeps for each | Integer | 0 |
# | logging level, num range [0, 1024], 0 means unlimited. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
logs:
level: debug
trace.enable: true
path: @MILVUS_DB_PATH@/logs
max_log_file_size: 1024MB
log_rotate_num: 0
#----------------------+------------------------------------------------------------+------------+-----------------+
# Metric Config | Description | Type | Default |
#----------------------+------------------------------------------------------------+------------+-----------------+
# enable | Enable monitoring function or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# address | Pushgateway address | IP | 127.0.0.1 +
#----------------------+------------------------------------------------------------+------------+-----------------+
# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 |
#----------------------+------------------------------------------------------------+------------+-----------------+
metric:
enable: false
address: 127.0.0.1
port: 9091

1
conf/storage.toml Normal file
View File

@ -0,0 +1 @@
driver="S3"

0
core/milvus_core.cpp Normal file
View File

12
go.mod Normal file
View File

@ -0,0 +1,12 @@
module github.com/czs007/suvlim
go 1.14
require (
github.com/BurntSushi/toml v0.3.1
github.com/golang/protobuf v1.4.2
github.com/minio/minio-go/v7 v7.0.5
github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48
github.com/stretchr/testify v1.6.1
github.com/tikv/client-go v0.0.0-20200723074018-095b94dc2430
)

258
go.sum Normal file
View File

@ -0,0 +1,258 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48 h1:/EMHruHCFXR9xClkGV/t0rmHrdhX4+trQUcBqjwc9xE=
code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa v0.0.1 h1:r3ncXbtIiad9owWu22r8ryYogBEV9NbJykk8k6K+u0w=
github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.12+incompatible h1:pAWNwdf7QiT1zfaWyqCtNZQWCLByQyA3JrSQyuYAqnQ=
github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190212144455-93d5ec2c7f76 h1:FE783w8WFh+Rvg+7bZ5g8p7gP4SeVS4AoNwkvazlsBg=
github.com/coreos/go-systemd v0.0.0-20190212144455-93d5ec2c7f76/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o=
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
github.com/envoyproxy/data-plane-api v0.0.0-20200823234036-b215ae4c0e16 h1:GLGI0UaYAaQ3udUVbWNUWyLwp+agjr+++DW0N43HEcU=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9 h1:EGUd+AQfZoi1OwZAoqekLbl4kq6tafFtKQSiN8nL21Y=
github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U=
github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.8.1 h1:VNUuLKyFcJ5IektwBKcZU4J5GJKEt+Odb8dl1d61BGQ=
github.com/grpc-ecosystem/grpc-gateway v1.8.1/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
github.com/minio/minio-go v1.0.0 h1:ooSujki+Z1PRGZsYffJw5jnF5eMBvzMVV86TLAlM0UM=
github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o=
github.com/minio/minio-go/v7 v7.0.5 h1:I2NIJ2ojwJqD/YByemC1M59e1b4FW9kS7NlOar7HPV4=
github.com/minio/minio-go/v7 v7.0.5/go.mod h1:TA0CQCjJZHM5SJj9IjqR0NmpmQJ6bCbXifAJ3mUU6Hw=
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk=
github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg=
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg=
github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3 h1:04yuCf5NMvLU8rB2m4Qs3rynH7EYpMno3lHkewIOdMo=
github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3/go.mod h1:DazNTg0PTldtpsQiT9I5tVJwV1onHMKBBgXzmJUlMns=
github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rGrobssy1nVy2VaVpNCuLpCbr+FEaTA8=
github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=
github.com/pingcap/kvproto v0.0.0-20190305055742-ab7debc182d9 h1:EsTt42btov+tFchxOFKnxBNmXOWyPKiddOwvr/WO90g=
github.com/pingcap/kvproto v0.0.0-20190305055742-ab7debc182d9/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY=
github.com/pingcap/pd v2.1.5+incompatible h1:vOLV2tSQdRjjmxaTXtJULoC94dYQOd+6fzn2yChODHc=
github.com/pingcap/pd v2.1.5+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E=
github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48 h1:2JCf+JCLBs7IUZzYdIrSDN+GWYacKOdToIAt5zcga54=
github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48/go.mod h1:43j3yLP9UiXa0z95/W3hN7yTjoxsQoOll5rrGBgBcnE=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/protocolbuffers/protobuf v3.13.0+incompatible h1:omZA3Tuq+U2kJ2uMuqMR9c1VO5qLEgZ19m9878fXNtg=
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME=
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tikv/client-go v0.0.0-20200723074018-095b94dc2430 h1:qrLc3hp4FJfZ+AzQHkf2BmlYYyj+RhSaYC8JeliV1KE=
github.com/tikv/client-go v0.0.0-20200723074018-095b94dc2430/go.mod h1:I3o4nCR8z2GtgGnqN/YCK5wgr/9bGtkJvCQgtKkTmo8=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ugorji/go v1.1.2 h1:JON3E2/GPW2iDNGoSAusl1KDf5TRQ8k8q7Tp097pZGs=
github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43 h1:BasDe+IErOQKrMVXab7UayvSlIpiyGwRvuX3EKYY7UA=
github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA=
github.com/unrolled/render v1.0.0 h1:XYtvhA3UkpB7PqkvhUFYmpKD55OudoIeygcfus4vcd4=
github.com/unrolled/render v1.0.0/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o=
go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f h1:FU37niK8AQ59mHcskRyQL7H0ErSeNh650vdcj8HqdSI=
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

0
master/master.go Normal file
View File

203
proxy/CMakeLists.txt Normal file
View File

@ -0,0 +1,203 @@
#-------------------------------------------------------------------------------
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
cmake_minimum_required( VERSION 3.14 )
message( STATUS "Building using CMake version: ${CMAKE_VERSION}" )
set( CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake" )
include( Utils )
# **************************** Build time, type and code version ****************************
get_current_time( BUILD_TIME )
message( STATUS "Build time = ${BUILD_TIME}" )
get_build_type( TARGET BUILD_TYPE
DEFAULT "Release" )
message( STATUS "Build type = ${BUILD_TYPE}" )
get_milvus_version( TARGET MILVUS_VERSION
DEFAULT "0.10.0" )
message( STATUS "Build version = ${MILVUS_VERSION}" )
get_last_commit_id( LAST_COMMIT_ID )
message( STATUS "LAST_COMMIT_ID = ${LAST_COMMIT_ID}" )
configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/src/version.h.in
${CMAKE_CURRENT_SOURCE_DIR}/src/version.h @ONLY )
# unset(CMAKE_EXPORT_COMPILE_COMMANDS CACHE)
set( CMAKE_EXPORT_COMPILE_COMMANDS ON )
# **************************** Project ****************************
project( milvus VERSION "${MILVUS_VERSION}" )
set( CMAKE_CXX_STANDARD 17 )
set( CMAKE_CXX_STANDARD_REQUIRED on )
set( MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR} )
set( MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR} )
set( MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src )
set( MILVUS_THIRDPARTY_SRC ${PROJECT_SOURCE_DIR}/thirdparty )
# This will set RPATH to all excutable TARGET
# self-installed dynamic libraries will be correctly linked by excutable
set( CMAKE_INSTALL_RPATH "/usr/lib" "${CMAKE_INSTALL_PREFIX}/lib" )
set( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE )
# **************************** Dependencies ****************************
include( CTest )
include( BuildUtils )
include( DefineOptions )
include( ExternalProject )
include( FetchContent )
set( FETCHCONTENT_BASE_DIR ${MILVUS_BINARY_DIR}/3rdparty_download )
set(FETCHCONTENT_QUIET OFF)
include( ThirdPartyPackages )
# **************************** Compiler arguments ****************************
message( STATUS "Building Milvus CPU version" )
append_flags( CMAKE_CXX_FLAGS FLAGS "-O3" )
append_flags( CMAKE_CXX_FLAGS
FLAGS
"-fPIC"
"-DELPP_THREAD_SAFE"
"-fopenmp"
"-Werror"
)
# **************************** Coding style check tools ****************************
find_package( ClangTools )
set( BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support" )
message(STATUS "CMAKE_SOURCE_DIR is at ${CMAKE_SOURCE_DIR}" )
if("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND)
# Generate a Clang compile_commands.json "compilation database" file for use
# with various development tools, such as Vim's YouCompleteMe plugin.
# See http://clang.llvm.org/docs/JSONCompilationDatabase.html
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
endif()
#
# "make lint" target
#
if ( NOT MILVUS_VERBOSE_LINT )
set( MILVUS_LINT_QUIET "--quiet" )
endif ()
if ( NOT LINT_EXCLUSIONS_FILE )
# source files matching a glob from a line in this file
# will be excluded from linting (cpplint, clang-tidy, clang-format)
set( LINT_EXCLUSIONS_FILE ${BUILD_SUPPORT_DIR}/lint_exclusions.txt )
endif ()
find_program( CPPLINT_BIN NAMES cpplint cpplint.py HINTS ${BUILD_SUPPORT_DIR} )
message( STATUS "Found cpplint executable at ${CPPLINT_BIN}" )
#
# "make lint" targets
#
add_custom_target( lint
${PYTHON_EXECUTABLE} ${BUILD_SUPPORT_DIR}/run_cpplint.py
--cpplint_binary ${CPPLINT_BIN}
--exclude_globs ${LINT_EXCLUSIONS_FILE}
--source_dir ${CMAKE_CURRENT_SOURCE_DIR}
${MILVUS_LINT_QUIET}
)
#
# "make clang-format" and "make check-clang-format" targets
#
if ( ${CLANG_FORMAT_FOUND} )
# runs clang format and updates files in place.
add_custom_target( clang-format
${PYTHON_EXECUTABLE} ${BUILD_SUPPORT_DIR}/run_clang_format.py
--clang_format_binary ${CLANG_FORMAT_BIN}
--exclude_globs ${LINT_EXCLUSIONS_FILE}
--source_dir ${CMAKE_CURRENT_SOURCE_DIR}/src
--fix
${MILVUS_LINT_QUIET} )
# runs clang format and exits with a non-zero exit code if any files need to be reformatted
add_custom_target( check-clang-format
${PYTHON_EXECUTABLE} ${BUILD_SUPPORT_DIR}/run_clang_format.py
--clang_format_binary ${CLANG_FORMAT_BIN}
--exclude_globs ${LINT_EXCLUSIONS_FILE}
--source_dir ${CMAKE_CURRENT_SOURCE_DIR}/src
${MILVUS_LINT_QUIET} )
endif ()
#
# "make clang-tidy" and "make check-clang-tidy" targets
#
if ( ${CLANG_TIDY_FOUND} )
# runs clang-tidy and attempts to fix any warning automatically
add_custom_target( clang-tidy
${PYTHON_EXECUTABLE} ${BUILD_SUPPORT_DIR}/run_clang_tidy.py
--clang_tidy_binary ${CLANG_TIDY_BIN}
--exclude_globs ${LINT_EXCLUSIONS_FILE}
--compile_commands ${CMAKE_BINARY_DIR}/compile_commands.json
--source_dir ${CMAKE_CURRENT_SOURCE_DIR}/src
--fix
${MILVUS_LINT_QUIET} )
# runs clang-tidy and exits with a non-zero exit code if any errors are found.
add_custom_target( check-clang-tidy
${PYTHON_EXECUTABLE} ${BUILD_SUPPORT_DIR}/run_clang_tidy.py
--clang_tidy_binary ${CLANG_TIDY_BIN}
--exclude_globs ${LINT_EXCLUSIONS_FILE}
--compile_commands ${CMAKE_BINARY_DIR}/compile_commands.json
--source_dir ${CMAKE_CURRENT_SOURCE_DIR}/src
${MILVUS_LINT_QUIET} )
endif ()
#
# Validate and print out Milvus configuration options
#
config_summary()
# **************************** Source files ****************************
add_subdirectory( thirdparty )
add_subdirectory( src )
# Unittest lib
if ( BUILD_UNIT_TEST STREQUAL "ON" )
if ( BUILD_COVERAGE STREQUAL "ON" )
append_flags( CMAKE_CXX_FLAGS
FLAGS
"-fprofile-arcs"
"-ftest-coverage"
)
endif ()
append_flags( CMAKE_CXX_FLAGS FLAGS "-DELPP_DISABLE_LOGS")
add_subdirectory( ${CMAKE_CURRENT_SOURCE_DIR}/unittest )
endif ()
add_custom_target( Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean )
# **************************** Install ****************************
if ( NOT MILVUS_DB_PATH )
set( MILVUS_DB_PATH "${CMAKE_INSTALL_PREFIX}" )
endif ()
if ( MILVUS_GPU_VERSION )
set( GPU_ENABLE "true" )
else ()
set( GPU_ENABLE "false" )
endif ()

38
proxy/build-support/code_style_clion.xml Normal file
View File

@ -0,0 +1,38 @@
<code_scheme name="milvus" version="173">
<Objective-C>
<option name="INDENT_NAMESPACE_MEMBERS" value="0" />
<option name="INDENT_VISIBILITY_KEYWORDS" value="1" />
<option name="KEEP_STRUCTURES_IN_ONE_LINE" value="true" />
<option name="KEEP_CASE_EXPRESSIONS_IN_ONE_LINE" value="true" />
<option name="FUNCTION_NON_TOP_AFTER_RETURN_TYPE_WRAP" value="0" />
<option name="FUNCTION_TOP_AFTER_RETURN_TYPE_WRAP" value="2" />
<option name="FUNCTION_PARAMETERS_WRAP" value="5" />
<option name="FUNCTION_CALL_ARGUMENTS_WRAP" value="5" />
<option name="TEMPLATE_CALL_ARGUMENTS_WRAP" value="5" />
<option name="TEMPLATE_CALL_ARGUMENTS_ALIGN_MULTILINE" value="true" />
<option name="CLASS_CONSTRUCTOR_INIT_LIST_WRAP" value="5" />
<option name="ALIGN_INIT_LIST_IN_COLUMNS" value="false" />
<option name="SPACE_BEFORE_PROTOCOLS_BRACKETS" value="false" />
<option name="SPACE_BEFORE_POINTER_IN_DECLARATION" value="false" />
<option name="SPACE_AFTER_POINTER_IN_DECLARATION" value="true" />
<option name="SPACE_BEFORE_REFERENCE_IN_DECLARATION" value="false" />
<option name="SPACE_AFTER_REFERENCE_IN_DECLARATION" value="true" />
<option name="KEEP_BLANK_LINES_BEFORE_END" value="1" />
</Objective-C>
<codeStyleSettings language="ObjectiveC">
<option name="KEEP_BLANK_LINES_IN_DECLARATIONS" value="1" />
<option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
<option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="1" />
<option name="BLANK_LINES_AROUND_CLASS" value="0" />
<option name="BLANK_LINES_AROUND_METHOD_IN_INTERFACE" value="0" />
<option name="BLANK_LINES_AFTER_CLASS_HEADER" value="1" />
<option name="SPACE_AFTER_TYPE_CAST" value="false" />
<option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
<option name="KEEP_SIMPLE_BLOCKS_IN_ONE_LINE" value="false" />
<option name="FOR_STATEMENT_WRAP" value="1" />
<option name="ASSIGNMENT_WRAP" value="1" />
<indentOptions>
<option name="CONTINUATION_INDENT_SIZE" value="4" />
</indentOptions>
</codeStyleSettings>
</code_scheme>

6476
proxy/build-support/cpplint.py vendored Executable file
View File

File diff suppressed because it is too large Load Diff

7
proxy/build-support/lint_exclusions.txt Normal file
View File

@ -0,0 +1,7 @@
*cmake-build-debug*
*cmake-build-release*
*cmake_build*
*thirdparty*
*src/grpc*
*milvus/include*
*unittest*

110
proxy/build-support/lintutils.py Executable file
View File

@ -0,0 +1,110 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import multiprocessing as mp
import os
from fnmatch import fnmatch
from subprocess import Popen
def chunk(seq, n):
"""
divide a sequence into equal sized chunks
(the last chunk may be smaller, but won't be empty)
"""
chunks = []
some = []
for element in seq:
if len(some) == n:
chunks.append(some)
some = []
some.append(element)
if len(some) > 0:
chunks.append(some)
return chunks
def dechunk(chunks):
"flatten chunks into a single list"
seq = []
for chunk in chunks:
seq.extend(chunk)
return seq
def run_parallel(cmds, **kwargs):
"""
Run each of cmds (with shared **kwargs) using subprocess.Popen
then wait for all of them to complete.
Runs batches of multiprocessing.cpu_count() * 2 from cmds
returns a list of tuples containing each process'
returncode, stdout, stderr
"""
complete = []
for cmds_batch in chunk(cmds, mp.cpu_count() * 2):
procs_batch = [Popen(cmd, **kwargs) for cmd in cmds_batch]
for proc in procs_batch:
stdout, stderr = proc.communicate()
complete.append((proc.returncode, stdout, stderr))
return complete
_source_extensions = '''
.h
.cc
.cpp
'''.split()
def get_sources(source_dir, exclude_globs=[]):
sources = []
for directory, subdirs, basenames in os.walk(source_dir):
for path in [os.path.join(directory, basename)
for basename in basenames]:
# filter out non-source files
if os.path.splitext(path)[1] not in _source_extensions:
continue
path = os.path.abspath(path)
# filter out files that match the globs in the globs file
if any([fnmatch(path, glob) for glob in exclude_globs]):
continue
sources.append(path)
return sources
def stdout_pathcolonline(completed_process, filenames):
"""
given a completed process which may have reported some files as problematic
by printing the path name followed by ':' then a line number, examine
stdout and return the set of actually reported file names
"""
returncode, stdout, stderr = completed_process
bfilenames = set()
for filename in filenames:
bfilenames.add(filename.encode('utf-8') + b':')
problem_files = set()
for line in stdout.splitlines():
for filename in bfilenames:
if line.startswith(filename):
problem_files.add(filename.decode('utf-8'))
bfilenames.remove(filename)
break
return problem_files, stdout

142
proxy/build-support/run_clang_format.py Executable file
View File

@ -0,0 +1,142 @@
#!/usr/bin/env python2
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from __future__ import print_function
import lintutils
from subprocess import PIPE
import argparse
import difflib
import multiprocessing as mp
import sys
from functools import partial
# examine the output of clang-format and if changes are
# present assemble a (unified)patch of the difference
def _check_one_file(completed_processes, filename):
with open(filename, "rb") as reader:
original = reader.read()
returncode, stdout, stderr = completed_processes[filename]
formatted = stdout
if formatted != original:
# Run the equivalent of diff -u
diff = list(difflib.unified_diff(
original.decode('utf8').splitlines(True),
formatted.decode('utf8').splitlines(True),
fromfile=filename,
tofile="{} (after clang format)".format(
filename)))
else:
diff = None
return filename, diff
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Runs clang-format on all of the source "
"files. If --fix is specified enforce format by "
"modifying in place, otherwise compare the output "
"with the existing file and output any necessary "
"changes as a patch in unified diff format")
parser.add_argument("--clang_format_binary",
required=True,
help="Path to the clang-format binary")
parser.add_argument("--exclude_globs",
help="Filename containing globs for files "
"that should be excluded from the checks")
parser.add_argument("--source_dir",
required=True,
help="Root directory of the source code")
parser.add_argument("--fix", default=False,
action="store_true",
help="If specified, will re-format the source "
"code instead of comparing the re-formatted "
"output, defaults to %(default)s")
parser.add_argument("--quiet", default=False,
action="store_true",
help="If specified, only print errors")
arguments = parser.parse_args()
exclude_globs = []
if arguments.exclude_globs:
for line in open(arguments.exclude_globs):
exclude_globs.append(line.strip())
formatted_filenames = []
for path in lintutils.get_sources(arguments.source_dir, exclude_globs):
formatted_filenames.append(str(path))
if arguments.fix:
if not arguments.quiet:
print("\n".join(map(lambda x: "Formatting {}".format(x),
formatted_filenames)))
# Break clang-format invocations into chunks: each invocation formats
# 16 files. Wait for all processes to complete
results = lintutils.run_parallel([
[arguments.clang_format_binary, "-i"] + some
for some in lintutils.chunk(formatted_filenames, 16)
])
for returncode, stdout, stderr in results:
# if any clang-format reported a parse error, bubble it
if returncode != 0:
sys.exit(returncode)
else:
# run an instance of clang-format for each source file in parallel,
# then wait for all processes to complete
results = lintutils.run_parallel([
[arguments.clang_format_binary, filename]
for filename in formatted_filenames
], stdout=PIPE, stderr=PIPE)
for returncode, stdout, stderr in results:
# if any clang-format reported a parse error, bubble it
if returncode != 0:
sys.exit(returncode)
error = False
checker = partial(_check_one_file, {
filename: result
for filename, result in zip(formatted_filenames, results)
})
pool = mp.Pool()
try:
# check the output from each invocation of clang-format in parallel
for filename, diff in pool.imap(checker, formatted_filenames):
if not arguments.quiet:
print("Checking {}".format(filename))
if diff:
print("{} had clang-format style issues".format(filename))
# Print out the diff to stderr
error = True
# pad with a newline
print(file=sys.stderr)
diff_out = []
for diff_str in diff:
diff_out.append(diff_str.encode('raw_unicode_escape'))
sys.stderr.writelines(diff_out)
except Exception:
error = True
raise
finally:
pool.terminate()
pool.join()
sys.exit(1 if error else 0)

125
proxy/build-support/run_clang_tidy.py Executable file
View File

@ -0,0 +1,125 @@
#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from __future__ import print_function
import argparse
import multiprocessing as mp
import lintutils
from subprocess import PIPE
import sys
from functools import partial
def _get_chunk_key(filenames):
# lists are not hashable so key on the first filename in a chunk
return filenames[0]
# clang-tidy outputs complaints in '/path:line_number: complaint' format,
# so we can scan its output to get a list of files to fix
def _check_some_files(completed_processes, filenames):
result = completed_processes[_get_chunk_key(filenames)]
return lintutils.stdout_pathcolonline(result, filenames)
def _check_all(cmd, filenames):
# each clang-tidy instance will process 16 files
chunks = lintutils.chunk(filenames, 16)
cmds = [cmd + some for some in chunks]
results = lintutils.run_parallel(cmds, stderr=PIPE, stdout=PIPE)
error = False
# record completed processes (keyed by the first filename in the input
# chunk) for lookup in _check_some_files
completed_processes = {
_get_chunk_key(some): result
for some, result in zip(chunks, results)
}
checker = partial(_check_some_files, completed_processes)
pool = mp.Pool()
try:
# check output of completed clang-tidy invocations in parallel
for problem_files, stdout in pool.imap(checker, chunks):
if problem_files:
msg = "clang-tidy suggested fixes for {}"
print("\n".join(map(msg.format, problem_files)))
print(stdout.decode("utf-8"))
error = True
except Exception:
error = True
raise
finally:
pool.terminate()
pool.join()
if error:
sys.exit(1)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Runs clang-tidy on all ")
parser.add_argument("--clang_tidy_binary",
required=True,
help="Path to the clang-tidy binary")
parser.add_argument("--exclude_globs",
help="Filename containing globs for files "
"that should be excluded from the checks")
parser.add_argument("--compile_commands",
required=True,
help="compile_commands.json to pass clang-tidy")
parser.add_argument("--source_dir",
required=True,
help="Root directory of the source code")
parser.add_argument("--fix", default=False,
action="store_true",
help="If specified, will attempt to fix the "
"source code instead of recommending fixes, "
"defaults to %(default)s")
parser.add_argument("--quiet", default=False,
action="store_true",
help="If specified, only print errors")
arguments = parser.parse_args()
exclude_globs = []
if arguments.exclude_globs:
for line in open(arguments.exclude_globs):
exclude_globs.append(line.strip())
linted_filenames = []
for path in lintutils.get_sources(arguments.source_dir, exclude_globs):
linted_filenames.append(path)
if not arguments.quiet:
msg = 'Tidying {}' if arguments.fix else 'Checking {}'
print("\n".join(map(msg.format, linted_filenames)))
cmd = [
arguments.clang_tidy_binary,
'-p',
arguments.compile_commands
]
if arguments.fix:
cmd.append('-fix')
results = lintutils.run_parallel(
[cmd + some for some in lintutils.chunk(linted_filenames, 16)])
for returncode, stdout, stderr in results:
if returncode != 0:
sys.exit(returncode)
else:
_check_all(cmd, linted_filenames)

132
proxy/build-support/run_cpplint.py Executable file
View File

@ -0,0 +1,132 @@
#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from __future__ import print_function
import lintutils
from subprocess import PIPE, STDOUT
import argparse
import multiprocessing as mp
import sys
import platform
from functools import partial
# NOTE(wesm):
#
# * readability/casting is disabled as it aggressively warns about functions
# with names like "int32", so "int32(x)", where int32 is a function name,
# warns with
_filters = '''
-whitespace/comments
-readability/casting
-readability/todo
-readability/alt_tokens
-build/header_guard
-build/c++11
-runtime/references
-build/include_order
'''.split()
def _get_chunk_key(filenames):
# lists are not hashable so key on the first filename in a chunk
return filenames[0]
def _check_some_files(completed_processes, filenames):
# cpplint outputs complaints in '/path:line_number: complaint' format,
# so we can scan its output to get a list of files to fix
result = completed_processes[_get_chunk_key(filenames)]
return lintutils.stdout_pathcolonline(result, filenames)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Runs cpplint on all of the source files.")
parser.add_argument("--cpplint_binary",
required=True,
help="Path to the cpplint binary")
parser.add_argument("--exclude_globs",
help="Filename containing globs for files "
"that should be excluded from the checks")
parser.add_argument("--source_dir",
required=True,
help="Root directory of the source code")
parser.add_argument("--quiet", default=False,
action="store_true",
help="If specified, only print errors")
arguments = parser.parse_args()
exclude_globs = []
if arguments.exclude_globs:
for line in open(arguments.exclude_globs):
exclude_globs.append(line.strip())
linted_filenames = []
for path in lintutils.get_sources(arguments.source_dir, exclude_globs):
linted_filenames.append(str(path))
cmd = [
arguments.cpplint_binary,
'--verbose=2',
'--linelength=120',
'--filter=' + ','.join(_filters)
]
if (arguments.cpplint_binary.endswith('.py') and
platform.system() == 'Windows'):
# Windows doesn't support executable scripts; execute with
# sys.executable
cmd.insert(0, sys.executable)
if arguments.quiet:
cmd.append('--quiet')
else:
print("\n".join(map(lambda x: "Linting {}".format(x),
linted_filenames)))
# lint files in chunks: each invocation of cpplint will process 16 files
chunks = lintutils.chunk(linted_filenames, 16)
cmds = [cmd + some for some in chunks]
results = lintutils.run_parallel(cmds, stdout=PIPE, stderr=STDOUT)
error = False
# record completed processes (keyed by the first filename in the input
# chunk) for lookup in _check_some_files
completed_processes = {
_get_chunk_key(filenames): result
for filenames, result in zip(chunks, results)
}
checker = partial(_check_some_files, completed_processes)
pool = mp.Pool()
try:
# scan the outputs of various cpplint invocations in parallel to
# distill a list of problematic files
for problem_files, stdout in pool.imap(checker, chunks):
if problem_files:
if isinstance(stdout, bytes):
stdout = stdout.decode('utf8')
print(stdout, file=sys.stderr)
error = True
except Exception:
error = True
raise
finally:
pool.terminate()
pool.join()
sys.exit(1 if error else 0)

118
proxy/build.sh Executable file
View File

@ -0,0 +1,118 @@
#!/bin/bash
# Compile jobs variable; Usage: $ jobs=12 ./build.sh ...
if [[ ! ${jobs+1} ]]; then
jobs=$(nproc)
fi
BUILD_OUTPUT_DIR="cmake_build"
BUILD_TYPE="Release"
BUILD_UNITTEST="OFF"
INSTALL_PREFIX=$(pwd)/milvus
MAKE_CLEAN="OFF"
DB_PATH="/tmp/milvus"
RUN_CPPLINT="OFF"
while getopts "p:d:t:s:ulrcghzme" arg; do
case $arg in
p)
INSTALL_PREFIX=$OPTARG
;;
d)
DB_PATH=$OPTARG
;;
t)
BUILD_TYPE=$OPTARG # BUILD_TYPE
;;
u)
echo "Build and run unittest cases"
BUILD_UNITTEST="ON"
;;
l)
RUN_CPPLINT="ON"
;;
r)
if [[ -d ${BUILD_OUTPUT_DIR} ]]; then
MAKE_CLEAN="ON"
fi
;;
h) # help
echo "
parameter:
-p: install prefix(default: $(pwd)/milvus)
-d: db data path(default: /tmp/milvus)
-t: build type(default: Debug)
-u: building unit test options(default: OFF)
-l: run cpplint, clang-format and clang-tidy(default: OFF)
-r: remove previous build directory(default: OFF)
-h: help
usage:
./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-h]
"
exit 0
;;
?)
echo "ERROR! unknown argument"
exit 1
;;
esac
done
if [[ ! -d ${BUILD_OUTPUT_DIR} ]]; then
mkdir ${BUILD_OUTPUT_DIR}
fi
cd ${BUILD_OUTPUT_DIR}
# remove make cache since build.sh -l use default variables
# force update the variables each time
make rebuild_cache >/dev/null 2>&1
if [[ ${MAKE_CLEAN} == "ON" ]]; then
echo "Runing make clean in ${BUILD_OUTPUT_DIR} ..."
make clean
exit 0
fi
CMAKE_CMD="cmake \
-DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-DOpenBLAS_SOURCE=AUTO \
-DMILVUS_DB_PATH=${DB_PATH} \
../"
echo ${CMAKE_CMD}
${CMAKE_CMD}
if [[ ${RUN_CPPLINT} == "ON" ]]; then
# cpplint check
make lint
if [ $? -ne 0 ]; then
echo "ERROR! cpplint check failed"
exit 1
fi
echo "cpplint check passed!"
# clang-format check
make check-clang-format
if [ $? -ne 0 ]; then
echo "ERROR! clang-format check failed"
exit 1
fi
echo "clang-format check passed!"
# clang-tidy check
# make check-clang-tidy
# if [ $? -ne 0 ]; then
# echo "ERROR! clang-tidy check failed"
# exit 1
# fi
# echo "clang-tidy check passed!"
else
# compile and build
make -j ${jobs} install || exit 1
fi

231
proxy/cmake/BuildUtils.cmake Normal file
View File

@ -0,0 +1,231 @@
# Define a function that check last file modification
function(Check_Last_Modify cache_check_lists_file_path working_dir last_modified_commit_id)
if(EXISTS "${working_dir}")
if(EXISTS "${cache_check_lists_file_path}")
set(GIT_LOG_SKIP_NUM 0)
set(_MATCH_ALL ON CACHE BOOL "Match all")
set(_LOOP_STATUS ON CACHE BOOL "Whether out of loop")
file(STRINGS ${cache_check_lists_file_path} CACHE_IGNORE_TXT)
while(_LOOP_STATUS)
foreach(_IGNORE_ENTRY ${CACHE_IGNORE_TXT})
if(NOT _IGNORE_ENTRY MATCHES "^[^#]+")
continue()
endif()
set(_MATCH_ALL OFF)
execute_process(COMMAND git log --no-merges -1 --skip=${GIT_LOG_SKIP_NUM} --name-status --pretty= WORKING_DIRECTORY ${working_dir} OUTPUT_VARIABLE CHANGE_FILES)
if(NOT CHANGE_FILES STREQUAL "")
string(REPLACE "\n" ";" _CHANGE_FILES ${CHANGE_FILES})
foreach(_FILE_ENTRY ${_CHANGE_FILES})
string(REGEX MATCH "[^ \t]+$" _FILE_NAME ${_FILE_ENTRY})
execute_process(COMMAND sh -c "echo ${_FILE_NAME} | grep ${_IGNORE_ENTRY}" RESULT_VARIABLE return_code)
if (return_code EQUAL 0)
execute_process(COMMAND git log --no-merges -1 --skip=${GIT_LOG_SKIP_NUM} --pretty=%H WORKING_DIRECTORY ${working_dir} OUTPUT_VARIABLE LAST_MODIFIED_COMMIT_ID)
set (${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE)
set(_LOOP_STATUS OFF)
endif()
endforeach()
else()
set(_LOOP_STATUS OFF)
endif()
endforeach()
if(_MATCH_ALL)
execute_process(COMMAND git log --no-merges -1 --skip=${GIT_LOG_SKIP_NUM} --pretty=%H WORKING_DIRECTORY ${working_dir} OUTPUT_VARIABLE LAST_MODIFIED_COMMIT_ID)
set (${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE)
set(_LOOP_STATUS OFF)
endif()
math(EXPR GIT_LOG_SKIP_NUM "${GIT_LOG_SKIP_NUM} + 1")
endwhile(_LOOP_STATUS)
else()
execute_process(COMMAND git log --no-merges -1 --skip=${GIT_LOG_SKIP_NUM} --pretty=%H WORKING_DIRECTORY ${working_dir} OUTPUT_VARIABLE LAST_MODIFIED_COMMIT_ID)
set (${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE)
endif()
else()
message(FATAL_ERROR "The directory ${working_dir} does not exist")
endif()
endfunction()
# Define a function that extracts a cached package
function(ExternalProject_Use_Cache project_name package_file install_path)
message(STATUS "Will use cached package file: ${package_file}")
ExternalProject_Add(${project_name}
DOWNLOAD_COMMAND ${CMAKE_COMMAND} -E echo
"No download step needed (using cached package)"
CONFIGURE_COMMAND ${CMAKE_COMMAND} -E echo
"No configure step needed (using cached package)"
BUILD_COMMAND ${CMAKE_COMMAND} -E echo
"No build step needed (using cached package)"
INSTALL_COMMAND ${CMAKE_COMMAND} -E echo
"No install step needed (using cached package)"
)
# We want our tar files to contain the Install/<package> prefix (not for any
# very special reason, only for consistency and so that we can identify them
# in the extraction logs) which means that we must extract them in the
# binary (top-level build) directory to have them installed in the right
# place for subsequent ExternalProjects to pick them up. It seems that the
# only way to control the working directory is with Add_Step!
ExternalProject_Add_Step(${project_name} extract
ALWAYS 1
COMMAND
${CMAKE_COMMAND} -E echo
"Extracting ${package_file} to ${install_path}"
COMMAND
${CMAKE_COMMAND} -E tar xzf ${package_file} ${install_path}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
)
ExternalProject_Add_StepTargets(${project_name} extract)
endfunction()
# Define a function that to create a new cached package
function(ExternalProject_Create_Cache project_name package_file install_path cache_username cache_password cache_path)
if(EXISTS ${package_file})
message(STATUS "Removing existing package file: ${package_file}")
file(REMOVE ${package_file})
endif()
string(REGEX REPLACE "(.+)/.+$" "\\1" package_dir ${package_file})
if(NOT EXISTS ${package_dir})
file(MAKE_DIRECTORY ${package_dir})
endif()
message(STATUS "Will create cached package file: ${package_file}")
ExternalProject_Add_Step(${project_name} package
DEPENDEES install
BYPRODUCTS ${package_file}
COMMAND ${CMAKE_COMMAND} -E echo "Updating cached package file: ${package_file}"
COMMAND ${CMAKE_COMMAND} -E tar czvf ${package_file} ${install_path}
COMMAND ${CMAKE_COMMAND} -E echo "Uploading package file ${package_file} to ${cache_path}"
COMMAND curl -u${cache_username}:${cache_password} -T ${package_file} ${cache_path}
)
ExternalProject_Add_StepTargets(${project_name} package)
endfunction()
function(ADD_THIRDPARTY_LIB LIB_NAME)
set(options)
set(one_value_args SHARED_LIB STATIC_LIB)
set(multi_value_args DEPS INCLUDE_DIRECTORIES)
cmake_parse_arguments(ARG
"${options}"
"${one_value_args}"
"${multi_value_args}"
${ARGN})
if(ARG_UNPARSED_ARGUMENTS)
message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
endif()
if(ARG_STATIC_LIB AND ARG_SHARED_LIB)
if(NOT ARG_STATIC_LIB)
message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
endif()
set(AUG_LIB_NAME "${LIB_NAME}_static")
add_library(${AUG_LIB_NAME} STATIC IMPORTED)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
if(ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
message(STATUS "Added static library dependency ${AUG_LIB_NAME}: ${ARG_STATIC_LIB}")
if(ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
set(AUG_LIB_NAME "${LIB_NAME}_shared")
add_library(${AUG_LIB_NAME} SHARED IMPORTED)
if(WIN32)
# Mark the ".lib" location as part of a Windows DLL
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
else()
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
endif()
if(ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
message(STATUS "Added shared library dependency ${AUG_LIB_NAME}: ${ARG_SHARED_LIB}")
if(ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
elseif(ARG_STATIC_LIB)
set(AUG_LIB_NAME "${LIB_NAME}_static")
add_library(${AUG_LIB_NAME} STATIC IMPORTED)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
if(ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
message(STATUS "Added static library dependency ${AUG_LIB_NAME}: ${ARG_STATIC_LIB}")
if(ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
elseif(ARG_SHARED_LIB)
set(AUG_LIB_NAME "${LIB_NAME}_shared")
add_library(${AUG_LIB_NAME} SHARED IMPORTED)
if(WIN32)
# Mark the ".lib" location as part of a Windows DLL
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
else()
set_target_properties(${AUG_LIB_NAME}
PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
endif()
message(STATUS "Added shared library dependency ${AUG_LIB_NAME}: ${ARG_SHARED_LIB}")
if(ARG_DEPS)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}")
endif()
if(ARG_INCLUDE_DIRECTORIES)
set_target_properties(${AUG_LIB_NAME}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${ARG_INCLUDE_DIRECTORIES}")
endif()
else()
message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
endif()
endfunction()
MACRO (import_mysql_inc)
find_path (MYSQL_INCLUDE_DIR
NAMES "mysql.h"
PATH_SUFFIXES "mysql")
if (${MYSQL_INCLUDE_DIR} STREQUAL "MYSQL_INCLUDE_DIR-NOTFOUND")
message(FATAL_ERROR "Could not found MySQL include directory")
else ()
include_directories(${MYSQL_INCLUDE_DIR})
endif ()
ENDMACRO (import_mysql_inc)
MACRO(using_ccache_if_defined MILVUS_USE_CCACHE)
if (MILVUS_USE_CCACHE)
find_program(CCACHE_FOUND ccache)
if (CCACHE_FOUND)
message(STATUS "Using ccache: ${CCACHE_FOUND}")
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
# let ccache preserve C++ comments, because some of them may be
# meaningful to the compiler
set(ENV{CCACHE_COMMENTS} "1")
endif (CCACHE_FOUND)
endif ()
ENDMACRO(using_ccache_if_defined)

155
proxy/cmake/DefineOptions.cmake Normal file
View File

@ -0,0 +1,155 @@
macro(set_option_category name)
set(MILVUS_OPTION_CATEGORY ${name})
list(APPEND "MILVUS_OPTION_CATEGORIES" ${name})
endmacro()
macro(define_option name description default)
option(${name} ${description} ${default})
list(APPEND "MILVUS_${MILVUS_OPTION_CATEGORY}_OPTION_NAMES" ${name})
set("${name}_OPTION_DESCRIPTION" ${description})
set("${name}_OPTION_DEFAULT" ${default})
set("${name}_OPTION_TYPE" "bool")
endmacro()
function(list_join lst glue out)
if ("${${lst}}" STREQUAL "")
set(${out} "" PARENT_SCOPE)
return()
endif ()
list(GET ${lst} 0 joined)
list(REMOVE_AT ${lst} 0)
foreach (item ${${lst}})
set(joined "${joined}${glue}${item}")
endforeach ()
set(${out} ${joined} PARENT_SCOPE)
endfunction()
macro(define_option_string name description default)
set(${name} ${default} CACHE STRING ${description})
list(APPEND "MILVUS_${MILVUS_OPTION_CATEGORY}_OPTION_NAMES" ${name})
set("${name}_OPTION_DESCRIPTION" ${description})
set("${name}_OPTION_DEFAULT" "\"${default}\"")
set("${name}_OPTION_TYPE" "string")
set("${name}_OPTION_ENUM" ${ARGN})
list_join("${name}_OPTION_ENUM" "|" "${name}_OPTION_ENUM")
if (NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
set_property(CACHE ${name} PROPERTY STRINGS ${ARGN})
endif ()
endmacro()
#----------------------------------------------------------------------
set_option_category("Milvus Build Option")
define_option(MILVUS_GPU_VERSION "Build GPU version" OFF)
#----------------------------------------------------------------------
set_option_category("Thirdparty")
set(MILVUS_DEPENDENCY_SOURCE_DEFAULT "BUNDLED")
define_option_string(MILVUS_DEPENDENCY_SOURCE
"Method to use for acquiring MILVUS's build dependencies"
"${MILVUS_DEPENDENCY_SOURCE_DEFAULT}"
"AUTO"
"BUNDLED"
"SYSTEM")
define_option(MILVUS_USE_CCACHE "Use ccache when compiling (if available)" ON)
define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD
"Show output from ExternalProjects rather than just logging to files" ON)
define_option(MILVUS_WITH_EASYLOGGINGPP "Build with Easylogging++ library" ON)
define_option(MILVUS_WITH_GRPC "Build with GRPC" ON)
define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON)
define_option(MILVUS_WITH_OPENTRACING "Build with Opentracing" ON)
define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON)
#----------------------------------------------------------------------
set_option_category("Test and benchmark")
unset(MILVUS_BUILD_TESTS CACHE)
if (BUILD_UNIT_TEST)
define_option(MILVUS_BUILD_TESTS "Build the MILVUS googletest unit tests" ON)
else ()
define_option(MILVUS_BUILD_TESTS "Build the MILVUS googletest unit tests" OFF)
endif (BUILD_UNIT_TEST)
#----------------------------------------------------------------------
macro(config_summary)
message(STATUS "---------------------------------------------------------------------")
message(STATUS "MILVUS version: ${MILVUS_VERSION}")
message(STATUS)
message(STATUS "Build configuration summary:")
message(STATUS " Generator: ${CMAKE_GENERATOR}")
message(STATUS " Build type: ${CMAKE_BUILD_TYPE}")
message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}")
if (${CMAKE_EXPORT_COMPILE_COMMANDS})
message(
STATUS " Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json")
endif ()
foreach (category ${MILVUS_OPTION_CATEGORIES})
message(STATUS)
message(STATUS "${category} options:")
set(option_names ${MILVUS_${category}_OPTION_NAMES})
set(max_value_length 0)
foreach (name ${option_names})
string(LENGTH "\"${${name}}\"" value_length)
if (${max_value_length} LESS ${value_length})
set(max_value_length ${value_length})
endif ()
endforeach ()
foreach (name ${option_names})
if ("${${name}_OPTION_TYPE}" STREQUAL "string")
set(value "\"${${name}}\"")
else ()
set(value "${${name}}")
endif ()
set(default ${${name}_OPTION_DEFAULT})
set(description ${${name}_OPTION_DESCRIPTION})
string(LENGTH ${description} description_length)
if (${description_length} LESS 70)
string(
SUBSTRING
" "
${description_length} -1 description_padding)
else ()
set(description_padding "
")
endif ()
set(comment "[${name}]")
if ("${value}" STREQUAL "${default}")
set(comment "[default] ${comment}")
endif ()
if (NOT ("${${name}_OPTION_ENUM}" STREQUAL ""))
set(comment "${comment} [${${name}_OPTION_ENUM}]")
endif ()
string(
SUBSTRING "${value} "
0 ${max_value_length} value)
message(STATUS " ${description} ${description_padding} ${value} ${comment}")
endforeach ()
endforeach ()
endmacro()

111
proxy/cmake/FindClangTools.cmake Normal file
View File

@ -0,0 +1,111 @@
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Tries to find the clang-tidy and clang-format modules
#
# Usage of this module as follows:
#
# find_package(ClangTools)
#
# Variables used by this module, they can change the default behaviour and need
# to be set before calling find_package:
#
# ClangToolsBin_HOME -
# When set, this path is inspected instead of standard library binary locations
# to find clang-tidy and clang-format
#
# This module defines
# CLANG_TIDY_BIN, The path to the clang tidy binary
# CLANG_TIDY_FOUND, Whether clang tidy was found
# CLANG_FORMAT_BIN, The path to the clang format binary
# CLANG_TIDY_FOUND, Whether clang format was found
find_program(CLANG_TIDY_BIN
NAMES
clang-tidy-7.0
clang-tidy-7
clang-tidy-6.0
clang-tidy-5.0
clang-tidy-4.0
clang-tidy-3.9
clang-tidy-3.8
clang-tidy-3.7
clang-tidy-3.6
clang-tidy
PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
NO_DEFAULT_PATH
)
if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
set(CLANG_TIDY_FOUND 0)
message("clang-tidy not found")
else()
set(CLANG_TIDY_FOUND 1)
message("clang-tidy found at ${CLANG_TIDY_BIN}")
endif()
if (CLANG_FORMAT_VERSION)
find_program(CLANG_FORMAT_BIN
NAMES clang-format-${CLANG_FORMAT_VERSION}
PATHS
${ClangTools_PATH}
$ENV{CLANG_TOOLS_PATH}
/usr/local/bin /usr/bin
NO_DEFAULT_PATH
)
# If not found yet, search alternative locations
if (("${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND") AND APPLE)
# Homebrew ships older LLVM versions in /usr/local/opt/llvm@version/
STRING(REGEX REPLACE "^([0-9]+)\\.[0-9]+" "\\1" CLANG_FORMAT_MAJOR_VERSION "${CLANG_FORMAT_VERSION}")
STRING(REGEX REPLACE "^[0-9]+\\.([0-9]+)" "\\1" CLANG_FORMAT_MINOR_VERSION "${CLANG_FORMAT_VERSION}")
if ("${CLANG_FORMAT_MINOR_VERSION}" STREQUAL "0")
find_program(CLANG_FORMAT_BIN
NAMES clang-format
PATHS /usr/local/opt/llvm@${CLANG_FORMAT_MAJOR_VERSION}/bin
NO_DEFAULT_PATH
)
else()
find_program(CLANG_FORMAT_BIN
NAMES clang-format
PATHS /usr/local/opt/llvm@${CLANG_FORMAT_VERSION}/bin
NO_DEFAULT_PATH
)
endif()
endif()
else()
find_program(CLANG_FORMAT_BIN
NAMES
clang-format-7.0
clang-format-7
clang-format-6.0
clang-format-5.0
clang-format-4.0
clang-format-3.9
clang-format-3.8
clang-format-3.7
clang-format-3.6
clang-format
PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
NO_DEFAULT_PATH
)
endif()
if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
set(CLANG_FORMAT_FOUND 0)
message("clang-format not found")
else()
set(CLANG_FORMAT_FOUND 1)
message("clang-format found at ${CLANG_FORMAT_BIN}")
endif()

172
proxy/cmake/ThirdPartyPackages.cmake Normal file
View File

@ -0,0 +1,172 @@
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
message(STATUS "Using ${MILVUS_DEPENDENCY_SOURCE} approach to find dependencies")
# For each dependency, set dependency source to global default, if unset
foreach (DEPENDENCY ${MILVUS_THIRDPARTY_DEPENDENCIES})
if ("${${DEPENDENCY}_SOURCE}" STREQUAL "")
set(${DEPENDENCY}_SOURCE ${MILVUS_DEPENDENCY_SOURCE})
endif ()
endforeach ()
# ----------------------------------------------------------------------
# Identify OS
if (UNIX)
if (APPLE)
set(CMAKE_OS_NAME "osx" CACHE STRING "Operating system name" FORCE)
else (APPLE)
## Check for Debian GNU/Linux ________________
find_file(DEBIAN_FOUND debian_version debconf.conf
PATHS /etc
)
if (DEBIAN_FOUND)
set(CMAKE_OS_NAME "debian" CACHE STRING "Operating system name" FORCE)
endif (DEBIAN_FOUND)
## Check for Fedora _________________________
find_file(FEDORA_FOUND fedora-release
PATHS /etc
)
if (FEDORA_FOUND)
set(CMAKE_OS_NAME "fedora" CACHE STRING "Operating system name" FORCE)
endif (FEDORA_FOUND)
## Check for RedHat _________________________
find_file(REDHAT_FOUND redhat-release inittab.RH
PATHS /etc
)
if (REDHAT_FOUND)
set(CMAKE_OS_NAME "redhat" CACHE STRING "Operating system name" FORCE)
endif (REDHAT_FOUND)
## Extra check for Ubuntu ____________________
if (DEBIAN_FOUND)
## At its core Ubuntu is a Debian system, with
## a slightly altered configuration; hence from
## a first superficial inspection a system will
## be considered as Debian, which signifies an
## extra check is required.
find_file(UBUNTU_EXTRA legal issue
PATHS /etc
)
if (UBUNTU_EXTRA)
## Scan contents of file
file(STRINGS ${UBUNTU_EXTRA} UBUNTU_FOUND
REGEX Ubuntu
)
## Check result of string search
if (UBUNTU_FOUND)
set(CMAKE_OS_NAME "ubuntu" CACHE STRING "Operating system name" FORCE)
set(DEBIAN_FOUND FALSE)
find_program(LSB_RELEASE_EXEC lsb_release)
execute_process(COMMAND ${LSB_RELEASE_EXEC} -rs
OUTPUT_VARIABLE LSB_RELEASE_ID_SHORT
OUTPUT_STRIP_TRAILING_WHITESPACE
)
STRING(REGEX REPLACE "\\." "_" UBUNTU_VERSION "${LSB_RELEASE_ID_SHORT}")
endif (UBUNTU_FOUND)
endif (UBUNTU_EXTRA)
endif (DEBIAN_FOUND)
endif (APPLE)
endif (UNIX)
# ----------------------------------------------------------------------
# thirdparty directory
set(THIRDPARTY_DIR "${MILVUS_SOURCE_DIR}/thirdparty")
# ----------------------------------------------------------------------
# ExternalProject options
string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}")
set(EP_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}}")
# Set -fPIC on all external projects
set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC")
set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC")
# CC/CXX environment variables are captured on the first invocation of the
# builder (e.g make or ninja) instead of when CMake is invoked into to build
# directory. This leads to issues if the variables are exported in a subshell
# and the invocation of make/ninja is in distinct subshell without the same
# environment (CC/CXX).
set(EP_COMMON_TOOLCHAIN -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER})
if (CMAKE_AR)
set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_AR=${CMAKE_AR})
endif ()
if (CMAKE_RANLIB)
set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_RANLIB=${CMAKE_RANLIB})
endif ()
# External projects are still able to override the following declarations.
# cmake command line will favor the last defined variable when a duplicate is
# encountered. This requires that `EP_COMMON_CMAKE_ARGS` is always the first
# argument.
set(EP_COMMON_CMAKE_ARGS
${EP_COMMON_TOOLCHAIN}
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_C_FLAGS=${EP_C_FLAGS}
-DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}
-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}
-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS})
if (NOT MILVUS_VERBOSE_THIRDPARTY_BUILD)
set(EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1)
else ()
set(EP_LOG_OPTIONS)
endif ()
# Ensure that a default make is set
if ("${MAKE}" STREQUAL "")
find_program(MAKE make)
endif ()
if (NOT DEFINED MAKE_BUILD_ARGS)
set(MAKE_BUILD_ARGS "-j8")
endif ()
message(STATUS "Third Party MAKE_BUILD_ARGS = ${MAKE_BUILD_ARGS}")
# ----------------------------------------------------------------------
# Find pthreads
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
# ----------------------------------------------------------------------
# Versions and URLs for toolchain builds, which also can be used to configure
# offline builds
# Read toolchain versions from cpp/thirdparty/versions.txt
file(STRINGS "${THIRDPARTY_DIR}/versions.txt" TOOLCHAIN_VERSIONS_TXT)
foreach (_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
# Exclude comments
if (NOT _VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_VERSION=")
continue()
endif ()
string(REGEX MATCH "^[^=]*" _LIB_NAME ${_VERSION_ENTRY})
string(REPLACE "${_LIB_NAME}=" "" _LIB_VERSION ${_VERSION_ENTRY})
# Skip blank or malformed lines
if (${_LIB_VERSION} STREQUAL "")
continue()
endif ()
# For debugging
#message(STATUS "${_LIB_NAME}: ${_LIB_VERSION}")
set(${_LIB_NAME} "${_LIB_VERSION}")
endforeach ()

102
proxy/cmake/Utils.cmake Normal file
View File

@ -0,0 +1,102 @@
# get build time
MACRO(get_current_time CURRENT_TIME)
execute_process(COMMAND "date" "+%Y-%m-%d %H:%M.%S" OUTPUT_VARIABLE ${CURRENT_TIME})
string(REGEX REPLACE "\n" "" ${CURRENT_TIME} ${${CURRENT_TIME}})
ENDMACRO(get_current_time)
# get build type
MACRO(get_build_type)
cmake_parse_arguments(BUILD_TYPE "" "TARGET;DEFAULT" "" ${ARGN})
if (NOT DEFINED CMAKE_BUILD_TYPE)
set(${BUILD_TYPE_TARGET} ${BUILD_TYPE_DEFAULT})
elseif (CMAKE_BUILD_TYPE STREQUAL "Release")
set(${BUILD_TYPE_TARGET} "Release")
elseif (CMAKE_BUILD_TYPE STREQUAL "Debug")
set(${BUILD_TYPE_TARGET} "Debug")
else ()
set(${BUILD_TYPE_TARGET} ${BUILD_TYPE_DEFAULT})
endif ()
ENDMACRO(get_build_type)
# get git branch name
MACRO(get_git_branch_name GIT_BRANCH_NAME)
set(GIT_BRANCH_NAME_REGEX "[0-9]+\\.[0-9]+\\.[0-9]")
execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.*, //' | sed 's=[a-zA-Z]*\/==g'"
OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
if (NOT GIT_BRANCH_NAME MATCHES "${GIT_BRANCH_NAME_REGEX}")
execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
endif ()
if (NOT GIT_BRANCH_NAME MATCHES "${GIT_BRANCH_NAME_REGEX}")
execute_process(COMMAND "git" symbolic-ref -q --short HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
endif ()
message(DEBUG "GIT_BRANCH_NAME = ${GIT_BRANCH_NAME}")
# Some unexpected case
if (NOT GIT_BRANCH_NAME STREQUAL "")
string(REGEX REPLACE "\n" "" GIT_BRANCH_NAME ${GIT_BRANCH_NAME})
else ()
set(GIT_BRANCH_NAME "#")
endif ()
ENDMACRO(get_git_branch_name)
# get last commit id
MACRO(get_last_commit_id LAST_COMMIT_ID)
execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | awk '{print $2}'"
OUTPUT_VARIABLE ${LAST_COMMIT_ID})
message(DEBUG "LAST_COMMIT_ID = ${${LAST_COMMIT_ID}}")
if (NOT LAST_COMMIT_ID STREQUAL "")
string(REGEX REPLACE "\n" "" ${LAST_COMMIT_ID} ${${LAST_COMMIT_ID}})
else ()
set(LAST_COMMIT_ID "Unknown")
endif ()
ENDMACRO(get_last_commit_id)
# get milvus version
MACRO(get_milvus_version)
cmake_parse_arguments(VER "" "TARGET;DEFAULT" "" ${ARGN})
# Step 1: get branch name
get_git_branch_name(GIT_BRANCH_NAME)
message(DEBUG ${GIT_BRANCH_NAME})
# Step 2: match MAJOR.MINOR.PATCH format or set DEFAULT value
string(REGEX MATCH "([0-9]+)\\.([0-9]+)\\.([0-9]+)" ${VER_TARGET} ${GIT_BRANCH_NAME})
if (NOT ${VER_TARGET})
set(${VER_TARGET} ${VER_DEFAULT})
endif()
ENDMACRO(get_milvus_version)
# set definition
MACRO(set_milvus_definition DEF_PASS_CMAKE MILVUS_DEF)
if (${${DEF_PASS_CMAKE}})
add_compile_definitions(${MILVUS_DEF})
endif()
ENDMACRO(set_milvus_definition)
MACRO(append_flags target)
cmake_parse_arguments(M "" "" "FLAGS" ${ARGN})
foreach(FLAG IN ITEMS ${M_FLAGS})
set(${target} "${${target}} ${FLAG}")
endforeach()
ENDMACRO(append_flags)
macro(create_executable)
cmake_parse_arguments(E "" "TARGET" "SRCS;LIBS;DEFS" ${ARGN})
add_executable(${E_TARGET})
target_sources(${E_TARGET} PRIVATE ${E_SRCS})
target_link_libraries(${E_TARGET} PRIVATE ${E_LIBS})
target_compile_definitions(${E_TARGET} PRIVATE ${E_DEFS})
endmacro()
macro(create_library)
cmake_parse_arguments(L "" "TARGET" "SRCS;LIBS;DEFS" ${ARGN})
add_library(${L_TARGET} ${L_SRCS})
target_link_libraries(${L_TARGET} PRIVATE ${L_LIBS})
target_compile_definitions(${L_TARGET} PRIVATE ${L_DEFS})
endmacro()

74
proxy/src/CMakeLists.txt Normal file
View File

@ -0,0 +1,74 @@
#-------------------------------------------------------------------------------
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
include_directories(${MILVUS_SOURCE_DIR})
include_directories(${MILVUS_ENGINE_SRC})
include_directories(${MILVUS_THIRDPARTY_SRC})
include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status)
include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus)
add_subdirectory( tracing )
add_subdirectory( utils )
add_subdirectory( config )
add_subdirectory( query )
add_subdirectory( log )
add_subdirectory( server )
set(link_lib
config
query
utils
log
)
set(link_lib
${link_lib}
curl
)
set( GRPC_LIB libprotobuf
grpc++_reflection
grpc++
)
set( THIRD_PARTY_LIBS
yaml-cpp
${GRPC_LIB}
)
target_link_libraries( server
PUBLIC ${link_lib}
tracing
${THIRD_PARTY_LIBS}
)
# **************************** Get&Print Include Directories ****************************
get_property( dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES )
foreach ( dir ${dirs} )
message( STATUS "Current Include DIRS: " ${dir} )
endforeach ()
set( SERVER_LIBS server )
add_executable( milvus_server ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp
)
target_link_libraries( milvus_server PRIVATE ${SERVER_LIBS} )
install( TARGETS milvus_server DESTINATION bin )

64
proxy/src/config/CMakeLists.txt Normal file
View File

@ -0,0 +1,64 @@
#-------------------------------------------------------------------------------
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
# library
set( CONFIG_SRCS ConfigMgr.h
ConfigMgr.cpp
ConfigType.h
ConfigType.cpp
ServerConfig.h
ServerConfig.cpp
)
set( CONFIG_LIBS yaml-cpp
)
create_library(
TARGET config
SRCS ${CONFIG_SRCS}
LIBS ${CONFIG_LIBS}
)
if ( BUILD_UNIT_TEST )
create_library(
TARGET config-fiu
SRCS ${CONFIG_SRCS}
LIBS ${CONFIG_LIBS}
DEFS FIU_ENABLE
)
target_compile_definitions(config-fiu PRIVATE FIU_ENABLE)
set(GTEST_LIBS gtest gtest_main gmock gmock_main)
create_executable(
TARGET ServerConfigTest
SRCS ServerConfigTest
LIBS config-fiu ${GTEST_LIBS}
DEFS FIU_ENABLE
)
create_executable(
TARGET ConfigTypeTest
SRCS ConfigTypeTest1 ConfigTypeTest2
LIBS config-fiu ${GTEST_LIBS}
DEFS FIU_ENABLE
)
add_test ( NAME ServerConfigTest
COMMAND $<TARGET_FILE:ServerConfigTest>
)
add_test ( NAME ConfigTypeTest
COMMAND $<TARGET_FILE:ConfigTypeTest>
)
endif()

275
proxy/src/config/ConfigMgr.cpp Normal file
View File

@ -0,0 +1,275 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <yaml-cpp/yaml.h>
#include <cstring>
#include <limits>
#include <unordered_map>
#include "config/ConfigMgr.h"
#include "config/ServerConfig.h"
namespace {
const int64_t MB = (1024ll * 1024);
const int64_t GB = (1024ll * 1024 * 1024);
void
Flatten(const YAML::Node& node, std::unordered_map<std::string, std::string>& target, const std::string& prefix) {
for (auto& it : node) {
auto key = prefix.empty() ? it.first.as<std::string>() : prefix + "." + it.first.as<std::string>();
switch (it.second.Type()) {
case YAML::NodeType::Null: {
target[key] = "";
break;
}
case YAML::NodeType::Scalar: {
target[key] = it.second.as<std::string>();
break;
}
case YAML::NodeType::Sequence: {
std::string value;
for (auto& sub : it.second) value += sub.as<std::string>() + ",";
target[key] = value;
break;
}
case YAML::NodeType::Map: {
Flatten(it.second, target, key);
break;
}
case YAML::NodeType::Undefined: {
throw "Unexpected";
}
default:
break;
}
}
}
void
ThrowIfNotSuccess(const milvus::ConfigStatus& cs) {
if (cs.set_return != milvus::SetReturn::SUCCESS) {
throw cs;
}
}
}; // namespace
namespace milvus {
ConfigMgr ConfigMgr::instance;
ConfigMgr::ConfigMgr() {
config_list_ = {
/* version */
{"version", CreateStringConfig("version", false, &config.version.value, "unknown", nullptr, nullptr)},
/* cluster */
{"cluster.enable",
CreateBoolConfig("cluster.enable", false, &config.cluster.enable.value, false, nullptr, nullptr)},
{"cluster.role", CreateEnumConfig("cluster.role", false, &ClusterRoleMap, &config.cluster.role.value,
ClusterRole::RW, nullptr, nullptr)},
/* general */
{"general.timezone",
CreateStringConfig("general.timezone", false, &config.general.timezone.value, "UTC+8", nullptr, nullptr)},
{"general.meta_uri", CreateStringConfig("general.meta_uri", false, &config.general.meta_uri.value,
"sqlite://:@:/", nullptr, nullptr)},
/* network */
{"network.bind.address", CreateStringConfig("network.bind.address", false, &config.network.bind.address.value,
"0.0.0.0", nullptr, nullptr)},
{"network.bind.port", CreateIntegerConfig("network.bind.port", false, 0, 65535, &config.network.bind.port.value,
19530, nullptr, nullptr)},
{"network.http.enable",
CreateBoolConfig("network.http.enable", false, &config.network.http.enable.value, true, nullptr, nullptr)},
{"network.http.port", CreateIntegerConfig("network.http.port", false, 0, 65535, &config.network.http.port.value,
19121, nullptr, nullptr)},
/* storage */
{"storage.path",
CreateStringConfig("storage.path", false, &config.storage.path.value, "/var/lib/milvus", nullptr, nullptr)},
{"storage.auto_flush_interval",
CreateIntegerConfig("storage.auto_flush_interval", true, 0, std::numeric_limits<int64_t>::max(),
&config.storage.auto_flush_interval.value, 1, nullptr, nullptr)},
/* wal */
{"wal.enable", CreateBoolConfig("wal.enable", false, &config.wal.enable.value, true, nullptr, nullptr)},
{"wal.recovery_error_ignore",
CreateBoolConfig("wal.recovery_error_ignore", false, &config.wal.recovery_error_ignore.value, false, nullptr,
nullptr)},
{"wal.buffer_size", CreateSizeConfig("wal.buffer_size", false, 64 * MB, 4096 * MB,
&config.wal.buffer_size.value, 256 * MB, nullptr, nullptr)},
{"wal.path",
CreateStringConfig("wal.path", false, &config.wal.path.value, "/var/lib/milvus/wal", nullptr, nullptr)},
/* cache */
{"cache.cache_size", CreateSizeConfig("cache.cache_size", true, 0, std::numeric_limits<int64_t>::max(),
&config.cache.cache_size.value, 4 * GB, nullptr, nullptr)},
{"cache.cpu_cache_threshold",
CreateFloatingConfig("cache.cpu_cache_threshold", false, 0.0, 1.0, &config.cache.cpu_cache_threshold.value,
0.7, nullptr, nullptr)},
{"cache.insert_buffer_size",
CreateSizeConfig("cache.insert_buffer_size", false, 0, std::numeric_limits<int64_t>::max(),
&config.cache.insert_buffer_size.value, 1 * GB, nullptr, nullptr)},
{"cache.cache_insert_data", CreateBoolConfig("cache.cache_insert_data", false,
&config.cache.cache_insert_data.value, false, nullptr, nullptr)},
{"cache.preload_collection", CreateStringConfig("cache.preload_collection", false,
&config.cache.preload_collection.value, "", nullptr, nullptr)},
/* gpu */
{"gpu.enable", CreateBoolConfig("gpu.enable", false, &config.gpu.enable.value, false, nullptr, nullptr)},
{"gpu.cache_size", CreateSizeConfig("gpu.cache_size", true, 0, std::numeric_limits<int64_t>::max(),
&config.gpu.cache_size.value, 1 * GB, nullptr, nullptr)},
{"gpu.cache_threshold", CreateFloatingConfig("gpu.cache_threshold", false, 0.0, 1.0,
&config.gpu.cache_threshold.value, 0.7, nullptr, nullptr)},
{"gpu.gpu_search_threshold",
CreateIntegerConfig("gpu.gpu_search_threshold", true, 0, std::numeric_limits<int64_t>::max(),
&config.gpu.gpu_search_threshold.value, 1000, nullptr, nullptr)},
{"gpu.search_devices",
CreateStringConfig("gpu.search_devices", false, &config.gpu.search_devices.value, "gpu0", nullptr, nullptr)},
{"gpu.build_index_devices",
CreateStringConfig("gpu.build_index_devices", false, &config.gpu.build_index_devices.value, "gpu0", nullptr,
nullptr)},
/* log */
{"logs.level", CreateStringConfig("logs.level", false, &config.logs.level.value, "debug", nullptr, nullptr)},
{"logs.trace.enable",
CreateBoolConfig("logs.trace.enable", false, &config.logs.trace.enable.value, true, nullptr, nullptr)},
{"logs.path",
CreateStringConfig("logs.path", false, &config.logs.path.value, "/var/lib/milvus/logs", nullptr, nullptr)},
{"logs.max_log_file_size", CreateSizeConfig("logs.max_log_file_size", false, 512 * MB, 4096 * MB,
&config.logs.max_log_file_size.value, 1024 * MB, nullptr, nullptr)},
{"logs.log_rotate_num", CreateIntegerConfig("logs.log_rotate_num", false, 0, 1024,
&config.logs.log_rotate_num.value, 0, nullptr, nullptr)},
/* metric */
{"metric.enable",
CreateBoolConfig("metric.enable", false, &config.metric.enable.value, false, nullptr, nullptr)},
{"metric.address",
CreateStringConfig("metric.address", false, &config.metric.address.value, "127.0.0.1", nullptr, nullptr)},
{"metric.port",
CreateIntegerConfig("metric.port", false, 1024, 65535, &config.metric.port.value, 9091, nullptr, nullptr)},
/* tracing */
{"tracing.json_config_path", CreateStringConfig("tracing.json_config_path", false,
&config.tracing.json_config_path.value, "", nullptr, nullptr)},
/* invisible */
/* engine */
{"engine.build_index_threshold",
CreateIntegerConfig("engine.build_index_threshold", false, 0, std::numeric_limits<int64_t>::max(),
&config.engine.build_index_threshold.value, 4096, nullptr, nullptr)},
{"engine.search_combine_nq",
CreateIntegerConfig("engine.search_combine_nq", true, 0, std::numeric_limits<int64_t>::max(),
&config.engine.search_combine_nq.value, 64, nullptr, nullptr)},
{"engine.use_blas_threshold",
CreateIntegerConfig("engine.use_blas_threshold", true, 0, std::numeric_limits<int64_t>::max(),
&config.engine.use_blas_threshold.value, 1100, nullptr, nullptr)},
{"engine.omp_thread_num",
CreateIntegerConfig("engine.omp_thread_num", true, 0, std::numeric_limits<int64_t>::max(),
&config.engine.omp_thread_num.value, 0, nullptr, nullptr)},
{"engine.simd_type", CreateEnumConfig("engine.simd_type", false, &SimdMap, &config.engine.simd_type.value,
SimdType::AUTO, nullptr, nullptr)},
};
}
void
ConfigMgr::Init() {
std::lock_guard<std::mutex> lock(GetConfigMutex());
for (auto& kv : config_list_) {
kv.second->Init();
}
}
void
ConfigMgr::Load(const std::string& path) {
/* load from milvus.yaml */
auto yaml = YAML::LoadFile(path);
/* make it flattened */
std::unordered_map<std::string, std::string> flattened;
Flatten(yaml, flattened, "");
/* update config */
for (auto& it : flattened) Set(it.first, it.second, false);
}
void
ConfigMgr::Set(const std::string& name, const std::string& value, bool update) {
try {
auto& config = config_list_.at(name);
std::unique_lock<std::mutex> lock(GetConfigMutex());
/* update=false when loading from config file */
if (not update) {
ThrowIfNotSuccess(config->Set(value, update));
} else if (config->modifiable_) {
/* set manually */
ThrowIfNotSuccess(config->Set(value, update));
lock.unlock();
Notify(name);
} else {
throw ConfigStatus(SetReturn::IMMUTABLE, "Config " + name + " is not modifiable");
}
} catch (ConfigStatus& cs) {
throw cs;
} catch (...) {
throw "Config " + name + " not found.";
}
}
std::string
ConfigMgr::Get(const std::string& name) const {
try {
auto& config = config_list_.at(name);
std::lock_guard<std::mutex> lock(GetConfigMutex());
return config->Get();
} catch (...) {
throw "Config " + name + " not found.";
}
}
std::string
ConfigMgr::Dump() const {
std::stringstream ss;
for (auto& kv : config_list_) {
auto& config = kv.second;
ss << config->name_ << ": " << config->Get() << std::endl;
}
return ss.str();
}
void
ConfigMgr::Attach(const std::string& name, ConfigObserver* observer) {
std::lock_guard<std::mutex> lock(observer_mutex_);
observers_[name].push_back(observer);
}
void
ConfigMgr::Detach(const std::string& name, ConfigObserver* observer) {
std::lock_guard<std::mutex> lock(observer_mutex_);
if (observers_.find(name) == observers_.end())
return;
auto& ob_list = observers_[name];
ob_list.remove(observer);
}
void
ConfigMgr::Notify(const std::string& name) {
std::lock_guard<std::mutex> lock(observer_mutex_);
if (observers_.find(name) == observers_.end())
return;
auto& ob_list = observers_[name];
for (auto& ob : ob_list) {
ob->ConfigUpdate(name);
}
}
} // namespace milvus

92
proxy/src/config/ConfigMgr.h Normal file
View File

@ -0,0 +1,92 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <list>
#include <memory>
#include <mutex>
#include <string>
#include <unordered_map>
#include <vector>
#include "config/ServerConfig.h"
namespace milvus {
class ConfigObserver {
public:
virtual ~ConfigObserver() {
}
virtual void
ConfigUpdate(const std::string& name) = 0;
};
using ConfigObserverPtr = std::shared_ptr<ConfigObserver>;
class ConfigMgr {
public:
static ConfigMgr&
GetInstance() {
return instance;
}
private:
static ConfigMgr instance;
public:
ConfigMgr();
ConfigMgr(const ConfigMgr&) = delete;
ConfigMgr&
operator=(const ConfigMgr&) = delete;
ConfigMgr(ConfigMgr&&) = delete;
ConfigMgr&
operator=(ConfigMgr&&) = delete;
public:
void
Init();
void
Load(const std::string& path);
void
Set(const std::string& name, const std::string& value, bool update = true);
std::string
Get(const std::string& name) const;
std::string
Dump() const;
public:
// Shared pointer should not be used here
void
Attach(const std::string& name, ConfigObserver* observer);
void
Detach(const std::string& name, ConfigObserver* observer);
private:
void
Notify(const std::string& name);
private:
std::unordered_map<std::string, BaseConfigPtr> config_list_;
std::mutex mutex_;
std::unordered_map<std::string, std::list<ConfigObserver*>> observers_;
std::mutex observer_mutex_;
};
} // namespace milvus

528
proxy/src/config/ConfigType.cpp Normal file
View File

@ -0,0 +1,528 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "config/ConfigType.h"
#include "config/ServerConfig.h"
#include <strings.h>
#include <algorithm>
#include <cassert>
#include <functional>
#include <sstream>
#include <string>
namespace {
std::unordered_map<std::string, int64_t> BYTE_UNITS = {
{"b", 1},
{"k", 1024},
{"m", 1024 * 1024},
{"g", 1024 * 1024 * 1024},
};
bool
is_integer(const std::string& s) {
if (not s.empty() && (std::isdigit(s[0]) || s[0] == '-')) {
auto ss = s.substr(1);
return std::find_if(ss.begin(), ss.end(), [](unsigned char c) { return !std::isdigit(c); }) == ss.end();
}
return false;
}
bool
is_number(const std::string& s) {
return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isdigit(c); }) == s.end();
}
bool
is_alpha(const std::string& s) {
return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isalpha(c); }) == s.end();
}
template <typename T>
bool
boundary_check(T val, T lower_bound, T upper_bound) {
return lower_bound <= val && val <= upper_bound;
}
bool
parse_bool(const std::string& str, std::string& err) {
if (!strcasecmp(str.c_str(), "true"))
return true;
else if (!strcasecmp(str.c_str(), "false"))
return false;
else
err = "The specified value must be true or false";
return false;
}
std::string
str_tolower(std::string s) {
std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); });
return s;
}
int64_t
parse_bytes(const std::string& str, std::string& err) {
try {
if (str.find_first_of('-') != std::string::npos) {
std::stringstream ss;
ss << "The specified value for memory (" << str << ") should be a positive integer.";
err = ss.str();
return 0;
}
std::string s = str;
if (is_number(s))
return std::stoll(s);
if (s.length() == 0)
return 0;
auto last_two = s.substr(s.length() - 2, 2);
auto last_one = s.substr(s.length() - 1);
if (is_alpha(last_two) && is_alpha(last_one))
if (last_one == "b" or last_one == "B")
s = s.substr(0, s.length() - 1);
auto& units = BYTE_UNITS;
auto suffix = str_tolower(s.substr(s.length() - 1));
std::string digits_part;
if (is_number(suffix)) {
digits_part = s;
suffix = 'b';
} else {
digits_part = s.substr(0, s.length() - 1);
}
if (is_number(digits_part) && (units.find(suffix) != units.end() || is_number(suffix))) {
auto digits = std::stoll(digits_part);
return digits * units[suffix];
} else {
std::stringstream ss;
ss << "The specified value for memory (" << str << ") should specify the units."
<< "The postfix should be one of the `b` `k` `m` `g` characters";
err = ss.str();
}
} catch (...) {
err = "Unknown error happened on parse bytes.";
}
return 0;
}
} // namespace
// Use (void) to silent unused warnings.
#define assertm(exp, msg) assert(((void)msg, exp))
namespace milvus {
std::vector<std::string>
OptionValue(const configEnum& ce) {
std::vector<std::string> ret;
for (auto& e : ce) {
ret.emplace_back(e.first);
}
return ret;
}
BaseConfig::BaseConfig(const char* name, const char* alias, bool modifiable)
: name_(name), alias_(alias), modifiable_(modifiable) {
}
void
BaseConfig::Init() {
assertm(not inited_, "already initialized");
inited_ = true;
}
BoolConfig::BoolConfig(const char* name, const char* alias, bool modifiable, bool* config, bool default_value,
std::function<bool(bool val, std::string& err)> is_valid_fn,
std::function<bool(bool val, bool prev, std::string& err)> update_fn)
: BaseConfig(name, alias, modifiable),
config_(config),
default_value_(default_value),
is_valid_fn_(std::move(is_valid_fn)),
update_fn_(std::move(update_fn)) {
}
void
BoolConfig::Init() {
BaseConfig::Init();
assert(config_ != nullptr);
*config_ = default_value_;
}
ConfigStatus
BoolConfig::Set(const std::string& val, bool update) {
assertm(inited_, "uninitialized");
try {
if (update and not modifiable_) {
std::stringstream ss;
ss << "Config " << name_ << " is immutable.";
return ConfigStatus(SetReturn::IMMUTABLE, ss.str());
}
std::string err;
bool value = parse_bool(val, err);
if (not err.empty())
return ConfigStatus(SetReturn::INVALID, err);
if (is_valid_fn_ && not is_valid_fn_(value, err))
return ConfigStatus(SetReturn::INVALID, err);
bool prev = *config_;
*config_ = value;
if (update && update_fn_ && not update_fn_(value, prev, err)) {
*config_ = prev;
return ConfigStatus(SetReturn::UPDATE_FAILURE, err);
}
return ConfigStatus(SetReturn::SUCCESS, "");
} catch (std::exception& e) {
return ConfigStatus(SetReturn::EXCEPTION, e.what());
} catch (...) {
return ConfigStatus(SetReturn::UNEXPECTED, "unexpected");
}
}
std::string
BoolConfig::Get() {
assertm(inited_, "uninitialized");
return *config_ ? "true" : "false";
}
StringConfig::StringConfig(
const char* name, const char* alias, bool modifiable, std::string* config, const char* default_value,
std::function<bool(const std::string& val, std::string& err)> is_valid_fn,
std::function<bool(const std::string& val, const std::string& prev, std::string& err)> update_fn)
: BaseConfig(name, alias, modifiable),
config_(config),
default_value_(default_value),
is_valid_fn_(std::move(is_valid_fn)),
update_fn_(std::move(update_fn)) {
}
void
StringConfig::Init() {
BaseConfig::Init();
assert(config_ != nullptr);
*config_ = default_value_;
}
ConfigStatus
StringConfig::Set(const std::string& val, bool update) {
assertm(inited_, "uninitialized");
try {
if (update and not modifiable_) {
std::stringstream ss;
ss << "Config " << name_ << " is immutable.";
return ConfigStatus(SetReturn::IMMUTABLE, ss.str());
}
std::string err;
if (is_valid_fn_ && not is_valid_fn_(val, err))
return ConfigStatus(SetReturn::INVALID, err);
std::string prev = *config_;
*config_ = val;
if (update && update_fn_ && not update_fn_(val, prev, err)) {
*config_ = prev;
return ConfigStatus(SetReturn::UPDATE_FAILURE, err);
}
return ConfigStatus(SetReturn::SUCCESS, "");
} catch (std::exception& e) {
return ConfigStatus(SetReturn::EXCEPTION, e.what());
} catch (...) {
return ConfigStatus(SetReturn::UNEXPECTED, "unexpected");
}
}
std::string
StringConfig::Get() {
assertm(inited_, "uninitialized");
return *config_;
}
EnumConfig::EnumConfig(const char* name, const char* alias, bool modifiable, configEnum* enumd, int64_t* config,
int64_t default_value, std::function<bool(int64_t val, std::string& err)> is_valid_fn,
std::function<bool(int64_t val, int64_t prev, std::string& err)> update_fn)
: BaseConfig(name, alias, modifiable),
config_(config),
enum_value_(enumd),
default_value_(default_value),
is_valid_fn_(std::move(is_valid_fn)),
update_fn_(std::move(update_fn)) {
}
void
EnumConfig::Init() {
BaseConfig::Init();
assert(enum_value_ != nullptr);
assertm(not enum_value_->empty(), "enum value empty");
assert(config_ != nullptr);
*config_ = default_value_;
}
ConfigStatus
EnumConfig::Set(const std::string& val, bool update) {
assertm(inited_, "uninitialized");
try {
if (update and not modifiable_) {
std::stringstream ss;
ss << "Config " << name_ << " is immutable.";
return ConfigStatus(SetReturn::IMMUTABLE, ss.str());
}
if (enum_value_->find(val) == enum_value_->end()) {
auto option_values = OptionValue(*enum_value_);
std::stringstream ss;
ss << "Config " << name_ << "(" << val << ") must be one of following: ";
for (size_t i = 0; i < option_values.size() - 1; ++i) {
ss << option_values[i] << ", ";
}
ss << option_values.back() << ".";
return ConfigStatus(SetReturn::ENUM_VALUE_NOTFOUND, ss.str());
}
int64_t value = enum_value_->at(val);
std::string err;
if (is_valid_fn_ && not is_valid_fn_(value, err)) {
return ConfigStatus(SetReturn::INVALID, err);
}
int64_t prev = *config_;
*config_ = value;
if (update && update_fn_ && not update_fn_(value, prev, err)) {
*config_ = prev;
return ConfigStatus(SetReturn::UPDATE_FAILURE, err);
}
return ConfigStatus(SetReturn::SUCCESS, "");
} catch (std::exception& e) {
return ConfigStatus(SetReturn::EXCEPTION, e.what());
} catch (...) {
return ConfigStatus(SetReturn::UNEXPECTED, "unexpected");
}
}
std::string
EnumConfig::Get() {
assertm(inited_, "uninitialized");
for (auto& it : *enum_value_) {
if (*config_ == it.second) {
return it.first;
}
}
return "unknown";
}
IntegerConfig::IntegerConfig(const char* name, const char* alias, bool modifiable, int64_t lower_bound,
int64_t upper_bound, int64_t* config, int64_t default_value,
std::function<bool(int64_t val, std::string& err)> is_valid_fn,
std::function<bool(int64_t val, int64_t prev, std::string& err)> update_fn)
: BaseConfig(name, alias, modifiable),
config_(config),
lower_bound_(lower_bound),
upper_bound_(upper_bound),
default_value_(default_value),
is_valid_fn_(std::move(is_valid_fn)),
update_fn_(std::move(update_fn)) {
}
void
IntegerConfig::Init() {
BaseConfig::Init();
assert(config_ != nullptr);
*config_ = default_value_;
}
ConfigStatus
IntegerConfig::Set(const std::string& val, bool update) {
assertm(inited_, "uninitialized");
try {
if (update and not modifiable_) {
std::stringstream ss;
ss << "Config " << name_ << " is immutable.";
return ConfigStatus(SetReturn::IMMUTABLE, ss.str());
}
if (not is_integer(val)) {
std::stringstream ss;
ss << "Config " << name_ << "(" << val << ") must be a integer.";
return ConfigStatus(SetReturn::INVALID, ss.str());
}
int64_t value = std::stoll(val);
if (not boundary_check<int64_t>(value, lower_bound_, upper_bound_)) {
std::stringstream ss;
ss << "Config " << name_ << "(" << val << ") must in range [" << lower_bound_ << ", " << upper_bound_
<< "].";
return ConfigStatus(SetReturn::OUT_OF_RANGE, ss.str());
}
std::string err;
if (is_valid_fn_ && not is_valid_fn_(value, err))
return ConfigStatus(SetReturn::INVALID, err);
int64_t prev = *config_;
*config_ = value;
if (update && update_fn_ && not update_fn_(value, prev, err)) {
*config_ = prev;
return ConfigStatus(SetReturn::UPDATE_FAILURE, err);
}
return ConfigStatus(SetReturn::SUCCESS, "");
} catch (std::exception& e) {
return ConfigStatus(SetReturn::EXCEPTION, e.what());
} catch (...) {
return ConfigStatus(SetReturn::UNEXPECTED, "unexpected");
}
}
std::string
IntegerConfig::Get() {
assertm(inited_, "uninitialized");
return std::to_string(*config_);
}
FloatingConfig::FloatingConfig(const char* name, const char* alias, bool modifiable, double lower_bound,
double upper_bound, double* config, double default_value,
std::function<bool(double val, std::string& err)> is_valid_fn,
std::function<bool(double val, double prev, std::string& err)> update_fn)
: BaseConfig(name, alias, modifiable),
config_(config),
lower_bound_(lower_bound),
upper_bound_(upper_bound),
default_value_(default_value),
is_valid_fn_(std::move(is_valid_fn)),
update_fn_(std::move(update_fn)) {
}
void
FloatingConfig::Init() {
BaseConfig::Init();
assert(config_ != nullptr);
*config_ = default_value_;
}
ConfigStatus
FloatingConfig::Set(const std::string& val, bool update) {
assertm(inited_, "uninitialized");
try {
if (update and not modifiable_) {
std::stringstream ss;
ss << "Config " << name_ << " is immutable.";
return ConfigStatus(SetReturn::IMMUTABLE, ss.str());
}
double value = std::stod(val);
if (not boundary_check<double>(value, lower_bound_, upper_bound_)) {
std::stringstream ss;
ss << "Config " << name_ << "(" << val << ") must in range [" << lower_bound_ << ", " << upper_bound_
<< "].";
return ConfigStatus(SetReturn::OUT_OF_RANGE, ss.str());
}
std::string err;
if (is_valid_fn_ && not is_valid_fn_(value, err))
return ConfigStatus(SetReturn::INVALID, err);
double prev = *config_;
*config_ = value;
if (update && update_fn_ && not update_fn_(value, prev, err)) {
*config_ = prev;
return ConfigStatus(SetReturn::UPDATE_FAILURE, err);
}
return ConfigStatus(SetReturn::SUCCESS, "");
} catch (std::exception& e) {
return ConfigStatus(SetReturn::EXCEPTION, e.what());
} catch (...) {
return ConfigStatus(SetReturn::UNEXPECTED, "unexpected");
}
}
std::string
FloatingConfig::Get() {
assertm(inited_, "uninitialized");
return std::to_string(*config_);
}
SizeConfig::SizeConfig(const char* name, const char* alias, bool modifiable, int64_t lower_bound, int64_t upper_bound,
int64_t* config, int64_t default_value,
std::function<bool(int64_t val, std::string& err)> is_valid_fn,
std::function<bool(int64_t val, int64_t prev, std::string& err)> update_fn)
: BaseConfig(name, alias, modifiable),
config_(config),
lower_bound_(lower_bound),
upper_bound_(upper_bound),
default_value_(default_value),
is_valid_fn_(std::move(is_valid_fn)),
update_fn_(std::move(update_fn)) {
}
void
SizeConfig::Init() {
BaseConfig::Init();
assert(config_ != nullptr);
*config_ = default_value_;
}
ConfigStatus
SizeConfig::Set(const std::string& val, bool update) {
assertm(inited_, "uninitialized");
try {
if (update and not modifiable_) {
std::stringstream ss;
ss << "Config " << name_ << " is immutable.";
return ConfigStatus(SetReturn::IMMUTABLE, ss.str());
}
std::string err;
int64_t value = parse_bytes(val, err);
if (not err.empty()) {
return ConfigStatus(SetReturn::INVALID, err);
}
if (not boundary_check<int64_t>(value, lower_bound_, upper_bound_)) {
std::stringstream ss;
ss << "Config " << name_ << "(" << val << ") must in range [" << lower_bound_ << " Byte, " << upper_bound_
<< " Byte].";
return ConfigStatus(SetReturn::OUT_OF_RANGE, ss.str());
}
if (is_valid_fn_ && not is_valid_fn_(value, err)) {
return ConfigStatus(SetReturn::INVALID, err);
}
int64_t prev = *config_;
*config_ = value;
if (update && update_fn_ && not update_fn_(value, prev, err)) {
*config_ = prev;
return ConfigStatus(SetReturn::UPDATE_FAILURE, err);
}
return ConfigStatus(SetReturn::SUCCESS, "");
} catch (std::exception& e) {
return ConfigStatus(SetReturn::EXCEPTION, e.what());
} catch (...) {
return ConfigStatus(SetReturn::UNEXPECTED, "unexpected");
}
}
std::string
SizeConfig::Get() {
assertm(inited_, "uninitialized");
return std::to_string(*config_);
}
} // namespace milvus

235
proxy/src/config/ConfigType.h Normal file
View File

@ -0,0 +1,235 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <functional>
#include <memory>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
namespace milvus {
using configEnum = const std::unordered_map<std::string, int64_t>;
std::vector<std::string>
OptionValue(const configEnum& ce);
enum SetReturn {
SUCCESS = 1,
IMMUTABLE,
ENUM_VALUE_NOTFOUND,
INVALID,
OUT_OF_RANGE,
UPDATE_FAILURE,
EXCEPTION,
UNEXPECTED,
};
struct ConfigStatus {
ConfigStatus(SetReturn sr, std::string msg) : set_return(sr), message(std::move(msg)) {
}
SetReturn set_return;
std::string message;
};
class BaseConfig {
public:
BaseConfig(const char* name, const char* alias, bool modifiable);
virtual ~BaseConfig() = default;
public:
bool inited_ = false;
const char* name_;
const char* alias_;
const bool modifiable_;
public:
virtual void
Init();
virtual ConfigStatus
Set(const std::string& value, bool update) = 0;
virtual std::string
Get() = 0;
};
using BaseConfigPtr = std::shared_ptr<BaseConfig>;
class BoolConfig : public BaseConfig {
public:
BoolConfig(const char* name, const char* alias, bool modifiable, bool* config, bool default_value,
std::function<bool(bool val, std::string& err)> is_valid_fn,
std::function<bool(bool val, bool prev, std::string& err)> update_fn);
private:
bool* config_;
const bool default_value_;
std::function<bool(bool val, std::string& err)> is_valid_fn_;
std::function<bool(bool val, bool prev, std::string& err)> update_fn_;
public:
void
Init() override;
ConfigStatus
Set(const std::string& value, bool update) override;
std::string
Get() override;
};
class StringConfig : public BaseConfig {
public:
StringConfig(const char* name, const char* alias, bool modifiable, std::string* config, const char* default_value,
std::function<bool(const std::string& val, std::string& err)> is_valid_fn,
std::function<bool(const std::string& val, const std::string& prev, std::string& err)> update_fn);
private:
std::string* config_;
const char* default_value_;
std::function<bool(const std::string& val, std::string& err)> is_valid_fn_;
std::function<bool(const std::string& val, const std::string& prev, std::string& err)> update_fn_;
public:
void
Init() override;
ConfigStatus
Set(const std::string& value, bool update) override;
std::string
Get() override;
};
class EnumConfig : public BaseConfig {
public:
EnumConfig(const char* name, const char* alias, bool modifiable, configEnum* enumd, int64_t* config,
int64_t default_value, std::function<bool(int64_t val, std::string& err)> is_valid_fn,
std::function<bool(int64_t val, int64_t prev, std::string& err)> update_fn);
private:
int64_t* config_;
configEnum* enum_value_;
const int64_t default_value_;
std::function<bool(int64_t val, std::string& err)> is_valid_fn_;
std::function<bool(int64_t val, int64_t prev, std::string& err)> update_fn_;
public:
void
Init() override;
ConfigStatus
Set(const std::string& value, bool update) override;
std::string
Get() override;
};
class IntegerConfig : public BaseConfig {
public:
IntegerConfig(const char* name, const char* alias, bool modifiable, int64_t lower_bound, int64_t upper_bound,
int64_t* config, int64_t default_value,
std::function<bool(int64_t val, std::string& err)> is_valid_fn,
std::function<bool(int64_t val, int64_t prev, std::string& err)> update_fn);
private:
int64_t* config_;
int64_t lower_bound_;
int64_t upper_bound_;
const int64_t default_value_;
std::function<bool(int64_t val, std::string& err)> is_valid_fn_;
std::function<bool(int64_t val, int64_t prev, std::string& err)> update_fn_;
public:
void
Init() override;
ConfigStatus
Set(const std::string& value, bool update) override;
std::string
Get() override;
};
class FloatingConfig : public BaseConfig {
public:
FloatingConfig(const char* name, const char* alias, bool modifiable, double lower_bound, double upper_bound,
double* config, double default_value, std::function<bool(double val, std::string& err)> is_valid_fn,
std::function<bool(double val, double prev, std::string& err)> update_fn);
private:
double* config_;
double lower_bound_;
double upper_bound_;
const double default_value_;
std::function<bool(double val, std::string& err)> is_valid_fn_;
std::function<bool(double val, double prev, std::string& err)> update_fn_;
public:
void
Init() override;
ConfigStatus
Set(const std::string& value, bool update) override;
std::string
Get() override;
};
class SizeConfig : public BaseConfig {
public:
SizeConfig(const char* name, const char* alias, bool modifiable, int64_t lower_bound, int64_t upper_bound,
int64_t* config, int64_t default_value, std::function<bool(int64_t val, std::string& err)> is_valid_fn,
std::function<bool(int64_t val, int64_t prev, std::string& err)> update_fn);
private:
int64_t* config_;
int64_t lower_bound_;
int64_t upper_bound_;
const int64_t default_value_;
std::function<bool(int64_t val, std::string& err)> is_valid_fn_;
std::function<bool(int64_t val, int64_t prev, std::string& err)> update_fn_;
public:
void
Init() override;
ConfigStatus
Set(const std::string& value, bool update) override;
std::string
Get() override;
};
#define CreateBoolConfig(name, modifiable, config_addr, default, is_valid, update) \
std::make_shared<BoolConfig>(name, nullptr, modifiable, config_addr, (default), is_valid, update)
#define CreateStringConfig(name, modifiable, config_addr, default, is_valid, update) \
std::make_shared<StringConfig>(name, nullptr, modifiable, config_addr, (default), is_valid, update)
#define CreateEnumConfig(name, modifiable, enumd, config_addr, default, is_valid, update) \
std::make_shared<EnumConfig>(name, nullptr, modifiable, enumd, config_addr, (default), is_valid, update)
#define CreateIntegerConfig(name, modifiable, lower_bound, upper_bound, config_addr, default, is_valid, update) \
std::make_shared<IntegerConfig>(name, nullptr, modifiable, lower_bound, upper_bound, config_addr, (default), \
is_valid, update)
#define CreateFloatingConfig(name, modifiable, lower_bound, upper_bound, config_addr, default, is_valid, update) \
std::make_shared<FloatingConfig>(name, nullptr, modifiable, lower_bound, upper_bound, config_addr, (default), \
is_valid, update)
#define CreateSizeConfig(name, modifiable, lower_bound, upper_bound, config_addr, default, is_valid, update) \
std::make_shared<SizeConfig>(name, nullptr, modifiable, lower_bound, upper_bound, config_addr, (default), \
is_valid, update)
} // namespace milvus

493
proxy/src/config/ConfigTypeTest1.cpp Normal file
View File

@ -0,0 +1,493 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <cstring>
#include <functional>
#include "config/ServerConfig.h"
#include "gtest/gtest.h"
namespace milvus {
#define _MODIFIABLE (true)
#define _IMMUTABLE (false)
template <typename T>
class Utils {
public:
bool
validate_fn(const T& value, std::string& err) {
validate_value = value;
return true;
}
bool
update_fn(const T& value, const T& prev, std::string& err) {
new_value = value;
prev_value = prev;
return true;
}
protected:
T validate_value;
T new_value;
T prev_value;
};
/* ValidBoolConfigTest */
class ValidBoolConfigTest : public testing::Test, public Utils<bool> {
protected:
};
TEST_F(ValidBoolConfigTest, init_load_update_get_test) {
auto validate = std::bind(&ValidBoolConfigTest::validate_fn, this, std::placeholders::_1, std::placeholders::_2);
auto update = std::bind(&ValidBoolConfigTest::update_fn, this, std::placeholders::_1, std::placeholders::_2,
std::placeholders::_3);
bool bool_value = true;
auto bool_config = CreateBoolConfig("b", _MODIFIABLE, &bool_value, false, validate, update);
ASSERT_EQ(bool_value, true);
ASSERT_EQ(bool_config->modifiable_, true);
bool_config->Init();
ASSERT_EQ(bool_value, false);
ASSERT_EQ(bool_config->Get(), "false");
{
// now `bool_value` is `false`, calling Set(update=false) to set it to `true`, but not notify update_fn()
validate_value = false;
new_value = false;
prev_value = true;
ConfigStatus status(SetReturn::SUCCESS, "");
status = bool_config->Set("true", false);
EXPECT_EQ(status.set_return, SetReturn::SUCCESS);
EXPECT_EQ(bool_value, true);
EXPECT_EQ(bool_config->Get(), "true");
// expect change
EXPECT_EQ(validate_value, true);
// expect not change
EXPECT_EQ(new_value, false);
EXPECT_EQ(prev_value, true);
}
{
// now `bool_value` is `true`, calling Set(update=true) to set it to `false`, will notify update_fn()
validate_value = true;
new_value = true;
prev_value = false;
ConfigStatus status(SetReturn::SUCCESS, "");
status = bool_config->Set("false", true);
EXPECT_EQ(status.set_return, SetReturn::SUCCESS);
EXPECT_EQ(bool_value, false);
EXPECT_EQ(bool_config->Get(), "false");
// expect change
EXPECT_EQ(validate_value, false);
EXPECT_EQ(new_value, false);
EXPECT_EQ(prev_value, true);
}
}
/* ValidStringConfigTest */
class ValidStringConfigTest : public testing::Test, public Utils<std::string> {
protected:
};
TEST_F(ValidStringConfigTest, init_load_update_get_test) {
auto validate = std::bind(&ValidStringConfigTest::validate_fn, this, std::placeholders::_1, std::placeholders::_2);
auto update = std::bind(&ValidStringConfigTest::update_fn, this, std::placeholders::_1, std::placeholders::_2,
std::placeholders::_3);
std::string string_value;
auto string_config = CreateStringConfig("s", _MODIFIABLE, &string_value, "Magic", validate, update);
ASSERT_EQ(string_value, "");
ASSERT_EQ(string_config->modifiable_, true);
string_config->Init();
ASSERT_EQ(string_value, "Magic");
ASSERT_EQ(string_config->Get(), "Magic");
{
// now `string_value` is `Magic`, calling Set(update=false) to set it to `cigaM`, but not notify update_fn()
validate_value = "";
new_value = "";
prev_value = "";
ConfigStatus status(SetReturn::SUCCESS, "");
status = string_config->Set("cigaM", false);
EXPECT_EQ(status.set_return, SetReturn::SUCCESS);
EXPECT_EQ(string_value, "cigaM");
EXPECT_EQ(string_config->Get(), "cigaM");
// expect change
EXPECT_EQ(validate_value, "cigaM");
// expect not change
EXPECT_EQ(new_value, "");
EXPECT_EQ(prev_value, "");
}
{
// now `string_value` is `cigaM`, calling Set(update=true) to set it to `Check`, will notify update_fn()
validate_value = "";
new_value = "";
prev_value = "";
ConfigStatus status(SetReturn::SUCCESS, "");
status = string_config->Set("Check", true);
EXPECT_EQ(status.set_return, SetReturn::SUCCESS);
EXPECT_EQ(string_value, "Check");
EXPECT_EQ(string_config->Get(), "Check");
// expect change
EXPECT_EQ(validate_value, "Check");
EXPECT_EQ(new_value, "Check");
EXPECT_EQ(prev_value, "cigaM");
}
}
/* ValidIntegerConfigTest */
class ValidIntegerConfigTest : public testing::Test, public Utils<int64_t> {
protected:
};
TEST_F(ValidIntegerConfigTest, init_load_update_get_test) {
auto validate = std::bind(&ValidIntegerConfigTest::validate_fn, this, std::placeholders::_1, std::placeholders::_2);
auto update = std::bind(&ValidIntegerConfigTest::update_fn, this, std::placeholders::_1, std::placeholders::_2,
std::placeholders::_3);
int64_t integer_value = 0;
auto integer_config = CreateIntegerConfig("i", _MODIFIABLE, -100, 100, &integer_value, 42, validate, update);
ASSERT_EQ(integer_value, 0);
ASSERT_EQ(integer_config->modifiable_, true);
integer_config->Init();
ASSERT_EQ(integer_value, 42);
ASSERT_EQ(integer_config->Get(), "42");
{
// now `integer_value` is `42`, calling Set(update=false) to set it to `24`, but not notify update_fn()
validate_value = 0;
new_value = 0;
prev_value = 0;
ConfigStatus status(SetReturn::SUCCESS, "");
status = integer_config->Set("24", false);
EXPECT_EQ(status.set_return, SetReturn::SUCCESS);
EXPECT_EQ(integer_value, 24);
EXPECT_EQ(integer_config->Get(), "24");
// expect change
EXPECT_EQ(validate_value, 24);
// expect not change
EXPECT_EQ(new_value, 0);
EXPECT_EQ(prev_value, 0);
}
{
// now `integer_value` is `24`, calling Set(update=true) to set it to `36`, will notify update_fn()
validate_value = 0;
new_value = 0;
prev_value = 0;
ConfigStatus status(SetReturn::SUCCESS, "");
status = integer_config->Set("36", true);
EXPECT_EQ(status.set_return, SetReturn::SUCCESS);
EXPECT_EQ(integer_value, 36);
EXPECT_EQ(integer_config->Get(), "36");
// expect change
EXPECT_EQ(validate_value, 36);
EXPECT_EQ(new_value, 36);
EXPECT_EQ(prev_value, 24);
}
}
/* ValidFloatingConfigTest */
class ValidFloatingConfigTest : public testing::Test, public Utils<double> {
protected:
};
TEST_F(ValidFloatingConfigTest, init_load_update_get_test) {
auto validate =
std::bind(&ValidFloatingConfigTest::validate_fn, this, std::placeholders::_1, std::placeholders::_2);
auto update = std::bind(&ValidFloatingConfigTest::update_fn, this, std::placeholders::_1, std::placeholders::_2,
std::placeholders::_3);
double floating_value = 0.0;
auto floating_config = CreateFloatingConfig("f", _MODIFIABLE, -10.0, 10.0, &floating_value, 3.14, validate, update);
ASSERT_FLOAT_EQ(floating_value, 0.0);
ASSERT_EQ(floating_config->modifiable_, true);
floating_config->Init();
ASSERT_FLOAT_EQ(floating_value, 3.14);
ASSERT_FLOAT_EQ(std::stof(floating_config->Get()), 3.14);
{
// now `floating_value` is `3.14`, calling Set(update=false) to set it to `6.22`, but not notify update_fn()
validate_value = 0.0;
new_value = 0.0;
prev_value = 0.0;
ConfigStatus status(SetReturn::SUCCESS, "");
status = floating_config->Set("6.22", false);
EXPECT_EQ(status.set_return, SetReturn::SUCCESS);
ASSERT_FLOAT_EQ(floating_value, 6.22);
ASSERT_FLOAT_EQ(std::stof(floating_config->Get()), 6.22);
// expect change
ASSERT_FLOAT_EQ(validate_value, 6.22);
// expect not change
ASSERT_FLOAT_EQ(new_value, 0.0);
ASSERT_FLOAT_EQ(prev_value, 0.0);
}
{
// now `integer_value` is `6.22`, calling Set(update=true) to set it to `-3.14`, will notify update_fn()
validate_value = 0.0;
new_value = 0.0;
prev_value = 0.0;
ConfigStatus status(SetReturn::SUCCESS, "");
status = floating_config->Set("-3.14", true);
EXPECT_EQ(status.set_return, SetReturn::SUCCESS);
ASSERT_FLOAT_EQ(floating_value, -3.14);
ASSERT_FLOAT_EQ(std::stof(floating_config->Get()), -3.14);
// expect change
ASSERT_FLOAT_EQ(validate_value, -3.14);
ASSERT_FLOAT_EQ(new_value, -3.14);
ASSERT_FLOAT_EQ(prev_value, 6.22);
}
}
/* ValidEnumConfigTest */
class ValidEnumConfigTest : public testing::Test, public Utils<int64_t> {
protected:
};
// template <>
// int64_t Utils<int64_t>::validate_value = 0;
// template <>
// int64_t Utils<int64_t>::new_value = 0;
// template <>
// int64_t Utils<int64_t>::prev_value = 0;
TEST_F(ValidEnumConfigTest, init_load_update_get_test) {
auto validate = std::bind(&ValidEnumConfigTest::validate_fn, this, std::placeholders::_1, std::placeholders::_2);
auto update = std::bind(&ValidEnumConfigTest::update_fn, this, std::placeholders::_1, std::placeholders::_2,
std::placeholders::_3);
configEnum testEnum{
{"a", 1},
{"b", 2},
{"c", 3},
};
int64_t enum_value = 0;
auto enum_config = CreateEnumConfig("e", _MODIFIABLE, &testEnum, &enum_value, 1, validate, update);
ASSERT_EQ(enum_value, 0);
ASSERT_EQ(enum_config->modifiable_, true);
enum_config->Init();
ASSERT_EQ(enum_value, 1);
ASSERT_EQ(enum_config->Get(), "a");
{
// now `enum_value` is `a`, calling Set(update=false) to set it to `b`, but not notify update_fn()
validate_value = 0;
new_value = 0;
prev_value = 0;
ConfigStatus status(SetReturn::SUCCESS, "");
status = enum_config->Set("b", false);
EXPECT_EQ(status.set_return, SetReturn::SUCCESS);
ASSERT_EQ(enum_value, 2);
ASSERT_EQ(enum_config->Get(), "b");
// expect change
ASSERT_EQ(validate_value, 2);
// expect not change
ASSERT_EQ(new_value, 0);
ASSERT_EQ(prev_value, 0);
}
{
// now `enum_value` is `b`, calling Set(update=true) to set it to `c`, will notify update_fn()
validate_value = 0;
new_value = 0;
prev_value = 0;
ConfigStatus status(SetReturn::SUCCESS, "");
status = enum_config->Set("c", true);
EXPECT_EQ(status.set_return, SetReturn::SUCCESS);
ASSERT_EQ(enum_value, 3);
ASSERT_EQ(enum_config->Get(), "c");
// expect change
ASSERT_EQ(validate_value, 3);
ASSERT_EQ(new_value, 3);
ASSERT_EQ(prev_value, 2);
}
}
/* ValidSizeConfigTest */
class ValidSizeConfigTest : public testing::Test, public Utils<int64_t> {
protected:
};
// template <>
// int64_t Utils<int64_t>::validate_value = 0;
// template <>
// int64_t Utils<int64_t>::new_value = 0;
// template <>
// int64_t Utils<int64_t>::prev_value = 0;
TEST_F(ValidSizeConfigTest, init_load_update_get_test) {
auto validate = std::bind(&ValidSizeConfigTest::validate_fn, this, std::placeholders::_1, std::placeholders::_2);
auto update = std::bind(&ValidSizeConfigTest::update_fn, this, std::placeholders::_1, std::placeholders::_2,
std::placeholders::_3);
int64_t size_value = 0;
auto size_config = CreateSizeConfig("i", _MODIFIABLE, 0, 1024 * 1024, &size_value, 1024, validate, update);
ASSERT_EQ(size_value, 0);
ASSERT_EQ(size_config->modifiable_, true);
size_config->Init();
ASSERT_EQ(size_value, 1024);
ASSERT_EQ(size_config->Get(), "1024");
{
// now `size_value` is `1024`, calling Set(update=false) to set it to `4096`, but not notify update_fn()
validate_value = 0;
new_value = 0;
prev_value = 0;
ConfigStatus status(SetReturn::SUCCESS, "");
status = size_config->Set("4096", false);
EXPECT_EQ(status.set_return, SetReturn::SUCCESS);
EXPECT_EQ(size_value, 4096);
EXPECT_EQ(size_config->Get(), "4096");
// expect change
EXPECT_EQ(validate_value, 4096);
// expect not change
EXPECT_EQ(new_value, 0);
EXPECT_EQ(prev_value, 0);
}
{
// now `size_value` is `4096`, calling Set(update=true) to set it to `256kb`, will notify update_fn()
validate_value = 0;
new_value = 0;
prev_value = 0;
ConfigStatus status(SetReturn::SUCCESS, "");
status = size_config->Set("256kb", true);
EXPECT_EQ(status.set_return, SetReturn::SUCCESS);
EXPECT_EQ(size_value, 256 * 1024);
EXPECT_EQ(size_config->Get(), "262144");
// expect change
EXPECT_EQ(validate_value, 262144);
EXPECT_EQ(new_value, 262144);
EXPECT_EQ(prev_value, 4096);
}
}
class ValidTest : public testing::Test {
protected:
configEnum family{
{"ipv4", 1},
{"ipv6", 2},
};
struct Server {
bool running = true;
std::string hostname;
int64_t family = 0;
int64_t port = 0;
double uptime = 0;
};
Server server;
protected:
void
SetUp() override {
config_list = {
CreateBoolConfig("running", true, &server.running, true, nullptr, nullptr),
CreateStringConfig("hostname", true, &server.hostname, "Magic", nullptr, nullptr),
CreateEnumConfig("socket_family", false, &family, &server.family, 2, nullptr, nullptr),
CreateIntegerConfig("port", true, 1024, 65535, &server.port, 19530, nullptr, nullptr),
CreateFloatingConfig("uptime", true, 0, 9999.0, &server.uptime, 0, nullptr, nullptr),
};
}
void
TearDown() override {
}
protected:
void
Init() {
for (auto& config : config_list) {
config->Init();
}
}
void
Load() {
std::unordered_map<std::string, std::string> config_file{
{"running", "false"},
};
for (auto& c : config_file) Set(c.first, c.second, false);
}
void
Set(const std::string& name, const std::string& value, bool update = true) {
for (auto& config : config_list) {
if (std::strcmp(name.c_str(), config->name_) == 0) {
config->Set(value, update);
return;
}
}
throw "Config " + name + " not found.";
}
std::string
Get(const std::string& name) {
for (auto& config : config_list) {
if (std::strcmp(name.c_str(), config->name_) == 0) {
return config->Get();
}
}
throw "Config " + name + " not found.";
}
std::vector<BaseConfigPtr> config_list;
};
} // namespace milvus

861
proxy/src/config/ConfigTypeTest2.cpp Normal file
View File

@ -0,0 +1,861 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "config/ServerConfig.h"
#include "gtest/gtest.h"
namespace milvus {
#define _MODIFIABLE (true)
#define _IMMUTABLE (false)
template <typename T>
class Utils {
public:
static bool
valid_check_failure(const T& value, std::string& err) {
err = "Value is invalid.";
return false;
}
static bool
update_failure(const T& value, const T& prev, std::string& err) {
err = "Update is failure";
return false;
}
static bool
valid_check_raise_string(const T& value, std::string& err) {
throw "string exception";
}
static bool
valid_check_raise_exception(const T& value, std::string& err) {
throw std::bad_alloc();
}
};
/* BoolConfigTest */
class BoolConfigTest : public testing::Test, public Utils<bool> {};
TEST_F(BoolConfigTest, nullptr_init_test) {
auto bool_config = CreateBoolConfig("b", _MODIFIABLE, nullptr, true, nullptr, nullptr);
ASSERT_DEATH(bool_config->Init(), "nullptr");
}
TEST_F(BoolConfigTest, init_twice_test) {
bool bool_value;
auto bool_config = CreateBoolConfig("b", _MODIFIABLE, &bool_value, true, nullptr, nullptr);
ASSERT_DEATH(
{
bool_config->Init();
bool_config->Init();
},
"initialized");
}
TEST_F(BoolConfigTest, non_init_test) {
bool bool_value;
auto bool_config = CreateBoolConfig("b", _MODIFIABLE, &bool_value, true, nullptr, nullptr);
ASSERT_DEATH(bool_config->Set("false", true), "uninitialized");
ASSERT_DEATH(bool_config->Get(), "uninitialized");
}
TEST_F(BoolConfigTest, immutable_update_test) {
bool bool_value = false;
auto bool_config = CreateBoolConfig("b", _IMMUTABLE, &bool_value, true, nullptr, nullptr);
bool_config->Init();
ASSERT_EQ(bool_value, true);
ConfigStatus status(SUCCESS, "");
status = bool_config->Set("false", true);
ASSERT_EQ(status.set_return, SetReturn::IMMUTABLE);
ASSERT_EQ(bool_value, true);
}
TEST_F(BoolConfigTest, set_invalid_value_test) {
bool bool_value;
auto bool_config = CreateBoolConfig("b", _MODIFIABLE, &bool_value, true, nullptr, nullptr);
bool_config->Init();
ConfigStatus status(SUCCESS, "");
status = bool_config->Set(" false", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(bool_config->Get(), "true");
status = bool_config->Set("false ", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(bool_config->Get(), "true");
status = bool_config->Set("afalse", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(bool_config->Get(), "true");
status = bool_config->Set("falsee", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(bool_config->Get(), "true");
status = bool_config->Set("abcdefg", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(bool_config->Get(), "true");
status = bool_config->Set("123456", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(bool_config->Get(), "true");
status = bool_config->Set("", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(bool_config->Get(), "true");
}
TEST_F(BoolConfigTest, valid_check_fail_test) {
bool bool_value;
auto bool_config = CreateBoolConfig("b", _MODIFIABLE, &bool_value, true, valid_check_failure, nullptr);
bool_config->Init();
ConfigStatus status(SUCCESS, "");
status = bool_config->Set("123456", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(bool_config->Get(), "true");
}
TEST_F(BoolConfigTest, update_fail_test) {
bool bool_value;
auto bool_config = CreateBoolConfig("b", _MODIFIABLE, &bool_value, true, nullptr, update_failure);
bool_config->Init();
ConfigStatus status(SUCCESS, "");
status = bool_config->Set("false", true);
ASSERT_EQ(status.set_return, SetReturn::UPDATE_FAILURE);
ASSERT_EQ(bool_config->Get(), "true");
}
TEST_F(BoolConfigTest, string_exception_test) {
bool bool_value;
auto bool_config = CreateBoolConfig("b", _MODIFIABLE, &bool_value, true, valid_check_raise_string, nullptr);
bool_config->Init();
ConfigStatus status(SUCCESS, "");
status = bool_config->Set("false", true);
ASSERT_EQ(status.set_return, SetReturn::UNEXPECTED);
ASSERT_EQ(bool_config->Get(), "true");
}
TEST_F(BoolConfigTest, standard_exception_test) {
bool bool_value;
auto bool_config = CreateBoolConfig("b", _MODIFIABLE, &bool_value, true, valid_check_raise_exception, nullptr);
bool_config->Init();
ConfigStatus status(SUCCESS, "");
status = bool_config->Set("false", true);
ASSERT_EQ(status.set_return, SetReturn::EXCEPTION);
ASSERT_EQ(bool_config->Get(), "true");
}
/* StringConfigTest */
class StringConfigTest : public testing::Test, public Utils<std::string> {};
TEST_F(StringConfigTest, nullptr_init_test) {
auto string_config = CreateStringConfig("s", true, nullptr, "Magic", nullptr, nullptr);
ASSERT_DEATH(string_config->Init(), "nullptr");
}
TEST_F(StringConfigTest, init_twice_test) {
std::string string_value;
auto string_config = CreateStringConfig("s", _MODIFIABLE, &string_value, "Magic", nullptr, nullptr);
ASSERT_DEATH(
{
string_config->Init();
string_config->Init();
},
"initialized");
}
TEST_F(StringConfigTest, non_init_test) {
std::string string_value;
auto string_config = CreateStringConfig("s", _MODIFIABLE, &string_value, "Magic", nullptr, nullptr);
ASSERT_DEATH(string_config->Set("value", true), "uninitialized");
ASSERT_DEATH(string_config->Get(), "uninitialized");
}
TEST_F(StringConfigTest, immutable_update_test) {
std::string string_value;
auto string_config = CreateStringConfig("s", _IMMUTABLE, &string_value, "Magic", nullptr, nullptr);
string_config->Init();
ASSERT_EQ(string_value, "Magic");
ConfigStatus status(SUCCESS, "");
status = string_config->Set("cigaM", true);
ASSERT_EQ(status.set_return, SetReturn::IMMUTABLE);
ASSERT_EQ(string_value, "Magic");
}
TEST_F(StringConfigTest, valid_check_fail_test) {
std::string string_value;
auto string_config = CreateStringConfig("s", _MODIFIABLE, &string_value, "Magic", valid_check_failure, nullptr);
string_config->Init();
ConfigStatus status(SUCCESS, "");
status = string_config->Set("123456", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(string_config->Get(), "Magic");
}
TEST_F(StringConfigTest, update_fail_test) {
std::string string_value;
auto string_config = CreateStringConfig("s", _MODIFIABLE, &string_value, "Magic", nullptr, update_failure);
string_config->Init();
ConfigStatus status(SUCCESS, "");
status = string_config->Set("Mi", true);
ASSERT_EQ(status.set_return, SetReturn::UPDATE_FAILURE);
ASSERT_EQ(string_config->Get(), "Magic");
}
TEST_F(StringConfigTest, string_exception_test) {
std::string string_value;
auto string_config =
CreateStringConfig("s", _MODIFIABLE, &string_value, "Magic", valid_check_raise_string, nullptr);
string_config->Init();
ConfigStatus status(SUCCESS, "");
status = string_config->Set("any", true);
ASSERT_EQ(status.set_return, SetReturn::UNEXPECTED);
ASSERT_EQ(string_config->Get(), "Magic");
}
TEST_F(StringConfigTest, standard_exception_test) {
std::string string_value;
auto string_config =
CreateStringConfig("s", _MODIFIABLE, &string_value, "Magic", valid_check_raise_exception, nullptr);
string_config->Init();
ConfigStatus status(SUCCESS, "");
status = string_config->Set("any", true);
ASSERT_EQ(status.set_return, SetReturn::EXCEPTION);
ASSERT_EQ(string_config->Get(), "Magic");
}
/* IntegerConfigTest */
class IntegerConfigTest : public testing::Test, public Utils<int64_t> {};
TEST_F(IntegerConfigTest, nullptr_init_test) {
auto integer_config = CreateIntegerConfig("i", true, 1024, 65535, nullptr, 19530, nullptr, nullptr);
ASSERT_DEATH(integer_config->Init(), "nullptr");
}
TEST_F(IntegerConfigTest, init_twice_test) {
int64_t integer_value;
auto integer_config = CreateIntegerConfig("i", true, 1024, 65535, &integer_value, 19530, nullptr, nullptr);
ASSERT_DEATH(
{
integer_config->Init();
integer_config->Init();
},
"initialized");
}
TEST_F(IntegerConfigTest, non_init_test) {
int64_t integer_value;
auto integer_config = CreateIntegerConfig("i", true, 1024, 65535, &integer_value, 19530, nullptr, nullptr);
ASSERT_DEATH(integer_config->Set("42", true), "uninitialized");
ASSERT_DEATH(integer_config->Get(), "uninitialized");
}
TEST_F(IntegerConfigTest, immutable_update_test) {
int64_t integer_value;
auto integer_config = CreateIntegerConfig("i", _IMMUTABLE, 1024, 65535, &integer_value, 19530, nullptr, nullptr);
integer_config->Init();
ASSERT_EQ(integer_value, 19530);
ConfigStatus status(SUCCESS, "");
status = integer_config->Set("2048", true);
ASSERT_EQ(status.set_return, SetReturn::IMMUTABLE);
ASSERT_EQ(integer_value, 19530);
}
TEST_F(IntegerConfigTest, set_invalid_value_test) {
}
TEST_F(IntegerConfigTest, valid_check_fail_test) {
int64_t integer_value;
auto integer_config =
CreateIntegerConfig("i", true, 1024, 65535, &integer_value, 19530, valid_check_failure, nullptr);
integer_config->Init();
ConfigStatus status(SUCCESS, "");
status = integer_config->Set("2048", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(integer_config->Get(), "19530");
}
TEST_F(IntegerConfigTest, update_fail_test) {
int64_t integer_value;
auto integer_config = CreateIntegerConfig("i", true, 1024, 65535, &integer_value, 19530, nullptr, update_failure);
integer_config->Init();
ConfigStatus status(SUCCESS, "");
status = integer_config->Set("2048", true);
ASSERT_EQ(status.set_return, SetReturn::UPDATE_FAILURE);
ASSERT_EQ(integer_config->Get(), "19530");
}
TEST_F(IntegerConfigTest, string_exception_test) {
int64_t integer_value;
auto integer_config =
CreateIntegerConfig("i", true, 1024, 65535, &integer_value, 19530, valid_check_raise_string, nullptr);
integer_config->Init();
ConfigStatus status(SUCCESS, "");
status = integer_config->Set("2048", true);
ASSERT_EQ(status.set_return, SetReturn::UNEXPECTED);
ASSERT_EQ(integer_config->Get(), "19530");
}
TEST_F(IntegerConfigTest, standard_exception_test) {
int64_t integer_value;
auto integer_config =
CreateIntegerConfig("i", true, 1024, 65535, &integer_value, 19530, valid_check_raise_exception, nullptr);
integer_config->Init();
ConfigStatus status(SUCCESS, "");
status = integer_config->Set("2048", true);
ASSERT_EQ(status.set_return, SetReturn::EXCEPTION);
ASSERT_EQ(integer_config->Get(), "19530");
}
TEST_F(IntegerConfigTest, out_of_range_test) {
int64_t integer_value;
auto integer_config = CreateIntegerConfig("i", true, 1024, 65535, &integer_value, 19530, nullptr, nullptr);
integer_config->Init();
{
ConfigStatus status(SUCCESS, "");
status = integer_config->Set("1023", true);
ASSERT_EQ(status.set_return, SetReturn::OUT_OF_RANGE);
ASSERT_EQ(integer_config->Get(), "19530");
}
{
ConfigStatus status(SUCCESS, "");
status = integer_config->Set("65536", true);
ASSERT_EQ(status.set_return, SetReturn::OUT_OF_RANGE);
ASSERT_EQ(integer_config->Get(), "19530");
}
}
TEST_F(IntegerConfigTest, invalid_bound_test) {
int64_t integer_value;
auto integer_config = CreateIntegerConfig("i", true, 100, 0, &integer_value, 50, nullptr, nullptr);
integer_config->Init();
ConfigStatus status(SUCCESS, "");
status = integer_config->Set("30", true);
ASSERT_EQ(status.set_return, SetReturn::OUT_OF_RANGE);
ASSERT_EQ(integer_config->Get(), "50");
}
TEST_F(IntegerConfigTest, invalid_format_test) {
int64_t integer_value;
auto integer_config = CreateIntegerConfig("i", true, 0, 100, &integer_value, 50, nullptr, nullptr);
integer_config->Init();
{
ConfigStatus status(SUCCESS, "");
status = integer_config->Set("3-0", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(integer_config->Get(), "50");
}
{
ConfigStatus status(SUCCESS, "");
status = integer_config->Set("30-", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(integer_config->Get(), "50");
}
{
ConfigStatus status(SUCCESS, "");
status = integer_config->Set("+30", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(integer_config->Get(), "50");
}
{
ConfigStatus status(SUCCESS, "");
status = integer_config->Set("a30", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(integer_config->Get(), "50");
}
{
ConfigStatus status(SUCCESS, "");
status = integer_config->Set("30a", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(integer_config->Get(), "50");
}
{
ConfigStatus status(SUCCESS, "");
status = integer_config->Set("3a0", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(integer_config->Get(), "50");
}
}
/* FloatingConfigTest */
class FloatingConfigTest : public testing::Test, public Utils<double> {};
TEST_F(FloatingConfigTest, nullptr_init_test) {
auto floating_config = CreateFloatingConfig("f", true, 1.0, 9.9, nullptr, 4.5, nullptr, nullptr);
ASSERT_DEATH(floating_config->Init(), "nullptr");
}
TEST_F(FloatingConfigTest, init_twice_test) {
double floating_value;
auto floating_config = CreateFloatingConfig("f", true, 1.0, 9.9, &floating_value, 4.5, nullptr, nullptr);
ASSERT_DEATH(
{
floating_config->Init();
floating_config->Init();
},
"initialized");
}
TEST_F(FloatingConfigTest, non_init_test) {
double floating_value;
auto floating_config = CreateFloatingConfig("f", true, 1.0, 9.9, &floating_value, 4.5, nullptr, nullptr);
ASSERT_DEATH(floating_config->Set("3.14", true), "uninitialized");
ASSERT_DEATH(floating_config->Get(), "uninitialized");
}
TEST_F(FloatingConfigTest, immutable_update_test) {
double floating_value;
auto floating_config = CreateFloatingConfig("f", _IMMUTABLE, 1.0, 9.9, &floating_value, 4.5, nullptr, nullptr);
floating_config->Init();
ASSERT_FLOAT_EQ(floating_value, 4.5);
ConfigStatus status(SUCCESS, "");
status = floating_config->Set("1.23", true);
ASSERT_EQ(status.set_return, SetReturn::IMMUTABLE);
ASSERT_FLOAT_EQ(std::stof(floating_config->Get()), 4.5);
}
TEST_F(FloatingConfigTest, set_invalid_value_test) {
}
TEST_F(FloatingConfigTest, valid_check_fail_test) {
double floating_value;
auto floating_config =
CreateFloatingConfig("f", true, 1.0, 9.9, &floating_value, 4.5, valid_check_failure, nullptr);
floating_config->Init();
ConfigStatus status(SUCCESS, "");
status = floating_config->Set("1.23", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_FLOAT_EQ(std::stof(floating_config->Get()), 4.5);
}
TEST_F(FloatingConfigTest, update_fail_test) {
double floating_value;
auto floating_config = CreateFloatingConfig("f", true, 1.0, 9.9, &floating_value, 4.5, nullptr, update_failure);
floating_config->Init();
ConfigStatus status(SUCCESS, "");
status = floating_config->Set("1.23", true);
ASSERT_EQ(status.set_return, SetReturn::UPDATE_FAILURE);
ASSERT_FLOAT_EQ(std::stof(floating_config->Get()), 4.5);
}
TEST_F(FloatingConfigTest, string_exception_test) {
double floating_value;
auto floating_config =
CreateFloatingConfig("f", true, 1.0, 9.9, &floating_value, 4.5, valid_check_raise_string, nullptr);
floating_config->Init();
ConfigStatus status(SUCCESS, "");
status = floating_config->Set("1.23", true);
ASSERT_EQ(status.set_return, SetReturn::UNEXPECTED);
ASSERT_FLOAT_EQ(std::stof(floating_config->Get()), 4.5);
}
TEST_F(FloatingConfigTest, standard_exception_test) {
double floating_value;
auto floating_config =
CreateFloatingConfig("f", true, 1.0, 9.9, &floating_value, 4.5, valid_check_raise_exception, nullptr);
floating_config->Init();
ConfigStatus status(SUCCESS, "");
status = floating_config->Set("1.23", true);
ASSERT_EQ(status.set_return, SetReturn::EXCEPTION);
ASSERT_FLOAT_EQ(std::stof(floating_config->Get()), 4.5);
}
TEST_F(FloatingConfigTest, out_of_range_test) {
double floating_value;
auto floating_config =
CreateFloatingConfig("f", true, 1.0, 9.9, &floating_value, 4.5, valid_check_raise_exception, nullptr);
floating_config->Init();
{
ConfigStatus status(SUCCESS, "");
status = floating_config->Set("0.99", true);
ASSERT_EQ(status.set_return, SetReturn::OUT_OF_RANGE);
ASSERT_FLOAT_EQ(std::stof(floating_config->Get()), 4.5);
}
{
ConfigStatus status(SUCCESS, "");
status = floating_config->Set("10.00", true);
ASSERT_EQ(status.set_return, SetReturn::OUT_OF_RANGE);
ASSERT_FLOAT_EQ(std::stof(floating_config->Get()), 4.5);
}
}
TEST_F(FloatingConfigTest, invalid_bound_test) {
double floating_value;
auto floating_config =
CreateFloatingConfig("f", true, 9.9, 1.0, &floating_value, 4.5, valid_check_raise_exception, nullptr);
floating_config->Init();
ConfigStatus status(SUCCESS, "");
status = floating_config->Set("6.0", true);
ASSERT_EQ(status.set_return, SetReturn::OUT_OF_RANGE);
ASSERT_FLOAT_EQ(std::stof(floating_config->Get()), 4.5);
}
TEST_F(FloatingConfigTest, DISABLED_invalid_format_test) {
double floating_value;
auto floating_config = CreateFloatingConfig("f", true, 1.0, 100.0, &floating_value, 4.5, nullptr, nullptr);
floating_config->Init();
{
ConfigStatus status(SUCCESS, "");
status = floating_config->Set("6.0.1", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_FLOAT_EQ(std::stof(floating_config->Get()), 4.5);
}
{
ConfigStatus status(SUCCESS, "");
status = floating_config->Set("6a0", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_FLOAT_EQ(std::stof(floating_config->Get()), 4.5);
}
}
/* EnumConfigTest */
class EnumConfigTest : public testing::Test, public Utils<int64_t> {};
TEST_F(EnumConfigTest, nullptr_init_test) {
configEnum testEnum{
{"e", 1},
};
int64_t testEnumValue;
auto enum_config_1 = CreateEnumConfig("e", _MODIFIABLE, &testEnum, nullptr, 2, nullptr, nullptr);
ASSERT_DEATH(enum_config_1->Init(), "nullptr");
auto enum_config_2 = CreateEnumConfig("e", _MODIFIABLE, nullptr, &testEnumValue, 2, nullptr, nullptr);
ASSERT_DEATH(enum_config_2->Init(), "nullptr");
auto enum_config_3 = CreateEnumConfig("e", _MODIFIABLE, nullptr, nullptr, 2, nullptr, nullptr);
ASSERT_DEATH(enum_config_3->Init(), "nullptr");
}
TEST_F(EnumConfigTest, init_twice_test) {
configEnum testEnum{
{"e", 1},
};
int64_t enum_value;
auto enum_config = CreateEnumConfig("e", _MODIFIABLE, &testEnum, &enum_value, 2, nullptr, nullptr);
ASSERT_DEATH(
{
enum_config->Init();
enum_config->Init();
},
"initialized");
}
TEST_F(EnumConfigTest, non_init_test) {
configEnum testEnum{
{"e", 1},
};
int64_t enum_value;
auto enum_config = CreateEnumConfig("e", _MODIFIABLE, &testEnum, &enum_value, 2, nullptr, nullptr);
ASSERT_DEATH(enum_config->Set("e", true), "uninitialized");
ASSERT_DEATH(enum_config->Get(), "uninitialized");
}
TEST_F(EnumConfigTest, immutable_update_test) {
configEnum testEnum{
{"a", 1},
{"b", 2},
{"c", 3},
};
int64_t enum_value = 0;
auto enum_config = CreateEnumConfig("e", _IMMUTABLE, &testEnum, &enum_value, 1, nullptr, nullptr);
enum_config->Init();
ASSERT_EQ(enum_value, 1);
ConfigStatus status(SUCCESS, "");
status = enum_config->Set("b", true);
ASSERT_EQ(status.set_return, SetReturn::IMMUTABLE);
ASSERT_EQ(enum_value, 1);
}
TEST_F(EnumConfigTest, set_invalid_value_check) {
configEnum testEnum{
{"a", 1},
};
int64_t enum_value = 0;
auto enum_config = CreateEnumConfig("e", _MODIFIABLE, &testEnum, &enum_value, 1, nullptr, nullptr);
enum_config->Init();
ConfigStatus status(SUCCESS, "");
status = enum_config->Set("b", true);
ASSERT_EQ(status.set_return, SetReturn::ENUM_VALUE_NOTFOUND);
ASSERT_EQ(enum_config->Get(), "a");
}
TEST_F(EnumConfigTest, empty_enum_test) {
configEnum testEnum{};
int64_t enum_value;
auto enum_config = CreateEnumConfig("e", _MODIFIABLE, &testEnum, &enum_value, 2, nullptr, nullptr);
ASSERT_DEATH(enum_config->Init(), "empty");
}
TEST_F(EnumConfigTest, valid_check_fail_test) {
configEnum testEnum{
{"a", 1},
{"b", 2},
{"c", 3},
};
int64_t enum_value;
auto enum_config = CreateEnumConfig("e", _MODIFIABLE, &testEnum, &enum_value, 1, valid_check_failure, nullptr);
enum_config->Init();
ConfigStatus status(SUCCESS, "");
status = enum_config->Set("b", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(enum_config->Get(), "a");
}
TEST_F(EnumConfigTest, update_fail_test) {
configEnum testEnum{
{"a", 1},
{"b", 2},
{"c", 3},
};
int64_t enum_value;
auto enum_config = CreateEnumConfig("e", _MODIFIABLE, &testEnum, &enum_value, 1, nullptr, update_failure);
enum_config->Init();
ConfigStatus status(SUCCESS, "");
status = enum_config->Set("b", true);
ASSERT_EQ(status.set_return, SetReturn::UPDATE_FAILURE);
ASSERT_EQ(enum_config->Get(), "a");
}
TEST_F(EnumConfigTest, string_exception_test) {
configEnum testEnum{
{"a", 1},
{"b", 2},
{"c", 3},
};
int64_t enum_value;
auto enum_config = CreateEnumConfig("e", _MODIFIABLE, &testEnum, &enum_value, 1, valid_check_raise_string, nullptr);
enum_config->Init();
ConfigStatus status(SUCCESS, "");
status = enum_config->Set("b", true);
ASSERT_EQ(status.set_return, SetReturn::UNEXPECTED);
ASSERT_EQ(enum_config->Get(), "a");
}
TEST_F(EnumConfigTest, standard_exception_test) {
configEnum testEnum{
{"a", 1},
{"b", 2},
{"c", 3},
};
int64_t enum_value;
auto enum_config =
CreateEnumConfig("e", _MODIFIABLE, &testEnum, &enum_value, 1, valid_check_raise_exception, nullptr);
enum_config->Init();
ConfigStatus status(SUCCESS, "");
status = enum_config->Set("b", true);
ASSERT_EQ(status.set_return, SetReturn::EXCEPTION);
ASSERT_EQ(enum_config->Get(), "a");
}
/* SizeConfigTest */
class SizeConfigTest : public testing::Test, public Utils<int64_t> {};
TEST_F(SizeConfigTest, nullptr_init_test) {
auto size_config = CreateSizeConfig("i", true, 1024, 4096, nullptr, 2048, nullptr, nullptr);
ASSERT_DEATH(size_config->Init(), "nullptr");
}
TEST_F(SizeConfigTest, init_twice_test) {
int64_t size_value;
auto size_config = CreateSizeConfig("i", true, 1024, 4096, &size_value, 2048, nullptr, nullptr);
ASSERT_DEATH(
{
size_config->Init();
size_config->Init();
},
"initialized");
}
TEST_F(SizeConfigTest, non_init_test) {
int64_t size_value;
auto size_config = CreateSizeConfig("i", true, 1024, 4096, &size_value, 2048, nullptr, nullptr);
ASSERT_DEATH(size_config->Set("3000", true), "uninitialized");
ASSERT_DEATH(size_config->Get(), "uninitialized");
}
TEST_F(SizeConfigTest, immutable_update_test) {
int64_t size_value = 0;
auto size_config = CreateSizeConfig("i", _IMMUTABLE, 1024, 4096, &size_value, 2048, nullptr, nullptr);
size_config->Init();
ASSERT_EQ(size_value, 2048);
ConfigStatus status(SUCCESS, "");
status = size_config->Set("3000", true);
ASSERT_EQ(status.set_return, SetReturn::IMMUTABLE);
ASSERT_EQ(size_value, 2048);
}
TEST_F(SizeConfigTest, set_invalid_value_test) {
}
TEST_F(SizeConfigTest, valid_check_fail_test) {
int64_t size_value;
auto size_config = CreateSizeConfig("i", true, 1024, 4096, &size_value, 2048, valid_check_failure, nullptr);
size_config->Init();
ConfigStatus status(SUCCESS, "");
status = size_config->Set("3000", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(size_config->Get(), "2048");
}
TEST_F(SizeConfigTest, update_fail_test) {
int64_t size_value;
auto size_config = CreateSizeConfig("i", true, 1024, 4096, &size_value, 2048, nullptr, update_failure);
size_config->Init();
ConfigStatus status(SUCCESS, "");
status = size_config->Set("3000", true);
ASSERT_EQ(status.set_return, SetReturn::UPDATE_FAILURE);
ASSERT_EQ(size_config->Get(), "2048");
}
TEST_F(SizeConfigTest, string_exception_test) {
int64_t size_value;
auto size_config = CreateSizeConfig("i", true, 1024, 4096, &size_value, 2048, valid_check_raise_string, nullptr);
size_config->Init();
ConfigStatus status(SUCCESS, "");
status = size_config->Set("3000", true);
ASSERT_EQ(status.set_return, SetReturn::UNEXPECTED);
ASSERT_EQ(size_config->Get(), "2048");
}
TEST_F(SizeConfigTest, standard_exception_test) {
int64_t size_value;
auto size_config = CreateSizeConfig("i", true, 1024, 4096, &size_value, 2048, valid_check_raise_exception, nullptr);
size_config->Init();
ConfigStatus status(SUCCESS, "");
status = size_config->Set("3000", true);
ASSERT_EQ(status.set_return, SetReturn::EXCEPTION);
ASSERT_EQ(size_config->Get(), "2048");
}
TEST_F(SizeConfigTest, out_of_range_test) {
int64_t size_value;
auto size_config = CreateSizeConfig("i", true, 1024, 4096, &size_value, 2048, nullptr, nullptr);
size_config->Init();
{
ConfigStatus status(SUCCESS, "");
status = size_config->Set("1023", true);
ASSERT_EQ(status.set_return, SetReturn::OUT_OF_RANGE);
ASSERT_EQ(size_config->Get(), "2048");
}
{
ConfigStatus status(SUCCESS, "");
status = size_config->Set("4097", true);
ASSERT_EQ(status.set_return, SetReturn::OUT_OF_RANGE);
ASSERT_EQ(size_config->Get(), "2048");
}
}
TEST_F(SizeConfigTest, negative_integer_test) {
int64_t size_value;
auto size_config = CreateSizeConfig("i", true, 1024, 4096, &size_value, 2048, nullptr, nullptr);
size_config->Init();
ConfigStatus status(SUCCESS, "");
status = size_config->Set("-3KB", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(size_config->Get(), "2048");
}
TEST_F(SizeConfigTest, invalid_bound_test) {
int64_t size_value;
auto size_config = CreateSizeConfig("i", true, 100, 0, &size_value, 50, nullptr, nullptr);
size_config->Init();
ConfigStatus status(SUCCESS, "");
status = size_config->Set("30", true);
ASSERT_EQ(status.set_return, SetReturn::OUT_OF_RANGE);
ASSERT_EQ(size_config->Get(), "50");
}
TEST_F(SizeConfigTest, invalid_unit_test) {
int64_t size_value;
auto size_config = CreateSizeConfig("i", true, 1024, 4096, &size_value, 2048, nullptr, nullptr);
size_config->Init();
ConfigStatus status(SUCCESS, "");
status = size_config->Set("1 TB", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(size_config->Get(), "2048");
}
TEST_F(SizeConfigTest, invalid_format_test) {
int64_t size_value;
auto size_config = CreateSizeConfig("i", true, 1024, 4096, &size_value, 2048, nullptr, nullptr);
size_config->Init();
{
ConfigStatus status(SUCCESS, "");
status = size_config->Set("a10GB", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(size_config->Get(), "2048");
}
{
ConfigStatus status(SUCCESS, "");
status = size_config->Set("200*0", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(size_config->Get(), "2048");
}
{
ConfigStatus status(SUCCESS, "");
status = size_config->Set("10AB", true);
ASSERT_EQ(status.set_return, SetReturn::INVALID);
ASSERT_EQ(size_config->Get(), "2048");
}
}
} // namespace milvus

62
proxy/src/config/ServerConfig.cpp Normal file
View File

@ -0,0 +1,62 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <iostream>
#include <sstream>
#include <string>
#include <unordered_set>
#include <vector>
#include "config/ServerConfig.h"
namespace milvus {
std::mutex config_mutex;
std::mutex&
GetConfigMutex() {
return config_mutex;
}
ServerConfig config;
std::vector<std::string>
ParsePreloadCollection(const std::string& str) {
std::stringstream ss(str);
std::vector<std::string> collections;
std::string collection;
while (std::getline(ss, collection, ',')) {
collections.push_back(collection);
}
return collections;
}
std::vector<int64_t>
ParseGPUDevices(const std::string& str) {
std::stringstream ss(str);
std::vector<int64_t> devices;
std::unordered_set<int64_t> device_set;
std::string device;
while (std::getline(ss, device, ',')) {
if (device.length() < 4) {
/* Invalid format string */
return {};
}
device_set.insert(std::stoll(device.substr(3)));
}
for (auto dev : device_set) devices.push_back(dev);
return devices;
}
} // namespace milvus

161
proxy/src/config/ServerConfig.h Normal file
View File

@ -0,0 +1,161 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <mutex>
#include <string>
#include <utility>
#include <vector>
#include "config/ConfigType.h"
namespace milvus {
extern std::mutex&
GetConfigMutex();
template <typename T>
class ConfigValue {
public:
explicit ConfigValue(T init_value) : value(std::move(init_value)) {
}
const T&
operator()() {
std::lock_guard<std::mutex> lock(GetConfigMutex());
return value;
}
public:
T value;
};
enum ClusterRole {
RW = 1,
RO,
};
const configEnum ClusterRoleMap{
{"rw", ClusterRole::RW},
{"ro", ClusterRole::RO},
};
enum SimdType {
AUTO = 1,
SSE,
AVX2,
AVX512,
};
const configEnum SimdMap{
{"auto", SimdType::AUTO},
{"sse", SimdType::SSE},
{"avx2", SimdType::AVX2},
{"avx512", SimdType::AVX512},
};
struct ServerConfig {
using String = ConfigValue<std::string>;
using Bool = ConfigValue<bool>;
using Integer = ConfigValue<int64_t>;
using Floating = ConfigValue<double>;
String version{"unknown"};
struct Cluster {
Bool enable{false};
Integer role{0};
} cluster;
struct General {
String timezone{"unknown"};
String meta_uri{"unknown"};
} general;
struct Network {
struct Bind {
String address{"unknown"};
Integer port{0};
} bind;
struct Http {
Bool enable{false};
Integer port{0};
} http;
} network;
struct Storage {
String path{"unknown"};
Integer auto_flush_interval{0};
} storage;
struct Cache {
Integer cache_size{0};
Floating cpu_cache_threshold{0.0};
Integer insert_buffer_size{0};
Bool cache_insert_data{false};
String preload_collection{"unknown"};
} cache;
struct Metric {
Bool enable{false};
String address{"unknown"};
Integer port{0};
} metric;
struct Engine {
Integer build_index_threshold{4096};
Integer search_combine_nq{0};
Integer use_blas_threshold{0};
Integer omp_thread_num{0};
Integer simd_type{0};
} engine;
struct GPU {
Bool enable{false};
Integer cache_size{0};
Floating cache_threshold{0.0};
Integer gpu_search_threshold{0};
String search_devices{"unknown"};
String build_index_devices{"unknown"};
} gpu;
struct Tracing {
String json_config_path{"unknown"};
} tracing;
struct WAL {
Bool enable{false};
Bool recovery_error_ignore{false};
Integer buffer_size{0};
String path{"unknown"};
} wal;
struct Logs {
String level{"unknown"};
struct Trace {
Bool enable{false};
} trace;
String path{"unknown"};
Integer max_log_file_size{0};
Integer log_rotate_num{0};
} logs;
};
extern ServerConfig config;
extern std::mutex _config_mutex;
std::vector<std::string>
ParsePreloadCollection(const std::string&);
std::vector<int64_t>
ParseGPUDevices(const std::string&);
} // namespace milvus

22
proxy/src/config/ServerConfigTest.cpp Normal file
View File

@ -0,0 +1,22 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <fiu-control.h>
#include <gtest/gtest.h>
#include "config/ServerConfig.h"
TEST(ServerConfigTest, parse_invalid_devices) {
fiu_init(0);
fiu_enable("ParseGPUDevices.invalid_format", 1, nullptr, 0);
auto collections = milvus::ParseGPUDevices("gpu0,gpu1");
ASSERT_EQ(collections.size(), 0);
}

9
proxy/src/grpc/cpp_gen.sh Executable file
View File

@ -0,0 +1,9 @@
#!/bin/bash
../../cmake-build/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --grpc_out=./gen-status --plugin=protoc-gen-grpc="../../cmake-build/thirdparty/grpc/grpc-build/grpc_cpp_plugin" status.proto
../../cmake-build/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --cpp_out=./gen-status status.proto
../../cmake-build/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --grpc_out=./gen-milvus --plugin=protoc-gen-grpc="../../cmake-build/thirdparty/grpc/grpc-build/grpc_cpp_plugin" milvus.proto
../../cmake-build/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --cpp_out=./gen-milvus milvus.proto

1093
proxy/src/grpc/gen-milvus/milvus.grpc.pb.cc Normal file
View File

File diff suppressed because it is too large Load Diff

4556
proxy/src/grpc/gen-milvus/milvus.grpc.pb.h Normal file
View File

File diff suppressed because it is too large Load Diff

16962
proxy/src/grpc/gen-milvus/milvus.pb.cc Normal file
View File

File diff suppressed because it is too large Load Diff

11538
proxy/src/grpc/gen-milvus/milvus.pb.h Normal file
View File

File diff suppressed because it is too large Load Diff

24
proxy/src/grpc/gen-status/status.grpc.pb.cc Normal file
View File

@ -0,0 +1,24 @@
// Generated by the gRPC C++ plugin.
// If you make any local change, they will be lost.
// source: status.proto
#include "status.pb.h"
#include "status.grpc.pb.h"
#include <functional>
#include <grpcpp/impl/codegen/async_stream.h>
#include <grpcpp/impl/codegen/async_unary_call.h>
#include <grpcpp/impl/codegen/channel_interface.h>
#include <grpcpp/impl/codegen/client_unary_call.h>
#include <grpcpp/impl/codegen/client_callback.h>
#include <grpcpp/impl/codegen/method_handler_impl.h>
#include <grpcpp/impl/codegen/rpc_service_method.h>
#include <grpcpp/impl/codegen/server_callback.h>
#include <grpcpp/impl/codegen/service_type.h>
#include <grpcpp/impl/codegen/sync_stream.h>
namespace milvus {
namespace grpc {
} // namespace milvus
} // namespace grpc

46
proxy/src/grpc/gen-status/status.grpc.pb.h Normal file
View File

@ -0,0 +1,46 @@
// Generated by the gRPC C++ plugin.
// If you make any local change, they will be lost.
// source: status.proto
#ifndef GRPC_status_2eproto__INCLUDED
#define GRPC_status_2eproto__INCLUDED
#include "status.pb.h"
#include <functional>
#include <grpcpp/impl/codegen/async_generic_service.h>
#include <grpcpp/impl/codegen/async_stream.h>
#include <grpcpp/impl/codegen/async_unary_call.h>
#include <grpcpp/impl/codegen/client_callback.h>
#include <grpcpp/impl/codegen/client_context.h>
#include <grpcpp/impl/codegen/completion_queue.h>
#include <grpcpp/impl/codegen/method_handler_impl.h>
#include <grpcpp/impl/codegen/proto_utils.h>
#include <grpcpp/impl/codegen/rpc_method.h>
#include <grpcpp/impl/codegen/server_callback.h>
#include <grpcpp/impl/codegen/server_context.h>
#include <grpcpp/impl/codegen/service_type.h>
#include <grpcpp/impl/codegen/status.h>
#include <grpcpp/impl/codegen/stub_options.h>
#include <grpcpp/impl/codegen/sync_stream.h>
namespace grpc_impl {
class CompletionQueue;
class ServerCompletionQueue;
class ServerContext;
} // namespace grpc_impl
namespace grpc {
namespace experimental {
template <typename RequestT, typename ResponseT>
class MessageAllocator;
} // namespace experimental
} // namespace grpc
namespace milvus {
namespace grpc {
} // namespace grpc
} // namespace milvus
#endif // GRPC_status_2eproto__INCLUDED

461
proxy/src/grpc/gen-status/status.pb.cc Normal file
View File

@ -0,0 +1,461 @@
// Generated by the protocol buffer compiler. DO NOT EDIT!
// source: status.proto
#include "status.pb.h"
#include <algorithm>
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/extension_set.h>
#include <google/protobuf/wire_format_lite.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/generated_message_reflection.h>
#include <google/protobuf/reflection_ops.h>
#include <google/protobuf/wire_format.h>
// @@protoc_insertion_point(includes)
#include <google/protobuf/port_def.inc>
namespace milvus {
namespace grpc {
class StatusDefaultTypeInternal {
public:
::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed<Status> _instance;
} _Status_default_instance_;
} // namespace grpc
} // namespace milvus
static void InitDefaultsscc_info_Status_status_2eproto() {
GOOGLE_PROTOBUF_VERIFY_VERSION;
{
void* ptr = &::milvus::grpc::_Status_default_instance_;
new (ptr) ::milvus::grpc::Status();
::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr);
}
::milvus::grpc::Status::InitAsDefaultInstance();
}
::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Status_status_2eproto =
{{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_Status_status_2eproto}, {}};
static ::PROTOBUF_NAMESPACE_ID::Metadata file_level_metadata_status_2eproto[1];
static const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* file_level_enum_descriptors_status_2eproto[1];
static constexpr ::PROTOBUF_NAMESPACE_ID::ServiceDescriptor const** file_level_service_descriptors_status_2eproto = nullptr;
const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_status_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
~0u, // no _has_bits_
PROTOBUF_FIELD_OFFSET(::milvus::grpc::Status, _internal_metadata_),
~0u, // no _extensions_
~0u, // no _oneof_case_
~0u, // no _weak_field_map_
PROTOBUF_FIELD_OFFSET(::milvus::grpc::Status, error_code_),
PROTOBUF_FIELD_OFFSET(::milvus::grpc::Status, reason_),
};
static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
{ 0, -1, sizeof(::milvus::grpc::Status)},
};
static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = {
reinterpret_cast<const ::PROTOBUF_NAMESPACE_ID::Message*>(&::milvus::grpc::_Status_default_instance_),
};
const char descriptor_table_protodef_status_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) =
"\n\014status.proto\022\013milvus.grpc\"D\n\006Status\022*\n"
"\nerror_code\030\001 \001(\0162\026.milvus.grpc.ErrorCod"
"e\022\016\n\006reason\030\002 \001(\t*\242\004\n\tErrorCode\022\013\n\007SUCCE"
"SS\020\000\022\024\n\020UNEXPECTED_ERROR\020\001\022\022\n\016CONNECT_FA"
"ILED\020\002\022\025\n\021PERMISSION_DENIED\020\003\022\031\n\025COLLECT"
"ION_NOT_EXISTS\020\004\022\024\n\020ILLEGAL_ARGUMENT\020\005\022\025"
"\n\021ILLEGAL_DIMENSION\020\007\022\026\n\022ILLEGAL_INDEX_T"
"YPE\020\010\022\033\n\027ILLEGAL_COLLECTION_NAME\020\t\022\020\n\014IL"
"LEGAL_TOPK\020\n\022\025\n\021ILLEGAL_ROWRECORD\020\013\022\025\n\021I"
"LLEGAL_VECTOR_ID\020\014\022\031\n\025ILLEGAL_SEARCH_RES"
"ULT\020\r\022\022\n\016FILE_NOT_FOUND\020\016\022\017\n\013META_FAILED"
"\020\017\022\020\n\014CACHE_FAILED\020\020\022\030\n\024CANNOT_CREATE_FO"
"LDER\020\021\022\026\n\022CANNOT_CREATE_FILE\020\022\022\030\n\024CANNOT"
"_DELETE_FOLDER\020\023\022\026\n\022CANNOT_DELETE_FILE\020\024"
"\022\025\n\021BUILD_INDEX_ERROR\020\025\022\021\n\rILLEGAL_NLIST"
"\020\026\022\027\n\023ILLEGAL_METRIC_TYPE\020\027\022\021\n\rOUT_OF_ME"
"MORY\020\030b\006proto3"
;
static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_status_2eproto_deps[1] = {
};
static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_status_2eproto_sccs[1] = {
&scc_info_Status_status_2eproto.base,
};
static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_status_2eproto_once;
static bool descriptor_table_status_2eproto_initialized = false;
const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_status_2eproto = {
&descriptor_table_status_2eproto_initialized, descriptor_table_protodef_status_2eproto, "status.proto", 654,
&descriptor_table_status_2eproto_once, descriptor_table_status_2eproto_sccs, descriptor_table_status_2eproto_deps, 1, 0,
schemas, file_default_instances, TableStruct_status_2eproto::offsets,
file_level_metadata_status_2eproto, 1, file_level_enum_descriptors_status_2eproto, file_level_service_descriptors_status_2eproto,
};
// Force running AddDescriptors() at dynamic initialization time.
static bool dynamic_init_dummy_status_2eproto = ( ::PROTOBUF_NAMESPACE_ID::internal::AddDescriptors(&descriptor_table_status_2eproto), true);
namespace milvus {
namespace grpc {
const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* ErrorCode_descriptor() {
::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&descriptor_table_status_2eproto);
return file_level_enum_descriptors_status_2eproto[0];
}
bool ErrorCode_IsValid(int value) {
switch (value) {
case 0:
case 1:
case 2:
case 3:
case 4:
case 5:
case 7:
case 8:
case 9:
case 10:
case 11:
case 12:
case 13:
case 14:
case 15:
case 16:
case 17:
case 18:
case 19:
case 20:
case 21:
case 22:
case 23:
case 24:
return true;
default:
return false;
}
}
// ===================================================================
void Status::InitAsDefaultInstance() {
}
class Status::_Internal {
public:
};
Status::Status()
: ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) {
SharedCtor();
// @@protoc_insertion_point(constructor:milvus.grpc.Status)
}
Status::Status(const Status& from)
: ::PROTOBUF_NAMESPACE_ID::Message(),
_internal_metadata_(nullptr) {
_internal_metadata_.MergeFrom(from._internal_metadata_);
reason_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from.reason().empty()) {
reason_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.reason_);
}
error_code_ = from.error_code_;
// @@protoc_insertion_point(copy_constructor:milvus.grpc.Status)
}
void Status::SharedCtor() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_Status_status_2eproto.base);
reason_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
error_code_ = 0;
}
Status::~Status() {
// @@protoc_insertion_point(destructor:milvus.grpc.Status)
SharedDtor();
}
void Status::SharedDtor() {
reason_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
void Status::SetCachedSize(int size) const {
_cached_size_.Set(size);
}
const Status& Status::default_instance() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_Status_status_2eproto.base);
return *internal_default_instance();
}
void Status::Clear() {
// @@protoc_insertion_point(message_clear_start:milvus.grpc.Status)
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
reason_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
error_code_ = 0;
_internal_metadata_.Clear();
}
#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
const char* Status::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) {
#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure
while (!ctx->Done(&ptr)) {
::PROTOBUF_NAMESPACE_ID::uint32 tag;
ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag);
CHK_(ptr);
switch (tag >> 3) {
// .milvus.grpc.ErrorCode error_code = 1;
case 1:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 8)) {
::PROTOBUF_NAMESPACE_ID::uint64 val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
CHK_(ptr);
set_error_code(static_cast<::milvus::grpc::ErrorCode>(val));
} else goto handle_unusual;
continue;
// string reason = 2;
case 2:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) {
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_reason(), ptr, ctx, "milvus.grpc.Status.reason");
CHK_(ptr);
} else goto handle_unusual;
continue;
default: {
handle_unusual:
if ((tag & 7) == 4 || tag == 0) {
ctx->SetLastTag(tag);
goto success;
}
ptr = UnknownFieldParse(tag, &_internal_metadata_, ptr, ctx);
CHK_(ptr != nullptr);
continue;
}
} // switch
} // while
success:
return ptr;
failure:
ptr = nullptr;
goto success;
#undef CHK_
}
#else // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
bool Status::MergePartialFromCodedStream(
::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) {
#define DO_(EXPRESSION) if (!PROTOBUF_PREDICT_TRUE(EXPRESSION)) goto failure
::PROTOBUF_NAMESPACE_ID::uint32 tag;
// @@protoc_insertion_point(parse_start:milvus.grpc.Status)
for (;;) {
::std::pair<::PROTOBUF_NAMESPACE_ID::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u);
tag = p.first;
if (!p.second) goto handle_unusual;
switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) {
// .milvus.grpc.ErrorCode error_code = 1;
case 1: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (8 & 0xFF)) {
int value = 0;
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive<
int, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_ENUM>(
input, &value)));
set_error_code(static_cast< ::milvus::grpc::ErrorCode >(value));
} else {
goto handle_unusual;
}
break;
}
// string reason = 2;
case 2: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) {
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString(
input, this->mutable_reason()));
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(
this->reason().data(), static_cast<int>(this->reason().length()),
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE,
"milvus.grpc.Status.reason"));
} else {
goto handle_unusual;
}
break;
}
default: {
handle_unusual:
if (tag == 0) {
goto success;
}
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SkipField(
input, tag, _internal_metadata_.mutable_unknown_fields()));
break;
}
}
}
success:
// @@protoc_insertion_point(parse_success:milvus.grpc.Status)
return true;
failure:
// @@protoc_insertion_point(parse_failure:milvus.grpc.Status)
return false;
#undef DO_
}
#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
void Status::SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const {
// @@protoc_insertion_point(serialize_start:milvus.grpc.Status)
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void) cached_has_bits;
// .milvus.grpc.ErrorCode error_code = 1;
if (this->error_code() != 0) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnum(
1, this->error_code(), output);
}
// string reason = 2;
if (this->reason().size() > 0) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(
this->reason().data(), static_cast<int>(this->reason().length()),
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE,
"milvus.grpc.Status.reason");
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased(
2, this->reason(), output);
}
if (_internal_metadata_.have_unknown_fields()) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields(
_internal_metadata_.unknown_fields(), output);
}
// @@protoc_insertion_point(serialize_end:milvus.grpc.Status)
}
::PROTOBUF_NAMESPACE_ID::uint8* Status::InternalSerializeWithCachedSizesToArray(
::PROTOBUF_NAMESPACE_ID::uint8* target) const {
// @@protoc_insertion_point(serialize_to_array_start:milvus.grpc.Status)
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void) cached_has_bits;
// .milvus.grpc.ErrorCode error_code = 1;
if (this->error_code() != 0) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray(
1, this->error_code(), target);
}
// string reason = 2;
if (this->reason().size() > 0) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(
this->reason().data(), static_cast<int>(this->reason().length()),
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE,
"milvus.grpc.Status.reason");
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray(
2, this->reason(), target);
}
if (_internal_metadata_.have_unknown_fields()) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray(
_internal_metadata_.unknown_fields(), target);
}
// @@protoc_insertion_point(serialize_to_array_end:milvus.grpc.Status)
return target;
}
size_t Status::ByteSizeLong() const {
// @@protoc_insertion_point(message_byte_size_start:milvus.grpc.Status)
size_t total_size = 0;
if (_internal_metadata_.have_unknown_fields()) {
total_size +=
::PROTOBUF_NAMESPACE_ID::internal::WireFormat::ComputeUnknownFieldsSize(
_internal_metadata_.unknown_fields());
}
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
// string reason = 2;
if (this->reason().size() > 0) {
total_size += 1 +
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
this->reason());
}
// .milvus.grpc.ErrorCode error_code = 1;
if (this->error_code() != 0) {
total_size += 1 +
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::EnumSize(this->error_code());
}
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
SetCachedSize(cached_size);
return total_size;
}
void Status::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {
// @@protoc_insertion_point(generalized_merge_from_start:milvus.grpc.Status)
GOOGLE_DCHECK_NE(&from, this);
const Status* source =
::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated<Status>(
&from);
if (source == nullptr) {
// @@protoc_insertion_point(generalized_merge_from_cast_fail:milvus.grpc.Status)
::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this);
} else {
// @@protoc_insertion_point(generalized_merge_from_cast_success:milvus.grpc.Status)
MergeFrom(*source);
}
}
void Status::MergeFrom(const Status& from) {
// @@protoc_insertion_point(class_specific_merge_from_start:milvus.grpc.Status)
GOOGLE_DCHECK_NE(&from, this);
_internal_metadata_.MergeFrom(from._internal_metadata_);
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void) cached_has_bits;
if (from.reason().size() > 0) {
reason_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.reason_);
}
if (from.error_code() != 0) {
set_error_code(from.error_code());
}
}
void Status::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {
// @@protoc_insertion_point(generalized_copy_from_start:milvus.grpc.Status)
if (&from == this) return;
Clear();
MergeFrom(from);
}
void Status::CopyFrom(const Status& from) {
// @@protoc_insertion_point(class_specific_copy_from_start:milvus.grpc.Status)
if (&from == this) return;
Clear();
MergeFrom(from);
}
bool Status::IsInitialized() const {
return true;
}
void Status::InternalSwap(Status* other) {
using std::swap;
_internal_metadata_.Swap(&other->_internal_metadata_);
reason_.Swap(&other->reason_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
GetArenaNoVirtual());
swap(error_code_, other->error_code_);
}
::PROTOBUF_NAMESPACE_ID::Metadata Status::GetMetadata() const {
return GetMetadataStatic();
}
// @@protoc_insertion_point(namespace_scope)
} // namespace grpc
} // namespace milvus
PROTOBUF_NAMESPACE_OPEN
template<> PROTOBUF_NOINLINE ::milvus::grpc::Status* Arena::CreateMaybeMessage< ::milvus::grpc::Status >(Arena* arena) {
return Arena::CreateInternal< ::milvus::grpc::Status >(arena);
}
PROTOBUF_NAMESPACE_CLOSE
// @@protoc_insertion_point(global_scope)
#include <google/protobuf/port_undef.inc>

360
proxy/src/grpc/gen-status/status.pb.h Normal file
View File

@ -0,0 +1,360 @@
// Generated by the protocol buffer compiler. DO NOT EDIT!
// source: status.proto
#ifndef GOOGLE_PROTOBUF_INCLUDED_status_2eproto
#define GOOGLE_PROTOBUF_INCLUDED_status_2eproto
#include <limits>
#include <string>
#include <google/protobuf/port_def.inc>
#if PROTOBUF_VERSION < 3009000
#error This file was generated by a newer version of protoc which is
#error incompatible with your Protocol Buffer headers. Please update
#error your headers.
#endif
#if 3009000 < PROTOBUF_MIN_PROTOC_VERSION
#error This file was generated by an older version of protoc which is
#error incompatible with your Protocol Buffer headers. Please
#error regenerate this file with a newer version of protoc.
#endif
#include <google/protobuf/port_undef.inc>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/arena.h>
#include <google/protobuf/arenastring.h>
#include <google/protobuf/generated_message_table_driven.h>
#include <google/protobuf/generated_message_util.h>
#include <google/protobuf/inlined_string_field.h>
#include <google/protobuf/metadata.h>
#include <google/protobuf/generated_message_reflection.h>
#include <google/protobuf/message.h>
#include <google/protobuf/repeated_field.h> // IWYU pragma: export
#include <google/protobuf/extension_set.h> // IWYU pragma: export
#include <google/protobuf/generated_enum_reflection.h>
#include <google/protobuf/unknown_field_set.h>
// @@protoc_insertion_point(includes)
#include <google/protobuf/port_def.inc>
#define PROTOBUF_INTERNAL_EXPORT_status_2eproto
PROTOBUF_NAMESPACE_OPEN
namespace internal {
class AnyMetadata;
} // namespace internal
PROTOBUF_NAMESPACE_CLOSE
// Internal implementation detail -- do not use these members.
struct TableStruct_status_2eproto {
static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTableField entries[]
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
static const ::PROTOBUF_NAMESPACE_ID::internal::AuxillaryParseTableField aux[]
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[1]
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
static const ::PROTOBUF_NAMESPACE_ID::internal::FieldMetadata field_metadata[];
static const ::PROTOBUF_NAMESPACE_ID::internal::SerializationTable serialization_table[];
static const ::PROTOBUF_NAMESPACE_ID::uint32 offsets[];
};
extern const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_status_2eproto;
namespace milvus {
namespace grpc {
class Status;
class StatusDefaultTypeInternal;
extern StatusDefaultTypeInternal _Status_default_instance_;
} // namespace grpc
} // namespace milvus
PROTOBUF_NAMESPACE_OPEN
template<> ::milvus::grpc::Status* Arena::CreateMaybeMessage<::milvus::grpc::Status>(Arena*);
PROTOBUF_NAMESPACE_CLOSE
namespace milvus {
namespace grpc {
enum ErrorCode : int {
SUCCESS = 0,
UNEXPECTED_ERROR = 1,
CONNECT_FAILED = 2,
PERMISSION_DENIED = 3,
COLLECTION_NOT_EXISTS = 4,
ILLEGAL_ARGUMENT = 5,
ILLEGAL_DIMENSION = 7,
ILLEGAL_INDEX_TYPE = 8,
ILLEGAL_COLLECTION_NAME = 9,
ILLEGAL_TOPK = 10,
ILLEGAL_ROWRECORD = 11,
ILLEGAL_VECTOR_ID = 12,
ILLEGAL_SEARCH_RESULT = 13,
FILE_NOT_FOUND = 14,
META_FAILED = 15,
CACHE_FAILED = 16,
CANNOT_CREATE_FOLDER = 17,
CANNOT_CREATE_FILE = 18,
CANNOT_DELETE_FOLDER = 19,
CANNOT_DELETE_FILE = 20,
BUILD_INDEX_ERROR = 21,
ILLEGAL_NLIST = 22,
ILLEGAL_METRIC_TYPE = 23,
OUT_OF_MEMORY = 24,
ErrorCode_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits<::PROTOBUF_NAMESPACE_ID::int32>::min(),
ErrorCode_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits<::PROTOBUF_NAMESPACE_ID::int32>::max()
};
bool ErrorCode_IsValid(int value);
constexpr ErrorCode ErrorCode_MIN = SUCCESS;
constexpr ErrorCode ErrorCode_MAX = OUT_OF_MEMORY;
constexpr int ErrorCode_ARRAYSIZE = ErrorCode_MAX + 1;
const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* ErrorCode_descriptor();
template<typename T>
inline const std::string& ErrorCode_Name(T enum_t_value) {
static_assert(::std::is_same<T, ErrorCode>::value ||
::std::is_integral<T>::value,
"Incorrect type passed to function ErrorCode_Name.");
return ::PROTOBUF_NAMESPACE_ID::internal::NameOfEnum(
ErrorCode_descriptor(), enum_t_value);
}
inline bool ErrorCode_Parse(
const std::string& name, ErrorCode* value) {
return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<ErrorCode>(
ErrorCode_descriptor(), name, value);
}
// ===================================================================
class Status :
public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.grpc.Status) */ {
public:
Status();
virtual ~Status();
Status(const Status& from);
Status(Status&& from) noexcept
: Status() {
*this = ::std::move(from);
}
inline Status& operator=(const Status& from) {
CopyFrom(from);
return *this;
}
inline Status& operator=(Status&& from) noexcept {
if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
if (this != &from) InternalSwap(&from);
} else {
CopyFrom(from);
}
return *this;
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() {
return GetDescriptor();
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() {
return GetMetadataStatic().descriptor;
}
static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() {
return GetMetadataStatic().reflection;
}
static const Status& default_instance();
static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY
static inline const Status* internal_default_instance() {
return reinterpret_cast<const Status*>(
&_Status_default_instance_);
}
static constexpr int kIndexInFileMessages =
0;
friend void swap(Status& a, Status& b) {
a.Swap(&b);
}
inline void Swap(Status* other) {
if (other == this) return;
InternalSwap(other);
}
// implements Message ----------------------------------------------
inline Status* New() const final {
return CreateMaybeMessage<Status>(nullptr);
}
Status* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final {
return CreateMaybeMessage<Status>(arena);
}
void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void CopyFrom(const Status& from);
void MergeFrom(const Status& from);
PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final;
bool IsInitialized() const final;
size_t ByteSizeLong() const final;
#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final;
#else
bool MergePartialFromCodedStream(
::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final;
#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
void SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final;
::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray(
::PROTOBUF_NAMESPACE_ID::uint8* target) const final;
int GetCachedSize() const final { return _cached_size_.Get(); }
private:
inline void SharedCtor();
inline void SharedDtor();
void SetCachedSize(int size) const final;
void InternalSwap(Status* other);
friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata;
static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() {
return "milvus.grpc.Status";
}
private:
inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const {
return nullptr;
}
inline void* MaybeArenaPtr() const {
return nullptr;
}
public:
::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final;
private:
static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() {
::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_status_2eproto);
return ::descriptor_table_status_2eproto.file_level_metadata[kIndexInFileMessages];
}
public:
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
enum : int {
kReasonFieldNumber = 2,
kErrorCodeFieldNumber = 1,
};
// string reason = 2;
void clear_reason();
const std::string& reason() const;
void set_reason(const std::string& value);
void set_reason(std::string&& value);
void set_reason(const char* value);
void set_reason(const char* value, size_t size);
std::string* mutable_reason();
std::string* release_reason();
void set_allocated_reason(std::string* reason);
// .milvus.grpc.ErrorCode error_code = 1;
void clear_error_code();
::milvus::grpc::ErrorCode error_code() const;
void set_error_code(::milvus::grpc::ErrorCode value);
// @@protoc_insertion_point(class_scope:milvus.grpc.Status)
private:
class _Internal;
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr reason_;
int error_code_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_status_2eproto;
};
// ===================================================================
// ===================================================================
#ifdef __GNUC__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wstrict-aliasing"
#endif // __GNUC__
// Status
// .milvus.grpc.ErrorCode error_code = 1;
inline void Status::clear_error_code() {
error_code_ = 0;
}
inline ::milvus::grpc::ErrorCode Status::error_code() const {
// @@protoc_insertion_point(field_get:milvus.grpc.Status.error_code)
return static_cast< ::milvus::grpc::ErrorCode >(error_code_);
}
inline void Status::set_error_code(::milvus::grpc::ErrorCode value) {
error_code_ = value;
// @@protoc_insertion_point(field_set:milvus.grpc.Status.error_code)
}
// string reason = 2;
inline void Status::clear_reason() {
reason_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
inline const std::string& Status::reason() const {
// @@protoc_insertion_point(field_get:milvus.grpc.Status.reason)
return reason_.GetNoArena();
}
inline void Status::set_reason(const std::string& value) {
reason_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value);
// @@protoc_insertion_point(field_set:milvus.grpc.Status.reason)
}
inline void Status::set_reason(std::string&& value) {
reason_.SetNoArena(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
// @@protoc_insertion_point(field_set_rvalue:milvus.grpc.Status.reason)
}
inline void Status::set_reason(const char* value) {
GOOGLE_DCHECK(value != nullptr);
reason_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
// @@protoc_insertion_point(field_set_char:milvus.grpc.Status.reason)
}
inline void Status::set_reason(const char* value, size_t size) {
reason_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
::std::string(reinterpret_cast<const char*>(value), size));
// @@protoc_insertion_point(field_set_pointer:milvus.grpc.Status.reason)
}
inline std::string* Status::mutable_reason() {
// @@protoc_insertion_point(field_mutable:milvus.grpc.Status.reason)
return reason_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
inline std::string* Status::release_reason() {
// @@protoc_insertion_point(field_release:milvus.grpc.Status.reason)
return reason_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
inline void Status::set_allocated_reason(std::string* reason) {
if (reason != nullptr) {
} else {
}
reason_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), reason);
// @@protoc_insertion_point(field_set_allocated:milvus.grpc.Status.reason)
}
#ifdef __GNUC__
#pragma GCC diagnostic pop
#endif // __GNUC__
// @@protoc_insertion_point(namespace_scope)
} // namespace grpc
} // namespace milvus
PROTOBUF_NAMESPACE_OPEN
template <> struct is_proto_enum< ::milvus::grpc::ErrorCode> : ::std::true_type {};
template <>
inline const EnumDescriptor* GetEnumDescriptor< ::milvus::grpc::ErrorCode>() {
return ::milvus::grpc::ErrorCode_descriptor();
}
PROTOBUF_NAMESPACE_CLOSE
// @@protoc_insertion_point(global_scope)
#include <google/protobuf/port_undef.inc>
#endif // GOOGLE_PROTOBUF_INCLUDED_GOOGLE_PROTOBUF_INCLUDED_status_2eproto

714
proxy/src/grpc/milvus.proto Normal file
View File

@ -0,0 +1,714 @@
syntax = "proto3";
import "status.proto";
package milvus.grpc;
/**
* @brief Field data type
*/
enum DataType {
NONE = 0;
BOOL = 1;
INT8 = 2;
INT16 = 3;
INT32 = 4;
INT64 = 5;
FLOAT = 10;
DOUBLE = 11;
STRING = 20;
VECTOR_BINARY = 100;
VECTOR_FLOAT = 101;
}
/**
* @brief General usage
*/
message KeyValuePair {
string key = 1;
string value = 2;
}
/**
* @brief Collection name
*/
message CollectionName {
string collection_name = 1;
}
/**
* @brief Collection name list
*/
message CollectionNameList {
Status status = 1;
repeated string collection_names = 2;
}
/**
* @brief Field name
*/
message FieldName {
string collection_name = 1;
string field_name = 2;
}
/**
* @brief Collection mapping
* @extra_params: key-value pair for extra parameters of the collection
* typically usage:
* extra_params["params"] = {segment_row_count: 1000000, auto_id: true}
* Note:
* the segment_row_count specify segment row count limit for merging
* the auto_id = true means entity id is auto-generated by milvus
*/
message Mapping {
Status status = 1;
string collection_name = 2;
repeated FieldParam fields = 3;
repeated KeyValuePair extra_params = 4;
}
/**
* @brief Collection mapping list
*/
message MappingList {
Status status = 1;
repeated Mapping mapping_list = 2;
}
/**
* @brief Parameters of partition
*/
message PartitionParam {
string collection_name = 1;
string tag = 2;
}
/**
* @brief Partition list
*/
message PartitionList {
Status status = 1;
repeated string partition_tag_array = 2;
}
/**
* @brief Vector row record
*/
message VectorRowRecord {
repeated float float_data = 1; //float vector data
bytes binary_data = 2; //binary vector data
}
/**
* @brief Attribute record
*/
message AttrRecord {
repeated int32 int32_value = 1;
repeated int64 int64_value = 2;
repeated float float_value = 3;
repeated double double_value = 4;
}
/**
* @brief Vector records
*/
message VectorRecord {
repeated VectorRowRecord records = 1;
}
/**
* @brief Field values
*/
message FieldValue {
string field_name = 1;
DataType type = 2;
AttrRecord attr_record = 3;
VectorRecord vector_record = 4;
}
/**
* @brief Parameters for insert action
*/
message InsertParam {
string collection_name = 1;
repeated FieldValue fields = 2;
repeated int64 entity_id_array = 3; //optional
string partition_tag = 4;
repeated KeyValuePair extra_params = 5;
}
/**
* @brief Entity ids
*/
message EntityIds {
Status status = 1;
repeated int64 entity_id_array = 2;
}
/**
* @brief Search vector parameters
*/
message VectorParam {
string json = 1;
VectorRecord row_record = 2;
}
/**
* @brief Parameters for search action
* @dsl example:
* {
* "query": {
* "bool": {
* "must": [
* {
* "must":[
* {
* "should": [
* {
* "term": {
* "gender": ["male"]
* }
* },
* {
* "range": {
* "height": {"gte": "170.0", "lte": "180.0"}
* }
* }
* ]
* },
* {
* "must_not": [
* {
* "term": {
* "age": [20, 21, 22, 23, 24, 25]
* }
* },
* {
* "Range": {
* "weight": {"lte": "100"}
* }
* }
* ]
* }
* ]
* },
* {
* "must": [
* {
* "vector": {
* "face_img": {
* "topk": 10,
* "metric_type": "L2",
* "query": [],
* "params": {
* "nprobe": 10
* }
* }
* }
* }
* ]
* }
* ]
* }
* },
* "fields": ["age", "face_img"]
* }
*/
message SearchParam {
string collection_name = 1;
repeated string partition_tag_array = 2;
repeated VectorParam vector_param = 3;
string dsl = 4;
repeated KeyValuePair extra_params = 5;
}
/**
* @brief Parameters for searching in segments
*/
message SearchInSegmentParam {
repeated string file_id_array = 1;
SearchParam search_param = 2;
}
/**
* @brief Entities
*/
message Entities {
Status status = 1;
repeated int64 ids = 2;
repeated bool valid_row = 3;
repeated FieldValue fields = 4;
}
/**
* @brief Query result
*/
message QueryResult {
Status status = 1;
Entities entities = 2;
int64 row_num = 3;
repeated float scores = 4;
repeated float distances = 5;
repeated KeyValuePair extra_params = 6;
}
/**
* @brief Server string Reply
*/
message StringReply {
Status status = 1;
string string_reply = 2;
}
/**
* @brief Server bool Reply
*/
message BoolReply {
Status status = 1;
bool bool_reply = 2;
}
/**
* @brief Return collection row count
*/
message CollectionRowCount {
Status status = 1;
int64 collection_row_count = 2;
}
/**
* @brief Server command parameters
*/
message Command {
string cmd = 1;
}
/**
* @brief Index params
* @collection_name: target collection
* @field_name: target field
* @index_name: a name for index provided by user, unique within this field
* @extra_params: index parameters in json format
* for vector field:
* extra_params["index_type"] = one of the values: FLAT, IVF_LAT, IVF_SQ8, NSGMIX, IVFSQ8H,
* PQ, HNSW, HNSW_SQ8NM, ANNOY
* extra_params["metric_type"] = one of the values: L2, IP, HAMMING, JACCARD, TANIMOTO
* SUBSTRUCTURE, SUPERSTRUCTURE
* extra_params["params"] = extra parameters for index, for example ivflat: {nlist: 2048}
* for structured field:
* extra_params["index_type"] = one of the values: SORTED
*/
message IndexParam {
Status status = 1;
string collection_name = 2;
string field_name = 3;
string index_name = 4;
repeated KeyValuePair extra_params = 5;
}
/**
* @brief Parameters for flush action
*/
message FlushParam {
repeated string collection_name_array = 1;
}
/**
* @brief Parameters for flush action
*/
message CompactParam {
string collection_name = 1;
double threshold = 2;
}
/**
* @brief Parameters for deleting entities by id
*/
message DeleteByIDParam {
string collection_name = 1;
repeated int64 id_array = 2;
}
/**
* @brief Return collection stats
* @json_info: collection stats in json format, typically, the format is like:
* {
* row_count: xxx,
* data_size: xxx,
* partitions: [
* {
* tag: xxx,
* id: xxx,
* row_count: xxx,
* data_size: xxx,
* segments: [
* {
* id: xxx,
* row_count: xxx,
* data_size: xxx,
* files: [
* {
* field: xxx,
* name: xxx,
* index_type: xxx,
* path: xxx,
* data_size: xxx,
* }
* ]
* }
* ]
* }
* ]
* }
*/
message CollectionInfo {
Status status = 1;
string json_info = 2;
}
/**
* @brief Parameters for returning entities id of a segment
*/
message GetEntityIDsParam {
string collection_name = 1;
int64 segment_id = 2;
}
/**
* @brief Entities identity
*/
message EntityIdentity {
string collection_name = 1;
repeated int64 id_array = 2;
repeated string field_names = 3;
}
/********************************************SearchPB interface***************************************************/
/**
* @brief Vector field parameters
*/
message VectorFieldParam {
int64 dimension = 1;
}
/**
* @brief Field type
*/
message FieldType {
oneof value {
DataType data_type = 1;
VectorFieldParam vector_param = 2;
}
}
/**
* @brief Field parameters
*/
message FieldParam {
uint64 id = 1;
string name = 2;
DataType type = 3;
repeated KeyValuePair index_params = 4;
repeated KeyValuePair extra_params = 5;
}
/**
* @brief Vector field record
*/
message VectorFieldRecord {
repeated VectorRowRecord value = 1;
}
///////////////////////////////////////////////////////////////////
message TermQuery {
string field_name = 1;
repeated int64 int_value = 2;
repeated double double_value = 3;
int64 value_num = 4;
float boost = 5;
repeated KeyValuePair extra_params = 6;
}
enum CompareOperator {
LT = 0;
LTE = 1;
EQ = 2;
GT = 3;
GTE = 4;
NE = 5;
}
message CompareExpr {
CompareOperator operator = 1;
string operand = 2;
}
message RangeQuery {
string field_name = 1;
repeated CompareExpr operand = 2;
float boost = 3;
repeated KeyValuePair extra_params = 4;
}
message VectorQuery {
string field_name = 1;
float query_boost = 2;
repeated VectorRowRecord records = 3;
int64 topk = 4;
repeated KeyValuePair extra_params = 5;
}
enum Occur {
INVALID = 0;
MUST = 1;
SHOULD = 2;
MUST_NOT = 3;
}
message BooleanQuery {
Occur occur = 1;
repeated GeneralQuery general_query = 2;
}
message GeneralQuery {
oneof query {
BooleanQuery boolean_query = 1;
TermQuery term_query = 2;
RangeQuery range_query = 3;
VectorQuery vector_query = 4;
}
}
message SearchParamPB {
string collection_name = 1;
repeated string partition_tag_array = 2;
GeneralQuery general_query = 3;
repeated KeyValuePair extra_params = 4;
}
service MilvusService {
/**
* @brief This method is used to create collection
*
* @param CollectionSchema, use to provide collection information to be created.
*
* @return Status
*/
rpc CreateCollection(Mapping) returns (Status){}
/**
* @brief This method is used to test collection existence.
*
* @param CollectionName, collection name is going to be tested.
*
* @return BoolReply
*/
rpc HasCollection(CollectionName) returns (BoolReply) {}
/**
* @brief This method is used to get collection schema.
*
* @param CollectionName, target collection name.
*
* @return CollectionSchema
*/
rpc DescribeCollection(CollectionName) returns (Mapping) {}
/**
* @brief This method is used to get collection schema.
*
* @param CollectionName, target collection name.
*
* @return CollectionRowCount
*/
rpc CountCollection(CollectionName) returns (CollectionRowCount) {}
/**
* @brief This method is used to list all collections.
*
* @param Command, dummy parameter.
*
* @return CollectionNameList
*/
rpc ShowCollections(Command) returns (CollectionNameList) {}
/**
* @brief This method is used to get collection detail information.
*
* @param CollectionName, target collection name.
*
* @return CollectionInfo
*/
rpc ShowCollectionInfo(CollectionName) returns (CollectionInfo) {}
/**
* @brief This method is used to delete collection.
*
* @param CollectionName, collection name is going to be deleted.
*
* @return Status
*/
rpc DropCollection(CollectionName) returns (Status) {}
/**
* @brief This method is used to build index by collection in sync mode.
*
* @param IndexParam, index paramters.
*
* @return Status
*/
rpc CreateIndex(IndexParam) returns (Status) {}
/**
* @brief This method is used to describe index
*
* @param IndexParam, target index.
*
* @return IndexParam
*/
rpc DescribeIndex(IndexParam) returns (IndexParam) {}
/**
* @brief This method is used to drop index
*
* @param IndexParam, target field. if the IndexParam.field_name is empty, will drop all index of the collection
*
* @return Status
*/
rpc DropIndex(IndexParam) returns (Status) {}
/**
* @brief This method is used to create partition
*
* @param PartitionParam, partition parameters.
*
* @return Status
*/
rpc CreatePartition(PartitionParam) returns (Status) {}
/**
* @brief This method is used to test partition existence.
*
* @param PartitionParam, target partition.
*
* @return BoolReply
*/
rpc HasPartition(PartitionParam) returns (BoolReply) {}
/**
* @brief This method is used to show partition information
*
* @param CollectionName, target collection name.
*
* @return PartitionList
*/
rpc ShowPartitions(CollectionName) returns (PartitionList) {}
/**
* @brief This method is used to drop partition
*
* @param PartitionParam, target partition.
*
* @return Status
*/
rpc DropPartition(PartitionParam) returns (Status) {}
/**
* @brief This method is used to add vector array to collection.
*
* @param InsertParam, insert parameters.
*
* @return VectorIds
*/
rpc Insert(InsertParam) returns (EntityIds) {}
/**
* @brief This method is used to get entities data by id array.
*
* @param EntitiesIdentity, target entity id array.
*
* @return EntitiesData
*/
rpc GetEntityByID(EntityIdentity) returns (Entities) {}
/**
* @brief This method is used to get vector ids from a segment
*
* @param GetVectorIDsParam, target collection and segment
*
* @return VectorIds
*/
rpc GetEntityIDs(GetEntityIDsParam) returns (EntityIds) {}
/**
* @brief This method is used to query vector in collection.
*
* @param SearchParam, search parameters.
*
* @return KQueryResult
*/
rpc Search(SearchParam) returns (QueryResult) {}
/**
* @brief This method is used to query vector in specified files.
*
* @param SearchInSegmentParam, target segments to search.
*
* @return TopKQueryResult
*/
rpc SearchInSegment(SearchInSegmentParam) returns (QueryResult) {}
/**
* @brief This method is used to give the server status.
*
* @param Command, command string
*
* @return StringReply
*/
rpc Cmd(Command) returns (StringReply) {}
/**
* @brief This method is used to delete vector by id
*
* @param DeleteByIDParam, delete parameters.
*
* @return status
*/
rpc DeleteByID(DeleteByIDParam) returns (Status) {}
/**
* @brief This method is used to preload collection
*
* @param CollectionName, target collection name.
*
* @return Status
*/
rpc PreloadCollection(CollectionName) returns (Status) {}
/**
* @brief This method is used to flush buffer into storage.
*
* @param FlushParam, flush parameters
*
* @return Status
*/
rpc Flush(FlushParam) returns (Status) {}
/**
* @brief This method is used to compact collection
*
* @param CompactParam, compact parameters
*
* @return Status
*/
rpc Compact(CompactParam) returns (Status) {}
/********************************New Interface********************************************/
rpc SearchPB(SearchParamPB) returns (QueryResult) {}
}

35
proxy/src/grpc/status.proto Normal file
View File

@ -0,0 +1,35 @@
syntax = "proto3";
package milvus.grpc;
enum ErrorCode {
SUCCESS = 0;
UNEXPECTED_ERROR = 1;
CONNECT_FAILED = 2;
PERMISSION_DENIED = 3;
COLLECTION_NOT_EXISTS = 4;
ILLEGAL_ARGUMENT = 5;
ILLEGAL_DIMENSION = 7;
ILLEGAL_INDEX_TYPE = 8;
ILLEGAL_COLLECTION_NAME = 9;
ILLEGAL_TOPK = 10;
ILLEGAL_ROWRECORD = 11;
ILLEGAL_VECTOR_ID = 12;
ILLEGAL_SEARCH_RESULT = 13;
FILE_NOT_FOUND = 14;
META_FAILED = 15;
CACHE_FAILED = 16;
CANNOT_CREATE_FOLDER = 17;
CANNOT_CREATE_FILE = 18;
CANNOT_DELETE_FOLDER = 19;
CANNOT_DELETE_FILE = 20;
BUILD_INDEX_ERROR = 21;
ILLEGAL_NLIST = 22;
ILLEGAL_METRIC_TYPE = 23;
OUT_OF_MEMORY = 24;
}
message Status {
ErrorCode error_code = 1;
string reason = 2;
}

21
proxy/src/log/CMakeLists.txt Normal file
View File

@ -0,0 +1,21 @@
#-------------------------------------------------------------------------------
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
set(LOG_FILES ${MILVUS_ENGINE_SRC}/log/Log.cpp
${MILVUS_ENGINE_SRC}/log/Log.h
${MILVUS_ENGINE_SRC}/log/LogMgr.cpp
${MILVUS_ENGINE_SRC}/log/LogMgr.h
${MILVUS_THIRDPARTY_SRC}/easyloggingpp/easylogging++.cc
${MILVUS_THIRDPARTY_SRC}/easyloggingpp/easylogging++.h
)
add_library(log STATIC ${LOG_FILES})

54
proxy/src/log/Log.cpp Normal file
View File

@ -0,0 +1,54 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "log/Log.h"
#include <cstdarg>
#include <cstdio>
#include <memory>
#include <string>
namespace milvus {
std::string
LogOut(const char* pattern, ...) {
size_t len = strnlen(pattern, 1024) + 256;
auto str_p = std::make_unique<char[]>(len);
memset(str_p.get(), 0, len);
va_list vl;
va_start(vl, pattern);
vsnprintf(str_p.get(), len, pattern, vl); // NOLINT
va_end(vl);
return std::string(str_p.get());
}
void
SetThreadName(const std::string& name) {
// Note: the name cannot exceed 16 bytes
pthread_setname_np(pthread_self(), name.c_str());
}
std::string
GetThreadName() {
std::string thread_name = "unamed";
char name[16];
size_t len = 16;
auto err = pthread_getname_np(pthread_self(), name, len);
if (not err) {
thread_name = name;
}
return thread_name;
}
} // namespace milvus

135
proxy/src/log/Log.h Normal file
View File

@ -0,0 +1,135 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <string>
#include "easyloggingpp/easylogging++.h"
namespace milvus {
/*
* Please use LOG_MODULE_LEVEL_C macro in member function of class
* and LOG_MODULE_LEVEL_ macro in other functions.
*/
/////////////////////////////////////////////////////////////////////////////////////////////////
#define SERVER_MODULE_NAME "SERVER"
#define SERVER_MODULE_CLASS_FUNCTION \
LogOut("[%s][%s::%s][%s] ", SERVER_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str())
#define SERVER_MODULE_FUNCTION LogOut("[%s][%s][%s] ", SERVER_MODULE_NAME, __FUNCTION__, GetThreadName().c_str())
#define LOG_SERVER_TRACE_C LOG(TRACE) << SERVER_MODULE_CLASS_FUNCTION
#define LOG_SERVER_DEBUG_C LOG(DEBUG) << SERVER_MODULE_CLASS_FUNCTION
#define LOG_SERVER_INFO_C LOG(INFO) << SERVER_MODULE_CLASS_FUNCTION
#define LOG_SERVER_WARNING_C LOG(WARNING) << SERVER_MODULE_CLASS_FUNCTION
#define LOG_SERVER_ERROR_C LOG(ERROR) << SERVER_MODULE_CLASS_FUNCTION
#define LOG_SERVER_FATAL_C LOG(FATAL) << SERVER_MODULE_CLASS_FUNCTION
#define LOG_SERVER_TRACE_ LOG(TRACE) << SERVER_MODULE_FUNCTION
#define LOG_SERVER_DEBUG_ LOG(DEBUG) << SERVER_MODULE_FUNCTION
#define LOG_SERVER_INFO_ LOG(INFO) << SERVER_MODULE_FUNCTION
#define LOG_SERVER_WARNING_ LOG(WARNING) << SERVER_MODULE_FUNCTION
#define LOG_SERVER_ERROR_ LOG(ERROR) << SERVER_MODULE_FUNCTION
#define LOG_SERVER_FATAL_ LOG(FATAL) << SERVER_MODULE_FUNCTION
/////////////////////////////////////////////////////////////////////////////////////////////////
#define ENGINE_MODULE_NAME "ENGINE"
#define ENGINE_MODULE_CLASS_FUNCTION \
LogOut("[%s][%s::%s][%s] ", ENGINE_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str())
#define ENGINE_MODULE_FUNCTION LogOut("[%s][%s][%s] ", ENGINE_MODULE_NAME, __FUNCTION__, GetThreadName().c_str())
#define LOG_ENGINE_TRACE_C LOG(TRACE) << ENGINE_MODULE_CLASS_FUNCTION
#define LOG_ENGINE_DEBUG_C LOG(DEBUG) << ENGINE_MODULE_CLASS_FUNCTION
#define LOG_ENGINE_INFO_C LOG(INFO) << ENGINE_MODULE_CLASS_FUNCTION
#define LOG_ENGINE_WARNING_C LOG(WARNING) << ENGINE_MODULE_CLASS_FUNCTION
#define LOG_ENGINE_ERROR_C LOG(ERROR) << ENGINE_MODULE_CLASS_FUNCTION
#define LOG_ENGINE_FATAL_C LOG(FATAL) << ENGINE_MODULE_CLASS_FUNCTION
#define LOG_ENGINE_TRACE_ LOG(TRACE) << ENGINE_MODULE_FUNCTION
#define LOG_ENGINE_DEBUG_ LOG(DEBUG) << ENGINE_MODULE_FUNCTION
#define LOG_ENGINE_INFO_ LOG(INFO) << ENGINE_MODULE_FUNCTION
#define LOG_ENGINE_WARNING_ LOG(WARNING) << ENGINE_MODULE_FUNCTION
#define LOG_ENGINE_ERROR_ LOG(ERROR) << ENGINE_MODULE_FUNCTION
#define LOG_ENGINE_FATAL_ LOG(FATAL) << ENGINE_MODULE_FUNCTION
/////////////////////////////////////////////////////////////////////////////////////////////////
#define WRAPPER_MODULE_NAME "WRAPPER"
#define WRAPPER_MODULE_CLASS_FUNCTION \
LogOut("[%s][%s::%s][%s] ", WRAPPER_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str())
#define WRAPPER_MODULE_FUNCTION LogOut("[%s][%s][%s] ", WRAPPER_MODULE_NAME, __FUNCTION__, GetThreadName().c_str())
#define LOG_WRAPPER_TRACE_C LOG(TRACE) << WRAPPER_MODULE_CLASS_FUNCTION
#define LOG_WRAPPER_DEBUG_C LOG(DEBUG) << WRAPPER_MODULE_CLASS_FUNCTION
#define LOG_WRAPPER_INFO_C LOG(INFO) << WRAPPER_MODULE_CLASS_FUNCTION
#define LOG_WRAPPER_WARNING_C LOG(WARNING) << WRAPPER_MODULE_CLASS_FUNCTION
#define LOG_WRAPPER_ERROR_C LOG(ERROR) << WRAPPER_MODULE_CLASS_FUNCTION
#define LOG_WRAPPER_FATAL_C LOG(FATAL) << WRAPPER_MODULE_CLASS_FUNCTION
#define LOG_WRAPPER_TRACE_ LOG(TRACE) << WRAPPER_MODULE_FUNCTION
#define LOG_WRAPPER_DEBUG_ LOG(DEBUG) << WRAPPER_MODULE_FUNCTION
#define LOG_WRAPPER_INFO_ LOG(INFO) << WRAPPER_MODULE_FUNCTION
#define LOG_WRAPPER_WARNING_ LOG(WARNING) << WRAPPER_MODULE_FUNCTION
#define LOG_WRAPPER_ERROR_ LOG(ERROR) << WRAPPER_MODULE_FUNCTION
#define LOG_WRAPPER_FATAL_ LOG(FATAL) << WRAPPER_MODULE_FUNCTION
/////////////////////////////////////////////////////////////////////////////////////////////////
#define STORAGE_MODULE_NAME "STORAGE"
#define STORAGE_MODULE_CLASS_FUNCTION \
LogOut("[%s][%s::%s][%s] ", STORAGE_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str())
#define STORAGE_MODULE_FUNCTION LogOut("[%s][%s][%s] ", STORAGE_MODULE_NAME, __FUNCTION__, GetThreadName().c_str())
#define LOG_STORAGE_TRACE_C LOG(TRACE) << STORAGE_MODULE_CLASS_FUNCTION
#define LOG_STORAGE_DEBUG_C LOG(DEBUG) << STORAGE_MODULE_CLASS_FUNCTION
#define LOG_STORAGE_INFO_C LOG(INFO) << STORAGE_MODULE_CLASS_FUNCTION
#define LOG_STORAGE_WARNING_C LOG(WARNING) << STORAGE_MODULE_CLASS_FUNCTION
#define LOG_STORAGE_ERROR_C LOG(ERROR) << STORAGE_MODULE_CLASS_FUNCTION
#define LOG_STORAGE_FATAL_C LOG(FATAL) << STORAGE_MODULE_CLASS_FUNCTION
#define LOG_STORAGE_TRACE_ LOG(TRACE) << STORAGE_MODULE_FUNCTION
#define LOG_STORAGE_DEBUG_ LOG(DEBUG) << STORAGE_MODULE_FUNCTION
#define LOG_STORAGE_INFO_ LOG(INFO) << STORAGE_MODULE_FUNCTION
#define LOG_STORAGE_WARNING_ LOG(WARNING) << STORAGE_MODULE_FUNCTION
#define LOG_STORAGE_ERROR_ LOG(ERROR) << STORAGE_MODULE_FUNCTION
#define LOG_STORAGE_FATAL_ LOG(FATAL) << STORAGE_MODULE_FUNCTION
/////////////////////////////////////////////////////////////////////////////////////////////////
#define WAL_MODULE_NAME "WAL"
#define WAL_MODULE_CLASS_FUNCTION \
LogOut("[%s][%s::%s][%s] ", WAL_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str())
#define WAL_MODULE_FUNCTION LogOut("[%s][%s][%s] ", WAL_MODULE_NAME, __FUNCTION__, GetThreadName().c_str())
#define LOG_WAL_TRACE_C LOG(TRACE) << WAL_MODULE_CLASS_FUNCTION
#define LOG_WAL_DEBUG_C LOG(DEBUG) << WAL_MODULE_CLASS_FUNCTION
#define LOG_WAL_INFO_C LOG(INFO) << WAL_MODULE_CLASS_FUNCTION
#define LOG_WAL_WARNING_C LOG(WARNING) << WAL_MODULE_CLASS_FUNCTION
#define LOG_WAL_ERROR_C LOG(ERROR) << WAL_MODULE_CLASS_FUNCTION
#define LOG_WAL_FATAL_C LOG(FATAL) << WAL_MODULE_CLASS_FUNCTION
#define LOG_WAL_TRACE_ LOG(TRACE) << WAL_MODULE_FUNCTION
#define LOG_WAL_DEBUG_ LOG(DEBUG) << WAL_MODULE_FUNCTION
#define LOG_WAL_INFO_ LOG(INFO) << WAL_MODULE_FUNCTION
#define LOG_WAL_WARNING_ LOG(WARNING) << WAL_MODULE_FUNCTION
#define LOG_WAL_ERROR_ LOG(ERROR) << WAL_MODULE_FUNCTION
#define LOG_WAL_FATAL_ LOG(FATAL) << WAL_MODULE_FUNCTION
/////////////////////////////////////////////////////////////////////////////////////////////////////
std::string
LogOut(const char* pattern, ...);
void
SetThreadName(const std::string& name);
std::string
GetThreadName();
} // namespace milvus

245
proxy/src/log/LogMgr.cpp Normal file
View File

@ -0,0 +1,245 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <libgen.h>
#include <cctype>
#include <string>
#include <unordered_map>
#include <boost/filesystem.hpp>
#include "config/ServerConfig.h"
#include "log/LogMgr.h"
#include "utils/Status.h"
namespace milvus {
namespace {
static int global_idx = 0;
static int debug_idx = 0;
static int warning_idx = 0;
static int trace_idx = 0;
static int error_idx = 0;
static int fatal_idx = 0;
static int64_t logs_delete_exceeds = 1;
static bool enable_log_delete = false;
/* module constant */
const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN = 536870912; /* 512 MB */
const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX = 4294967296; /* 4 GB */
const int64_t CONFIG_LOGS_LOG_ROTATE_NUM_MIN = 0;
const int64_t CONFIG_LOGS_LOG_ROTATE_NUM_MAX = 1024;
} // namespace
// TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename
void
RolloutHandler(const char* filename, std::size_t size, el::Level level) {
char* dirc = strdup(filename);
char* basec = strdup(filename);
char* dir = dirname(dirc);
char* base = basename(basec);
std::string s(base);
std::string list[] = {"\\", " ", "\'", "\"", "*", "\?", "{", "}", ";", "<",
">", "|", "^", "&", "$", "#", "!", "`", "~"};
std::string::size_type position;
for (auto substr : list) {
position = 0;
while ((position = s.find_first_of(substr, position)) != std::string::npos) {
s.insert(position, "\\");
position += 2;
}
}
std::string m(std::string(dir) + "/" + s);
s = m;
try {
switch (level) {
case el::Level::Debug: {
s.append("." + std::to_string(++debug_idx));
rename(m.c_str(), s.c_str());
if (enable_log_delete && debug_idx - logs_delete_exceeds > 0) {
std::string to_delete = m + "." + std::to_string(debug_idx - logs_delete_exceeds);
// std::cout << "remote " << to_delete << std::endl;
boost::filesystem::remove(to_delete);
}
break;
}
case el::Level::Warning: {
s.append("." + std::to_string(++warning_idx));
rename(m.c_str(), s.c_str());
if (enable_log_delete && warning_idx - logs_delete_exceeds > 0) {
std::string to_delete = m + "." + std::to_string(warning_idx - logs_delete_exceeds);
boost::filesystem::remove(to_delete);
}
break;
}
case el::Level::Trace: {
s.append("." + std::to_string(++trace_idx));
rename(m.c_str(), s.c_str());
if (enable_log_delete && trace_idx - logs_delete_exceeds > 0) {
std::string to_delete = m + "." + std::to_string(trace_idx - logs_delete_exceeds);
boost::filesystem::remove(to_delete);
}
break;
}
case el::Level::Error: {
s.append("." + std::to_string(++error_idx));
rename(m.c_str(), s.c_str());
if (enable_log_delete && error_idx - logs_delete_exceeds > 0) {
std::string to_delete = m + "." + std::to_string(error_idx - logs_delete_exceeds);
boost::filesystem::remove(to_delete);
}
break;
}
case el::Level::Fatal: {
s.append("." + std::to_string(++fatal_idx));
rename(m.c_str(), s.c_str());
if (enable_log_delete && fatal_idx - logs_delete_exceeds > 0) {
std::string to_delete = m + "." + std::to_string(fatal_idx - logs_delete_exceeds);
boost::filesystem::remove(to_delete);
}
break;
}
default: {
s.append("." + std::to_string(++global_idx));
rename(m.c_str(), s.c_str());
if (enable_log_delete && global_idx - logs_delete_exceeds > 0) {
std::string to_delete = m + "." + std::to_string(global_idx - logs_delete_exceeds);
boost::filesystem::remove(to_delete);
}
break;
}
}
} catch (const std::exception& exc) {
std::cerr << exc.what() << ". Exception throws from RolloutHandler." << std::endl;
}
}
Status
LogMgr::InitLog(bool trace_enable, const std::string& level, const std::string& logs_path, int64_t max_log_file_size,
int64_t delete_exceeds) {
std::unordered_map<std::string, int64_t> level_to_int{
{"debug", 5}, {"info", 4}, {"warning", 3}, {"error", 2}, {"fatal", 1},
};
bool debug_enable = false;
bool info_enable = false;
bool warning_enable = false;
bool error_enable = false;
bool fatal_enable = false;
switch (level_to_int[level]) {
case 5:
debug_enable = true;
case 4:
info_enable = true;
case 3:
warning_enable = true;
case 2:
error_enable = true;
case 1:
fatal_enable = true;
break;
default:
return Status(SERVER_UNEXPECTED_ERROR, "invalid log level");
}
el::Configurations defaultConf;
defaultConf.setToDefault();
defaultConf.setGlobally(el::ConfigurationType::Format, "[%datetime][%level]%msg");
defaultConf.setGlobally(el::ConfigurationType::ToFile, "true");
defaultConf.setGlobally(el::ConfigurationType::ToStandardOutput, "false");
defaultConf.setGlobally(el::ConfigurationType::SubsecondPrecision, "3");
defaultConf.setGlobally(el::ConfigurationType::PerformanceTracking, "false");
std::string logs_reg_path = logs_path.rfind('/') == logs_path.length() - 1 ? logs_path : logs_path + "/";
std::string global_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-global.log";
defaultConf.set(el::Level::Global, el::ConfigurationType::Filename, global_log_path.c_str());
defaultConf.set(el::Level::Global, el::ConfigurationType::Enabled, "true");
std::string info_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-info.log";
defaultConf.set(el::Level::Info, el::ConfigurationType::Filename, info_log_path.c_str());
if (info_enable) {
defaultConf.set(el::Level::Info, el::ConfigurationType::Enabled, "true");
} else {
defaultConf.set(el::Level::Info, el::ConfigurationType::Enabled, "false");
}
std::string debug_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-debug.log";
defaultConf.set(el::Level::Debug, el::ConfigurationType::Filename, debug_log_path.c_str());
if (debug_enable) {
defaultConf.set(el::Level::Debug, el::ConfigurationType::Enabled, "true");
} else {
defaultConf.set(el::Level::Debug, el::ConfigurationType::Enabled, "false");
}
std::string warning_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-warning.log";
defaultConf.set(el::Level::Warning, el::ConfigurationType::Filename, warning_log_path.c_str());
if (warning_enable) {
defaultConf.set(el::Level::Warning, el::ConfigurationType::Enabled, "true");
} else {
defaultConf.set(el::Level::Warning, el::ConfigurationType::Enabled, "false");
}
std::string trace_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-trace.log";
defaultConf.set(el::Level::Trace, el::ConfigurationType::Filename, trace_log_path.c_str());
if (trace_enable) {
defaultConf.set(el::Level::Trace, el::ConfigurationType::Enabled, "true");
} else {
defaultConf.set(el::Level::Trace, el::ConfigurationType::Enabled, "false");
}
std::string error_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-error.log";
defaultConf.set(el::Level::Error, el::ConfigurationType::Filename, error_log_path.c_str());
if (error_enable) {
defaultConf.set(el::Level::Error, el::ConfigurationType::Enabled, "true");
} else {
defaultConf.set(el::Level::Error, el::ConfigurationType::Enabled, "false");
}
std::string fatal_log_path = logs_reg_path + "milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log";
defaultConf.set(el::Level::Fatal, el::ConfigurationType::Filename, fatal_log_path.c_str());
if (fatal_enable) {
defaultConf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "true");
} else {
defaultConf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "false");
}
if (max_log_file_size < CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN ||
max_log_file_size > CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX) {
return Status(SERVER_UNEXPECTED_ERROR, "max_log_file_size must in range[" +
std::to_string(CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN) + ", " +
std::to_string(CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX) + "], now is " +
std::to_string(max_log_file_size));
}
defaultConf.setGlobally(el::ConfigurationType::MaxLogFileSize, std::to_string(max_log_file_size));
el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck);
el::Helpers::installPreRollOutCallback(RolloutHandler);
el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog);
// set delete_exceeds = 0 means disable throw away log file even they reach certain limit.
if (delete_exceeds != 0) {
if (delete_exceeds < CONFIG_LOGS_LOG_ROTATE_NUM_MIN || delete_exceeds > CONFIG_LOGS_LOG_ROTATE_NUM_MAX) {
return Status(SERVER_UNEXPECTED_ERROR, "delete_exceeds must in range[" +
std::to_string(CONFIG_LOGS_LOG_ROTATE_NUM_MIN) + ", " +
std::to_string(CONFIG_LOGS_LOG_ROTATE_NUM_MAX) + "], now is " +
std::to_string(delete_exceeds));
}
enable_log_delete = true;
logs_delete_exceeds = delete_exceeds;
}
el::Loggers::reconfigureLogger("default", defaultConf);
return Status::OK();
}
} // namespace milvus

29
proxy/src/log/LogMgr.h Normal file
View File

@ -0,0 +1,29 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "easyloggingpp/easylogging++.h"
#include "utils/Status.h"
#include <sstream>
#include <string>
namespace milvus {
class LogMgr {
public:
static Status
InitLog(bool trace_enable, const std::string& level, const std::string& logs_path, int64_t max_log_file_size,
int64_t delete_exceeds);
};
} // namespace milvus

159
proxy/src/main.cpp Normal file
View File

@ -0,0 +1,159 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <getopt.h>
#include <unistd.h>
#include <csignal>
#include <cstring>
#include <string>
#include "config/ConfigMgr.h"
#include "easyloggingpp/easylogging++.h"
#include "server/Server.h"
#include "src/version.h"
#include "utils/SignalHandler.h"
#include "utils/Status.h"
INITIALIZE_EASYLOGGINGPP
void
print_help(const std::string& app_name) {
std::cout << std::endl << "Usage: " << app_name << " [OPTIONS]" << std::endl;
std::cout << R"(
Options:
-h --help Print this help.
-c --conf_file filename Read configuration from the file.
-d --daemon Daemonize this application.
-p --pid_file filename PID file used by daemonized app.
)" << std::endl;
}
void
print_banner() {
std::cout << std::endl;
std::cout << " __ _________ _ ____ ______ " << std::endl;
std::cout << " / |/ / _/ /| | / / / / / __/ " << std::endl;
std::cout << " / /|_/ // // /_| |/ / /_/ /\\ \\ " << std::endl;
std::cout << " /_/ /_/___/____/___/\\____/___/ " << std::endl;
std::cout << std::endl;
std::cout << "Welcome to use Milvus!" << std::endl;
std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME << ", with "
#ifdef WITH_MKL
<< "MKL"
#else
<< "OpenBLAS"
#endif
<< " library." << std::endl;
#ifdef MILVUS_GPU_VERSION
std::cout << "You are using Milvus GPU edition" << std::endl;
#else
std::cout << "You are using Milvus CPU edition" << std::endl;
#endif
std::cout << "Last commit id: " << LAST_COMMIT_ID << std::endl;
std::cout << std::endl;
}
int
main(int argc, char* argv[]) {
print_banner();
static struct option long_options[] = {{"conf_file", required_argument, nullptr, 'c'},
{"help", no_argument, nullptr, 'h'},
{"daemon", no_argument, nullptr, 'd'},
{"pid_file", required_argument, nullptr, 'p'},
{nullptr, 0, nullptr, 0}};
int option_index = 0;
int64_t start_daemonized = 0;
std::string config_filename;
std::string pid_filename;
std::string app_name = argv[0];
milvus::Status s;
milvus::server::Server& server = milvus::server::Server::GetInstance();
if (argc < 2) {
print_help(app_name);
goto FAIL;
}
int value;
while ((value = getopt_long(argc, argv, "c:p:dh", long_options, &option_index)) != -1) {
switch (value) {
case 'c': {
char* config_filename_ptr = strdup(optarg);
config_filename = config_filename_ptr;
free(config_filename_ptr);
std::cout << "Loading configuration from: " << config_filename << std::endl;
break;
}
case 'p': {
char* pid_filename_ptr = strdup(optarg);
pid_filename = pid_filename_ptr;
free(pid_filename_ptr);
std::cout << pid_filename << std::endl;
break;
}
case 'd':
start_daemonized = 1;
break;
case 'h':
print_help(app_name);
return EXIT_SUCCESS;
case '?':
print_help(app_name);
return EXIT_FAILURE;
default:
print_help(app_name);
break;
}
}
/* Handle Signal */
milvus::signal_routine_func = [](int32_t exit_code) {
milvus::server::Server::GetInstance().Stop();
exit(exit_code);
};
signal(SIGHUP, milvus::HandleSignal);
signal(SIGINT, milvus::HandleSignal);
signal(SIGUSR1, milvus::HandleSignal);
signal(SIGSEGV, milvus::HandleSignal);
signal(SIGUSR2, milvus::HandleSignal);
signal(SIGTERM, milvus::HandleSignal);
try {
milvus::ConfigMgr::GetInstance().Init();
milvus::ConfigMgr::GetInstance().Load(config_filename);
} catch (milvus::ConfigStatus& cs) {
std::cerr << "Load config(" << config_filename << ") failed: " << cs.message << std::endl;
goto FAIL;
}
server.Init(start_daemonized, pid_filename, config_filename);
s = server.Start();
if (s.ok()) {
std::cout << "Milvus server started successfully!" << std::endl;
} else {
std::cout << s.message() << std::endl;
goto FAIL;
}
/* wait signal */
pause();
return EXIT_SUCCESS;
FAIL:
std::cout << "Milvus server exit..." << std::endl;
return EXIT_FAILURE;
}

302
proxy/src/query/BinaryQuery.cpp Normal file
View File

@ -0,0 +1,302 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <memory>
#include <set>
#include <stack>
#include <string>
#include <utility>
#include <vector>
#include "query/BinaryQuery.h"
namespace milvus {
namespace query {
BinaryQueryPtr
ConstructBinTree(std::vector<BooleanQueryPtr> queries, QueryRelation relation, uint64_t idx) {
if (idx == queries.size()) {
return nullptr;
} else if (idx == queries.size() - 1) {
return queries[idx]->getBinaryQuery();
} else {
BinaryQueryPtr bquery = std::make_shared<BinaryQuery>();
bquery->relation = relation;
bquery->left_query = std::make_shared<GeneralQuery>();
bquery->right_query = std::make_shared<GeneralQuery>();
bquery->left_query->bin = queries[idx]->getBinaryQuery();
++idx;
bquery->right_query->bin = ConstructBinTree(queries, relation, idx);
return bquery;
}
}
Status
ConstructLeafBinTree(std::vector<LeafQueryPtr> leaf_queries, BinaryQueryPtr binary_query, uint64_t idx) {
if (idx == leaf_queries.size()) {
return Status::OK();
}
binary_query->left_query = std::make_shared<GeneralQuery>();
binary_query->right_query = std::make_shared<GeneralQuery>();
if (leaf_queries.size() == leaf_queries.size() - 1) {
binary_query->left_query->leaf = leaf_queries[idx];
return Status::OK();
} else if (idx == leaf_queries.size() - 2) {
binary_query->left_query->leaf = leaf_queries[idx];
++idx;
binary_query->right_query->leaf = leaf_queries[idx];
return Status::OK();
} else {
binary_query->left_query->bin->relation = binary_query->relation;
binary_query->right_query->leaf = leaf_queries[idx];
++idx;
return ConstructLeafBinTree(leaf_queries, binary_query->left_query->bin, idx);
}
}
Status
GenBinaryQuery(BooleanQueryPtr query, BinaryQueryPtr& binary_query) {
if (query->getBooleanQueries().size() == 0) {
if (binary_query->relation == QueryRelation::AND || binary_query->relation == QueryRelation::OR) {
// Put VectorQuery to the end of leaf queries
auto query_size = query->getLeafQueries().size();
for (uint64_t i = 0; i < query_size; ++i) {
if (query->getLeafQueries()[i]->vector_placeholder.size() > 0) {
std::swap(query->getLeafQueries()[i], query->getLeafQueries()[0]);
break;
}
}
return ConstructLeafBinTree(query->getLeafQueries(), binary_query, 0);
} else {
switch (query->getOccur()) {
case Occur::MUST: {
binary_query->relation = QueryRelation::AND;
return GenBinaryQuery(query, binary_query);
}
case Occur::MUST_NOT:
case Occur::SHOULD: {
binary_query->relation = QueryRelation::OR;
return GenBinaryQuery(query, binary_query);
}
default:
return Status::OK();
}
}
}
if (query->getBooleanQueries().size() == 1) {
auto bc = query->getBooleanQueries()[0];
binary_query->left_query = std::make_shared<GeneralQuery>();
switch (bc->getOccur()) {
case Occur::MUST: {
binary_query->relation = QueryRelation::AND;
return GenBinaryQuery(bc, binary_query);
}
case Occur::MUST_NOT:
case Occur::SHOULD: {
binary_query->relation = QueryRelation::OR;
return GenBinaryQuery(bc, binary_query);
}
default:
return Status::OK();
}
}
// Construct binary query for every single boolean query
std::vector<BooleanQueryPtr> must_queries;
std::vector<BooleanQueryPtr> must_not_queries;
std::vector<BooleanQueryPtr> should_queries;
Status status;
for (auto& _query : query->getBooleanQueries()) {
status = GenBinaryQuery(_query, _query->getBinaryQuery());
if (!status.ok()) {
return status;
}
if (_query->getOccur() == Occur::MUST) {
must_queries.emplace_back(_query);
} else if (_query->getOccur() == Occur::MUST_NOT) {
must_not_queries.emplace_back(_query);
} else {
should_queries.emplace_back(_query);
}
}
// Construct binary query for combine boolean queries
BinaryQueryPtr must_bquery, should_bquery, must_not_bquery;
uint64_t bquery_num = 0;
if (must_queries.size() > 1) {
// Construct a must binary tree
must_bquery = ConstructBinTree(must_queries, QueryRelation::R1, 0);
++bquery_num;
} else if (must_queries.size() == 1) {
must_bquery = must_queries[0]->getBinaryQuery();
++bquery_num;
}
if (should_queries.size() > 1) {
// Construct a should binary tree
should_bquery = ConstructBinTree(should_queries, QueryRelation::R2, 0);
++bquery_num;
} else if (should_queries.size() == 1) {
should_bquery = should_queries[0]->getBinaryQuery();
++bquery_num;
}
if (must_not_queries.size() > 1) {
// Construct a must_not binary tree
must_not_bquery = ConstructBinTree(must_not_queries, QueryRelation::R1, 0);
++bquery_num;
} else if (must_not_queries.size() == 1) {
must_not_bquery = must_not_queries[0]->getBinaryQuery();
++bquery_num;
}
binary_query->left_query = std::make_shared<GeneralQuery>();
binary_query->right_query = std::make_shared<GeneralQuery>();
BinaryQueryPtr must_should_query = std::make_shared<BinaryQuery>();
must_should_query->left_query = std::make_shared<GeneralQuery>();
must_should_query->right_query = std::make_shared<GeneralQuery>();
if (bquery_num == 3) {
must_should_query->relation = QueryRelation::R3;
must_should_query->left_query->bin = must_bquery;
must_should_query->right_query->bin = should_bquery;
binary_query->relation = QueryRelation::R1;
binary_query->left_query->bin = must_should_query;
binary_query->right_query->bin = must_not_bquery;
} else if (bquery_num == 2) {
if (must_bquery == nullptr) {
binary_query->relation = QueryRelation::R3;
binary_query->left_query->bin = must_not_bquery;
binary_query->right_query->bin = should_bquery;
} else if (should_bquery == nullptr) {
binary_query->relation = QueryRelation::R4;
binary_query->left_query->bin = must_bquery;
binary_query->right_query->bin = must_not_bquery;
} else {
binary_query->relation = QueryRelation::R3;
binary_query->left_query->bin = must_bquery;
binary_query->right_query->bin = should_bquery;
}
} else {
if (must_bquery != nullptr) {
binary_query = must_bquery;
} else if (should_bquery != nullptr) {
binary_query = should_bquery;
} else {
binary_query = must_not_bquery;
}
}
return Status::OK();
}
uint64_t
BinaryQueryHeight(BinaryQueryPtr& binary_query) {
if (binary_query == nullptr) {
return 1;
}
uint64_t left_height = 0, right_height = 0;
if (binary_query->left_query != nullptr) {
left_height = BinaryQueryHeight(binary_query->left_query->bin);
}
if (binary_query->right_query != nullptr) {
right_height = BinaryQueryHeight(binary_query->right_query->bin);
}
return left_height > right_height ? left_height + 1 : right_height + 1;
}
/**
* rules:
* 1. The child node of 'should' and 'must_not' can only be 'term query' and 'range query'.
* 2. One layer cannot include bool query and leaf query.
* 3. The direct child node of 'bool' node cannot be 'should' node or 'must_not' node.
* 4. All filters are pre-filtered(Do structure query first, then use the result to do filtering for vector query).
*
*/
Status
rule_1(BooleanQueryPtr& boolean_query, std::stack<BooleanQueryPtr>& path_stack) {
auto status = Status::OK();
if (boolean_query != nullptr) {
path_stack.push(boolean_query);
for (const auto& leaf_query : boolean_query->getLeafQueries()) {
if (!leaf_query->vector_placeholder.empty()) {
while (!path_stack.empty()) {
auto query = path_stack.top();
if (query->getOccur() == Occur::SHOULD || query->getOccur() == Occur::MUST_NOT) {
std::string msg =
"The child node of 'should' and 'must_not' can only be 'term query' and 'range query'.";
return Status{SERVER_INVALID_DSL_PARAMETER, msg};
}
path_stack.pop();
}
}
}
for (auto query : boolean_query->getBooleanQueries()) {
status = rule_1(query, path_stack);
if (!status.ok()) {
return status;
}
}
}
return status;
}
Status
rule_2(BooleanQueryPtr& boolean_query) {
auto status = Status::OK();
if (boolean_query != nullptr) {
if (!boolean_query->getBooleanQueries().empty() && !boolean_query->getLeafQueries().empty()) {
std::string msg = "One layer cannot include bool query and leaf query.";
return Status{SERVER_INVALID_DSL_PARAMETER, msg};
} else {
for (auto query : boolean_query->getBooleanQueries()) {
status = rule_2(query);
if (!status.ok()) {
return status;
}
}
}
}
return status;
}
Status
ValidateBooleanQuery(BooleanQueryPtr& boolean_query) {
auto status = Status::OK();
if (boolean_query != nullptr) {
for (auto& query : boolean_query->getBooleanQueries()) {
if (query->getOccur() == Occur::SHOULD || query->getOccur() == Occur::MUST_NOT) {
std::string msg = "The direct child node of 'bool' node cannot be 'should' node or 'must_not' node.";
return Status{SERVER_INVALID_DSL_PARAMETER, msg};
}
}
std::stack<BooleanQueryPtr> path_stack;
status = rule_1(boolean_query, path_stack);
if (!status.ok()) {
return status;
}
status = rule_2(boolean_query);
if (!status.ok()) {
return status;
}
}
return status;
}
bool
ValidateBinaryQuery(BinaryQueryPtr& binary_query) {
uint64_t height = BinaryQueryHeight(binary_query);
return height > 1;
}
} // namespace query
} // namespace milvus

41
proxy/src/query/BinaryQuery.h Normal file
View File

@ -0,0 +1,41 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <memory>
#include <vector>
#include "BooleanQuery.h"
namespace milvus {
namespace query {
BinaryQueryPtr
ConstructBinTree(std::vector<BooleanQueryPtr> clauses, QueryRelation relation, uint64_t idx);
Status
ConstructLeafBinTree(std::vector<LeafQueryPtr> leaf_clauses, BinaryQueryPtr binary_query, uint64_t idx);
Status
GenBinaryQuery(BooleanQueryPtr clause, BinaryQueryPtr& binary_query);
uint64_t
BinaryQueryHeight(BinaryQueryPtr& binary_query);
Status
ValidateBooleanQuery(BooleanQueryPtr& boolean_query);
bool
ValidateBinaryQuery(BinaryQueryPtr& binary_query);
} // namespace query
} // namespace milvus

87
proxy/src/query/BooleanQuery.h Normal file
View File

@ -0,0 +1,87 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <memory>
#include <vector>
#include "GeneralQuery.h"
#include "utils/Status.h"
namespace milvus {
namespace query {
enum class Occur {
INVALID = 0,
MUST,
MUST_NOT,
SHOULD,
};
class BooleanQuery {
public:
BooleanQuery() {
}
explicit BooleanQuery(Occur occur) : occur_(occur) {
}
Occur
getOccur() {
return occur_;
}
void
SetOccur(Occur occur) {
occur_ = occur;
}
void
AddBooleanQuery(std::shared_ptr<BooleanQuery> boolean_clause) {
boolean_clauses_.emplace_back(boolean_clause);
}
void
AddLeafQuery(LeafQueryPtr leaf_query) {
leaf_queries_.emplace_back(leaf_query);
}
void
SetLeafQuery(std::vector<LeafQueryPtr> leaf_queries) {
leaf_queries_ = leaf_queries;
}
std::vector<std::shared_ptr<BooleanQuery>>
getBooleanQueries() {
return boolean_clauses_;
}
BinaryQueryPtr&
getBinaryQuery() {
return binary_query_;
}
std::vector<LeafQueryPtr>&
getLeafQueries() {
return leaf_queries_;
}
private:
Occur occur_ = Occur::INVALID;
std::vector<std::shared_ptr<BooleanQuery>> boolean_clauses_;
std::vector<LeafQueryPtr> leaf_queries_;
BinaryQueryPtr binary_query_ = std::make_shared<BinaryQuery>();
};
using BooleanQueryPtr = std::shared_ptr<BooleanQuery>;
} // namespace query
} // namespace milvus

17
proxy/src/query/CMakeLists.txt Normal file
View File

@ -0,0 +1,17 @@
#-------------------------------------------------------------------------------
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
aux_source_directory( ${MILVUS_ENGINE_SRC}/query QUERY_FILES )
add_library( query STATIC )
target_sources( query PRIVATE ${QUERY_FILES} )
target_include_directories( query PUBLIC ${MILVUS_ENGINE_SRC}/query )

127
proxy/src/query/GeneralQuery.h Normal file
View File

@ -0,0 +1,127 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <iostream>
#include <memory>
#include <set>
#include <string>
#include <unordered_map>
#include <vector>
// #include "db/Types.h"
#include "utils/Json.h"
namespace milvus {
namespace query {
enum class CompareOperator {
LT = 0,
LTE,
EQ,
GT,
GTE,
NE,
};
enum class QueryRelation {
INVALID = 0,
R1,
R2,
R3,
R4,
AND,
OR,
};
struct QueryColumn {
std::string name;
std::string column_value;
};
struct TermQuery {
milvus::json json_obj;
// std::string field_name;
// std::vector<uint8_t> field_value;
// float boost;
};
using TermQueryPtr = std::shared_ptr<TermQuery>;
struct CompareExpr {
CompareOperator compare_operator;
std::string operand;
};
struct RangeQuery {
milvus::json json_obj;
// std::string field_name;
// std::vector<CompareExpr> compare_expr;
// float boost;
};
using RangeQueryPtr = std::shared_ptr<RangeQuery>;
struct VectorRecord {
std::vector<float> float_data;
std::vector<uint8_t> binary_data;
};
struct VectorQuery {
std::string field_name;
milvus::json extra_params = {};
int64_t topk;
int64_t nq;
std::string metric_type = "";
float boost;
VectorRecord query_vector;
};
using VectorQueryPtr = std::shared_ptr<VectorQuery>;
struct LeafQuery;
using LeafQueryPtr = std::shared_ptr<LeafQuery>;
struct BinaryQuery;
using BinaryQueryPtr = std::shared_ptr<BinaryQuery>;
struct GeneralQuery {
LeafQueryPtr leaf;
BinaryQueryPtr bin = std::make_shared<BinaryQuery>();
};
using GeneralQueryPtr = std::shared_ptr<GeneralQuery>;
struct LeafQuery {
TermQueryPtr term_query;
RangeQueryPtr range_query;
std::string vector_placeholder;
float query_boost;
};
struct BinaryQuery {
GeneralQueryPtr left_query;
GeneralQueryPtr right_query;
QueryRelation relation;
float query_boost;
};
struct Query {
GeneralQueryPtr root;
std::unordered_map<std::string, VectorQueryPtr> vectors;
std::string collection_id;
std::vector<std::string> partitions;
std::vector<std::string> field_names;
std::set<std::string> index_fields;
std::unordered_map<std::string, std::string> metric_types;
};
using QueryPtr = std::shared_ptr<Query>;
} // namespace query
} // namespace milvus

48
proxy/src/server/CMakeLists.txt Normal file
View File

@ -0,0 +1,48 @@
#-------------------------------------------------------------------------------
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
set( GRPC_SERVICE_FILES ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.grpc.pb.cc
${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.pb.cc
${MILVUS_ENGINE_SRC}/grpc/gen-status/status.grpc.pb.cc
${MILVUS_ENGINE_SRC}/grpc/gen-status/status.pb.cc
)
aux_source_directory( ${MILVUS_ENGINE_SRC}/server SERVER_SERVICE_FILES )
aux_source_directory( ${MILVUS_ENGINE_SRC}/server/init SERVER_INIT_FILES )
aux_source_directory( ${MILVUS_ENGINE_SRC}/server/delivery/request DELIVERY_REQUEST_FILES )
aux_source_directory( ${MILVUS_ENGINE_SRC}/server/delivery DELIVERY_FILES )
set( SERVER_FILES ${SERVER_INIT_FILES}
${SERVER_SERVICE_FILES}
${SERVER_INIT_FILES}
${DELIVERY_REQUEST_FILES}
${DELIVERY_FILES}
)
aux_source_directory( ${MILVUS_ENGINE_SRC}/server/grpc_impl GRPC_IMPL_FILES )
aux_source_directory( ${MILVUS_ENGINE_SRC}/server/grpc_impl/interceptor GRPC_INTERCEPTOR_FILES )
set( GRPC_SERVER_FILES ${GRPC_IMPL_FILES}
${GRPC_INTERCEPTOR_FILES}
)
aux_source_directory( ${MILVUS_ENGINE_SRC}/server/context SERVER_CONTEXT_FILES )
add_library( server STATIC )
target_sources( server
PRIVATE ${GRPC_SERVER_FILES}
${GRPC_SERVICE_FILES}
${SERVER_FILES}
${SERVER_CONTEXT_FILES}
)

347
proxy/src/server/Server.cpp Normal file
View File

@ -0,0 +1,347 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/Server.h"
#include "server/init/InstanceLockCheck.h"
#include <fcntl.h>
#include <unistd.h>
#include <boost/filesystem.hpp>
#include <cstring>
#include <unordered_map>
#include "config/ServerConfig.h"
#include "log/LogMgr.h"
// #include "scheduler/SchedInst.h"
#include "server/grpc_impl/GrpcServer.h"
#include "server/init/CpuChecker.h"
// #include "server/init/GpuChecker.h"
#include "server/init/StorageChecker.h"
#include "src/version.h"
#include <yaml-cpp/yaml.h>
#include "utils/Log.h"
#include "utils/SignalHandler.h"
#include "utils/TimeRecorder.h"
namespace milvus {
namespace server {
Server&
Server::GetInstance() {
static Server server;
return server;
}
void
Server::Init(int64_t daemonized, const std::string& pid_filename, const std::string& config_filename) {
daemonized_ = daemonized;
pid_filename_ = pid_filename;
config_filename_ = config_filename;
}
void
Server::Daemonize() {
if (daemonized_ == 0) {
return;
}
std::cout << "Milvus server run in daemonize mode";
pid_t pid = 0;
// Fork off the parent process
pid = fork();
// An error occurred
if (pid < 0) {
exit(EXIT_FAILURE);
}
// Success: terminate parent
if (pid > 0) {
exit(EXIT_SUCCESS);
}
// On success: The child process becomes session leader
if (setsid() < 0) {
exit(EXIT_FAILURE);
}
// Ignore signal sent from child to parent process
signal(SIGCHLD, SIG_IGN);
// Fork off for the second time
pid = fork();
// An error occurred
if (pid < 0) {
exit(EXIT_FAILURE);
}
// Terminate the parent
if (pid > 0) {
exit(EXIT_SUCCESS);
}
// Set new file permissions
umask(0);
// Change the working directory to root
int ret = chdir("/");
if (ret != 0) {
return;
}
// Close all open fd
for (int64_t fd = sysconf(_SC_OPEN_MAX); fd > 0; fd--) {
close(fd);
}
std::cout << "Redirect stdin/stdout/stderr to /dev/null";
// Redirect stdin/stdout/stderr to /dev/null
stdin = fopen("/dev/null", "r");
stdout = fopen("/dev/null", "w+");
stderr = fopen("/dev/null", "w+");
// Try to write PID of daemon to lockfile
if (!pid_filename_.empty()) {
pid_fd_ = open(pid_filename_.c_str(), O_RDWR | O_CREAT, 0640);
if (pid_fd_ < 0) {
std::cerr << "Can't open filename: " + pid_filename_ + ", Error: " + strerror(errno);
exit(EXIT_FAILURE);
}
if (lockf(pid_fd_, F_TLOCK, 0) < 0) {
std::cerr << "Can't lock filename: " + pid_filename_ + ", Error: " + strerror(errno);
exit(EXIT_FAILURE);
}
std::string pid_file_context = std::to_string(getpid());
ssize_t res = write(pid_fd_, pid_file_context.c_str(), pid_file_context.size());
if (res != 0) {
return;
}
}
}
Status
Server::Start() {
if (daemonized_ != 0) {
Daemonize();
}
try {
auto meta_uri = config.general.meta_uri();
if (meta_uri.length() > 6 && strcasecmp("sqlite", meta_uri.substr(0, 6).c_str()) == 0) {
std::cout << "WARNING: You are using SQLite as the meta data management, "
"which can't be used in production. Please change it to MySQL!"
<< std::endl;
}
/* Init opentracing tracer from config */
std::string tracing_config_path = config.tracing.json_config_path();
tracing_config_path.empty() ? tracing::TracerUtil::InitGlobal()
: tracing::TracerUtil::InitGlobal(tracing_config_path);
auto time_zone = config.general.timezone();
if (time_zone.length() == 3) {
time_zone = "CUT";
} else {
int time_bias = std::stoi(time_zone.substr(3, std::string::npos));
if (time_bias == 0) {
time_zone = "CUT";
} else if (time_bias > 0) {
time_zone = "CUT" + std::to_string(-time_bias);
} else {
time_zone = "CUT+" + std::to_string(-time_bias);
}
}
if (setenv("TZ", time_zone.c_str(), 1) != 0) {
return Status(SERVER_UNEXPECTED_ERROR, "Fail to setenv");
}
tzset();
/* log path is defined in Config file, so InitLog must be called after LoadConfig */
STATUS_CHECK(LogMgr::InitLog(config.logs.trace.enable(), config.logs.level(), config.logs.path(),
config.logs.max_log_file_size(), config.logs.log_rotate_num()));
bool cluster_enable = config.cluster.enable();
auto cluster_role = config.cluster.role();
Status s;
if ((not cluster_enable) || cluster_role == ClusterRole::RW) {
try {
// True if a new directory was created, otherwise false.
boost::filesystem::create_directories(config.storage.path());
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, "Cannot create db directory, " + std::string(ex.what()));
} catch (...) {
return Status(SERVER_UNEXPECTED_ERROR, "Cannot create db directory");
}
s = InstanceLockCheck::Check(config.storage.path());
if (!s.ok()) {
if (not cluster_enable) {
std::cerr << "single instance lock db path failed." << s.message() << std::endl;
} else {
std::cerr << cluster_role << " instance lock db path failed." << s.message() << std::endl;
}
return s;
}
if (config.wal.enable()) {
std::string wal_path = config.wal.path();
try {
// True if a new directory was created, otherwise false.
boost::filesystem::create_directories(wal_path);
} catch (...) {
return Status(SERVER_UNEXPECTED_ERROR, "Cannot create wal directory");
}
s = InstanceLockCheck::Check(wal_path);
if (!s.ok()) {
if (not cluster_enable) {
std::cerr << "single instance lock wal path failed." << s.message() << std::endl;
} else {
std::cerr << cluster_role << " instance lock wal path failed." << s.message() << std::endl;
}
return s;
}
}
}
// print version information
LOG_SERVER_INFO_ << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME;
#ifdef MILVUS_GPU_VERSION
LOG_SERVER_INFO_ << "GPU edition";
#else
LOG_SERVER_INFO_ << "CPU edition";
#endif
STATUS_CHECK(StorageChecker::CheckStoragePermission());
STATUS_CHECK(CpuChecker::CheckCpuInstructionSet());
/* record config and hardware information into log */
LogConfigInFile(config_filename_);
LogCpuInfo();
LOG_SERVER_INFO_ << "\n\n"
<< std::string(15, '*') << "Config in memory" << std::string(15, '*') << "\n\n"
<< ConfigMgr::GetInstance().Dump();
server::Metrics::GetInstance().Init();
server::SystemInfo::GetInstance().Init();
return StartService();
} catch (std::exception& ex) {
std::string str = "Milvus server encounter exception: " + std::string(ex.what());
return Status(SERVER_UNEXPECTED_ERROR, str);
}
}
void
Server::Stop() {
std::cerr << "Milvus server is going to shutdown ..." << std::endl;
/* Unlock and close lockfile */
if (pid_fd_ != -1) {
int ret = lockf(pid_fd_, F_ULOCK, 0);
if (ret != 0) {
std::cerr << "ERROR: Can't lock file: " << strerror(errno) << std::endl;
exit(0);
}
ret = close(pid_fd_);
if (ret != 0) {
std::cerr << "ERROR: Can't close file: " << strerror(errno) << std::endl;
exit(0);
}
}
/* delete lockfile */
if (!pid_filename_.empty()) {
int ret = unlink(pid_filename_.c_str());
if (ret != 0) {
std::cerr << "ERROR: Can't unlink file: " << strerror(errno) << std::endl;
exit(0);
}
}
StopService();
std::cerr << "Milvus server exit..." << std::endl;
}
Status
Server::StartService() {
Status stat;
stat = engine::KnowhereResource::Initialize();
if (!stat.ok()) {
LOG_SERVER_ERROR_ << "KnowhereResource initialize fail: " << stat.message();
goto FAIL;
}
grpc::GrpcServer::GetInstance().Start();
// stat = storage::S3ClientWrapper::GetInstance().StartService();
// if (!stat.ok()) {
// LOG_SERVER_ERROR_ << "S3Client start service fail: " << stat.message();
// goto FAIL;
// }
return Status::OK();
FAIL:
std::cerr << "Milvus initializes fail: " << stat.message() << std::endl;
return stat;
}
void
Server::StopService() {
// storage::S3ClientWrapper::GetInstance().StopService();
grpc::GrpcServer::GetInstance().Stop();
}
void
Server::LogConfigInFile(const std::string& path) {
// TODO(yhz): Check if file exists
auto node = YAML::LoadFile(path);
YAML::Emitter out;
out << node;
LOG_SERVER_INFO_ << "\n\n"
<< std::string(15, '*') << "Config in file" << std::string(15, '*') << "\n\n"
<< out.c_str();
}
void
Server::LogCpuInfo() {
/*CPU information*/
std::fstream fcpu("/proc/cpuinfo", std::ios::in);
if (!fcpu.is_open()) {
LOG_SERVER_WARNING_ << "Cannot obtain CPU information. Open file /proc/cpuinfo fail: " << strerror(errno)
<< "(errno: " << errno << ")";
return;
}
std::stringstream cpu_info_ss;
cpu_info_ss << fcpu.rdbuf();
fcpu.close();
std::string cpu_info = cpu_info_ss.str();
auto processor_pos = cpu_info.rfind("processor");
if (std::string::npos == processor_pos) {
LOG_SERVER_WARNING_ << "Cannot obtain CPU information. No sub string \'processor\'";
return;
}
auto sub_str = cpu_info.substr(processor_pos);
LOG_SERVER_INFO_ << "\n\n" << std::string(15, '*') << "CPU" << std::string(15, '*') << "\n\n" << sub_str;
}
} // namespace server
} // namespace milvus

62
proxy/src/server/Server.h Normal file
View File

@ -0,0 +1,62 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <string>
#include "config/ConfigMgr.h"
#include "utils/Status.h"
namespace milvus {
namespace server {
class Server {
public:
static Server&
GetInstance();
void
Init(int64_t daemonized, const std::string& pid_filename, const std::string& config_filename);
Status
Start();
void
Stop();
private:
Server() = default;
~Server() = default;
void
Daemonize();
Status
StartService();
void
StopService();
private:
static void
LogConfigInFile(const std::string& path);
static void
LogCpuInfo();
private:
int64_t daemonized_ = 0;
int pid_fd_ = -1;
std::string pid_filename_;
std::string config_filename_;
// ConfigMgrPtr config_mgr_;
}; // Server
} // namespace server
} // namespace milvus

113
proxy/src/server/ValidationUtil.cpp Normal file
View File

@ -0,0 +1,113 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/ValidationUtil.h"
// #include "db/Constants.h"
// #include "db/Utils.h"
// #include "knowhere/index/vector_index/ConfAdapter.h"
// #include "knowhere/index/vector_index/helpers/IndexParameter.h"
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
#include <algorithm>
#include <limits>
#include <set>
#include <string>
namespace milvus {
namespace server {
namespace {
Status
CheckParameterRange(const milvus::json& json_params, const std::string& param_name, int64_t min, int64_t max,
bool min_close = true, bool max_closed = true) {
return Status::OK();
}
Status
CheckParameterExistence(const milvus::json& json_params, const std::string& param_name) {
return Status::OK();
}
} // namespace
Status
ValidateCollectionName(const std::string& collection_name) {
return Status::OK();
}
Status
ValidateFieldName(const std::string& field_name) {
return Status::OK();
}
Status
ValidateIndexType(std::string& index_type) {
return Status::OK();
}
Status
ValidateDimension(int64_t dim, bool is_binary) {
return Status::OK();
}
Status
ValidateIndexParams(const milvus::json& index_params, int64_t dimension, const std::string& index_type) {
return Status::OK();
}
Status
ValidateSegmentRowCount(int64_t segment_row_count) {
return Status::OK();
}
Status
ValidateIndexMetricType(const std::string& metric_type, const std::string& index_type) {
return Status::OK();
}
Status
ValidateSearchTopk(int64_t top_k) {
return Status::OK();
}
Status
ValidatePartitionTags(const std::vector<std::string>& partition_tags) {
return Status::OK();
}
Status
ValidateInsertDataSize(const engine::DataChunkPtr& data) {
return Status::OK();
}
} // namespace server
} // namespace milvus

56
proxy/src/server/ValidationUtil.h Normal file
View File

@ -0,0 +1,56 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
// #include "db/Types.h"
#include "utils/Json.h"
#include "utils/Status.h"
#include <string>
#include <vector>
namespace milvus {
namespace server {
constexpr int64_t QUERY_MAX_TOPK = 2048;
extern Status
ValidateCollectionName(const std::string& collection_name);
extern Status
ValidateFieldName(const std::string& field_name);
extern Status
ValidateDimension(int64_t dimension, bool is_binary);
extern Status
ValidateIndexType(std::string& index_type);
extern Status
ValidateIndexParams(const milvus::json& index_params, int64_t dimension, const std::string& index_type);
extern Status
ValidateSegmentRowCount(int64_t segment_row_count);
extern Status
ValidateIndexMetricType(const std::string& metric_type, const std::string& index_type);
extern Status
ValidateSearchTopk(int64_t top_k);
extern Status
ValidatePartitionTags(const std::vector<std::string>& partition_tags);
extern Status
ValidateInsertDataSize(const engine::DataChunkPtr& data);
} // namespace server
} // namespace milvus

30
proxy/src/server/context/ConnectionContext.h Normal file
View File

@ -0,0 +1,30 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <memory>
namespace milvus {
namespace server {
class ConnectionContext {
public:
virtual ~ConnectionContext() {
}
virtual bool
IsConnectionBroken() const = 0;
};
using ConnectionContextPtr = std::shared_ptr<ConnectionContext>;
} // namespace server
} // namespace milvus

108
proxy/src/server/context/Context.cpp Normal file
View File

@ -0,0 +1,108 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/context/Context.h"
namespace milvus {
namespace server {
Context::Context(const std::string& req_id) : req_id_(req_id) {
}
const std::shared_ptr<tracing::TraceContext>&
Context::GetTraceContext() const {
return trace_context_;
}
void
Context::SetTraceContext(const std::shared_ptr<tracing::TraceContext>& trace_context) {
trace_context_ = trace_context;
}
std::shared_ptr<Context>
Context::Child(const std::string& operation_name) const {
auto new_context = std::make_shared<Context>(req_id_);
new_context->SetTraceContext(trace_context_->Child(operation_name));
return new_context;
}
std::shared_ptr<Context>
Context::Follower(const std::string& operation_name) const {
auto new_context = std::make_shared<Context>(req_id_);
new_context->SetTraceContext(trace_context_->Follower(operation_name));
return new_context;
}
void
Context::SetConnectionContext(ConnectionContextPtr& context) {
context_ = context;
}
bool
Context::IsConnectionBroken() const {
if (context_ == nullptr) {
return false;
}
return context_->IsConnectionBroken();
}
ReqType
Context::GetReqType() const {
return req_type_;
}
void
Context::SetReqType(ReqType type) {
req_type_ = type;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
ContextChild::ContextChild(const ContextPtr& context, const std::string& operation_name) {
if (context) {
context_ = context->Child(operation_name);
}
}
ContextChild::~ContextChild() {
Finish();
}
void
ContextChild::Finish() {
if (context_) {
context_->GetTraceContext()->GetSpan()->Finish();
context_ = nullptr;
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////
ContextFollower::ContextFollower(const ContextPtr& context, const std::string& operation_name) {
if (context) {
context_ = context->Follower(operation_name);
}
}
ContextFollower::~ContextFollower() {
if (context_) {
context_->GetTraceContext()->GetSpan()->Finish();
}
}
void
ContextFollower::Finish() {
if (context_) {
context_->GetTraceContext()->GetSpan()->Finish();
context_ = nullptr;
}
}
} // namespace server
} // namespace milvus

104
proxy/src/server/context/Context.h Normal file
View File

@ -0,0 +1,104 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <memory>
#include <string>
#include <unordered_map>
#include <grpcpp/server_context.h>
#include "server/context/ConnectionContext.h"
#include "server/delivery/request/Types.h"
#include "tracing/TraceContext.h"
namespace milvus {
namespace server {
class Context {
public:
explicit Context(const std::string& request_id);
inline std::string
ReqID() const {
return req_id_;
}
std::shared_ptr<Context>
Child(const std::string& operation_name) const;
std::shared_ptr<Context>
Follower(const std::string& operation_name) const;
void
SetTraceContext(const std::shared_ptr<tracing::TraceContext>& trace_context);
const std::shared_ptr<tracing::TraceContext>&
GetTraceContext() const;
void
SetConnectionContext(ConnectionContextPtr& context);
bool
IsConnectionBroken() const;
ReqType
GetReqType() const;
void
SetReqType(ReqType type);
private:
std::string req_id_;
ReqType req_type_;
std::shared_ptr<tracing::TraceContext> trace_context_;
ConnectionContextPtr context_;
};
using ContextPtr = std::shared_ptr<milvus::server::Context>;
class ContextChild {
public:
explicit ContextChild(const ContextPtr& context, const std::string& operation_name);
~ContextChild();
ContextPtr
Context() {
return context_;
}
void
Finish();
private:
ContextPtr context_;
};
class ContextFollower {
public:
explicit ContextFollower(const ContextPtr& context, const std::string& operation_name);
~ContextFollower();
ContextPtr
Context() {
return context_;
}
void
Finish();
private:
ContextPtr context_;
};
} // namespace server
} // namespace milvus

221
proxy/src/server/delivery/ReqHandler.cpp Normal file
View File

@ -0,0 +1,221 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/ReqHandler.h"
#include <set>
#include "server/delivery/ReqScheduler.h"
#include "server/delivery/request/BaseReq.h"
#include "server/delivery/request/CmdReq.h"
#include "server/delivery/request/CompactReq.h"
#include "server/delivery/request/CountEntitiesReq.h"
#include "server/delivery/request/CreateCollectionReq.h"
#include "server/delivery/request/CreateIndexReq.h"
#include "server/delivery/request/CreatePartitionReq.h"
#include "server/delivery/request/DeleteEntityByIDReq.h"
#include "server/delivery/request/DescribeIndexReq.h"
#include "server/delivery/request/DropCollectionReq.h"
#include "server/delivery/request/DropIndexReq.h"
#include "server/delivery/request/DropPartitionReq.h"
#include "server/delivery/request/FlushReq.h"
#include "server/delivery/request/GetCollectionInfoReq.h"
#include "server/delivery/request/GetCollectionStatsReq.h"
#include "server/delivery/request/GetEntityByIDReq.h"
#include "server/delivery/request/HasCollectionReq.h"
#include "server/delivery/request/HasPartitionReq.h"
#include "server/delivery/request/InsertReq.h"
#include "server/delivery/request/ListCollectionsReq.h"
#include "server/delivery/request/ListIDInSegmentReq.h"
#include "server/delivery/request/ListPartitionsReq.h"
#include "server/delivery/request/LoadCollectionReq.h"
#include "server/delivery/request/SearchReq.h"
namespace milvus {
namespace server {
Status
ReqHandler::CreateCollection(const ContextPtr& context, const std::string& collection_name, FieldsType& fields,
milvus::json& json_param) {
BaseReqPtr req_ptr = CreateCollectionReq::Create(context, collection_name, fields, json_param);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::DropCollection(const ContextPtr& context, const std::string& collection_name) {
BaseReqPtr req_ptr = DropCollectionReq::Create(context, collection_name);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::HasCollection(const ContextPtr& context, const std::string& collection_name, bool& has_collection) {
BaseReqPtr req_ptr = HasCollectionReq::Create(context, collection_name, has_collection);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::ListCollections(const ContextPtr& context, std::vector<std::string>& collections) {
BaseReqPtr req_ptr = ListCollectionsReq::Create(context, collections);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::GetCollectionInfo(const ContextPtr& context, const std::string& collection_name,
CollectionSchema& collection_schema) {
BaseReqPtr req_ptr = GetCollectionInfoReq::Create(context, collection_name, collection_schema);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::GetCollectionStats(const ContextPtr& context, const std::string& collection_name,
std::string& collection_stats) {
BaseReqPtr req_ptr = GetCollectionStatsReq::Create(context, collection_name, collection_stats);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::CountEntities(const ContextPtr& context, const std::string& collection_name, int64_t& count) {
BaseReqPtr req_ptr = CountEntitiesReq::Create(context, collection_name, count);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::CreatePartition(const ContextPtr& context, const std::string& collection_name, const std::string& tag) {
BaseReqPtr req_ptr = CreatePartitionReq::Create(context, collection_name, tag);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::DropPartition(const ContextPtr& context, const std::string& collection_name, const std::string& tag) {
BaseReqPtr req_ptr = DropPartitionReq::Create(context, collection_name, tag);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::HasPartition(const ContextPtr& context, const std::string& collection_name, const std::string& tag,
bool& has_partition) {
BaseReqPtr req_ptr = HasPartitionReq::Create(context, collection_name, tag, has_partition);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::ListPartitions(const ContextPtr& context, const std::string& collection_name,
std::vector<std::string>& partitions) {
BaseReqPtr req_ptr = ListPartitionsReq::Create(context, collection_name, partitions);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::CreateIndex(const ContextPtr& context, const std::string& collection_name, const std::string& field_name,
const std::string& index_name, const milvus::json& json_params) {
BaseReqPtr req_ptr = CreateIndexReq::Create(context, collection_name, field_name, index_name, json_params);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::DescribeIndex(const ContextPtr& context, const std::string& collection_name, const std::string& field_name,
std::string& index_name, milvus::json& json_params) {
BaseReqPtr req_ptr = DescribeIndexReq::Create(context, collection_name, field_name, index_name, json_params);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::DropIndex(const ContextPtr& context, const std::string& collection_name, const std::string& field_name,
const std::string& index_name) {
BaseReqPtr req_ptr = DropIndexReq::Create(context, collection_name, index_name, field_name);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::Insert(const ContextPtr& context, const std::string& collection_name, const std::string& partition_name,
const int64_t& row_count, std::unordered_map<std::string, std::vector<uint8_t>>& chunk_data) {
BaseReqPtr req_ptr = InsertReq::Create(context, collection_name, partition_name, row_count, chunk_data);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::GetEntityByID(const ContextPtr& context, const std::string& collection_name, const engine::IDNumbers& ids,
std::vector<std::string>& field_names, std::vector<bool>& valid_row,
engine::snapshot::FieldElementMappings& field_mappings, engine::DataChunkPtr& data_chunk) {
BaseReqPtr req_ptr =
GetEntityByIDReq::Create(context, collection_name, ids, field_names, valid_row, field_mappings, data_chunk);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::DeleteEntityByID(const ContextPtr& context, const std::string& collection_name,
const engine::IDNumbers& ids) {
BaseReqPtr req_ptr = DeleteEntityByIDReq::Create(context, collection_name, ids);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::Search(const ContextPtr& context, const query::QueryPtr& query_ptr, const milvus::json& json_params,
engine::snapshot::FieldElementMappings& collection_mappings, engine::QueryResultPtr& result) {
BaseReqPtr req_ptr = SearchReq::Create(context, query_ptr, json_params, collection_mappings, result);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::ListIDInSegment(const ContextPtr& context, const std::string& collection_name, int64_t segment_id,
engine::IDNumbers& ids) {
BaseReqPtr req_ptr = ListIDInSegmentReq::Create(context, collection_name, segment_id, ids);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::LoadCollection(const ContextPtr& context, const std::string& collection_name) {
BaseReqPtr req_ptr = LoadCollectionReq::Create(context, collection_name);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::Flush(const ContextPtr& context, const std::vector<std::string>& collection_names) {
BaseReqPtr req_ptr = FlushReq::Create(context, collection_names);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::Compact(const ContextPtr& context, const std::string& collection_name, double compact_threshold) {
BaseReqPtr req_ptr = CompactReq::Create(context, collection_name, compact_threshold);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
Status
ReqHandler::Cmd(const ContextPtr& context, const std::string& cmd, std::string& reply) {
BaseReqPtr req_ptr = CmdReq::Create(context, cmd, reply);
ReqScheduler::ExecReq(req_ptr);
return req_ptr->status();
}
} // namespace server
} // namespace milvus

115
proxy/src/server/delivery/ReqHandler.h Normal file
View File

@ -0,0 +1,115 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
// #include <src/db/snapshot/Context.h>
// #include <src/segment/Segment.h>
#include <memory>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
// #include "query/BooleanQuery.h"
#include "server/delivery/request/BaseReq.h"
#include "utils/Status.h"
namespace milvus {
namespace server {
class ReqHandler {
public:
ReqHandler() = default;
Status
CreateCollection(const ContextPtr& context, const std::string& collection_name, FieldsType& fields,
milvus::json& json_params);
Status
DropCollection(const ContextPtr& context, const std::string& collection_name);
Status
HasCollection(const ContextPtr& context, const std::string& collection_name, bool& has_collection);
Status
ListCollections(const ContextPtr& context, std::vector<std::string>& collections);
Status
GetCollectionInfo(const ContextPtr& context, const std::string& collection_name,
CollectionSchema& collection_schema);
Status
GetCollectionStats(const ContextPtr& context, const std::string& collection_name, std::string& collection_stats);
Status
CountEntities(const ContextPtr& context, const std::string& collection_name, int64_t& count);
Status
CreatePartition(const ContextPtr& context, const std::string& collection_name, const std::string& tag);
Status
DropPartition(const ContextPtr& context, const std::string& collection_name, const std::string& tag);
Status
HasPartition(const ContextPtr& context, const std::string& collection_name, const std::string& tag,
bool& has_partition);
Status
ListPartitions(const ContextPtr& context, const std::string& collection_name, std::vector<std::string>& partitions);
Status
CreateIndex(const ContextPtr& context, const std::string& collection_name, const std::string& field_name,
const std::string& index_name, const milvus::json& json_params);
Status
DescribeIndex(const ContextPtr& context, const std::string& collection_name, const std::string& field_name,
std::string& index_name, milvus::json& json_params);
Status
DropIndex(const ContextPtr& context, const std::string& collection_name, const std::string& field_name,
const std::string& index_name);
Status
Insert(const ContextPtr& context, const std::string& collection_name, const std::string& partition_name,
const int64_t& row_count, std::unordered_map<std::string, std::vector<uint8_t>>& chunk_data);
Status
GetEntityByID(const ContextPtr& context, const std::string& collection_name, const engine::IDNumbers& ids,
std::vector<std::string>& field_names, std::vector<bool>& valid_row,
engine::snapshot::FieldElementMappings& field_mappings, engine::DataChunkPtr& data_chunk);
Status
DeleteEntityByID(const ContextPtr& context, const std::string& collection_name, const engine::IDNumbers& ids);
Status
Search(const ContextPtr& context, const query::QueryPtr& query_ptr, const milvus::json& json_params,
engine::snapshot::FieldElementMappings& collection_mappings, engine::QueryResultPtr& result);
Status
ListIDInSegment(const ContextPtr& context, const std::string& collection_name, int64_t segment_id,
engine::IDNumbers& ids);
Status
LoadCollection(const ContextPtr& context, const std::string& collection_name);
Status
Flush(const ContextPtr& context, const std::vector<std::string>& collection_names);
Status
Compact(const ContextPtr& context, const std::string& collection_name, double compact_threshold);
Status
Cmd(const ContextPtr& context, const std::string& cmd, std::string& reply);
};
} // namespace server
} // namespace milvus

71
proxy/src/server/delivery/ReqQueue.cpp Normal file
View File

@ -0,0 +1,71 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/ReqQueue.h"
#include "server/delivery/strategy/ReqStrategy.h"
#include "server/delivery/strategy/SearchReqStrategy.h"
#include "utils/Log.h"
#include <unistd.h>
#include <queue>
#include <utility>
namespace milvus {
namespace server {
namespace {
Status
ScheduleReq(const BaseReqPtr& req, std::queue<BaseReqPtr>& queue) {
if (req == nullptr) {
return Status(SERVER_NULL_POINTER, "request schedule cannot handle null object");
}
if (queue.empty()) {
queue.push(req);
return Status::OK();
}
static std::map<ReqType, ReqStrategyPtr> s_schedulers = {{ReqType::kSearch, std::make_shared<SearchReqStrategy>()}};
auto iter = s_schedulers.find(req->type());
if (iter == s_schedulers.end() || iter->second == nullptr) {
queue.push(req);
} else {
iter->second->ReScheduleQueue(req, queue);
}
return Status::OK();
}
} // namespace
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
ReqQueue::ReqQueue() {
}
ReqQueue::~ReqQueue() {
}
BaseReqPtr
ReqQueue::TakeReq() {
return Take();
}
Status
ReqQueue::PutReq(const BaseReqPtr& req_ptr) {
std::unique_lock<std::mutex> lock(mtx);
full_.wait(lock, [this] { return (queue_.size() < capacity_); });
auto status = ScheduleReq(req_ptr, queue_);
empty_.notify_all();
return status;
}
} // namespace server
} // namespace milvus

44
proxy/src/server/delivery/ReqQueue.h Normal file
View File

@ -0,0 +1,44 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "server/delivery/request/BaseReq.h"
#include "utils/BlockingQueue.h"
#include "utils/Status.h"
#include <map>
#include <memory>
#include <string>
#include <thread>
#include <vector>
namespace milvus {
namespace server {
using BlockingReqQueue = BlockingQueue<BaseReqPtr>;
class ReqQueue : public BlockingReqQueue {
public:
ReqQueue();
virtual ~ReqQueue();
BaseReqPtr
TakeReq();
Status
PutReq(const BaseReqPtr& req_ptr);
};
using ReqQueuePtr = std::shared_ptr<ReqQueue>;
} // namespace server
} // namespace milvus

156
proxy/src/server/delivery/ReqScheduler.cpp Normal file
View File

@ -0,0 +1,156 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/ReqScheduler.h"
#include "utils/Log.h"
#include <unistd.h>
#include <utility>
namespace milvus {
namespace server {
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
ReqScheduler::ReqScheduler() : stopped_(false) {
Start();
}
ReqScheduler::~ReqScheduler() {
Stop();
}
void
ReqScheduler::ExecReq(const BaseReqPtr& req_ptr) {
if (req_ptr == nullptr) {
return;
}
ReqScheduler& scheduler = ReqScheduler::GetInstance();
scheduler.ExecuteReq(req_ptr);
}
void
ReqScheduler::Start() {
if (!stopped_) {
return;
}
stopped_ = false;
}
void
ReqScheduler::Stop() {
if (stopped_ && req_groups_.empty() && execute_threads_.empty()) {
return;
}
LOG_SERVER_INFO_ << "Scheduler gonna stop...";
{
std::lock_guard<std::mutex> lock(queue_mtx_);
for (auto& iter : req_groups_) {
if (iter.second != nullptr) {
iter.second->Put(nullptr);
}
}
}
for (auto& iter : execute_threads_) {
if (iter == nullptr)
continue;
iter->join();
}
req_groups_.clear();
execute_threads_.clear();
stopped_ = true;
LOG_SERVER_INFO_ << "Scheduler stopped";
}
Status
ReqScheduler::ExecuteReq(const BaseReqPtr& req_ptr) {
if (req_ptr == nullptr) {
return Status::OK();
}
auto status = req_ptr->PreExecute();
if (!status.ok()) {
req_ptr->Done();
return status;
}
status = PutToQueue(req_ptr);
if (!status.ok()) {
LOG_SERVER_ERROR_ << "Put request to queue failed with code: " << status.ToString();
req_ptr->Done();
return status;
}
if (req_ptr->async()) {
return Status::OK(); // async execution, caller need to call WaitToFinish at somewhere
}
status = req_ptr->WaitToFinish(); // sync execution
if (!status.ok()) {
return status;
}
return req_ptr->PostExecute();
}
void
ReqScheduler::TakeToExecute(ReqQueuePtr req_queue) {
SetThreadName("reqsched_thread");
if (req_queue == nullptr) {
return;
}
while (true) {
BaseReqPtr req = req_queue->TakeReq();
if (req == nullptr) {
LOG_SERVER_ERROR_ << "Take null from request queue, stop thread";
break; // stop the thread
}
try {
auto status = req->Execute();
if (!status.ok()) {
LOG_SERVER_ERROR_ << "Req failed with code: " << status.ToString();
}
} catch (std::exception& ex) {
LOG_SERVER_ERROR_ << "Req failed to execute: " << ex.what();
}
}
}
Status
ReqScheduler::PutToQueue(const BaseReqPtr& req_ptr) {
std::lock_guard<std::mutex> lock(queue_mtx_);
std::string group_name = req_ptr->req_group();
if (req_groups_.count(group_name) > 0) {
req_groups_[group_name]->PutReq(req_ptr);
} else {
ReqQueuePtr queue = std::make_shared<ReqQueue>();
queue->PutReq(req_ptr);
req_groups_.insert(std::make_pair(group_name, queue));
// start a thread
ThreadPtr thread = std::make_shared<std::thread>(&ReqScheduler::TakeToExecute, this, queue);
execute_threads_.push_back(thread);
LOG_SERVER_INFO_ << "Create new thread for request group: " << group_name;
}
return Status::OK();
}
} // namespace server
} // namespace milvus

70
proxy/src/server/delivery/ReqScheduler.h Normal file
View File

@ -0,0 +1,70 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "server/delivery/ReqQueue.h"
#include "utils/Status.h"
#include <map>
#include <memory>
#include <string>
#include <thread>
#include <vector>
namespace milvus {
namespace server {
using ThreadPtr = std::shared_ptr<std::thread>;
class ReqScheduler {
public:
static ReqScheduler&
GetInstance() {
static ReqScheduler scheduler;
return scheduler;
}
void
Start();
void
Stop();
Status
ExecuteReq(const BaseReqPtr& req_ptr);
static void
ExecReq(const BaseReqPtr& req_ptr);
protected:
ReqScheduler();
virtual ~ReqScheduler();
void
TakeToExecute(ReqQueuePtr req_queue);
Status
PutToQueue(const BaseReqPtr& req_ptr);
private:
mutable std::mutex queue_mtx_;
std::map<std::string, ReqQueuePtr> req_groups_;
std::vector<ThreadPtr> execute_threads_;
bool stopped_;
};
} // namespace server
} // namespace milvus

81
proxy/src/server/delivery/request/BaseReq.cpp Normal file
View File

@ -0,0 +1,81 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/request/BaseReq.h"
namespace milvus {
namespace server {
BaseReq::BaseReq(const ContextPtr& context, ReqType type, bool async)
: context_(context), type_(type), async_(async), done_(false) {
req_group_ = GetReqGroup(type);
if (nullptr != context_) {
context_->SetReqType(type_);
}
}
BaseReq::~BaseReq() {
WaitToFinish();
}
Status
BaseReq::PreExecute() {
status_ = OnPreExecute();
if (!status_.ok()) {
Done();
}
return status_;
}
Status
BaseReq::Execute() {
status_ = OnExecute();
Done();
return status_;
}
Status
BaseReq::PostExecute() {
status_ = OnPostExecute();
return status_;
}
Status
BaseReq::OnPreExecute() {
return Status::OK();
}
Status
BaseReq::OnPostExecute() {
return Status::OK();
}
void
BaseReq::Done() {
std::unique_lock<std::mutex> lock(finish_mtx_);
done_ = true;
finish_cond_.notify_all();
}
void
BaseReq::SetStatus(const Status& status) {
status_ = status;
}
Status
BaseReq::WaitToFinish() {
std::unique_lock<std::mutex> lock(finish_mtx_);
finish_cond_.wait(lock, [this] { return done_; });
return status_;
}
} // namespace server
} // namespace milvus

103
proxy/src/server/delivery/request/BaseReq.h Normal file
View File

@ -0,0 +1,103 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "server/context/Context.h"
#include "server/delivery/request/Types.h"
#include "utils/Status.h"
#include <condition_variable>
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
namespace milvus {
namespace server {
class BaseReq {
protected:
BaseReq(const ContextPtr& context, ReqType type, bool async = false);
virtual ~BaseReq();
public:
const ContextPtr&
context() const {
return context_;
}
ReqType
type() const {
return type_;
}
std::string
req_group() const {
return req_group_;
}
const Status&
status() const {
return status_;
}
bool
async() const {
return async_;
}
Status
PreExecute();
Status
Execute();
Status
PostExecute();
void
Done();
Status
WaitToFinish();
void
SetStatus(const Status& status);
protected:
virtual Status
OnPreExecute();
virtual Status
OnExecute() = 0;
virtual Status
OnPostExecute();
protected:
const ContextPtr context_;
ReqType type_;
std::string req_group_;
bool async_;
Status status_;
private:
mutable std::mutex finish_mtx_;
std::condition_variable finish_cond_;
bool done_;
};
using BaseReqPtr = std::shared_ptr<BaseReq>;
} // namespace server
} // namespace milvus

64
proxy/src/server/delivery/request/CmdReq.cpp Normal file
View File

@ -0,0 +1,64 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/request/CmdReq.h"
#include "config/ConfigMgr.h"
// #include "metrics/SystemInfo.h"
// #include "scheduler/SchedInst.h"
#include "src/version.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include <algorithm>
#include <cctype>
#include <memory>
#include <vector>
namespace milvus {
namespace server {
CmdReq::CmdReq(const ContextPtr& context, const std::string& cmd, std::string& result)
: BaseReq(context, ReqType::kCmd), origin_cmd_(cmd), cmd_(tolower(cmd)), result_(result) {
}
BaseReqPtr
CmdReq::Create(const ContextPtr& context, const std::string& cmd, std::string& result) {
return std::shared_ptr<BaseReq>(new CmdReq(context, cmd, result));
}
Status
CmdReq::OnExecute() {
std::string hdr = "CmdReq(cmd=" + cmd_ + ")";
TimeRecorderAuto rc(hdr);
Status stat = Status::OK();
return stat;
}
std::vector<std::string>
CmdReq::split(const std::string& src, char delimiter) {
std::stringstream ss(src);
std::vector<std::string> words;
std::string word;
while (std::getline(ss, word, delimiter)) {
words.push_back(word);
}
return words;
}
std::string
CmdReq::tolower(std::string s) {
std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); });
return s;
}
} // namespace server
} // namespace milvus

48
proxy/src/server/delivery/request/CmdReq.h Normal file
View File

@ -0,0 +1,48 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "BaseReq.h"
#include <memory>
#include <string>
#include <vector>
namespace milvus {
namespace server {
class CmdReq : public BaseReq {
public:
static BaseReqPtr
Create(const ContextPtr& context, const std::string& cmd, std::string& result);
protected:
CmdReq(const ContextPtr& context, const std::string& cmd, std::string& result);
Status
OnExecute() override;
private:
static std::vector<std::string>
split(const std::string& src, char delimiter);
static std::string
tolower(std::string s);
private:
const std::string origin_cmd_;
const std::string cmd_;
std::string& result_;
};
} // namespace server
} // namespace milvus

46
proxy/src/server/delivery/request/CompactReq.cpp Normal file
View File

@ -0,0 +1,46 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "server/delivery/request/CompactReq.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include <memory>
#include <unordered_map>
#include <vector>
namespace milvus {
namespace server {
CompactReq::CompactReq(const ContextPtr& context, const std::string& collection_name, double compact_threshold)
: BaseReq(context, ReqType::kCompact), collection_name_(collection_name), compact_threshold_(compact_threshold) {
}
BaseReqPtr
CompactReq::Create(const ContextPtr& context, const std::string& collection_name, double compact_threshold) {
return std::shared_ptr<BaseReq>(new CompactReq(context, collection_name, compact_threshold));
}
Status
CompactReq::OnExecute() {
return Status::OK();
}
} // namespace server
} // namespace milvus

45
proxy/src/server/delivery/request/CompactReq.h Normal file
View File

@ -0,0 +1,45 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "server/delivery/request/BaseReq.h"
#include <memory>
#include <string>
namespace milvus {
namespace server {
class CompactReq : public BaseReq {
public:
static BaseReqPtr
Create(const ContextPtr& context, const std::string& collection_name, double compact_threshold);
protected:
CompactReq(const ContextPtr& context, const std::string& collection_name, double compact_threshold);
Status
OnExecute() override;
private:
const std::string collection_name_;
double compact_threshold_ = 0.0;
};
} // namespace server
} // namespace milvus

50
proxy/src/server/delivery/request/CountEntitiesReq.cpp Normal file
View File

@ -0,0 +1,50 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/request/CountEntitiesReq.h"
#include "BaseReq.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include <memory>
#include <unordered_map>
#include <vector>
namespace milvus {
namespace server {
CountEntitiesReq::CountEntitiesReq(const ContextPtr& context, const std::string& collection_name, int64_t& row_count)
: BaseReq(context, ReqType::kCountEntities), collection_name_(collection_name), row_count_(row_count) {
}
BaseReqPtr
CountEntitiesReq::Create(const ContextPtr& context, const std::string& collection_name, int64_t& row_count) {
return std::shared_ptr<BaseReq>(new CountEntitiesReq(context, collection_name, row_count));
}
Status
CountEntitiesReq::OnExecute() {
try {
std::string hdr = "CountEntitiesReq(collection=" + collection_name_ + ")";
TimeRecorderAuto rc(hdr);
rc.ElapseFromBegin("done");
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}
return Status::OK();
}
} // namespace server
} // namespace milvus

39
proxy/src/server/delivery/request/CountEntitiesReq.h Normal file
View File

@ -0,0 +1,39 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "server/delivery/request/BaseReq.h"
#include <memory>
#include <string>
namespace milvus {
namespace server {
class CountEntitiesReq : public BaseReq {
public:
static BaseReqPtr
Create(const ContextPtr& context, const std::string& collection_name, int64_t& row_count);
protected:
CountEntitiesReq(const ContextPtr& context, const std::string& collection_name, int64_t& row_count);
Status
OnExecute() override;
private:
const std::string collection_name_;
int64_t& row_count_;
};
} // namespace server
} // namespace milvus

45
proxy/src/server/delivery/request/CreateCollectionReq.cpp Normal file
View File

@ -0,0 +1,45 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/request/CreateCollectionReq.h"
// #include "db/Utils.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include <set>
namespace milvus {
namespace server {
CreateCollectionReq::CreateCollectionReq(const ContextPtr& context, const std::string& collection_name,
FieldsType& fields, milvus::json& extra_params)
: BaseReq(context, ReqType::kCreateCollection),
collection_name_(collection_name),
fields_(fields),
extra_params_(extra_params) {
}
BaseReqPtr
CreateCollectionReq::Create(const ContextPtr& context, const std::string& collection_name, FieldsType& fields,
milvus::json& extra_params) {
return std::shared_ptr<BaseReq>(new CreateCollectionReq(context, collection_name, fields, extra_params));
}
Status
CreateCollectionReq::OnExecute() {
return Status::OK();
}
} // namespace server
} // namespace milvus

43
proxy/src/server/delivery/request/CreateCollectionReq.h Normal file
View File

@ -0,0 +1,43 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <memory>
#include <string>
#include <unordered_map>
#include "server/delivery/request/BaseReq.h"
namespace milvus {
namespace server {
class CreateCollectionReq : public BaseReq {
public:
static BaseReqPtr
Create(const ContextPtr& context, const std::string& collection_name, FieldsType& fields,
milvus::json& extra_params);
protected:
CreateCollectionReq(const ContextPtr& context, const std::string& collection_name, FieldsType& fields,
milvus::json& extra_params);
Status
OnExecute() override;
private:
const std::string collection_name_;
std::unordered_map<std::string, FieldSchema> fields_;
milvus::json extra_params_;
};
} // namespace server
} // namespace milvus

49
proxy/src/server/delivery/request/CreateIndexReq.cpp Normal file
View File

@ -0,0 +1,49 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/request/CreateIndexReq.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
namespace milvus {
namespace server {
CreateIndexReq::CreateIndexReq(const ContextPtr& context, const std::string& collection_name,
const std::string& field_name, const std::string& index_name,
const milvus::json& json_params)
: BaseReq(context, ReqType::kCreateIndex),
collection_name_(collection_name),
field_name_(field_name),
index_name_(index_name),
json_params_(json_params) {
}
BaseReqPtr
CreateIndexReq::Create(const ContextPtr& context, const std::string& collection_name, const std::string& field_name,
const std::string& index_name, const milvus::json& json_params) {
return std::shared_ptr<BaseReq>(new CreateIndexReq(context, collection_name, field_name, index_name, json_params));
}
Status
CreateIndexReq::OnExecute() {
return Status::OK();
}
} // namespace server
} // namespace milvus

42
proxy/src/server/delivery/request/CreateIndexReq.h Normal file
View File

@ -0,0 +1,42 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <memory>
#include <string>
#include "server/delivery/request/BaseReq.h"
namespace milvus {
namespace server {
class CreateIndexReq : public BaseReq {
public:
static BaseReqPtr
Create(const ContextPtr& context, const std::string& collection_name, const std::string& field_name,
const std::string& index_name, const milvus::json& json_params);
protected:
CreateIndexReq(const ContextPtr& context, const std::string& collection_name, const std::string& field_name,
const std::string& index_name, const milvus::json& json_params);
Status
OnExecute() override;
private:
const std::string collection_name_;
const std::string field_name_;
const std::string index_name_;
const milvus::json json_params_;
};
} // namespace server
} // namespace milvus

44
proxy/src/server/delivery/request/CreatePartitionReq.cpp Normal file
View File

@ -0,0 +1,44 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/request/CreatePartitionReq.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include <memory>
#include <string>
#include <vector>
namespace milvus {
namespace server {
constexpr uint64_t MAX_PARTITION_NUM = 4096;
CreatePartitionReq::CreatePartitionReq(const ContextPtr& context, const std::string& collection_name,
const std::string& tag)
: BaseReq(context, ReqType::kCreatePartition), collection_name_(collection_name), tag_(tag) {
}
BaseReqPtr
CreatePartitionReq::Create(const ContextPtr& context, const std::string& collection_name, const std::string& tag) {
return std::shared_ptr<BaseReq>(new CreatePartitionReq(context, collection_name, tag));
}
Status
CreatePartitionReq::OnExecute() {
return Status::OK();
}
} // namespace server
} // namespace milvus

38
proxy/src/server/delivery/request/CreatePartitionReq.h Normal file
View File

@ -0,0 +1,38 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <memory>
#include <string>
#include "server/delivery/request/BaseReq.h"
namespace milvus {
namespace server {
class CreatePartitionReq : public BaseReq {
public:
static BaseReqPtr
Create(const ContextPtr& context, const std::string& collection_name, const std::string& tag);
protected:
CreatePartitionReq(const ContextPtr& context, const std::string& collection_name, const std::string& tag);
Status
OnExecute() override;
private:
const std::string collection_name_;
const std::string tag_;
};
} // namespace server
} // namespace milvus

50
proxy/src/server/delivery/request/DeleteEntityByIDReq.cpp Normal file
View File

@ -0,0 +1,50 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "server/delivery/request/DeleteEntityByIDReq.h"
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
namespace milvus {
namespace server {
DeleteEntityByIDReq::DeleteEntityByIDReq(const ContextPtr& context, const std::string& collection_name,
const engine::IDNumbers& entity_ids)
: BaseReq(context, ReqType::kDeleteEntityByID), collection_name_(collection_name), entity_ids_(entity_ids) {
}
BaseReqPtr
DeleteEntityByIDReq::Create(const ContextPtr& context, const std::string& collection_name,
const engine::IDNumbers& entity_ids) {
return std::shared_ptr<BaseReq>(new DeleteEntityByIDReq(context, collection_name, entity_ids));
}
Status
DeleteEntityByIDReq::OnExecute() {
return Status::OK();
}
} // namespace server
} // namespace milvus

47
proxy/src/server/delivery/request/DeleteEntityByIDReq.h Normal file
View File

@ -0,0 +1,47 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <memory>
#include <string>
#include <vector>
#include "server/delivery/request/BaseReq.h"
namespace milvus {
namespace server {
class DeleteEntityByIDReq : public BaseReq {
public:
static BaseReqPtr
Create(const ContextPtr& context, const std::string& collection_name, const engine::IDNumbers& entity_ids);
protected:
DeleteEntityByIDReq(const ContextPtr& context, const std::string& collection_name,
const engine::IDNumbers& entity_ids);
Status
OnExecute() override;
private:
const std::string collection_name_;
const engine::IDNumbers& entity_ids_;
};
} // namespace server
} // namespace milvus

56
proxy/src/server/delivery/request/DescribeIndexReq.cpp Normal file
View File

@ -0,0 +1,56 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/request/DescribeIndexReq.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
namespace milvus {
namespace server {
DescribeIndexReq::DescribeIndexReq(const std::shared_ptr<milvus::server::Context>& context,
const std::string& collection_name, const std::string& field_name,
std::string& index_name, milvus::json& json_params)
: BaseReq(context, ReqType::kDescribeIndex),
collection_name_(collection_name),
field_name_(field_name),
index_name_(index_name),
json_params_(json_params) {
}
BaseReqPtr
DescribeIndexReq::Create(const std::shared_ptr<milvus::server::Context>& context, const std::string& collection_name,
const std::string& field_name, std::string& index_name, milvus::json& json_params) {
return std::shared_ptr<BaseReq>(
new DescribeIndexReq(context, collection_name, field_name, index_name, json_params));
}
Status
DescribeIndexReq::OnExecute() {
try {
std::string hdr = "DescribeIndexReq(collection=" + collection_name_ + ")";
TimeRecorderAuto rc(hdr);
// step 1: check arguments
STATUS_CHECK(ValidateCollectionName(collection_name_));
STATUS_CHECK(ValidateFieldName(field_name_));
return Status::OK();
}
} // namespace server
} // namespace milvus

42
proxy/src/server/delivery/request/DescribeIndexReq.h Normal file
View File

@ -0,0 +1,42 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <memory>
#include <string>
#include "server/delivery/request/BaseReq.h"
namespace milvus {
namespace server {
class DescribeIndexReq : public BaseReq {
public:
static BaseReqPtr
Create(const std::shared_ptr<milvus::server::Context>& context, const std::string& collection_name,
const std::string& field_name, std::string& index_name, milvus::json& json_params);
protected:
DescribeIndexReq(const std::shared_ptr<milvus::server::Context>& context, const std::string& collection_name,
const std::string& field_name, std::string& index_name, milvus::json& json_params);
Status
OnExecute() override;
private:
const std::string collection_name_;
const std::string field_name_;
std::string& index_name_;
milvus::json& json_params_;
};
} // namespace server
} // namespace milvus

42
proxy/src/server/delivery/request/DropCollectionReq.cpp Normal file
View File

@ -0,0 +1,42 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/request/DropCollectionReq.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
namespace milvus {
namespace server {
DropCollectionReq::DropCollectionReq(const ContextPtr& context, const std::string& collection_name)
: BaseReq(context, ReqType::kDropCollection), collection_name_(collection_name) {
}
BaseReqPtr
DropCollectionReq::Create(const ContextPtr& context, const std::string& collection_name) {
return std::shared_ptr<BaseReq>(new DropCollectionReq(context, collection_name));
}
Status
DropCollectionReq::OnExecute() {
try {
std::string hdr = "DropCollectionReq(collection=" + collection_name_ + ")";
TimeRecorder rc(hdr);
STATUS_CHECK(ValidateCollectionName(collection_name_));
return Status::OK();
}
} // namespace server
} // namespace milvus

38
proxy/src/server/delivery/request/DropCollectionReq.h Normal file
View File

@ -0,0 +1,38 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "server/delivery/request/BaseReq.h"
#include <memory>
#include <string>
namespace milvus {
namespace server {
class DropCollectionReq : public BaseReq {
public:
static BaseReqPtr
Create(const ContextPtr& context, const std::string& collection_name);
protected:
DropCollectionReq(const ContextPtr& context, const std::string& collection_name);
Status
OnExecute() override;
private:
std::string collection_name_;
};
} // namespace server
} // namespace milvus

48
proxy/src/server/delivery/request/DropIndexReq.cpp Normal file
View File

@ -0,0 +1,48 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/request/DropIndexReq.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include <memory>
#include <unordered_map>
#include <vector>
namespace milvus {
namespace server {
DropIndexReq::DropIndexReq(const ContextPtr& context, const std::string& collection_name, const std::string& field_name,
const std::string& index_name)
: BaseReq(context, ReqType::kDropIndex),
collection_name_(collection_name),
field_name_(field_name),
index_name_(index_name) {
}
BaseReqPtr
DropIndexReq::Create(const ContextPtr& context, const std::string& collection_name, const std::string& field_name,
const std::string& index_name) {
return std::shared_ptr<BaseReq>(new DropIndexReq(context, collection_name, field_name, index_name));
}
Status
DropIndexReq::OnExecute() {
try {
std::string hdr = "DropIndexReq(collection=" + collection_name_ + ")";
TimeRecorderAuto rc(hdr);
return Status::OK();
}
} // namespace server
} // namespace milvus

42
proxy/src/server/delivery/request/DropIndexReq.h Normal file
View File

@ -0,0 +1,42 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "server/delivery/request/BaseReq.h"
#include <memory>
#include <string>
namespace milvus {
namespace server {
class DropIndexReq : public BaseReq {
public:
static BaseReqPtr
Create(const ContextPtr& context, const std::string& collection_name, const std::string& field_name,
const std::string& index_name);
protected:
DropIndexReq(const ContextPtr& context, const std::string& collection_name, const std::string& field_name,
const std::string& index_name);
Status
OnExecute() override;
private:
const std::string collection_name_;
const std::string field_name_;
const std::string index_name_;
};
} // namespace server
} // namespace milvus

40
proxy/src/server/delivery/request/DropPartitionReq.cpp Normal file
View File

@ -0,0 +1,40 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/request/DropPartitionReq.h"
#include "server/ValidationUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include <memory>
#include <string>
namespace milvus {
namespace server {
DropPartitionReq::DropPartitionReq(const ContextPtr& context, const std::string& collection_name,
const std::string& tag)
: BaseReq(context, ReqType::kDropPartition), collection_name_(collection_name), tag_(tag) {
}
BaseReqPtr
DropPartitionReq::Create(const ContextPtr& context, const std::string& collection_name, const std::string& tag) {
return std::shared_ptr<BaseReq>(new DropPartitionReq(context, collection_name, tag));
}
Status
DropPartitionReq::OnExecute() {
return Status::OK();
}
} // namespace server
} // namespace milvus

38
proxy/src/server/delivery/request/DropPartitionReq.h Normal file
View File

@ -0,0 +1,38 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <memory>
#include <string>
#include "server/delivery/request/BaseReq.h"
namespace milvus {
namespace server {
class DropPartitionReq : public BaseReq {
public:
static BaseReqPtr
Create(const ContextPtr& context, const std::string& collection_name, const std::string& tag);
protected:
DropPartitionReq(const ContextPtr& context, const std::string& collection_name, const std::string& tag);
Status
OnExecute() override;
private:
const std::string collection_name_;
const std::string tag_;
};
} // namespace server
} // namespace milvus

53
proxy/src/server/delivery/request/FlushReq.cpp Normal file
View File

@ -0,0 +1,53 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "server/delivery/request/FlushReq.h"
#include <memory>
#include <unordered_map>
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
namespace milvus {
namespace server {
FlushReq::FlushReq(const ContextPtr& context, const std::vector<std::string>& collection_names)
: BaseReq(context, ReqType::kFlush), collection_names_(collection_names) {
}
BaseReqPtr
FlushReq::Create(const ContextPtr& context, const std::vector<std::string>& collection_names) {
return std::shared_ptr<BaseReq>(new FlushReq(context, collection_names));
}
Status
FlushReq::OnExecute() {
std::string hdr = "FlushReq flush collections: ";
for (auto& name : collection_names_) {
hdr += name;
hdr += ", ";
}
TimeRecorderAuto rc(hdr);
LOG_SERVER_DEBUG_ << hdr;
return Status::OK();
}
} // namespace server
} // namespace milvus

45
proxy/src/server/delivery/request/FlushReq.h Normal file
View File

@ -0,0 +1,45 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "BaseReq.h"
#include <memory>
#include <string>
#include <vector>
namespace milvus {
namespace server {
class FlushReq : public BaseReq {
public:
static BaseReqPtr
Create(const ContextPtr& context, const std::vector<std::string>& collection_names);
protected:
FlushReq(const ContextPtr& context, const std::vector<std::string>& collection_names);
Status
OnExecute() override;
private:
std::vector<std::string> collection_names_;
};
} // namespace server
} // namespace milvus

Some files were not shown because too many files have changed in this diff Show More