milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[skip e2e]remove files under tests-deprecating (#13615) 

Signed-off-by: Jenny Li <jing.li@zilliz.com>
pull/13656/head
Jenny Li 2021-12-17 18:48:13 +08:00 committed by GitHub
parent 43fbb0fd16
commit b5758e6342
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
269 changed files with 0 additions and 29310 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
tests-deprecating/OWNERS
View File

@ -1,13 +0,0 @@
reviewers:
- binbinlv
- ThreadDao
- wangting0128
- yanliang567
approvers:
- maintainers
labels:
- area/test
- sig/testing

91
tests-deprecating/README.md
View File

@ -1,91 +0,0 @@
## Tests
### E2E Test
#### 配置清单
##### 操作系统
| 操作系统 | 版本 |
| ------ | --------- |
| CentOS | 7.5 或以上 |
| Ubuntu | 16.04 或以上 |
| Mac | 10.14 或以上 |
##### 硬件
| 硬件名称 | 建议配置 |
| ---- | --------------------------------------------------------------------------------------------------- |
| CPU | x86_64 平台<br> Intel CPU Sandy Bridge 或以上<br> CPU 指令集<br> _ SSE42<br> _ AVX<br> _ AVX2<br> _ AVX512 |
| 内存 | 16 GB 或以上 |
##### 软件
| 软件名称 | 版本 |
| -------------- | ---------- |
| Docker | 19.05 或以上 |
| Docker Compose | 1.25.5 或以上 |
| jq | 1.3 或以上 |
| kubectl | 1.14 或以上 |
| helm | 3.0 或以上 |
| kind | 0.10.0 或以上 |
#### 安装依赖
##### 检查 Docker 和 Docker Compose 状态
1. 确认 Docker Daemon 正在运行：
```shell
$ docker info
```
- 安装 Docker 步骤见 [Docker CE/EE 官方安装说明](https://docs.docker.com/get-docker/)进行安装
- 如果无法正常打印 Docker 相关信息，请启动 Docker Daemon。
- 要在没有 `root` 权限的情况下运行 Docker 命令，请创建 `docker` 组并添加用户，以运行：`sudo usermod -aG docker $USER`， 退出终端并重新登录，以使更改生效 ，详见 [使用非 root 用户管理 docker](https://docs.docker.com/install/linux/linux-postinstall/)。
2. 确认 Docker Compose 版本
```shell
$ docker-compose version
docker-compose version 1.25.5, build 8a1c60f6
docker-py version: 4.1.0
CPython version: 3.7.5
OpenSSL version: OpenSSL 1.1.1f 31 Mar 2020
```
- 安装 Docker Compose 步骤见 [Install Docker Compose](https://docs.docker.com/compose/install/)
##### 安装 jq
- 安装方式见 <https://stedolan.github.io/jq/download/>
##### 安装 kubectl
- 安装方式见 <https://kubernetes.io/docs/tasks/tools/>
##### 安装 helm
- 安装方式见 <https://helm.sh/docs/intro/install/>
##### 安装 kind
- 安装方式见 <https://kind.sigs.k8s.io/docs/user/quick-start/#installation>
#### 运行 E2E Test
```shell
$ cd tests/scripts
$ ./e2e-k8s.sh
```
> Getting help
>
> 你可以执行以下命令获取帮助
>
> ```shell
> $ ./e2e-k8s.sh --help
> ```

7
tests-deprecating/docker/.env
View File

@ -1,7 +0,0 @@
MILVUS_SERVICE_IP=127.0.0.1
MILVUS_SERVICE_PORT=19530
MILVUS_PYTEST_WORKSPACE=/milvus/tests/python_test
MILVUS_PYTEST_LOG_PATH=/milvus/_artifacts/tests/pytest_logs
IMAGE_REPO=milvusdb
IMAGE_TAG=20210802-87c5a49
LATEST_IMAGE_TAG=20210802-87c5a49

18
tests-deprecating/docker/Dockerfile
View File

@ -1,18 +0,0 @@
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
FROM python:3.6.8-jessie
COPY ./tests/python_test/requirements.txt /requirements.txt
RUN python3 -m pip install --no-cache-dir -r /requirements.txt
CMD ["tail", "-f", "/dev/null"]

23
tests-deprecating/docker/docker-compose.yml
View File

@ -1,23 +0,0 @@
version: '3.5'
services:
pytest:
image: ${IMAGE_REPO}/pytest:${IMAGE_TAG}
build:
context: ../..
dockerfile: tests/docker/Dockerfile
cache_from:
- ${IMAGE_REPO}/pytest:${LATEST_IMAGE_TAG}
shm_size: 2G
environment:
MILVUS_SERVICE_IP: ${MILVUS_SERVICE_IP}
MILVUS_SERVICE_PORT: ${MILVUS_SERVICE_PORT}
CI_LOG_PATH: ${MILVUS_PYTEST_LOG_PATH}
volumes:
- ../../:/milvus:delegated
working_dir: ${MILVUS_PYTEST_WORKSPACE}
networks:
default:
external: ${PRE_EXIST_NETWORK:-false}
name: ${PYTEST_NETWORK:-milvus_dev}

57
tests-deprecating/go/insert_test.go
View File

@ -1,57 +0,0 @@
package test
import (
//"context"
//"sync"
"testing"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
type Timestamp = typeutil.Timestamp
//func GetInsertMsg(collectionName string, partitionTag string, entityId UniqueID) *msgpb.InsertOrDeleteMsg {
// return &msgpb.InsertOrDeleteMsg{
// CollectionName: collectionName,
// PartitionTag: partitionTag,
// SegmentID: UniqueID(entityId / 100),
// Uid: UniqueID(entityId),
// Timestamp: Timestamp(entityId),
// ClientId: 0,
// }
//}
//
//func GetDeleteMsg(collectionName string, entityId UniqueID) *msgpb.InsertOrDeleteMsg {
// return &msgpb.InsertOrDeleteMsg{
// CollectionName: collectionName,
// Uid: entityId,
// Timestamp: Timestamp(entityId + 100),
// }
//}
func TestInsert(t *testing.T) {
// TODO: fix test
//return
//ctx := context.Background()
//var topics []string
//topics = append(topics, "test")
//topics = append(topics, "test1")
//writerNode, _ := writer.NewWriteNode(ctx, "null", topics, 0)
//var insertMsgs []*msgpb.InsertOrDeleteMsg
//for i := 0; i < 120; i++ {
// insertMsgs = append(insertMsgs, GetInsertMsg("collection0", "tag01", UniqueID(i)))
//}
//wg := sync.WaitGroup{}
//wg.Add(3)
////var wg sync.WaitGroup
//writerNode.InsertBatchData(ctx, insertMsgs, &wg)
//var insertMsgs2 []*msgpb.InsertOrDeleteMsg
//for i := 120; i < 200; i++ {
// insertMsgs2 = append(insertMsgs2, GetInsertMsg("collection0", "tag02", UniqueID(i)))
//}
//writerNode.InsertBatchData(ctx, insertMsgs2, &wg)
//var deleteMsgs []*msgpb.InsertOrDeleteMsg
//deleteMsgs = append(deleteMsgs, GetDeleteMsg("collection0", 2))
//deleteMsgs = append(deleteMsgs, GetDeleteMsg("collection0", 120))
//writerNode.DeleteBatchData(ctx, deleteMsgs)
}

39
tests-deprecating/go/key2seg_test.go
View File

@ -1,39 +0,0 @@
package test
import (
"testing"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
type UniqueID = typeutil.UniqueID
func TestKey2Seg(t *testing.T) {
// TODO: fix test
//return
//
//lookupUrl := "pulsar://localhost:6650"
//client, err := pulsar.NewClient(pulsar.ClientOptions{
// URL: lookupUrl,
//})
//if err != nil {
// log.Fatal(err)
//}
//
//consumer, err := client.Subscribe(pulsar.ConsumerOptions{
// Topic: "Key2Seg",
// SubscriptionName: "sub-1",
//})
//
//obj := msgpb.Key2SegMsg{}
//msg, err := consumer.Receive(context.Background())
//proto.Unmarshal(msg.Payload(), &obj)
//assert.Equal(t, obj.Uid, UniqueID(0))
//consumer.Ack(msg)
//msg, err = consumer.Receive(context.Background())
//proto.Unmarshal(msg.Payload(), &obj)
//assert.Equal(t, obj.Uid, UniqueID(0))
//consumer.Ack(msg)
//consumer.Close()
//client.Close()
}

13
tests-deprecating/milvus_benchmark/.gitignore vendored
View File

@ -1,13 +0,0 @@
random_data
benchmark_logs/
db/
*idmap*.txt
__pycache__/
venv
.idea
nohup.out
*.swp
*.swo
.DS_Store
.vscode

30
tests-deprecating/milvus_benchmark/Dockerfile
View File

@ -1,30 +0,0 @@
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
FROM python:3.6.8-jessie
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN apt-get update && apt-get install -y --no-install-recommends wget apt-transport-https && \
wget -qO- "https://get.helm.sh/helm-v3.6.1-linux-amd64.tar.gz" | tar --strip-components=1 -xz -C /usr/local/bin linux-amd64/helm && \
wget -P /tmp https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg && \
apt-key add /tmp/apt-key.gpg && \
sh -c 'echo deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main > /etc/apt/sources.list.d/kubernetes.list' && \
apt-get update && apt-get install -y --no-install-recommends \
build-essential kubectl && \
apt-get remove --purge -y && \
rm -rf /var/lib/apt/lists/*
COPY requirements.txt /requirements.txt
RUN python3 -m pip install --no-cache-dir -r /requirements.txt
WORKDIR /root

184
tests-deprecating/milvus_benchmark/README.md
View File

@ -1,184 +0,0 @@
The milvus_benchmark is a non-functional testing tool or service which allows users to run tests on k8s cluster or at local, the primary use case is performance/load/stability testing, the objective is to expose problems in milvus project.
## Quick start
### Description
- Test cases in `milvus_benchmark` can be organized with `yaml`
- Test can run with local mode or helm mode
- local: install and start your local server, and pass the host/port param when starting the tests
- helm: install the server by helm, which will manage the milvus in k8s cluster, and you can integrate the test stage into argo workflow or jenkins pipeline
### Usage
- Using jenkins:
Use `ci/main_jenkinsfile` as the jenkins pipeline file
- Using argo：
example argo workflow yaml configuration: `ci/argo.yaml`
- Local test：
1. set PYTHONPATH:
`export PYTHONPATH=/your/project/path/milvus_benchmark`
2. prepare data:
if we need to use the sift/deep dataset as the raw data input, we need to mount NAS and update `RAW_DATA_DIR` in `config.py`, the example mount command:
`sudo mount -t cifs -o username=test,vers=1.0 //172.16.70.249/test /test`
3. install requirements:
`pip install -r requirements.txt`
4. write test yaml and run with the yaml param:
`cd milvus-benchmark/ && python main.py --local --host=* --port=19530 --suite=suites/2_insert_data.yaml`
### Test suite
#### Description
Test suite yaml defines the test process, users need to add test suite yaml if adding a customized test into the current test framework.
#### Example
Take the test file `2_insert_data.yaml` as an example
```
insert_performance:
collections:
-
milvus:
db_config.primary_path: /test/milvus/db_data_2/cluster/sift_1m_128_l2
wal_enable: true
collection_name: sift_1m_128_l2
ni_per: 50000
build_index: false
index_type: ivf_sq8
index_param:
nlist: 1024
```
- `insert_performance`
The top level is the runner type: the other test types including: `search_performance/build_performance/insert_performance/accuracy/locust_insert/...`, each test type corresponds to the different runner conponent defined in directory `runnners`
- other fields under runner type
The other parts in the test yaml is the params pass to the runner, such as:
- The field `collection_name` means which kind of collection will be created in milvus
- The field `ni_per` means the batch size
- The filed `build_index` means that whether to create index during inserting
While using argo workflow as benchmark pipeline, the test suite is made of both `client` and `server` configmap, an example will be like this:
`server`
```
kind: ConfigMap
apiVersion: v1
metadata:
name: server-cluster-8c16m
namespace: qa
uid: 3752f85c-c840-40c6-a5db-ae44146ad8b5
resourceVersion: '42213135'
creationTimestamp: '2021-05-14T07:00:53Z'
managedFields:
- manager: dashboard
operation: Update
apiVersion: v1
time: '2021-05-14T07:00:53Z'
fieldsType: FieldsV1
fieldsV1:
'f:data':
.: {}
'f:config.yaml': {}
data:
config.yaml: |
server:
server_tag: "8c16m"
milvus:
deploy_mode: "cluster"
```
`client`
```
kind: ConfigMap
apiVersion: v1
metadata:
name: client-insert-batch-1000
namespace: qa
uid: 8604c277-f00f-47c7-8fcb-9b3bc97efa74
resourceVersion: '42988547'
creationTimestamp: '2021-07-09T08:33:02Z'
managedFields:
- manager: dashboard
operation: Update
apiVersion: v1
fieldsType: FieldsV1
fieldsV1:
'f:data':
.: {}
'f:config.yaml': {}
data:
config.yaml: |
insert_performance:
collections:
-
milvus:
wal_enable: true
collection_name: sift_1m_128_l2
ni_per: 1000
build_index: false
index_type: ivf_sq8
index_param:
nlist: 1024
```
## Overview of the benchmark
### Components
- `main.py`
The entry file: parse the input params and initialize the other components: `metric`, `env`, `runner`
- `metric`
The test result can be used to analyze the regression or improvement of the milvus system, so we upload the metrics of the test result when a test suite run finished, and then use `redash` to make sense of our data
- `db`
Currently we use the `mongodb` to store the test result
- `env`
The `env` component defines the server environment and environment management, the instance of the `env` corresponds to the run mode of the benchmark
- `local`: Only defines the host and port for testing
- `helm/docker`: Install and uninstall the server in benchmark stage
- `runner`
The actual executor in benchmark, each test type defined in test suite will generate the corresponding runner instance, there are three stages in `runner`:
- `extract_cases`: There are several test cases defined in each test suite yaml, and each case shares the same server environment and shares the same `prepare` stage, but the `metric` for each case is different, so we need to extract cases from the test suite before the cases runs
- `prepare`: Prepare the data and operations, for example, before running searching, index needs to be created and data needs to be loaded
- `run_case`: Do the core operation and set `metric` value
- `suites`: There are two ways to take the content to be tested as input parameters：
- Test suite files under `suites` directory
- Test suite configmap name including `server_config_map` and `client_config_map` if using argo workflow
- `update.py`: While using argo workflow as benchmark pipeline, we have two steps in workflow template: `install-milvus` and `client-test`
- In stage `install-milvus`, `update.py` is used to generate a new `values.yaml` which will be a param while in `helm install` operation
- In stage `client-test`, it runs `main.py` and receives the milvus host and port as the cmd params, with the run mode `local`
### Conceptual overview
The following diagram shows the runtime execution graph of the benchmark (local mode based on argo workflow)
<img src="asserts/uml.jpg" />

BIN
tests-deprecating/milvus_benchmark/asserts/uml.jpg
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 138 KiB

232
tests-deprecating/milvus_benchmark/ci/argo.yaml
View File

@ -1,232 +0,0 @@
metadata:
name: benchmark
namespace: qa
uid: e8a51212-9b27-441d-b357-e73c63854ccf
resourceVersion: '63697833'
generation: 41
creationTimestamp: '2021-05-25T11:04:40Z'
labels:
workflows.argoproj.io/creator: system-serviceaccount-argo-argo-server
managedFields:
- manager: argo
operation: Update
apiVersion: argoproj.io/v1alpha1
fieldsType: FieldsV1
fieldsV1:
'f:metadata':
'f:labels':
.: {}
'f:workflows.argoproj.io/creator': {}
'f:spec':
.: {}
'f:arguments':
.: {}
'f:parameters': {}
'f:entrypoint': {}
'f:nodeSelector':
.: {}
'f:node-role.kubernetes.io/benchmark': {}
'f:onExit': {}
'f:serviceAccountName': {}
'f:templates': {}
'f:tolerations': {}
'f:volumes': {}
spec:
templates:
- name: benchmark-loop
inputs: {}
outputs: {}
metadata: {}
steps:
- - name: call-benchmark-test
template: benchmark
arguments:
parameters:
- name: server-instance
value: '{{workflow.name}}-{{item.instanceId}}'
- name: server-configmap
value: '{{item.server-configmap}}'
- name: client-configmap
value: '{{item.client-configmap}}'
withParam: '{{workflow.parameters.configmaps}}'
- name: uninstall-all
inputs: {}
outputs: {}
metadata: {}
steps:
- - name: uninstall-milvus
template: uninstall-milvus
arguments:
parameters:
- name: server-instance
value: '{{workflow.name}}-{{item.instanceId}}'
withParam: '{{workflow.parameters.configmaps}}'
- name: benchmark
inputs:
parameters:
- name: server-instance
- name: server-configmap
- name: client-configmap
outputs: {}
metadata: {}
steps:
- - name: install-milvus
template: install-milvus
arguments:
parameters:
- name: server-instance
value: '{{inputs.parameters.server-instance}}'
- name: server-configmap
value: '{{inputs.parameters.server-configmap}}'
- - name: client-test
template: client-test
arguments:
parameters:
- name: server-instance
value: '{{inputs.parameters.server-instance}}'
- name: server-configmap
value: '{{inputs.parameters.server-configmap}}'
- name: client-configmap
value: '{{inputs.parameters.client-configmap}}'
- name: uninstall-milvus
inputs:
parameters:
- name: server-instance
outputs: {}
metadata: {}
container:
name: ''
image: 'registry.zilliz.com/milvus/milvus-test-env:v0.5'
command:
- /bin/sh
- '-c'
args:
- ' helm uninstall -n qa-milvus {{inputs.parameters.server-instance}} && kubectl delete pvc -l app.kubernetes.io/instance={{inputs.parameters.server-instance}} -n qa-milvus '
resources: {}
volumeMounts:
- name: kube-config
mountPath: /root/.kube
- name: install-milvus
inputs:
parameters:
- name: server-instance
- name: server-configmap
artifacts:
- name: charts
path: /src/helm
git:
repo: 'git@github.com:milvus-io/milvus-helm.git'
revision: master
sshPrivateKeySecret:
name: github-key
key: ssh-private-key
- name: benchmark-src
path: /src/benchmark
git:
repo: 'git@github.com:zilliztech/milvus_benchmark.git'
revision: '{{workflow.parameters.test-client-branch}}'
sshPrivateKeySecret:
name: github-key
key: ssh-private-key
outputs: {}
metadata: {}
container:
name: ''
image: 'registry.zilliz.com/milvus/milvus-test-env:v0.5'
command:
- /bin/sh
- '-c'
args:
- ' cd /src/helm/charts/milvus && cp -r /src/benchmark/milvus_benchmark/* . && cp /configmap-server/config.yaml . && python update.py --src-values=values.yaml --deploy-params=config.yaml && cat values.yaml && helm install -n qa-milvus --set image.all.repository={{workflow.parameters.milvus-image-repository}} --set image.all.tag={{workflow.parameters.milvus-image-tag}} --set image.all.pullPolicy=Always --set etcd.persistence.enabled=false --set servicemonitor.enabled=true --wait --timeout 15m {{inputs.parameters.server-instance}} . && kubectl get pods -n qa-milvus -l app.kubernetes.io/instance={{inputs.parameters.server-instance}} '
resources: {}
volumeMounts:
- name: kube-config
readOnly: true
mountPath: /root/.kube
- name: benchmark-server-configmap
mountPath: /configmap-server
volumes:
- name: benchmark-server-configmap
configMap:
name: '{{inputs.parameters.server-configmap}}'
- name: client-test
inputs:
parameters:
- name: server-instance
- name: server-configmap
- name: client-configmap
artifacts:
- name: source
path: /src
git:
repo: 'git@github.com:zilliztech/milvus_benchmark.git'
revision: '{{workflow.parameters.test-client-branch}}'
sshPrivateKeySecret:
name: github-key
key: ssh-private-key
outputs: {}
metadata: {}
container:
name: ''
image: 'registry.zilliz.com/milvus/milvus-test-env:v0.5'
command:
- /bin/sh
- '-c'
args:
- ' cd /src && pip install -r requirements.txt -i https://pypi.doubanio.com/simple/ --trusted-host pypi.doubanio.com && pip install -i https://test.pypi.org/simple/ pymilvus=={{workflow.parameters.test-sdk-version}} && cd milvus_benchmark && export PYTHONPATH=/src && python main.py --host={{inputs.parameters.server-instance}}-milvus.qa-milvus.svc.cluster.local --local --suite=/configmap-client/config.yaml --server-config=/configmap-server/config.yaml'
resources:
limits:
cpu: '4'
memory: 4Gi
volumeMounts:
- name: kube-config
readOnly: true
mountPath: /root/.kube
- name: benchmark-server-configmap
mountPath: /configmap-server
- name: benchmark-client-configmap
mountPath: /configmap-client
- name: db-data-path
mountPath: /test
volumes:
- name: benchmark-server-configmap
configMap:
name: '{{inputs.parameters.server-configmap}}'
- name: benchmark-client-configmap
configMap:
name: '{{inputs.parameters.client-configmap}}'
- name: db-data-path
flexVolume:
driver: fstab/cifs
fsType: cifs
secretRef:
name: cifs-test-secret
options:
mountOptions: vers=1.0
networkPath: //172.16.70.249/test
activeDeadlineSeconds: 21600
entrypoint: benchmark-loop
arguments:
parameters:
- name: milvus-image-repository
value: harbor.zilliz.cc/dockerhub/milvusdb/milvus-dev
- name: milvus-image-tag
value: master-latest
- name: test-client-branch
value: master
- name: test-sdk-version
value: 2.0.0rc4.dev1
- name: configmaps
value: ' [ {"instanceId":"1", "server-configmap": "server-single-8c16m", "client-configmap": "client-acc-sift-ivf-flat" } ]'
serviceAccountName: qa-admin
volumes:
- name: kube-config
secret:
secretName: qa-admin-config
nodeSelector:
node-role.kubernetes.io/benchmark: ''
tolerations:
- key: node-role.kubernetes.io/benchmark
operator: Exists
effect: NoSchedule
onExit: uninstall-all

10
tests-deprecating/milvus_benchmark/ci/function/file_transfer.groovy
View File

@ -1,10 +0,0 @@
def FileTransfer (sourceFiles, remoteDirectory, remoteIP, protocol = "ftp", makeEmptyDirs = true) {
if (protocol == "ftp") {
ftpPublisher masterNodeName: '', paramPublish: [parameterName: ''], alwaysPublishFromMaster: false, continueOnError: false, failOnError: true, publishers: [
[configName: "${remoteIP}", transfers: [
[asciiMode: false, cleanRemote: false, excludes: '', flatten: false, makeEmptyDirs: "${makeEmptyDirs}", noDefaultExcludes: false, patternSeparator: '[, ]+', remoteDirectory: "${remoteDirectory}", remoteDirectorySDF: false, removePrefix: '', sourceFiles: "${sourceFiles}"]], usePromotionTimestamp: true, useWorkspaceInPromotion: false, verbose: true
]
]
}
}
return this

13
tests-deprecating/milvus_benchmark/ci/jenkinsfile/cleanup.groovy
View File

@ -1,13 +0,0 @@
try {
def result = sh script: "helm status -n milvus ${env.HELM_RELEASE_NAME}", returnStatus: true
if (!result) {
sh "helm uninstall -n milvus ${env.HELM_RELEASE_NAME}"
}
} catch (exc) {
def result = sh script: "helm status -n milvus ${env.HELM_RELEASE_NAME}", returnStatus: true
if (!result) {
sh "helm uninstall -n milvus ${env.HELM_RELEASE_NAME}"
}
throw exc
}

24
tests-deprecating/milvus_benchmark/ci/jenkinsfile/deploy_test.groovy
View File

@ -1,24 +0,0 @@
try {
dir ("milvus-helm-charts") {
// sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
// sh 'helm repo update'
checkout([$class: 'GitSCM', branches: [[name: "${HELM_BRANCH}"]], userRemoteConfigs: [[url: "${HELM_URL}", name: 'origin', refspec: "+refs/heads/${HELM_BRANCH}:refs/remotes/origin/${HELM_BRANCH}"]]])
}
// dir ("milvus_benchmark") {
// print "Git clone url: ${TEST_URL}:${TEST_BRANCH}"
// checkout([$class: 'GitSCM', branches: [[name: "${TEST_BRANCH}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "${TEST_URL}", name: 'origin', refspec: "+refs/heads/${TEST_BRANCH}:refs/remotes/origin/${TEST_BRANCH}"]]])
print "Install requirements"
// sh "python3 -m pip install pymilvus-distributed==0.0.57"
// sh "python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com"
sh "python3 -m pip install -r requirements.txt"
if ("${params.CLUSTER_NAME}" == "idc-kubernetes") {
sh "export KUBECONFIG=/root/kube/.kube/config && cd milvus_benchmark && export PYTHONPATH=${env.WORKSPACE}/ && python3 main.py --image-version=${params.IMAGE_VERSION} --schedule-conf=scheduler/${params.CONFIG_FILE}"
} else {
sh "cd milvus_benchmark && export PYTHONPATH=${env.WORKSPACE}/ && python3 main.py --image-version=${params.IMAGE_VERSION} --schedule-conf=scheduler/${params.CONFIG_FILE}"
}
// }
} catch (exc) {
echo 'Deploy Test Failed !'
throw exc
}

15
tests-deprecating/milvus_benchmark/ci/jenkinsfile/notify.groovy
View File

@ -1,15 +0,0 @@
def notify() {
if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
// Send an email only if the build status has changed from green/unstable to red
emailext subject: '$DEFAULT_SUBJECT',
body: '$DEFAULT_CONTENT',
recipientProviders: [
[$class: 'DevelopersRecipientProvider'],
[$class: 'RequesterRecipientProvider']
],
replyTo: '$DEFAULT_REPLYTO',
to: '$DEFAULT_RECIPIENTS'
}
}
return this

151
tests-deprecating/milvus_benchmark/ci/main_jenkinsfile
View File

@ -1,151 +0,0 @@
pipeline {
agent none
options {
timestamps()
}
parameters{
string defaultValue: 'master', description: 'server image version', name: 'IMAGE_VERSION', trim: true
choice choices: ['kubernetes', 'idc-kubernetes'], description: 'cluster name', name: 'CLUSTER_NAME'
string defaultValue: '2_data.json', description: 'test suite config yaml', name: 'CONFIG_FILE', trim: true
string defaultValue: 'd0928627-efb6-4cfd-8030-9bf635988d85', description: 'git credentials', name: 'GIT_USER', trim: true
}
environment {
HELM_URL = "https://github.com/zilliztech/milvus-helm-charts.git"
HELM_BRANCH = "main"
TEST_URL = "https://github.com/zilliztech/milvus_benchmark.git"
TEST_BRANCH = "distributed"
HELM_RELEASE_NAME = "distributed-benchmark-test-${env.BUILD_NUMBER}"
}
stages {
stage("Setup env") {
agent {
kubernetes {
cloud "${params.CLUSTER_NAME}"
label "test-benchmark-${env.JOB_NAME}-${env.BUILD_NUMBER}"
defaultContainer 'jnlp'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
app: milvus
componet: test
spec:
containers:
- name: milvus-test-env
image: registry.zilliz.com/milvus/milvus-test-env:v0.3
command:
- cat
tty: true
volumeMounts:
- name: kubeconf
mountPath: /root/.kube/
readOnly: true
- name: kubeconf2
mountPath: /root/kube/.kube/
readOnly: true
- name: db-data-path
mountPath: /test
readOnly: false
nodeSelector:
kubernetes.io/hostname: idc-sh002
tolerations:
- key: worker
operator: Equal
value: performance
effect: NoSchedule
volumes:
- name: kubeconf
secret:
secretName: test-cluster-config
- name: kubeconf2
secret:
secretName: idc-cluster-config
- name: db-data-path
flexVolume:
driver: "fstab/cifs"
fsType: "cifs"
secretRef:
name: "cifs-test-secret"
options:
networkPath: "//172.16.70.249/test"
mountOptions: "vers=1.0"
"""
}
}
stages {
stage("Publish Daily Docker images") {
steps {
container('milvus-test-env') {
script {
boolean isNightlyTest = isTimeTriggeredBuild()
if (isNightlyTest) {
// build job: 'milvus-publish-daily-docker', parameters: [[$class: 'StringParameterValue', name: 'BRANCH', value: "${params.IMAGE_VERSION}"]], wait: false
build job: 'milvus-publish-daily-docker', parameters: [string(name: 'LOCAL_DOKCER_REGISTRY_URL', value: 'registry.zilliz.com'), string(name: 'REMOTE_DOKCER_REGISTRY_URL', value: 'registry-1.docker.io'), string(name: 'REMOTE_DOCKER_CREDENTIALS_ID', value: 'milvus-docker-access-token'), string(name: 'BRANCH', value: String.valueOf(IMAGE_VERSION))], wait: false
} else {
echo "Skip publish daily docker images ..."
}
}
}
}
}
stage("Deploy Test") {
steps {
container('milvus-test-env') {
script {
print "In Deploy Test Stage"
// use the idc context
// sh 'kubectl config use-context idc001'
if ("${params.CLUSTER_NAME}" == "idc-kubernetes") {
print "Use cluster name idc001"
sh 'export KUBECONFIG=/root/kube/.kube/config'
}
load "${env.WORKSPACE}/ci/jenkinsfile/deploy_test.groovy"
}
}
}
}
stage ("Cleanup Env") {
steps {
container('milvus-test-env') {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/cleanup.groovy"
}
}
}
}
}
post {
success {
script {
echo "Milvus benchmark test success !"
}
}
aborted {
script {
echo "Milvus benchmark test aborted !"
}
}
failure {
script {
echo "Milvus benchmark test failed !"
}
}
}
}
}
}
boolean isTimeTriggeredBuild() {
if (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0) {
return true
}
return false
}

13
tests-deprecating/milvus_benchmark/ci/pod_containers/milvus-testframework.yaml
View File

@ -1,13 +0,0 @@
apiVersion: v1
kind: Pod
metadata:
labels:
app: milvus
componet: testframework
spec:
containers:
- name: milvus-testframework
image: registry.zilliz.com/milvus/milvus-test:v0.2
command:
- cat
tty: true

104
tests-deprecating/milvus_benchmark/ci/publish_jenkinsfile
View File

@ -1,104 +0,0 @@
pipeline {
agent none
options {
timestamps()
}
parameters{
string defaultValue: 'registry.zilliz.com', description: 'Local Docker registry URL', name: 'LOCAL_DOKCER_REGISTRY_URL', trim: true
string defaultValue: 'registry-1.docker.io', description: 'Remote Docker registry URL', name: 'REMOTE_DOKCER_REGISTRY_URL', trim: true
string defaultValue: 'milvus-docker-access-token', description: 'Remote Docker credentials id', name: 'REMOTE_DOCKER_CREDENTIALS_ID', trim: true
string(defaultValue: "master", description: 'Milvus server version', name: 'BRANCH')
}
environment {
DAILY_BUILD_VERSION = VersionNumber([
versionNumberString : '${BUILD_DATE_FORMATTED, "yyyyMMdd"}'
]);
}
stages {
stage('Push Daily Docker Images') {
matrix {
agent none
axes {
axis {
name 'OS_NAME'
values 'ubuntu18.04', 'centos7'
}
axis {
name 'CPU_ARCH'
values 'amd64'
}
axis {
name 'BINARY_VERSION'
values 'gpu', 'cpu'
}
}
stages {
stage("Publish Docker Images") {
environment {
DOCKER_VERSION = "${params.BRANCH}-${BINARY_VERSION}-${OS_NAME}-release"
REMOTE_DOCKER_VERSION = "${params.BRANCH}-${OS_NAME}-${BINARY_VERSION}-${DAILY_BUILD_VERSION}"
REMOTE_DOCKER_LATEST_VERSION = "${params.BRANCH}-${OS_NAME}-${BINARY_VERSION}-latest"
}
agent {
kubernetes {
label "${OS_NAME}-${BINARY_VERSION}-publish-${env.BUILD_NUMBER}"
defaultContainer 'jnlp'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
app: publish
componet: docker
spec:
containers:
- name: publish-images
image: registry.zilliz.com/library/docker:v1.0.0
securityContext:
privileged: true
command:
- cat
tty: true
resources:
limits:
memory: "4Gi"
cpu: "1.0"
requests:
memory: "2Gi"
cpu: "0.5"
volumeMounts:
- name: docker-sock
mountPath: /var/run/docker.sock
volumes:
- name: docker-sock
hostPath:
path: /var/run/docker.sock
"""
}
}
stages {
stage('Publish') {
steps {
container('publish-images') {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/publishDailyImages.groovy"
}
}
}
}
}
}
}
}
}
}
}

536
tests-deprecating/milvus_benchmark/ci/scripts/yaml_processor.py
View File

@ -1,536 +0,0 @@
#!/usr/bin/env python3
import sys
import argparse
from argparse import Namespace
import os, shutil
import getopt
from ruamel.yaml import YAML, yaml_object
from ruamel.yaml.comments import CommentedSeq, CommentedMap
from ruamel.yaml.tokens import CommentToken
##
yaml = YAML(typ="rt")
## format yaml file
yaml.indent(mapping=2, sequence=4, offset=2)
############################################
# Comment operation
#
############################################
def _extract_comment(_comment):
"""
remove '#' at start of comment
"""
# if _comment is empty, do nothing
if not _comment:
return _comment
# str_ = _comment.lstrip(" ")
str_ = _comment.strip()
str_ = str_.lstrip("#")
return str_
def _add_eol_comment(element, *args, **kwargs):
"""
add_eol_comment
args --> (comment, key)
"""
if element is None or \
(not isinstance(element, CommentedMap) and
not isinstance(element, CommentedSeq)) or \
args[0] is None or \
len(args[0]) == 0:
return
comment = args[0]
# comment is empty, do nothing
if not comment:
return
key = args[1]
try:
element.yaml_add_eol_comment(*args, **kwargs)
except Exception:
element.ca.items.pop(key, None)
element.yaml_add_eol_comment(*args, **kwargs)
def _map_comment(_element, _key):
origin_comment = ""
token = _element.ca.items.get(_key, None)
if token is not None:
try:
origin_comment = token[2].value
except Exception:
try:
# comment is below element, add profix "#\n"
col = _element.lc.col + 2
space_list = [" " for i in range(col)]
space_str = "".join(space_list)
origin_comment = "\n" + "".join([space_str + t.value for t in token[3]])
except Exception:
pass
return origin_comment
def _seq_comment(_element, _index):
# get target comment
_comment = ""
token = _element.ca.items.get(_index, None)
if token is not None:
_comment = token[0].value
return _comment
def _start_comment(_element):
_comment = ""
cmt = _element.ca.comment
try:
_comment = cmt[1][0].value
except Exception:
pass
return _comment
def _comment_counter(_comment):
"""
counter comment tips and split into list
"""
x = lambda l: l.strip().strip("#").strip()
_counter = []
if _comment.startswith("\n"):
_counter.append("")
_counter.append(x(_comment[1:]))
return _counter
elif _comment.startswith("#\n"):
_counter.append("")
_counter.append(x(_comment[2:]))
else:
index = _comment.find("\n")
_counter.append(x(_comment[:index]))
_counter.append(x(_comment[index + 1:]))
return _counter
def _obtain_comment(_m_comment, _t_comment):
if not _m_comment or not _t_comment:
return _m_comment or _t_comment
_m_counter = _comment_counter(_m_comment)
_t_counter = _comment_counter(_t_comment)
if not _m_counter[0] and not _t_counter[1]:
comment = _t_comment + _m_comment
elif not _m_counter[1] and not _t_counter[0]:
comment = _m_comment + _t_comment
elif _t_counter[0] and _t_counter[1]:
comment = _t_comment
elif not _t_counter[0] and not _t_counter[1]:
comment = _m_comment
elif not _m_counter[0] and not _m_counter[1]:
comment = _t_comment
else:
if _t_counter[0]:
comment = _m_comment.replace(_m_counter[0], _t_counter[0], 1)
else:
comment = _m_comment.replace(_m_counter[1], _t_counter[1], 1)
i = comment.find("\n\n")
while i >= 0:
comment = comment.replace("\n\n\n", "\n\n", 1)
i = comment.find("\n\n\n")
return comment
############################################
# Utils
#
############################################
def _get_update_par(_args):
_dict = _args.__dict__
# file path
_in_file = _dict.get("f", None) or _dict.get("file", None)
# tips
_tips = _dict.get('tips', None) or "Input \"-h\" for more information"
# update
_u = _dict.get("u", None) or _dict.get("update", None)
# apppend
_a = _dict.get('a', None) or _dict.get('append', None)
# out stream group
_i = _dict.get("i", None) or _dict.get("inplace", None)
_o = _dict.get("o", None) or _dict.get("out_file", None)
return _in_file, _u, _a, _i, _o, _tips
############################################
# Element operation
#
############################################
def update_map_element(element, key, value, comment, _type):
"""
element:
key:
value:
comment:
_type: value type.
"""
if element is None or not isinstance(element, CommentedMap):
print("Only key-value update support")
sys.exit(1)
origin_comment = _map_comment(element, key)
sub_element = element.get(key, None)
if isinstance(sub_element, CommentedMap) or isinstance(sub_element, CommentedSeq):
print("Only support update a single value")
element.update({key: value})
comment = _obtain_comment(origin_comment, comment)
_add_eol_comment(element, _extract_comment(comment), key)
def update_seq_element(element, value, comment, _type):
if element is None or not isinstance(element, CommentedSeq):
print("Param `-a` only use to append yaml list")
sys.exit(1)
element.append(str(value))
comment = _obtain_comment("", comment)
_add_eol_comment(element, _extract_comment(comment), len(element) - 1)
def run_update(code, keys, value, comment, _app):
key_list = keys.split(".")
space_str = ":\n "
key_str = "{}".format(key_list[0])
for key in key_list[1:]:
key_str = key_str + space_str + key
space_str = space_str + " "
if not _app:
yaml_str = """{}: {}""".format(key_str, value)
else:
yaml_str = "{}{}- {}".format(key_str, space_str, value)
if comment:
yaml_str = "{} # {}".format(yaml_str, comment)
mcode = yaml.load(yaml_str)
_merge(code, mcode)
def _update(code, _update, _app, _tips):
if not _update:
return code
_update_list = [l.strip() for l in _update.split(",")]
for l in _update_list:
try:
variant, comment = l.split("#")
except ValueError:
variant = l
comment = None
try:
keys, value = variant.split("=")
run_update(code, keys, value, comment, _app)
except ValueError:
print("Invalid format. print command \"--help\" get more info.")
sys.exit(1)
return code
def _backup(in_file_p):
backup_p = in_file_p + ".bak"
if os.path.exists(backup_p):
os.remove(backup_p)
if not os.path.exists(in_file_p):
print("File {} not exists.".format(in_file_p))
sys.exit(1)
shutil.copyfile(in_file_p, backup_p) # 复制文件
def _recovery(in_file_p):
backup_p = in_file_p + ".bak"
if not os.path.exists(in_file_p):
print("File {} not exists.".format(in_file_p))
sys.exit(1)
elif not os.path.exists(backup_p):
print("Backup file not exists")
sys.exit(0)
os.remove(in_file_p)
os.rename(backup_p, in_file_p)
# master merge target
def _merge(master, target):
if type(master) != type(target):
print("yaml format not match:\n")
yaml.dump(master, sys.stdout)
print("\n&&\n")
yaml.dump(target, sys.stdout)
sys.exit(1)
## item is a sequence
if isinstance(target, CommentedSeq):
for index in range(len(target)):
# get target comment
target_comment = _seq_comment(target, index)
master_index = len(master)
target_item = target[index]
if isinstance(target_item, CommentedMap):
merge_flag = False
for idx in range(len(master)):
if isinstance(master[idx], CommentedMap):
if master[idx].keys() == target_item.keys():
_merge(master[idx], target_item)
# nonlocal merge_flag
master_index = idx
merge_flag = True
break
if merge_flag is False:
master.append(target_item)
elif target_item not in master:
master.append(target[index])
else:
# merge(master[index], target[index])
pass
# # remove enter signal in previous item
previous_comment = _seq_comment(master, master_index - 1)
_add_eol_comment(master, _extract_comment(previous_comment), master_index - 1)
origin_comment = _seq_comment(master, master_index)
comment = _obtain_comment(origin_comment, target_comment)
if len(comment) > 0:
_add_eol_comment(master, _extract_comment(comment) + "\n\n", len(master) - 1)
## item is a map
elif isinstance(target, CommentedMap):
for item in target:
if item == "flag":
print("")
origin_comment = _map_comment(master, item)
target_comment = _map_comment(target, item)
# get origin start comment
origin_start_comment = _start_comment(master)
# get target start comment
target_start_comment = _start_comment(target)
m = master.get(item, default=None)
if m is None or \
(not (isinstance(m, CommentedMap) or
isinstance(m, CommentedSeq))):
master.update({item: target[item]})
else:
_merge(master[item], target[item])
comment = _obtain_comment(origin_comment, target_comment)
if len(comment) > 0:
_add_eol_comment(master, _extract_comment(comment), item)
start_comment = _obtain_comment(origin_start_comment, target_start_comment)
if len(start_comment) > 0:
master.yaml_set_start_comment(_extract_comment(start_comment))
def _save(_code, _file):
with open(_file, 'w') as wf:
yaml.dump(_code, wf)
def _load(_file):
with open(_file, 'r') as rf:
code = yaml.load(rf)
return code
############################################
# sub parser process operation
#
############################################
def merge_yaml(_args):
_dict = _args.__dict__
_m_file = _dict.get("merge_file", None)
_in_file, _u, _a, _i, _o, _tips = _get_update_par(_args)
if not (_in_file and _m_file):
print(_tips)
sys.exit(1)
code = _load(_in_file)
mcode = _load(_m_file)
_merge(code, mcode)
_update(code, _u, _a, _tips)
if _i:
_backup(_in_file)
_save(code, _in_file)
elif _o:
_save(code, _o)
else:
print(_tips)
sys.exit(1)
def update_yaml(_args):
_in_file, _u, _a, _i, _o, _tips = _get_update_par(_args)
if not _in_file or not _u:
print(_tips)
sys.exit(1)
code = _load(_in_file)
if _i and _o:
print(_tips)
sys.exit(1)
_update(code, _u, _a, _tips)
if _i:
_backup(_in_file)
_save(code, _in_file)
elif _o:
_save(code, _o)
def reset(_args):
_dict = _args.__dict__
_f = _dict.get('f', None) or _dict.get('file', None)
if _f:
_recovery(_f)
else:
_t = _dict.get('tips', None) or "Input \"-h\" for more information"
print(_t)
############################################
# Cli operation
#
############################################
def _set_merge_parser(_parsers):
"""
config merge parser
"""
merge_parser = _parsers.add_parser("merge", help="merge with another yaml file")
_set_merge_parser_arg(merge_parser)
_set_update_parser_arg(merge_parser)
merge_parser.set_defaults(
function=merge_yaml,
tips=merge_parser.format_help()
)
def _set_merge_parser_arg(_parser):
"""
config parser argument for merging
"""
_parser.add_argument("-m", "--merge-file", help="indicate merge yaml file")
def _set_update_parser(_parsers):
"""
config merge parser
"""
update_parser = _parsers.add_parser("update", help="update with another yaml file")
_set_update_parser_arg(update_parser)
update_parser.set_defaults(
function=update_yaml,
tips=update_parser.format_help()
)
def _set_update_parser_arg(_parser):
"""
config parser argument for updating
"""
_parser.add_argument("-f", "--file", help="source yaml file")
_parser.add_argument('-u', '--update', help="update with args, instance as \"a.b.c=d# d comment\"")
_parser.add_argument('-a', '--append', action="store_true", help="append to a seq")
group = _parser.add_mutually_exclusive_group()
group.add_argument("-o", "--out-file", help="indicate output yaml file")
group.add_argument("-i", "--inplace", action="store_true", help="indicate whether result store in origin file")
def _set_reset_parser(_parsers):
"""
config merge parser
"""
reset_parser = _parsers.add_parser("reset", help="reset yaml file")
# indicate yaml file
reset_parser.add_argument('-f', '--file', help="indicate input yaml file")
reset_parser.set_defaults(
function=reset,
tips=reset_parser.format_help()
)
def main():
parser = argparse.ArgumentParser()
sub_parsers = parser.add_subparsers()
# set merge command
_set_merge_parser(sub_parsers)
# set update command
_set_update_parser(sub_parsers)
# set reset command
_set_reset_parser(sub_parsers)
# parse argument and run func
args = parser.parse_args()
args.function(args)
if __name__ == '__main__':
main()

2
tests-deprecating/milvus_benchmark/milvus_benchmark/__init__.py
View File

@ -1,2 +0,0 @@
from locust import User, events
import gevent

0
tests-deprecating/milvus_benchmark/milvus_benchmark/chaos/__init__.py
View File

71
tests-deprecating/milvus_benchmark/milvus_benchmark/chaos/chaos_mesh.py
View File

@ -1,71 +0,0 @@
import logging
import os
from yaml import full_load
from milvus_benchmark.chaos import utils
logger = logging.getLogger("milvus_benchmark.chaos.base")
class BaseChaos(object):
cur_path = os.path.abspath(os.path.dirname(__file__))
def __init__(self, api_version, kind, metadata, spec):
self.api_version = api_version
self.kind = kind
self.metadata = metadata
self.spec = spec
def gen_experiment_config(self):
pass
"""
1. load dict from default yaml
2. merge dict between dict and self.x
"""
def check_config(self):
if not self.kind:
raise Exception("kind is must be specified")
if not self.spec:
raise Exception("spec is must be specified")
if "action" not in self.spec:
raise Exception("action is must be specified in spec")
if "selector" not in self.spec:
raise Exception("selector is must be specified in spec")
return True
def replace_label_selector(self):
self.check_config()
label_selectors_dict = self.spec["selector"]["labelSelectors"]
label_selector = next(iter(label_selectors_dict.items()))
label_selector_value = label_selector[1]
# pods = utils.list_pod_for_namespace(label_selector[0] + "=" + label_selector_value)
pods = utils.list_pod_for_namespace()
real_label_selector_value = list(map(lambda pod: pod, filter(lambda pod: label_selector_value in pod, pods)))[0]
self.spec["selector"]["labelSelectors"].update({label_selector[0]: real_label_selector_value})
class PodChaos(BaseChaos):
default_yaml = BaseChaos.cur_path + '/template/PodChaos.yaml'
def __init__(self, api_version, kind, metadata, spec):
super(PodChaos, self).__init__(api_version, kind, metadata, spec)
def gen_experiment_config(self):
with open(self.default_yaml) as f:
default_config = full_load(f)
f.close()
self.replace_label_selector()
experiment_config = default_config
experiment_config.update({"apiVersion": self.api_version})
experiment_config.update({"kind": self.kind})
experiment_config["metadata"].update(self.metadata)
experiment_config["spec"].update(self.spec)
return experiment_config
class NetworkChaos(BaseChaos):
def __init__(self, api_version, kind, metadata, spec):
super(NetworkChaos, self).__init__(api_version, kind, metadata, spec)
def gen_experiment_config(self):
pass

65
tests-deprecating/milvus_benchmark/milvus_benchmark/chaos/chaos_opt.py
View File

@ -1,65 +0,0 @@
from __future__ import print_function
from utils import *
import logging
from pprint import pprint
from kubernetes import client, config
from kubernetes.client.rest import ApiException
from milvus_benchmark import config as cf
logger = logging.getLogger("milvus_benchmark.chaos.chaosOpt")
class ChaosOpt(object):
def __init__(self, kind, group=cf.DEFAULT_GROUP, version=cf.DEFAULT_VERSION, namespace=cf.CHAOS_NAMESPACE):
self.group = group
self.version = version
self.namespace = namespace
self.plural = kind.lower()
# def get_metadata_name(self):
# return self.metadata_name
def create_chaos_object(self, body):
# body = create_chaos_config(self.plural, self.metadata_name, spec_params)
# logger.info(body)
pretty = 'true'
config.load_kube_config()
api_instance = client.CustomObjectsApi()
try:
api_response = api_instance.create_namespaced_custom_object(self.group, self.version, self.namespace,
plural=self.plural, body=body, pretty=pretty)
print(api_response)
logging.getLogger().info(api_instance)
except ApiException as e:
logger.error("Exception when calling CustomObjectsApi->create_namespaced_custom_object: %s\n" % e)
raise Exception(str(e))
def delete_chaos_object(self, metadata_name):
print(metadata_name)
try:
config.load_kube_config()
api_instance = client.CustomObjectsApi()
data = api_instance.delete_namespaced_custom_object(self.group, self.version, self.namespace, self.plural,
metadata_name)
logger.info(data)
except ApiException as e:
logger.error("Exception when calling CustomObjectsApi->delete_namespaced_custom_object: %s\n" % e)
raise Exception(str(e))
def list_chaos_object(self):
try:
config.load_kube_config()
api_instance = client.CustomObjectsApi()
data = api_instance.list_namespaced_custom_object(self.group, self.version, self.namespace,
plural=self.plural)
# pprint(data)
except ApiException as e:
logger.error("Exception when calling CustomObjectsApi->list_namespaced_custom_object: %s\n" % e)
raise Exception(str(e))
return data
def delete_all_chaos_object(self):
chaos_objects = self.list_chaos_object()
if len(chaos_objects["items"]) > 0:
for item in chaos_objects["items"]:
metadata_name = item["metadata"]["name"]
self.delete_chaos_object(metadata_name)

17
tests-deprecating/milvus_benchmark/milvus_benchmark/chaos/pod-new.yaml
View File

@ -1,17 +0,0 @@
apiVersion: chaos-mesh.org/v1alpha1
kind: PodChaos
metadata:
name: milvus-podchaos
namespace: chaos-testing
spec:
action: pod-kill
duration: 30s
mode: one
scheduler:
cron: '@every 20s'
selector:
labelSelectors:
app.kubernetes.io/name: zong-single-etcd-0
namespaces:
- milvus
value: ''

11
tests-deprecating/milvus_benchmark/milvus_benchmark/chaos/pod.yaml
View File

@ -1,11 +0,0 @@
chaos:
kind: PodChaos
spec:
action: pod-kill
selector:
namespaces:
- milvus
labelSelectors:
"app.kubernetes.io/name": etcd
scheduler:
cron: "@every 20s"

13
tests-deprecating/milvus_benchmark/milvus_benchmark/chaos/template/PodChaos.yaml
View File

@ -1,13 +0,0 @@
apiVersion: chaos-mesh.org/v1alpha1
kind: PodChaos
metadata:
name: pod-failure-example
namespace: chaos-testing
spec:
action: pod-failure
mode: one
selector:
labelSelectors:
'app.kubernetes.io/component': 'tikv'
scheduler:
cron: '@every 2m'

36
tests-deprecating/milvus_benchmark/milvus_benchmark/chaos/test.py
View File

@ -1,36 +0,0 @@
from gevent import monkey
monkey.patch_all()
from yaml import full_load, dump
from chaos.chaos_opt import ChaosOpt
from milvus_benchmark.chaos.chaos_mesh import PodChaos, NetworkChaos
from milvus_benchmark import config
kind_chaos_mapping = {
"PodChaos": PodChaos,
"NetworkChaos": NetworkChaos
}
if __name__ == '__main__':
with open('./pod.yaml') as f:
conf = full_load(f)
f.close()
chaos_config = conf["chaos"]
kind = chaos_config["kind"]
spec = chaos_config["spec"]
metadata_name = config.NAMESPACE + "-" + kind.lower()
metadata = {"name": metadata_name}
chaos_mesh = kind_chaos_mapping[kind](config.DEFAULT_API_VERSION, kind, metadata, spec)
experiment_params = chaos_mesh.gen_experiment_config()
# print(experiment_params)
# with open('./pod-new-chaos.yaml', "w") as f:
# dump(experiment_params, f)
# f.close()
chaos_opt = ChaosOpt(chaos_mesh.kind)
res = chaos_opt.list_chaos_object()
print(res)
if len(res["items"]) != 0:
# chaos_opt.delete_chaos_object("milvus-pod-chaos")
print(res["items"][0]["metadata"]["name"])
chaos_opt.delete_all_chaos_object()
print(chaos_opt.list_chaos_object())

38
tests-deprecating/milvus_benchmark/milvus_benchmark/chaos/utils.py
View File

@ -1,38 +0,0 @@
import logging
from operator import methodcaller
from kubernetes import client, config
from milvus_benchmark import config as cf
logger = logging.getLogger("milvus_benchmark.chaos.utils")
def list_pod_for_namespace(label_selector="app.kubernetes.io/instance=zong-standalone"):
config.load_kube_config()
v1 = client.CoreV1Api()
ret = v1.list_namespaced_pod(namespace=cf.NAMESPACE, label_selector=label_selector)
pods = []
# label_selector = 'release=zong-single'
for i in ret.items:
pods.append(i.metadata.name)
# print("%s\t%s\t%s" % (i.status.pod_ip, i.metadata.namespace, i.metadata.name))
return pods
def assert_fail(func, milvus_client, **params):
try:
methodcaller(func, **params)(milvus_client)
except Exception as e:
logger.debug("11111111111111111111111111")
logger.info(str(e))
pass
else:
raise Exception("fail-assert failed")
def assert_pass(func, milvus_client, **params):
try:
methodcaller(func, **params)(milvus_client)
logger.debug("&&&&&&&&&&&&&&&&&&&&")
except Exception as e:
raise

491
tests-deprecating/milvus_benchmark/milvus_benchmark/client.py
View File

@ -1,491 +0,0 @@
import sys
import pdb
import random
import logging
import json
import time, datetime
import traceback
from multiprocessing import Process
from pymilvus import Milvus, DataType
import numpy as np
import utils
import config
from milvus_benchmark.runners import utils
logger = logging.getLogger("milvus_benchmark.client")
INDEX_MAP = {
"flat": "FLAT",
"ivf_flat": "IVF_FLAT",
"ivf_sq8": "IVF_SQ8",
"nsg": "NSG",
"ivf_sq8h": "IVF_SQ8_HYBRID",
"ivf_pq": "IVF_PQ",
"hnsw": "HNSW",
"annoy": "ANNOY",
"bin_flat": "BIN_FLAT",
"bin_ivf_flat": "BIN_IVF_FLAT",
"rhnsw_pq": "RHNSW_PQ",
"rhnsw_sq": "RHNSW_SQ"
}
epsilon = 0.1
DEFAULT_WARM_QUERY_TOPK = 1
DEFAULT_WARM_QUERY_NQ = 1
def time_wrapper(func):
"""
This decorator prints the execution time for the decorated function.
"""
def wrapper(*args, **kwargs):
start = time.time()
# logger.debug("Milvus {} start".format(func.__name__))
log = kwargs.get("log", True)
kwargs.pop("log", None)
result = func(*args, **kwargs)
end = time.time()
if log:
logger.debug("Milvus {} run in {}s".format(func.__name__, round(end - start, 2)))
return result
return wrapper
class MilvusClient(object):
def __init__(self, collection_name=None, host=None, port=None, timeout=300):
self._collection_name = collection_name
self._collection_info = None
start_time = time.time()
if not host:
host = config.SERVER_HOST_DEFAULT
if not port:
port = config.SERVER_PORT_DEFAULT
# retry connect remote server
i = 0
while time.time() < start_time + timeout:
try:
self._milvus = Milvus(
host=host,
port=port,
try_connect=False,
pre_ping=False)
break
except Exception as e:
logger.error(str(e))
logger.error("Milvus connect failed: %d times" % i)
i = i + 1
time.sleep(30)
if time.time() > start_time + timeout:
raise Exception("Server connect timeout")
# self._metric_type = None
def __str__(self):
return 'Milvus collection %s' % self._collection_name
def set_collection(self, collection_name):
self._collection_name = collection_name
# TODO: server not support
# def check_status(self, status):
# if not status.OK():
# logger.error(status.message)
# logger.error(self._milvus.server_status())
# logger.error(self.count())
# raise Exception("Status not ok")
def check_result_ids(self, result):
for index, item in enumerate(result):
if item[0].distance >= epsilon:
logger.error(index)
logger.error(item[0].distance)
raise Exception("Distance wrong")
@property
def collection_name(self):
return self._collection_name
# only support the given field name
def create_collection(self, dimension, data_type=DataType.FLOAT_VECTOR, auto_id=False,
collection_name=None, other_fields=None):
self._dimension = dimension
if not collection_name:
collection_name = self._collection_name
vec_field_name = utils.get_default_field_name(data_type)
fields = [
{"name": vec_field_name, "type": data_type, "params": {"dim": dimension}},
{"name": "id", "type": DataType.INT64, "is_primary": True}
]
if other_fields:
other_fields = other_fields.split(",")
for other_field_name in other_fields:
if other_field_name.startswith("int"):
field_type = DataType.INT64
elif other_field_name.startswith("float"):
field_type = DataType.FLOAT
elif other_field_name.startswith("double"):
field_type = DataType.DOUBLE
else:
raise Exception("Field name not supported")
fields.append({"name": other_field_name, "type": field_type})
create_param = {
"fields": fields,
"auto_id": auto_id}
try:
self._milvus.create_collection(collection_name, create_param)
logger.info("Create collection: <%s> successfully" % collection_name)
except Exception as e:
logger.error(str(e))
raise
def create_partition(self, tag, collection_name=None):
if not collection_name:
collection_name = self._collection_name
self._milvus.create_partition(collection_name, tag)
@time_wrapper
def insert(self, entities, collection_name=None, timeout=None):
tmp_collection_name = self._collection_name if collection_name is None else collection_name
try:
insert_res = self._milvus.insert(tmp_collection_name, entities, timeout=timeout)
return insert_res.primary_keys
except Exception as e:
logger.error(str(e))
@time_wrapper
def insert_flush(self, entities, _async=False, collection_name=None):
tmp_collection_name = self._collection_name if collection_name is None else collection_name
try:
insert_res = self._milvus.insert(tmp_collection_name, entities)
return insert_res.primary_keys
except Exception as e:
logger.error(str(e))
self._milvus.flush([tmp_collection_name], _async=_async)
def get_dimension(self):
info = self.get_info()
for field in info["fields"]:
if field["type"] in [DataType.FLOAT_VECTOR, DataType.BINARY_VECTOR]:
return field["params"]["dim"]
def get_rand_ids(self, length):
segment_ids = []
while True:
stats = self.get_stats()
segments = stats["partitions"][0]["segments"]
# random choice one segment
segment = random.choice(segments)
try:
segment_ids = self._milvus.list_id_in_segment(self._collection_name, segment["id"])
except Exception as e:
logger.error(str(e))
if not len(segment_ids):
continue
elif len(segment_ids) > length:
return random.sample(segment_ids, length)
else:
logger.debug("Reset length: %d" % len(segment_ids))
return segment_ids
# def get_rand_ids_each_segment(self, length):
# res = []
# status, stats = self._milvus.get_collection_stats(self._collection_name)
# self.check_status(status)
# segments = stats["partitions"][0]["segments"]
# segments_num = len(segments)
# # random choice from each segment
# for segment in segments:
# status, segment_ids = self._milvus.list_id_in_segment(self._collection_name, segment["name"])
# self.check_status(status)
# res.extend(segment_ids[:length])
# return segments_num, res
# def get_rand_entities(self, length):
# ids = self.get_rand_ids(length)
# status, get_res = self._milvus.get_entity_by_id(self._collection_name, ids)
# self.check_status(status)
# return ids, get_res
@time_wrapper
def get_entities(self, get_ids):
get_res = self._milvus.get_entity_by_id(self._collection_name, get_ids)
return get_res
@time_wrapper
def delete(self, ids, collection_name=None):
tmp_collection_name = self._collection_name if collection_name is None else collection_name
self._milvus.delete_entity_by_id(tmp_collection_name, ids)
def delete_rand(self):
delete_id_length = random.randint(1, 100)
count_before = self.count()
logger.debug("%s: length to delete: %d" % (self._collection_name, delete_id_length))
delete_ids = self.get_rand_ids(delete_id_length)
self.delete(delete_ids)
self.flush()
logger.info("%s: count after delete: %d" % (self._collection_name, self.count()))
get_res = self._milvus.get_entity_by_id(self._collection_name, delete_ids)
for item in get_res:
assert not item
# if count_before - len(delete_ids) < self.count():
# logger.error(delete_ids)
# raise Exception("Error occured")
@time_wrapper
def flush(self, _async=False, collection_name=None, timeout=None):
tmp_collection_name = self._collection_name if collection_name is None else collection_name
self._milvus.flush([tmp_collection_name], _async=_async, timeout=timeout)
@time_wrapper
def compact(self, collection_name=None):
tmp_collection_name = self._collection_name if collection_name is None else collection_name
status = self._milvus.compact(tmp_collection_name)
self.check_status(status)
# only support "in" in expr
@time_wrapper
def get(self, ids, collection_name=None, timeout=None):
tmp_collection_name = self._collection_name if collection_name is None else collection_name
# res = self._milvus.get(tmp_collection_name, ids, output_fields=None, partition_names=None)
ids_expr = "id in %s" % (str(ids))
res = self._milvus.query(tmp_collection_name, ids_expr, output_fields=None, partition_names=None, timeout=timeout)
return res
@time_wrapper
def create_index(self, field_name, index_type, metric_type, _async=False, index_param=None):
index_type = INDEX_MAP[index_type]
metric_type = utils.metric_type_trans(metric_type)
logger.info("Building index start, collection_name: %s, index_type: %s, metric_type: %s" % (
self._collection_name, index_type, metric_type))
if index_param:
logger.info(index_param)
index_params = {
"index_type": index_type,
"metric_type": metric_type,
"params": index_param
}
self._milvus.create_index(self._collection_name, field_name, index_params, _async=_async)
# TODO: need to check
def describe_index(self, field_name, collection_name=None):
# stats = self.get_stats()
tmp_collection_name = self._collection_name if collection_name is None else collection_name
info = self._milvus.describe_index(tmp_collection_name, field_name)
logger.info(info)
index_info = {"index_type": "flat", "metric_type": None, "index_param": None}
if info:
index_info = {"index_type": info["index_type"], "metric_type": info["metric_type"], "index_param": info["params"]}
# transfer index type name
for k, v in INDEX_MAP.items():
if index_info['index_type'] == v:
index_info['index_type'] = k
return index_info
def drop_index(self, field_name):
logger.info("Drop index: %s" % self._collection_name)
return self._milvus.drop_index(self._collection_name, field_name)
@time_wrapper
def query(self, vector_query, filter_query=None, collection_name=None, timeout=300):
tmp_collection_name = self._collection_name if collection_name is None else collection_name
must_params = [vector_query]
if filter_query:
must_params.extend(filter_query)
query = {
"bool": {"must": must_params}
}
result = self._milvus.search(tmp_collection_name, query, timeout=timeout)
return result
@time_wrapper
def warm_query(self, index_field_name, search_param, metric_type, times=2):
query_vectors = [[random.random() for _ in range(self._dimension)] for _ in range(DEFAULT_WARM_QUERY_NQ)]
# index_info = self.describe_index(index_field_name)
vector_query = {"vector": {index_field_name: {
"topk": DEFAULT_WARM_QUERY_TOPK,
"query": query_vectors,
"metric_type": metric_type,
"params": search_param}
}}
must_params = [vector_query]
query = {
"bool": {"must": must_params}
}
logger.debug("Start warm up query")
for i in range(times):
self._milvus.search(self._collection_name, query)
logger.debug("End warm up query")
@time_wrapper
def load_and_query(self, vector_query, filter_query=None, collection_name=None, timeout=120):
tmp_collection_name = self._collection_name if collection_name is None else collection_name
must_params = [vector_query]
if filter_query:
must_params.extend(filter_query)
query = {
"bool": {"must": must_params}
}
self.load_collection(tmp_collection_name)
result = self._milvus.search(tmp_collection_name, query, timeout=timeout)
return result
def get_ids(self, result):
# idss = result._entities.ids
ids = []
# len_idss = len(idss)
# len_r = len(result)
# top_k = len_idss // len_r
# for offset in range(0, len_idss, top_k):
# ids.append(idss[offset: min(offset + top_k, len_idss)])
for res in result:
ids.append(res.ids)
return ids
def query_rand(self, nq_max=100, timeout=None):
# for ivf search
dimension = 128
top_k = random.randint(1, 100)
nq = random.randint(1, nq_max)
nprobe = random.randint(1, 100)
search_param = {"nprobe": nprobe}
query_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)]
metric_type = random.choice(["l2", "ip"])
logger.info("%s, Search nq: %d, top_k: %d, nprobe: %d" % (self._collection_name, nq, top_k, nprobe))
vec_field_name = utils.get_default_field_name()
vector_query = {"vector": {vec_field_name: {
"topk": top_k,
"query": query_vectors,
"metric_type": utils.metric_type_trans(metric_type),
"params": search_param}
}}
self.query(vector_query, timeout=timeout)
def load_query_rand(self, nq_max=100, timeout=None):
# for ivf search
dimension = 128
top_k = random.randint(1, 100)
nq = random.randint(1, nq_max)
nprobe = random.randint(1, 100)
search_param = {"nprobe": nprobe}
query_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)]
metric_type = random.choice(["l2", "ip"])
logger.info("%s, Search nq: %d, top_k: %d, nprobe: %d" % (self._collection_name, nq, top_k, nprobe))
vec_field_name = utils.get_default_field_name()
vector_query = {"vector": {vec_field_name: {
"topk": top_k,
"query": query_vectors,
"metric_type": utils.metric_type_trans(metric_type),
"params": search_param}
}}
self.load_and_query(vector_query, timeout=timeout)
# TODO: need to check
def count(self, collection_name=None):
if collection_name is None:
collection_name = self._collection_name
row_count = self._milvus.get_collection_stats(collection_name)["row_count"]
logger.debug("Row count: %d in collection: <%s>" % (row_count, collection_name))
return row_count
def drop(self, timeout=120, collection_name=None):
timeout = int(timeout)
if collection_name is None:
collection_name = self._collection_name
logger.info("Start delete collection: %s" % collection_name)
self._milvus.drop_collection(collection_name)
i = 0
while i < timeout:
try:
row_count = self.count(collection_name=collection_name)
if row_count:
time.sleep(1)
i = i + 1
continue
else:
break
except Exception as e:
logger.warning("Collection count failed: {}".format(str(e)))
break
if i >= timeout:
logger.error("Delete collection timeout")
def get_stats(self):
return self._milvus.get_collection_stats(self._collection_name)
def get_info(self, collection_name=None):
if collection_name is None:
collection_name = self._collection_name
return self._milvus.describe_collection(collection_name)
def show_collections(self):
return self._milvus.list_collections()
def exists_collection(self, collection_name=None):
if collection_name is None:
collection_name = self._collection_name
res = self._milvus.has_collection(collection_name)
return res
def clean_db(self):
collection_names = self.show_collections()
for name in collection_names:
self.drop(collection_name=name)
@time_wrapper
def load_collection(self, collection_name=None, timeout=3000):
if collection_name is None:
collection_name = self._collection_name
return self._milvus.load_collection(collection_name, timeout=timeout)
@time_wrapper
def release_collection(self, collection_name=None, timeout=3000):
if collection_name is None:
collection_name = self._collection_name
return self._milvus.release_collection(collection_name, timeout=timeout)
@time_wrapper
def load_partitions(self, tag_names, collection_name=None, timeout=3000):
if collection_name is None:
collection_name = self._collection_name
return self._milvus.load_partitions(collection_name, tag_names, timeout=timeout)
@time_wrapper
def release_partitions(self, tag_names, collection_name=None, timeout=3000):
if collection_name is None:
collection_name = self._collection_name
return self._milvus.release_partitions(collection_name, tag_names, timeout=timeout)
# TODO: remove
# def get_server_version(self):
# return self._milvus.server_version()
# def get_server_mode(self):
# return self.cmd("mode")
# def get_server_commit(self):
# return self.cmd("build_commit_id")
# def get_server_config(self):
# return json.loads(self.cmd("get_milvus_config"))
# def get_mem_info(self):
# result = json.loads(self.cmd("get_system_info"))
# result_human = {
# # unit: Gb
# "memory_used": round(int(result["memory_used"]) / (1024 * 1024 * 1024), 2)
# }
# return result_human
# def cmd(self, command):
# res = self._milvus._cmd(command)
# logger.info("Server command: %s, result: %s" % (command, res))
# return res
# @time_wrapper
# def set_config(self, parent_key, child_key, value):
# self._milvus.set_config(parent_key, child_key, value)
# def get_config(self, key):
# return self._milvus.get_config(key)

42
tests-deprecating/milvus_benchmark/milvus_benchmark/config.py
View File

@ -1,42 +0,0 @@
MONGO_SERVER = 'mongodb://192.168.1.234:27017/'
# MONGO_SERVER = 'mongodb://mongodb.test:27017/'
SCHEDULER_DB = "scheduler"
JOB_COLLECTION = "jobs"
REGISTRY_URL = "registry.zilliz.com/milvus/milvus"
IDC_NAS_URL = "//172.16.70.249/test"
DEFAULT_IMAGE = "milvusdb/milvus:latest"
SERVER_HOST_DEFAULT = "127.0.0.1"
SERVER_PORT_DEFAULT = 19530
SERVER_VERSION = "2.0.0-RC3"
DEFUALT_DEPLOY_MODE = "single"
HELM_NAMESPACE = "milvus"
BRANCH = "master"
DEFAULT_CPUS = 48
RAW_DATA_DIR = "/test/milvus/raw_data/"
# nars log
LOG_PATH = "/test/milvus/benchmark/logs/{}/".format(BRANCH)
DEFAULT_DEPLOY_MODE = "single"
SINGLE_DEPLOY_MODE = "single"
CLUSTER_DEPLOY_MODE = "cluster"
NAMESPACE = "milvus"
CHAOS_NAMESPACE = "chaos-testing"
DEFAULT_API_VERSION = 'chaos-mesh.org/v1alpha1'
DEFAULT_GROUP = 'chaos-mesh.org'
DEFAULT_VERSION = 'v1alpha1'
# minio config
MINIO_HOST = "milvus-test-minio.qa-milvus.svc.cluster.local"
MINIO_PORT = 9000
MINIO_ACCESS_KEY = "minioadmin"
MINIO_SECRET_KEY = "minioadmin"
MINIO_BUCKET_NAME = "test"

14
tests-deprecating/milvus_benchmark/milvus_benchmark/env/__init__.py vendored
View File

@ -1,14 +0,0 @@
import logging
from .helm import HelmEnv
from .docker import DockerEnv
from .local import LocalEnv
logger = logging.getLogger("milvus_benchmark.env")
def get_env(env_mode, deploy_mode=None):
return {
"helm": HelmEnv(deploy_mode),
"docker": DockerEnv(None),
"local": LocalEnv(None),
}.get(env_mode)

46
tests-deprecating/milvus_benchmark/milvus_benchmark/env/base.py vendored
View File

@ -1,46 +0,0 @@
import logging
from milvus_benchmark import utils
from milvus_benchmark import config
logger = logging.getLogger("milvus_benchmark.env.env")
class BaseEnv(object):
"""docstring for Env"""
def __init__(self, deploy_mode="single"):
self.deploy_mode = deploy_mode
self._name = utils.get_unique_name()
self._hostname = None
self._port = config.SERVER_PORT_DEFAULT
def start_up(self):
logger.debug("IN ENV CLASS")
pass
def tear_down(self):
pass
def restart(self):
pass
def set_hostname(self, hostname):
self._hostname = hostname
def set_port(self, port):
self._port = port
def resources(self):
pass
@property
def name(self):
return self._name
@property
def hostname(self):
return self._hostname
@property
def port(self):
return self._port

12
tests-deprecating/milvus_benchmark/milvus_benchmark/env/docker.py vendored
View File

@ -1,12 +0,0 @@
import logging
from milvus_benchmark.env.base import BaseEnv
logger = logging.getLogger("milvus_benchmark.env.docker")
class DockerEnv(BaseEnv):
"""docker env class wrapper"""
env_mode = "docker"
def __init__(self, deploy_mode=None):
super(DockerEnv, self).__init__(deploy_mode)

72
tests-deprecating/milvus_benchmark/milvus_benchmark/env/helm.py vendored
View File

@ -1,72 +0,0 @@
import os
import time
import pdb
import logging
import traceback
from milvus_benchmark.env import helm_utils
from milvus_benchmark.env.base import BaseEnv
from milvus_benchmark import config
logger = logging.getLogger("milvus_benchmark.env.helm")
TIMEOUT = 5
class HelmEnv(BaseEnv):
"""helm env class wrapper"""
env_mode = "helm"
def __init__(self, deploy_mode="single"):
super(HelmEnv, self).__init__(deploy_mode)
self._name_space = config.HELM_NAMESPACE
def start_up(self, helm_path, helm_install_params):
if "namespace" in helm_install_params:
self._name_space = helm_install_params["namespace"]
server_name = helm_install_params["server_name"]
server_tag = helm_install_params["server_tag"] if "server_tag" in helm_install_params else None
server_config = helm_install_params["server_config"] if "server_config" in helm_install_params else None
milvus_config = helm_install_params["milvus_config"]
image_tag = helm_install_params["image_tag"]
image_type = helm_install_params["image_type"]
logger.debug(self.deploy_mode)
server_config = helm_utils.update_server_config(server_name, server_tag, server_config)
# update values
values_file_path = helm_path + "/values.yaml"
if not os.path.exists(values_file_path):
raise Exception("File {} not existed".format(values_file_path))
lock_file_path = helm_path + "/values.yaml.lock"
start_time = time.time()
while os.path.exists(lock_file_path) and time.time() < start_time+TIMEOUT:
logger.debug("Waiting for the lock file to release")
time.sleep(1)
if not os.path.exists(lock_file_path):
# generate lock file
open(lock_file_path, 'a').close()
try:
if milvus_config:
helm_utils.update_values(values_file_path, self.deploy_mode, server_name, server_tag, milvus_config, server_config)
logger.debug("Config file has been updated, remove the lock file")
os.system("rm -rf %s" % lock_file_path)
logger.debug("Start install server")
hostname = helm_utils.helm_install_server(helm_path, self.deploy_mode, image_tag, image_type, self.name,
self._name_space)
status_cmd = 'kubectl get pods -n milvus -l release=zong-standalone -o=jsonpath=\'{range .items[*]}{.metadata.name}{"\t"}{.status.phase}{"\n"}{end}\''
if not hostname:
logger.error("Helm install server failed")
return False
else:
self.set_hostname(hostname)
while not helm_utils.running_status(self.name, self._name_space):
pass
else:
return hostname
except Exception as e:
os.system("rm -rf %s" % lock_file_path)
logger.error("Helm install server failed: %s" % (str(e)))
logger.error(traceback.format_exc())
return False
def tear_down(self):
logger.debug("Start clean up: {}.{}".format(self.name, self._name_space))
helm_utils.helm_del_server(self.name, self._name_space)

473
tests-deprecating/milvus_benchmark/milvus_benchmark/env/helm_utils.py vendored
View File

@ -1,473 +0,0 @@
import os
import pdb
import time
import logging
import hashlib
import traceback
from yaml import full_load, dump
from milvus_benchmark import utils
from milvus_benchmark import config
logger = logging.getLogger("milvus_benchmark.env.helm_utils")
BOOKKEEPER_PULSAR_MEM = '\"-Xms512m -Xmx1024m -XX:MaxDirectMemorySize=1024m -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.linkCapacity=1024 -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintHeapAtGC -verbosegc -XX:G1LogLevel=finest\"'
BROKER_PULSAR_MEM = '\"-Xms512m -Xmx1024m -XX:MaxDirectMemorySize=1024m -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.linkCapacity=1024 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem\"'
def get_host_cpus(hostname):
from kubernetes import client, config
config.load_kube_config()
client.rest.logger.setLevel(logging.WARNING)
try:
v1 = client.CoreV1Api()
cpus = v1.read_node(hostname).status.allocatable.get("cpu")
except Exception as e:
logger.error(traceback.format_exc())
logger.error(str(e))
cpus = 0
finally:
return cpus
def update_server_config(server_name, server_tag, server_config):
cpus = config.DEFAULT_CPUS
if server_name:
try:
cpus = get_host_cpus(server_name)
if not cpus:
cpus = config.DEFAULT_CPUS
except Exception as e:
logger.error("Get cpus on host: {} failed".format(server_name))
logger.error(str(e))
if server_config:
if "cpus" in server_config.keys():
cpus = server_config["cpus"]
# self.hardware = Hardware(name=self.hostname, cpus=cpus)
if server_tag:
cpus = int(server_tag.split("c")[0])
kv = {"cpus": cpus}
logger.debug(kv)
if server_config:
server_config.update(kv)
else:
server_config = kv
return server_config
"""
description: update values.yaml
return: no return
"""
def update_values(file_path, deploy_mode, hostname, server_tag, milvus_config, server_config=None):
# bak values.yaml
file_name = os.path.basename(file_path)
bak_file_name = file_name + ".bak"
file_parent_path = os.path.dirname(file_path)
bak_file_path = file_parent_path + '/' + bak_file_name
if os.path.exists(bak_file_path):
os.system("cp %s %s" % (bak_file_path, file_path))
else:
os.system("cp %s %s" % (file_path, bak_file_path))
with open(file_path) as f:
values_dict = full_load(f)
f.close()
cluster = False
if deploy_mode == "cluster":
cluster = True
# TODO: disable change config
# cluster = False
# if "cluster" in milvus_config and milvus_config["cluster"]:
# cluster = True
# for k, v in milvus_config.items():
# if k.find("primary_path") != -1:
# suffix_path = milvus_config["suffix_path"] if "suffix_path" in milvus_config else None
# path_value = v
# if suffix_path:
# path_value = v + "_" + str(int(time.time()))
# values_dict["primaryPath"] = path_value
# values_dict['wal']['path'] = path_value + "/wal"
# values_dict['logs']['path'] = path_value + "/logs"
# # elif k.find("use_blas_threshold") != -1:
# # values_dict['useBLASThreshold'] = int(v)
# elif k.find("gpu_search_threshold") != -1:
# values_dict['gpu']['gpuSearchThreshold'] = int(v)
# if cluster:
# values_dict['readonly']['gpu']['gpuSearchThreshold'] = int(v)
# elif k.find("cpu_cache_capacity") != -1:
# values_dict['cache']['cacheSize'] = v
# if cluster:
# values_dict['readonly']['cache']['cacheSize'] = v
# # elif k.find("cache_insert_data") != -1:
# # values_dict['cache']['cacheInsertData'] = v
# elif k.find("insert_buffer_size") != -1:
# values_dict['cache']['insertBufferSize'] = v
# if cluster:
# values_dict['readonly']['cache']['insertBufferSize'] = v
# elif k.find("gpu_resource_config.enable") != -1:
# values_dict['gpu']['enabled'] = v
# if cluster:
# values_dict['readonly']['gpu']['enabled'] = v
# elif k.find("gpu_resource_config.cache_capacity") != -1:
# values_dict['gpu']['cacheSize'] = v
# if cluster:
# values_dict['readonly']['gpu']['cacheSize'] = v
# elif k.find("build_index_resources") != -1:
# values_dict['gpu']['buildIndexDevices'] = v
# if cluster:
# values_dict['readonly']['gpu']['buildIndexDevices'] = v
# elif k.find("search_resources") != -1:
# values_dict['gpu']['searchDevices'] = v
# if cluster:
# values_dict['readonly']['gpu']['searchDevices'] = v
# # wal
# elif k.find("auto_flush_interval") != -1:
# values_dict['storage']['autoFlushInterval'] = v
# if cluster:
# values_dict['readonly']['storage']['autoFlushInterval'] = v
# elif k.find("wal_enable") != -1:
# values_dict['wal']['enabled'] = v
# # if values_dict['nodeSelector']:
# # logger.warning("nodeSelector has been set: %s" % str(values_dict['engine']['nodeSelector']))
# # return
# values_dict["wal"]["recoveryErrorIgnore"] = True
# # enable monitor
# values_dict["metrics"]["enabled"] = True
# values_dict["metrics"]["address"] = "192.168.1.237"
# values_dict["metrics"]["port"] = 9091
# # only test avx2
# values_dict["extraConfiguration"].update({"engine": {"simd_type": "avx2"}})
# # stat_optimizer_enable
# values_dict["extraConfiguration"]["engine"].update({"stat_optimizer_enable": False})
# # enable read-write mode
# if cluster:
# values_dict["cluster"]["enabled"] = True
# # update readonly log path
# values_dict["readonly"]['logs']['path'] = values_dict['logs']['path'] + "/readonly"
# if "readonly" in milvus_config:
# if "replicas" in milvus_config["readonly"]:
# values_dict["readonly"]["replicas"] = milvus_config["readonly"]["replicas"]
# use_external_mysql = False
# if "external_mysql" in milvus_config and milvus_config["external_mysql"]:
# use_external_mysql = True
# # meta mysql
# if use_external_mysql:
# values_dict["mysql"]["enabled"] = False
# # values_dict["mysql"]["persistence"]["enabled"] = True
# # values_dict["mysql"]["persistence"]["existingClaim"] = hashlib.md5(path_value.encode(encoding='UTF-8')).hexdigest()
# values_dict['externalMysql']['enabled'] = True
# if deploy_mode == "local":
# values_dict['externalMysql']["ip"] = "192.168.1.238"
# else:
# values_dict['externalMysql']["ip"] = "milvus-mysql.test"
# values_dict['externalMysql']["port"] = 3306
# values_dict['externalMysql']["user"] = "root"
# values_dict['externalMysql']["password"] = "milvus"
# values_dict['externalMysql']["database"] = "db"
# else:
# values_dict["mysql"]["enabled"] = False
# # update values.yaml with the given host
node_config = None
perf_tolerations = [{
"key": "worker",
"operator": "Equal",
"value": "performance",
"effect": "NoSchedule"
}]
if hostname:
node_config = {'kubernetes.io/hostname': hostname}
elif server_tag:
# server tag
node_config = {'instance-type': server_tag}
cpus = server_config["cpus"]
logger.debug(hostname)
if cluster is False:
if node_config:
values_dict['standalone']['nodeSelector'] = node_config
values_dict['minio']['nodeSelector'] = node_config
values_dict['etcd']['nodeSelector'] = node_config
# TODO: disable
# set limit/request cpus in resources
values_dict['standalone']['resources'] = {
"limits": {
# "cpu": str(int(cpus)) + ".0"
"cpu": str(int(cpus)) + ".0"
},
"requests": {
"cpu": str(int(cpus) // 2 + 1) + ".0"
# "cpu": "4.0"
}
}
logger.debug("Add tolerations into standalone server")
values_dict['standalone']['tolerations'] = perf_tolerations
values_dict['minio']['tolerations'] = perf_tolerations
values_dict['etcd']['tolerations'] = perf_tolerations
else:
# values_dict['pulsar']["broker"]["configData"].update({"maxMessageSize": "52428800", "PULSAR_MEM": BOOKKEEPER_PULSAR_MEM})
# values_dict['pulsar']["bookkeeper"]["configData"].update({"nettyMaxFrameSizeBytes": "52428800", "PULSAR_MEM": BROKER_PULSAR_MEM})
values_dict['proxynode']['nodeSelector'] = node_config
values_dict['querynode']['nodeSelector'] = node_config
values_dict['indexnode']['nodeSelector'] = node_config
values_dict['datanode']['nodeSelector'] = node_config
values_dict['minio']['nodeSelector'] = node_config
# values_dict['pulsar']["enabled"] = True
# values_dict['pulsar']['autoRecovery']['nodeSelector'] = node_config
# values_dict['pulsar']['proxy']['nodeSelector'] = node_config
# values_dict['pulsar']['broker']['nodeSelector'] = node_config
# values_dict['pulsar']['bookkeeper']['nodeSelector'] = node_config
# values_dict['pulsar']['zookeeper']['nodeSelector'] = node_config
values_dict['pulsarStandalone']['nodeSelector'] = node_config
if hostname:
logger.debug("Add tolerations into cluster server")
values_dict['proxynode']['tolerations'] = perf_tolerations
values_dict['querynode']['tolerations'] = perf_tolerations
values_dict['indexnode']['tolerations'] = perf_tolerations
values_dict['datanode']['tolerations'] = perf_tolerations
values_dict['etcd']['tolerations'] = perf_tolerations
values_dict['minio']['tolerations'] = perf_tolerations
values_dict['pulsarStandalone']['tolerations'] = perf_tolerations
# values_dict['pulsar']['autoRecovery']['tolerations'] = perf_tolerations
# values_dict['pulsar']['proxy']['tolerations'] = perf_tolerations
# values_dict['pulsar']['broker']['tolerations'] = perf_tolerations
# values_dict['pulsar']['bookkeeper']['tolerations'] = perf_tolerations
# values_dict['pulsar']['zookeeper']['tolerations'] = perf_tolerations
# add extra volumes
values_dict['extraVolumes'] = [{
'name': 'test',
'flexVolume': {
'driver': "fstab/cifs",
'fsType': "cifs",
'secretRef': {
'name': "cifs-test-secret"
},
'options': {
'networkPath': config.IDC_NAS_URL,
'mountOptions': "vers=1.0"
}
}
}]
values_dict['extraVolumeMounts'] = [{
'name': 'test',
'mountPath': '/test'
}]
with open(file_path, 'w') as f:
dump(values_dict, f, default_flow_style=False)
f.close()
# DEBUG
with open(file_path) as f:
for line in f.readlines():
line = line.strip("\n")
logger.debug(line)
# deploy server
def helm_install_server(helm_path, deploy_mode, image_tag, image_type, name, namespace):
logger.debug("Server deploy mode: %s" % deploy_mode)
host = "%s-milvus-ha.%s.svc.cluster.local" % (name, namespace)
# TODO: update etcd config
etcd_config_map_cmd = "kubectl create configmap -n %s %s --from-literal=ETCD_QUOTA_BACKEND_BYTES=8589934592 --from-literal=ETCD_SNAPSHOT_COUNT=5000 --from-literal=ETCD_AUTO_COMPACTION_MODE=revision --from-literal=ETCD_AUTO_COMPACTION_RETENTION=1" % (
namespace, name)
if os.system(etcd_config_map_cmd):
raise Exception("Create configmap: {} failed".format(name))
logger.debug("Create configmap: {} successfully".format(name))
log_path = config.LOG_PATH + "install.log"
install_cmd = "helm install \
--set standalone.service.type=ClusterIP \
--set image.all.repository=%s \
--set image.all.tag=%s \
--set minio.persistence.enabled=false \
--set etcd.persistence.enabled=false \
--set etcd.envVarsConfigMap=%s \
--namespace %s \
%s . >>%s >&1" % (config.REGISTRY_URL, image_tag, name, namespace, name, log_path)
# --set image.all.pullPolicy=Always \
if deploy_mode == "cluster":
install_cmd = "helm install \
--set standalone.enabled=false \
--set image.all.repository=%s \
--set image.all.tag=%s \
--set minio.persistence.enabled=false \
--set etcd.persistence.enabled=false \
--set etcd.envVarsConfigMap=%s \
--namespace %s \
%s . >>%s >&1" % (config.REGISTRY_URL, image_tag, name, namespace, name, log_path)
# --set image.all.pullPolicy=Always \
elif deploy_mode != "single":
raise Exception("Deploy mode: {} not support".format(deploy_mode))
logger.debug(install_cmd)
logger.debug(host)
if os.system("cd %s && %s" % (helm_path, install_cmd)):
logger.error("Helm install failed: %s" % name)
return None
logger.debug("Wait for 60s ..")
time.sleep(60)
# config.load_kube_config()
# v1 = client.CoreV1Api()
# pod_name = None
# pod_id = None
# pods = v1.list_namespaced_pod(namespace)
# for i in pods.items:
# if i.metadata.name.find(name) != -1:
# pod_name = i.metadata.name
# pod_ip = i.status.pod_ip
# logger.debug(pod_name)
# logger.debug(pod_ip)
# return pod_name, pod_ip
return host
# delete server
@utils.retry(3)
def helm_del_server(name, namespace):
# logger.debug("Sleep 600s before uninstall server")
# time.sleep(600)
delete_etcd_config_map_cmd = "kubectl delete configmap -n %s %s" % (namespace, name)
logger.info(delete_etcd_config_map_cmd)
if os.system(delete_etcd_config_map_cmd):
logger.error("Delete configmap %s:%s failed" % (namespace, name))
del_cmd = "helm uninstall -n milvus %s" % name
logger.info(del_cmd)
if os.system(del_cmd):
logger.error("Helm delete name:%s failed" % name)
return False
return True
def restart_server(helm_release_name, namespace):
res = True
timeout = 120000
# service_name = "%s.%s.svc.cluster.local" % (helm_release_name, namespace)
config.load_kube_config()
v1 = client.CoreV1Api()
pod_name = None
# config_map_names = v1.list_namespaced_config_map(namespace, pretty='true')
# body = {"replicas": 0}
pods = v1.list_namespaced_pod(namespace)
for i in pods.items:
if i.metadata.name.find(helm_release_name) != -1 and i.metadata.name.find("mysql") == -1:
pod_name = i.metadata.name
break
# v1.patch_namespaced_config_map(config_map_name, namespace, body, pretty='true')
# status_res = v1.read_namespaced_service_status(helm_release_name, namespace, pretty='true')
logger.debug("Pod name: %s" % pod_name)
if pod_name is not None:
try:
v1.delete_namespaced_pod(pod_name, namespace)
except Exception as e:
logger.error(str(e))
logger.error("Exception when calling CoreV1Api->delete_namespaced_pod")
res = False
return res
logger.error("Sleep 10s after pod deleted")
time.sleep(10)
# check if restart successfully
pods = v1.list_namespaced_pod(namespace)
for i in pods.items:
pod_name_tmp = i.metadata.name
logger.error(pod_name_tmp)
if pod_name_tmp == pod_name:
continue
elif pod_name_tmp.find(helm_release_name) == -1 or pod_name_tmp.find("mysql") != -1:
continue
else:
status_res = v1.read_namespaced_pod_status(pod_name_tmp, namespace, pretty='true')
logger.error(status_res.status.phase)
start_time = time.time()
ready_break = False
while time.time() - start_time <= timeout:
logger.error(time.time())
status_res = v1.read_namespaced_pod_status(pod_name_tmp, namespace, pretty='true')
if status_res.status.phase == "Running":
logger.error("Already running")
ready_break = True
break
else:
time.sleep(5)
if time.time() - start_time > timeout:
logger.error("Restart pod: %s timeout" % pod_name_tmp)
res = False
return res
if ready_break:
break
else:
raise Exception("Pod: %s not found" % pod_name)
follow = True
pretty = True
previous = True # bool | Return previous terminated container logs. Defaults to false. (optional)
since_seconds = 56 # int | A relative time in seconds before the current time from which to show logs. If this value precedes the time a pod was started, only logs since the pod start will be returned. If this value is in the future, no logs will be returned. Only one of sinceSeconds or sinceTime may be specified. (optional)
timestamps = True # bool | If true, add an RFC3339 or RFC3339Nano timestamp at the beginning of every line of log output. Defaults to false. (optional)
container = "milvus"
# start_time = time.time()
# while time.time() - start_time <= timeout:
# try:
# api_response = v1.read_namespaced_pod_log(pod_name_tmp, namespace, container=container, follow=follow,
# pretty=pretty, previous=previous, since_seconds=since_seconds,
# timestamps=timestamps)
# logging.error(api_response)
# return res
# except Exception as e:
# logging.error("Exception when calling CoreV1Api->read_namespaced_pod_log: %s\n" % e)
# # waiting for server start
# time.sleep(2)
# # res = False
# # return res
# if time.time() - start_time > timeout:
# logging.error("Restart pod: %s timeout" % pod_name_tmp)
# res = False
return res
def get_pod_status(helm_release_name, namespace):
from kubernetes import client, config
config.load_kube_config()
v1 = client.CoreV1Api()
pod_status = []
label_selector = 'app.kubernetes.io/instance={}'.format(helm_release_name)
# pods = v1.list_namespaced_pod(namespace, label_selector=label_selector)
pods = v1.list_namespaced_pod(namespace)
for i in pods.items:
if i.metadata.name.find(helm_release_name) != -1:
pod_name = i.metadata.name
result = v1.read_namespaced_pod_status(pod_name, namespace)
pod_status.append({"pod": pod_name, "status": result.status.phase})
# print(pod_status)
return pod_status
def running_status(helm_release_name, namespace):
pod_status = get_pod_status(helm_release_name, namespace)
for pod in pod_status:
if pod["status"] != "Running":
return False
return True
if __name__ == '__main__':
def ff():
namespace = 'milvus'
helm_release_name = 'zong-standalone'
# st = get_pod_status(helm_release_name, namespace)
status = get_pod_status(helm_release_name, namespace)
print(status)
for s in status:
if s["status"] != "Runningk":
return False
return True
def fff():
print(time.time())
while not ff():
print("retry")
else:
print("gogog")
print("hhhh")

21
tests-deprecating/milvus_benchmark/milvus_benchmark/env/local.py vendored
View File

@ -1,21 +0,0 @@
import logging
from milvus_benchmark.env.base import BaseEnv
logger = logging.getLogger("milvus_benchmark.env.local")
class LocalEnv(BaseEnv):
"""docker env class wrapper"""
env_mode = "local"
def __init__(self, deploy_mode=None):
super(LocalEnv, self).__init__(deploy_mode)
def start_up(self, hostname, port):
res = True
try:
self.set_hostname(hostname)
except Exception as e:
logger.error(str(e))
res = False
return res

24
tests-deprecating/milvus_benchmark/milvus_benchmark/logs/log.py
View File

@ -1,24 +0,0 @@
import logging.config
from datetime import datetime
import os
import yaml
import config
cur_path = os.path.abspath(os.path.dirname(__file__))
LOG_CONFIG_PATH = cur_path + "/logging.yaml"
FILE_NAME = config.LOG_PATH + 'benchmark-{:%Y-%m-%d}.log'.format(datetime.now())
def setup_logging(config_path=LOG_CONFIG_PATH, default_level=logging.INFO):
"""
Setup logging configuration
"""
print(FILE_NAME)
try:
with open(config_path, 'rt') as f:
log_config = yaml.safe_load(f.read())
log_config["handlers"]["info_file_handler"].update({"filename": FILE_NAME})
logging.config.dictConfig(log_config)
except Exception:
raise
logging.error('Failed to open file', exc_info=True)

37
tests-deprecating/milvus_benchmark/milvus_benchmark/logs/logging.yaml
View File

@ -1,37 +0,0 @@
version: 1
disable_existing_loggers: False
formatters:
simple:
format: "[%(asctime)-15s] [%(levelname)8s] - %(message)s (%(name)s:%(lineno)s)"
handlers:
console:
class: logging.StreamHandler
level: DEBUG
formatter: simple
stream: ext://sys.stdout
info_file_handler:
class: logging.FileHandler
formatter: simple
level: DEBUG
filename: info.log
# error_file_handler:
# class: logging.handlers.RotatingFileHandler
# level: ERROR
# formatter: simple
# filename: errors.log
# maxBytes: 10485760 # 10MB
# backupCount: 20
# encoding: utf8
loggers:
milvus_benchmark:
level: DEBUG
handlers: [console, info_file_handler]
propagate: no
root:
level: DEBUG
handlers: [console, info_file_handler]

273
tests-deprecating/milvus_benchmark/milvus_benchmark/main.py
View File

@ -1,273 +0,0 @@
import os
import sys
import time
import argparse
import logging
import traceback
# from queue import Queue
from yaml import full_load, dump
from milvus_benchmark.metrics.models.server import Server
from milvus_benchmark.metrics.models.hardware import Hardware
from milvus_benchmark.metrics.models.env import Env
from milvus_benchmark.env import get_env
from milvus_benchmark.runners import get_runner
from milvus_benchmark.metrics import api
from milvus_benchmark import config, utils
from milvus_benchmark import parser
# from scheduler import back_scheduler
from logs import log
log.setup_logging()
logger = logging.getLogger("milvus_benchmark.main")
# q = Queue()
def positive_int(s):
i = None
try:
i = int(s)
except ValueError:
pass
if not i or i < 1:
raise argparse.ArgumentTypeError("%r is not a positive integer" % s)
return i
def get_image_tag(image_version):
return "%s-latest" % (image_version)
# def shutdown(event):
# logger.info("Check if there is scheduled jobs in scheduler")
# if not back_scheduler.get_jobs():
# logger.info("No job in scheduler, will shutdown the scheduler")
# back_scheduler.shutdown(wait=False)
def run_suite(run_type, suite, env_mode, env_params, timeout=None):
try:
start_status = False
metric = api.Metric()
deploy_mode = env_params["deploy_mode"]
deploy_opology = env_params["deploy_opology"] if "deploy_opology" in env_params else None
env = get_env(env_mode, deploy_mode)
metric.set_run_id()
metric.set_mode(env_mode)
metric.env = Env()
metric.server = Server(version=config.SERVER_VERSION, mode=deploy_mode, deploy_opology=deploy_opology)
logger.info(env_params)
if env_mode == "local":
metric.hardware = Hardware("")
if "server_tag" in env_params and env_params["server_tag"]:
metric.hardware = Hardware("server_tag")
start_status = env.start_up(env_params["host"], env_params["port"])
elif env_mode == "helm":
helm_params = env_params["helm_params"]
helm_path = env_params["helm_path"]
server_name = helm_params["server_name"] if "server_name" in helm_params else None
server_tag = helm_params["server_tag"] if "server_tag" in helm_params else None
if not server_name and not server_tag:
metric.hardware = Hardware("")
else:
metric.hardware = Hardware(server_name) if server_name else Hardware(server_tag)
start_status = env.start_up(helm_path, helm_params)
if start_status:
metric.update_status(status="DEPLOYE_SUCC")
logger.debug("Get runner")
runner = get_runner(run_type, env, metric)
cases, case_metrics = runner.extract_cases(suite)
# TODO: only run when the as_group is equal to True
logger.info("Prepare to run cases")
runner.prepare(**cases[0])
logger.info("Start run case")
suite_status = True
for index, case in enumerate(cases):
case_metric = case_metrics[index]
result = None
err_message = ""
try:
result = runner.run_case(case_metric, **case)
except Exception as e:
err_message = str(e) + "\n" + traceback.format_exc()
logger.error(traceback.format_exc())
logger.info(result)
if result:
case_metric.update_status(status="RUN_SUCC")
case_metric.update_result(result)
else:
case_metric.update_status(status="RUN_FAILED")
case_metric.update_message(err_message)
suite_status = False
logger.debug(case_metric.metrics)
if deploy_mode:
api.save(case_metric)
if suite_status:
metric.update_status(status="RUN_SUCC")
else:
metric.update_status(status="RUN_FAILED")
else:
logger.info("Deploy failed on server")
metric.update_status(status="DEPLOYE_FAILED")
except Exception as e:
logger.error(str(e))
logger.error(traceback.format_exc())
metric.update_status(status="RUN_FAILED")
finally:
if deploy_mode:
api.save(metric)
# time.sleep(10)
env.tear_down()
if metric.status != "RUN_SUCC":
return False
else:
return True
def main():
arg_parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# helm mode with scheduler
arg_parser.add_argument(
"--image-version",
default="",
help="image version")
arg_parser.add_argument(
"--schedule-conf",
metavar='FILE',
default='',
help="load test schedule from FILE")
# local mode
arg_parser.add_argument(
'--local',
action='store_true',
help='use local milvus server')
arg_parser.add_argument(
'--host',
help='server host ip param for local mode',
default='127.0.0.1')
arg_parser.add_argument(
'--port',
help='server port param for local mode',
default=config.SERVER_PORT_DEFAULT)
arg_parser.add_argument(
'--suite',
metavar='FILE',
help='load test suite from FILE',
default='')
arg_parser.add_argument(
'--server-config',
metavar='FILE',
help='load server config from FILE',
default='')
args = arg_parser.parse_args()
if args.schedule_conf:
if args.local:
raise Exception("Helm mode with scheduler and other mode are incompatible")
if not args.image_version:
raise Exception("Image version not given")
env_mode = "helm"
image_version = args.image_version
with open(args.schedule_conf) as f:
schedule_config = full_load(f)
f.close()
helm_path = os.path.join(os.getcwd(), "..//milvus-helm-charts/charts/milvus-ha")
for item in schedule_config:
server_host = item["server"] if "server" in item else ""
server_tag = item["server_tag"] if "server_tag" in item else ""
deploy_mode = item["deploy_mode"] if "deploy_mode" in item else config.DEFAULT_DEPLOY_MODE
suite_params = item["suite_params"]
for suite_param in suite_params:
suite_file = "suites/" + suite_param["suite"]
with open(suite_file) as f:
suite_dict = full_load(f)
f.close()
logger.debug(suite_dict)
run_type, run_params = parser.operations_parser(suite_dict)
collections = run_params["collections"]
image_type = suite_param["image_type"]
image_tag = get_image_tag(image_version)
for suite in collections:
# run test cases
milvus_config = suite["milvus"] if "milvus" in suite else None
server_config = suite["server"] if "server" in suite else None
logger.debug(milvus_config)
logger.debug(server_config)
helm_params = {
"server_name": server_host,
"server_tag": server_tag,
"server_config": server_config,
"milvus_config": milvus_config,
"image_tag": image_tag,
"image_type": image_type
}
env_params = {
"deploy_mode": deploy_mode,
"helm_path": helm_path,
"helm_params": helm_params
}
# job = back_scheduler.add_job(run_suite, args=[run_type, suite, env_mode, env_params],
# misfire_grace_time=36000)
# logger.info(job)
# logger.info(job.id)
elif args.local:
# for local mode
deploy_params = args.server_config
deploy_params_dict = None
if deploy_params:
with open(deploy_params) as f:
deploy_params_dict = full_load(f)
f.close()
logger.debug(deploy_params_dict)
deploy_mode = utils.get_deploy_mode(deploy_params_dict)
server_tag = utils.get_server_tag(deploy_params_dict)
env_params = {
"host": args.host,
"port": args.port,
"deploy_mode": deploy_mode,
"server_tag": server_tag,
"deploy_opology": deploy_params_dict
}
suite_file = args.suite
with open(suite_file) as f:
suite_dict = full_load(f)
f.close()
logger.debug(suite_dict)
run_type, run_params = parser.operations_parser(suite_dict)
collections = run_params["collections"]
if len(collections) > 1:
raise Exception("Multi collections not supported in Local Mode")
# ensure there is only one case in suite
# suite = {"run_type": run_type, "run_params": collections[0]}
suite = collections[0]
timeout = suite["timeout"] if "timeout" in suite else None
env_mode = "local"
return run_suite(run_type, suite, env_mode, env_params, timeout=timeout)
# job = back_scheduler.add_job(run_suite, args=[run_type, suite, env_mode, env_params], misfire_grace_time=36000)
# logger.info(job)
# logger.info(job.id)
if __name__ == "__main__":
try:
if not main():
sys.exit(-1)
# from apscheduler.events import EVENT_JOB_MISSED
# back_scheduler.add_listener(listen_miss, EVENT_JOB_MISSED)
# back_scheduler.start()
# except (KeyboardInterrupt, SystemExit):
# logger.error("Received interruption")
# # back_scheduler.shutdown(wait=False)
# sys.exit(0)
except Exception as e:
logger.error(traceback.format_exc())
# back_scheduler.shutdown(wait=False)
sys.exit(-2)
# block_scheduler.shutdown(wait=False)
logger.info("All tests run finshed")
sys.exit(0)

0
tests-deprecating/milvus_benchmark/milvus_benchmark/metrics/__init__.py
View File

55
tests-deprecating/milvus_benchmark/milvus_benchmark/metrics/api.py
View File

@ -1,55 +0,0 @@
import pdb
import logging
from pymongo import MongoClient
from .models.env import Env
from .models.hardware import Hardware
from .models.metric import Metric
from .models.server import Server
from .config import DB, UNIQUE_ID_COLLECTION, DOC_COLLECTION
from milvus_benchmark import config
_client = MongoClient(config.MONGO_SERVER)
logger = logging.getLogger("milvus_benchmark.metric.api")
def insert_or_get(md5):
collection = _client[DB][UNIQUE_ID_COLLECTION]
found = collection.find_one({'md5': md5})
if not found:
return collection.insert_one({'md5': md5}).inserted_id
return found['_id']
def save(obj):
if not isinstance(obj, Metric):
logger.error("obj is not instance of Metric")
return False
logger.debug(vars(obj))
if not isinstance(obj.server, Server):
logger.error("obj.server is not instance of Server")
return False
if not isinstance(obj.hardware, Hardware):
logger.error("obj.hardware is not instance of Hardware")
return False
if not isinstance(obj.env, Env):
logger.error("obj.env is not instance of Env")
return False
md5 = obj.server.json_md5()
server_doc_id = insert_or_get(md5)
obj.server = {"id": server_doc_id, "value": vars(obj.server)}
md5 = obj.hardware.json_md5()
hardware_doc_id = insert_or_get(md5)
obj.hardware = {"id": hardware_doc_id, "value": vars(obj.hardware)}
md5 = obj.env.json_md5()
env_doc_id = insert_or_get(md5)
obj.env = {"id": env_doc_id, "value": vars(obj.env)}
collection = _client[DB][DOC_COLLECTION]
collection.insert_one(vars(obj))

3
tests-deprecating/milvus_benchmark/milvus_benchmark/metrics/config.py
View File

@ -1,3 +0,0 @@
DB = 'test'
UNIQUE_ID_COLLECTION = 'unique_id'
DOC_COLLECTION = 'doc'

4
tests-deprecating/milvus_benchmark/milvus_benchmark/metrics/models/__init__.py
View File

@ -1,4 +0,0 @@
from .env import Env
from .hardware import Hardware
from .metric import Metric
from .server import Server

23
tests-deprecating/milvus_benchmark/milvus_benchmark/metrics/models/env.py
View File

@ -1,23 +0,0 @@
import json
import hashlib
class Env:
"""
{
"_version": "0.1",
"_type": "env",
"server_config": dict,
"OMP_NUM_THREADS": string,
}
"""
def __init__(self, server_config=None, omp_num_threads=None):
self._version = '0.1'
self._type = 'env'
self.server_config = server_config
self.OMP_NUM_THREADS = omp_num_threads
def json_md5(self):
json_str = json.dumps(vars(self), sort_keys=True)
return hashlib.md5(json_str.encode('utf-8')).hexdigest()

24
tests-deprecating/milvus_benchmark/milvus_benchmark/metrics/models/hardware.py
View File

@ -1,24 +0,0 @@
import json
import hashlib
class Hardware:
"""
{
"_version": "0.1",
"_type": "hardware",
"name": string,
"cpus": float
}
"""
def __init__(self, name=None, cpus=0.0):
self._version = '0.1'
self._type = 'hardware'
self.name = name
self.cpus = cpus
def json_md5(self):
json_str = json.dumps(vars(self), sort_keys=True)
return hashlib.md5(json_str.encode('utf-8')).hexdigest()

131
tests-deprecating/milvus_benchmark/milvus_benchmark/metrics/models/metric.py
View File

@ -1,131 +0,0 @@
import time
import datetime
import json
import hashlib
from .env import Env
from .server import Server
from .hardware import Hardware
class Metric(object):
"""
A template for reporting data:
{
"_id" : ObjectId("6126865855aba6fb8e742f05"),
"_version" : "0.1",
"_type" : "case",
"run_id" : NumberInt(1629914593),
"mode" : "local",
"server" : {
"id" : ObjectId("6126865855aba6fb8e742f04"),
"value" : {
"_version" : "0.1",
"_type" : "server",
"version" : "2.0.0-RC5",
"mode" : "single",
"build_commit" : null,
"deploy_opology" : {
"server" : {
"server_tag" : "8c16m"
},
"milvus" : {
"deploy_mode" : "single"
}
}
}
},
"hardware" : {
"id" : ObjectId("60f078c5d8aad7192f9baf80"),
"value" : {
"_version" : "0.1",
"_type" : "hardware",
"name" : "server_tag",
"cpus" : 0.0
}
},
"env" : {
"id" : ObjectId("604b54df90fbee981a6ed81d"),
"value" : {
"_version" : "0.1",
"_type" : "env",
"server_config" : null,
"OMP_NUM_THREADS" : null
}
},
"status" : "RUN_SUCC",
"err_message" : "",
"collection" : {
"dimension" : NumberInt(128),
"metric_type" : "l2",
"dataset_name" : "sift_128_euclidean"
},
"index" : {
"index_type" : "ivf_sq8",
"index_param" : {
"nlist" : NumberInt(1024)
}
},
"search" : {
"nq" : NumberInt(10000),
"topk" : NumberInt(10),
"search_param" : {
"nprobe" : NumberInt(1)
},
"filter" : [
]
},
"run_params" : null,
"metrics" : {
"type" : "ann_accuracy",
"value" : {
"acc" : 0.377
}
},
"datetime" : "2021-08-25 18:03:13.820593",
"type" : "metric"
}
"""
def __init__(self):
self._version = '0.1'
self._type = 'metric'
self.run_id = None
self.mode = None
self.server = Server()
self.hardware = Hardware()
self.env = Env()
self.status = "INIT"
self.err_message = ""
self.collection = {}
self.index = {}
self.search = {}
self.run_params = {}
self.metrics = {
"type": "",
"value": None,
}
self.datetime = str(datetime.datetime.now())
def set_run_id(self):
self.run_id = int(time.time())
def set_mode(self, mode):
self.mode = mode
# including: metric, suite_metric
def set_case_metric_type(self):
self._type = "case"
def json_md5(self):
json_str = json.dumps(vars(self), sort_keys=True)
return hashlib.md5(json_str.encode('utf-8')).hexdigest()
def update_status(self, status):
self.status = status
def update_result(self, result):
self.metrics["value"].update(result)
def update_message(self, err_message):
self.err_message = err_message

27
tests-deprecating/milvus_benchmark/milvus_benchmark/metrics/models/server.py
View File

@ -1,27 +0,0 @@
import json
import hashlib
class Server:
"""
{
"_version": "0.1",
"_type": "server",
"version": string,
"build_commit": string,
# "md5": string,
}
"""
def __init__(self, version=None, mode=None, build_commit=None, deploy_opology=None):
self._version = '0.1'
self._type = 'server'
self.version = version
self.mode = mode
self.build_commit = build_commit
self.deploy_opology = deploy_opology
# self.md5 = md5
def json_md5(self):
json_str = json.dumps(vars(self), sort_keys=True)
return hashlib.md5(json_str.encode('utf-8')).hexdigest()

88
tests-deprecating/milvus_benchmark/milvus_benchmark/parser.py
View File

@ -1,88 +0,0 @@
import pdb
import logging
logger = logging.getLogger("milvus_benchmark.parser")
def operations_parser(operations):
if not operations:
raise Exception("No operations in suite defined")
for run_type, run_params in operations.items():
logger.debug(run_type)
return (run_type, run_params)
def collection_parser(collection_name):
""" Collection size analysis """
tmp = collection_name.split("_")
# if len(tmp) != 5:
# return None
data_type = tmp[0]
collection_size_unit = tmp[1][-1]
collection_size = tmp[1][0:-1]
if collection_size_unit == "w":
collection_size = int(collection_size) * 10000
elif collection_size_unit == "m":
collection_size = int(collection_size) * 1000000
elif collection_size_unit == "b":
collection_size = int(collection_size) * 1000000000
dimension = int(tmp[2])
metric_type = str(tmp[3])
return (data_type, collection_size, dimension, metric_type)
def parse_ann_collection_name(collection_name):
data_type = collection_name.split("_")[0]
dimension = int(collection_name.split("_")[1])
metric = collection_name.split("_")[2]
# metric = collection_name.attrs['distance']
# dimension = len(collection_name["train"][0])
if metric == "euclidean":
metric_type = "l2"
elif metric == "angular":
metric_type = "ip"
elif metric == "jaccard":
metric_type = "jaccard"
elif metric == "hamming":
metric_type = "hamming"
return (data_type, dimension, metric_type)
def search_params_parser(param):
# parse top-k, set default value if top-k not in param
if "top_ks" not in param:
top_ks = [10]
else:
top_ks = param["top_ks"]
if isinstance(top_ks, int):
top_ks = [top_ks]
elif isinstance(top_ks, list):
top_ks = list(top_ks)
else:
logger.warning("Invalid format top-ks: %s" % str(top_ks))
# parse nqs, set default value if nq not in param
if "nqs" not in param:
nqs = [10]
else:
nqs = param["nqs"]
if isinstance(nqs, int):
nqs = [nqs]
elif isinstance(nqs, list):
nqs = list(nqs)
else:
logger.warning("Invalid format nqs: %s" % str(nqs))
# parse nprobes
if "nprobes" not in param:
nprobes = [1]
else:
nprobes = param["nprobes"]
if isinstance(nprobes, int):
nprobes = [nprobes]
elif isinstance(nprobes, list):
nprobes = list(nprobes)
else:
logger.warning("Invalid format nprobes: %s" % str(nprobes))
return top_ks, nqs, nprobes

26
tests-deprecating/milvus_benchmark/milvus_benchmark/runners/__init__.py
View File

@ -1,26 +0,0 @@
from .insert import InsertRunner, BPInsertRunner
from .locust import LocustInsertRunner, LocustSearchRunner, LocustRandomRunner
from .search import SearchRunner, InsertSearchRunner
from .build import BuildRunner, InsertBuildRunner
from .get import InsertGetRunner
from .accuracy import AccuracyRunner
from .accuracy import AccAccuracyRunner
from .chaos import SimpleChaosRunner
def get_runner(name, env, metric):
return {
"insert_performance": InsertRunner(env, metric),
"bp_insert_performance": BPInsertRunner(env, metric),
"search_performance": SearchRunner(env, metric),
"insert_search_performance": InsertSearchRunner(env, metric),
"locust_insert_performance": LocustInsertRunner(env, metric),
"locust_search_performance": LocustSearchRunner(env, metric),
"locust_random_performance": LocustRandomRunner(env, metric),
"insert_build_performance": InsertBuildRunner(env, metric),
"insert_get_performance": InsertGetRunner(env, metric),
"build_performance": BuildRunner(env, metric),
"accuracy": AccuracyRunner(env, metric),
"ann_accuracy": AccAccuracyRunner(env, metric),
"simple_chaos": SimpleChaosRunner(env, metric)
}.get(name)

263
tests-deprecating/milvus_benchmark/milvus_benchmark/runners/accuracy.py
View File

@ -1,263 +0,0 @@
import json
import time
import copy
import logging
import numpy as np
from milvus_benchmark import parser
from milvus_benchmark.runners import utils
from milvus_benchmark.runners.base import BaseRunner
logger = logging.getLogger("milvus_benchmark.runners.accuracy")
INSERT_INTERVAL = 50000
class AccuracyRunner(BaseRunner):
"""run accuracy"""
name = "accuracy"
def __init__(self, env, metric):
super(AccuracyRunner, self).__init__(env, metric)
def extract_cases(self, collection):
collection_name = collection["collection_name"] if "collection_name" in collection else None
(data_type, collection_size, dimension, metric_type) = parser.collection_parser(collection_name)
vector_type = utils.get_vector_type(data_type)
index_field_name = utils.get_default_field_name(vector_type)
base_query_vectors = utils.get_vectors_from_binary(utils.MAX_NQ, dimension, data_type)
# collection_info and index_info are used to report
collection_info = {
"dimension": dimension,
"metric_type": metric_type,
"dataset_name": collection_name,
"collection_size": collection_size
}
index_info = self.milvus.describe_index(index_field_name, collection_name)
filters = collection["filters"] if "filters" in collection else []
filter_query = []
top_ks = collection["top_ks"]
nqs = collection["nqs"]
search_params = collection["search_params"]
search_params = utils.generate_combinations(search_params)
cases = list()
case_metrics = list()
self.init_metric(self.name, collection_info, index_info, search_info=None)
for search_param in search_params:
if not filters:
filters.append(None)
for filter in filters:
filter_param = []
if isinstance(filter, dict) and "range" in filter:
filter_query.append(eval(filter["range"]))
filter_param.append(filter["range"])
if isinstance(filter, dict) and "term" in filter:
filter_query.append(eval(filter["term"]))
filter_param.append(filter["term"])
for nq in nqs:
query_vectors = base_query_vectors[0:nq]
for top_k in top_ks:
search_info = {
"topk": top_k,
"query": query_vectors,
"metric_type": utils.metric_type_trans(metric_type),
"params": search_param}
# TODO: only update search_info
case_metric = copy.deepcopy(self.metric)
case_metric.set_case_metric_type()
case_metric.search = {
"nq": nq,
"topk": top_k,
"search_param": search_param,
"filter": filter_param
}
vector_query = {"vector": {index_field_name: search_info}}
case = {
"collection_name": collection_name,
"index_field_name": index_field_name,
"dimension": dimension,
"data_type": data_type,
"metric_type": metric_type,
"vector_type": vector_type,
"collection_size": collection_size,
"filter_query": filter_query,
"vector_query": vector_query
}
cases.append(case)
case_metrics.append(case_metric)
return cases, case_metrics
def prepare(self, **case_param):
collection_name = case_param["collection_name"]
self.milvus.set_collection(collection_name)
if not self.milvus.exists_collection():
logger.info("collection not exist")
self.milvus.load_collection(timeout=600)
def run_case(self, case_metric, **case_param):
collection_size = case_param["collection_size"]
nq = case_metric.search["nq"]
top_k = case_metric.search["topk"]
query_res = self.milvus.query(case_param["vector_query"], filter_query=case_param["filter_query"])
true_ids = utils.get_ground_truth_ids(collection_size)
logger.debug({"true_ids": [len(true_ids[0]), len(true_ids[0])]})
result_ids = self.milvus.get_ids(query_res)
logger.debug({"result_ids": len(result_ids[0])})
acc_value = utils.get_recall_value(true_ids[:nq, :top_k].tolist(), result_ids)
tmp_result = {"acc": acc_value}
return tmp_result
class AccAccuracyRunner(AccuracyRunner):
"""run ann accuracy"""
"""
1. entities from hdf5
2. one collection test different index
"""
name = "ann_accuracy"
def __init__(self, env, metric):
super(AccAccuracyRunner, self).__init__(env, metric)
def extract_cases(self, collection):
collection_name = collection["collection_name"] if "collection_name" in collection else None
(data_type, dimension, metric_type) = parser.parse_ann_collection_name(collection_name)
hdf5_source_file = collection["source_file"]
index_types = collection["index_types"]
index_params = collection["index_params"]
top_ks = collection["top_ks"]
nqs = collection["nqs"]
search_params = collection["search_params"]
vector_type = utils.get_vector_type(data_type)
index_field_name = utils.get_default_field_name(vector_type)
dataset = utils.get_dataset(hdf5_source_file)
collection_info = {
"dimension": dimension,
"metric_type": metric_type,
"dataset_name": collection_name
}
filters = collection["filters"] if "filters" in collection else []
filter_query = []
search_params = utils.generate_combinations(search_params)
index_params = utils.generate_combinations(index_params)
cases = list()
case_metrics = list()
self.init_metric(self.name, collection_info, {}, search_info=None)
true_ids = np.array(dataset["neighbors"])
for index_type in index_types:
for index_param in index_params:
index_info = {
"index_type": index_type,
"index_param": index_param
}
for search_param in search_params:
if not filters:
filters.append(None)
for filter in filters:
filter_param = []
if isinstance(filter, dict) and "range" in filter:
filter_query.append(eval(filter["range"]))
filter_param.append(filter["range"])
if isinstance(filter, dict) and "term" in filter:
filter_query.append(eval(filter["term"]))
filter_param.append(filter["term"])
for nq in nqs:
query_vectors = utils.normalize(metric_type, np.array(dataset["test"][:nq]))
for top_k in top_ks:
search_info = {
"topk": top_k,
"query": query_vectors,
"metric_type": utils.metric_type_trans(metric_type),
"params": search_param}
# TODO: only update search_info
case_metric = copy.deepcopy(self.metric)
case_metric.set_case_metric_type()
case_metric.index = index_info
case_metric.search = {
"nq": nq,
"topk": top_k,
"search_param": search_param,
"filter": filter_param
}
vector_query = {"vector": {index_field_name: search_info}}
case = {
"collection_name": collection_name,
"dataset": dataset,
"index_field_name": index_field_name,
"dimension": dimension,
"data_type": data_type,
"metric_type": metric_type,
"vector_type": vector_type,
"index_type": index_type,
"index_param": index_param,
"filter_query": filter_query,
"vector_query": vector_query,
"true_ids": true_ids
}
cases.append(case)
case_metrics.append(case_metric)
return cases, case_metrics
def prepare(self, **case_param):
collection_name = case_param["collection_name"]
metric_type = case_param["metric_type"]
dimension = case_param["dimension"]
vector_type = case_param["vector_type"]
index_type = case_param["index_type"]
index_param = case_param["index_param"]
index_field_name = case_param["index_field_name"]
self.milvus.set_collection(collection_name)
if self.milvus.exists_collection(collection_name):
logger.info("Re-create collection: %s" % collection_name)
self.milvus.drop()
dataset = case_param["dataset"]
self.milvus.create_collection(dimension, data_type=vector_type)
insert_vectors = utils.normalize(metric_type, np.array(dataset["train"]))
if len(insert_vectors) != dataset["train"].shape[0]:
raise Exception("Row count of insert vectors: %d is not equal to dataset size: %d" % (
len(insert_vectors), dataset["train"].shape[0]))
logger.debug("The row count of entities to be inserted: %d" % len(insert_vectors))
# Insert batch once
# milvus_instance.insert(insert_vectors)
info = self.milvus.get_info(collection_name)
loops = len(insert_vectors) // INSERT_INTERVAL + 1
for i in range(loops):
start = i * INSERT_INTERVAL
end = min((i + 1) * INSERT_INTERVAL, len(insert_vectors))
if start < end:
tmp_vectors = insert_vectors[start:end]
ids = [i for i in range(start, end)]
if not isinstance(tmp_vectors, list):
entities = utils.generate_entities(info, tmp_vectors.tolist(), ids)
res_ids = self.milvus.insert(entities)
else:
entities = utils.generate_entities(tmp_vectors, ids)
res_ids = self.milvus.insert(entities)
assert res_ids == ids
logger.debug("End insert, start flush")
self.milvus.flush()
logger.debug("End flush")
res_count = self.milvus.count()
logger.info("Table: %s, row count: %d" % (collection_name, res_count))
if res_count != len(insert_vectors):
raise Exception("Table row count is not equal to insert vectors")
if self.milvus.describe_index(index_field_name):
self.milvus.drop_index(index_field_name)
logger.info("Re-create index: %s" % collection_name)
self.milvus.create_index(index_field_name, index_type, metric_type, index_param=index_param)
logger.info(self.milvus.describe_index(index_field_name))
logger.info("Start load collection: %s" % collection_name)
# self.milvus.release_collection()
self.milvus.load_collection(timeout=600)
logger.info("End load collection: %s" % collection_name)
def run_case(self, case_metric, **case_param):
true_ids = case_param["true_ids"]
nq = case_metric.search["nq"]
top_k = case_metric.search["topk"]
query_res = self.milvus.query(case_param["vector_query"], filter_query=case_param["filter_query"])
result_ids = self.milvus.get_ids(query_res)
acc_value = utils.get_recall_value(true_ids[:nq, :top_k].tolist(), result_ids)
tmp_result = {"acc": acc_value}
return tmp_result

152
tests-deprecating/milvus_benchmark/milvus_benchmark/runners/base.py
View File

@ -1,152 +0,0 @@
import time
import pdb
import logging
import threading
import traceback
import grpc
import numpy as np
from milvus_benchmark.env import get_env
from milvus_benchmark import config
from milvus_benchmark.client import MilvusClient
from . import utils
logger = logging.getLogger("milvus_benchmark.runners.base")
class BaseRunner(object):
"""runner is actually the executors"""
def __init__(self, env, metric):
self._metric = metric
self._env = env
self._run_as_group = False
self._result = dict()
self._milvus = MilvusClient(host=self._env.hostname)
def run(self, run_params):
pass
def stop(self):
logger.debug("Stop runner...")
pass
@property
def hostname(self):
return self._env.hostname
@property
def port(self):
return self._env.port
@property
def milvus(self):
return self._milvus
@property
def metric(self):
return self._metric
@property
def result(self):
return self._result
@property
def run_as_group(self):
return self._run_as_group
def init_metric(self, name, collection_info=None, index_info=None, search_info=None, run_params=None, t="metric"):
self._metric.collection = collection_info
self._metric.index = index_info
self._metric.search = search_info
self._metric.type = t
self._metric.run_params = run_params
self._metric.metrics = {
"type": name,
"value": self._result
}
# TODO: need an easy method to change value in metric
def update_metric(self, key, value):
pass
def insert_core(self, milvus, info, start_id, vectors):
# start insert vectors
end_id = start_id + len(vectors)
logger.debug("Start id: %s, end id: %s" % (start_id, end_id))
ids = [k for k in range(start_id, end_id)]
entities = utils.generate_entities(info, vectors, ids)
ni_start_time = time.time()
try:
_res_ids = milvus.insert(entities)
except Exception as e:
logger.error("Insert failed")
logger.error(traceback.format_exc())
raise e
# assert ids == res_ids
# milvus.flush()
ni_end_time = time.time()
logger.debug(milvus.count())
return ni_end_time-ni_start_time
# TODO: need to improve
def insert(self, milvus, collection_name, data_type, dimension, size, ni):
total_time = 0.0
rps = 0.0
ni_time = 0.0
vectors_per_file = utils.get_len_vectors_per_file(data_type, dimension)
if size % vectors_per_file or size % ni:
logger.error("Not invalid collection size or ni")
return False
i = 0
info = milvus.get_info(collection_name)
if data_type == "local" or not data_type:
# insert local
info = milvus.get_info(collection_name)
while i < (size // vectors_per_file):
vectors = []
for j in range(vectors_per_file // ni):
# vectors = src_vectors[j * ni:(j + 1) * ni]
vectors = utils.generate_vectors(ni, dimension)
if vectors:
start_id = i * vectors_per_file + j * ni
ni_time = self.insert_core(milvus, info, start_id, vectors)
total_time = total_time+ni_time
i += 1
else:
# insert from file
while i < (size // vectors_per_file):
vectors = []
if vectors_per_file >= ni:
file_name = utils.gen_file_name(i, dimension, data_type)
# logger.info("Load npy file: %s start" % file_name)
data = np.load(file_name)
# logger.info("Load npy file: %s end" % file_name)
for j in range(vectors_per_file // ni):
vectors = data[j * ni:(j + 1) * ni].tolist()
if vectors:
start_id = i * vectors_per_file + j * ni
ni_time = self.insert_core(milvus, info, start_id, vectors)
total_time = total_time+ni_time
i += 1
else:
vectors.clear()
loops = ni // vectors_per_file
for j in range(loops):
file_name = utils.gen_file_name(loops * i + j, dimension, data_type)
data = np.load(file_name)
vectors.extend(data.tolist())
if vectors:
start_id = i * vectors_per_file
ni_time = self.insert_core(milvus, info, start_id, vectors)
total_time = total_time+ni_time
i += loops
rps = round(size / total_time, 2)
ni_time = round(total_time / (size / ni), 2)
result = {
"total_time": round(total_time, 2),
"rps": rps,
"ni_time": ni_time
}
logger.info(result)
return result

106
tests-deprecating/milvus_benchmark/milvus_benchmark/runners/build.py
View File

@ -1,106 +0,0 @@
import time
import copy
import logging
from milvus_benchmark import parser
from milvus_benchmark.runners import utils
from milvus_benchmark.runners.base import BaseRunner
logger = logging.getLogger("milvus_benchmark.runners.build")
class BuildRunner(BaseRunner):
"""run build"""
name = "build_performance"
def __init__(self, env, metric):
super(BuildRunner, self).__init__(env, metric)
def extract_cases(self, collection):
collection_name = collection["collection_name"] if "collection_name" in collection else None
(data_type, collection_size, dimension, metric_type) = parser.collection_parser(collection_name)
ni_per = collection["ni_per"]
vector_type = utils.get_vector_type(data_type)
other_fields = collection["other_fields"] if "other_fields" in collection else None
collection_info = {
"dimension": dimension,
"metric_type": metric_type,
"dataset_name": collection_name,
"collection_size": collection_size,
"other_fields": other_fields,
"ni_per": ni_per
}
index_field_name = utils.get_default_field_name(vector_type)
index_type = collection["index_type"]
index_param = collection["index_param"]
index_info = {
"index_type": index_type,
"index_param": index_param
}
flush = True
if "flush" in collection and collection["flush"] == "no":
flush = False
self.init_metric(self.name, collection_info, index_info, search_info=None)
case_metric = copy.deepcopy(self.metric)
case_metric.set_case_metric_type()
case_metrics = list()
case_params = list()
case_metrics.append(case_metric)
case_param = {
"collection_name": collection_name,
"data_type": data_type,
"dimension": dimension,
"collection_size": collection_size,
"ni_per": ni_per,
"metric_type": metric_type,
"vector_type": vector_type,
"other_fields": other_fields,
"flush_after_insert": flush,
"index_field_name": index_field_name,
"index_type": index_type,
"index_param": index_param,
}
case_params.append(case_param)
return case_params, case_metrics
def prepare(self, **case_param):
collection_name = case_param["collection_name"]
self.milvus.set_collection(collection_name)
if not self.milvus.exists_collection():
logger.info("collection not exist")
logger.debug({"collection count": self.milvus.count()})
def run_case(self, case_metric, **case_param):
index_field_name = case_param["index_field_name"]
start_time = time.time()
self.milvus.create_index(index_field_name, case_param["index_type"], case_param["metric_type"],
index_param=case_param["index_param"])
build_time = round(time.time() - start_time, 2)
tmp_result = {"build_time": build_time}
return tmp_result
class InsertBuildRunner(BuildRunner):
"""run insert and build"""
name = "insert_build_performance"
def __init__(self, env, metric):
super(InsertBuildRunner, self).__init__(env, metric)
def prepare(self, **case_param):
collection_name = case_param["collection_name"]
dimension = case_param["dimension"]
vector_type = case_param["vector_type"]
other_fields = case_param["other_fields"]
self.milvus.set_collection(collection_name)
if self.milvus.exists_collection():
logger.debug("Start drop collection")
self.milvus.drop()
time.sleep(utils.DELETE_INTERVAL_TIME)
self.milvus.create_collection(dimension, data_type=vector_type, other_fields=other_fields)
self.insert(self.milvus, collection_name, case_param["data_type"], dimension,
case_param["collection_size"], case_param["ni_per"])
start_time = time.time()
self.milvus.flush()
flush_time = round(time.time() - start_time, 2)
logger.debug({"collection count": self.milvus.count()})
logger.debug({"flush_time": flush_time})

127
tests-deprecating/milvus_benchmark/milvus_benchmark/runners/chaos.py
View File

@ -1,127 +0,0 @@
import copy
import logging
import pdb
import time
from operator import methodcaller
from yaml import full_load, dump
import threading
from milvus_benchmark import utils
from milvus_benchmark.runners import utils as runner_utils
from milvus_benchmark.chaos import utils as chaos_utils
from milvus_benchmark.runners.base import BaseRunner
from chaos.chaos_opt import ChaosOpt
from milvus_benchmark import config
from milvus_benchmark.chaos.chaos_mesh import PodChaos, NetworkChaos
logger = logging.getLogger("milvus_benchmark.runners.chaos")
kind_chaos_mapping = {
"PodChaos": PodChaos,
"NetworkChaos": NetworkChaos
}
assert_func_mapping = {
"fail": chaos_utils.assert_fail,
"pass": chaos_utils.assert_pass
}
class SimpleChaosRunner(BaseRunner):
"""run chaos"""
name = "simple_chaos"
def __init__(self, env, metric):
super(SimpleChaosRunner, self).__init__(env, metric)
async def async_call(self, func, **kwargs):
future = methodcaller(func, **kwargs)(self.milvus)
def run_step(self, interface_name, interface_params):
if interface_name == "create_collection":
collection_name = utils.get_unique_name("chaos")
self.data_type = interface_params["data_type"]
self.dimension = interface_params["dimension"]
self.milvus.set_collection(collection_name)
vector_type = runner_utils.get_vector_type(self.data_type)
self.milvus.create_collection(self.dimension, data_type=vector_type)
elif interface_name == "insert":
batch_size = interface_params["batch_size"]
collection_size = interface_params["collection_size"]
self.insert(self.milvus, self.milvus.collection_name, self.data_type, self.dimension, collection_size,
batch_size)
elif interface_name == "create_index":
metric_type = interface_params["metric_type"]
index_type = interface_params["index_type"]
index_param = interface_params["index_param"]
vector_type = runner_utils.get_vector_type(self.data_type)
field_name = runner_utils.get_default_field_name(vector_type)
self.milvus.create_index(field_name, index_type, metric_type, index_param=index_param)
elif interface_name == "flush":
self.milvus.flush()
def extract_cases(self, collection):
before_steps = collection["before"]
after = collection["after"] if "after" in collection else None
processing = collection["processing"]
case_metrics = []
case_params = [{
"before_steps": before_steps,
"after": after,
"processing": processing
}]
self.init_metric(self.name, {}, {}, None)
case_metric = copy.deepcopy(self.metric)
case_metric.set_case_metric_type()
case_metrics.append(case_metric)
return case_params, case_metrics
def prepare(self, **case_param):
steps = case_param["before_steps"]
for step in steps:
interface_name = step["interface_name"]
params = step["params"]
self.run_step(interface_name, params)
def run_case(self, case_metric, **case_param):
processing = case_param["processing"]
after = case_param["after"]
user_chaos = processing["chaos"]
kind = user_chaos["kind"]
spec = user_chaos["spec"]
metadata_name = config.NAMESPACE + "-" + kind.lower()
metadata = {"name": metadata_name}
process_assertion = processing["assertion"]
after_assertion = after["assertion"]
# load yaml from default template to generate stand chaos dict
chaos_mesh = kind_chaos_mapping[kind](config.DEFAULT_API_VERSION, kind, metadata, spec)
experiment_config = chaos_mesh.gen_experiment_config()
process_func = processing["interface_name"]
process_params = processing["params"] if "params" in processing else {}
after_func = after["interface_name"]
after_params = after["params"] if "params" in after else {}
logger.debug(chaos_mesh.kind)
chaos_opt = ChaosOpt(chaos_mesh.kind)
chaos_objects = chaos_opt.list_chaos_object()
if len(chaos_objects["items"]) != 0:
logger.debug(chaos_objects["items"])
chaos_opt.delete_chaos_object(chaos_mesh.metadata["name"])
# with open('./pod-newq.yaml', "w") as f:
# dump(experiment_config, f)
# f.close()
# concurrent inject chaos and run func
# logger.debug(experiment_config)
t_milvus = threading.Thread(target=assert_func_mapping[process_assertion], args=(process_func, self.milvus,), kwargs=process_params)
try:
t_milvus.start()
chaos_opt.create_chaos_object(experiment_config)
# processing assert exception
except Exception as e:
logger.info("exception {}".format(str(e)))
else:
chaos_opt.delete_chaos_object(chaos_mesh.metadata["name"])
# TODO retry connect milvus
time.sleep(15)
assert_func_mapping[after_assertion](after_func, self.milvus, **after_params)
finally:
chaos_opt.delete_all_chaos_object()
logger.info(chaos_opt.list_chaos_object())

366
tests-deprecating/milvus_benchmark/milvus_benchmark/runners/docker_runner.py
View File

@ -1,366 +0,0 @@
import os
import logging
import pdb
import time
import random
from multiprocessing import Process
import numpy as np
from client import MilvusClient
import utils
import parser
from runner import Runner
logger = logging.getLogger("milvus_benchmark.docker")
class DockerRunner(Runner):
"""run docker mode"""
def __init__(self, image):
super(DockerRunner, self).__init__()
self.image = image
def run(self, definition, run_type=None):
if run_type == "performance":
for op_type, op_value in definition.items():
# run docker mode
run_count = op_value["run_count"]
run_params = op_value["params"]
container = None
if op_type == "insert":
if not run_params:
logger.debug("No run params")
continue
for index, param in enumerate(run_params):
logger.info("Definition param: %s" % str(param))
collection_name = param["collection_name"]
volume_name = param["db_path_prefix"]
print(collection_name)
(data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name)
for k, v in param.items():
if k.startswith("server."):
# Update server config
utils.modify_config(k, v, type="server", db_slave=None)
container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None)
time.sleep(2)
milvus = MilvusClient(collection_name)
# Check has collection or not
if milvus.exists_collection():
milvus.delete()
time.sleep(10)
milvus.create_collection(collection_name, dimension, index_file_size, metric_type)
# debug
# milvus.create_index("ivf_sq8", 16384)
res = self.do_insert(milvus, collection_name, data_type, dimension, collection_size, param["ni_per"])
logger.info(res)
# wait for file merge
time.sleep(collection_size * dimension / 5000000)
# Clear up
utils.remove_container(container)
elif op_type == "query":
for index, param in enumerate(run_params):
logger.info("Definition param: %s" % str(param))
collection_name = param["dataset"]
volume_name = param["db_path_prefix"]
(data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name)
for k, v in param.items():
if k.startswith("server."):
utils.modify_config(k, v, type="server")
container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None)
time.sleep(2)
milvus = MilvusClient(collection_name)
logger.debug(milvus.show_collections())
# Check has collection or not
if not milvus.exists_collection():
logger.warning("Table %s not existed, continue exec next params ..." % collection_name)
continue
# parse index info
index_types = param["index.index_types"]
nlists = param["index.nlists"]
# parse top-k, nq, nprobe
top_ks, nqs, nprobes = parser.search_params_parser(param)
for index_type in index_types:
for nlist in nlists:
result = milvus.describe_index()
logger.info(result)
# milvus.drop_index()
# milvus.create_index(index_type, nlist)
result = milvus.describe_index()
logger.info(result)
logger.info(milvus.count())
# preload index
milvus.preload_collection()
logger.info("Start warm up query")
res = self.do_query(milvus, collection_name, [1], [1], 1, 1)
logger.info("End warm up query")
# Run query test
for nprobe in nprobes:
logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe))
res = self.do_query(milvus, collection_name, top_ks, nqs, nprobe, run_count)
headers = ["Nq/Top-k"]
headers.extend([str(top_k) for top_k in top_ks])
utils.print_collection(headers, nqs, res)
utils.remove_container(container)
elif run_type == "insert_performance":
for op_type, op_value in definition.items():
# run docker mode
run_count = op_value["run_count"]
run_params = op_value["params"]
container = None
if not run_params:
logger.debug("No run params")
continue
for index, param in enumerate(run_params):
logger.info("Definition param: %s" % str(param))
collection_name = param["collection_name"]
volume_name = param["db_path_prefix"]
print(collection_name)
(data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name)
for k, v in param.items():
if k.startswith("server."):
# Update server config
utils.modify_config(k, v, type="server", db_slave=None)
container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None)
time.sleep(2)
milvus = MilvusClient(collection_name)
# Check has collection or not
if milvus.exists_collection():
milvus.delete()
time.sleep(10)
milvus.create_collection(collection_name, dimension, index_file_size, metric_type)
# debug
# milvus.create_index("ivf_sq8", 16384)
res = self.do_insert(milvus, collection_name, data_type, dimension, collection_size, param["ni_per"])
logger.info(res)
# wait for file merge
time.sleep(collection_size * dimension / 5000000)
# Clear up
utils.remove_container(container)
elif run_type == "search_performance":
for op_type, op_value in definition.items():
# run docker mode
run_count = op_value["run_count"]
run_params = op_value["params"]
container = None
for index, param in enumerate(run_params):
logger.info("Definition param: %s" % str(param))
collection_name = param["dataset"]
volume_name = param["db_path_prefix"]
(data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name)
for k, v in param.items():
if k.startswith("server."):
utils.modify_config(k, v, type="server")
container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None)
time.sleep(2)
milvus = MilvusClient(collection_name)
logger.debug(milvus.show_collections())
# Check has collection or not
if not milvus.exists_collection():
logger.warning("Table %s not existed, continue exec next params ..." % collection_name)
continue
# parse index info
index_types = param["index.index_types"]
nlists = param["index.nlists"]
# parse top-k, nq, nprobe
top_ks, nqs, nprobes = parser.search_params_parser(param)
for index_type in index_types:
for nlist in nlists:
result = milvus.describe_index()
logger.info(result)
# milvus.drop_index()
# milvus.create_index(index_type, nlist)
result = milvus.describe_index()
logger.info(result)
logger.info(milvus.count())
# preload index
milvus.preload_collection()
logger.info("Start warm up query")
res = self.do_query(milvus, collection_name, [1], [1], 1, 1)
logger.info("End warm up query")
# Run query test
for nprobe in nprobes:
logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe))
res = self.do_query(milvus, collection_name, top_ks, nqs, nprobe, run_count)
headers = ["Nq/Top-k"]
headers.extend([str(top_k) for top_k in top_ks])
utils.print_collection(headers, nqs, res)
utils.remove_container(container)
elif run_type == "accuracy":
"""
{
"dataset": "random_50m_1024_512",
"index.index_types": ["flat", ivf_flat", "ivf_sq8"],
"index.nlists": [16384],
"nprobes": [1, 32, 128],
"nqs": [100],
"top_ks": [1, 64],
"server.use_blas_threshold": 1100,
"server.cpu_cache_capacity": 256
}
"""
for op_type, op_value in definition.items():
if op_type != "query":
logger.warning("invalid operation: %s in accuracy test, only support query operation" % op_type)
break
run_count = op_value["run_count"]
run_params = op_value["params"]
container = None
for index, param in enumerate(run_params):
logger.info("Definition param: %s" % str(param))
collection_name = param["dataset"]
sift_acc = False
if "sift_acc" in param:
sift_acc = param["sift_acc"]
(data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name)
for k, v in param.items():
if k.startswith("server."):
utils.modify_config(k, v, type="server")
volume_name = param["db_path_prefix"]
container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None)
time.sleep(2)
milvus = MilvusClient(collection_name)
# Check has collection or not
if not milvus.exists_collection():
logger.warning("Table %s not existed, continue exec next params ..." % collection_name)
continue
# parse index info
index_types = param["index.index_types"]
nlists = param["index.nlists"]
# parse top-k, nq, nprobe
top_ks, nqs, nprobes = parser.search_params_parser(param)
if sift_acc is True:
# preload groundtruth data
true_ids_all = self.get_groundtruth_ids(collection_size)
acc_dict = {}
for index_type in index_types:
for nlist in nlists:
result = milvus.describe_index()
logger.info(result)
milvus.create_index(index_type, nlist)
# preload index
milvus.preload_collection()
# Run query test
for nprobe in nprobes:
logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe))
for top_k in top_ks:
for nq in nqs:
result_ids = []
id_prefix = "%s_index_%s_nlist_%s_metric_type_%s_nprobe_%s_top_k_%s_nq_%s" % \
(collection_name, index_type, nlist, metric_type, nprobe, top_k, nq)
if sift_acc is False:
self.do_query_acc(milvus, collection_name, top_k, nq, nprobe, id_prefix)
if index_type != "flat":
# Compute accuracy
base_name = "%s_index_flat_nlist_%s_metric_type_%s_nprobe_%s_top_k_%s_nq_%s" % \
(collection_name, nlist, metric_type, nprobe, top_k, nq)
avg_acc = self.compute_accuracy(base_name, id_prefix)
logger.info("Query: <%s> accuracy: %s" % (id_prefix, avg_acc))
else:
result_ids, result_distances = self.do_query_ids(milvus, collection_name, top_k, nq, nprobe)
debug_file_ids = "0.5.3_result_ids"
debug_file_distances = "0.5.3_result_distances"
with open(debug_file_ids, "w+") as fd:
total = 0
for index, item in enumerate(result_ids):
true_item = true_ids_all[:nq, :top_k].tolist()[index]
tmp = set(item).intersection(set(true_item))
total = total + len(tmp)
fd.write("query: N-%d, intersection: %d, total: %d\n" % (index, len(tmp), total))
fd.write("%s\n" % str(item))
fd.write("%s\n" % str(true_item))
acc_value = self.get_recall_value(true_ids_all[:nq, :top_k].tolist(), result_ids)
logger.info("Query: <%s> accuracy: %s" % (id_prefix, acc_value))
# # print accuracy collection
# headers = [collection_name]
# headers.extend([str(top_k) for top_k in top_ks])
# utils.print_collection(headers, nqs, res)
# remove container, and run next definition
logger.info("remove container, and run next definition")
utils.remove_container(container)
elif run_type == "stability":
for op_type, op_value in definition.items():
if op_type != "query":
logger.warning("invalid operation: %s in accuracy test, only support query operation" % op_type)
break
run_count = op_value["run_count"]
run_params = op_value["params"]
container = None
for index, param in enumerate(run_params):
logger.info("Definition param: %s" % str(param))
collection_name = param["dataset"]
index_type = param["index_type"]
volume_name = param["db_path_prefix"]
(data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name)
# set default test time
if "during_time" not in param:
during_time = 100 # seconds
else:
during_time = int(param["during_time"]) * 60
# set default query process num
if "query_process_num" not in param:
query_process_num = 10
else:
query_process_num = int(param["query_process_num"])
for k, v in param.items():
if k.startswith("server."):
utils.modify_config(k, v, type="server")
container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None)
time.sleep(2)
milvus = MilvusClient(collection_name)
# Check has collection or not
if not milvus.exists_collection():
logger.warning("Table %s not existed, continue exec next params ..." % collection_name)
continue
start_time = time.time()
insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(10000)]
i = 0
while time.time() < start_time + during_time:
i = i + 1
processes = []
# do query
# for i in range(query_process_num):
# milvus_instance = MilvusClient(collection_name)
# top_k = random.choice([x for x in range(1, 100)])
# nq = random.choice([x for x in range(1, 100)])
# nprobe = random.choice([x for x in range(1, 1000)])
# # logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe))
# p = Process(target=self.do_query, args=(milvus_instance, collection_name, [top_k], [nq], [nprobe], run_count, ))
# processes.append(p)
# p.start()
# time.sleep(0.1)
# for p in processes:
# p.join()
milvus_instance = MilvusClient(collection_name)
top_ks = random.sample([x for x in range(1, 100)], 3)
nqs = random.sample([x for x in range(1, 1000)], 3)
nprobe = random.choice([x for x in range(1, 500)])
res = self.do_query(milvus, collection_name, top_ks, nqs, nprobe, run_count)
if i % 10 == 0:
status, res = milvus_instance.insert(insert_vectors, ids=[x for x in range(len(insert_vectors))])
if not status.OK():
logger.error(status)
# status = milvus_instance.drop_index()
# if not status.OK():
# logger.error(status)
# index_type = random.choice(["flat", "ivf_flat", "ivf_sq8"])
milvus_instance.create_index(index_type, 16384)
result = milvus.describe_index()
logger.info(result)
# milvus_instance.create_index("ivf_sq8", 16384)
utils.remove_container(container)
else:
logger.warning("Run type: %s not supported" % run_type)

126
tests-deprecating/milvus_benchmark/milvus_benchmark/runners/docker_utils.py
View File

@ -1,126 +0,0 @@
# def pull_image(image):
# registry = image.split(":")[0]
# image_tag = image.split(":")[1]
# client = docker.APIClient(base_url='unix://var/run/docker.sock')
# logger.info("Start pulling image: %s" % image)
# return client.pull(registry, image_tag)
# def run_server(image, mem_limit=None, timeout=30, test_type="local", volume_name=None, db_slave=None):
# import colors
# client = docker.from_env()
# # if mem_limit is None:
# # mem_limit = psutil.virtual_memory().available
# # logger.info('Memory limit:', mem_limit)
# # cpu_limit = "0-%d" % (multiprocessing.cpu_count() - 1)
# # logger.info('Running on CPUs:', cpu_limit)
# for dir_item in ['logs', 'db']:
# try:
# os.mkdir(os.path.abspath(dir_item))
# except Exception as e:
# pass
# if test_type == "local":
# volumes = {
# os.path.abspath('conf'):
# {'bind': '/opt/milvus/conf', 'mode': 'ro'},
# os.path.abspath('logs'):
# {'bind': '/opt/milvus/logs', 'mode': 'rw'},
# os.path.abspath('db'):
# {'bind': '/opt/milvus/db', 'mode': 'rw'},
# }
# elif test_type == "remote":
# if volume_name is None:
# raise Exception("No volume name")
# remote_log_dir = volume_name+'/logs'
# remote_db_dir = volume_name+'/db'
# for dir_item in [remote_log_dir, remote_db_dir]:
# if not os.path.isdir(dir_item):
# os.makedirs(dir_item, exist_ok=True)
# volumes = {
# os.path.abspath('conf'):
# {'bind': '/opt/milvus/conf', 'mode': 'ro'},
# remote_log_dir:
# {'bind': '/opt/milvus/logs', 'mode': 'rw'},
# remote_db_dir:
# {'bind': '/opt/milvus/db', 'mode': 'rw'}
# }
# # add volumes
# if db_slave and isinstance(db_slave, int):
# for i in range(2, db_slave+1):
# remote_db_dir = volume_name+'/data'+str(i)
# if not os.path.isdir(remote_db_dir):
# os.makedirs(remote_db_dir, exist_ok=True)
# volumes[remote_db_dir] = {'bind': '/opt/milvus/data'+str(i), 'mode': 'rw'}
# container = client.containers.run(
# image,
# volumes=volumes,
# runtime="nvidia",
# ports={'19530/tcp': 19530, '8080/tcp': 8080},
# # environment=["OMP_NUM_THREADS=48"],
# # cpuset_cpus=cpu_limit,
# # mem_limit=mem_limit,
# # environment=[""],
# detach=True)
# def stream_logs():
# for line in container.logs(stream=True):
# logger.info(colors.color(line.decode().rstrip(), fg='blue'))
# if sys.version_info >= (3, 0):
# t = threading.Thread(target=stream_logs, daemon=True)
# else:
# t = threading.Thread(target=stream_logs)
# t.daemon = True
# t.start()
# logger.info('Container: %s started' % container)
# return container
# # exit_code = container.wait(timeout=timeout)
# # # Exit if exit code
# # if exit_code == 0:
# # return container
# # elif exit_code is not None:
# # print(colors.color(container.logs().decode(), fg='red'))
# def restart_server(container):
# client = docker.APIClient(base_url='unix://var/run/docker.sock')
# client.restart(container.name)
# logger.info('Container: %s restarted' % container.name)
# return container
# def remove_container(container):
# container.remove(force=True)
# logger.info('Container: %s removed' % container)
# def remove_all_containers(image):
# client = docker.from_env()
# try:
# for container in client.containers.list():
# if image in container.image.tags:
# container.stop(timeout=30)
# container.remove(force=True)
# except Exception as e:
# logger.error("Containers removed failed")
# def container_exists(image):
# '''
# Check if container existed with the given image name
# @params: image name
# @return: container if exists
# '''
# res = False
# client = docker.from_env()
# for container in client.containers.list():
# if image in container.image.tags:
# # True
# res = container
# return res

121
tests-deprecating/milvus_benchmark/milvus_benchmark/runners/get.py
View File

@ -1,121 +0,0 @@
import time
import copy
import logging
from milvus_benchmark import parser
from milvus_benchmark.runners import utils
from milvus_benchmark.runners.base import BaseRunner
logger = logging.getLogger("milvus_benchmark.runners.get")
def get_ids(length, size):
ids_list = []
step = size // length
for i in range(length):
ids_list.append(step * i)
return ids_list
class GetRunner(BaseRunner):
"""run get"""
name = "get_performance"
def __init__(self, env, metric):
super(GetRunner, self).__init__(env, metric)
def extract_cases(self, collection):
collection_name = collection["collection_name"] if "collection_name" in collection else None
(data_type, collection_size, dimension, metric_type) = parser.collection_parser(collection_name)
ni_per = collection["ni_per"]
vector_type = utils.get_vector_type(data_type)
other_fields = collection["other_fields"] if "other_fields" in collection else None
ids_length_list = collection["ids_length_list"]
collection_info = {
"dimension": dimension,
"metric_type": metric_type,
"dataset_name": collection_name,
"collection_size": collection_size,
"other_fields": other_fields,
"ni_per": ni_per
}
index_field_name = utils.get_default_field_name(vector_type)
index_type = collection["index_type"]
index_param = collection["index_param"]
index_info = {
"index_type": index_type,
"index_param": index_param
}
flush = True
if "flush" in collection and collection["flush"] == "no":
flush = False
self.init_metric(self.name, collection_info, index_info, search_info=None)
case_metrics = list()
for ids_length in ids_length_list:
ids = get_ids(ids_length, collection_size)
case_metric = copy.deepcopy(self.metric)
case_metric.set_case_metric_type()
case_params = list()
case_metric.run_params = {"ids_length": ids_length}
case_metrics.append(case_metric)
case_param = {
"collection_name": collection_name,
"data_type": data_type,
"dimension": dimension,
"collection_size": collection_size,
"ni_per": ni_per,
"metric_type": metric_type,
"vector_type": vector_type,
"other_fields": other_fields,
"flush_after_insert": flush,
"index_field_name": index_field_name,
"index_type": index_type,
"index_param": index_param,
"ids": ids
}
case_params.append(case_param)
return case_params, case_metrics
def prepare(self, **case_param):
collection_name = case_param["collection_name"]
self.milvus.set_collection(collection_name)
if not self.milvus.exists_collection():
logger.info("collection not exist")
logger.debug({"collection count": self.milvus.count()})
def run_case(self, case_metric, **case_param):
ids = case_param["ids"]
start_time = time.time()
self.milvus.get(ids)
get_time = round(time.time() - start_time, 2)
tmp_result = {"get_time": get_time}
return tmp_result
class InsertGetRunner(GetRunner):
"""run insert and get"""
name = "insert_get_performance"
def __init__(self, env, metric):
super(InsertGetRunner, self).__init__(env, metric)
def prepare(self, **case_param):
collection_name = case_param["collection_name"]
dimension = case_param["dimension"]
vector_type = case_param["vector_type"]
other_fields = case_param["other_fields"]
self.milvus.set_collection(collection_name)
if self.milvus.exists_collection():
logger.debug("Start drop collection")
self.milvus.drop()
time.sleep(utils.DELETE_INTERVAL_TIME)
self.milvus.create_collection(dimension, data_type=vector_type, other_fields=other_fields)
self.insert(self.milvus, collection_name, case_param["data_type"], dimension,
case_param["collection_size"], case_param["ni_per"])
start_time = time.time()
self.milvus.flush()
flush_time = round(time.time() - start_time, 2)
logger.debug({"collection count": self.milvus.count()})
logger.debug({"flush_time": flush_time})
logger.debug("Start load collection")
self.milvus.load_collection(timeout=1200)
logger.debug("Load collection end")

243
tests-deprecating/milvus_benchmark/milvus_benchmark/runners/insert.py
View File

@ -1,243 +0,0 @@
import time
import pdb
import copy
import logging
from milvus_benchmark import parser
from milvus_benchmark.runners import utils
from milvus_benchmark.runners.base import BaseRunner
logger = logging.getLogger("milvus_benchmark.runners.insert")
class InsertRunner(BaseRunner):
"""run insert"""
name = "insert_performance"
def __init__(self, env, metric):
super(InsertRunner, self).__init__(env, metric)
def extract_cases(self, collection):
collection_name = collection["collection_name"] if "collection_name" in collection else None
(data_type, collection_size, dimension, metric_type) = parser.collection_parser(collection_name)
ni_per = collection["ni_per"]
build_index = collection["build_index"] if "build_index" in collection else False
index_info = None
vector_type = utils.get_vector_type(data_type)
other_fields = collection["other_fields"] if "other_fields" in collection else None
collection_info = {
"dimension": dimension,
"metric_type": metric_type,
"dataset_name": collection_name,
"collection_size": collection_size,
"other_fields": other_fields,
"ni_per": ni_per
}
index_field_name = None
index_type = None
index_param = None
if build_index is True:
index_type = collection["index_type"]
index_param = collection["index_param"]
index_info = {
"index_type": index_type,
"index_param": index_param
}
index_field_name = utils.get_default_field_name(vector_type)
flush = True
if "flush" in collection and collection["flush"] == "no":
flush = False
self.init_metric(self.name, collection_info, index_info, None)
case_metric = copy.deepcopy(self.metric)
case_metric.set_case_metric_type()
case_metrics = list()
case_params = list()
case_metrics.append(case_metric)
case_param = {
"collection_name": collection_name,
"data_type": data_type,
"dimension": dimension,
"collection_size": collection_size,
"ni_per": ni_per,
"metric_type": metric_type,
"vector_type": vector_type,
"other_fields": other_fields,
"build_index": build_index,
"flush_after_insert": flush,
"index_field_name": index_field_name,
"index_type": index_type,
"index_param": index_param,
}
case_params.append(case_param)
return case_params, case_metrics
def prepare(self, **case_param):
collection_name = case_param["collection_name"]
dimension = case_param["dimension"]
vector_type = case_param["vector_type"]
other_fields = case_param["other_fields"]
index_field_name = case_param["index_field_name"]
build_index = case_param["build_index"]
self.milvus.set_collection(collection_name)
if self.milvus.exists_collection():
logger.debug("Start drop collection")
self.milvus.drop()
time.sleep(utils.DELETE_INTERVAL_TIME)
self.milvus.create_collection(dimension, data_type=vector_type,
other_fields=other_fields)
# TODO: update fields in collection_info
# fields = self.get_fields(self.milvus, collection_name)
# collection_info = {
# "dimension": dimension,
# "metric_type": metric_type,
# "dataset_name": collection_name,
# "fields": fields
# }
if build_index is True:
if case_param["index_type"]:
self.milvus.create_index(index_field_name, case_param["index_type"], case_param["metric_type"], index_param=case_param["index_param"])
logger.debug(self.milvus.describe_index(index_field_name))
else:
build_index = False
logger.warning("Please specify the index_type")
# TODO: error handler
def run_case(self, case_metric, **case_param):
collection_name = case_param["collection_name"]
dimension = case_param["dimension"]
index_field_name = case_param["index_field_name"]
build_index = case_param["build_index"]
tmp_result = self.insert(self.milvus, collection_name, case_param["data_type"], dimension, case_param["collection_size"], case_param["ni_per"])
flush_time = 0.0
build_time = 0.0
if case_param["flush_after_insert"] is True:
start_time = time.time()
self.milvus.flush()
flush_time = round(time.time()-start_time, 2)
logger.debug(self.milvus.count())
if build_index is True:
logger.debug("Start build index for last file")
start_time = time.time()
self.milvus.create_index(index_field_name, case_param["index_type"], case_param["metric_type"], index_param=case_param["index_param"])
build_time = round(time.time()-start_time, 2)
tmp_result.update({"flush_time": flush_time, "build_time": build_time})
return tmp_result
class BPInsertRunner(BaseRunner):
"""run insert"""
name = "bp_insert_performance"
def __init__(self, env, metric):
super(BPInsertRunner, self).__init__(env, metric)
def extract_cases(self, collection):
collection_name = collection["collection_name"] if "collection_name" in collection else None
(data_type, collection_size, dimension, metric_type) = parser.collection_parser(collection_name)
ni_pers = collection["ni_pers"]
build_index = collection["build_index"] if "build_index" in collection else False
index_info = None
vector_type = utils.get_vector_type(data_type)
other_fields = collection["other_fields"] if "other_fields" in collection else None
index_field_name = None
index_type = None
index_param = None
if build_index is True:
index_type = collection["index_type"]
index_param = collection["index_param"]
index_info = {
"index_type": index_type,
"index_param": index_param
}
index_field_name = utils.get_default_field_name(vector_type)
flush = True
if "flush" in collection and collection["flush"] == "no":
flush = False
case_metrics = list()
case_params = list()
for ni_per in ni_pers:
collection_info = {
"dimension": dimension,
"metric_type": metric_type,
"dataset_name": collection_name,
"collection_size": collection_size,
"other_fields": other_fields,
"ni_per": ni_per
}
self.init_metric(self.name, collection_info, index_info, None)
case_metric = copy.deepcopy(self.metric)
case_metric.set_case_metric_type()
case_metrics.append(case_metric)
case_param = {
"collection_name": collection_name,
"data_type": data_type,
"dimension": dimension,
"collection_size": collection_size,
"ni_per": ni_per,
"metric_type": metric_type,
"vector_type": vector_type,
"other_fields": other_fields,
"build_index": build_index,
"flush_after_insert": flush,
"index_field_name": index_field_name,
"index_type": index_type,
"index_param": index_param,
}
case_params.append(case_param)
return case_params, case_metrics
def prepare(self, **case_param):
collection_name = case_param["collection_name"]
dimension = case_param["dimension"]
vector_type = case_param["vector_type"]
other_fields = case_param["other_fields"]
index_field_name = case_param["index_field_name"]
build_index = case_param["build_index"]
self.milvus.set_collection(collection_name)
if self.milvus.exists_collection():
logger.debug("Start drop collection")
self.milvus.drop()
time.sleep(utils.DELETE_INTERVAL_TIME)
self.milvus.create_collection(dimension, data_type=vector_type,
other_fields=other_fields)
# TODO: update fields in collection_info
# fields = self.get_fields(self.milvus, collection_name)
# collection_info = {
# "dimension": dimension,
# "metric_type": metric_type,
# "dataset_name": collection_name,
# "fields": fields
# }
if build_index is True:
if case_param["index_type"]:
self.milvus.create_index(index_field_name, case_param["index_type"], case_param["metric_type"], index_param=case_param["index_param"])
logger.debug(self.milvus.describe_index(index_field_name))
else:
build_index = False
logger.warning("Please specify the index_type")
# TODO: error handler
def run_case(self, case_metric, **case_param):
collection_name = case_param["collection_name"]
dimension = case_param["dimension"]
index_field_name = case_param["index_field_name"]
build_index = case_param["build_index"]
# TODO:
tmp_result = self.insert(self.milvus, collection_name, case_param["data_type"], dimension, case_param["collection_size"], case_param["ni_per"])
flush_time = 0.0
build_time = 0.0
if case_param["flush_after_insert"] is True:
start_time = time.time()
self.milvus.flush()
flush_time = round(time.time()-start_time, 2)
logger.debug(self.milvus.count())
if build_index is True:
logger.debug("Start build index for last file")
start_time = time.time()
self.milvus.create_index(index_field_name, case_param["index_type"], case_param["metric_type"], index_param=case_param["index_param"])
build_time = round(time.time()-start_time, 2)
tmp_result.update({"flush_time": flush_time, "build_time": build_time})
return tmp_result

400
tests-deprecating/milvus_benchmark/milvus_benchmark/runners/locust.py
View File

@ -1,400 +0,0 @@
import pdb
import time
import copy
import logging
from . import locust_user
from .base import BaseRunner
from milvus_benchmark import parser
from milvus_benchmark import utils
from milvus_benchmark.runners import utils as runner_utils
logger = logging.getLogger("milvus_benchmark.runners.locust")
class LocustRunner(BaseRunner):
def __init__(self, env, metric):
super(LocustRunner, self).__init__(env, metric)
def run_case(self, case_metric, **case_param):
""" start running locust test """
collection_name = case_param["collection_name"]
task = case_param["task"]
connection_type = case_param["connection_type"]
# spawn locust requests
task["during_time"] = utils.timestr_to_int(task["during_time"])
task_types = task["types"]
run_params = {"tasks": {}}
run_params.update(task)
info_in_params = {
"index_field_name": case_param["index_field_name"],
"vector_field_name": case_param["vector_field_name"],
"dimension": case_param["dimension"],
"collection_info": self.milvus.get_info(collection_name)}
logger.info(info_in_params)
run_params.update({"op_info": info_in_params})
for task_type in task_types:
run_params["tasks"].update({
task_type["type"]: {
"weight": task_type["weight"] if "weight" in task_type else 1,
"params": task_type["params"] if "params" in task_type else None,
}
})
# collect stats
# pdb.set_trace()
logger.info(run_params)
locust_stats = locust_user.locust_executor(self.hostname, self.port, collection_name,
connection_type=connection_type, run_params=run_params)
return locust_stats
class LocustInsertRunner(LocustRunner):
"""run insert"""
name = "locust_insert_performance"
def __init__(self, env, metric):
super(LocustInsertRunner, self).__init__(env, metric)
def extract_cases(self, collection):
collection_name = collection["collection_name"] if "collection_name" in collection else None
(data_type, collection_size, dimension, metric_type) = parser.collection_parser(collection_name)
ni_per = collection["ni_per"]
build_index = collection["build_index"] if "build_index" in collection else False
vector_type = runner_utils.get_vector_type(data_type)
other_fields = collection["other_fields"] if "other_fields" in collection else None
collection_info = {
"dimension": dimension,
"metric_type": metric_type,
"dataset_name": collection_name,
"collection_size": collection_size,
"other_fields": other_fields,
"ni_per": ni_per
}
index_field_name = None
index_type = None
index_param = None
index_info = None
vector_field_name = runner_utils.get_default_field_name(vector_type)
if build_index is True:
index_type = collection["index_type"]
index_param = collection["index_param"]
index_info = {
"index_type": index_type,
"index_param": index_param
}
index_field_name = runner_utils.get_default_field_name(vector_type)
task = collection["task"]
connection_type = "single"
connection_num = task["connection_num"]
if connection_num > 1:
connection_type = "multi"
run_params = {
"task": collection["task"],
"connection_type": connection_type,
}
self.init_metric(self.name, collection_info, index_info, None, run_params)
case_metric = copy.deepcopy(self.metric)
case_metric.set_case_metric_type()
case_metrics = list()
case_params = list()
case_metrics.append(case_metric)
case_param = {
"collection_name": collection_name,
"data_type": data_type,
"dimension": dimension,
"collection_size": collection_size,
"ni_per": ni_per,
"metric_type": metric_type,
"vector_type": vector_type,
"other_fields": other_fields,
"build_index": build_index,
"index_field_name": index_field_name,
"vector_field_name": vector_field_name,
"index_type": index_type,
"index_param": index_param,
"task": collection["task"],
"connection_type": connection_type,
}
case_params.append(case_param)
return case_params, case_metrics
def prepare(self, **case_param):
collection_name = case_param["collection_name"]
dimension = case_param["dimension"]
vector_type = case_param["vector_type"]
other_fields = case_param["other_fields"]
index_field_name = case_param["index_field_name"]
build_index = case_param["build_index"]
self.milvus.set_collection(collection_name)
if self.milvus.exists_collection():
logger.debug("Start drop collection")
self.milvus.drop()
time.sleep(runner_utils.DELETE_INTERVAL_TIME)
self.milvus.create_collection(dimension, data_type=vector_type,
other_fields=other_fields)
# TODO: update fields in collection_info
# fields = self.get_fields(self.milvus, collection_name)
# collection_info = {
# "dimension": dimension,
# "metric_type": metric_type,
# "dataset_name": collection_name,
# "fields": fields
# }
if build_index is True:
if case_param["index_type"]:
self.milvus.create_index(index_field_name, case_param["index_type"], case_param["metric_type"], index_param=case_param["index_param"])
logger.debug(self.milvus.describe_index(index_field_name))
else:
build_index = False
logger.warning("Please specify the index_type")
class LocustSearchRunner(LocustRunner):
"""run search"""
name = "locust_search_performance"
def __init__(self, env, metric):
super(LocustSearchRunner, self).__init__(env, metric)
def extract_cases(self, collection):
collection_name = collection["collection_name"] if "collection_name" in collection else None
(data_type, collection_size, dimension, metric_type) = parser.collection_parser(collection_name)
ni_per = collection["ni_per"]
build_index = collection["build_index"] if "build_index" in collection else False
vector_type = runner_utils.get_vector_type(data_type)
other_fields = collection["other_fields"] if "other_fields" in collection else None
collection_info = {
"dimension": dimension,
"metric_type": metric_type,
"dataset_name": collection_name,
"collection_size": collection_size,
"other_fields": other_fields,
"ni_per": ni_per
}
index_field_name = None
index_type = None
index_param = None
index_info = None
if build_index is True:
index_type = collection["index_type"]
index_param = collection["index_param"]
index_info = {
"index_type": index_type,
"index_param": index_param
}
index_field_name = runner_utils.get_default_field_name(vector_type)
vector_field_name = runner_utils.get_default_field_name(vector_type)
task = collection["task"]
connection_type = "single"
connection_num = task["connection_num"]
if connection_num > 1:
connection_type = "multi"
run_params = {
"task": collection["task"],
"connection_type": connection_type,
}
self.init_metric(self.name, collection_info, index_info, None, run_params)
case_metric = copy.deepcopy(self.metric)
case_metric.set_case_metric_type()
case_metrics = list()
case_params = list()
case_metrics.append(case_metric)
case_param = {
"collection_name": collection_name,
"data_type": data_type,
"dimension": dimension,
"collection_size": collection_size,
"ni_per": ni_per,
"metric_type": metric_type,
"vector_type": vector_type,
"other_fields": other_fields,
"build_index": build_index,
"index_field_name": index_field_name,
"vector_field_name": vector_field_name,
"index_type": index_type,
"index_param": index_param,
"task": collection["task"],
"connection_type": connection_type,
}
case_params.append(case_param)
return case_params, case_metrics
def prepare(self, **case_param):
collection_name = case_param["collection_name"]
dimension = case_param["dimension"]
vector_type = case_param["vector_type"]
other_fields = case_param["other_fields"]
index_field_name = case_param["index_field_name"]
metric_type = case_param["metric_type"]
build_index = case_param["build_index"]
self.milvus.set_collection(collection_name)
if self.milvus.exists_collection():
logger.debug("Start drop collection")
self.milvus.drop()
time.sleep(runner_utils.DELETE_INTERVAL_TIME)
self.milvus.create_collection(dimension, data_type=vector_type,
other_fields=other_fields)
# TODO: update fields in collection_info
# fields = self.get_fields(self.milvus, collection_name)
# collection_info = {
# "dimension": dimension,
# "metric_type": metric_type,
# "dataset_name": collection_name,
# "fields": fields
# }
if build_index is True:
if case_param["index_type"]:
self.milvus.create_index(index_field_name, case_param["index_type"], case_param["metric_type"], index_param=case_param["index_param"])
logger.debug(self.milvus.describe_index(index_field_name))
else:
build_index = False
logger.warning("Please specify the index_type")
self.insert(self.milvus, collection_name, case_param["data_type"], dimension, case_param["collection_size"], case_param["ni_per"])
build_time = 0.0
start_time = time.time()
self.milvus.flush()
flush_time = round(time.time()-start_time, 2)
logger.debug(self.milvus.count())
if build_index is True:
logger.debug("Start build index for last file")
start_time = time.time()
self.milvus.create_index(index_field_name, case_param["index_type"], case_param["metric_type"], index_param=case_param["index_param"])
build_time = round(time.time()-start_time, 2)
logger.debug({"flush_time": flush_time, "build_time": build_time})
logger.info(self.milvus.count())
logger.info("Start load collection")
load_start_time = time.time()
self.milvus.load_collection()
logger.debug({"load_time": round(time.time()-load_start_time, 2)})
# search_param = None
# for op in case_param["task"]["types"]:
# if op["type"] == "query":
# search_param = op["params"]["search_param"]
# break
# logger.info("index_field_name: {}".format(index_field_name))
# TODO: enable warm query
# self.milvus.warm_query(index_field_name, search_param, metric_type, times=2)
class LocustRandomRunner(LocustRunner):
"""run random interface"""
name = "locust_random_performance"
def __init__(self, env, metric):
super(LocustRandomRunner, self).__init__(env, metric)
def extract_cases(self, collection):
collection_name = collection["collection_name"] if "collection_name" in collection else None
(data_type, collection_size, dimension, metric_type) = parser.collection_parser(collection_name)
ni_per = collection["ni_per"]
build_index = collection["build_index"] if "build_index" in collection else False
vector_type = runner_utils.get_vector_type(data_type)
other_fields = collection["other_fields"] if "other_fields" in collection else None
collection_info = {
"dimension": dimension,
"metric_type": metric_type,
"dataset_name": collection_name,
"collection_size": collection_size,
"other_fields": other_fields,
"ni_per": ni_per
}
index_field_name = None
index_type = None
index_param = None
index_info = None
vector_field_name = runner_utils.get_default_field_name(vector_type)
if build_index is True:
index_type = collection["index_type"]
index_param = collection["index_param"]
index_info = {
"index_type": index_type,
"index_param": index_param
}
index_field_name = runner_utils.get_default_field_name(vector_type)
task = collection["task"]
connection_type = "single"
connection_num = task["connection_num"]
if connection_num > 1:
connection_type = "multi"
run_params = {
"task": collection["task"],
"connection_type": connection_type,
}
self.init_metric(self.name, collection_info, index_info, None, run_params)
case_metric = copy.deepcopy(self.metric)
case_metric.set_case_metric_type()
case_metrics = list()
case_params = list()
case_metrics.append(case_metric)
case_param = {
"collection_name": collection_name,
"data_type": data_type,
"dimension": dimension,
"collection_size": collection_size,
"ni_per": ni_per,
"metric_type": metric_type,
"vector_type": vector_type,
"other_fields": other_fields,
"build_index": build_index,
"index_field_name": index_field_name,
"vector_field_name": vector_field_name,
"index_type": index_type,
"index_param": index_param,
"task": collection["task"],
"connection_type": connection_type,
}
case_params.append(case_param)
return case_params, case_metrics
def prepare(self, **case_param):
collection_name = case_param["collection_name"]
dimension = case_param["dimension"]
vector_type = case_param["vector_type"]
other_fields = case_param["other_fields"]
index_field_name = case_param["index_field_name"]
build_index = case_param["build_index"]
self.milvus.set_collection(collection_name)
if self.milvus.exists_collection():
logger.debug("Start drop collection")
self.milvus.drop()
time.sleep(runner_utils.DELETE_INTERVAL_TIME)
self.milvus.create_collection(dimension, data_type=vector_type,
other_fields=other_fields)
# TODO: update fields in collection_info
# fields = self.get_fields(self.milvus, collection_name)
# collection_info = {
# "dimension": dimension,
# "metric_type": metric_type,
# "dataset_name": collection_name,
# "fields": fields
# }
if build_index is True:
if case_param["index_type"]:
self.milvus.create_index(index_field_name, case_param["index_type"], case_param["metric_type"], index_param=case_param["index_param"])
logger.debug(self.milvus.describe_index(index_field_name))
else:
build_index = False
logger.warning("Please specify the index_type")
self.insert(self.milvus, collection_name, case_param["data_type"], dimension, case_param["collection_size"], case_param["ni_per"])
build_time = 0.0
start_time = time.time()
self.milvus.flush()
flush_time = round(time.time()-start_time, 2)
logger.debug(self.milvus.count())
if build_index is True:
logger.debug("Start build index for last file")
start_time = time.time()
self.milvus.create_index(index_field_name, case_param["index_type"], case_param["metric_type"], index_param=case_param["index_param"])
build_time = round(time.time()-start_time, 2)
logger.debug({"flush_time": flush_time, "build_time": build_time})
logger.info(self.milvus.count())
logger.info("Start load collection")
load_start_time = time.time()
self.milvus.load_collection()
logger.debug({"load_time": round(time.time()-load_start_time, 2)})

30
tests-deprecating/milvus_benchmark/milvus_benchmark/runners/locust_file.py
View File

@ -1,30 +0,0 @@
import random
from locust import HttpUser, task, between
collection_name = "random_1m_2048_512_ip_sq8"
headers = {'Content-Type': "application/json"}
url = '/collections/%s/vectors' % collection_name
top_k = 2
nq = 1
dim = 512
vectors = [[random.random() for _ in range(dim)] for _ in range(nq)]
data = {
"search":{
"topk": top_k,
"vectors": vectors,
"params": {
"nprobe": 1
}
}
}
class MyUser(HttpUser):
wait_time = between(0, 0.1)
host = "http://192.168.1.112:19122"
@task
def search(self):
response = self.client.put(url=url, json=data, headers=headers, timeout=2)
print(response)

38
tests-deprecating/milvus_benchmark/milvus_benchmark/runners/locust_task.py
View File

@ -1,38 +0,0 @@
import time
import pdb
import random
import logging
from locust import User, events
from milvus_benchmark.client import MilvusClient
logger = logging.getLogger("milvus_benchmark.runners.locust_task")
class MilvusTask(object):
def __init__(self, *args, **kwargs):
self.request_type = "grpc"
connection_type = kwargs.get("connection_type")
if connection_type == "single":
self.m = kwargs.get("m")
elif connection_type == "multi":
host = kwargs.get("host")
port = kwargs.get("port")
collection_name = kwargs.get("collection_name")
self.m = MilvusClient(host=host, port=port, collection_name=collection_name)
def __getattr__(self, name):
func = getattr(self.m, name)
def wrapper(*args, **kwargs):
start_time = time.time()
try:
result = func(*args, **kwargs)
total_time = int((time.time() - start_time) * 1000)
events.request_success.fire(request_type=self.request_type, name=name, response_time=total_time,
response_length=0)
except Exception as e:
total_time = int((time.time() - start_time) * 1000)
events.request_failure.fire(request_type=self.request_type, name=name, response_time=total_time,
exception=e, response_length=0)
return wrapper

79
tests-deprecating/milvus_benchmark/milvus_benchmark/runners/locust_tasks.py
View File

@ -1,79 +0,0 @@
import pdb
import random
import time
import logging
import math
from locust import TaskSet, task
from . import utils
logger = logging.getLogger("milvus_benchmark.runners.locust_tasks")
class Tasks(TaskSet):
@task
def query(self):
op = "query"
# X = utils.generate_vectors(self.params[op]["nq"], self.op_info["dimension"])
vector_query = {"vector": {self.op_info["vector_field_name"]: {
"topk": self.params[op]["top_k"],
"query": self.values["X"][:self.params[op]["nq"]],
"metric_type": self.params[op]["metric_type"] if "metric_type" in self.params[op] else utils.DEFAULT_METRIC_TYPE,
"params": self.params[op]["search_param"]}
}}
filter_query = []
if "filters" in self.params[op]:
for filter in self.params[op]["filters"]:
if isinstance(filter, dict) and "range" in filter:
filter_query.append(eval(filter["range"]))
if isinstance(filter, dict) and "term" in filter:
filter_query.append(eval(filter["term"]))
# logger.debug(filter_query)
self.client.query(vector_query, filter_query=filter_query, log=False, timeout=30)
@task
def flush(self):
self.client.flush(log=False, timeout=30)
@task
def load(self):
self.client.load_collection(timeout=30)
@task
def release(self):
self.client.release_collection()
self.client.load_collection(timeout=30)
# @task
# def release_index(self):
# self.client.release_index()
# @task
# def create_index(self):
# self.client.release_index()
@task
def insert(self):
op = "insert"
# ids = [random.randint(1000000, 10000000) for _ in range(self.params[op]["ni_per"])]
# X = [[random.random() for _ in range(self.op_info["dimension"])] for _ in range(self.params[op]["ni_per"])]
entities = utils.generate_entities(self.op_info["collection_info"], self.values["X"][:self.params[op]["ni_per"]], self.values["ids"][:self.params[op]["ni_per"]])
self.client.insert(entities, log=False)
@task
def insert_flush(self):
op = "insert_flush"
# ids = [random.randint(1000000, 10000000) for _ in range(self.params[op]["ni_per"])]
# X = [[random.random() for _ in range(self.op_info["dimension"])] for _ in range(self.params[op]["ni_per"])]
entities = utils.generate_entities(self.op_info["collection_info"], self.values["X"][:self.params[op]["ni_per"]], self.values["ids"][:self.params[op]["ni_per"]])
self.client.insert(entities, log=False)
self.client.flush(log=False)
@task
def insert_rand(self):
self.client.insert_rand(log=False)
@task
def get(self):
op = "get"
# ids = [random.randint(1, 10000000) for _ in range(self.params[op]["ids_length"])]
self.client.get(self.values["get_ids"][:self.params[op]["ids_length"]])

109
tests-deprecating/milvus_benchmark/milvus_benchmark/runners/locust_user.py
View File

@ -1,109 +0,0 @@
import logging
import random
import pdb
import gevent
# import gevent.monkey
# gevent.monkey.patch_all()
from locust import User, between, events, stats
from locust.env import Environment
import locust.stats
import math
from locust import LoadTestShape
from locust.stats import stats_printer, print_stats
from locust.log import setup_logging, greenlet_exception_logger
from milvus_benchmark.client import MilvusClient
from .locust_task import MilvusTask
from .locust_tasks import Tasks
from . import utils
locust.stats.CONSOLE_STATS_INTERVAL_SEC = 20
logger = logging.getLogger("milvus_benchmark.runners.locust_user")
nq = 10000
nb = 100000
class StepLoadShape(LoadTestShape):
"""
A step load shape
Keyword arguments:
step_time -- Time between steps
step_load -- User increase amount at each step
spawn_rate -- Users to stop/start per second at every step
time_limit -- Time limit in seconds
"""
def init(self, step_time, step_load, spawn_rate, time_limit):
self.step_time = step_time
self.step_load = step_load
self.spawn_rate = spawn_rate
self.time_limit = time_limit
def tick(self):
run_time = self.get_run_time()
if run_time > self.time_limit:
return None
current_step = math.floor(run_time / self.step_time) + 1
return (current_step * self.step_load, self.spawn_rate)
class MyUser(User):
# task_set = None
# wait_time = between(0.001, 0.002)
pass
def locust_executor(host, port, collection_name, connection_type="single", run_params=None):
m = MilvusClient(host=host, port=port, collection_name=collection_name)
MyUser.tasks = {}
MyUser.op_info = run_params["op_info"]
MyUser.params = {}
tasks = run_params["tasks"]
for op, value in tasks.items():
task = {eval("Tasks." + op): value["weight"]}
MyUser.tasks.update(task)
MyUser.params[op] = value["params"] if "params" in value else None
logger.info(MyUser.tasks)
MyUser.values = {
"ids": [random.randint(1000000, 10000000) for _ in range(nb)],
"get_ids": [random.randint(1, 10000000) for _ in range(nb)],
"X": utils.generate_vectors(nq, MyUser.op_info["dimension"])
}
# MyUser.tasks = {Tasks.query: 1, Tasks.flush: 1}
MyUser.client = MilvusTask(host=host, port=port, collection_name=collection_name, connection_type=connection_type,
m=m)
if "load_shape" in run_params and run_params["load_shape"]:
test = StepLoadShape()
test.init(run_params["step_time"], run_params["step_load"], run_params["spawn_rate"], run_params["during_time"])
env = Environment(events=events, user_classes=[MyUser], shape_class=test)
runner = env.create_local_runner()
env.runner.start_shape()
else:
env = Environment(events=events, user_classes=[MyUser])
runner = env.create_local_runner()
# setup logging
# setup_logging("WARNING", "/dev/null")
# greenlet_exception_logger(logger=logger)
gevent.spawn(stats_printer(env.stats))
# env.create_web_ui("127.0.0.1", 8089)
# gevent.spawn(stats_printer(env.stats), env, "test", full_history=True)
# events.init.fire(environment=env, runner=runner)
clients_num = run_params["clients_num"] if "clients_num" in run_params else 0
step_load = run_params["step_load"] if "step_load" in run_params else 0
step_time = run_params["step_time"] if "step_time" in run_params else 0
spawn_rate = run_params["spawn_rate"]
during_time = run_params["during_time"]
runner.start(clients_num, spawn_rate=spawn_rate)
gevent.spawn_later(during_time, lambda: runner.quit())
runner.greenlet.join()
print_stats(env.stats)
result = {
"rps": round(env.stats.total.current_rps, 1),
"fail_ratio": env.stats.total.fail_ratio,
"max_response_time": round(env.stats.total.max_response_time, 1),
"avg_response_time": round(env.stats.total.avg_response_time, 1)
}
runner.stop()
return result

290
tests-deprecating/milvus_benchmark/milvus_benchmark/runners/search.py
View File

@ -1,290 +0,0 @@
import time
import pdb
import copy
import json
import logging
from milvus_benchmark import parser
from milvus_benchmark.runners import utils
from milvus_benchmark.runners.base import BaseRunner
logger = logging.getLogger("milvus_benchmark.runners.search")
class SearchRunner(BaseRunner):
"""run search"""
name = "search_performance"
def __init__(self, env, metric):
super(SearchRunner, self).__init__(env, metric)
def extract_cases(self, collection):
collection_name = collection["collection_name"] if "collection_name" in collection else None
(data_type, collection_size, dimension, metric_type) = parser.collection_parser(collection_name)
run_count = collection["run_count"]
top_ks = collection["top_ks"]
nqs = collection["nqs"]
filters = collection["filters"] if "filters" in collection else []
search_params = collection["search_params"]
# TODO: get fields by describe_index
# fields = self.get_fields(self.milvus, collection_name)
fields = None
collection_info = {
"dimension": dimension,
"metric_type": metric_type,
"dataset_name": collection_name,
"collection_size": collection_size,
"fields": fields
}
# TODO: need to get index_info
index_info = None
vector_type = utils.get_vector_type(data_type)
index_field_name = utils.get_default_field_name(vector_type)
base_query_vectors = utils.get_vectors_from_binary(utils.MAX_NQ, dimension, data_type)
cases = list()
case_metrics = list()
self.init_metric(self.name, collection_info, index_info, None)
for search_param in search_params:
logger.info("Search param: %s" % json.dumps(search_param))
for filter in filters:
filter_query = []
filter_param = []
if filter and isinstance(filter, dict):
if "range" in filter:
filter_query.append(eval(filter["range"]))
filter_param.append(filter["range"])
elif "term" in filter:
filter_query.append(eval(filter["term"]))
filter_param.append(filter["term"])
else:
raise Exception("%s not supported" % filter)
logger.info("filter param: %s" % json.dumps(filter_param))
for nq in nqs:
query_vectors = base_query_vectors[0:nq]
for top_k in top_ks:
search_info = {
"topk": top_k,
"query": query_vectors,
"metric_type": utils.metric_type_trans(metric_type),
"params": search_param}
# TODO: only update search_info
case_metric = copy.deepcopy(self.metric)
case_metric.set_case_metric_type()
case_metric.search = {
"nq": nq,
"topk": top_k,
"search_param": search_param,
"filter": filter_param
}
vector_query = {"vector": {index_field_name: search_info}}
case = {
"collection_name": collection_name,
"index_field_name": index_field_name,
"run_count": run_count,
"filter_query": filter_query,
"vector_query": vector_query,
}
cases.append(case)
case_metrics.append(case_metric)
return cases, case_metrics
def prepare(self, **case_param):
collection_name = case_param["collection_name"]
self.milvus.set_collection(collection_name)
if not self.milvus.exists_collection():
logger.error("collection name: {} not existed".format(collection_name))
return False
logger.debug(self.milvus.count())
logger.info("Start load collection")
self.milvus.load_collection(timeout=1200)
# TODO: enable warm query
# self.milvus.warm_query(index_field_name, search_params[0], times=2)
def run_case(self, case_metric, **case_param):
# index_field_name = case_param["index_field_name"]
run_count = case_param["run_count"]
avg_query_time = 0.0
min_query_time = 0.0
total_query_time = 0.0
for i in range(run_count):
logger.debug("Start run query, run %d of %s" % (i+1, run_count))
start_time = time.time()
_query_res = self.milvus.query(case_param["vector_query"], filter_query=case_param["filter_query"])
interval_time = time.time() - start_time
total_query_time += interval_time
if (i == 0) or (min_query_time > interval_time):
min_query_time = round(interval_time, 2)
avg_query_time = round(total_query_time/run_count, 2)
tmp_result = {"search_time": min_query_time, "avc_search_time": avg_query_time}
return tmp_result
class InsertSearchRunner(BaseRunner):
"""run insert and search"""
name = "insert_search_performance"
def __init__(self, env, metric):
super(InsertSearchRunner, self).__init__(env, metric)
self.build_time = None
self.insert_result = None
def extract_cases(self, collection):
collection_name = collection["collection_name"] if "collection_name" in collection else None
(data_type, collection_size, dimension, metric_type) = parser.collection_parser(collection_name)
build_index = collection["build_index"] if "build_index" in collection else False
index_type = collection["index_type"] if "index_type" in collection else None
index_param = collection["index_param"] if "index_param" in collection else None
run_count = collection["run_count"]
top_ks = collection["top_ks"]
nqs = collection["nqs"]
other_fields = collection["other_fields"] if "other_fields" in collection else None
filters = collection["filters"] if "filters" in collection else []
filter_query = []
search_params = collection["search_params"]
ni_per = collection["ni_per"]
# TODO: get fields by describe_index
# fields = self.get_fields(self.milvus, collection_name)
fields = None
collection_info = {
"dimension": dimension,
"metric_type": metric_type,
"dataset_name": collection_name,
"fields": fields
}
index_info = {
"index_type": index_type,
"index_param": index_param
}
vector_type = utils.get_vector_type(data_type)
index_field_name = utils.get_default_field_name(vector_type)
base_query_vectors = utils.get_vectors_from_binary(utils.MAX_NQ, dimension, data_type)
cases = list()
case_metrics = list()
self.init_metric(self.name, collection_info, index_info, None)
for search_param in search_params:
if not filters:
filters.append(None)
for filter in filters:
# filter_param = []
filter_query = []
if isinstance(filter, dict) and "range" in filter:
filter_query.append(eval(filter["range"]))
# filter_param.append(filter["range"])
if isinstance(filter, dict) and "term" in filter:
filter_query.append(eval(filter["term"]))
# filter_param.append(filter["term"])
# logger.info("filter param: %s" % json.dumps(filter_param))
for nq in nqs:
query_vectors = base_query_vectors[0:nq]
for top_k in top_ks:
search_info = {
"topk": top_k,
"query": query_vectors,
"metric_type": utils.metric_type_trans(metric_type),
"params": search_param}
# TODO: only update search_info
case_metric = copy.deepcopy(self.metric)
case_metric.set_case_metric_type()
case_metric.search = {
"nq": nq,
"topk": top_k,
"search_param": search_param,
"filter": filter_query
}
vector_query = {"vector": {index_field_name: search_info}}
case = {
"collection_name": collection_name,
"index_field_name": index_field_name,
"other_fields": other_fields,
"dimension": dimension,
"data_type": data_type,
"vector_type": vector_type,
"collection_size": collection_size,
"ni_per": ni_per,
"build_index": build_index,
"index_type": index_type,
"index_param": index_param,
"metric_type": metric_type,
"run_count": run_count,
"filter_query": filter_query,
"vector_query": vector_query,
}
cases.append(case)
case_metrics.append(case_metric)
return cases, case_metrics
def prepare(self, **case_param):
collection_name = case_param["collection_name"]
dimension = case_param["dimension"]
vector_type = case_param["vector_type"]
other_fields = case_param["other_fields"]
index_field_name = case_param["index_field_name"]
build_index = case_param["build_index"]
self.milvus.set_collection(collection_name)
if self.milvus.exists_collection():
logger.debug("Start drop collection")
self.milvus.drop()
time.sleep(utils.DELETE_INTERVAL_TIME)
self.milvus.create_collection(dimension, data_type=vector_type,
other_fields=other_fields)
# TODO: update fields in collection_info
# fields = self.get_fields(self.milvus, collection_name)
# collection_info = {
# "dimension": dimension,
# "metric_type": metric_type,
# "dataset_name": collection_name,
# "fields": fields
# }
if build_index is True:
if case_param["index_type"]:
self.milvus.create_index(index_field_name, case_param["index_type"], case_param["metric_type"], index_param=case_param["index_param"])
logger.debug(self.milvus.describe_index(index_field_name))
else:
build_index = False
logger.warning("Please specify the index_type")
insert_result = self.insert(self.milvus, collection_name, case_param["data_type"], dimension, case_param["collection_size"], case_param["ni_per"])
self.insert_result = insert_result
build_time = 0.0
start_time = time.time()
self.milvus.flush()
flush_time = round(time.time()-start_time, 2)
logger.debug(self.milvus.count())
if build_index is True:
logger.debug("Start build index for last file")
start_time = time.time()
self.milvus.create_index(index_field_name, case_param["index_type"], case_param["metric_type"], index_param=case_param["index_param"])
build_time = round(time.time()-start_time, 2)
logger.debug({"flush_time": flush_time, "build_time": build_time})
self.build_time = build_time
logger.info(self.milvus.count())
logger.info("Start load collection")
load_start_time = time.time()
self.milvus.load_collection(timeout=1200)
logger.debug({"load_time": round(time.time()-load_start_time, 2)})
def run_case(self, case_metric, **case_param):
run_count = case_param["run_count"]
avg_query_time = 0.0
min_query_time = 0.0
total_query_time = 0.0
for i in range(run_count):
logger.debug("Start run query, run %d of %s" % (i+1, run_count))
logger.info(case_metric.search)
start_time = time.time()
_query_res = self.milvus.query(case_param["vector_query"], filter_query=case_param["filter_query"])
interval_time = time.time() - start_time
total_query_time += interval_time
if (i == 0) or (min_query_time > interval_time):
min_query_time = round(interval_time, 2)
avg_query_time = round(total_query_time/run_count, 2)
logger.info("Min query time: %.2f, avg query time: %.2f" % (min_query_time, avg_query_time))
tmp_result = {"insert": self.insert_result, "build_time": self.build_time, "search_time": min_query_time, "avc_search_time": avg_query_time}
#
# logger.info("Start load collection")
# self.milvus.load_collection(timeout=1200)
# logger.info("Release load collection")
# self.milvus.release_collection()
return tmp_result

40
tests-deprecating/milvus_benchmark/milvus_benchmark/runners/test.py
View File

@ -1,40 +0,0 @@
import math
from locust import User, TaskSet, task, constant
from locust import LoadTestShape
class StepLoadShape(LoadTestShape):
"""
A step load shape
Keyword arguments:
step_time -- Time between steps
step_load -- User increase amount at each step
spawn_rate -- Users to stop/start per second at every step
time_limit -- Time limit in seconds
"""
step_time = 30
step_load = 10
spawn_rate = 10
time_limit = 600
def tick(self):
run_time = self.get_run_time()
if run_time > self.time_limit:
return None
current_step = math.floor(run_time / self.step_time) + 1
return (current_step * self.step_load, self.spawn_rate)
class UserTasks(TaskSet):
@task
def get_root(self):
print("in usertasks")
class WebsiteUser(User):
wait_time = constant(0.5)
tasks = [UserTasks]
shape = StepLoadShape

265
tests-deprecating/milvus_benchmark/milvus_benchmark/runners/utils.py
View File

@ -1,265 +0,0 @@
import os
import pdb
import logging
import numpy as np
import sklearn.preprocessing
import h5py
import random
from itertools import product
from pymilvus import DataType
from milvus_benchmark import config
logger = logging.getLogger("milvus_benchmark.runners.utils")
DELETE_INTERVAL_TIME = 2
VECTORS_PER_FILE = 1000000
SIFT_VECTORS_PER_FILE = 100000
BINARY_VECTORS_PER_FILE = 2000000
MAX_NQ = 10001
FILE_PREFIX = "binary_"
WARM_TOP_K = 1
WARM_NQ = 1
DEFAULT_DIM = 512
DEFAULT_METRIC_TYPE = "L2"
RANDOM_SRC_DATA_DIR = config.RAW_DATA_DIR + 'random/'
SIFT_SRC_DATA_DIR = config.RAW_DATA_DIR + 'sift1b/' # has organized the corresponding query.npy file
DEEP_SRC_DATA_DIR = config.RAW_DATA_DIR + 'deep1b/'
JACCARD_SRC_DATA_DIR = config.RAW_DATA_DIR + 'jaccard/' # has organized the corresponding query.npy file
HAMMING_SRC_DATA_DIR = config.RAW_DATA_DIR + 'hamming/' # The current data set is not organized on the NAS
STRUCTURE_SRC_DATA_DIR = config.RAW_DATA_DIR + 'structure/' # The current data set is not organized on the NAS
BINARY_SRC_DATA_DIR = config.RAW_DATA_DIR + 'binary/' # has organized the corresponding query.npy file
SIFT_SRC_GROUNDTRUTH_DATA_DIR = SIFT_SRC_DATA_DIR + 'gnd'
DEFAULT_F_FIELD_NAME = 'float_vector'
DEFAULT_B_FIELD_NAME = 'binary_vector'
DEFAULT_INT_FIELD_NAME = 'int64'
DEFAULT_FLOAT_FIELD_NAME = 'float'
DEFAULT_DOUBLE_FIELD_NAME = "double"
GROUNDTRUTH_MAP = {
"1000000": "idx_1M.ivecs",
"2000000": "idx_2M.ivecs",
"5000000": "idx_5M.ivecs",
"10000000": "idx_10M.ivecs",
"20000000": "idx_20M.ivecs",
"50000000": "idx_50M.ivecs",
"100000000": "idx_100M.ivecs",
"200000000": "idx_200M.ivecs",
"500000000": "idx_500M.ivecs",
"1000000000": "idx_1000M.ivecs",
}
METRIC_MAP = {
"l2": "L2",
"ip": "IP",
"jaccard": "JACCARD",
"hamming": "HAMMING",
"sub": "SUBSTRUCTURE",
"super": "SUPERSTRUCTURE"
}
def get_len_vectors_per_file(data_type, dimension):
if data_type == "random":
if dimension == 512:
vectors_per_file = VECTORS_PER_FILE
elif dimension == 4096:
vectors_per_file = 100000
elif dimension == 16384:
vectors_per_file = 10000
elif data_type == "sift":
vectors_per_file = SIFT_VECTORS_PER_FILE
elif data_type in ["binary"]:
vectors_per_file = BINARY_VECTORS_PER_FILE
elif data_type == "local":
vectors_per_file = SIFT_VECTORS_PER_FILE
else:
raise Exception("data_type: %s not supported" % data_type)
return vectors_per_file
def get_vectors_from_binary(nq, dimension, data_type):
# use the first file, nq should be less than VECTORS_PER_FILE
if nq > MAX_NQ:
raise Exception("Over size nq")
if data_type == "local":
return generate_vectors(nq, dimension)
elif data_type == "random":
file_name = RANDOM_SRC_DATA_DIR + 'query_%d.npy' % dimension
elif data_type == "sift":
file_name = SIFT_SRC_DATA_DIR + 'query.npy'
elif data_type == "deep":
file_name = DEEP_SRC_DATA_DIR + 'query.npy'
elif data_type == "binary":
file_name = BINARY_SRC_DATA_DIR + 'query.npy'
data = np.load(file_name)
vectors = data[0:nq].tolist()
return vectors
def generate_vectors(nb, dim):
return [[random.random() for _ in range(dim)] for _ in range(nb)]
def generate_values(data_type, vectors, ids):
values = None
if data_type in [DataType.INT32, DataType.INT64]:
values = ids
elif data_type in [DataType.FLOAT, DataType.DOUBLE]:
values = [(i + 0.0) for i in ids]
elif data_type in [DataType.FLOAT_VECTOR, DataType.BINARY_VECTOR]:
values = vectors
return values
def generate_entities(info, vectors, ids=None):
entities = []
for field in info["fields"]:
# if field["name"] == "_id":
# continue
field_type = field["type"]
entities.append(
{"name": field["name"], "type": field_type, "values": generate_values(field_type, vectors, ids)})
return entities
def metric_type_trans(metric_type):
if metric_type in METRIC_MAP.keys():
return METRIC_MAP[metric_type]
else:
raise Exception("metric_type: %s not in METRIC_MAP" % metric_type)
def get_dataset(hdf5_file_path):
if not os.path.exists(hdf5_file_path):
raise Exception("%s not existed" % hdf5_file_path)
dataset = h5py.File(hdf5_file_path)
return dataset
def get_default_field_name(data_type=DataType.FLOAT_VECTOR):
if data_type == DataType.FLOAT_VECTOR:
field_name = DEFAULT_F_FIELD_NAME
elif data_type == DataType.BINARY_VECTOR:
field_name = DEFAULT_B_FIELD_NAME
elif data_type == DataType.INT64:
field_name = DEFAULT_INT_FIELD_NAME
elif data_type == DataType.FLOAT:
field_name = DEFAULT_FLOAT_FIELD_NAME
else:
logger.error(data_type)
raise Exception("Not supported data type")
return field_name
def get_vector_type(data_type):
vector_type = ''
if data_type in ["random", "sift", "deep", "glove", "local"]:
vector_type = DataType.FLOAT_VECTOR
elif data_type in ["binary"]:
vector_type = DataType.BINARY_VECTOR
else:
raise Exception("Data type: %s not defined" % data_type)
return vector_type
def get_vector_type_from_metric(metric_type):
vector_type = ''
if metric_type in ["hamming", "jaccard"]:
vector_type = DataType.BINARY_VECTOR
else:
vector_type = DataType.FLOAT_VECTOR
return vector_type
def normalize(metric_type, X):
if metric_type == "ip":
logger.info("Set normalize for metric_type: %s" % metric_type)
X = sklearn.preprocessing.normalize(X, axis=1, norm='l2')
X = X.astype(np.float32)
elif metric_type == "l2":
X = X.astype(np.float32)
elif metric_type in ["jaccard", "hamming", "sub", "super"]:
tmp = []
for item in X:
new_vector = bytes(np.packbits(item, axis=-1).tolist())
tmp.append(new_vector)
X = tmp
return X
def generate_combinations(args):
if isinstance(args, list):
args = [el if isinstance(el, list) else [el] for el in args]
return [list(x) for x in product(*args)]
elif isinstance(args, dict):
flat = []
for k, v in args.items():
if isinstance(v, list):
flat.append([(k, el) for el in v])
else:
flat.append([(k, v)])
return [dict(x) for x in product(*flat)]
else:
raise TypeError("No args handling exists for %s" % type(args).__name__)
def gen_file_name(idx, dimension, data_type):
s = "%05d" % idx
fname = FILE_PREFIX + str(dimension) + "d_" + s + ".npy"
if data_type == "random":
fname = RANDOM_SRC_DATA_DIR + fname
elif data_type == "sift":
fname = SIFT_SRC_DATA_DIR + fname
elif data_type == "deep":
fname = DEEP_SRC_DATA_DIR + fname
elif data_type == "jaccard":
fname = JACCARD_SRC_DATA_DIR + fname
elif data_type == "hamming":
fname = HAMMING_SRC_DATA_DIR + fname
elif data_type == "sub" or data_type == "super":
fname = STRUCTURE_SRC_DATA_DIR + fname
return fname
def get_recall_value(true_ids, result_ids):
"""
Use the intersection length
"""
sum_radio = 0.0
for index, item in enumerate(result_ids):
# tmp = set(item).intersection(set(flat_id_list[index]))
tmp = set(true_ids[index]).intersection(set(item))
sum_radio = sum_radio + len(tmp) / len(item)
# logger.debug(sum_radio)
return round(sum_radio / len(result_ids), 3)
def get_ground_truth_ids(collection_size):
fname = GROUNDTRUTH_MAP[str(collection_size)]
fname = SIFT_SRC_GROUNDTRUTH_DATA_DIR + "/" + fname
a = np.fromfile(fname, dtype='int32')
d = a[0]
true_ids = a.reshape(-1, d + 1)[:, 1:].copy()
return true_ids
def normalize(metric_type, X):
if metric_type == "ip":
logger.info("Set normalize for metric_type: %s" % metric_type)
X = sklearn.preprocessing.normalize(X, axis=1, norm='l2')
X = X.astype(np.float32)
elif metric_type == "l2":
X = X.astype(np.float32)
elif metric_type in ["jaccard", "hamming", "sub", "super"]:
tmp = []
for item in X:
new_vector = bytes(np.packbits(item, axis=-1).tolist())
tmp.append(new_vector)
X = tmp
return X

27
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler.py
View File

@ -1,27 +0,0 @@
# import logging
# from apscheduler.schedulers.background import BackgroundScheduler
# from apscheduler.schedulers.blocking import BlockingScheduler
# from apscheduler.jobstores.mongodb import MongoDBJobStore
# from apscheduler.executors.pool import ProcessPoolExecutor, ThreadPoolExecutor
# from apscheduler.executors.debug import DebugExecutor
# import config
# from pymongo import MongoClient
# logger = logging.basicConfig()
# mongo_client = MongoClient(config.MONGO_SERVER)
# jobstores = {
# 'default': MongoDBJobStore(database=config.SCHEDULER_DB, collection=config.JOB_COLLECTION, client=mongo_client)
# }
# executors = {
# 'default': ThreadPoolExecutor(max_workers=100)
# }
# job_defaults = {
# 'coalesce': True,
# 'max_instances': 32
# }
# # TODO:
# back_scheduler = BackgroundScheduler(executors=executors, job_defaults=job_defaults, logger=logger)

65
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/010_data.json
View File

@ -1,65 +0,0 @@
[
{
"server": "athena",
"suite_params": [
{
"suite": "080_gpu_accuracy.yaml",
"image_type": "gpu"
},
{
"suite": "080_search_stability.yaml",
"image_type": "gpu"
},
{
"suite": "gpu_accuracy_ann.yaml",
"image_type": "gpu"
}
]
},
{
"server": "poseidon",
"suite_params": [
{
"suite": "080_gpu_search.yaml",
"image_type": "gpu"
},
{
"suite": "080_cpu_search.yaml",
"image_type": "cpu"
},
{
"suite": "080_gpu_build.yaml",
"image_type": "gpu"
},
{
"suite": "080_cpu_accuracy.yaml",
"image_type": "cpu"
},
{
"suite": "locust_search.yaml",
"image_type": "cpu"
}
]
},
{
"server": "apollo",
"suite_params": [
{
"suite": "cpu_accuracy_ann.yaml",
"image_type": "cpu"
},
{
"suite": "080_cpu_build.yaml",
"image_type": "cpu"
},
{
"suite": "080_insert_performance.yaml",
"image_type": "cpu"
},
{
"suite": "add_flush_performance.yaml",
"image_type": "cpu"
}
]
}
]

62
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/011_data.json
View File

@ -1,62 +0,0 @@
[
{
"server": "idc-sh002",
"suite_params": [
{
"suite": "011_cpu_accuracy_ann.yaml",
"image_type": "cpu"
},
{
"suite": "011_gpu_accuracy_ann.yaml",
"image_type": "gpu"
}
]
},
{
"server": "idc-sh003",
"suite_params": [
{
"suite": "locust_mix.yaml",
"image_type": "gpu"
}
]
},
{
"server": "idc-sh004",
"suite_params": [
{
"suite": "011_insert_performance.yaml",
"image_type": "cpu"
},
{
"suite": "011_gpu_accuracy.yaml",
"image_type": "gpu"
},
{
"suite": "011_gpu_build.yaml",
"image_type": "gpu"
}
]
},
{
"server": "idc-sh005",
"suite_params": [
{
"suite": "011_gpu_search.yaml",
"image_type": "gpu"
},
{
"suite": "011_cpu_search.yaml",
"image_type": "cpu"
},
{
"suite": "011_cpu_accuracy.yaml",
"image_type": "cpu"
},
{
"suite": "011_locust_search.yaml",
"image_type": "cpu"
}
]
}
]

11
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/011_data_acc_debug.json
View File

@ -1,11 +0,0 @@
[
{
"server": "apollo",
"suite_params": [
{
"suite": "011_cpu_accuracy_ann.yaml",
"image_type": "cpu"
}
]
}
]

11
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/011_data_gpu_build.json
View File

@ -1,11 +0,0 @@
[
{
"server": "eros",
"suite_params": [
{
"suite": "011_gpu_build_sift10m.yaml",
"image_type": "gpu"
}
]
}
]

11
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/011_data_insert.json
View File

@ -1,11 +0,0 @@
[
{
"server": "eros",
"suite_params": [
{
"suite": "011_insert_data.yaml",
"image_type": "cpu"
}
]
}
]

15
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/011_delete.json
View File

@ -1,15 +0,0 @@
[
{
"server": "apollo",
"suite_params": [
{
"suite": "011_insert_performance.yaml",
"image_type": "cpu"
},
{
"suite": "011_delete_performance.yaml",
"image_type": "cpu"
}
]
}
]

41
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/2_cluster_data.json
View File

@ -1,41 +0,0 @@
[
{
"server": "idc-sh002",
"deploy_mode": "cluster",
"suite_params": [
{
"suite": "2_insert_search_sift10m_512.yaml",
"image_type": "cpu"
}
]
},
{
"deploy_mode": "cluster",
"suite_params": [
{
"suite": "2_cpu_ann_accuracy.yaml",
"image_type": "cpu"
}
]
},
{
"server": "idc-sh003",
"deploy_mode": "cluster",
"suite_params": [
{
"suite": "2_insert_search_sift10m_2048.yaml",
"image_type": "cpu"
}
]
},
{
"server": "idc-sh005",
"deploy_mode": "cluster",
"suite_params": [
{
"suite": "2_insert_search_sift10m_4096.yaml",
"image_type": "cpu"
}
]
}
]

34
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/2_data.json
View File

@ -1,34 +0,0 @@
[
{
"deploy_mode": "cluster",
"server": "idc-sh002",
"suite_params": [
{
"suite": "2_insert_search_sift10m_2048.yaml",
"image_type": "cpu"
}
]
},
{
"server": "idc-sh005",
"suite_params": [
{
"suite": "2_insert_search_sift50m_2048.yaml",
"image_type": "cpu"
}
]
},
{
"server": "idc-sh004",
"suite_params": [
{
"suite": "2_locust_search.yaml",
"image_type": "cpu"
},
{
"suite": "2_locust_random.yaml",
"image_type": "cpu"
}
]
}
]

10
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/acc.json
View File

@ -1,10 +0,0 @@
[
{
"suite_params": [
{
"suite": "2_accuracy_ann_debug.yaml",
"image_type": "cpu"
}
]
}
]

11
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/build.json
View File

@ -1,11 +0,0 @@
[
{
"server": "idc-sh004",
"suite_params": [
{
"suite": "2_insert_build.yaml",
"image_type": "cpu"
}
]
}
]

11
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/clean.json
View File

@ -1,11 +0,0 @@
[
{
"server": "poseidon",
"suite_params": [
{
"suite": "clean.yaml",
"image_type": "cpu"
}
]
}
]

11
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/cluster.json
View File

@ -1,11 +0,0 @@
[
{
"server": "idc-sh004",
"suite_params": [
{
"suite": "2_insert_cluster.yaml",
"image_type": "cpu"
}
]
}
]

19
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/debug.json
View File

@ -1,19 +0,0 @@
[
{
"deploy_mode": "cluster",
"suite_params": [
{
"suite": "2_cpu_ann_accuracy.yaml",
"image_type": "cpu"
}
]
},
{
"suite_params": [
{
"suite": "2_cpu_ann_accuracy.yaml",
"image_type": "cpu"
}
]
}
]

20
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/debug1.json
View File

@ -1,20 +0,0 @@
[
{
"server_tag": "8c16m1g",
"suite_params": [
{
"suite": "080_gpu_search_debug.yaml",
"image_type": "gpu"
}
]
},
{
"server_tag": "16c32m1g",
"suite_params": [
{
"suite": "080_gpu_search_debug.yaml",
"image_type": "gpu"
}
]
}
]

12
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/debug2.json
View File

@ -1,12 +0,0 @@
[
{
"server": "idc-sh005",
"deploy_mode": "cluster",
"suite_params": [
{
"suite": "2_insert_search_sift10m_512.yaml",
"image_type": "cpu"
}
]
}
]

11
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/filter.json
View File

@ -1,11 +0,0 @@
[
{
"server": "poseidon",
"suite_params": [
{
"suite": "011_search_dsl.yaml",
"image_type": "gpu"
}
]
}
]

11
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/idc.json
View File

@ -1,11 +0,0 @@
[
{
"server": "idc-sh004",
"suite_params": [
{
"suite": "011_cpu_search_debug.yaml",
"image_type": "gpu"
}
]
}
]

11
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/insert.json
View File

@ -1,11 +0,0 @@
[
{
"server": "idc-sh002",
"suite_params": [
{
"suite": "2_insert_data.yaml",
"image_type": "cpu"
}
]
}
]

11
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/insert2.json
View File

@ -1,11 +0,0 @@
[
{
"server": "idc-sh002",
"suite_params": [
{
"suite": "2_insert_data.yaml",
"image_type": "cpu"
}
]
}
]

11
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/jaccard.json
View File

@ -1,11 +0,0 @@
[
{
"server": "athena",
"suite_params": [
{
"suite": "011_cpu_search_binary.yaml",
"image_type": "cpu"
}
]
}
]

15
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/locust.json
View File

@ -1,15 +0,0 @@
[
{
"server": "idc-sh002",
"suite_params": [
{
"suite": "2_locust_search.yaml",
"image_type": "cpu"
},
{
"suite": "2_locust_search_index.yaml",
"image_type": "cpu"
}
]
}
]

11
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/locust_insert.json
View File

@ -1,11 +0,0 @@
[
{
"server": "idc-sh005",
"suite_params": [
{
"suite": "2_locust_insert.yaml",
"image_type": "cpu"
}
]
}
]

11
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/locust_mix.json
View File

@ -1,11 +0,0 @@
[
{
"server": "athena",
"suite_params": [
{
"suite": "locust_mix.yaml",
"image_type": "gpu"
}
]
}
]

10
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/locust_mix_debug.json
View File

@ -1,10 +0,0 @@
[
{
"suite_params": [
{
"suite": "locust_mix.yaml",
"image_type": "cpu"
}
]
}
]

11
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/locust_search.json
View File

@ -1,11 +0,0 @@
[
{
"server": "idc-sh002",
"suite_params": [
{
"suite": "2_locust_search.yaml",
"image_type": "cpu"
}
]
}
]

10
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/loop.json
View File

@ -1,10 +0,0 @@
[
{
"suite_params": [
{
"suite": "2_locust_insert_5h.yaml",
"image_type": "cpu"
}
]
}
]

10
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/loop_search.json
View File

@ -1,10 +0,0 @@
[
{
"suite_params": [
{
"suite": "2_locust_search_5h.yaml",
"image_type": "cpu"
}
]
}
]

11
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/nlist.json
View File

@ -1,11 +0,0 @@
[
{
"server": "idc-sh003",
"suite_params": [
{
"suite": "2_insert_search_sift50m_2048.yaml",
"image_type": "cpu"
}
]
}
]

11
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/search.json
View File

@ -1,11 +0,0 @@
[
{
"server": "idc-sh003",
"suite_params": [
{
"suite": "2_insert_search.yaml",
"image_type": "cpu"
}
]
}
]

11
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/search2.json
View File

@ -1,11 +0,0 @@
[
{
"server_tag": "16c32m0g",
"suite_params": [
{
"suite": "2_cpu_search.yaml",
"image_type": "cpu"
}
]
}
]

11
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/search_debug.json
View File

@ -1,11 +0,0 @@
[
{
"server": "idc-sh005",
"suite_params": [
{
"suite": "2_insert_search_sift10m_4096.yaml",
"image_type": "cpu"
}
]
}
]

10
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/shards_ann.json
View File

@ -1,10 +0,0 @@
[
{
"suite_params": [
{
"suite": "shards_ann_debug.yaml",
"image_type": "cpu"
}
]
}
]

15
tests-deprecating/milvus_benchmark/milvus_benchmark/scheduler/shards_debug.json
View File

@ -1,15 +0,0 @@
[
{
"server": "apollo",
"suite_params": [
{
"suite": "shards_insert_performance_sift1m.yaml",
"image_type": "cpu"
},
{
"suite": "shards_search_performance_sift1m.yaml",
"image_type": "cpu"
}
]
}
]

Some files were not shown because too many files have changed in this diff Show More