milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/branch-0.3.1' into mem-0.3.1 

Former-commit-id: 18256c0946e15c41cc7446adc5ee14127c4c1058
pull/191/head
zhiru 2019-07-08 11:14:51 +08:00
parent a951dd14c0 9ec9b3fc26
commit 404e1db012
26 changed files with 1026 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOGS.md
View File

@ -15,3 +15,4 @@ Please mark all change in change log and use the ticket from JIRA.
### Task
- MS-1 - Add CHANGELOG.md
- MS-161 - Add CI / CD Module to Milvus Project

10
ci/function/file_transfer.groovy Normal file
View File

@ -0,0 +1,10 @@
def FileTransfer (sourceFiles, remoteDirectory, remoteIP, protocol = "ftp", makeEmptyDirs = true) {
if (protocol == "ftp") {
ftpPublisher masterNodeName: '', paramPublish: [parameterName: ''], alwaysPublishFromMaster: false, continueOnError: false, failOnError: true, publishers: [
[configName: "${remoteIP}", transfers: [
[asciiMode: false, cleanRemote: false, excludes: '', flatten: false, makeEmptyDirs: "${makeEmptyDirs}", noDefaultExcludes: false, patternSeparator: '[, ]+', remoteDirectory: "${remoteDirectory}", remoteDirectorySDF: false, removePrefix: '', sourceFiles: "${sourceFiles}"]], usePromotionTimestamp: true, useWorkspaceInPromotion: false, verbose: true
]
]
}
}
return this

12
ci/jenkinsfile/cleanup_dev.groovy Normal file
View File

@ -0,0 +1,12 @@
try {
sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
if (currentBuild.result == 'ABORTED') {
throw new hudson.AbortException("Dev Test Aborted !")
} else if (currentBuild.result == 'FAILURE') {
error("Dev Test Failure !")
}
} catch (exc) {
updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed'
throw exc
}

11
ci/jenkinsfile/deploy2dev.groovy Normal file
View File

@ -0,0 +1,11 @@
try {
sh 'helm init --client-only --skip-refresh'
sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus'
sh 'helm repo update'
sh "helm install --set engine.image.repository=registry.zilliz.com/${PROJECT_NAME}/engine --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.0 milvus/milvus-gpu"
} catch (exc) {
updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed'
echo 'Helm running failed!'
sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
throw exc
}

17
ci/jenkinsfile/dev_test.groovy Normal file
View File

@ -0,0 +1,17 @@
container('milvus-testframework') {
timeout(time: 10, unit: 'MINUTES') {
gitlabCommitStatus(name: 'Dev Test') {
try {
dir ("${PROJECT_NAME}_test") {
checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git"]]])
sh 'python3 -m pip install -r requirements.txt'
sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local"
}
} catch (exc) {
updateGitlabCommitStatus name: 'Dev Test', state: 'failed'
currentBuild.result = 'FAILURE'
echo 'Milvus Test Failed !'
}
}
}
}

17
ci/jenkinsfile/milvus_build.groovy Normal file
View File

@ -0,0 +1,17 @@
container('milvus-build-env') {
timeout(time: 20, unit: 'MINUTES') {
gitlabCommitStatus(name: 'Build Engine') {
dir ("milvus_engine") {
try {
checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git"]]])
dir ("cpp") {
sh "./build.sh -t ${params.BUILD_TYPE} -u -c"
}
} catch (exc) {
updateGitlabCommitStatus name: 'Build Engine', state: 'failed'
throw exc
}
}
}
}
}

17
ci/jenkinsfile/milvus_build_no_ut.groovy Normal file
View File

@ -0,0 +1,17 @@
container('milvus-build-env') {
timeout(time: 20, unit: 'MINUTES') {
gitlabCommitStatus(name: 'Build Engine') {
dir ("milvus_engine") {
try {
checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git"]]])
dir ("cpp") {
sh "./build.sh -t ${params.BUILD_TYPE}"
}
} catch (exc) {
updateGitlabCommitStatus name: 'Build Engine', state: 'failed'
throw exc
}
}
}
}
}

44
ci/jenkinsfile/packaged_milvus.groovy Normal file
View File

@ -0,0 +1,44 @@
container('milvus-build-env') {
timeout(time: 5, unit: 'MINUTES') {
dir ("milvus_engine") {
dir ("cpp") {
gitlabCommitStatus(name: 'Packaged Engine') {
if (fileExists('milvus')) {
try {
sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus"
def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage')
if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
echo "Download Milvus Engine Binary Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz\""
}
} catch (exc) {
updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed'
throw exc
}
} else {
updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed'
error("Milvus binary directory don't exists!")
}
}
gitlabCommitStatus(name: 'Packaged Engine lcov') {
if (fileExists('lcov_out')) {
try {
def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
fileTransfer.FileTransfer("lcov_out/", "${PROJECT_NAME}/lcov/${JOB_NAME}-${BUILD_ID}", 'nas storage')
if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
echo "Milvus lcov out Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/lcov/${JOB_NAME}-${BUILD_ID}/lcov_out/\""
}
} catch (exc) {
updateGitlabCommitStatus name: 'Packaged Engine lcov', state: 'failed'
throw exc
}
} else {
updateGitlabCommitStatus name: 'Packaged Engine lcov', state: 'failed'
error("Milvus lcov out directory don't exists!")
}
}
}
}
}
}

26
ci/jenkinsfile/packaged_milvus_no_ut.groovy Normal file
View File

@ -0,0 +1,26 @@
container('milvus-build-env') {
timeout(time: 5, unit: 'MINUTES') {
dir ("milvus_engine") {
dir ("cpp") {
gitlabCommitStatus(name: 'Packaged Engine') {
if (fileExists('milvus')) {
try {
sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus"
def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage')
if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
echo "Download Milvus Engine Binary Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz\""
}
} catch (exc) {
updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed'
throw exc
}
} else {
updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed'
error("Milvus binary directory don't exists!")
}
}
}
}
}
}

31
ci/jenkinsfile/publish_docker.groovy Normal file
View File

@ -0,0 +1,31 @@
container('publish-docker') {
timeout(time: 15, unit: 'MINUTES') {
gitlabCommitStatus(name: 'Publish Engine Docker') {
try {
dir ("${PROJECT_NAME}_build") {
checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:build/milvus_build.git"]]])
dir ("docker/deploy/ubuntu16.04/free_version") {
sh "curl -O -u anonymous: ftp://192.168.1.126/data/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz"
sh "tar zxvf ${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz"
try {
docker.withRegistry('https://registry.zilliz.com', 'a54e38ef-c424-4ea9-9224-b25fc20e3924') {
def customImage = docker.build("${PROJECT_NAME}/engine:${DOCKER_VERSION}")
customImage.push()
}
echo "Docker Pull Command: docker pull registry.zilliz.com/${PROJECT_NAME}/engine:${DOCKER_VERSION}"
} catch (exc) {
updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'canceled'
throw exc
} finally {
sh "docker rmi ${PROJECT_NAME}/engine:${DOCKER_VERSION}"
}
}
}
} catch (exc) {
updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'failed'
echo 'Publish docker failed!'
throw exc
}
}
}
}

26
ci/jenkinsfile/upload_dev_test_out.groovy Normal file
View File

@ -0,0 +1,26 @@
container('milvus-testframework') {
timeout(time: 5, unit: 'MINUTES') {
dir ("${PROJECT_NAME}_test") {
gitlabCommitStatus(name: 'Upload Dev Test Out') {
if (fileExists('test_out')) {
try {
def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
fileTransfer.FileTransfer("test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage')
if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\""
}
} catch (hudson.AbortException ae) {
updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'canceled'
currentBuild.result = 'ABORTED'
} catch (exc) {
updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'failed'
currentBuild.result = 'FAILURE'
}
} else {
updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'failed'
echo "Milvus Dev Test Out directory don't exists!"
}
}
}
}
}

256
ci/main_jenkinsfile Normal file
View File

@ -0,0 +1,256 @@
pipeline {
agent none
options {
timestamps()
}
environment {
PROJECT_NAME = "milvus"
LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase()
SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}"
GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}"
SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}"
DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}"
}
stages {
stage("Ubuntu 16.04") {
environment {
PACKAGE_VERSION = VersionNumber([
versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}'
]);
DOCKER_VERSION = VersionNumber([
versionNumberString : '${DOCKER_VERSION_STR}'
]);
}
stages {
stage("Run Build") {
agent {
kubernetes {
cloud 'build-kubernetes'
label 'build'
defaultContainer 'jnlp'
containerTemplate {
name 'milvus-build-env'
image 'registry.zilliz.com/milvus/milvus-build-env:v0.10'
ttyEnabled true
command 'cat'
}
}
}
stages {
stage('Build') {
steps {
gitlabCommitStatus(name: 'Build') {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build.groovy"
load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus.groovy"
}
}
}
}
}
post {
aborted {
script {
updateGitlabCommitStatus name: 'Build', state: 'canceled'
echo "Milvus Build aborted !"
}
}
failure {
script {
updateGitlabCommitStatus name: 'Build', state: 'failed'
echo "Milvus Build failure !"
}
}
}
}
stage("Publish docker and helm") {
agent {
kubernetes {
label 'publish'
defaultContainer 'jnlp'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
app: publish
componet: docker
spec:
containers:
- name: publish-docker
image: registry.zilliz.com/library/zilliz_docker:v1.0.0
securityContext:
privileged: true
command:
- cat
tty: true
volumeMounts:
- name: docker-sock
mountPath: /var/run/docker.sock
volumes:
- name: docker-sock
hostPath:
path: /var/run/docker.sock
"""
}
}
stages {
stage('Publish Docker') {
steps {
gitlabCommitStatus(name: 'Publish Docker') {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy"
}
}
}
}
}
post {
aborted {
script {
updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled'
echo "Milvus Publish Docker aborted !"
}
}
failure {
script {
updateGitlabCommitStatus name: 'Publish Docker', state: 'failed'
echo "Milvus Publish Docker failure !"
}
}
}
}
stage("Deploy to Development") {
stages {
stage("Deploy to Dev") {
agent {
kubernetes {
label 'jenkins-slave'
defaultContainer 'jnlp'
}
}
stages {
stage('Deploy') {
steps {
gitlabCommitStatus(name: 'Deloy to Dev') {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy"
}
}
}
}
}
post {
aborted {
script {
updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled'
echo "Milvus Deloy to Dev aborted !"
}
}
failure {
script {
updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed'
echo "Milvus Deloy to Dev failure !"
}
}
}
}
stage("Dev Test") {
agent {
kubernetes {
label 'test'
defaultContainer 'jnlp'
containerTemplate {
name 'milvus-testframework'
image 'registry.zilliz.com/milvus/milvus-test:v0.1'
ttyEnabled true
command 'cat'
}
}
}
stages {
stage('Test') {
steps {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy"
load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy"
}
}
}
}
}
stage ("Cleanup Dev") {
agent {
kubernetes {
label 'jenkins-slave'
defaultContainer 'jnlp'
}
}
stages {
stage('Cleanup') {
steps {
gitlabCommitStatus(name: 'Cleanup Dev') {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy"
}
}
}
}
}
post {
aborted {
script {
updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled'
echo "Milvus Cleanup Dev aborted !"
}
}
failure {
script {
updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed'
echo "Milvus Cleanup Dev failure !"
}
}
}
}
}
}
}
}
}
post {
success {
script {
updateGitlabCommitStatus name: 'CI/CD', state: 'success'
echo "Milvus CI/CD success !"
}
}
aborted {
script {
updateGitlabCommitStatus name: 'CI/CD', state: 'canceled'
echo "Milvus CI/CD aborted !"
}
}
failure {
script {
updateGitlabCommitStatus name: 'CI/CD', state: 'failed'
echo "Milvus CI/CD failure !"
}
}
}
}

256
ci/main_jenkinsfile_no_ut Normal file
View File

@ -0,0 +1,256 @@
pipeline {
agent none
options {
timestamps()
}
environment {
PROJECT_NAME = "milvus"
LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase()
SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}"
GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}"
SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}"
DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}"
}
stages {
stage("Ubuntu 16.04") {
environment {
PACKAGE_VERSION = VersionNumber([
versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}'
]);
DOCKER_VERSION = VersionNumber([
versionNumberString : '${DOCKER_VERSION_STR}'
]);
}
stages {
stage("Run Build") {
agent {
kubernetes {
cloud 'build-kubernetes'
label 'build'
defaultContainer 'jnlp'
containerTemplate {
name 'milvus-build-env'
image 'registry.zilliz.com/milvus/milvus-build-env:v0.10'
ttyEnabled true
command 'cat'
}
}
}
stages {
stage('Build') {
steps {
gitlabCommitStatus(name: 'Build') {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build_no_ut.groovy"
load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus_no_ut.groovy"
}
}
}
}
}
post {
aborted {
script {
updateGitlabCommitStatus name: 'Build', state: 'canceled'
echo "Milvus Build aborted !"
}
}
failure {
script {
updateGitlabCommitStatus name: 'Build', state: 'failed'
echo "Milvus Build failure !"
}
}
}
}
stage("Publish docker and helm") {
agent {
kubernetes {
label 'publish'
defaultContainer 'jnlp'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
app: publish
componet: docker
spec:
containers:
- name: publish-docker
image: registry.zilliz.com/library/zilliz_docker:v1.0.0
securityContext:
privileged: true
command:
- cat
tty: true
volumeMounts:
- name: docker-sock
mountPath: /var/run/docker.sock
volumes:
- name: docker-sock
hostPath:
path: /var/run/docker.sock
"""
}
}
stages {
stage('Publish Docker') {
steps {
gitlabCommitStatus(name: 'Publish Docker') {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy"
}
}
}
}
}
post {
aborted {
script {
updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled'
echo "Milvus Publish Docker aborted !"
}
}
failure {
script {
updateGitlabCommitStatus name: 'Publish Docker', state: 'failed'
echo "Milvus Publish Docker failure !"
}
}
}
}
stage("Deploy to Development") {
stages {
stage("Deploy to Dev") {
agent {
kubernetes {
label 'jenkins-slave'
defaultContainer 'jnlp'
}
}
stages {
stage('Deploy') {
steps {
gitlabCommitStatus(name: 'Deloy to Dev') {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy"
}
}
}
}
}
post {
aborted {
script {
updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled'
echo "Milvus Deloy to Dev aborted !"
}
}
failure {
script {
updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed'
echo "Milvus Deloy to Dev failure !"
}
}
}
}
stage("Dev Test") {
agent {
kubernetes {
label 'test'
defaultContainer 'jnlp'
containerTemplate {
name 'milvus-testframework'
image 'registry.zilliz.com/milvus/milvus-test:v0.1'
ttyEnabled true
command 'cat'
}
}
}
stages {
stage('Test') {
steps {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy"
load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy"
}
}
}
}
}
stage ("Cleanup Dev") {
agent {
kubernetes {
label 'jenkins-slave'
defaultContainer 'jnlp'
}
}
stages {
stage('Cleanup') {
steps {
gitlabCommitStatus(name: 'Cleanup Dev') {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy"
}
}
}
}
}
post {
aborted {
script {
updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled'
echo "Milvus Cleanup Dev aborted !"
}
}
failure {
script {
updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed'
echo "Milvus Cleanup Dev failure !"
}
}
}
}
}
}
}
}
}
post {
success {
script {
updateGitlabCommitStatus name: 'CI/CD', state: 'success'
echo "Milvus CI/CD success !"
}
}
aborted {
script {
updateGitlabCommitStatus name: 'CI/CD', state: 'canceled'
echo "Milvus CI/CD aborted !"
}
}
failure {
script {
updateGitlabCommitStatus name: 'CI/CD', state: 'failed'
echo "Milvus CI/CD failure !"
}
}
}
}

13
ci/pod_containers/milvus-engine-build.yaml Normal file
View File

@ -0,0 +1,13 @@
apiVersion: v1
kind: Pod
metadata:
labels:
app: milvus
componet: build-env
spec:
containers:
- name: milvus-build-env
image: registry.zilliz.com/milvus/milvus-build-env:v0.9
command:
- cat
tty: true

13
ci/pod_containers/milvus-testframework.yaml Normal file
View File

@ -0,0 +1,13 @@
apiVersion: v1
kind: Pod
metadata:
labels:
app: milvus
componet: testframework
spec:
containers:
- name: milvus-testframework
image: registry.zilliz.com/milvus/milvus-test:v0.1
command:
- cat
tty: true

22
ci/pod_containers/publish-docker.yaml Normal file
View File

@ -0,0 +1,22 @@
apiVersion: v1
kind: Pod
metadata:
labels:
app: publish
componet: docker
spec:
containers:
- name: publish-docker
image: registry.zilliz.com/library/zilliz_docker:v1.0.0
securityContext:
privileged: true
command:
- cat
tty: true
volumeMounts:
- name: docker-sock
mountPath: /var/run/docker.sock
volumes:
- name: docker-sock
hostPath:
path: /var/run/docker.sock

68
cpp/src/server/RequestTask.cpp
View File

@ -8,6 +8,7 @@
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ValidationUtil.h"
#include "DBWrapper.h"
#include "version.h"
@ -108,7 +109,13 @@ namespace {
}
long days = (tt_end > tt_start) ? (tt_end - tt_start)/DAY_SECONDS : (tt_start - tt_end)/DAY_SECONDS;
for(long i = 0; i <= days; i++) {
if(days == 0) {
error_code = SERVER_INVALID_TIME_RANGE;
error_msg = "Invalid time range: " + range.start_value + " to " + range.end_value;
return ;
}
for(long i = 0; i < days; i++) {
time_t tt_day = tt_start + DAY_SECONDS*i;
tm tm_day;
CommonUtil::ConvertTime(tt_day, tm_day);
@ -133,19 +140,23 @@ BaseTaskPtr CreateTableTask::Create(const thrift::TableSchema& schema) {
ServerError CreateTableTask::OnExecute() {
TimeRecorder rc("CreateTableTask");
try {
//step 1: check arguments
if(schema_.table_name.empty()) {
return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name");
}
if(schema_.dimension <= 0) {
return SetError(SERVER_INVALID_TABLE_DIMENSION, "Invalid table dimension: " + std::to_string(schema_.dimension));
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(schema_.table_name);
if(res != SERVER_SUCCESS) {
return res;
}
engine::EngineType engine_type = EngineType(schema_.index_type);
if(engine_type == engine::EngineType::INVALID) {
return SetError(SERVER_INVALID_INDEX_TYPE, "Invalid index type: " + std::to_string(schema_.index_type));
res = ValidateTableDimension(schema_.dimension);
if(res != SERVER_SUCCESS) {
return res;
}
res = ValidateTableIndexType(schema_.index_type);
if(res != SERVER_SUCCESS) {
return res;
}
//step 2: construct table schema
@ -187,8 +198,10 @@ ServerError DescribeTableTask::OnExecute() {
try {
//step 1: check arguments
if(table_name_.empty()) {
return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name");
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return res;
}
//step 2: get table info
@ -230,10 +243,11 @@ ServerError HasTableTask::OnExecute() {
TimeRecorder rc("HasTableTask");
//step 1: check arguments
if(table_name_.empty()) {
return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name");
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return res;
}
//step 2: check table existence
engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table_);
if(!stat.ok()) {
@ -264,8 +278,10 @@ ServerError DeleteTableTask::OnExecute() {
TimeRecorder rc("DeleteTableTask");
//step 1: check arguments
if (table_name_.empty()) {
return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name");
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return res;
}
//step 2: check table existence
@ -346,8 +362,10 @@ ServerError AddVectorTask::OnExecute() {
TimeRecorder rc("AddVectorTask");
//step 1: check arguments
if (table_name_.empty()) {
return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name");
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return res;
}
if(record_array_.empty()) {
@ -435,8 +453,10 @@ ServerError SearchVectorTask::OnExecute() {
TimeRecorder rc("SearchVectorTask");
//step 1: check arguments
if (table_name_.empty()) {
return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name");
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return res;
}
if(top_k_ <= 0) {
@ -548,8 +568,10 @@ ServerError GetTableRowCountTask::OnExecute() {
TimeRecorder rc("GetTableRowCountTask");
//step 1: check arguments
if (table_name_.empty()) {
return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name");
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return res;
}
//step 2: get row count

74
cpp/src/utils/ValidationUtil.cpp Normal file
View File

@ -0,0 +1,74 @@
#include <src/db/ExecutionEngine.h>
#include "ValidationUtil.h"
#include "Log.h"
namespace zilliz {
namespace milvus {
namespace server {
constexpr size_t table_name_size_limit = 255;
constexpr int64_t table_dimension_limit = 16384;
ServerError
ValidateTableName(const std::string &table_name) {
// Table name shouldn't be empty.
if (table_name.empty()) {
SERVER_LOG_ERROR << "Empty table name";
return SERVER_INVALID_TABLE_NAME;
}
// Table name size shouldn't exceed 16384.
if (table_name.size() > table_name_size_limit) {
SERVER_LOG_ERROR << "Table name size exceed the limitation";
return SERVER_INVALID_TABLE_NAME;
}
// Table name first character should be underscore or character.
char first_char = table_name[0];
if (first_char != '_' && std::isalpha(first_char) == 0) {
SERVER_LOG_ERROR << "Table name first character isn't underscore or character: " << first_char;
return SERVER_INVALID_TABLE_NAME;
}
int64_t table_name_size = table_name.size();
for (int64_t i = 1; i < table_name_size; ++i) {
char name_char = table_name[i];
if (name_char != '_' && std::isalnum(name_char) == 0) {
SERVER_LOG_ERROR << "Table name character isn't underscore or alphanumber: " << name_char;
return SERVER_INVALID_TABLE_NAME;
}
}
return SERVER_SUCCESS;
}
ServerError
ValidateTableDimension(int64_t dimension) {
if (dimension <= 0 || dimension > table_dimension_limit) {
SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << table_dimension_limit;
return SERVER_INVALID_VECTOR_DIMENSION;
} else {
return SERVER_SUCCESS;
}
}
ServerError
ValidateTableIndexType(int32_t index_type) {
auto engine_type = engine::EngineType(index_type);
switch (engine_type) {
case engine::EngineType::FAISS_IDMAP:
case engine::EngineType::FAISS_IVFFLAT: {
SERVER_LOG_DEBUG << "Index type: " << index_type;
return SERVER_SUCCESS;
}
default: {
return SERVER_INVALID_INDEX_TYPE;
}
}
}
}
}
}

20
cpp/src/utils/ValidationUtil.h Normal file
View File

@ -0,0 +1,20 @@
#pragma once
#include "Error.h"
namespace zilliz {
namespace milvus {
namespace server {
ServerError
ValidateTableName(const std::string& table_name);
ServerError
ValidateTableDimension(int64_t dimension);
ServerError
ValidateTableIndexType(int32_t index_type);
}
}
}

4
cpp/unittest/CMakeLists.txt
View File

@ -12,7 +12,6 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
set(unittest_srcs
${CMAKE_CURRENT_SOURCE_DIR}/main.cpp)
#${EASYLOGGINGPP_INCLUDE_DIR}/easylogging++.cc)
set(require_files
${MILVUS_ENGINE_SRC}/server/ServerConfig.cpp
@ -44,4 +43,5 @@ add_subdirectory(db)
add_subdirectory(faiss_wrapper)
#add_subdirectory(license)
add_subdirectory(metrics)
add_subdirectory(storage)
add_subdirectory(storage)
add_subdirectory(utils)

13
cpp/unittest/db/db_tests.cpp
View File

@ -3,17 +3,20 @@
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include <thread>
#include <easylogging++.h>
#include <boost/filesystem.hpp>
#include "utils.h"
#include "db/DB.h"
#include "db/DBImpl.h"
#include "db/MetaConsts.h"
#include "db/Factories.h"
#include <gtest/gtest.h>
#include <easylogging++.h>
#include <boost/filesystem.hpp>
#include <thread>
#include <random>
using namespace zilliz::milvus;
namespace {

12
cpp/unittest/db/mysql_db_test.cpp
View File

@ -3,17 +3,19 @@
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include <thread>
#include <easylogging++.h>
#include <boost/filesystem.hpp>
#include "utils.h"
#include "db/DB.h"
#include "db/DBImpl.h"
#include "db/MetaConsts.h"
#include "db/Factories.h"
#include <gtest/gtest.h>
#include <easylogging++.h>
#include <boost/filesystem.hpp>
#include <thread>
#include <random>
using namespace zilliz::milvus;
namespace {

5
cpp/unittest/db/search_test.cpp
View File

@ -3,10 +3,11 @@
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include "db/scheduler/task/SearchTask.h"
#include <gtest/gtest.h>
#include <cmath>
#include <vector>
using namespace zilliz::milvus;

5
cpp/unittest/faiss_wrapper/wrapper_test.cpp
View File

@ -4,12 +4,15 @@
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include "wrapper/Operand.h"
#include "wrapper/Index.h"
#include "wrapper/IndexBuilder.h"
#include <gtest/gtest.h>
#include <random>
using namespace zilliz::milvus::engine;

30
cpp/unittest/utils/CMakeLists.txt Normal file
View File

@ -0,0 +1,30 @@
#-------------------------------------------------------------------------------
# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
# Unauthorized copying of this file, via any medium is strictly prohibited.
# Proprietary and confidential.
#-------------------------------------------------------------------------------
# Make sure that your call to link_directories takes place before your call to the relevant add_executable.
include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
set(validation_util_src
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.h)
set(validation_util_test_src
${unittest_srcs}
${validation_util_src}
${require_files}
ValidationUtilTest.cpp
)
add_executable(valication_util_test
${validation_util_test_src}
${config_files})
target_link_libraries(valication_util_test
${unittest_libs}
boost_filesystem)
install(TARGETS valication_util_test DESTINATION bin)

61
cpp/unittest/utils/ValidationUtilTest.cpp Normal file
View File

@ -0,0 +1,61 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include "utils/ValidationUtil.h"
#include "utils/Error.h"
#include <string>
using namespace zilliz::milvus::server;
TEST(ValidationUtilTest, TableNameTest) {
std::string table_name = "Normal123_";
ServerError res = ValidateTableName(table_name);
ASSERT_EQ(res, SERVER_SUCCESS);
table_name = "12sds";
res = ValidateTableName(table_name);
ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
table_name = "";
res = ValidateTableName(table_name);
ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
table_name = "_asdasd";
res = ValidateTableName(table_name);
ASSERT_EQ(res, SERVER_SUCCESS);
table_name = "!@#!@";
res = ValidateTableName(table_name);
ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
table_name = "中文";
res = ValidateTableName(table_name);
ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
table_name = std::string('a', 32768);
res = ValidateTableName(table_name);
ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
}
TEST(ValidationUtilTest, TableDimensionTest) {
ASSERT_EQ(ValidateTableDimension(-1), SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(ValidateTableDimension(0), SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(ValidateTableDimension(16385), SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(ValidateTableDimension(16384), SERVER_SUCCESS);
ASSERT_EQ(ValidateTableDimension(1), SERVER_SUCCESS);
}
TEST(ValidationUtilTest, TableIndexTypeTest) {
ASSERT_EQ(ValidateTableIndexType(0), SERVER_INVALID_INDEX_TYPE);
ASSERT_EQ(ValidateTableIndexType(1), SERVER_SUCCESS);
ASSERT_EQ(ValidateTableIndexType(2), SERVER_SUCCESS);
ASSERT_EQ(ValidateTableIndexType(3), SERVER_INVALID_INDEX_TYPE);
ASSERT_EQ(ValidateTableIndexType(4), SERVER_INVALID_INDEX_TYPE);
}