From 46a63f43f36c495089cca3e9a78f7901bee6981e Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sun, 14 Jul 2019 09:33:35 +0800 Subject: [PATCH 01/18] MS-215 - add milvus cluster CI/CD groovy file Former-commit-id: 6d3159ab782c4acb0bc31831438e09015ed447fa --- ci/jenkinsfile/cluster_deploy2dev.groovy | 8 +++++--- ci/jenkinsfile/deploy2dev.groovy | 8 +++++--- ci/main_jenkinsfile | 22 ++++++++++++---------- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/ci/jenkinsfile/cluster_deploy2dev.groovy b/ci/jenkinsfile/cluster_deploy2dev.groovy index 97062922ca..4347ea0938 100644 --- a/ci/jenkinsfile/cluster_deploy2dev.groovy +++ b/ci/jenkinsfile/cluster_deploy2dev.groovy @@ -7,9 +7,11 @@ try { dir ("milvus/milvus-cluster") { sh "helm install --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.1.0 . " } - waitUntil { - def result = sh script: "nc -z -w 2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local 19530", returnStatus: true - return !result + timeout(time: 2, unit: 'MINUTES') { + waitUntil { + def result = sh script: "nc -z -w 2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local 19530", returnStatus: true + return !result + } } } } catch (exc) { diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy index f594d0ba57..607149f35d 100644 --- a/ci/jenkinsfile/deploy2dev.groovy +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -3,9 +3,11 @@ try { sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' sh 'helm repo update' sh "helm install --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.0 milvus/milvus-gpu" - waitUntil { - def result = sh script: "nc -z -w 2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local 19530", returnStatus: true - return !result + timeout(time: 2, unit: 'MINUTES') { + waitUntil { + def result = sh script: "nc -z -w 2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local 19530", returnStatus: true + return !result + } } } catch (exc) { updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index 1f0d0c3d7f..a324ff4441 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -337,16 +337,18 @@ spec: post { always { script { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' + if (env.gitlabAfter != null) { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } } } } From f45037322c1a50f05f68daa58837647053bf8c5d Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sun, 14 Jul 2019 09:34:47 +0800 Subject: [PATCH 02/18] change cpp CHANGELOGS.md Former-commit-id: f57d675bf412261f544708860ccdd486933ca79c --- CHANGELOGS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOGS.md b/CHANGELOGS.md index def4965a41..3cc1b28d82 100644 --- a/CHANGELOGS.md +++ b/CHANGELOGS.md @@ -16,3 +16,4 @@ Please mark all change in change log and use the ticket from JIRA. - MS-1 - Add CHANGELOG.md - MS-161 - Add CI / CD Module to Milvus Project +- MS-215 - Add Milvus cluster CI/CD groovy file From 94d88ec0cb9a82924cf4ec7ba87757e37e1a5739 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sun, 14 Jul 2019 11:24:47 +0800 Subject: [PATCH 03/18] Update Chart Version to 0.3.1 Former-commit-id: 8295125e067afeb2e45b58cfeba53f4c14a19db3 --- ci/jenkinsfile/cluster_deploy2dev.groovy | 4 ++-- ci/jenkinsfile/deploy2dev.groovy | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ci/jenkinsfile/cluster_deploy2dev.groovy b/ci/jenkinsfile/cluster_deploy2dev.groovy index 4347ea0938..dba2e52713 100644 --- a/ci/jenkinsfile/cluster_deploy2dev.groovy +++ b/ci/jenkinsfile/cluster_deploy2dev.groovy @@ -3,9 +3,9 @@ try { sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' sh 'helm repo update' dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) dir ("milvus/milvus-cluster") { - sh "helm install --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.1.0 . " + sh "helm install --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.3.1 . " } timeout(time: 2, unit: 'MINUTES') { waitUntil { diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy index 607149f35d..26e2aec562 100644 --- a/ci/jenkinsfile/deploy2dev.groovy +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -2,7 +2,12 @@ try { sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' sh 'helm repo update' - sh "helm install --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.0 milvus/milvus-gpu" + dir ("milvus-helm") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + dir ("milvus/milvus-gpu") { + helm install --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.1 . + } + } timeout(time: 2, unit: 'MINUTES') { waitUntil { def result = sh script: "nc -z -w 2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local 19530", returnStatus: true From eb14bc0d2d7cfe17e3fc3e66ab3501b20f39f512 Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 14 Jul 2019 11:43:14 +0800 Subject: [PATCH 04/18] Fix SQ8 row count bug Former-commit-id: 41749a648567e492bc9a718db9526ed98bb2532a --- cpp/CHANGELOG.md | 5 +++-- cpp/conf/server_config.template | 6 +++--- cpp/src/db/DBImpl.cpp | 2 +- cpp/src/db/DBMetaImpl.cpp | 22 ++++++++++++++-------- cpp/src/db/FaissExecutionEngine.cpp | 2 +- cpp/src/db/Options.h | 2 +- 6 files changed, 23 insertions(+), 16 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 9ac6cb11bc..3be585c7b6 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -9,8 +9,9 @@ Please mark all change in change log and use the ticket from JIRA. - MS-148 - Disable cleanup if mode is read only - MS-149 - Fixed searching only one index file issue in distributed mode -- MS-153 - fix c_str error when connecting to MySQL -- MS-157 - fix changelog +- MS-153 - Fix c_str error when connecting to MySQL +- MS-157 - Fix changelog +- MS-217 - Fix SQ8 row count bug ## Improvement - MS-156 - Add unittest for merge result functions diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 3f3b004456..6de96ae895 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -14,8 +14,8 @@ db_config: db_backend_url: sqlite://:@:/ index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB - archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB - archive_days_threshold: 30 # files older than x days will be archived, unit: day + archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB + archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day metric_config: is_startup: off # if monitoring start: on, off @@ -37,4 +37,4 @@ engine_config: nprobe: 10 nlist: 16384 use_blas_threshold: 20 - metric_type: L2 #L2 or Inner Product \ No newline at end of file + metric_type: L2 # compare vectors by euclidean distance(L2) or inner product(IP), optional: L2 or IP \ No newline at end of file diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 35a6f075f8..beae6c30d3 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -487,7 +487,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { //step 6: update meta table_file.file_type_ = meta::TableFileSchema::INDEX; - table_file.size_ = index->PhysicalSize(); + table_file.size_ = index->Size(); auto to_remove = file; to_remove.file_type_ = meta::TableFileSchema::TO_DELETE; diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index fc67c6b360..99ae3b6711 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -674,16 +674,22 @@ Status DBMetaImpl::Archive() { Status DBMetaImpl::Size(uint64_t &result) { result = 0; try { - auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::size_)), - where( - c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE - )); + auto files = ConnectorPtr->select(columns(&TableFileSchema::size_, + &TableFileSchema::file_type_, + &TableFileSchema::engine_type_), + where( + c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE + )); - for (auto &sub_query : selected) { - if (!std::get<0>(sub_query)) { - continue; + for (auto &file : files) { + auto file_size = std::get<0>(file); + auto file_type = std::get<1>(file); + auto engine_type = std::get<2>(file); + if(file_type == (int)TableFileSchema::INDEX && engine_type == (int)EngineType::FAISS_IVFSQ8) { + result += (uint64_t)file_size/4;//hardcode for sq8 + } else { + result += (uint64_t)file_size; } - result += (uint64_t) (*std::get<0>(sub_query)); } } catch (std::exception &e) { return HandleException("Encounter exception when calculte db size", e); diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index d2cb835364..7183f43017 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -110,7 +110,7 @@ Status FaissExecutionEngine::Merge(const std::string& location) { if (location == location_) { return Status::Error("Cannot Merge Self"); } - ENGINE_LOG_DEBUG << "Merge index file: " << location << " to: " << location_; + ENGINE_LOG_DEBUG << "Merge raw file: " << location << " to: " << location_; auto to_merge = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location); if (!to_merge) { diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 1c3c28b4ac..f6c09fb750 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -22,7 +22,7 @@ static constexpr uint64_t ONE_GB = ONE_KB*ONE_MB; static const std::string ARCHIVE_CONF_DISK = "disk"; static const std::string ARCHIVE_CONF_DAYS = "days"; -static const std::string ARCHIVE_CONF_DEFAULT = ARCHIVE_CONF_DISK + ":512"; +static const std::string ARCHIVE_CONF_DEFAULT = ""; struct ArchiveConf { using CriteriaT = std::map; From b30cefc5e5d402072e695eb2bd4ca460ddcc2baf Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sun, 14 Jul 2019 12:02:19 +0800 Subject: [PATCH 05/18] fix milvus single deploy error Former-commit-id: 4fabf7de507cb7eebbc532d3753c4b10cf0d391d --- ci/jenkinsfile/deploy2dev.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy index 26e2aec562..d20f1444bc 100644 --- a/ci/jenkinsfile/deploy2dev.groovy +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -5,7 +5,7 @@ try { dir ("milvus-helm") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) dir ("milvus/milvus-gpu") { - helm install --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.1 . + sh "helm install --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.1 ." } } timeout(time: 2, unit: 'MINUTES') { From 8fd3dd219fdca159ef90e263efe5aca585155894 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sun, 14 Jul 2019 14:43:47 +0800 Subject: [PATCH 06/18] format clean up stage Former-commit-id: cbe735d8a052e33816fba943057877135ed7212a --- ci/jenkinsfile/cleanup_dev.groovy | 6 -- ci/jenkinsfile/cluster_cleanup_dev.groovy | 6 -- ci/jenkinsfile/cluster_dev_test.groovy | 2 +- ci/jenkinsfile/dev_test.groovy | 2 +- ci/main_jenkinsfile | 89 ++++++----------- ci/main_jenkinsfile_no_ut | 111 ++++++++-------------- ci/nightly_main_jenkinsfile | 88 ++++++----------- 7 files changed, 98 insertions(+), 206 deletions(-) diff --git a/ci/jenkinsfile/cleanup_dev.groovy b/ci/jenkinsfile/cleanup_dev.groovy index 056b8fa8c5..664c452f34 100644 --- a/ci/jenkinsfile/cleanup_dev.groovy +++ b/ci/jenkinsfile/cleanup_dev.groovy @@ -1,11 +1,5 @@ try { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - - if (currentBuild.result == 'ABORTED') { - throw new hudson.AbortException("Dev Test Aborted !") - } else if (currentBuild.result == 'FAILURE') { - error("Dev Test Failure !") - } } catch (exc) { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' diff --git a/ci/jenkinsfile/cluster_cleanup_dev.groovy b/ci/jenkinsfile/cluster_cleanup_dev.groovy index dc18067cdd..9708669b9e 100644 --- a/ci/jenkinsfile/cluster_cleanup_dev.groovy +++ b/ci/jenkinsfile/cluster_cleanup_dev.groovy @@ -1,11 +1,5 @@ try { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" - - if (currentBuild.result == 'ABORTED') { - throw new hudson.AbortException("Cluster Dev Test Aborted !") - } else if (currentBuild.result == 'FAILURE') { - error("Dev Test Failure !") - } } catch (exc) { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' diff --git a/ci/jenkinsfile/cluster_dev_test.groovy b/ci/jenkinsfile/cluster_dev_test.groovy index 50fd67f803..d3c6a19832 100644 --- a/ci/jenkinsfile/cluster_dev_test.groovy +++ b/ci/jenkinsfile/cluster_dev_test.groovy @@ -9,8 +9,8 @@ container('milvus-testframework') { } } catch (exc) { updateGitlabCommitStatus name: 'Dev Test', state: 'failed' - currentBuild.result = 'FAILURE' echo 'Milvus Test Failed !' + throw exc } } } diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index 13e1c868b8..4034872615 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -9,8 +9,8 @@ container('milvus-testframework') { } } catch (exc) { updateGitlabCommitStatus name: 'Dev Test', state: 'failed' - currentBuild.result = 'FAILURE' echo 'Milvus Test Failed !' + throw exc } } } diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index a324ff4441..dc5e879817 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -191,38 +191,22 @@ spec: } } } - } - - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + post { + always { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } } } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" } } } @@ -290,38 +274,22 @@ spec: } } } - } - - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + post { + always { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } } } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" } } } @@ -375,4 +343,3 @@ spec: } } } - diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index 203b185e14..35ebc64778 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -191,38 +191,22 @@ spec: } } } - } - - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + post { + always { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } } } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" } } } @@ -290,38 +274,22 @@ spec: } } } - } - - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + post { + always { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } } } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" } } } @@ -337,16 +305,18 @@ spec: post { always { script { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' + if (env.gitlabAfter != null) { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } } } } @@ -373,4 +343,3 @@ spec: } } } - diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index c375cc87ea..7b39ed386c 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -191,38 +191,23 @@ spec: } } } - } - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + post { + always { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } } } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" } } } @@ -290,38 +275,22 @@ spec: } } } - } - - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + post { + always { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } } } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" } } } @@ -373,4 +342,3 @@ spec: } } } - From fdc159c27dab2b00290376fec10c5cf83901bd45 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sun, 14 Jul 2019 14:53:00 +0800 Subject: [PATCH 07/18] fix cleanup bug Former-commit-id: e1a28efe242625615c79b3a1a2c0f7dc3fc1ace5 --- ci/main_jenkinsfile | 44 ++++++++++++++++++++----------------- ci/main_jenkinsfile_no_ut | 44 ++++++++++++++++++++----------------- ci/nightly_main_jenkinsfile | 44 ++++++++++++++++++++----------------- 3 files changed, 72 insertions(+), 60 deletions(-) diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index dc5e879817..cde27f64d0 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -193,17 +193,19 @@ spec: } post { always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' + stages { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } } } } @@ -276,17 +278,19 @@ spec: } post { always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' + stages { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } } } } diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index 35ebc64778..c6a5914baa 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -193,17 +193,19 @@ spec: } post { always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' + stages { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } } } } @@ -276,17 +278,19 @@ spec: } post { always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' + stages { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } } } } diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index 7b39ed386c..bedfce378e 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -194,17 +194,19 @@ spec: post { always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' + stages { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } } } } @@ -277,17 +279,19 @@ spec: } post { always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' + stages { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } } } } From 753ba5a1514f2ccdc7939c1364bed5082c1effe3 Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 14 Jul 2019 15:18:22 +0800 Subject: [PATCH 08/18] Fix SQ8 row count bug Former-commit-id: 7cd10610f84b0398b36909d9cc45bcfa2d77487b --- cpp/src/cache/DataObj.h | 10 ++++++++++ cpp/src/db/DBImpl.cpp | 10 ++++++++-- cpp/src/db/FaissExecutionEngine.cpp | 5 +++-- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/cpp/src/cache/DataObj.h b/cpp/src/cache/DataObj.h index 1dff04027e..995711c6ab 100644 --- a/cpp/src/cache/DataObj.h +++ b/cpp/src/cache/DataObj.h @@ -20,6 +20,11 @@ public: : index_(index) {} + DataObj(const engine::Index_ptr& index, int64_t size) + : index_(index), + size_(size) + {} + engine::Index_ptr data() { return index_; } const engine::Index_ptr& data() const { return index_; } @@ -28,11 +33,16 @@ public: return 0; } + if(size_ > 0) { + return size_; + } + return index_->ntotal*(index_->dim*4); } private: engine::Index_ptr index_ = nullptr; + int64_t size_ = 0; }; using DataObjPtr = std::shared_ptr; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index beae6c30d3..5025c67adf 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -170,7 +170,10 @@ Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, } } - return QueryAsync(table_id, file_id_array, k, nq, vectors, dates, results); + cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info before query + status = QueryAsync(table_id, file_id_array, k, nq, vectors, dates, results); + cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info after query + return status; } Status DBImpl::Query(const std::string& table_id, const std::vector& file_ids, @@ -195,7 +198,10 @@ Status DBImpl::Query(const std::string& table_id, const std::vector return Status::Error("Invalid file id"); } - return QueryAsync(table_id, files_array, k, nq, vectors, dates, results); + cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info before query + status = QueryAsync(table_id, files_array, k, nq, vectors, dates, results); + cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info after query + return status; } Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index 7183f43017..e32363dd1d 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -165,8 +165,9 @@ Status FaissExecutionEngine::Search(long n, } Status FaissExecutionEngine::Cache() { - zilliz::milvus::cache::CpuCacheMgr::GetInstance( - )->InsertItem(location_, std::make_shared(pIndex_)); + auto index = std::make_shared(pIndex_); + cache::DataObjPtr data_obj = std::make_shared(index, PhysicalSize()); + zilliz::milvus::cache::CpuCacheMgr::GetInstance()->InsertItem(location_, data_obj); return Status::OK(); } From c1b312e257f9ce79c7c97e9ae510a70a27de6b0f Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 14 Jul 2019 16:09:09 +0800 Subject: [PATCH 09/18] Fix unittest error Former-commit-id: 106c17b0afc52ddc577c2bc8262b25dcb23f4902 --- cpp/src/db/Options.cpp | 4 ++++ cpp/src/db/scheduler/task/SearchTask.cpp | 2 +- cpp/src/sdk/examples/simple/src/ClientTest.cpp | 14 ++++++++------ cpp/src/server/RequestTask.cpp | 8 +++++++- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index 5f591dcb0f..81f97978ea 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -41,6 +41,10 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) { } for (auto& token : tokens) { + if(token.empty()) { + continue; + } + std::vector kv; boost::algorithm::split(kv, token, boost::is_any_of(":")); if (kv.size() != 2) { diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index 8036cf986d..5606ba2c84 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -107,7 +107,7 @@ Status SearchTask::ClusterResult(const std::vector &output_ids, uint64_t nq, uint64_t topk, SearchContext::ResultSet &result_set) { - if(output_ids.size() != nq*topk || output_distence.size() != nq*topk) { + if(output_ids.size() < nq*topk || output_distence.size() < nq*topk) { std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) + " distance array size: " + std::to_string(output_distence.size()); SERVER_LOG_ERROR << msg; diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index 495e6ae861..f90b6cfe84 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -18,11 +18,12 @@ namespace { static const std::string TABLE_NAME = GetTableName(); static constexpr int64_t TABLE_DIMENSION = 512; - static constexpr int64_t BATCH_ROW_COUNT = 100000; + static constexpr int64_t BATCH_ROW_COUNT = 10000; static constexpr int64_t NQ = 10; static constexpr int64_t TOP_K = 10; static constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different - static constexpr int64_t ADD_VECTOR_LOOP = 10; + static constexpr int64_t ADD_VECTOR_LOOP = 1; + static constexpr int64_t SECONDS_EACH_HOUR = 3600; #define BLOCK_SPLITER std::cout << "===========================================" << std::endl; @@ -59,7 +60,7 @@ namespace { std::string CurrentTime() { time_t tt; time( &tt ); - tt = tt + 8*3600; + tt = tt + 8*SECONDS_EACH_HOUR; tm* t= gmtime( &tt ); std::string str = std::to_string(t->tm_year + 1900) + "_" + std::to_string(t->tm_mon + 1) @@ -69,10 +70,11 @@ namespace { return str; } - std::string CurrentTmDate() { + std::string CurrentTmDate(int64_t offset_day = 0) { time_t tt; time( &tt ); - tt = tt + 8*3600; + tt = tt + 8*SECONDS_EACH_HOUR; + tt = tt + 24*SECONDS_EACH_HOUR*offset_day; tm* t= gmtime( &tt ); std::string str = std::to_string(t->tm_year + 1900) + "-" + std::to_string(t->tm_mon + 1) @@ -160,7 +162,7 @@ namespace { std::vector query_range_array; Range rg; rg.start_value = CurrentTmDate(); - rg.end_value = CurrentTmDate(); + rg.end_value = CurrentTmDate(1); query_range_array.emplace_back(rg); std::vector record_array; diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index 3f42b68531..86681623c0 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -110,7 +110,13 @@ namespace { } long days = (tt_end > tt_start) ? (tt_end - tt_start)/DAY_SECONDS : (tt_start - tt_end)/DAY_SECONDS; - for(long i = 0; i <= days; i++) { + if(days == 0) { + error_code = SERVER_INVALID_TIME_RANGE; + error_msg = "Invalid time range: " + range.start_value + " to " + range.end_value; + return ; + } + + for(long i = 0; i < days; i++) { time_t tt_day = tt_start + DAY_SECONDS*i; tm tm_day; CommonUtil::ConvertTime(tt_day, tm_day); From db65b9a0b4423b1717e9de08c1b312441d665d69 Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 14 Jul 2019 16:10:17 +0800 Subject: [PATCH 10/18] Fix unittest error Former-commit-id: d8dc208227a2904098d899fca71b936a64a528bd --- cpp/src/sdk/examples/simple/src/ClientTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index f90b6cfe84..5caab0961d 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -18,11 +18,11 @@ namespace { static const std::string TABLE_NAME = GetTableName(); static constexpr int64_t TABLE_DIMENSION = 512; - static constexpr int64_t BATCH_ROW_COUNT = 10000; + static constexpr int64_t BATCH_ROW_COUNT = 100000; static constexpr int64_t NQ = 10; static constexpr int64_t TOP_K = 10; static constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different - static constexpr int64_t ADD_VECTOR_LOOP = 1; + static constexpr int64_t ADD_VECTOR_LOOP = 10; static constexpr int64_t SECONDS_EACH_HOUR = 3600; #define BLOCK_SPLITER std::cout << "===========================================" << std::endl; From 14fb1c408ea34704ce7c939dbba4d8bd3c84eb8f Mon Sep 17 00:00:00 2001 From: Jeff Date: Sun, 14 Jul 2019 16:18:13 +0800 Subject: [PATCH 11/18] Revert "fix cleanup bug" This reverts commit 02fb15b7af3f7007aac7c70440db64a56a4244ca [formerly fcdd0de9624858dbaa62280c8fcaad5e37b8c277] Former-commit-id: ae863c72b256fc2917a4b7ad783e7f43bbbf8e38 --- ci/main_jenkinsfile | 44 +++++++++++++++++-------------------- ci/main_jenkinsfile_no_ut | 44 +++++++++++++++++-------------------- ci/nightly_main_jenkinsfile | 44 +++++++++++++++++-------------------- 3 files changed, 60 insertions(+), 72 deletions(-) diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index cde27f64d0..dc5e879817 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -193,19 +193,17 @@ spec: } post { always { - stages { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" } } } @@ -278,19 +276,17 @@ spec: } post { always { - stages { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" - } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" } } } diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index c6a5914baa..35ebc64778 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -193,19 +193,17 @@ spec: } post { always { - stages { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" } } } @@ -278,19 +276,17 @@ spec: } post { always { - stages { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" - } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" } } } diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index bedfce378e..7b39ed386c 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -194,19 +194,17 @@ spec: post { always { - stages { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" } } } @@ -279,19 +277,17 @@ spec: } post { always { - stages { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" - } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" } } } From 6adf5688076110334ed61aaea497313c19372e27 Mon Sep 17 00:00:00 2001 From: Jeff Date: Sun, 14 Jul 2019 16:19:09 +0800 Subject: [PATCH 12/18] Revert "format clean up stage" This reverts commit f2eb4ca5040ce3348695608e9d36e36ad618ccde [formerly 29cd25f823901ecd46fd174fa26ddb57ef3c3620] Former-commit-id: 53d31f7f21b81ab8c110f115e2b99e6fd4645519 --- ci/jenkinsfile/cleanup_dev.groovy | 6 ++ ci/jenkinsfile/cluster_cleanup_dev.groovy | 6 ++ ci/jenkinsfile/cluster_dev_test.groovy | 2 +- ci/jenkinsfile/dev_test.groovy | 2 +- ci/main_jenkinsfile | 93 +++++++++++------ ci/main_jenkinsfile_no_ut | 115 ++++++++++++++-------- ci/nightly_main_jenkinsfile | 92 +++++++++++------ 7 files changed, 212 insertions(+), 104 deletions(-) diff --git a/ci/jenkinsfile/cleanup_dev.groovy b/ci/jenkinsfile/cleanup_dev.groovy index 664c452f34..056b8fa8c5 100644 --- a/ci/jenkinsfile/cleanup_dev.groovy +++ b/ci/jenkinsfile/cleanup_dev.groovy @@ -1,5 +1,11 @@ try { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + + if (currentBuild.result == 'ABORTED') { + throw new hudson.AbortException("Dev Test Aborted !") + } else if (currentBuild.result == 'FAILURE') { + error("Dev Test Failure !") + } } catch (exc) { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' diff --git a/ci/jenkinsfile/cluster_cleanup_dev.groovy b/ci/jenkinsfile/cluster_cleanup_dev.groovy index 9708669b9e..dc18067cdd 100644 --- a/ci/jenkinsfile/cluster_cleanup_dev.groovy +++ b/ci/jenkinsfile/cluster_cleanup_dev.groovy @@ -1,5 +1,11 @@ try { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" + + if (currentBuild.result == 'ABORTED') { + throw new hudson.AbortException("Cluster Dev Test Aborted !") + } else if (currentBuild.result == 'FAILURE') { + error("Dev Test Failure !") + } } catch (exc) { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' diff --git a/ci/jenkinsfile/cluster_dev_test.groovy b/ci/jenkinsfile/cluster_dev_test.groovy index d3c6a19832..50fd67f803 100644 --- a/ci/jenkinsfile/cluster_dev_test.groovy +++ b/ci/jenkinsfile/cluster_dev_test.groovy @@ -9,8 +9,8 @@ container('milvus-testframework') { } } catch (exc) { updateGitlabCommitStatus name: 'Dev Test', state: 'failed' + currentBuild.result = 'FAILURE' echo 'Milvus Test Failed !' - throw exc } } } diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index 4034872615..13e1c868b8 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -9,8 +9,8 @@ container('milvus-testframework') { } } catch (exc) { updateGitlabCommitStatus name: 'Dev Test', state: 'failed' + currentBuild.result = 'FAILURE' echo 'Milvus Test Failed !' - throw exc } } } diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index dc5e879817..a324ff4441 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -191,22 +191,38 @@ spec: } } } + } + + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + } + } post { - always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" } } } @@ -274,22 +290,38 @@ spec: } } } + } + + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } + } + } + } + } post { - always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" - } - } - } + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" } } } @@ -343,3 +375,4 @@ spec: } } } + diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index 35ebc64778..203b185e14 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -191,22 +191,38 @@ spec: } } } + } + + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + } + } post { - always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" } } } @@ -274,22 +290,38 @@ spec: } } } + } + + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } + } + } + } + } post { - always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" - } - } - } + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" } } } @@ -305,18 +337,16 @@ spec: post { always { script { - if (env.gitlabAfter != null) { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' - } + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' } } } @@ -343,3 +373,4 @@ spec: } } } + diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index 7b39ed386c..c375cc87ea 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -191,23 +191,38 @@ spec: } } } + } + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + } + } post { - always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" } } } @@ -275,22 +290,38 @@ spec: } } } + } + + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } + } + } + } + } post { - always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" - } - } - } + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" } } } @@ -342,3 +373,4 @@ spec: } } } + From 46d64af09c567695bb513e0d0dc615d120a5dd3a Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 14 Jul 2019 16:46:05 +0800 Subject: [PATCH 13/18] Fix small size index build crash Former-commit-id: d1255df5fc62aa037f4670049cccfbb8a9851732 --- cpp/src/wrapper/Operand.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/cpp/src/wrapper/Operand.cpp b/cpp/src/wrapper/Operand.cpp index 62a863ae0e..49d007ba2a 100644 --- a/cpp/src/wrapper/Operand.cpp +++ b/cpp/src/wrapper/Operand.cpp @@ -28,6 +28,15 @@ IndexType resolveIndexType(const string &index_type) { return IndexType::Invalid_Option; } +int CalcBacketCount(int nb, size_t nlist) { + int backet_count = int(nb / 1000000.0 * nlist); + if(backet_count == 0) { + backet_count = 1; //avoid faiss rash + } + + return backet_count; +} + // nb at least 100 string Operand::get_index_type(const int &nb) { if (!index_str.empty()) { return index_str; } @@ -45,7 +54,7 @@ string Operand::get_index_type(const int &nb) { size_t nlist = engine_config.GetInt32Value(CONFIG_NLIST, 16384); index_str += (ncent != 0 ? index_type + std::to_string(ncent) : - index_type + std::to_string(int(nb / 1000000.0 * nlist))); + index_type + std::to_string(CalcBacketCount(nb, nlist))); // std::cout<<"nlist = "< Date: Sun, 14 Jul 2019 20:43:32 +0800 Subject: [PATCH 15/18] fix labels error Former-commit-id: c3e6ca2d5ff47b79e0210f9f5475974cc3e050d9 --- ci/main_jenkinsfile | 4 ++-- ci/main_jenkinsfile_no_ut | 4 ++-- ci/nightly_main_jenkinsfile | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index 8c636a7981..9577837bfd 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -134,7 +134,7 @@ spec: stage("Single Node") { agent { kubernetes { - label 'dev test' + label 'dev-test' defaultContainer 'jnlp' yaml """ apiVersion: v1 @@ -227,7 +227,7 @@ spec: stage("Cluster") { agent { kubernetes { - label 'dev test' + label 'dev-test' defaultContainer 'jnlp' yaml """ apiVersion: v1 diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index 0642a3723b..082433650c 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -134,7 +134,7 @@ spec: stage("Single Node") { agent { kubernetes { - label 'dev test' + label 'dev-test' defaultContainer 'jnlp' yaml """ apiVersion: v1 @@ -227,7 +227,7 @@ spec: stage("Cluster") { agent { kubernetes { - label 'dev test' + label 'dev-test' defaultContainer 'jnlp' yaml """ apiVersion: v1 diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index 89b4a14985..7f59561f82 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -134,7 +134,7 @@ spec: stage("Single Node") { agent { kubernetes { - label 'dev test' + label 'dev-test' defaultContainer 'jnlp' yaml """ apiVersion: v1 @@ -227,7 +227,7 @@ spec: stage("Cluster") { agent { kubernetes { - label 'dev test' + label 'dev-test' defaultContainer 'jnlp' yaml """ apiVersion: v1 From 2abe69ffb51f1b77e4fd3c551ee0bf061f117fb0 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sun, 14 Jul 2019 21:08:49 +0800 Subject: [PATCH 16/18] fix clean up bug Former-commit-id: 041f4f5f8beb650d9d0277d4745ef0d1b8820866 --- ci/jenkinsfile/cleanup_dev.groovy | 6 +++--- ci/jenkinsfile/cluster_cleanup_dev.groovy | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/jenkinsfile/cleanup_dev.groovy b/ci/jenkinsfile/cleanup_dev.groovy index 1b7e67e1b0..2e9332fa6e 100644 --- a/ci/jenkinsfile/cleanup_dev.groovy +++ b/ci/jenkinsfile/cleanup_dev.groovy @@ -1,11 +1,11 @@ try { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true - if (result) { + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" } } catch (exc) { def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true - if (result) { + if (!result) { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" } throw exc diff --git a/ci/jenkinsfile/cluster_cleanup_dev.groovy b/ci/jenkinsfile/cluster_cleanup_dev.groovy index 9a73d0a291..e57988fefe 100644 --- a/ci/jenkinsfile/cluster_cleanup_dev.groovy +++ b/ci/jenkinsfile/cluster_cleanup_dev.groovy @@ -1,11 +1,11 @@ try { def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true - if (result) { + if (!result) { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" } } catch (exc) { def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true - if (result) { + if (!result) { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" } throw exc From 2428ff11608f567744f3b76e68810045369b3b50 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 15 Jul 2019 09:25:55 +0800 Subject: [PATCH 17/18] Fix unittest error Former-commit-id: 9185fecbe5161694c6b741ad53fdccf89857899a --- cpp/unittest/db/db_tests.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 729317011d..a8f5d2bf8d 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -52,15 +52,13 @@ TEST_F(DBTest, CONFIG_TEST) { engine::ArchiveConf conf("delete"); ASSERT_EQ(conf.GetType(), "delete"); auto criterias = conf.GetCriterias(); - ASSERT_TRUE(criterias.size() == 1); - ASSERT_TRUE(criterias["disk"] == 512); + ASSERT_TRUE(criterias.size() == 0); } { engine::ArchiveConf conf("swap"); ASSERT_EQ(conf.GetType(), "swap"); auto criterias = conf.GetCriterias(); - ASSERT_TRUE(criterias.size() == 1); - ASSERT_TRUE(criterias["disk"] == 512); + ASSERT_TRUE(criterias.size() == 0); } { ASSERT_ANY_THROW(engine::ArchiveConf conf1("swap", "disk:")); From 2dffb947a94e275ca42138fed36044cf79965277 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 15 Jul 2019 10:29:28 +0800 Subject: [PATCH 18/18] add more unitest cases Former-commit-id: 6367f0601b449098514ac4770eb72daa81d91607 --- cpp/unittest/db/db_tests.cpp | 18 ++++++++++++++---- cpp/unittest/db/misc_test.cpp | 29 +++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index a8f5d2bf8d..c9a9d89780 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -204,11 +204,21 @@ TEST_F(DBTest, SEARCH_TEST) { ASSERT_STATS(stat); } - sleep(2); // wait until build index finish + db_->BuildIndex(TABLE_NAME); // wait until build index finish - engine::QueryResults results; - stat = db_->Query(TABLE_NAME, k, nq, xq.data(), results); - ASSERT_STATS(stat); + { + engine::QueryResults results; + stat = db_->Query(TABLE_NAME, k, nq, xq.data(), results); + ASSERT_STATS(stat); + } + + {//search by specify index file + engine::meta::DatesT dates; + std::vector file_ids = {"1", "2", "3", "4"}; + engine::QueryResults results; + stat = db_->Query(TABLE_NAME, file_ids, k, nq, xq.data(), dates, results); + ASSERT_STATS(stat); + } // TODO(linxj): add groundTruth assert }; diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index 4356746fc2..9dd07fac8d 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -14,6 +14,7 @@ #include "db/Options.h" #include "db/DBMetaImpl.h" #include "db/EngineFactory.h" +#include "db/Utils.h" #include @@ -134,4 +135,32 @@ TEST(DBMiscTest, META_TEST) { int delta = 10; engine::meta::DateT dt = impl.GetDate(tt, delta); ASSERT_GT(dt, 0); +} + +TEST(DBMiscTest, UTILS_TEST) { + engine::DBMetaOptions options; + options.path = "/tmp/milvus_test/main"; + options.slave_paths.push_back("/tmp/milvus_test/slave_1"); + options.slave_paths.push_back("/tmp/milvus_test/slave_2"); + + const std::string TABLE_NAME = "test_tbl"; + auto status = engine::utils::CreateTablePath(options, TABLE_NAME); + ASSERT_TRUE(status.ok()); + ASSERT_TRUE(boost::filesystem::exists(options.path)); + for(auto& path : options.slave_paths) { + ASSERT_TRUE(boost::filesystem::exists(path)); + } + + engine::meta::TableFileSchema file; + file.id_ = 50; + file.table_id_ = TABLE_NAME; + file.file_type_ = 3; + file.date_ = 155000; + status = engine::utils::GetTableFilePath(options, file); + ASSERT_FALSE(status.ok()); + ASSERT_TRUE(file.location_.empty()); + + status = engine::utils::DeleteTablePath(options, TABLE_NAME); + ASSERT_TRUE(status.ok()); + } \ No newline at end of file