From 28ff29ac342f42f0f9ba9154ae6f36aa78dc9ecb Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Wed, 20 Nov 2019 16:06:26 +0800 Subject: [PATCH 1/4] [skip ci] Split test reports into two folders #255 --- .../milvus_ivfsq8_test_report_detailed_version_cn.md | 0 .../milvus_ivfsq8h_test_report_detailed_version_cn.md | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename docs/{test_report => test_report_cn}/milvus_ivfsq8_test_report_detailed_version_cn.md (100%) rename docs/{test_report => test_report_cn}/milvus_ivfsq8h_test_report_detailed_version_cn.md (100%) diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md b/docs/test_report_cn/milvus_ivfsq8_test_report_detailed_version_cn.md similarity index 100% rename from docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md rename to docs/test_report_cn/milvus_ivfsq8_test_report_detailed_version_cn.md diff --git a/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md b/docs/test_report_cn/milvus_ivfsq8h_test_report_detailed_version_cn.md similarity index 100% rename from docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md rename to docs/test_report_cn/milvus_ivfsq8h_test_report_detailed_version_cn.md From 78a4b40cef7cad0a684682321d2a967b60d15935 Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Wed, 20 Nov 2019 16:28:57 +0800 Subject: [PATCH 2/4] [skip ci] Change test_report name #255 --- ...s_ivfsq8_test_report_detailed_version_cn.md => ivfsq8_test.md} | 0 ...ivfsq8h_test_report_detailed_version_cn.md => ivfsq8h_test.md} | 0 .../ivfsq8_test.md} | 0 .../ivfsq8h_test.md} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename docs/test_report_cn/{milvus_ivfsq8_test_report_detailed_version_cn.md => ivfsq8_test.md} (100%) rename docs/test_report_cn/{milvus_ivfsq8h_test_report_detailed_version_cn.md => ivfsq8h_test.md} (100%) rename docs/{test_report/milvus_ivfsq8_test_report_detailed_version.md => test_report_en/ivfsq8_test.md} (100%) rename docs/{test_report/milvus_ivfsq8h_test_report_detailed_version.md => test_report_en/ivfsq8h_test.md} (100%) diff --git a/docs/test_report_cn/milvus_ivfsq8_test_report_detailed_version_cn.md b/docs/test_report_cn/ivfsq8_test.md similarity index 100% rename from docs/test_report_cn/milvus_ivfsq8_test_report_detailed_version_cn.md rename to docs/test_report_cn/ivfsq8_test.md diff --git a/docs/test_report_cn/milvus_ivfsq8h_test_report_detailed_version_cn.md b/docs/test_report_cn/ivfsq8h_test.md similarity index 100% rename from docs/test_report_cn/milvus_ivfsq8h_test_report_detailed_version_cn.md rename to docs/test_report_cn/ivfsq8h_test.md diff --git a/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md b/docs/test_report_en/ivfsq8_test.md similarity index 100% rename from docs/test_report/milvus_ivfsq8_test_report_detailed_version.md rename to docs/test_report_en/ivfsq8_test.md diff --git a/docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md b/docs/test_report_en/ivfsq8h_test.md similarity index 100% rename from docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md rename to docs/test_report_en/ivfsq8h_test.md From 47abe4ff4749a378ea6fc7b11a05afcdea3ef47c Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Wed, 27 Nov 2019 14:21:53 +0800 Subject: [PATCH 3/4] #504 The code coverage rate of core/src/scheduler/optimizer is too low --- CHANGELOG.md | 1 + core/src/db/engine/ExecutionEngine.h | 4 +- core/src/db/engine/ExecutionEngineImpl.cpp | 29 +++++------- core/src/db/engine/ExecutionEngineImpl.h | 4 +- .../index/vector_index/IndexGPUIDMAP.cpp | 20 ++++----- .../index/vector_index/IndexGPUIDMAP.h | 4 +- .../index/vector_index/IndexGPUIVF.cpp | 10 ++--- .../knowhere/index/vector_index/IndexGPUIVF.h | 4 +- .../index/vector_index/IndexIDMAP.cpp | 18 ++++---- .../knowhere/index/vector_index/IndexIDMAP.h | 4 +- .../knowhere/index/vector_index/IndexIVF.cpp | 28 ++++++------ .../knowhere/index/vector_index/IndexIVF.h | 8 ++-- .../index/vector_index/IndexIVFPQ.cpp | 8 ++-- .../knowhere/index/vector_index/IndexIVFPQ.h | 4 +- .../index/vector_index/IndexIVFSQ.cpp | 8 ++-- .../knowhere/index/vector_index/IndexIVFSQ.h | 4 +- .../knowhere/index/vector_index/IndexNSG.cpp | 8 ++-- .../knowhere/index/vector_index/IndexNSG.h | 4 +- .../index/vector_index/IndexSPTAG.cpp | 14 ++++-- .../knowhere/index/vector_index/IndexSPTAG.h | 4 +- .../knowhere/index/vector_index/VectorIndex.h | 4 +- .../vector_index/helpers/IndexParameter.h | 8 ++-- core/src/index/unittest/test_idmap.cpp | 12 ++--- core/src/index/unittest/test_ivf.cpp | 1 + core/src/index/unittest/test_nsg/test_nsg.cpp | 4 +- .../scheduler/optimizer/BuildIndexPass.cpp | 5 +-- core/src/scheduler/optimizer/BuildIndexPass.h | 2 + .../src/scheduler/optimizer/FaissFlatPass.cpp | 5 +-- core/src/scheduler/optimizer/FaissFlatPass.h | 2 + .../scheduler/optimizer/FaissIVFFlatPass.cpp | 3 +- .../scheduler/optimizer/FaissIVFFlatPass.h | 2 + .../scheduler/optimizer/FaissIVFSQ8HPass.cpp | 7 ++- .../scheduler/optimizer/FaissIVFSQ8HPass.h | 2 + .../scheduler/optimizer/FaissIVFSQ8Pass.cpp | 3 +- .../src/scheduler/optimizer/FaissIVFSQ8Pass.h | 2 + core/src/wrapper/VecImpl.cpp | 14 +++--- core/src/wrapper/VecImpl.h | 4 +- core/src/wrapper/VecIndex.h | 4 +- core/unittest/db/test_db.cpp | 44 +++++++++++++++++++ core/unittest/db/test_engine.cpp | 10 ++--- core/unittest/scheduler/test_scheduler.cpp | 8 ++-- core/unittest/server/test_cache.cpp | 10 ++--- core/unittest/wrapper/test_wrapper.cpp | 12 ++++- 43 files changed, 214 insertions(+), 142 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7083cc12eb..c3e2f0153d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#458 - Index data is not compatible between 0.5 and 0.6 - \#465 - Server hang caused by searching with nsg index - \#486 - gpu no usage during index building +- \#504 - The code coverage rate of core/src/scheduler/optimizer is too low - \#509 - IVF_PQ index build trapped into dead loop caused by invalid params - \#513 - Unittest DELETE_BY_RANGE sometimes failed - \#523 - Erase file data from cache once the file is marked as deleted diff --git a/core/src/db/engine/ExecutionEngine.h b/core/src/db/engine/ExecutionEngine.h index c8784e8a90..9f3e6a86db 100644 --- a/core/src/db/engine/ExecutionEngine.h +++ b/core/src/db/engine/ExecutionEngine.h @@ -77,8 +77,8 @@ class ExecutionEngine { virtual Status CopyToCpu() = 0; - virtual std::shared_ptr - Clone() = 0; + // virtual std::shared_ptr + // Clone() = 0; virtual Status Merge(const std::string& location) = 0; diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 0f92b34eca..d1f21ebc42 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -257,11 +257,6 @@ ExecutionEngineImpl::PhysicalSize() const { Status ExecutionEngineImpl::Serialize() { auto status = write_index(index_, location_); - - // here we reset index size by file size, - // since some index type(such as SQ8) data size become smaller after serialized - index_->set_size(PhysicalSize()); - return status; } @@ -415,18 +410,18 @@ ExecutionEngineImpl::CopyToCpu() { return Status::OK(); } -ExecutionEnginePtr -ExecutionEngineImpl::Clone() { - if (index_ == nullptr) { - ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to clone"; - return nullptr; - } - - auto ret = std::make_shared(dim_, location_, index_type_, metric_type_, nlist_); - ret->Init(); - ret->index_ = index_->Clone(); - return ret; -} +// ExecutionEnginePtr +// ExecutionEngineImpl::Clone() { +// if (index_ == nullptr) { +// ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to clone"; +// return nullptr; +// } +// +// auto ret = std::make_shared(dim_, location_, index_type_, metric_type_, nlist_); +// ret->Init(); +// ret->index_ = index_->Clone(); +// return ret; +//} Status ExecutionEngineImpl::Merge(const std::string& location) { diff --git a/core/src/db/engine/ExecutionEngineImpl.h b/core/src/db/engine/ExecutionEngineImpl.h index da0e7cfb64..84604f519e 100644 --- a/core/src/db/engine/ExecutionEngineImpl.h +++ b/core/src/db/engine/ExecutionEngineImpl.h @@ -64,8 +64,8 @@ class ExecutionEngineImpl : public ExecutionEngine { Status CopyToCpu() override; - ExecutionEnginePtr - Clone() override; + // ExecutionEnginePtr + // Clone() override; Status Merge(const std::string& location) override; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp index 1aded3ddaa..3193b1e5c8 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp @@ -47,16 +47,16 @@ GPUIDMAP::CopyGpuToCpu(const Config& config) { return std::make_shared(new_index); } -VectorIndexPtr -GPUIDMAP::Clone() { - auto cpu_idx = CopyGpuToCpu(Config()); - - if (auto idmap = std::dynamic_pointer_cast(cpu_idx)) { - return idmap->CopyCpuToGpu(gpu_id_, Config()); - } else { - KNOWHERE_THROW_MSG("IndexType not Support GpuClone"); - } -} +// VectorIndexPtr +// GPUIDMAP::Clone() { +// auto cpu_idx = CopyGpuToCpu(Config()); +// +// if (auto idmap = std::dynamic_pointer_cast(cpu_idx)) { +// return idmap->CopyCpuToGpu(gpu_id_, Config()); +// } else { +// KNOWHERE_THROW_MSG("IndexType not Support GpuClone"); +// } +//} BinarySet GPUIDMAP::SerializeImpl() { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.h index f1dfe2f21a..d41e3479e0 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.h @@ -41,8 +41,8 @@ class GPUIDMAP : public IDMAP, public GPUIndex { int64_t* GetRawIds() override; - VectorIndexPtr - Clone() override; + // VectorIndexPtr + // Clone() override; VectorIndexPtr CopyGpuToGpu(const int64_t& device_id, const Config& config) override; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp index d69f87a061..923ca0db56 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp @@ -158,11 +158,11 @@ GPUIVF::CopyGpuToCpu(const Config& config) { } } -VectorIndexPtr -GPUIVF::Clone() { - auto cpu_idx = CopyGpuToCpu(Config()); - return knowhere::cloner::CopyCpuToGpu(cpu_idx, gpu_id_, Config()); -} +// VectorIndexPtr +// GPUIVF::Clone() { +// auto cpu_idx = CopyGpuToCpu(Config()); +// return knowhere::cloner::CopyCpuToGpu(cpu_idx, gpu_id_, Config()); +//} VectorIndexPtr GPUIVF::CopyGpuToGpu(const int64_t& device_id, const Config& config) { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.h index fa9a206c48..a27f61d693 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVF.h @@ -75,8 +75,8 @@ class GPUIVF : public IVF, public GPUIndex { VectorIndexPtr CopyGpuToGpu(const int64_t& device_id, const Config& config) override; - VectorIndexPtr - Clone() final; + // VectorIndexPtr + // Clone() final; protected: void diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp index 7aedf98613..11d434ea9e 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp @@ -164,15 +164,15 @@ IDMAP::Train(const Config& config) { index_.reset(index); } -VectorIndexPtr -IDMAP::Clone() { - std::lock_guard lk(mutex_); - - auto clone_index = faiss::clone_index(index_.get()); - std::shared_ptr new_index; - new_index.reset(clone_index); - return std::make_shared(new_index); -} +// VectorIndexPtr +// IDMAP::Clone() { +// std::lock_guard lk(mutex_); +// +// auto clone_index = faiss::clone_index(index_.get()); +// std::shared_ptr new_index; +// new_index.reset(clone_index); +// return std::make_shared(new_index); +//} VectorIndexPtr IDMAP::CopyCpuToGpu(const int64_t& device_id, const Config& config) { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h index 0f66e8fac0..ac85ac0319 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h @@ -42,8 +42,8 @@ class IDMAP : public VectorIndex, public FaissBaseIndex { Search(const DatasetPtr& dataset, const Config& config) override; int64_t Count() override; - VectorIndexPtr - Clone() override; + // VectorIndexPtr + // Clone() override; int64_t Dimension() override; void diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index 8b734abdc6..f99c6e3972 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -257,20 +257,20 @@ IVF::CopyCpuToGpu(const int64_t& device_id, const Config& config) { #endif } -VectorIndexPtr -IVF::Clone() { - std::lock_guard lk(mutex_); - - auto clone_index = faiss::clone_index(index_.get()); - std::shared_ptr new_index; - new_index.reset(clone_index); - return Clone_impl(new_index); -} - -VectorIndexPtr -IVF::Clone_impl(const std::shared_ptr& index) { - return std::make_shared(index); -} +// VectorIndexPtr +// IVF::Clone() { +// std::lock_guard lk(mutex_); +// +// auto clone_index = faiss::clone_index(index_.get()); +// std::shared_ptr new_index; +// new_index.reset(clone_index); +// return Clone_impl(new_index); +//} +// +// VectorIndexPtr +// IVF::Clone_impl(const std::shared_ptr& index) { +// return std::make_shared(index); +//} void IVF::Seal() { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h index e064b6f08c..83ef4bf3fe 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h @@ -38,8 +38,8 @@ class IVF : public VectorIndex, public FaissBaseIndex { explicit IVF(std::shared_ptr index) : FaissBaseIndex(std::move(index)) { } - VectorIndexPtr - Clone() override; + // VectorIndexPtr + // Clone() override; IndexModelPtr Train(const DatasetPtr& dataset, const Config& config) override; @@ -81,8 +81,8 @@ class IVF : public VectorIndex, public FaissBaseIndex { virtual std::shared_ptr GenParams(const Config& config); - virtual VectorIndexPtr - Clone_impl(const std::shared_ptr& index); + // virtual VectorIndexPtr + // Clone_impl(const std::shared_ptr& index); virtual void search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp index 841ef63245..5622e31a11 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp @@ -63,10 +63,10 @@ IVFPQ::GenParams(const Config& config) { return params; } -VectorIndexPtr -IVFPQ::Clone_impl(const std::shared_ptr& index) { - return std::make_shared(index); -} +// VectorIndexPtr +// IVFPQ::Clone_impl(const std::shared_ptr& index) { +// return std::make_shared(index); +//} VectorIndexPtr IVFPQ::CopyCpuToGpu(const int64_t& device_id, const Config& config) { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.h index fc50c68389..2e3192e2ca 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.h @@ -41,8 +41,8 @@ class IVFPQ : public IVF { std::shared_ptr GenParams(const Config& config) override; - VectorIndexPtr - Clone_impl(const std::shared_ptr& index) override; + // VectorIndexPtr + // Clone_impl(const std::shared_ptr& index) override; }; } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp index 7c86cd4dbd..273061d20d 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp @@ -54,10 +54,10 @@ IVFSQ::Train(const DatasetPtr& dataset, const Config& config) { return std::make_shared(ret_index); } -VectorIndexPtr -IVFSQ::Clone_impl(const std::shared_ptr& index) { - return std::make_shared(index); -} +// VectorIndexPtr +// IVFSQ::Clone_impl(const std::shared_ptr& index) { +// return std::make_shared(index); +//} VectorIndexPtr IVFSQ::CopyCpuToGpu(const int64_t& device_id, const Config& config) { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.h index cac95faebf..0dcb6555f6 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.h @@ -38,8 +38,8 @@ class IVFSQ : public IVF { CopyCpuToGpu(const int64_t& device_id, const Config& config) override; protected: - VectorIndexPtr - Clone_impl(const std::shared_ptr& index) override; + // VectorIndexPtr + // Clone_impl(const std::shared_ptr& index) override; }; } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp index 3cf0122233..16c0b9172f 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp @@ -158,10 +158,10 @@ NSG::Dimension() { return index_->dimension; } -VectorIndexPtr -NSG::Clone() { - KNOWHERE_THROW_MSG("not support"); -} +// VectorIndexPtr +// NSG::Clone() { +// KNOWHERE_THROW_MSG("not support"); +//} void NSG::Seal() { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.h index 04a146d58a..1af12b2ae1 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.h @@ -49,8 +49,8 @@ class NSG : public VectorIndex { Count() override; int64_t Dimension() override; - VectorIndexPtr - Clone() override; + // VectorIndexPtr + // Clone() override; void Seal() override; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp index 17a93fdcc7..9d1d693c14 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp @@ -210,6 +210,9 @@ CPUSPTAGRNG::Load(const BinarySet& binary_set) { IndexModelPtr CPUSPTAGRNG::Train(const DatasetPtr& origin, const Config& train_config) { SetParameters(train_config); + if (train_config != nullptr) { + train_config->CheckValid(); // throw exception + } DatasetPtr dataset = origin->Clone(); // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine @@ -295,6 +298,9 @@ CPUSPTAGRNG::SetParameters(const Config& config) { DatasetPtr CPUSPTAGRNG::Search(const DatasetPtr& dataset, const Config& config) { SetParameters(config); + if (config != nullptr) { + config->CheckValid(); // throw exception + } auto tensor = dataset->tensor()[0]; auto p = (float*)tensor->raw_mutable_data(); for (auto i = 0; i < 10; ++i) { @@ -325,10 +331,10 @@ CPUSPTAGRNG::Dimension() { return index_ptr_->GetFeatureDim(); } -VectorIndexPtr -CPUSPTAGRNG::Clone() { - KNOWHERE_THROW_MSG("not support"); -} +// VectorIndexPtr +// CPUSPTAGRNG::Clone() { +// KNOWHERE_THROW_MSG("not support"); +//} void CPUSPTAGRNG::Seal() { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h index 01380ce943..92942dd24f 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h @@ -36,8 +36,8 @@ class CPUSPTAGRNG : public VectorIndex { BinarySet Serialize() override; - VectorIndexPtr - Clone() override; + // VectorIndexPtr + // Clone() override; void Load(const BinarySet& index_array) override; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/VectorIndex.h b/core/src/index/knowhere/knowhere/index/vector_index/VectorIndex.h index 6509458b7b..6626d9d94d 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/VectorIndex.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/VectorIndex.h @@ -49,8 +49,8 @@ class VectorIndex : public Index { Seal() = 0; // TODO(linxj): Deprecated - virtual VectorIndexPtr - Clone() = 0; + // virtual VectorIndexPtr + // Clone() = 0; virtual int64_t Count() = 0; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h index e30088ecdf..2a274d474f 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h @@ -180,10 +180,10 @@ struct SPTAGCfg : public Cfg { SPTAGCfg() = default; - bool - CheckValid() override { - return true; - }; + // bool + // CheckValid() override { + // return true; + // }; }; using SPTAGConfig = std::shared_ptr; diff --git a/core/src/index/unittest/test_idmap.cpp b/core/src/index/unittest/test_idmap.cpp index 98a554199f..1ed750d9a8 100644 --- a/core/src/index/unittest/test_idmap.cpp +++ b/core/src/index/unittest/test_idmap.cpp @@ -139,9 +139,9 @@ TEST_F(IDMAPTest, copy_test) { { // clone - auto clone_index = index_->Clone(); - auto clone_result = clone_index->Search(query_dataset, conf); - AssertAnns(clone_result, nq, k); + // auto clone_index = index_->Clone(); + // auto clone_result = clone_index->Search(query_dataset, conf); + // AssertAnns(clone_result, nq, k); } { @@ -159,9 +159,9 @@ TEST_F(IDMAPTest, copy_test) { auto new_result = clone_index->Search(query_dataset, conf); AssertAnns(new_result, nq, k); - auto clone_gpu_idx = clone_index->Clone(); - auto clone_gpu_res = clone_gpu_idx->Search(query_dataset, conf); - AssertAnns(clone_gpu_res, nq, k); + // auto clone_gpu_idx = clone_index->Clone(); + // auto clone_gpu_res = clone_gpu_idx->Search(query_dataset, conf); + // AssertAnns(clone_gpu_res, nq, k); // gpu to cpu auto host_index = knowhere::cloner::CopyGpuToCpu(clone_index, conf); diff --git a/core/src/index/unittest/test_ivf.cpp b/core/src/index/unittest/test_ivf.cpp index 79ef1aa170..7438d5656a 100644 --- a/core/src/index/unittest/test_ivf.cpp +++ b/core/src/index/unittest/test_ivf.cpp @@ -62,6 +62,7 @@ class IVFTest : public DataGen, public TestWithParam<::std::tupleDump(); } void diff --git a/core/src/index/unittest/test_nsg/test_nsg.cpp b/core/src/index/unittest/test_nsg/test_nsg.cpp index 4722c7e8f6..5de826fabe 100644 --- a/core/src/index/unittest/test_nsg/test_nsg.cpp +++ b/core/src/index/unittest/test_nsg/test_nsg.cpp @@ -57,11 +57,13 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test { tmp_conf->candidate_pool_size = 100; tmp_conf->metric_type = knowhere::METRICTYPE::L2; train_conf = tmp_conf; + train_conf->Dump(); auto tmp2_conf = std::make_shared(); tmp2_conf->k = k; tmp2_conf->search_length = 30; search_conf = tmp2_conf; + search_conf->Dump(); } void @@ -92,7 +94,7 @@ TEST_F(NSGInterfaceTest, basic_test) { ASSERT_EQ(index_->Count(), nb); ASSERT_EQ(index_->Dimension(), dim); - ASSERT_THROW({ index_->Clone(); }, knowhere::KnowhereException); + // ASSERT_THROW({ index_->Clone(); }, knowhere::KnowhereException); ASSERT_NO_THROW({ index_->Add(base_dataset, knowhere::Config()); index_->Seal(); diff --git a/core/src/scheduler/optimizer/BuildIndexPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp index 67036f9d7b..770cfd333c 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.cpp +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -19,19 +19,17 @@ #include "scheduler/SchedInst.h" #include "scheduler/Utils.h" #include "scheduler/tasklabel/SpecResLabel.h" - +#ifdef MILVUS_GPU_VERSION namespace milvus { namespace scheduler { void BuildIndexPass::Init() { -#ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); Status s = config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids_); if (!s.ok()) { throw; } -#endif } bool @@ -56,3 +54,4 @@ BuildIndexPass::Run(const TaskPtr& task) { } // namespace scheduler } // namespace milvus +#endif diff --git a/core/src/scheduler/optimizer/BuildIndexPass.h b/core/src/scheduler/optimizer/BuildIndexPass.h index 3adf1259a7..8be1a2f3cf 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.h +++ b/core/src/scheduler/optimizer/BuildIndexPass.h @@ -14,6 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +#ifdef MILVUS_GPU_VERSION #pragma once #include @@ -52,3 +53,4 @@ using BuildIndexPassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus +#endif diff --git a/core/src/scheduler/optimizer/FaissFlatPass.cpp b/core/src/scheduler/optimizer/FaissFlatPass.cpp index 53256f7790..15ddc62f55 100644 --- a/core/src/scheduler/optimizer/FaissFlatPass.cpp +++ b/core/src/scheduler/optimizer/FaissFlatPass.cpp @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - +#ifdef MILVUS_GPU_VERSION #include "scheduler/optimizer/FaissFlatPass.h" #include "cache/GpuCacheMgr.h" #include "scheduler/SchedInst.h" @@ -29,7 +29,6 @@ namespace scheduler { void FaissFlatPass::Init() { -#ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { @@ -39,7 +38,6 @@ FaissFlatPass::Init() { if (!s.ok()) { throw; } -#endif } bool @@ -71,3 +69,4 @@ FaissFlatPass::Run(const TaskPtr& task) { } // namespace scheduler } // namespace milvus +#endif diff --git a/core/src/scheduler/optimizer/FaissFlatPass.h b/core/src/scheduler/optimizer/FaissFlatPass.h index f219bebdf3..a6d7b7320a 100644 --- a/core/src/scheduler/optimizer/FaissFlatPass.h +++ b/core/src/scheduler/optimizer/FaissFlatPass.h @@ -14,6 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +#ifdef MILVUS_GPU_VERSION #pragma once #include @@ -54,3 +55,4 @@ using FaissFlatPassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus +#endif diff --git a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp index 8ad51fb14a..70909e8cd1 100644 --- a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - +#ifdef MILVUS_GPU_VERSION #include "scheduler/optimizer/FaissIVFFlatPass.h" #include "cache/GpuCacheMgr.h" #include "scheduler/SchedInst.h" @@ -72,3 +72,4 @@ FaissIVFFlatPass::Run(const TaskPtr& task) { } // namespace scheduler } // namespace milvus +#endif diff --git a/core/src/scheduler/optimizer/FaissIVFFlatPass.h b/core/src/scheduler/optimizer/FaissIVFFlatPass.h index 2d15539014..7a8fca8ef8 100644 --- a/core/src/scheduler/optimizer/FaissIVFFlatPass.h +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.h @@ -14,6 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +#ifdef MILVUS_GPU_VERSION #pragma once #include @@ -54,3 +55,4 @@ using FaissIVFFlatPassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus +#endif diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp index cc04944fda..7bcee6b07e 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - +#ifdef MILVUS_GPU_VERSION #include "scheduler/optimizer/FaissIVFSQ8HPass.h" #include "cache/GpuCacheMgr.h" #include "scheduler/SchedInst.h" @@ -29,7 +29,7 @@ namespace scheduler { void FaissIVFSQ8HPass::Init() { -#ifdef MILVUS_GPU_VERSION +#ifdef CUSTOMIZATION server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { @@ -41,6 +41,7 @@ FaissIVFSQ8HPass::Init() { bool FaissIVFSQ8HPass::Run(const TaskPtr& task) { +#ifdef CUSTOMIZATION if (task->Type() != TaskType::SearchTask) { return false; } @@ -65,7 +66,9 @@ FaissIVFSQ8HPass::Run(const TaskPtr& task) { auto label = std::make_shared(res_ptr); task->label() = label; return true; +#endif } } // namespace scheduler } // namespace milvus +#endif diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h index 0d2892809f..4e1e37730e 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h @@ -14,6 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +#ifdef MILVUS_GPU_VERSION #pragma once #include @@ -54,3 +55,4 @@ using FaissIVFSQ8HPassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus +#endif diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp index c2a1858966..7df917000c 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - +#ifdef MILVUS_GPU_VERSION #include "scheduler/optimizer/FaissIVFSQ8Pass.h" #include "cache/GpuCacheMgr.h" #include "scheduler/SchedInst.h" @@ -72,3 +72,4 @@ FaissIVFSQ8Pass::Run(const TaskPtr& task) { } // namespace scheduler } // namespace milvus +#endif diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h index e92ea2fe4e..47033fc790 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h @@ -14,6 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +#ifdef MILVUS_GPU_VERSION #pragma once #include @@ -54,3 +55,4 @@ using FaissIVFSQ8PassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus +#endif diff --git a/core/src/wrapper/VecImpl.cpp b/core/src/wrapper/VecImpl.cpp index 74e9e94a2f..dda7452cd0 100644 --- a/core/src/wrapper/VecImpl.cpp +++ b/core/src/wrapper/VecImpl.cpp @@ -180,13 +180,13 @@ VecIndexImpl::CopyToCpu(const Config& cfg) { #endif } -VecIndexPtr -VecIndexImpl::Clone() { - // TODO(linxj): exception handle - auto clone_index = std::make_shared(index_->Clone(), type); - clone_index->dim = dim; - return clone_index; -} +// VecIndexPtr +// VecIndexImpl::Clone() { +// // TODO(linxj): exception handle +// auto clone_index = std::make_shared(index_->Clone(), type); +// clone_index->dim = dim; +// return clone_index; +//} int64_t VecIndexImpl::GetDeviceId() { diff --git a/core/src/wrapper/VecImpl.h b/core/src/wrapper/VecImpl.h index 2b6f07827e..e8dee45d42 100644 --- a/core/src/wrapper/VecImpl.h +++ b/core/src/wrapper/VecImpl.h @@ -60,8 +60,8 @@ class VecIndexImpl : public VecIndex { Status Load(const knowhere::BinarySet& index_binary) override; - VecIndexPtr - Clone() override; + // VecIndexPtr + // Clone() override; int64_t GetDeviceId() override; diff --git a/core/src/wrapper/VecIndex.h b/core/src/wrapper/VecIndex.h index e69655b087..536ebb32f6 100644 --- a/core/src/wrapper/VecIndex.h +++ b/core/src/wrapper/VecIndex.h @@ -75,8 +75,8 @@ class VecIndex : public cache::DataObj { CopyToCpu(const Config& cfg = Config()) = 0; // TODO(linxj): Deprecated - virtual VecIndexPtr - Clone() = 0; + // virtual VecIndexPtr + // Clone() = 0; virtual int64_t GetDeviceId() = 0; diff --git a/core/unittest/db/test_db.cpp b/core/unittest/db/test_db.cpp index 217fbe429e..8f53c0ed8a 100644 --- a/core/unittest/db/test_db.cpp +++ b/core/unittest/db/test_db.cpp @@ -229,6 +229,7 @@ TEST_F(DBTest, DB_TEST) { } TEST_F(DBTest, SEARCH_TEST) { + milvus::scheduler::OptimizerInst::GetInstance()->Init(); std::string config_path(CONFIG_PATH); config_path += CONFIG_FILE; milvus::server::Config& config = milvus::server::Config::GetInstance(); @@ -290,8 +291,51 @@ TEST_F(DBTest, SEARCH_TEST) { milvus::engine::ResultDistances result_distances; stat = db_->Query(TABLE_NAME, tags, k, nq, 10, xq.data(), result_ids, result_distances); ASSERT_TRUE(stat.ok()); + stat = db_->Query(TABLE_NAME, tags, k, 1100, 10, xq.data(), result_ids, result_distances); + ASSERT_TRUE(stat.ok()); } + index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFFLAT; + db_->CreateIndex(TABLE_NAME, index); // wait until build index finish + + { + std::vector tags; + milvus::engine::ResultIds result_ids; + milvus::engine::ResultDistances result_distances; + stat = db_->Query(TABLE_NAME, tags, k, nq, 10, xq.data(), result_ids, result_distances); + ASSERT_TRUE(stat.ok()); + stat = db_->Query(TABLE_NAME, tags, k, 1100, 10, xq.data(), result_ids, result_distances); + ASSERT_TRUE(stat.ok()); + } + + index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; + db_->CreateIndex(TABLE_NAME, index); // wait until build index finish + + { + std::vector tags; + milvus::engine::ResultIds result_ids; + milvus::engine::ResultDistances result_distances; + stat = db_->Query(TABLE_NAME, tags, k, nq, 10, xq.data(), result_ids, result_distances); + ASSERT_TRUE(stat.ok()); + stat = db_->Query(TABLE_NAME, tags, k, 1100, 10, xq.data(), result_ids, result_distances); + ASSERT_TRUE(stat.ok()); + } + +#ifdef CUSTOMIZATION + index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H; + db_->CreateIndex(TABLE_NAME, index); // wait until build index finish + + { + std::vector tags; + milvus::engine::ResultIds result_ids; + milvus::engine::ResultDistances result_distances; + stat = db_->Query(TABLE_NAME, tags, k, nq, 10, xq.data(), result_ids, result_distances); + ASSERT_TRUE(stat.ok()); + stat = db_->Query(TABLE_NAME, tags, k, 1100, 10, xq.data(), result_ids, result_distances); + ASSERT_TRUE(stat.ok()); + } +#endif + { // search by specify index file milvus::engine::meta::DatesT dates; std::vector file_ids = {"1", "2", "3", "4", "5", "6"}; diff --git a/core/unittest/db/test_engine.cpp b/core/unittest/db/test_engine.cpp index 5130a2c30d..67070df41b 100644 --- a/core/unittest/db/test_engine.cpp +++ b/core/unittest/db/test_engine.cpp @@ -88,12 +88,12 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) { status = engine_ptr->CopyToGpu(0, false); // ASSERT_TRUE(status.ok()); - auto new_engine = engine_ptr->Clone(); - ASSERT_EQ(new_engine->Dimension(), dimension); - ASSERT_EQ(new_engine->Count(), ids.size()); - status = new_engine->CopyToCpu(); +// auto new_engine = engine_ptr->Clone(); +// ASSERT_EQ(new_engine->Dimension(), dimension); +// ASSERT_EQ(new_engine->Count(), ids.size()); + status = engine_ptr->CopyToCpu(); // ASSERT_TRUE(status.ok()); - auto engine_build = new_engine->BuildIndex("/tmp/milvus_index_2", milvus::engine::EngineType::FAISS_IVFSQ8); + auto engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_2", milvus::engine::EngineType::FAISS_IVFSQ8); // ASSERT_TRUE(status.ok()); } diff --git a/core/unittest/scheduler/test_scheduler.cpp b/core/unittest/scheduler/test_scheduler.cpp index c839307958..dba8974698 100644 --- a/core/unittest/scheduler/test_scheduler.cpp +++ b/core/unittest/scheduler/test_scheduler.cpp @@ -37,10 +37,10 @@ class MockVecIndex : public engine::VecIndex { const float* xt = nullptr) { } - engine::VecIndexPtr - Clone() override { - return milvus::engine::VecIndexPtr(); - } +// engine::VecIndexPtr +// Clone() override { +// return milvus::engine::VecIndexPtr(); +// } int64_t GetDeviceId() override { diff --git a/core/unittest/server/test_cache.cpp b/core/unittest/server/test_cache.cpp index 92e09d4a26..ce8cfb304c 100644 --- a/core/unittest/server/test_cache.cpp +++ b/core/unittest/server/test_cache.cpp @@ -47,10 +47,10 @@ class MockVecIndex : public milvus::engine::VecIndex { return milvus::Status(); } - milvus::engine::VecIndexPtr - Clone() override { - return milvus::engine::VecIndexPtr(); - } +// milvus::engine::VecIndexPtr +// Clone() override { +// return milvus::engine::VecIndexPtr(); +// } int64_t GetDeviceId() override { @@ -120,7 +120,7 @@ TEST(CacheTest, DUMMY_TEST) { mock_index.Add(1, nullptr, nullptr); mock_index.BuildAll(1, nullptr, nullptr, cfg); mock_index.Search(1, nullptr, nullptr, nullptr, cfg); - mock_index.Clone(); +// mock_index.Clone(); mock_index.CopyToCpu(cfg); mock_index.CopyToGpu(1, cfg); mock_index.GetDeviceId(); diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index 4019c0f63c..37e5c31095 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -181,12 +181,22 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) { TEST(whatever, test_config) { milvus::engine::TempMetaConf conf; + conf.nprobe = 16; auto nsg_conf = std::make_shared(); nsg_conf->Match(conf); - nsg_conf->MatchSearch(conf, milvus::engine::IndexType::FAISS_IVFPQ_GPU); + nsg_conf->MatchSearch(conf, milvus::engine::IndexType::NSG_MIX); auto pq_conf = std::make_shared(); pq_conf->Match(conf); + pq_conf->MatchSearch(conf, milvus::engine::IndexType::FAISS_IVFPQ_MIX); + + auto kdt_conf = std::make_shared(); + kdt_conf->Match(conf); + kdt_conf->MatchSearch(conf, milvus::engine::IndexType::SPTAG_KDT_RNT_CPU); + + auto bkt_conf = std::make_shared(); + bkt_conf->Match(conf); + bkt_conf->MatchSearch(conf, milvus::engine::IndexType::SPTAG_BKT_RNT_CPU); } // #include "knowhere/index/vector_index/IndexIDMAP.h" From 46da58e2224e1f266c4d4098cefa038f32a74dcf Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Wed, 27 Nov 2019 14:29:05 +0800 Subject: [PATCH 4/4] #504 The code coverage rate of core/src/scheduler/optimizer is too low --- core/src/db/engine/ExecutionEngineImpl.cpp | 5 +++++ .../milvus_ivfsq8_test_report_detailed_version.md} | 0 .../milvus_ivfsq8_test_report_detailed_version_cn.md} | 0 .../milvus_ivfsq8h_test_report_detailed_version.md} | 0 .../milvus_ivfsq8h_test_report_detailed_version_cn.md} | 0 5 files changed, 5 insertions(+) rename docs/{test_report_en/ivfsq8_test.md => test_report/milvus_ivfsq8_test_report_detailed_version.md} (100%) rename docs/{test_report_cn/ivfsq8_test.md => test_report/milvus_ivfsq8_test_report_detailed_version_cn.md} (100%) rename docs/{test_report_en/ivfsq8h_test.md => test_report/milvus_ivfsq8h_test_report_detailed_version.md} (100%) rename docs/{test_report_cn/ivfsq8h_test.md => test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md} (100%) diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index d1f21ebc42..397ca89d68 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -257,6 +257,11 @@ ExecutionEngineImpl::PhysicalSize() const { Status ExecutionEngineImpl::Serialize() { auto status = write_index(index_, location_); + + // here we reset index size by file size, + // since some index type(such as SQ8) data size become smaller after serialized + index_->set_size(PhysicalSize()); + return status; } diff --git a/docs/test_report_en/ivfsq8_test.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version.md similarity index 100% rename from docs/test_report_en/ivfsq8_test.md rename to docs/test_report/milvus_ivfsq8_test_report_detailed_version.md diff --git a/docs/test_report_cn/ivfsq8_test.md b/docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md similarity index 100% rename from docs/test_report_cn/ivfsq8_test.md rename to docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md diff --git a/docs/test_report_en/ivfsq8h_test.md b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md similarity index 100% rename from docs/test_report_en/ivfsq8h_test.md rename to docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md diff --git a/docs/test_report_cn/ivfsq8h_test.md b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md similarity index 100% rename from docs/test_report_cn/ivfsq8h_test.md rename to docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md