milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow more choices for the parameter ‘m’ of CPU IVF_PQ (#3469) 

* fix the IVFPQ fixed M problem

Signed-off-by: cqy <yaya645@126.com>

* fix compile

Signed-off-by: shengjun.li <shengjun.li@zilliz.com>

Co-authored-by: shengjun.li <shengjun.li@zilliz.com>
pull/3525/head
cqy123456 2020-08-27 11:37:25 +08:00 committed by GitHub
parent 5897f4fa33
commit a75c0a572c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 85 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -6,9 +6,10 @@ Please mark all change in change log and use the issue from GitHub
## Bug
## Feature
- \#3213 Allow users to specify a distance type at runtime for Flat index
- \#3254 Allow more choices for the parameter ‘m’ of CPU IVF_PQ
## Improvement
- \#3213 Allow users to specify a distance type at runtime for Flat index
## Task

22
core/src/db/engine/ExecutionEngineImpl.cpp
View File

@ -182,6 +182,13 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
break;
}
case EngineType::FAISS_PQ: {
auto m = index_params_[knowhere::IndexParams::m];
if (!m.is_null() && !milvus::knowhere::IVFPQConfAdapter::GetValidM(dim_, m.get<int64_t>(), mode)) {
std::string err_msg = "dimension " + std::to_string(dim_) + " can't not be divided by m " +
std::to_string(m.get<int64_t>());
LOG_ENGINE_ERROR_ << err_msg;
break;
}
index = vec_index_factory.CreateVecIndex(knowhere::IndexEnum::INDEX_FAISS_IVFPQ, mode);
break;
}
@ -595,12 +602,17 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) {
index_reserve_ = index_;
if (gpu_cache_enable) {
gpu_cache_mgr->Reserve(index_->Size());
index_ = knowhere::cloner::CopyCpuToGpu(index_, device_id, knowhere::Config());
gpu_cache_mgr->InsertItem(location_, std::static_pointer_cast<cache::DataObj>(index_));
} else {
index_ = knowhere::cloner::CopyCpuToGpu(index_, device_id, knowhere::Config());
}
LOG_ENGINE_DEBUG_ << "CPU to GPU" << device_id << " finished";
index_ = knowhere::cloner::CopyCpuToGpu(index_, device_id, knowhere::Config());
if (index_ == nullptr) {
LOG_ENGINE_DEBUG_ << "copy to GPU faied, search on CPU";
index_ = index_reserve_;
} else {
if (gpu_cache_enable) {
gpu_cache_mgr->InsertItem(location_, std::static_pointer_cast<cache::DataObj>(index_));
}
LOG_ENGINE_DEBUG_ << "CPU to GPU" << device_id << " finished";
}
} catch (std::exception& e) {
LOG_ENGINE_ERROR_ << e.what();
return Status(DB_ERROR, e.what());

55
core/src/index/knowhere/knowhere/index/vector_index/ConfAdapter.cpp
View File

@ -156,18 +156,28 @@ IVFPQConfAdapter::CheckTrain(Config& oricfg, const IndexMode mode) {
// static int64_t MAX_POINTS_PER_CENTROID = 256;
// CheckIntByRange(knowhere::meta::ROWS, MIN_POINTS_PER_CENTROID * nlist, MAX_POINTS_PER_CENTROID * nlist);
std::vector<int64_t> resset;
int64_t dimension = oricfg[knowhere::meta::DIM].get<int64_t>();
IVFPQConfAdapter::GetValidMList(dimension, resset);
int64_t m = oricfg[knowhere::IndexParams::m].get<int64_t>();
IndexMode IVFPQ_mode = mode;
return GetValidM(dimension, m, IVFPQ_mode);
}
CheckIntByValues(knowhere::IndexParams::m, resset);
bool
IVFPQConfAdapter::GetValidM(int64_t dimension, int64_t m, IndexMode& mode) {
#ifdef MILVUS_GPU_VERSION
if (mode == knowhere::IndexMode::MODE_GPU && !IVFPQConfAdapter::GetValidGPUM(dimension, m)) {
mode = knowhere::IndexMode::MODE_CPU;
}
#endif
if (mode == knowhere::IndexMode::MODE_CPU && !IVFPQConfAdapter::GetValidCPUM(dimension, m)) {
return false;
}
return true;
}
void
IVFPQConfAdapter::GetValidMList(int64_t dimension, std::vector<int64_t>& resset) {
resset.clear();
bool
IVFPQConfAdapter::GetValidGPUM(int64_t dimension, int64_t m) {
/*
* Faiss 1.6
* Only 1, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32 dims per sub-quantizer are currently supported with
@ -176,15 +186,34 @@ IVFPQConfAdapter::GetValidMList(int64_t dimension, std::vector<int64_t>& resset)
static std::vector<int64_t> support_dim_per_subquantizer{32, 28, 24, 20, 16, 12, 10, 8, 6, 4, 3, 2, 1};
static std::vector<int64_t> support_subquantizer{96, 64, 56, 48, 40, 32, 28, 24, 20, 16, 12, 8, 4, 3, 2, 1};
for (const auto& dimperquantizer : support_dim_per_subquantizer) {
if (!(dimension % dimperquantizer)) {
auto subquantzier_num = dimension / dimperquantizer;
auto finder = std::find(support_subquantizer.begin(), support_subquantizer.end(), subquantzier_num);
if (finder != support_subquantizer.end()) {
resset.push_back(subquantzier_num);
if (!GetValidCPUM(dimension, m)) {
return false;
}
int64_t sub_dim = dimension / m;
return (std::find(std::begin(support_subquantizer), std::end(support_subquantizer), m) !=
support_subquantizer.end()) &&
(std::find(std::begin(support_dim_per_subquantizer), std::end(support_dim_per_subquantizer), sub_dim) !=
support_dim_per_subquantizer.end());
/*
std::vector<int64_t> resset;
resset.clear();
for (const auto& dimperquantizer : support_dim_per_subquantizer) {
if (!(dimension % dimperquantizer)) {
auto subquantzier_num = dimension / dimperquantizer;
auto finder = std::find(support_subquantizer.begin(), support_subquantizer.end(), subquantzier_num);
if (finder != support_subquantizer.end()) {
resset.push_back(subquantzier_num);
}
}
}
}
*/
}
bool
IVFPQConfAdapter::GetValidCPUM(int64_t dimension, int64_t m) {
return (dimension % m == 0);
}
bool

10
core/src/index/knowhere/knowhere/index/vector_index/ConfAdapter.h
View File

@ -51,8 +51,14 @@ class IVFPQConfAdapter : public IVFConfAdapter {
bool
CheckTrain(Config& oricfg, const IndexMode mode) override;
static void
GetValidMList(int64_t dimension, std::vector<int64_t>& resset);
static bool
GetValidM(int64_t dimension, int64_t m, IndexMode& mode);
static bool
GetValidGPUM(int64_t dimension, int64_t m);
static bool
GetValidCPUM(int64_t dimension, int64_t m);
};
class NSGConfAdapter : public IVFConfAdapter {

8
core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp
View File

@ -24,6 +24,7 @@
#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
#ifdef MILVUS_GPU_VERSION
#include "knowhere/index/vector_index/ConfAdapter.h"
#include "knowhere/index/vector_index/gpu/IndexGPUIVF.h"
#include "knowhere/index/vector_index/gpu/IndexGPUIVFPQ.h"
#endif
@ -47,6 +48,13 @@ IVFPQ::Train(const DatasetPtr& dataset_ptr, const Config& config) {
VecIndexPtr
IVFPQ::CopyCpuToGpu(const int64_t device_id, const Config& config) {
#ifdef MILVUS_GPU_VERSION
auto ivfpq_index = dynamic_cast<faiss::IndexIVFPQ*>(index_.get());
int64_t dim = ivfpq_index->d;
int64_t m = ivfpq_index->pq.M;
if (!IVFPQConfAdapter::GetValidGPUM(dim, m)) {
return nullptr;
}
if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) {
ResScope rs(res, device_id, false);
auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get());

5
core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp
View File

@ -63,8 +63,9 @@ CopyCpuToGpu(const VecIndexPtr& index, const int64_t device_id, const Config& co
} else {
KNOWHERE_THROW_MSG("this index type not support transfer to gpu");
}
CopyIndexData(result, index);
if (result != nullptr) {
CopyIndexData(result, index);
}
return result;
}

1
core/src/index/unittest/CMakeLists.txt
View File

@ -65,6 +65,7 @@ set(faiss_srcs
)
if (KNOWHERE_GPU_VERSION)
set(faiss_srcs ${faiss_srcs}
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ConfAdapter.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIDMAP.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.cpp

11
core/src/utils/ValidationUtil.cpp
View File

@ -215,16 +215,14 @@ ValidationUtil::ValidateIndexParams(const milvus::json& index_params,
}
// special check for 'm' parameter
std::vector<int64_t> resset;
milvus::knowhere::IVFPQConfAdapter::GetValidMList(collection_schema.dimension_, resset);
int64_t m_value = index_params[knowhere::IndexParams::m];
if (resset.empty()) {
std::string msg = "Invalid collection dimension, unable to get reasonable values for 'm'";
if (!milvus::knowhere::IVFPQConfAdapter::GetValidCPUM(collection_schema.dimension_, m_value)) {
std::string msg = "Invalid collection dimension, dimension can not be divided by m";
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_COLLECTION_DIMENSION, msg);
}
auto iter = std::find(std::begin(resset), std::end(resset), m_value);
/* auto iter = std::find(std::begin(resset), std::end(resset), m_value);
if (iter == std::end(resset)) {
std::string msg =
"Invalid " + std::string(knowhere::IndexParams::m) + ", must be one of the following values: ";
@ -237,8 +235,7 @@ ValidationUtil::ValidateIndexParams(const milvus::json& index_params,
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
}*/
break;
}
case (int32_t)engine::EngineType::NSG_MIX: {