milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix 6421, split raw vectors to serveral smaller one (#6449) 

Signed-off-by: dragondriver <jiquan.long@zilliz.com>
pull/6485/head
dragondriver 2021-07-13 16:05:36 +08:00 committed by GitHub
parent d822cfdbe2
commit 25b8c37383
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 16 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
internal/core/src/indexbuilder/IndexWrapper.cpp
View File

@ -21,6 +21,7 @@
#include "indexbuilder/utils.h"
#include "index/knowhere/knowhere/index/vector_index/ConfAdapterMgr.h"
#include "index/knowhere/knowhere/common/Timer.h"
#include "index/knowhere/knowhere/common/Utils.h"
namespace milvus {
namespace indexbuilder {
@ -239,6 +240,10 @@ IndexWrapper::Serialize() {
std::shared_ptr<uint8_t[]> raw_data(new uint8_t[raw_data_.size()], std::default_delete<uint8_t[]>());
memcpy(raw_data.get(), raw_data_.data(), raw_data_.size());
binarySet.Append(RAW_DATA, raw_data, raw_data_.size());
auto slice_size = get_index_file_slice_size();
// https://github.com/milvus-io/milvus/issues/6421
// Disassemble will only divide the raw vectors, other keys was already divided
knowhere::Disassemble(slice_size * 1024 * 1024, binarySet);
}
namespace indexcgo = milvus::proto::indexcgo;
@ -317,6 +322,14 @@ IndexWrapper::get_index_mode() {
return mode.has_value() ? mode_map[mode.value()] : knowhere::IndexMode::MODE_CPU;
}
int64_t
IndexWrapper::get_index_file_slice_size() {
if (config_.contains(knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE)) {
return config_[knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE].get<int64_t>();
}
return 4; // by default
}
std::unique_ptr<IndexWrapper::QueryResult>
IndexWrapper::Query(const knowhere::DatasetPtr& dataset) {
return std::move(QueryImpl(dataset, config_));

3
internal/core/src/indexbuilder/IndexWrapper.h
View File

@ -65,6 +65,9 @@ class IndexWrapper {
knowhere::IndexMode
get_index_mode();
int64_t
get_index_file_slice_size();
template <typename T>
std::optional<T>
get_config_by_name(std::string name);