milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix by reduce knng size

pull/381/head
xiaojun.lin 2019-11-18 09:53:38 +08:00
parent fa4f0a0703
commit d5eb5675b7
9 changed files with 188 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
core/src/index/CMakeLists.txt
View File

@ -88,14 +88,14 @@ endif ()
include(ThirdPartyPackagesCore)
if (CMAKE_BUILD_TYPE STREQUAL "Release")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt")
if (KNOWHERE_GPU_VERSION)
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3")
endif ()
else ()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -g -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt")
if (KNOWHERE_GPU_VERSION)
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3 -g")
endif ()
endif ()

14
core/src/index/knowhere/knowhere/common/Config.h
View File

@ -18,6 +18,8 @@
#pragma once
#include <memory>
#include <sstream>
#include "Log.h"
namespace knowhere {
@ -50,6 +52,18 @@ struct Cfg {
CheckValid() {
return true;
}
void
Dump() {
KNOWHERE_LOG_DEBUG << DumpImpl().str();
}
virtual std::stringstream
DumpImpl() {
std::stringstream ss;
ss << "dim: " << d << ", metric: " << int(metric_type) << ", gpuid: " << gpu_id << ", k: " << k;
return ss;
}
};
using Config = std::shared_ptr<Cfg>;

22
core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp
View File

@ -34,4 +34,26 @@ GetMetricType(METRICTYPE& type) {
KNOWHERE_THROW_MSG("Metric type is invalid");
}
std::stringstream
IVFCfg::DumpImpl() {
auto ss = Cfg::DumpImpl();
ss << ", nlist: " << nlist << ", nprobe: " << nprobe;
return ss;
}
std::stringstream
IVFSQCfg::DumpImpl() {
auto ss = IVFCfg::DumpImpl();
ss << ", nbits: " << nbits;
return ss;
}
std::stringstream
NSGCfg::DumpImpl() {
auto ss = IVFCfg::DumpImpl();
ss << ", knng: " << knng << ", search_length: " << search_length << ", out_degree: " << out_degree
<< ", candidate: " << candidate_pool_size;
return ss;
}
} // namespace knowhere

9
core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h
View File

@ -53,6 +53,9 @@ struct IVFCfg : public Cfg {
IVFCfg() = default;
std::stringstream
DumpImpl() override;
bool
CheckValid() override {
return true;
@ -69,6 +72,9 @@ struct IVFSQCfg : public IVFCfg {
: IVFCfg(dim, k, gpu_id, nlist, nprobe, type), nbits(nbits) {
}
std::stringstream
DumpImpl() override;
IVFSQCfg() = default;
bool
@ -119,6 +125,9 @@ struct NSGCfg : public IVFCfg {
NSGCfg() = default;
std::stringstream
DumpImpl() override;
bool
CheckValid() override {
return true;

27
core/src/wrapper/ConfAdapter.cpp
View File

@ -47,7 +47,8 @@ ConfAdapter::Match(const TempMetaConf& metaconf) {
auto conf = std::make_shared<knowhere::Cfg>();
conf->d = metaconf.dim;
conf->metric_type = metaconf.metric_type;
conf->gpu_id = conf->gpu_id;
conf->gpu_id = metaconf.gpu_id;
conf->k = metaconf.k;
MatchBase(conf);
return conf;
}
@ -65,7 +66,7 @@ IVFConfAdapter::Match(const TempMetaConf& metaconf) {
conf->nlist = MatchNlist(metaconf.size, metaconf.nlist);
conf->d = metaconf.dim;
conf->metric_type = metaconf.metric_type;
conf->gpu_id = conf->gpu_id;
conf->gpu_id = metaconf.gpu_id;
MatchBase(conf);
return conf;
}
@ -114,7 +115,7 @@ IVFSQConfAdapter::Match(const TempMetaConf& metaconf) {
conf->nlist = MatchNlist(metaconf.size, metaconf.nlist);
conf->d = metaconf.dim;
conf->metric_type = metaconf.metric_type;
conf->gpu_id = conf->gpu_id;
conf->gpu_id = metaconf.gpu_id;
conf->nbits = 8;
MatchBase(conf);
return conf;
@ -126,7 +127,7 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) {
conf->nlist = MatchNlist(metaconf.size, metaconf.nlist);
conf->d = metaconf.dim;
conf->metric_type = metaconf.metric_type;
conf->gpu_id = conf->gpu_id;
conf->gpu_id = metaconf.gpu_id;
conf->nbits = 8;
if (!(conf->d % 4))
@ -175,21 +176,17 @@ NSGConfAdapter::Match(const TempMetaConf& metaconf) {
conf->nlist = MatchNlist(metaconf.size, metaconf.nlist);
conf->d = metaconf.dim;
conf->metric_type = metaconf.metric_type;
conf->gpu_id = conf->gpu_id;
conf->gpu_id = metaconf.gpu_id;
conf->k = metaconf.k;
double factor = metaconf.size / TYPICAL_COUNT;
auto scale_factor = round(metaconf.dim / 128.0);
scale_factor = scale_factor >= 4 ? 4 : scale_factor;
conf->nprobe = conf->nlist > 10000 ? conf->nlist * 0.02 : conf->nlist * 0.1;
conf->knng = (100 + 100 * scale_factor) * factor;
conf->search_length = (40 + 5 * scale_factor) * factor;
conf->out_degree = (50 + 5 * scale_factor) * factor;
conf->candidate_pool_size = (200 + 100 * scale_factor) * factor;
conf->nprobe = int64_t(conf->nlist * 0.01);
conf->knng = 40 + 10 * scale_factor; // the size of knng
conf->search_length = 40 + 5 * scale_factor;
conf->out_degree = 50 + 5 * scale_factor;
conf->candidate_pool_size = 200 + 100 * scale_factor;
MatchBase(conf);
// WRAPPER_LOG_DEBUG << "nlist: " << conf->nlist
// << ", gpu_id: " << conf->gpu_id << ", d: " << conf->d
// << ", nprobe: " << conf->nprobe << ", knng: " << conf->knng;
return conf;
}

3
core/src/wrapper/ConfAdapter.h
View File

@ -46,9 +46,6 @@ class ConfAdapter {
virtual knowhere::Config
MatchSearch(const TempMetaConf& metaconf, const IndexType& type);
// virtual void
// Dump(){}
protected:
static void
MatchBase(knowhere::Config conf);

121
core/unittest/wrapper/test_wrapper.cpp
View File

@ -56,10 +56,6 @@ class KnowhereWrapperTest
index_ = GetVecIndexFactory(index_type);
conf = ParamGenerator::GetInstance().GenBuild(index_type, tempconf);
searchconf = ParamGenerator::GetInstance().GenSearchConf(index_type, tempconf);
// conf->k = k;
// conf->d = dim;
// conf->gpu_id = DEVICEID;
}
void TearDown() override {
@ -97,6 +93,7 @@ INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest,
TEST_P(KnowhereWrapperTest, BASE_TEST) {
EXPECT_EQ(index_->GetType(), index_type);
// conf->Dump();
auto elems = nq * k;
std::vector<int64_t> res_ids(elems);
@ -191,3 +188,119 @@ TEST(whatever, test_config) {
auto pq_conf = std::make_shared<milvus::engine::IVFPQConfAdapter>();
pq_conf->Match(conf);
}
// #include "knowhere/index/vector_index/IndexIDMAP.h"
// #include "src/wrapper/VecImpl.h"
// #include "src/index/unittest/utils.h"
// The two case below prove NSG is concern with data distribution
// Further work: 1. Use right basedata and pass it by milvus
// a. batch size is 100000 [Pass]
// b. transfer all at once [Pass]
// 2. Use SIFT1M in test and check time cost []
// TEST_P(KnowhereWrapperTest, nsgwithidmap) {
// auto idmap = GetVecIndexFactory(milvus::engine::IndexType::FAISS_IDMAP);
// auto ori_xb = xb;
// auto ori_ids = ids;
// std::vector<float> temp_xb;
// std::vector<int64_t> temp_ids;
// nb = 50000;
// for (int i = 0; i < 20; ++i) {
// GenData(dim, nb, nq, xb, xq, ids, k, gt_ids, gt_dis);
// assert(xb.size() == nb*dim);
// //#define IDMAP
// #ifdef IDMAP
// temp_xb.insert(temp_xb.end(), xb.data(), xb.data() + nb*dim);
// temp_ids.insert(temp_ids.end(), ori_ids.data()+nb*i, ori_ids.data() + nb*(i+1));
// if (i == 0) {
// idmap->BuildAll(nb, temp_xb.data(), temp_ids.data(), conf);
// } else {
// idmap->Add(nb, temp_xb.data(), temp_ids.data());
// }
// temp_xb.clear();
// temp_ids.clear();
// #else
// temp_xb.insert(temp_xb.end(), xb.data(), xb.data() + nb*dim);
// temp_ids.insert(temp_ids.end(), ori_ids.data()+nb*i, ori_ids.data() + nb*(i+1));
// #endif
// }
// #ifdef IDMAP
// auto idmap_idx = std::dynamic_pointer_cast<milvus::engine::BFIndex>(idmap);
// auto x = idmap_idx->Count();
// index_->BuildAll(idmap_idx->Count(), idmap_idx->GetRawVectors(), idmap_idx->GetRawIds(), conf);
// #else
// assert(temp_xb.size() == 1000000*128);
// index_->BuildAll(1000000, temp_xb.data(), ori_ids.data(), conf);
// #endif
// }
// TEST_P(KnowhereWrapperTest, nsgwithsidmap) {
// auto idmap = GetVecIndexFactory(milvus::engine::IndexType::FAISS_IDMAP);
// auto ori_xb = xb;
// std::vector<float> temp_xb;
// std::vector<int64_t> temp_ids;
// nb = 50000;
// for (int i = 0; i < 20; ++i) {
// #define IDMAP
// #ifdef IDMAP
// temp_xb.insert(temp_xb.end(), ori_xb.data()+nb*dim*i, ori_xb.data() + nb*dim*(i+1));
// temp_ids.insert(temp_ids.end(), ids.data()+nb*i, ids.data() + nb*(i+1));
// if (i == 0) {
// idmap->BuildAll(nb, temp_xb.data(), temp_ids.data(), conf);
// } else {
// idmap->Add(nb, temp_xb.data(), temp_ids.data());
// }
// temp_xb.clear();
// temp_ids.clear();
// #else
// temp_xb.insert(temp_xb.end(), ori_xb.data()+nb*dim*i, ori_xb.data() + nb*dim*(i+1));
// temp_ids.insert(temp_ids.end(), ids.data()+nb*i, ids.data() + nb*(i+1));
// #endif
// }
// #ifdef IDMAP
// auto idmap_idx = std::dynamic_pointer_cast<milvus::engine::BFIndex>(idmap);
// auto x = idmap_idx->Count();
// index_->BuildAll(idmap_idx->Count(), idmap_idx->GetRawVectors(), idmap_idx->GetRawIds(), conf);
// #else
// index_->BuildAll(1000000, temp_xb.data(), temp_ids.data(), conf);
// #endif
// // The code use to store raw base data
// FileIOWriter writer("/tmp/newraw");
// ori_xb.shrink_to_fit();
// std::cout << "size" << ori_xb.size();
// writer(static_cast<void*>(ori_xb.data()), ori_xb.size()* sizeof(float));
// std::cout << "Finish!" << std::endl;
// }
// void load_data(char* filename, float*& data, unsigned& num,
// unsigned& dim) { // load data with sift10K pattern
// std::ifstream in(filename, std::ios::binary);
// if (!in.is_open()) {
// std::cout << "open file error" << std::endl;
// exit(-1);
// }
// in.read((char*)&dim, 4);
// in.seekg(0, std::ios::end);
// std::ios::pos_type ss = in.tellg();
// size_t fsize = (size_t)ss;
// num = (unsigned)(fsize / (dim + 1) / 4);
// data = new float[(size_t)num * (size_t)dim];
// in.seekg(0, std::ios::beg);
// for (size_t i = 0; i < num; i++) {
// in.seekg(4, std::ios::cur);
// in.read((char*)(data + i * dim), dim * 4);
// }
// in.close();
// }
// TEST_P(KnowhereWrapperTest, Sift1M) {
// float* data = nullptr;
// unsigned points_num, dim;
// load_data("/mnt/112d53a6-5592-4360-a33b-7fd789456fce/workspace/Data/sift/sift_base.fvecs", data, points_num,
// dim); std::cout << points_num << " " << dim << std::endl;
// index_->BuildAll(points_num, data, ids.data(), conf);
// }

5
core/unittest/wrapper/utils.cpp
View File

@ -117,6 +117,11 @@ void
DataGenBase::GenData(const int& dim, const int& nb, const int& nq, std::vector<float>& xb, std::vector<float>& xq,
std::vector<int64_t>& ids, const int& k, std::vector<int64_t>& gt_ids,
std::vector<float>& gt_dis) {
xb.clear();
xq.clear();
ids.clear();
gt_ids.clear();
gt_dis.clear();
xb.resize(nb * dim);
xq.resize(nq * dim);
ids.resize(nb);

10
core/unittest/wrapper/utils.h
View File

@ -25,7 +25,6 @@
#include <cstdio>
#include <fstream>
#include "wrapper/VecIndex.h"
#include "wrapper/utils.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
@ -90,17 +89,20 @@ class ParamGenerator {
return instance;
}
knowhere::Config GenSearchConf(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) {
knowhere::Config
GenSearchConf(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) {
auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type);
return adapter->MatchSearch(conf, type);
}
knowhere::Config GenBuild(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) {
knowhere::Config
GenBuild(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) {
auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type);
return adapter->Match(conf);
}
knowhere::Config Gen(const milvus::engine::IndexType& type) {
knowhere::Config
Gen(const milvus::engine::IndexType& type) {
switch (type) {
case milvus::engine::IndexType::FAISS_IDMAP: {
auto tempconf = std::make_shared<knowhere::Cfg>();