diff --git a/core/src/index/CMakeLists.txt b/core/src/index/CMakeLists.txt index 4b5c1b1de3..53453d53aa 100644 --- a/core/src/index/CMakeLists.txt +++ b/core/src/index/CMakeLists.txt @@ -88,14 +88,14 @@ endif () include(ThirdPartyPackagesCore) if (CMAKE_BUILD_TYPE STREQUAL "Release") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt") if (KNOWHERE_GPU_VERSION) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3") endif () else () - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -g -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt") if (KNOWHERE_GPU_VERSION) - set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3 -g") endif () endif () diff --git a/core/src/index/knowhere/knowhere/common/Config.h b/core/src/index/knowhere/knowhere/common/Config.h index 6191ecb771..48c2de8b1b 100644 --- a/core/src/index/knowhere/knowhere/common/Config.h +++ b/core/src/index/knowhere/knowhere/common/Config.h @@ -18,6 +18,8 @@ #pragma once #include +#include +#include "Log.h" namespace knowhere { @@ -50,6 +52,18 @@ struct Cfg { CheckValid() { return true; } + + void + Dump() { + KNOWHERE_LOG_DEBUG << DumpImpl().str(); + } + + virtual std::stringstream + DumpImpl() { + std::stringstream ss; + ss << "dim: " << d << ", metric: " << int(metric_type) << ", gpuid: " << gpu_id << ", k: " << k; + return ss; + } }; using Config = std::shared_ptr; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp index 20f3388174..6e9ee55658 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp @@ -34,4 +34,26 @@ GetMetricType(METRICTYPE& type) { KNOWHERE_THROW_MSG("Metric type is invalid"); } +std::stringstream +IVFCfg::DumpImpl() { + auto ss = Cfg::DumpImpl(); + ss << ", nlist: " << nlist << ", nprobe: " << nprobe; + return ss; +} + +std::stringstream +IVFSQCfg::DumpImpl() { + auto ss = IVFCfg::DumpImpl(); + ss << ", nbits: " << nbits; + return ss; +} + +std::stringstream +NSGCfg::DumpImpl() { + auto ss = IVFCfg::DumpImpl(); + ss << ", knng: " << knng << ", search_length: " << search_length << ", out_degree: " << out_degree + << ", candidate: " << candidate_pool_size; + return ss; +} + } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h index b2854abef8..b931790b04 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h @@ -53,6 +53,9 @@ struct IVFCfg : public Cfg { IVFCfg() = default; + std::stringstream + DumpImpl() override; + bool CheckValid() override { return true; @@ -69,6 +72,9 @@ struct IVFSQCfg : public IVFCfg { : IVFCfg(dim, k, gpu_id, nlist, nprobe, type), nbits(nbits) { } + std::stringstream + DumpImpl() override; + IVFSQCfg() = default; bool @@ -119,6 +125,9 @@ struct NSGCfg : public IVFCfg { NSGCfg() = default; + std::stringstream + DumpImpl() override; + bool CheckValid() override { return true; diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index 461745f1fd..b121372a26 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -47,7 +47,8 @@ ConfAdapter::Match(const TempMetaConf& metaconf) { auto conf = std::make_shared(); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; + conf->k = metaconf.k; MatchBase(conf); return conf; } @@ -65,7 +66,7 @@ IVFConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; MatchBase(conf); return conf; } @@ -114,7 +115,7 @@ IVFSQConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; conf->nbits = 8; MatchBase(conf); return conf; @@ -126,7 +127,7 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; conf->nbits = 8; if (!(conf->d % 4)) @@ -175,21 +176,17 @@ NSGConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; + conf->k = metaconf.k; - double factor = metaconf.size / TYPICAL_COUNT; auto scale_factor = round(metaconf.dim / 128.0); scale_factor = scale_factor >= 4 ? 4 : scale_factor; - conf->nprobe = conf->nlist > 10000 ? conf->nlist * 0.02 : conf->nlist * 0.1; - conf->knng = (100 + 100 * scale_factor) * factor; - conf->search_length = (40 + 5 * scale_factor) * factor; - conf->out_degree = (50 + 5 * scale_factor) * factor; - conf->candidate_pool_size = (200 + 100 * scale_factor) * factor; + conf->nprobe = int64_t(conf->nlist * 0.01); + conf->knng = 40 + 10 * scale_factor; // the size of knng + conf->search_length = 40 + 5 * scale_factor; + conf->out_degree = 50 + 5 * scale_factor; + conf->candidate_pool_size = 200 + 100 * scale_factor; MatchBase(conf); - - // WRAPPER_LOG_DEBUG << "nlist: " << conf->nlist - // << ", gpu_id: " << conf->gpu_id << ", d: " << conf->d - // << ", nprobe: " << conf->nprobe << ", knng: " << conf->knng; return conf; } diff --git a/core/src/wrapper/ConfAdapter.h b/core/src/wrapper/ConfAdapter.h index 5ec3d52486..85637a4969 100644 --- a/core/src/wrapper/ConfAdapter.h +++ b/core/src/wrapper/ConfAdapter.h @@ -46,9 +46,6 @@ class ConfAdapter { virtual knowhere::Config MatchSearch(const TempMetaConf& metaconf, const IndexType& type); - // virtual void - // Dump(){} - protected: static void MatchBase(knowhere::Config conf); diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index 025601a1cd..11a8b8e01a 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -56,10 +56,6 @@ class KnowhereWrapperTest index_ = GetVecIndexFactory(index_type); conf = ParamGenerator::GetInstance().GenBuild(index_type, tempconf); searchconf = ParamGenerator::GetInstance().GenSearchConf(index_type, tempconf); - -// conf->k = k; -// conf->d = dim; -// conf->gpu_id = DEVICEID; } void TearDown() override { @@ -97,6 +93,7 @@ INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest, TEST_P(KnowhereWrapperTest, BASE_TEST) { EXPECT_EQ(index_->GetType(), index_type); + // conf->Dump(); auto elems = nq * k; std::vector res_ids(elems); @@ -191,3 +188,119 @@ TEST(whatever, test_config) { auto pq_conf = std::make_shared(); pq_conf->Match(conf); } + +// #include "knowhere/index/vector_index/IndexIDMAP.h" +// #include "src/wrapper/VecImpl.h" +// #include "src/index/unittest/utils.h" +// The two case below prove NSG is concern with data distribution +// Further work: 1. Use right basedata and pass it by milvus +// a. batch size is 100000 [Pass] +// b. transfer all at once [Pass] +// 2. Use SIFT1M in test and check time cost [] +// TEST_P(KnowhereWrapperTest, nsgwithidmap) { +// auto idmap = GetVecIndexFactory(milvus::engine::IndexType::FAISS_IDMAP); +// auto ori_xb = xb; +// auto ori_ids = ids; +// std::vector temp_xb; +// std::vector temp_ids; +// nb = 50000; +// for (int i = 0; i < 20; ++i) { +// GenData(dim, nb, nq, xb, xq, ids, k, gt_ids, gt_dis); +// assert(xb.size() == nb*dim); +// //#define IDMAP +// #ifdef IDMAP +// temp_xb.insert(temp_xb.end(), xb.data(), xb.data() + nb*dim); +// temp_ids.insert(temp_ids.end(), ori_ids.data()+nb*i, ori_ids.data() + nb*(i+1)); +// if (i == 0) { +// idmap->BuildAll(nb, temp_xb.data(), temp_ids.data(), conf); +// } else { +// idmap->Add(nb, temp_xb.data(), temp_ids.data()); +// } +// temp_xb.clear(); +// temp_ids.clear(); +// #else +// temp_xb.insert(temp_xb.end(), xb.data(), xb.data() + nb*dim); +// temp_ids.insert(temp_ids.end(), ori_ids.data()+nb*i, ori_ids.data() + nb*(i+1)); +// #endif +// } + +// #ifdef IDMAP +// auto idmap_idx = std::dynamic_pointer_cast(idmap); +// auto x = idmap_idx->Count(); +// index_->BuildAll(idmap_idx->Count(), idmap_idx->GetRawVectors(), idmap_idx->GetRawIds(), conf); +// #else +// assert(temp_xb.size() == 1000000*128); +// index_->BuildAll(1000000, temp_xb.data(), ori_ids.data(), conf); +// #endif +// } + +// TEST_P(KnowhereWrapperTest, nsgwithsidmap) { +// auto idmap = GetVecIndexFactory(milvus::engine::IndexType::FAISS_IDMAP); +// auto ori_xb = xb; +// std::vector temp_xb; +// std::vector temp_ids; +// nb = 50000; +// for (int i = 0; i < 20; ++i) { +// #define IDMAP +// #ifdef IDMAP +// temp_xb.insert(temp_xb.end(), ori_xb.data()+nb*dim*i, ori_xb.data() + nb*dim*(i+1)); +// temp_ids.insert(temp_ids.end(), ids.data()+nb*i, ids.data() + nb*(i+1)); +// if (i == 0) { +// idmap->BuildAll(nb, temp_xb.data(), temp_ids.data(), conf); +// } else { +// idmap->Add(nb, temp_xb.data(), temp_ids.data()); +// } +// temp_xb.clear(); +// temp_ids.clear(); +// #else +// temp_xb.insert(temp_xb.end(), ori_xb.data()+nb*dim*i, ori_xb.data() + nb*dim*(i+1)); +// temp_ids.insert(temp_ids.end(), ids.data()+nb*i, ids.data() + nb*(i+1)); +// #endif +// } + +// #ifdef IDMAP +// auto idmap_idx = std::dynamic_pointer_cast(idmap); +// auto x = idmap_idx->Count(); +// index_->BuildAll(idmap_idx->Count(), idmap_idx->GetRawVectors(), idmap_idx->GetRawIds(), conf); +// #else +// index_->BuildAll(1000000, temp_xb.data(), temp_ids.data(), conf); +// #endif + +// // The code use to store raw base data +// FileIOWriter writer("/tmp/newraw"); +// ori_xb.shrink_to_fit(); +// std::cout << "size" << ori_xb.size(); +// writer(static_cast(ori_xb.data()), ori_xb.size()* sizeof(float)); +// std::cout << "Finish!" << std::endl; +// } + +// void load_data(char* filename, float*& data, unsigned& num, +// unsigned& dim) { // load data with sift10K pattern +// std::ifstream in(filename, std::ios::binary); +// if (!in.is_open()) { +// std::cout << "open file error" << std::endl; +// exit(-1); +// } +// in.read((char*)&dim, 4); +// in.seekg(0, std::ios::end); +// std::ios::pos_type ss = in.tellg(); +// size_t fsize = (size_t)ss; +// num = (unsigned)(fsize / (dim + 1) / 4); +// data = new float[(size_t)num * (size_t)dim]; + +// in.seekg(0, std::ios::beg); +// for (size_t i = 0; i < num; i++) { +// in.seekg(4, std::ios::cur); +// in.read((char*)(data + i * dim), dim * 4); +// } +// in.close(); +// } + +// TEST_P(KnowhereWrapperTest, Sift1M) { +// float* data = nullptr; +// unsigned points_num, dim; +// load_data("/mnt/112d53a6-5592-4360-a33b-7fd789456fce/workspace/Data/sift/sift_base.fvecs", data, points_num, +// dim); std::cout << points_num << " " << dim << std::endl; + +// index_->BuildAll(points_num, data, ids.data(), conf); +// } \ No newline at end of file diff --git a/core/unittest/wrapper/utils.cpp b/core/unittest/wrapper/utils.cpp index 00a1db2c88..96b9e643f5 100644 --- a/core/unittest/wrapper/utils.cpp +++ b/core/unittest/wrapper/utils.cpp @@ -117,6 +117,11 @@ void DataGenBase::GenData(const int& dim, const int& nb, const int& nq, std::vector& xb, std::vector& xq, std::vector& ids, const int& k, std::vector& gt_ids, std::vector& gt_dis) { + xb.clear(); + xq.clear(); + ids.clear(); + gt_ids.clear(); + gt_dis.clear(); xb.resize(nb * dim); xq.resize(nq * dim); ids.resize(nb); diff --git a/core/unittest/wrapper/utils.h b/core/unittest/wrapper/utils.h index 1eaa7ae4ec..cc1926302d 100644 --- a/core/unittest/wrapper/utils.h +++ b/core/unittest/wrapper/utils.h @@ -25,7 +25,6 @@ #include #include - #include "wrapper/VecIndex.h" #include "wrapper/utils.h" #include "knowhere/index/vector_index/helpers/IndexParameter.h" @@ -90,17 +89,20 @@ class ParamGenerator { return instance; } - knowhere::Config GenSearchConf(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { + knowhere::Config + GenSearchConf(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type); return adapter->MatchSearch(conf, type); } - knowhere::Config GenBuild(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { + knowhere::Config + GenBuild(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) { auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type); return adapter->Match(conf); } - knowhere::Config Gen(const milvus::engine::IndexType& type) { + knowhere::Config + Gen(const milvus::engine::IndexType& type) { switch (type) { case milvus::engine::IndexType::FAISS_IDMAP: { auto tempconf = std::make_shared();