milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'add_unittest' into 'branch-0.4.0' 

MS-526 1. Fix knowhere unittest

See merge request megasearch/milvus!517

Former-commit-id: 948cfc7a2378f88670512732f064742b353392e1
pull/191/head
peng.xu 2019-09-09 15:23:51 +08:00
parent 01c2af4eb3 f333368663
commit ace2d9fc85
6 changed files with 48 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cpp/src/core/include/knowhere/index/vector_index/nsg/nsg.h
View File

@ -138,8 +138,8 @@ class NsgIndex {
void FindUnconnectedNode(boost::dynamic_bitset<> &flags, int64_t &root);
private:
void GetKnnGraphFromFile();
//private:
// void GetKnnGraphFromFile();
};
}

48
cpp/src/core/src/knowhere/index/vector_index/nsg/nsg.cpp
View File

@ -722,30 +722,30 @@ void NsgIndex::SetKnnGraph(Graph &g) {
knng = std::move(g);
}
void NsgIndex::GetKnnGraphFromFile() {
//std::string filename = "/home/zilliz/opt/workspace/wook/efanna_graph/tests/sift.1M.50NN.graph";
std::string filename = "/home/zilliz/opt/workspace/wook/efanna_graph/tests/sift.50NN.graph";
std::ifstream in(filename, std::ios::binary);
unsigned k;
in.read((char *) &k, sizeof(unsigned));
in.seekg(0, std::ios::end);
std::ios::pos_type ss = in.tellg();
size_t fsize = (size_t) ss;
size_t num = (unsigned) (fsize / (k + 1) / 4);
in.seekg(0, std::ios::beg);
knng.resize(num);
knng.reserve(num);
unsigned kk = (k + 3) / 4 * 4;
for (size_t i = 0; i < num; i++) {
in.seekg(4, std::ios::cur);
knng[i].resize(k);
knng[i].reserve(kk);
in.read((char *) knng[i].data(), k * sizeof(unsigned));
}
in.close();
}
//void NsgIndex::GetKnnGraphFromFile() {
// //std::string filename = "/home/zilliz/opt/workspace/wook/efanna_graph/tests/sift.1M.50NN.graph";
// std::string filename = "/home/zilliz/opt/workspace/wook/efanna_graph/tests/sift.50NN.graph";
//
// std::ifstream in(filename, std::ios::binary);
// unsigned k;
// in.read((char *) &k, sizeof(unsigned));
// in.seekg(0, std::ios::end);
// std::ios::pos_type ss = in.tellg();
// size_t fsize = (size_t) ss;
// size_t num = (unsigned) (fsize / (k + 1) / 4);
// in.seekg(0, std::ios::beg);
//
// knng.resize(num);
// knng.reserve(num);
// unsigned kk = (k + 3) / 4 * 4;
// for (size_t i = 0; i < num; i++) {
// in.seekg(4, std::ios::cur);
// knng[i].resize(k);
// knng[i].reserve(kk);
// in.read((char *) knng[i].data(), k * sizeof(unsigned));
// }
// in.close();
//}
}
}

2
cpp/src/core/test/CMakeLists.txt
View File

@ -78,6 +78,6 @@ install(TARGETS test_ivf DESTINATION unittest)
install(TARGETS test_idmap DESTINATION unittest)
install(TARGETS test_kdt DESTINATION unittest)
add_subdirectory(faiss_ori)
#add_subdirectory(faiss_ori)
add_subdirectory(test_nsg)

6
cpp/src/core/test/test_idmap.cpp
View File

@ -148,10 +148,6 @@ TEST_F(IDMAPTest, copy_test) {
{
// cpu to gpu
static int64_t device_id = 0;
FaissGpuResourceMgr::GetInstance().InitDevice(0);
FaissGpuResourceMgr::GetInstance().InitDevice(1);
auto clone_index = CopyCpuToGpu(index_, device_id, Config());
auto clone_result = clone_index->Search(query_dataset, Config::object{{"k", k}});
AssertAnns(clone_result, nq, k);
@ -169,7 +165,7 @@ TEST_F(IDMAPTest, copy_test) {
assert(std::static_pointer_cast<IDMAP>(host_index)->GetRawIds() != nullptr);
// gpu to gpu
auto device_index = CopyCpuToGpu(index_, 1, Config());
auto device_index = CopyCpuToGpu(index_, device_id, Config());
auto device_result = device_index->Search(query_dataset, Config::object{{"k", k}});
AssertAnns(device_result, nq, k);
//assert(std::static_pointer_cast<GPUIDMAP>(device_index)->GetRawVectors() != nullptr);

24
cpp/src/core/test/test_ivf.cpp
View File

@ -52,9 +52,9 @@ class IVFTest
void SetUp() override {
std::tie(index_type, preprocess_cfg, train_cfg, add_cfg, search_cfg) = GetParam();
//Init_with_default();
Generate(128, 1000000/5, 10);
Generate(128, 1000000/10, 10);
index_ = IndexFactory(index_type);
FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024*1024*200, 1024*1024*300, 2);
FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024*1024*200, 1024*1024*600, 2);
}
void TearDown() override {
FaissGpuResourceMgr::GetInstance().Free();
@ -77,21 +77,21 @@ INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest,
Config::object{{"nlist", 100}, {"metric_type", "L2"}},
Config(),
Config::object{{"k", 10}}),
//std::make_tuple("IVFPQ",
// Config(),
// Config::object{{"nlist", 100}, {"M", 8}, {"nbits", 8}, {"metric_type", "L2"}},
// Config(),
// Config::object{{"k", 10}}),
std::make_tuple("IVFPQ",
Config(),
Config::object{{"nlist", 100}, {"M", 8}, {"nbits", 8}, {"metric_type", "L2"}},
Config(),
Config::object{{"k", 10}}),
std::make_tuple("GPUIVF",
Config(),
Config::object{{"nlist", 1638}, {"gpu_id", device_id}, {"metric_type", "L2"}},
Config(),
Config::object{{"k", 10}}),
//std::make_tuple("GPUIVFPQ",
// Config(),
// Config::object{{"gpu_id", device_id}, {"nlist", 100}, {"M", 8}, {"nbits", 8}, {"metric_type", "L2"}},
// Config(),
// Config::object{{"k", 10}}),
std::make_tuple("GPUIVFPQ",
Config(),
Config::object{{"gpu_id", device_id}, {"nlist", 100}, {"M", 8}, {"nbits", 8}, {"metric_type", "L2"}},
Config(),
Config::object{{"k", 10}}),
std::make_tuple("IVFSQ",
Config(),
Config::object{{"nlist", 100}, {"nbits", 8}, {"metric_type", "L2"}},

8
cpp/src/core/test/test_nsg/test_nsg.cpp
View File

@ -7,6 +7,7 @@
#include <gtest/gtest.h>
#include <memory>
#include "knowhere/index/vector_index/gpu_ivf.h"
#include "knowhere/index/vector_index/nsg_index.h"
#include "knowhere/index/vector_index/nsg/nsg_io.h"
@ -18,15 +19,22 @@ using ::testing::TestWithParam;
using ::testing::Values;
using ::testing::Combine;
constexpr int64_t DEVICE_ID = 0;
class NSGInterfaceTest : public DataGen, public TestWithParam<::std::tuple<Config, Config>> {
protected:
void SetUp() override {
//Init_with_default();
FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID, 1024*1024*200, 1024*1024*300, 2);
Generate(256, 1000000, 1);
index_ = std::make_shared<NSG>();
std::tie(train_cfg, search_cfg) = GetParam();
}
void TearDown() override {
FaissGpuResourceMgr::GetInstance().Free();
}
protected:
std::shared_ptr<NSG> index_;
Config train_cfg;