From bc93735b18ca160394fc02654b868a1bf1094bb5 Mon Sep 17 00:00:00 2001 From: erdustiggen Date: Thu, 14 Nov 2019 18:48:02 +0100 Subject: [PATCH 01/10] Changed from std::cout to LOG for error messages --- core/src/scheduler/task/BuildIndexTask.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index d8602c141e..daab7c9dec 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -146,8 +146,7 @@ XBuildIndexTask::Execute() { status = meta_ptr->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; - std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough" - << std::endl; + ENGINE_LOG_ERROR << "Failed to build index, index file is too large or gpu memory is not enough"; build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = Status(DB_ERROR, msg); @@ -179,8 +178,8 @@ XBuildIndexTask::Execute() { status = meta_ptr->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; - std::cout << "ERROR: failed to persist index file: " << table_file.location_ - << ", possible out of disk space" << std::endl; + ENGINE_LOG_ERROR << "Failed to persist index file: " << table_file.location_ + << ", possible out of disk space"; build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = Status(DB_ERROR, msg); From d53385caa7b90eab2bb851d597f87bbe2c42cd43 Mon Sep 17 00:00:00 2001 From: erdustiggen Date: Thu, 14 Nov 2019 20:54:25 +0100 Subject: [PATCH 02/10] clang format fix --- core/src/scheduler/task/BuildIndexTask.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index daab7c9dec..f561fa947d 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -179,7 +179,7 @@ XBuildIndexTask::Execute() { ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; ENGINE_LOG_ERROR << "Failed to persist index file: " << table_file.location_ - << ", possible out of disk space"; + << ", possible out of disk space"; build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = Status(DB_ERROR, msg); From 4c71748bb2b423fc56a9c63c44c2f3acefae371b Mon Sep 17 00:00:00 2001 From: erdustiggen Date: Sat, 16 Nov 2019 20:55:16 +0100 Subject: [PATCH 03/10] Fix typos and grammatical errors --- core/src/wrapper/VecImpl.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/src/wrapper/VecImpl.cpp b/core/src/wrapper/VecImpl.cpp index c97900f839..931eef9dbf 100644 --- a/core/src/wrapper/VecImpl.cpp +++ b/core/src/wrapper/VecImpl.cpp @@ -26,7 +26,7 @@ /* * no parameter check in this layer. - * only responible for index combination + * only responsible for index combination */ namespace milvus { @@ -271,7 +271,7 @@ IVFHybridIndex::LoadQuantizer(const Config& conf) { if (auto new_idx = std::dynamic_pointer_cast(index_)) { return new_idx->LoadQuantizer(conf); } else { - WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); } } @@ -282,8 +282,8 @@ IVFHybridIndex::SetQuantizer(const knowhere::QuantizerPtr& q) { if (auto new_idx = std::dynamic_pointer_cast(index_)) { new_idx->SetQuantizer(q); } else { - WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); - return Status(KNOWHERE_ERROR, "not support"); + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); + return Status(KNOWHERE_ERROR, "not supported"); } } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); @@ -302,8 +302,8 @@ IVFHybridIndex::UnsetQuantizer() { if (auto new_idx = std::dynamic_pointer_cast(index_)) { new_idx->UnsetQuantizer(); } else { - WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); - return Status(KNOWHERE_ERROR, "not support"); + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); + return Status(KNOWHERE_ERROR, "not supported"); } } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); @@ -322,7 +322,7 @@ IVFHybridIndex::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { if (auto new_idx = std::dynamic_pointer_cast(index_)) { return std::make_shared(new_idx->LoadData(q, conf), type); } else { - WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); } } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); @@ -341,7 +341,7 @@ IVFHybridIndex::CopyToGpuWithQuantizer(const int64_t& device_id, const Config& c auto new_idx = std::make_shared(pair.first, type); return std::make_pair(new_idx, pair.second); } else { - WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); + WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); } } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); From 562b84a178afb90344e26bda3c021bff80e5f60c Mon Sep 17 00:00:00 2001 From: Shouyu Luo Date: Mon, 18 Nov 2019 23:03:59 +0800 Subject: [PATCH 04/10] commented out logs --- core/src/db/DBImpl.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index fc31846bd3..52cf1e668a 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -71,12 +71,12 @@ DBImpl::Start() { return Status::OK(); } - ENGINE_LOG_TRACE << "DB service start"; + // ENGINE_LOG_TRACE << "DB service start"; shutting_down_.store(false, std::memory_order_release); // for distribute version, some nodes are read only if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) { - ENGINE_LOG_TRACE << "StartTimerTasks"; + // ENGINE_LOG_TRACE << "StartTimerTasks"; bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this); } @@ -101,7 +101,7 @@ DBImpl::Stop() { meta_ptr_->CleanUp(); } - ENGINE_LOG_TRACE << "DB service stop"; + // ENGINE_LOG_TRACE << "DB service stop"; return Status::OK(); } @@ -506,7 +506,7 @@ DBImpl::StartMetricTask() { return; } - ENGINE_LOG_TRACE << "Start metric task"; + // ENGINE_LOG_TRACE << "Start metric task"; server::Metrics::GetInstance().KeepingAliveCounterIncrement(METRIC_ACTION_INTERVAL); int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); @@ -532,7 +532,7 @@ DBImpl::StartMetricTask() { server::Metrics::GetInstance().GPUTemperature(); server::Metrics::GetInstance().CPUTemperature(); - ENGINE_LOG_TRACE << "Metric task finished"; + // ENGINE_LOG_TRACE << "Metric task finished"; } Status @@ -692,7 +692,7 @@ DBImpl::BackgroundMergeFiles(const std::string& table_id) { void DBImpl::BackgroundCompaction(std::set table_ids) { - ENGINE_LOG_TRACE << " Background compaction thread start"; + // ENGINE_LOG_TRACE << " Background compaction thread start"; Status status; for (auto& table_id : table_ids) { @@ -715,7 +715,7 @@ DBImpl::BackgroundCompaction(std::set table_ids) { } meta_ptr_->CleanUpFilesWithTTL(ttl); - ENGINE_LOG_TRACE << " Background compaction thread exit"; + // ENGINE_LOG_TRACE << " Background compaction thread exit"; } void @@ -748,7 +748,7 @@ DBImpl::StartBuildIndexTask(bool force) { void DBImpl::BackgroundBuildIndex() { - ENGINE_LOG_TRACE << "Background build index thread start"; + // ENGINE_LOG_TRACE << "Background build index thread start"; std::unique_lock lock(build_index_mutex_); meta::TableFilesSchema to_index_files; @@ -771,7 +771,7 @@ DBImpl::BackgroundBuildIndex() { } } - ENGINE_LOG_TRACE << "Background build index thread exit"; + // ENGINE_LOG_TRACE << "Background build index thread exit"; } } // namespace engine From 94d11b99310599a0134c851194460e183013c8d6 Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Wed, 20 Nov 2019 15:03:44 +0800 Subject: [PATCH 05/10] [skip ci] add ivfsq8h test report detailed version #255 --- ...us_ivfsq8h_test_report_detailed_version.md | 179 +++++++++++++++++ ...ivfsq8h_test_report_detailed_version_cn.md | 180 ++++++++++++++++++ 2 files changed, 359 insertions(+) create mode 100755 docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md create mode 100644 docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md diff --git a/docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md new file mode 100755 index 0000000000..042d059684 --- /dev/null +++ b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version.md @@ -0,0 +1,179 @@ +# milvus_ivfsq8h_test_report_detailed_version + +## Summary + +This document contains the test reports of IVF_SQ8H index on Milvus single server. + + + +## Test objectives + +The time cost and recall when searching with different parameters. + + + +## Test method + +### Hardware/Software requirements + +Operating System: CentOS Linux release 7.6.1810 (Core) + +CPU: Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz + +GPU0: GeForce GTX 1080 + +GPU1: GeForce GTX 1080 + +Memory: 503GB + +Docker version: 18.09 + +NVIDIA Driver version: 430.34 + +Milvus version: 0.5.3 + +SDK interface: Python 3.6.8 + +pymilvus version: 0.2.5 + + + +### Data model + +The data used in the tests are: + +- Data source: sift1b +- Data type: hdf5 + +For details on this dataset, please check : http://corpus-texmex.irisa.fr/ . + + + +### Measures + +- Query Elapsed Time: Time cost (in seconds) to run a query. Variables that affect Query Elapsed Time: + + - nq (Number of queried vectors) + +> Note: In the query test of query elapsed time, we will test the following parameters with different values: + > + > nq - grouped by: [1, 5, 10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800]. + > + +- Recall: The fraction of the total amount of relevant instances that were actually retrieved . Variables that affect Recall: + + - nq (Number of queried vectors) + - topk (Top k result of a query) + + > Note: In the query test of recall, we will test the following parameters with different values: + > + > nq - grouped by: [10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800], + > + > topk - grouped by: [1, 10, 100] + + + +## Test reports + +### Test environment + +Data base: sift1b-1,000,000,000 vectors, 128-dimension + +Table Attributes + +- nlist: 16384 +- metric_type: L2 + +Query configuration + +- nprobe: 32 + +Milvus configuration + +- cpu_cache_capacity: 150 +- gpu_cache_capacity: 6 +- use_blas_threshold: 1100 +- gpu_search_threshold: 1200 +- search_resources: cpu, gpu0, gpu1 + +The definitions of Milvus configuration are on https://milvus.io/docs/en/reference/milvus_config/. + +Test method + +Test the query elapsed time and recall with several parameters, and once only change one parameter. + +- Whether to restart Milvus after each query: No + + + +### Performance test + +#### Data query + +**Test result** + +Query Elapsed Time + +topk = 100 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 0.34 | +| nq=5 | 0.72 | +| nq=10 | 0.91 | +| nq=50 | 1.51 | +| nq=100 | 2.49 | +| nq=200 | 4.09 | +| nq=400 | 7.32 | +| nq=600 | 10.63 | +| nq=800 | 13.84 | +| nq=1000 | 16.83 | +| nq=1200 | 18.20 | +| nq=1400 | 20.1 | +| nq=1600 | 20.0 | +| nq=1800 | 19.86 | + +When nq is 1800, the query time cost of a 128-dimension vector is around 11ms. + + + +**Conclusion** + +When nq < 1200, the query elapsed time increases quickly with nq; when nq > 1200, the query elapsed time increases much slower. It is because gpu_search_threshold is set to 1200, when nq < 1200, CPU is chosen to do the query, otherwise GPU is chosen. Compared with CPU, GPU has much more cores and stronger computing capability. When nq is large, it can better reflect GPU's advantages on computing. + +The query elapsed time consists of two parts: (1) index CPU-to-GPU copy time; (2) nprobe buckets search time. When nq is larger enough, index CPU-to-GPU copy time can be amortized efficiently. So Milvus performs well through setting suitable gpu_search_threshold. + + + +### Recall test + +**Test result** + +topk = 1 : recall - recall@1 + +topk = 10 : recall - recall@10 + +topk = 100 : recall - recall@100 + +We use the ground_truth in sift1b dataset to calculate the recall of query results. + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=50 | 0.980 | 0.950 | 0.941 | +| nq=100 | 0.970 | 0.937 | 0.931 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | +| nq=1200 | 0.937 | 0.943 | 0.931 | +| nq=1400 | 0.939 | 0.945 | 0.931 | +| nq=1600 | 0.936 | 0.945 | 0.931 | +| nq=1800 | 0.937 | 0.946 | 0.932 | + + + +**Conclusion** + +As nq increases, the recall gradually stabilizes to over 93%. The usage of CPU or GPU and different topk are not related to recall. \ No newline at end of file diff --git a/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md new file mode 100644 index 0000000000..b50d00f9bd --- /dev/null +++ b/docs/test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md @@ -0,0 +1,180 @@ +# milvus_ivfsq8h_test_report_detailed_version_cn + +## 概述 + +本文描述了ivfsq8h索引在milvus单机部署方式下的测试结果。 + + + +## 测试目标 + +参数不同情况下的查询时间和召回率。 + + + +## 测试方法 + +### 软硬件环境 + +操作系统：CentOS Linux release 7.6.1810 (Core) + +CPU：Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz + +GPU0：GeForce GTX 1080 + +GPU1：GeForce GTX 1080 + +内存：503GB + +Docker版本：18.09 + +NVIDIA Driver版本：430.34 + +Milvus版本：0.5.3 + +SDK接口：Python 3.6.8 + +pymilvus版本：0.2.5 + + + +### 数据模型 + +本测试中用到的主要数据: + +- 数据来源：sift1b +- 数据类型：hdf5 + +关于该数据集的详细信息请参考：http://corpus-texmex.irisa.fr/ 。 + + + +### 测试指标 + +- Query Elapsed Time: 数据库查询所有向量的时间（以秒计）。影响Query Elapsed Time的变量： + + - nq (被查询向量的数量) + +> 备注：在向量查询测试中，我们会测试下面参数不同的取值来观察结果： + > + > 被查询向量的数量nq将按照 [1, 5, 10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800]的数量分组。 + > + +- Recall: 实际返回的正确结果占总数之比。影响Recall的变量： + + - nq (被查询向量的数量) + - topk (单条查询中最相似的K个结果) + + > 备注：在向量准确性测试中，我们会测试下面参数不同的取值来观察结果： + > + > 被查询向量的数量nq将按照 [10, 50, 100, 200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800]的数量分组， + > + > 单条查询中最相似的K个结果topk将按照[1, 10, 100]的数量分组。 + + + +## 测试报告 + +### 测试环境 + +数据集: sift1b-1,000,000,000向量, 128维 + +表格属性： + +- nlist: 16384 +- metric_type: L2 + +查询设置： + +- nprobe: 32 + +Milvus设置： + +- cpu_cache_capacity: 150 +- gpu_cache_capacity: 6 +- use_blas_threshold: 1100 +- gpu_search_threshold: 1200 +- search_resources: cpu, gpu0, gpu1 + +Milvus设置的详细定义可以参考 https://milvus.io/docs/en/reference/milvus_config/ 。 + +测试方法 + +通过一次仅改变一个参数的值，测试查询向量时间和召回率。 + +- 查询后是否重启Milvus：否 + + + +### 性能测试 + +#### 数据查询 + +测试结果 + +Query Elapsed Time + +topk = 100 + +| nq/topk | topk=100 | +| :-----: | :------: | +| nq=1 | 0.34 | +| nq=5 | 0.72 | +| nq=10 | 0.91 | +| nq=50 | 1.51 | +| nq=100 | 2.49 | +| nq=200 | 4.09 | +| nq=400 | 7.32 | +| nq=600 | 10.63 | +| nq=800 | 13.84 | +| nq=1000 | 16.83 | +| nq=1200 | 18.20 | +| nq=1400 | 20.1 | +| nq=1600 | 20.0 | +| nq=1800 | 19.86 | + +当nq为1800时，查询一条128维向量需要耗时约11毫秒。 + + + +**总结** + +当nq小于1200时，查询耗时随nq的增长快速增大；当nq大于1200时，查询耗时的增大则缓慢许多。这是因为gpu_search_threshold这一参数的值被设为1200，当nq<1200时，选择CPU进行操作，否则选择GPU进行操作。与CPU。 + +在GPU模式下的查询耗时由两部分组成：（1）索引从CPU到GPU的拷贝时间；（2）所有分桶的查询时间。当nq小于500时，索引从CPU到GPU 的拷贝时间无法被有效均摊，此时CPU模式时一个更优的选择；当nq大于500时，选择GPU模式更合理。和CPU相比，GPU具有更多的核数和更强的算力。当nq较大时，GPU在计算上的优势能被更好地被体现。 + + + +### 召回率测试 + +**测试结果** + +topk = 1 : recall - recall@1 + +topk = 10 : recall - recall@10 + +topk = 100 : recall - recall@100 + +我们利用sift1b数据集中的ground_truth来计算查询结果的召回率。 + +| nq/topk | topk=1 | topk=10 | topk=100 | +| :-----: | :----: | :-----: | :------: | +| nq=10 | 0.900 | 0.910 | 0.939 | +| nq=50 | 0.980 | 0.950 | 0.941 | +| nq=100 | 0.970 | 0.937 | 0.931 | +| nq=200 | 0.955 | 0.941 | 0.929 | +| nq=400 | 0.958 | 0.944 | 0.932 | +| nq=600 | 0.952 | 0.946 | 0.934 | +| nq=800 | 0.941 | 0.943 | 0.930 | +| nq=1000 | 0.938 | 0.942 | 0.930 | +| nq=1200 | 0.937 | 0.943 | 0.931 | +| nq=1400 | 0.939 | 0.945 | 0.931 | +| nq=1600 | 0.936 | 0.945 | 0.931 | +| nq=1800 | 0.937 | 0.946 | 0.932 | + + + +**总结** + +随着nq的增大，召回率逐渐稳定至93%以上。CPU/GPU的使用以及topk的值与召回率的大小无关。 + From a77bd78924fb6fe0d6be82825ba79dbfaa7f969e Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Wed, 20 Nov 2019 16:05:14 +0800 Subject: [PATCH 06/10] Fix compiling error --- core/src/wrapper/VecImpl.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/wrapper/VecImpl.cpp b/core/src/wrapper/VecImpl.cpp index 2085daed24..9918a9a025 100644 --- a/core/src/wrapper/VecImpl.cpp +++ b/core/src/wrapper/VecImpl.cpp @@ -21,11 +21,13 @@ #include "knowhere/index/vector_index/IndexIDMAP.h" #include "utils/Log.h" #include "wrapper/WrapperException.h" +#include "wrapper/gpu/GPUVecImpl.h" #ifdef MILVUS_GPU_VERSION -#include -#include +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/helpers/Cloner.h" +#include "knowhere/index/vector_index/IndexIVFSQHybrid.h" #endif From bfd4c6c5224f35682725874b5c53a734fde276c4 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Wed, 20 Nov 2019 16:08:18 +0800 Subject: [PATCH 07/10] upgrade SPTAG and support KDT and BKT --- core/src/db/engine/ExecutionEngine.h | 4 +- core/src/db/engine/ExecutionEngineImpl.cpp | 8 + core/src/index/knowhere/CMakeLists.txt | 4 +- .../knowhere/index/vector_index/IndexKDT.cpp | 180 --------- .../knowhere/index/vector_index/IndexKDT.h | 87 ----- .../index/vector_index/IndexSPTAG.cpp | 348 ++++++++++++++++++ .../knowhere/index/vector_index/IndexSPTAG.h | 93 +++++ .../vector_index/helpers/IndexParameter.h | 79 +++- .../vector_index/helpers/KDTParameterMgr.cpp | 55 --- .../helpers/SPTAGParameterMgr.cpp | 75 ++++ ...{KDTParameterMgr.h => SPTAGParameterMgr.h} | 47 ++- .../AnnService/inc/Core/Common/Dataset.h | 2 +- core/src/index/unittest/CMakeLists.txt | 16 +- core/src/index/unittest/test_kdt.cpp | 144 -------- core/src/index/unittest/test_sptag.cpp | 154 ++++++++ core/src/index/unittest/utils.cpp | 14 +- core/src/wrapper/ConfAdapter.cpp | 30 ++ core/src/wrapper/ConfAdapter.h | 18 + core/src/wrapper/ConfAdapterMgr.cpp | 3 + core/src/wrapper/VecIndex.cpp | 8 +- core/src/wrapper/VecIndex.h | 4 + core/unittest/wrapper/test_wrapper.cpp | 33 +- 22 files changed, 883 insertions(+), 523 deletions(-) delete mode 100644 core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.cpp delete mode 100644 core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.h create mode 100644 core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp create mode 100644 core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h delete mode 100644 core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp create mode 100644 core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp rename core/src/index/knowhere/knowhere/index/vector_index/helpers/{KDTParameterMgr.h => SPTAGParameterMgr.h} (55%) delete mode 100644 core/src/index/unittest/test_kdt.cpp create mode 100644 core/src/index/unittest/test_sptag.cpp diff --git a/core/src/db/engine/ExecutionEngine.h b/core/src/db/engine/ExecutionEngine.h index 86a014cf66..c8784e8a90 100644 --- a/core/src/db/engine/ExecutionEngine.h +++ b/core/src/db/engine/ExecutionEngine.h @@ -35,7 +35,9 @@ enum class EngineType { NSG_MIX, FAISS_IVFSQ8H, FAISS_PQ, - MAX_VALUE = FAISS_PQ, + SPTAG_KDT, + SPTAG_BKT, + MAX_VALUE = SPTAG_BKT, }; enum class MetricType { diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 3a2b940cec..e2db7179be 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -124,6 +124,14 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) { #endif break; } + case EngineType::SPTAG_KDT: { + index = GetVecIndexFactory(IndexType::SPTAG_KDT_RNT_CPU); + break; + } + case EngineType::SPTAG_BKT: { + index = GetVecIndexFactory(IndexType::SPTAG_BKT_RNT_CPU); + break; + } default: { ENGINE_LOG_ERROR << "Unsupported index type"; return nullptr; diff --git a/core/src/index/knowhere/CMakeLists.txt b/core/src/index/knowhere/CMakeLists.txt index 5f8e4d6970..285461bdef 100644 --- a/core/src/index/knowhere/CMakeLists.txt +++ b/core/src/index/knowhere/CMakeLists.txt @@ -30,10 +30,10 @@ set(external_srcs set(index_srcs knowhere/index/preprocessor/Normalize.cpp - knowhere/index/vector_index/IndexKDT.cpp + knowhere/index/vector_index/IndexSPTAG.cpp knowhere/index/vector_index/IndexIDMAP.cpp knowhere/index/vector_index/IndexIVF.cpp - knowhere/index/vector_index/helpers/KDTParameterMgr.cpp + knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp knowhere/index/vector_index/IndexNSG.cpp knowhere/index/vector_index/nsg/NSG.cpp knowhere/index/vector_index/nsg/NSGIO.cpp diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.cpp deleted file mode 100644 index 1bd45075e3..0000000000 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.cpp +++ /dev/null @@ -1,180 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include -#include -#include -#include - -#undef mkdir - -#include "knowhere/index/vector_index/IndexKDT.h" -#include "knowhere/index/vector_index/helpers/Definitions.h" -//#include "knowhere/index/preprocessor/normalize.h" -#include "knowhere/adapter/SptagAdapter.h" -#include "knowhere/common/Exception.h" -#include "knowhere/index/vector_index/helpers/KDTParameterMgr.h" - -namespace knowhere { - -BinarySet -CPUKDTRNG::Serialize() { - std::vector index_blobs; - std::vector index_len; - - // TODO(zirui): dev - // index_ptr_->SaveIndexToMemory(index_blobs, index_len); - BinarySet binary_set; - - // - // auto sample = std::make_shared(); - // sample.reset(static_cast(index_blobs[0])); - // auto tree = std::make_shared(); - // tree.reset(static_cast(index_blobs[1])); - // auto graph = std::make_shared(); - // graph.reset(static_cast(index_blobs[2])); - // auto metadata = std::make_shared(); - // metadata.reset(static_cast(index_blobs[3])); - // - // binary_set.Append("samples", sample, index_len[0]); - // binary_set.Append("tree", tree, index_len[1]); - // binary_set.Append("graph", graph, index_len[2]); - // binary_set.Append("metadata", metadata, index_len[3]); - return binary_set; -} - -void -CPUKDTRNG::Load(const BinarySet& binary_set) { - // TODO(zirui): dev - - // std::vector index_blobs; - // - // auto samples = binary_set.GetByName("samples"); - // index_blobs.push_back(samples->data.get()); - // - // auto tree = binary_set.GetByName("tree"); - // index_blobs.push_back(tree->data.get()); - // - // auto graph = binary_set.GetByName("graph"); - // index_blobs.push_back(graph->data.get()); - // - // auto metadata = binary_set.GetByName("metadata"); - // index_blobs.push_back(metadata->data.get()); - // - // index_ptr_->LoadIndexFromMemory(index_blobs); -} - -// PreprocessorPtr -// CPUKDTRNG::BuildPreprocessor(const DatasetPtr &dataset, const Config &config) { -// return std::make_shared(); -//} - -IndexModelPtr -CPUKDTRNG::Train(const DatasetPtr& origin, const Config& train_config) { - SetParameters(train_config); - DatasetPtr dataset = origin->Clone(); - - // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine - // && preprocessor_) { - // preprocessor_->Preprocess(dataset); - //} - - auto vectorset = ConvertToVectorSet(dataset); - auto metaset = ConvertToMetadataSet(dataset); - index_ptr_->BuildIndex(vectorset, metaset); - - // TODO: return IndexModelPtr - return nullptr; -} - -void -CPUKDTRNG::Add(const DatasetPtr& origin, const Config& add_config) { - SetParameters(add_config); - DatasetPtr dataset = origin->Clone(); - - // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine - // && preprocessor_) { - // preprocessor_->Preprocess(dataset); - //} - - auto vectorset = ConvertToVectorSet(dataset); - auto metaset = ConvertToMetadataSet(dataset); - index_ptr_->AddIndex(vectorset, metaset); -} - -void -CPUKDTRNG::SetParameters(const Config& config) { - for (auto& para : KDTParameterMgr::GetInstance().GetKDTParameters()) { - // auto value = config.get_with_default(para.first, para.second); - index_ptr_->SetParameter(para.first, para.second); - } -} - -DatasetPtr -CPUKDTRNG::Search(const DatasetPtr& dataset, const Config& config) { - SetParameters(config); - auto tensor = dataset->tensor()[0]; - auto p = (float*)tensor->raw_mutable_data(); - for (auto i = 0; i < 10; ++i) { - for (auto j = 0; j < 10; ++j) { - std::cout << p[i * 10 + j] << " "; - } - std::cout << std::endl; - } - std::vector query_results = ConvertToQueryResult(dataset, config); - -#pragma omp parallel for - for (auto i = 0; i < query_results.size(); ++i) { - auto target = (float*)query_results[i].GetTarget(); - std::cout << target[0] << ", " << target[1] << ", " << target[2] << std::endl; - index_ptr_->SearchIndex(query_results[i]); - } - - return ConvertToDataset(query_results); -} - -int64_t -CPUKDTRNG::Count() { - index_ptr_->GetNumSamples(); -} - -int64_t -CPUKDTRNG::Dimension() { - index_ptr_->GetFeatureDim(); -} - -VectorIndexPtr -CPUKDTRNG::Clone() { - KNOWHERE_THROW_MSG("not support"); -} - -void -CPUKDTRNG::Seal() { - // do nothing -} - -// TODO(linxj): -BinarySet -CPUKDTRNGIndexModel::Serialize() { -} - -void -CPUKDTRNGIndexModel::Load(const BinarySet& binary) { -} - -} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.h deleted file mode 100644 index f6d436995b..0000000000 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.h +++ /dev/null @@ -1,87 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include "VectorIndex.h" -#include "knowhere/index/IndexModel.h" - -namespace knowhere { - -class CPUKDTRNG : public VectorIndex { - public: - CPUKDTRNG() { - index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::KDT, SPTAG::VectorValueType::Float); - index_ptr_->SetParameter("DistCalcMethod", "L2"); - } - - public: - BinarySet - Serialize() override; - VectorIndexPtr - Clone() override; - void - Load(const BinarySet& index_array) override; - - public: - // PreprocessorPtr - // BuildPreprocessor(const DatasetPtr &dataset, const Config &config) override; - int64_t - Count() override; - int64_t - Dimension() override; - - IndexModelPtr - Train(const DatasetPtr& dataset, const Config& config) override; - - void - Add(const DatasetPtr& dataset, const Config& config) override; - - DatasetPtr - Search(const DatasetPtr& dataset, const Config& config) override; - void - Seal() override; - - private: - void - SetParameters(const Config& config); - - private: - PreprocessorPtr preprocessor_; - std::shared_ptr index_ptr_; -}; - -using CPUKDTRNGPtr = std::shared_ptr; - -class CPUKDTRNGIndexModel : public IndexModel { - public: - BinarySet - Serialize() override; - - void - Load(const BinarySet& binary) override; - - private: - std::shared_ptr index_; -}; - -using CPUKDTRNGIndexModelPtr = std::shared_ptr; - -} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp new file mode 100644 index 0000000000..c64baebb93 --- /dev/null +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp @@ -0,0 +1,348 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include + +#include +#include +#include + +#undef mkdir + +#include "knowhere/index/vector_index/IndexSPTAG.h" +#include "knowhere/index/vector_index/helpers/Definitions.h" +#include "knowhere/adapter/SptagAdapter.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/helpers/SPTAGParameterMgr.h" + +namespace knowhere { + + CPUSPTAGRNG::CPUSPTAGRNG(const std::string& IndexType) { + if (IndexType == "KDT") { + index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::KDT, SPTAG::VectorValueType::Float); + index_ptr_->SetParameter("DistCalcMethod", "L2"); + index_type_ = SPTAG::IndexAlgoType::KDT; + } else { + index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::BKT, SPTAG::VectorValueType::Float); + index_ptr_->SetParameter("DistCalcMethod", "L2"); + index_type_ = SPTAG::IndexAlgoType::BKT; + } + } + + BinarySet + CPUSPTAGRNG::Serialize() { + std::string index_config; + std::vector index_blobs; + + std::shared_ptr> buffersize = index_ptr_->CalculateBufferSize(); + std::vector res(buffersize->size() + 1); + for (uint64_t i = 1; i < res.size(); i++) { + res[i] = new char[buffersize->at(i - 1)]; + auto ptr = &res[i][0]; + index_blobs.emplace_back(SPTAG::ByteArray((std::uint8_t*)ptr, buffersize->at(i - 1), false)); + } + + index_ptr_->SaveIndex(index_config, index_blobs); + + size_t length = index_config.length(); + char* cstr = new char[length]; + snprintf(cstr, length, "%s", index_config.c_str()); + + BinarySet binary_set; + auto sample = std::make_shared(); + sample.reset(static_cast(index_blobs[0].Data())); + auto tree = std::make_shared(); + tree.reset(static_cast(index_blobs[1].Data())); + auto graph = std::make_shared(); + graph.reset(static_cast(index_blobs[2].Data())); + auto deleteid = std::make_shared(); + deleteid.reset(static_cast(index_blobs[3].Data())); + auto metadata1 = std::make_shared(); + metadata1.reset(static_cast(index_blobs[4].Data())); + auto metadata2 = std::make_shared(); + metadata2.reset(static_cast(index_blobs[5].Data())); + auto config = std::make_shared(); + config.reset(static_cast((void*)cstr)); + + binary_set.Append("samples", sample, index_blobs[0].Length()); + binary_set.Append("tree", tree, index_blobs[1].Length()); + binary_set.Append("deleteid", deleteid, index_blobs[3].Length()); + binary_set.Append("metadata1", metadata1, index_blobs[4].Length()); + binary_set.Append("metadata2", metadata2, index_blobs[5].Length()); + binary_set.Append("config", config, length); + binary_set.Append("graph", graph, index_blobs[2].Length()); + +// MemoryIOWriter writer; +// size_t len = 0; +// for (int i = 0; i < 6; ++i) { +// len = index_blobs[i].Length(); +// assert(len != 0); +// writer(&len, sizeof(size_t), 1); +// writer(index_blobs[i].Data(), len, 1); +// len = 0; +// } +// writer(&length, sizeof(size_t), 1); +// writer(cstr, length, 1); +// auto data = std::make_shared(); +// data.reset(writer.data_); +// BinarySet binary_set; +// binary_set.Append("sptag", data, writer.total); + +// MemoryIOWriter writer; +// size_t len = 0; +// for (int i = 0; i < 6; ++i) { +// if (i == 2) continue; +// len = index_blobs[i].Length(); +// assert(len != 0); +// writer(&len, sizeof(size_t), 1); +// writer(index_blobs[i].Data(), len, 1); +// len = 0; +// } +// writer(&length, sizeof(size_t), 1); +// writer(cstr, length, 1); +// auto data = std::make_shared(); +// data.reset(writer.data_); +// BinarySet binary_set; +// binary_set.Append("sptag", data, writer.total); +// auto graph = std::make_shared(); +// graph.reset(static_cast(index_blobs[2].Data())); +// binary_set.Append("graph", graph, index_blobs[2].Length()); + + return binary_set; + } + + void + CPUSPTAGRNG::Load(const BinarySet& binary_set) { + std::string index_config; + std::vector index_blobs; + + auto samples = binary_set.GetByName("samples"); + index_blobs.push_back(SPTAG::ByteArray(samples->data.get(), samples->size, false)); + + auto tree = binary_set.GetByName("tree"); + index_blobs.push_back(SPTAG::ByteArray(tree->data.get(), tree->size, false)); + + auto graph = binary_set.GetByName("graph"); + index_blobs.push_back(SPTAG::ByteArray(graph->data.get(), graph->size, false)); + + auto deleteid = binary_set.GetByName("deleteid"); + index_blobs.push_back(SPTAG::ByteArray(deleteid->data.get(), deleteid->size, false)); + + auto metadata1 = binary_set.GetByName("metadata1"); + index_blobs.push_back(SPTAG::ByteArray(metadata1->data.get(), metadata1->size, false)); + + auto metadata2 = binary_set.GetByName("metadata2"); + index_blobs.push_back(SPTAG::ByteArray(metadata2->data.get(), metadata2->size, false)); + + auto config = binary_set.GetByName("config"); + index_config = reinterpret_cast(config->data.get()); + +// std::vector index_blobs; +// auto data = binary_set.GetByName("sptag"); +// MemoryIOReader reader; +// reader.total = data->size; +// reader.data_ = data->data.get(); +// size_t len = 0; +// for (int i = 0; i < 6; ++i) { +// reader(&len, sizeof(size_t), 1); +// assert(len != 0); +// auto binary = new uint8_t[len]; +// reader(binary, len, 1); +// index_blobs.emplace_back(SPTAG::ByteArray(binary, len, true)); +// len = 0; +// } +// reader(&len, sizeof(size_t), 1); +// assert(len != 0); +// auto config = new char[len]; +// reader(config, len, 1); +// std::string index_config = config; +// delete[] config; + +// std::vector index_blobs; +// auto data = binary_set.GetByName("sptag"); +// MemoryIOReader reader; +// reader.total = data->size; +// reader.data_ = data->data.get(); +// size_t len = 0; +// for (int i = 0; i < 6; ++i) { +// if (i == 2) { +// auto graph = binary_set.GetByName("graph"); +// index_blobs.emplace_back(SPTAG::ByteArray(graph->data.get(), graph->size, false)); +// continue; +// } +// reader(&len, sizeof(size_t), 1); +// assert(len != 0); +// auto binary = new uint8_t[len]; +// reader(binary, len, 1); +// index_blobs.emplace_back(SPTAG::ByteArray(binary, len, true)); +// len = 0; +// } +// reader(&len, sizeof(size_t), 1); +// assert(len != 0); +// auto config = new char[len]; +// reader(config, len, 1); +// std::string index_config = config; +// delete[] config; + index_ptr_->LoadIndex(index_config, index_blobs); + } + +// PreprocessorPtr +// CPUKDTRNG::BuildPreprocessor(const DatasetPtr &dataset, const Config &config) { +// return std::make_shared(); +//} + + IndexModelPtr + CPUSPTAGRNG::Train(const DatasetPtr& origin, const Config& train_config) { + SetParameters(train_config); + DatasetPtr dataset = origin->Clone(); + + // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine + // && preprocessor_) { + // preprocessor_->Preprocess(dataset); + //} + + auto vectorset = ConvertToVectorSet(dataset); + auto metaset = ConvertToMetadataSet(dataset); + index_ptr_->BuildIndex(vectorset, metaset); + + // TODO: return IndexModelPtr + return nullptr; + } + + void + CPUSPTAGRNG::Add(const DatasetPtr& origin, const Config& add_config) { + // SetParameters(add_config); + // DatasetPtr dataset = origin->Clone(); + // + // // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine + // // && preprocessor_) { + // // preprocessor_->Preprocess(dataset); + // //} + // + // auto vectorset = ConvertToVectorSet(dataset); + // auto metaset = ConvertToMetadataSet(dataset); + // index_ptr_->AddIndex(vectorset, metaset); + } + + void + CPUSPTAGRNG::SetParameters(const Config& config) { +#define Assign(param_name, str_name) \ + conf->param_name == INVALID_VALUE ? index_ptr_->SetParameter(str_name, std::to_string(build_cfg->param_name)) \ + : index_ptr_->SetParameter(str_name, std::to_string(conf->param_name)) + + if (index_type_ == SPTAG::IndexAlgoType::KDT) { + auto conf = std::dynamic_pointer_cast(config); + auto build_cfg = SPTAGParameterMgr::GetInstance().GetKDTParameters(); + + Assign(kdtnumber, "KDTNumber"); + Assign(numtopdimensionkdtsplit, "NumTopDimensionKDTSplit"); + Assign(samples, "Samples"); + Assign(tptnumber, "TPTNumber"); + Assign(tptleafsize, "TPTLeafSize"); + Assign(numtopdimensiontptsplit, "NumTopDimensionTPTSplit"); + Assign(neighborhoodsize, "NeighborhoodSize"); + Assign(graphneighborhoodscale, "GraphNeighborhoodScale"); + Assign(graphcefscale, "GraphCEFScale"); + Assign(refineiterations, "RefineIterations"); + Assign(cef, "CEF"); + Assign(maxcheckforrefinegraph, "MaxCheckForRefineGraph"); + Assign(numofthreads, "NumberOfThreads"); + Assign(maxcheck, "MaxCheck"); + Assign(thresholdofnumberofcontinuousnobetterpropagation, "ThresholdOfNumberOfContinuousNoBetterPropagation"); + Assign(numberofinitialdynamicpivots, "NumberOfInitialDynamicPivots"); + Assign(numberofotherdynamicpivots, "NumberOfOtherDynamicPivots"); + } else { + auto conf = std::dynamic_pointer_cast(config); + auto build_cfg = SPTAGParameterMgr::GetInstance().GetBKTParameters(); + + Assign(bktnumber, "BKTNumber"); + Assign(bktkmeansk, "BKTKMeansK"); + Assign(bktleafsize, "BKTLeafSize"); + Assign(samples, "Samples"); + Assign(tptnumber, "TPTNumber"); + Assign(tptleafsize, "TPTLeafSize"); + Assign(numtopdimensiontptsplit, "NumTopDimensionTPTSplit"); + Assign(neighborhoodsize, "NeighborhoodSize"); + Assign(graphneighborhoodscale, "GraphNeighborhoodScale"); + Assign(graphcefscale, "GraphCEFScale"); + Assign(refineiterations, "RefineIterations"); + Assign(cef, "CEF"); + Assign(maxcheckforrefinegraph, "MaxCheckForRefineGraph"); + Assign(numofthreads, "NumberOfThreads"); + Assign(maxcheck, "MaxCheck"); + Assign(thresholdofnumberofcontinuousnobetterpropagation, "ThresholdOfNumberOfContinuousNoBetterPropagation"); + Assign(numberofinitialdynamicpivots, "NumberOfInitialDynamicPivots"); + Assign(numberofotherdynamicpivots, "NumberOfOtherDynamicPivots"); + } + } + + DatasetPtr + CPUSPTAGRNG::Search(const DatasetPtr& dataset, const Config& config) { + SetParameters(config); + auto tensor = dataset->tensor()[0]; + auto p = (float*)tensor->raw_mutable_data(); + for (auto i = 0; i < 10; ++i) { + for (auto j = 0; j < 10; ++j) { + std::cout << p[i * 10 + j] << " "; + } + std::cout << std::endl; + } + std::vector query_results = ConvertToQueryResult(dataset, config); + +#pragma omp parallel for + for (auto i = 0; i < query_results.size(); ++i) { + auto target = (float*)query_results[i].GetTarget(); + std::cout << target[0] << ", " << target[1] << ", " << target[2] << std::endl; + index_ptr_->SearchIndex(query_results[i]); + } + + return ConvertToDataset(query_results); + } + + int64_t + CPUSPTAGRNG::Count() { + return index_ptr_->GetNumSamples(); + } + + int64_t + CPUSPTAGRNG::Dimension() { + return index_ptr_->GetFeatureDim(); + } + + VectorIndexPtr + CPUSPTAGRNG::Clone() { + KNOWHERE_THROW_MSG("not support"); + } + + void + CPUSPTAGRNG::Seal() { + return; // do nothing + } + + BinarySet + CPUSPTAGRNGIndexModel::Serialize() { + // KNOWHERE_THROW_MSG("not support"); // not support + } + + void + CPUSPTAGRNGIndexModel::Load(const BinarySet& binary) { + // KNOWHERE_THROW_MSG("not support"); // not support + } + +} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h new file mode 100644 index 0000000000..c78807a3ee --- /dev/null +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include + +#include "VectorIndex.h" +#include "knowhere/index/IndexModel.h" + +namespace knowhere { + + class CPUSPTAGRNG : public VectorIndex { + public: + explicit CPUSPTAGRNG(const std::string& IndexType); + + public: + BinarySet + Serialize() override; + + VectorIndexPtr + Clone() override; + + void + Load(const BinarySet& index_array) override; + + public: + // PreprocessorPtr + // BuildPreprocessor(const DatasetPtr &dataset, const Config &config) override; + + int64_t + Count() override; + + int64_t + Dimension() override; + + IndexModelPtr + Train(const DatasetPtr& dataset, const Config& config) override; + + void + Add(const DatasetPtr& dataset, const Config& config) override; + + DatasetPtr + Search(const DatasetPtr& dataset, const Config& config) override; + + void + Seal() override; + + private: + void + SetParameters(const Config& config); + + private: + PreprocessorPtr preprocessor_; + std::shared_ptr index_ptr_; + SPTAG::IndexAlgoType index_type_; + }; + + using CPUSPTAGRNGPtr = std::shared_ptr; + + class CPUSPTAGRNGIndexModel : public IndexModel { + public: + BinarySet + Serialize() override; + + void + Load(const BinarySet& binary) override; + + private: + std::shared_ptr index_; + }; + + using CPUSPTAGRNGIndexModelPtr = std::shared_ptr; + +} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h index b2854abef8..9bb3cac303 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h @@ -42,6 +42,32 @@ constexpr int64_t DEFAULT_OUT_DEGREE = INVALID_VALUE; constexpr int64_t DEFAULT_CANDIDATE_SISE = INVALID_VALUE; constexpr int64_t DEFAULT_NNG_K = INVALID_VALUE; +// SPTAG Config +constexpr int64_t DEFAULT_SAMPLES = INVALID_VALUE; +constexpr int64_t DEFAULT_TPTNUMBER = INVALID_VALUE; +constexpr int64_t DEFAULT_TPTLEAFSIZE = INVALID_VALUE; +constexpr int64_t DEFAULT_NUMTOPDIMENSIONTPTSPLIT = INVALID_VALUE; +constexpr int64_t DEFAULT_NEIGHBORHOODSIZE = INVALID_VALUE; +constexpr int64_t DEFAULT_GRAPHNEIGHBORHOODSCALE = INVALID_VALUE; +constexpr int64_t DEFAULT_GRAPHCEFSCALE = INVALID_VALUE; +constexpr int64_t DEFAULT_REFINEITERATIONS = INVALID_VALUE; +constexpr int64_t DEFAULT_CEF = INVALID_VALUE; +constexpr int64_t DEFAULT_MAXCHECKFORREFINEGRAPH = INVALID_VALUE; +constexpr int64_t DEFAULT_NUMOFTHREADS = INVALID_VALUE; +constexpr int64_t DEFAULT_MAXCHECK = INVALID_VALUE; +constexpr int64_t DEFAULT_THRESHOLDOFNUMBEROFCONTINUOUSNOBETTERPROPAGATION = INVALID_VALUE; +constexpr int64_t DEFAULT_NUMBEROFINITIALDYNAMICPIVOTS = INVALID_VALUE; +constexpr int64_t DEFAULT_NUMBEROFOTHERDYNAMICPIVOTS = INVALID_VALUE; + +// KDT Config +constexpr int64_t DEFAULT_KDTNUMBER = INVALID_VALUE; +constexpr int64_t DEFAULT_NUMTOPDIMENSIONKDTSPLIT = INVALID_VALUE; + +// BKT Config +constexpr int64_t DEFAULT_BKTNUMBER = INVALID_VALUE; +constexpr int64_t DEFAULT_BKTKMEANSK = INVALID_VALUE; +constexpr int64_t DEFAULT_BKTLEAFSIZE = INVALID_VALUE; + struct IVFCfg : public Cfg { int64_t nlist = DEFAULT_NLIST; int64_t nprobe = DEFAULT_NPROBE; @@ -126,8 +152,57 @@ struct NSGCfg : public IVFCfg { }; using NSGConfig = std::shared_ptr; -struct KDTCfg : public Cfg { - int64_t tptnubmber = -1; +struct SPTAGCfg : public Cfg { + int64_t samples = DEFAULT_SAMPLES; + int64_t tptnumber = DEFAULT_TPTNUMBER; + int64_t tptleafsize = DEFAULT_TPTLEAFSIZE; + int64_t numtopdimensiontptsplit = DEFAULT_NUMTOPDIMENSIONTPTSPLIT; + int64_t neighborhoodsize = DEFAULT_NEIGHBORHOODSIZE; + int64_t graphneighborhoodscale = DEFAULT_GRAPHNEIGHBORHOODSCALE; + int64_t graphcefscale = DEFAULT_GRAPHCEFSCALE; + int64_t refineiterations = DEFAULT_REFINEITERATIONS; + int64_t cef = DEFAULT_CEF; + int64_t maxcheckforrefinegraph = DEFAULT_MAXCHECKFORREFINEGRAPH; + int64_t numofthreads = DEFAULT_NUMOFTHREADS; + int64_t maxcheck = DEFAULT_MAXCHECK; + int64_t thresholdofnumberofcontinuousnobetterpropagation = DEFAULT_THRESHOLDOFNUMBEROFCONTINUOUSNOBETTERPROPAGATION; + int64_t numberofinitialdynamicpivots = DEFAULT_NUMBEROFINITIALDYNAMICPIVOTS; + int64_t numberofotherdynamicpivots = DEFAULT_NUMBEROFOTHERDYNAMICPIVOTS; + + SPTAGCfg() = default; + + bool + CheckValid() override { + return true; + }; }; +using SPTAGConfig = std::shared_ptr; + +struct KDTCfg : public SPTAGCfg { + int64_t kdtnumber = DEFAULT_KDTNUMBER; + int64_t numtopdimensionkdtsplit = DEFAULT_NUMTOPDIMENSIONKDTSPLIT; + + KDTCfg() = default; + + bool + CheckValid() override { + return true; + }; +}; +using KDTConfig = std::shared_ptr; + +struct BKTCfg : public SPTAGCfg { + int64_t bktnumber = DEFAULT_BKTNUMBER; + int64_t bktkmeansk = DEFAULT_BKTKMEANSK; + int64_t bktleafsize = DEFAULT_BKTLEAFSIZE; + + BKTCfg() = default; + + bool + CheckValid() override { + return true; + }; +}; +using BKTConfig = std::shared_ptr; } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp deleted file mode 100644 index 19bf070dba..0000000000 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include - -#include "knowhere/index/vector_index/helpers/KDTParameterMgr.h" - -namespace knowhere { - -const std::vector& -KDTParameterMgr::GetKDTParameters() { - return kdt_parameters_; -} - -KDTParameterMgr::KDTParameterMgr() { - kdt_parameters_ = std::vector{ - {"KDTNumber", "1"}, - {"NumTopDimensionKDTSplit", "5"}, - {"NumSamplesKDTSplitConsideration", "100"}, - - {"TPTNumber", "1"}, - {"TPTLeafSize", "2000"}, - {"NumTopDimensionTPTSplit", "5"}, - - {"NeighborhoodSize", "32"}, - {"GraphNeighborhoodScale", "2"}, - {"GraphCEFScale", "2"}, - {"RefineIterations", "0"}, - {"CEF", "1000"}, - {"MaxCheckForRefineGraph", "10000"}, - - {"NumberOfThreads", "1"}, - - {"MaxCheck", "8192"}, - {"ThresholdOfNumberOfContinuousNoBetterPropagation", "3"}, - {"NumberOfInitialDynamicPivots", "50"}, - {"NumberOfOtherDynamicPivots", "4"}, - }; -} - -} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp new file mode 100644 index 0000000000..f6e1409abd --- /dev/null +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "knowhere/index/vector_index/helpers/SPTAGParameterMgr.h" + +namespace knowhere { + + const KDTConfig& + SPTAGParameterMgr::GetKDTParameters() { + return kdt_config_; + } + + const BKTConfig& + SPTAGParameterMgr::GetBKTParameters() { + return bkt_config_; + } + + SPTAGParameterMgr::SPTAGParameterMgr() { + kdt_config_ = std::make_shared(); + kdt_config_->kdtnumber = 1; + kdt_config_->numtopdimensionkdtsplit = 5; + kdt_config_->samples = 100; + kdt_config_->tptnumber = 1; + kdt_config_->tptleafsize = 2000; + kdt_config_->numtopdimensiontptsplit = 5; + kdt_config_->neighborhoodsize = 32; + kdt_config_->graphneighborhoodscale = 2; + kdt_config_->graphcefscale = 2; + kdt_config_->refineiterations = 0; + kdt_config_->cef = 1000; + kdt_config_->maxcheckforrefinegraph = 10000; + kdt_config_->numofthreads = 1; + kdt_config_->maxcheck = 8192; + kdt_config_->thresholdofnumberofcontinuousnobetterpropagation = 3; + kdt_config_->numberofinitialdynamicpivots = 50; + kdt_config_->numberofotherdynamicpivots = 4; + + bkt_config_ = std::make_shared(); + bkt_config_->bktnumber = 1; + bkt_config_->bktkmeansk = 32; + bkt_config_->bktleafsize = 8; + bkt_config_->samples = 100; + bkt_config_->tptnumber = 1; + bkt_config_->tptleafsize = 2000; + bkt_config_->numtopdimensiontptsplit = 5; + bkt_config_->neighborhoodsize = 32; + bkt_config_->graphneighborhoodscale = 2; + bkt_config_->graphcefscale = 2; + bkt_config_->refineiterations = 0; + bkt_config_->cef = 1000; + bkt_config_->maxcheckforrefinegraph = 10000; + bkt_config_->numofthreads = 1; + bkt_config_->maxcheck = 8192; + bkt_config_->thresholdofnumberofcontinuousnobetterpropagation = 3; + bkt_config_->numberofinitialdynamicpivots = 50; + bkt_config_->numberofotherdynamicpivots = 4; + } + +} // namespace knowhere \ No newline at end of file diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h similarity index 55% rename from core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h rename to core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h index fe90761e17..d215d9e275 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h @@ -22,31 +22,40 @@ #include #include +#include +#include "IndexParameter.h" + namespace knowhere { -using KDTParameter = std::pair; + using KDTConfig = std::shared_ptr; + using BKTConfig = std::shared_ptr; -class KDTParameterMgr { - public: - const std::vector& - GetKDTParameters(); + class SPTAGParameterMgr { + public: + const KDTConfig& + GetKDTParameters(); - public: - static KDTParameterMgr& - GetInstance() { - static KDTParameterMgr instance; - return instance; - } + const BKTConfig& + GetBKTParameters(); - KDTParameterMgr(const KDTParameterMgr&) = delete; - KDTParameterMgr& - operator=(const KDTParameterMgr&) = delete; + public: + static SPTAGParameterMgr& + GetInstance() { + static SPTAGParameterMgr instance; + return instance; + } - private: - KDTParameterMgr(); + SPTAGParameterMgr(const SPTAGParameterMgr&) = delete; - private: - std::vector kdt_parameters_; -}; + SPTAGParameterMgr& + operator=(const SPTAGParameterMgr&) = delete; + + private: + SPTAGParameterMgr(); + + private: + KDTConfig kdt_config_; + BKTConfig bkt_config_; + }; } // namespace knowhere diff --git a/core/src/index/thirdparty/SPTAG/AnnService/inc/Core/Common/Dataset.h b/core/src/index/thirdparty/SPTAG/AnnService/inc/Core/Common/Dataset.h index d00ea45365..0208f6d983 100644 --- a/core/src/index/thirdparty/SPTAG/AnnService/inc/Core/Common/Dataset.h +++ b/core/src/index/thirdparty/SPTAG/AnnService/inc/Core/Common/Dataset.h @@ -195,7 +195,7 @@ namespace SPTAG C = *((DimensionType*)pDataPointsMemFile); pDataPointsMemFile += sizeof(DimensionType); - Initialize(R, C, (T*)pDataPointsMemFile); + Initialize(R, C, (T*)pDataPointsMemFile, false); std::cout << "Load " << name << " (" << R << ", " << C << ") Finish!" << std::endl; return true; } diff --git a/core/src/index/unittest/CMakeLists.txt b/core/src/index/unittest/CMakeLists.txt index d3ba84e557..93ae63a9ec 100644 --- a/core/src/index/unittest/CMakeLists.txt +++ b/core/src/index/unittest/CMakeLists.txt @@ -82,17 +82,17 @@ if (NOT TARGET test_idmap) endif () target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) -# -set(kdt_srcs +# +set(sptag_srcs ${INDEX_SOURCE_DIR}/knowhere/knowhere/adapter/SptagAdapter.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/preprocessor/Normalize.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexKDT.cpp + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp ) -if (NOT TARGET test_kdt) - add_executable(test_kdt test_kdt.cpp ${kdt_srcs} ${util_srcs}) +if (NOT TARGET test_sptag) + add_executable(test_sptag test_sptag.cpp ${sptag_srcs} ${util_srcs}) endif () -target_link_libraries(test_kdt +target_link_libraries(test_sptag SPTAGLibStatic ${depend_libs} ${unittest_libs} ${basic_libs}) @@ -106,7 +106,7 @@ endif () install(TARGETS test_ivf DESTINATION unittest) install(TARGETS test_idmap DESTINATION unittest) -install(TARGETS test_kdt DESTINATION unittest) +install(TARGETS test_sptag DESTINATION unittest) if (KNOWHERE_GPU_VERSION) install(TARGETS test_gpuresource DESTINATION unittest) install(TARGETS test_customized_index DESTINATION unittest) diff --git a/core/src/index/unittest/test_kdt.cpp b/core/src/index/unittest/test_kdt.cpp deleted file mode 100644 index bbc7dcf94c..0000000000 --- a/core/src/index/unittest/test_kdt.cpp +++ /dev/null @@ -1,144 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include - -#include -#include - -#include "knowhere/adapter/SptagAdapter.h" -#include "knowhere/adapter/Structure.h" -#include "knowhere/common/Exception.h" -#include "knowhere/index/vector_index/IndexKDT.h" -#include "knowhere/index/vector_index/helpers/Definitions.h" - -#include "unittest/utils.h" - -using ::testing::Combine; -using ::testing::TestWithParam; -using ::testing::Values; - -class KDTTest : public DataGen, public ::testing::Test { - protected: - void - SetUp() override { - Generate(96, 1000, 10); - index_ = std::make_shared(); - - auto tempconf = std::make_shared(); - tempconf->tptnubmber = 1; - tempconf->k = 10; - conf = tempconf; - - Init_with_default(); - } - - protected: - knowhere::Config conf; - std::shared_ptr index_ = nullptr; -}; - -// TODO(lxj): add test about count() and dimension() -TEST_F(KDTTest, kdt_basic) { - assert(!xb.empty()); - - auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); - index_->set_preprocessor(preprocessor); - - auto model = index_->Train(base_dataset, conf); - index_->set_index_model(model); - index_->Add(base_dataset, conf); - auto result = index_->Search(query_dataset, conf); - AssertAnns(result, nq, k); - - { - // auto ids = result->array()[0]; - // auto dists = result->array()[1]; - auto ids = result->ids(); - auto dists = result->dist(); - - std::stringstream ss_id; - std::stringstream ss_dist; - for (auto i = 0; i < nq; i++) { - for (auto j = 0; j < k; ++j) { - ss_id << *((int64_t*)(ids) + i * k + j) << " "; - ss_dist << *((float*)(dists) + i * k + j) << " "; - // ss_id << *ids->data()->GetValues(1, i * k + j) << " "; - // ss_dist << *dists->data()->GetValues(1, i * k + j) << " "; - } - ss_id << std::endl; - ss_dist << std::endl; - } - std::cout << "id\n" << ss_id.str() << std::endl; - std::cout << "dist\n" << ss_dist.str() << std::endl; - } -} - -// TODO(zirui): enable test -// TEST_F(KDTTest, kdt_serialize) { -// assert(!xb.empty()); -// -// auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); -// index_->set_preprocessor(preprocessor); -// -// auto model = index_->Train(base_dataset, conf); -// // index_->Add(base_dataset, conf); -// auto binaryset = index_->Serialize(); -// auto new_index = std::make_shared(); -// new_index->Load(binaryset); -// auto result = new_index->Search(query_dataset, conf); -// AssertAnns(result, nq, k); -// PrintResult(result, nq, k); -// ASSERT_EQ(new_index->Count(), nb); -// ASSERT_EQ(new_index->Dimension(), dim); -// ASSERT_THROW({ new_index->Clone(); }, knowhere::KnowhereException); -// ASSERT_NO_THROW({ new_index->Seal(); }); -// -// { -// int fileno = 0; -// const std::string& base_name = "/tmp/kdt_serialize_test_bin_"; -// std::vector filename_list; -// std::vector> meta_list; -// for (auto& iter : binaryset.binary_map_) { -// const std::string& filename = base_name + std::to_string(fileno); -// FileIOWriter writer(filename); -// writer(iter.second->data.get(), iter.second->size); -// -// meta_list.emplace_back(std::make_pair(iter.first, iter.second->size)); -// filename_list.push_back(filename); -// ++fileno; -// } -// -// knowhere::BinarySet load_data_list; -// for (int i = 0; i < filename_list.size() && i < meta_list.size(); ++i) { -// auto bin_size = meta_list[i].second; -// FileIOReader reader(filename_list[i]); -// -// auto load_data = new uint8_t[bin_size]; -// reader(load_data, bin_size); -// auto data = std::make_shared(); -// data.reset(load_data); -// load_data_list.Append(meta_list[i].first, data, bin_size); -// } -// -// auto new_index = std::make_shared(); -// new_index->Load(load_data_list); -// auto result = new_index->Search(query_dataset, conf); -// AssertAnns(result, nq, k); -// PrintResult(result, nq, k); -// } -//} diff --git a/core/src/index/unittest/test_sptag.cpp b/core/src/index/unittest/test_sptag.cpp new file mode 100644 index 0000000000..a0898a619b --- /dev/null +++ b/core/src/index/unittest/test_sptag.cpp @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include + +#include "knowhere/adapter/SptagAdapter.h" +#include "knowhere/adapter/Structure.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexSPTAG.h" +#include "knowhere/index/vector_index/helpers/Definitions.h" + +#include "unittest/utils.h" + +using ::testing::Combine; +using ::testing::TestWithParam; +using ::testing::Values; + +class SPTAGTest : public DataGen, public TestWithParam { + protected: + void + SetUp() override { + IndexType = GetParam(); + Generate(128, 100, 5); + index_ = std::make_shared(IndexType); + if (IndexType == "KDT") { + auto tempconf = std::make_shared(); + tempconf->tptnumber = 1; + tempconf->k = 10; + conf = tempconf; + } else { + auto tempconf = std::make_shared(); + tempconf->tptnumber = 1; + tempconf->k = 10; + conf = tempconf; + } + + Init_with_default(); + } + + protected: + knowhere::Config conf; + std::shared_ptr index_ = nullptr; + std::string IndexType; +}; + +INSTANTIATE_TEST_CASE_P(SPTAGParameters, SPTAGTest, Values("KDT", "BKT")); + +// TODO(lxj): add test about count() and dimension() +TEST_P(SPTAGTest, sptag_basic) { + assert(!xb.empty()); + + auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); + index_->set_preprocessor(preprocessor); + + auto model = index_->Train(base_dataset, conf); + index_->set_index_model(model); + index_->Add(base_dataset, conf); + auto result = index_->Search(query_dataset, conf); + AssertAnns(result, nq, k); + + { + //auto ids = result->array()[0]; + //auto dists = result->array()[1]; + auto ids = result->ids(); + auto dists = result->dist(); + + std::stringstream ss_id; + std::stringstream ss_dist; + for (auto i = 0; i < nq; i++) { + for (auto j = 0; j < k; ++j) { + //ss_id << *ids->data()->GetValues(1, i * k + j) << " "; + //ss_dist << *dists->data()->GetValues(1, i * k + j) << " "; + ss_id << *((int64_t*)(ids) + i * k + j) << " "; + ss_dist << *((float*)(dists) + i * k + j) << " "; + } + ss_id << std::endl; + ss_dist << std::endl; + } + std::cout << "id\n" << ss_id.str() << std::endl; + std::cout << "dist\n" << ss_dist.str() << std::endl; + } +} + +TEST_P(SPTAGTest, sptag_serialize) { + assert(!xb.empty()); + + auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); + index_->set_preprocessor(preprocessor); + + auto model = index_->Train(base_dataset, conf); + + index_->Add(base_dataset, conf); + auto binaryset = index_->Serialize(); + auto new_index = std::make_shared(IndexType); + new_index->Load(binaryset); + auto result = new_index->Search(query_dataset, conf); + AssertAnns(result, nq, k); + PrintResult(result, nq, k); + ASSERT_EQ(new_index->Count(), nb); + ASSERT_EQ(new_index->Dimension(), dim); + // ASSERT_THROW({ new_index->Clone(); }, knowhere::KnowhereException); + // ASSERT_NO_THROW({ new_index->Seal(); }); + + { + int fileno = 0; + const std::string& base_name = "/tmp/sptag_serialize_test_bin_"; + std::vector filename_list; + std::vector> meta_list; + for (auto& iter : binaryset.binary_map_) { + const std::string& filename = base_name + std::to_string(fileno); + FileIOWriter writer(filename); + writer(iter.second->data.get(), iter.second->size); + + meta_list.emplace_back(std::make_pair(iter.first, iter.second->size)); + filename_list.push_back(filename); + ++fileno; + } + + knowhere::BinarySet load_data_list; + for (int i = 0; i < filename_list.size() && i < meta_list.size(); ++i) { + auto bin_size = meta_list[i].second; + FileIOReader reader(filename_list[i]); + + auto load_data = new uint8_t[bin_size]; + reader(load_data, bin_size); + auto data = std::make_shared(); + data.reset(load_data); + load_data_list.Append(meta_list[i].first, data, bin_size); + } + + auto new_index = std::make_shared(IndexType); + new_index->Load(load_data_list); + auto result = new_index->Search(query_dataset, conf); + AssertAnns(result, nq, k); + PrintResult(result, nq, k); + } +} diff --git a/core/src/index/unittest/utils.cpp b/core/src/index/unittest/utils.cpp index 2556b60fad..110d8fbf92 100644 --- a/core/src/index/unittest/utils.cpp +++ b/core/src/index/unittest/utils.cpp @@ -153,22 +153,24 @@ void AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) { auto ids = result->ids(); for (auto i = 0; i < nq; i++) { - EXPECT_EQ(i, *((int64_t*)(ids) + i * k)); + EXPECT_EQ(i, *((int64_t*)(ids) + i * k)); // EXPECT_EQ(i, *(ids->data()->GetValues(1, i * k))); } } void PrintResult(const knowhere::DatasetPtr& result, const int& nq, const int& k) { - auto ids = result->array()[0]; - auto dists = result->array()[1]; + auto ids = result->ids(); + auto dists = result->dist(); std::stringstream ss_id; std::stringstream ss_dist; - for (auto i = 0; i < 10; i++) { + for (auto i = 0; i < nq; i++) { for (auto j = 0; j < k; ++j) { - ss_id << *(ids->data()->GetValues(1, i * k + j)) << " "; - ss_dist << *(dists->data()->GetValues(1, i * k + j)) << " "; + //ss_id << *(ids->data()->GetValues(1, i * k + j)) << " "; + //ss_dist << *(dists->data()->GetValues(1, i * k + j)) << " "; + ss_id << *((int64_t*)(ids) + i * k + j) << " "; + ss_dist << *((float*)(dists) + i * k + j) << " "; } ss_id << std::endl; ss_dist << std::endl; diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index 461745f1fd..4114eb2af4 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -204,5 +204,35 @@ NSGConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) return conf; } +knowhere::Config +SPTAGKDTConfAdapter::Match(const TempMetaConf& metaconf) { + auto conf = std::make_shared(); + conf->d = metaconf.dim; + conf->metric_type = metaconf.metric_type; + return conf; +} + +knowhere::Config +SPTAGKDTConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) { + auto conf = std::make_shared(); + conf->k = metaconf.k; + return conf; +} + +knowhere::Config +SPTAGBKTConfAdapter::Match(const TempMetaConf& metaconf) { + auto conf = std::make_shared(); + conf->d = metaconf.dim; + conf->metric_type = metaconf.metric_type; + return conf; +} + +knowhere::Config +SPTAGBKTConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) { + auto conf = std::make_shared(); + conf->k = metaconf.k; + return conf; +} + } // namespace engine } // namespace milvus diff --git a/core/src/wrapper/ConfAdapter.h b/core/src/wrapper/ConfAdapter.h index 5ec3d52486..600dd3a517 100644 --- a/core/src/wrapper/ConfAdapter.h +++ b/core/src/wrapper/ConfAdapter.h @@ -97,5 +97,23 @@ class NSGConfAdapter : public IVFConfAdapter { MatchSearch(const TempMetaConf& metaconf, const IndexType& type) final; }; +class SPTAGKDTConfAdapter : public ConfAdapter { + public: + knowhere::Config + Match(const TempMetaConf& metaconf) override; + + knowhere::Config + MatchSearch(const TempMetaConf& metaconf, const IndexType& type) override; +}; + +class SPTAGBKTConfAdapter : public ConfAdapter { + public: + knowhere::Config + Match(const TempMetaConf& metaconf) override; + + knowhere::Config + MatchSearch(const TempMetaConf& metaconf, const IndexType& type) override; +}; + } // namespace engine } // namespace milvus diff --git a/core/src/wrapper/ConfAdapterMgr.cpp b/core/src/wrapper/ConfAdapterMgr.cpp index d0eba04529..cf58c0110f 100644 --- a/core/src/wrapper/ConfAdapterMgr.cpp +++ b/core/src/wrapper/ConfAdapterMgr.cpp @@ -56,6 +56,9 @@ AdapterMgr::RegisterAdapter() { REGISTER_CONF_ADAPTER(IVFPQConfAdapter, IndexType::FAISS_IVFPQ_MIX, ivfpq_mix); REGISTER_CONF_ADAPTER(NSGConfAdapter, IndexType::NSG_MIX, nsg_mix); + + REGISTER_CONF_ADAPTER(SPTAGKDTConfAdapter, IndexType::SPTAG_KDT_RNT_CPU, sptag_kdt); + REGISTER_CONF_ADAPTER(SPTAGBKTConfAdapter, IndexType::SPTAG_BKT_RNT_CPU, sptag_bkt); } } // namespace engine diff --git a/core/src/wrapper/VecIndex.cpp b/core/src/wrapper/VecIndex.cpp index 81a13c60f9..9f68cac0a1 100644 --- a/core/src/wrapper/VecIndex.cpp +++ b/core/src/wrapper/VecIndex.cpp @@ -22,7 +22,7 @@ #include "knowhere/index/vector_index/IndexIVF.h" #include "knowhere/index/vector_index/IndexIVFPQ.h" #include "knowhere/index/vector_index/IndexIVFSQ.h" -#include "knowhere/index/vector_index/IndexKDT.h" +#include "knowhere/index/vector_index/IndexSPTAG.h" #include "knowhere/index/vector_index/IndexNSG.h" #include "utils/Log.h" @@ -128,7 +128,11 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) { break; } case IndexType::SPTAG_KDT_RNT_CPU: { - index = std::make_shared(); + index = std::make_shared("KDT"); + break; + } + case IndexType::SPTAG_BKT_RNT_CPU: { + index = std::make_shared("BKT"); break; } case IndexType::FAISS_IVFSQ8_CPU: { diff --git a/core/src/wrapper/VecIndex.h b/core/src/wrapper/VecIndex.h index efe01a25d7..e69655b087 100644 --- a/core/src/wrapper/VecIndex.h +++ b/core/src/wrapper/VecIndex.h @@ -49,6 +49,7 @@ enum class IndexType { FAISS_IVFSQ8_HYBRID, // only support build on gpu. NSG_MIX, FAISS_IVFPQ_MIX, + SPTAG_BKT_RNT_CPU, }; class VecIndex; @@ -139,6 +140,9 @@ write_index(VecIndexPtr index, const std::string& location); extern VecIndexPtr read_index(const std::string& location); +VecIndexPtr +read_index(const std::string& location, knowhere::BinarySet& index_binary); + extern VecIndexPtr GetVecIndexFactory(const IndexType& type, const Config& cfg = Config()); diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index 025601a1cd..413db60e37 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -29,15 +29,16 @@ INITIALIZE_EASYLOGGINGPP +using ::testing::Combine; using ::testing::TestWithParam; using ::testing::Values; -using ::testing::Combine; class KnowhereWrapperTest - : public DataGenBase, - public TestWithParam<::std::tuple> { + : public DataGenBase, + public TestWithParam<::std::tuple> { protected: - void SetUp() override { + void + SetUp() override { #ifdef MILVUS_GPU_VERSION knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICEID, PINMEM, TEMPMEM, RESNUM); #endif @@ -57,12 +58,13 @@ class KnowhereWrapperTest conf = ParamGenerator::GetInstance().GenBuild(index_type, tempconf); searchconf = ParamGenerator::GetInstance().GenSearchConf(index_type, tempconf); -// conf->k = k; -// conf->d = dim; -// conf->gpu_id = DEVICEID; + // conf->k = k; + // conf->d = dim; + // conf->gpu_id = DEVICEID; } - void TearDown() override { + void + TearDown() override { #ifdef MILVUS_GPU_VERSION knowhere::FaissGpuResourceMgr::GetInstance().Free(); #endif @@ -75,22 +77,21 @@ class KnowhereWrapperTest knowhere::Config searchconf; }; -INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest, - Values( - //["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"] +INSTANTIATE_TEST_CASE_P( + WrapperParam, KnowhereWrapperTest, + Values( +//["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"] #ifdef MILVUS_GPU_VERSION std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_GPU, "Default", DIM, NB, 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_MIX, "Default", 64, 1000, 10, 10), - // std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, - // 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IVFPQ_MIX, "Default", 64, 1000, 10, 10), - -// std::make_tuple(IndexType::NSG_MIX, "Default", 128, 250000, 10, 10), + // std::make_tuple(milvus::engine::IndexType::NSG_MIX, "Default", 128, 250000, 10, 10), #endif - // std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 250000, 10, 10), + // std::make_tuple(milvus::engine::IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 100, 10, 10), + // std::make_tuple(milvus::engine::IndexType::SPTAG_BKT_RNT_CPU, "Default", 128, 100, 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IDMAP, "Default", 64, 1000, 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_CPU, "Default", 64, 1000, 10, 10), std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10))); From da0b9e05ac8e341f8533073deabd9a4dadd355d8 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 20 Nov 2019 17:07:20 +0800 Subject: [PATCH 08/10] [skip ci] Add test report list --- docs/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/README.md b/docs/README.md index 64409d8240..7465fe839f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,3 +9,8 @@ The tests are run on [SIFT1B dataset](http://corpus-texmex.irisa.fr/), and provi - Recall: The fraction of the total amount of relevant instances that were actually retrieved. Test variables are `nq` and `topk`. + +The following is a list of existing test reports: + +- [IVF_SQ8](test_report/milvus_ivfsq8_test_report_detailed_version.md) +- [IVF_SQ8H](test_report/milvus_ivfsq8h_test_report_detailed_version.md) From 323e9cfd79cd8444b77cca602d8700f2590818ff Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 20 Nov 2019 17:12:42 +0800 Subject: [PATCH 09/10] [skip ci] Add cn version links --- docs/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/README.md b/docs/README.md index 7465fe839f..d57bbabaf2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,3 +14,8 @@ The following is a list of existing test reports: - [IVF_SQ8](test_report/milvus_ivfsq8_test_report_detailed_version.md) - [IVF_SQ8H](test_report/milvus_ivfsq8h_test_report_detailed_version.md) + +To read the CN version of these reports: + +- [IVF_SQ8_cn](test_report/milvus_ivfsq8_test_report_detailed_version_cn.md) +- [IVF_SQ8H_cn](test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md) From 24e05a87d09a3447e51bbb384800737ef2e93d2f Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 20 Nov 2019 17:15:07 +0800 Subject: [PATCH 10/10] [skip ci] minor update --- docs/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/README.md b/docs/README.md index d57bbabaf2..f5f964de0d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,8 @@ The tests are run on [SIFT1B dataset](http://corpus-texmex.irisa.fr/), and provi Test variables are `nq` and `topk`. +## Test reports + The following is a list of existing test reports: - [IVF_SQ8](test_report/milvus_ivfsq8_test_report_detailed_version.md)