milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Upgrade to knowhere-v1.1.12 (#17692) 

Signed-off-by: yudong.cai <yudong.cai@zilliz.com>
pull/17746/merge
Cai Yudong 2022-06-24 10:34:18 +08:00 committed by GitHub
parent 9cd454e563
commit 7385770014
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 517 additions and 487 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
internal/core/src/common/LoadInfo.h
View File

@ -20,7 +20,6 @@
#include <string>
#include "Types.h"
#include "knowhere/index/vector_index/VecIndex.h"
#include "common/CDataType.h"
#include "knowhere/index/Index.h"

16
internal/core/src/common/Types.cpp
View File

@ -27,17 +27,17 @@
namespace milvus {
using boost::algorithm::to_upper_copy;
namespace Metric = knowhere::Metric;
namespace metric = knowhere::metric;
static const auto metric_bimap = [] {
boost::bimap<std::string, MetricType> mapping;
using pos = boost::bimap<std::string, MetricType>::value_type;
mapping.insert(pos(std::string(Metric::L2), MetricType::METRIC_L2));
mapping.insert(pos(std::string(Metric::IP), MetricType::METRIC_INNER_PRODUCT));
mapping.insert(pos(std::string(Metric::JACCARD), MetricType::METRIC_Jaccard));
mapping.insert(pos(std::string(Metric::TANIMOTO), MetricType::METRIC_Tanimoto));
mapping.insert(pos(std::string(Metric::HAMMING), MetricType::METRIC_Hamming));
mapping.insert(pos(std::string(Metric::SUBSTRUCTURE), MetricType::METRIC_Substructure));
mapping.insert(pos(std::string(Metric::SUPERSTRUCTURE), MetricType::METRIC_Superstructure));
mapping.insert(pos(std::string(metric::L2), MetricType::METRIC_L2));
mapping.insert(pos(std::string(metric::IP), MetricType::METRIC_INNER_PRODUCT));
mapping.insert(pos(std::string(metric::JACCARD), MetricType::METRIC_Jaccard));
mapping.insert(pos(std::string(metric::TANIMOTO), MetricType::METRIC_Tanimoto));
mapping.insert(pos(std::string(metric::HAMMING), MetricType::METRIC_Hamming));
mapping.insert(pos(std::string(metric::SUBSTRUCTURE), MetricType::METRIC_Substructure));
mapping.insert(pos(std::string(metric::SUPERSTRUCTURE), MetricType::METRIC_Superstructure));
return mapping;
}();

4
internal/core/src/index/BoolIndex.h
View File

@ -22,8 +22,8 @@ class BoolIndex : public ScalarIndexSort<bool> {
public:
void
BuildWithDataset(const DatasetPtr& dataset) override {
auto size = dataset->Get<int64_t>(knowhere::meta::ROWS);
auto data = dataset->Get<const void*>(knowhere::meta::TENSOR);
auto size = knowhere::GetDatasetRows(dataset);
auto data = knowhere::GetDatasetTensor(dataset);
proto::schema::BoolArray arr;
arr.ParseFromArray(data, size);
Build(arr.data().size(), arr.data().data());

12
internal/core/src/index/ScalarIndex-inl.h
View File

@ -9,9 +9,11 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include <string>
#include <iostream>
#include <string>
#include "index/Meta.h"
#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
namespace milvus::scalar {
template <typename T>
@ -36,14 +38,14 @@ ScalarIndex<T>::Query(const DatasetPtr& dataset) {
}
case OpType::In: {
auto n = dataset->Get<int64_t>(knowhere::meta::ROWS);
auto values = dataset->Get<const void*>(knowhere::meta::TENSOR);
auto n = knowhere::GetDatasetRows(dataset);
auto values = knowhere::GetDatasetTensor(dataset);
return In(n, reinterpret_cast<const T*>(values));
}
case OpType::NotIn: {
auto n = dataset->Get<int64_t>(knowhere::meta::ROWS);
auto values = dataset->Get<const void*>(knowhere::meta::TENSOR);
auto n = knowhere::GetDatasetRows(dataset);
auto values = knowhere::GetDatasetTensor(dataset);
return NotIn(n, reinterpret_cast<const T*>(values));
}

4
internal/core/src/index/ScalarIndexSort-inl.h
View File

@ -33,8 +33,8 @@ inline ScalarIndexSort<T>::ScalarIndexSort(const size_t n, const T* values) : is
template <typename T>
inline void
ScalarIndexSort<T>::BuildWithDataset(const DatasetPtr& dataset) {
auto size = dataset->Get<int64_t>(knowhere::meta::ROWS);
auto data = dataset->Get<const void*>(knowhere::meta::TENSOR);
auto size = knowhere::GetDatasetRows(dataset);
auto data = knowhere::GetDatasetTensor(dataset);
Build(size, reinterpret_cast<const T*>(data));
}

4
internal/core/src/index/StringIndex.h
View File

@ -24,8 +24,8 @@ class StringIndex : public ScalarIndex<std::string> {
public:
void
BuildWithDataset(const DatasetPtr& dataset) override {
auto size = dataset->Get<int64_t>(knowhere::meta::ROWS);
auto data = dataset->Get<const void*>(knowhere::meta::TENSOR);
auto size = knowhere::GetDatasetRows(dataset);
auto data = knowhere::GetDatasetTensor(dataset);
proto::schema::StringArray arr;
arr.ParseFromArray(data, size);

2
internal/core/src/index/StringIndexMarisa.cpp
View File

@ -15,7 +15,7 @@
#include <stdlib.h>
#include <stdio.h>
#include <fcntl.h>
#include <knowhere/common/Utils.h>
#include <knowhere/index/VecIndex.h>
#include "index/StringIndexMarisa.h"
#include "index/Utils.h"

7
internal/core/src/index/StringIndexSort.h
View File

@ -13,9 +13,10 @@
#include <vector>
#include <string>
#include "common/Utils.h"
#include "index/ScalarIndexSort.h"
#include "index/StringIndex.h"
#include "common/Utils.h"
#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
namespace milvus::scalar {
// TODO: should inherit from StringIndex?
@ -23,8 +24,8 @@ class StringIndexSort : public ScalarIndexSort<std::string> {
public:
void
BuildWithDataset(const DatasetPtr& dataset) override {
auto size = dataset->Get<int64_t>(knowhere::meta::ROWS);
auto data = dataset->Get<const void*>(knowhere::meta::TENSOR);
auto size = knowhere::GetDatasetRows(dataset);
auto data = knowhere::GetDatasetTensor(dataset);
proto::schema::StringArray arr;
arr.ParseFromArray(data, size);

125
internal/core/src/indexbuilder/VecIndexCreator.cpp
View File

@ -14,14 +14,15 @@
#include <google/protobuf/text_format.h>
#include "exceptions/EasyAssert.h"
#include "pb/index_cgo_msg.pb.h"
#include "indexbuilder/VecIndexCreator.h"
#include "indexbuilder/utils.h"
#include "knowhere/common/Timer.h"
#include "knowhere/common/Utils.h"
#include "knowhere/index/VecIndex.h"
#include "knowhere/index/VecIndexFactory.h"
#include "knowhere/index/vector_index/ConfAdapterMgr.h"
#include "knowhere/index/vector_index/VecIndexFactory.h"
#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
#include "pb/index_cgo_msg.pb.h"
namespace milvus::indexbuilder {
@ -57,54 +58,56 @@ VecIndexCreator::parse_impl(const std::string& serialized_params_str, knowhere::
conf[key] = value;
}
auto stoi_closure = [](const std::string& s) -> int { return std::stoi(s); };
auto stof_closure = [](const std::string& s) -> float { return std::stof(s); };
auto stoi_closure = [](const std::string_view& s) -> int { return std::stoi(std::string(s)); };
auto stof_closure = [](const std::string_view& s) -> float { return std::stof(std::string(s)); };
/***************************** meta *******************************/
check_parameter<int>(conf, knowhere::meta::SLICE_SIZE, stoi_closure, std::optional{4});
check_parameter<int>(conf, knowhere::meta::DIM, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::meta::TOPK, stoi_closure, std::nullopt);
/***************************** IVF Params *******************************/
check_parameter<int>(conf, knowhere::IndexParams::nprobe, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::nlist, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::m, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::nbits, stoi_closure, std::nullopt);
/************************** NSG Parameter **************************/
check_parameter<int>(conf, knowhere::IndexParams::knng, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::search_length, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::out_degree, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::candidate, stoi_closure, std::nullopt);
/************************** HNSW Params *****************************/
check_parameter<int>(conf, knowhere::IndexParams::efConstruction, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::M, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::ef, stoi_closure, std::nullopt);
/************************** Annoy Params *****************************/
check_parameter<int>(conf, knowhere::IndexParams::n_trees, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::search_k, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::indexparam::NPROBE, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::indexparam::NLIST, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::indexparam::M, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::indexparam::NBITS, stoi_closure, std::nullopt);
/************************** PQ Params *****************************/
check_parameter<int>(conf, knowhere::IndexParams::PQM, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::indexparam::PQ_M, stoi_closure, std::nullopt);
#ifdef MILVUS_SUPPORT_NSG
/************************** NSG Parameter **************************/
check_parameter<int>(conf, knowhere::indexparam::KNNG, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::indexparam::SEARCH_LENGTH, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::indexparam::OUT_DEGREE, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::indexparam::CANDIDATE, stoi_closure, std::nullopt);
#endif
/************************** HNSW Params *****************************/
check_parameter<int>(conf, knowhere::indexparam::EFCONSTRUCTION, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::indexparam::HNSW_M, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::indexparam::EF, stoi_closure, std::nullopt);
/************************** Annoy Params *****************************/
check_parameter<int>(conf, knowhere::indexparam::N_TREES, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::indexparam::SEARCH_K, stoi_closure, std::nullopt);
#ifdef MILVUS_SUPPORT_NGT
/************************** NGT Params *****************************/
check_parameter<int>(conf, knowhere::IndexParams::edge_size, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::indexparam::EDGE_SIZE, stoi_closure, std::nullopt);
/************************** NGT Search Params *****************************/
check_parameter<float>(conf, knowhere::IndexParams::epsilon, stof_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::max_search_edges, stoi_closure, std::nullopt);
check_parameter<float>(conf, knowhere::indexparam::EPSILON, stof_closure, std::nullopt);
check_parameter<int>(conf, knowhere::indexparam::MAX_SEARCH_EDGES, stoi_closure, std::nullopt);
/************************** NGT_PANNG Params *****************************/
check_parameter<int>(conf, knowhere::IndexParams::forcedly_pruned_edge_size, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::selectively_pruned_edge_size, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::indexparam::FORCEDLY_PRUNED_EDGE_SIZE, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::indexparam::SELECTIVELY_PRUNED_EDGE_SIZE, stoi_closure, std::nullopt);
/************************** NGT_ONNG Params *****************************/
check_parameter<int>(conf, knowhere::IndexParams::outgoing_edge_size, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::incoming_edge_size, stoi_closure, std::nullopt);
/************************** Serialize Params *******************************/
check_parameter<int>(conf, knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, stoi_closure, std::optional{4});
check_parameter<int>(conf, knowhere::indexparam::OUTGOING_EDGE_SIZE, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::indexparam::INCOMING_EDGE_SIZE, stoi_closure, std::nullopt);
#endif
}
void
@ -121,24 +124,24 @@ VecIndexCreator::parse() {
template <typename T>
void
VecIndexCreator::check_parameter(knowhere::Config& conf,
const std::string& key,
const std::string_view& key,
std::function<T(std::string)> fn,
std::optional<T> default_v) {
if (!conf.contains(key)) {
if (default_v.has_value()) {
conf[key] = default_v.value();
conf[std::string(key)] = default_v.value();
}
} else {
auto value = conf[key];
conf[key] = fn(value);
auto value = conf[std::string(key)];
conf[std::string(key)] = fn(value);
}
}
template <typename T>
std::optional<T>
VecIndexCreator::get_config_by_name(std::string name) {
VecIndexCreator::get_config_by_name(std::string_view name) {
if (config_.contains(name)) {
return {config_[name].get<T>()};
return knowhere::GetValueFromConfig<T>(config_, name);
}
return std::nullopt;
}
@ -154,10 +157,10 @@ void
VecIndexCreator::BuildWithoutIds(const knowhere::DatasetPtr& dataset) {
auto index_type = get_index_type();
auto index_mode = get_index_mode();
config_[knowhere::meta::ROWS] = dataset->Get<int64_t>(knowhere::meta::ROWS);
knowhere::SetMetaRows(config_, knowhere::GetDatasetRows(dataset));
if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFPQ) {
if (!config_.contains(knowhere::IndexParams::nbits)) {
config_[knowhere::IndexParams::nbits] = 8;
if (!config_.contains(knowhere::indexparam::NBITS)) {
knowhere::SetIndexParamNbits(config_, 8);
}
}
auto conf_adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_type);
@ -187,14 +190,12 @@ VecIndexCreator::BuildWithoutIds(const knowhere::DatasetPtr& dataset) {
void
VecIndexCreator::BuildWithIds(const knowhere::DatasetPtr& dataset) {
AssertInfo(dataset->data().find(knowhere::meta::IDS) != dataset->data().end(),
"[VecIndexCreator]Can't find ids field in dataset");
auto index_type = get_index_type();
auto index_mode = get_index_mode();
config_[knowhere::meta::ROWS] = dataset->Get<int64_t>(knowhere::meta::ROWS);
knowhere::SetMetaRows(config_, knowhere::GetDatasetRows(dataset));
if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFPQ) {
if (!config_.contains(knowhere::IndexParams::nbits)) {
config_[knowhere::IndexParams::nbits] = 8;
if (!config_.contains(knowhere::indexparam::NBITS)) {
knowhere::SetIndexParamNbits(config_, 8);
}
}
auto conf_adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_type);
@ -212,9 +213,9 @@ void
VecIndexCreator::StoreRawData(const knowhere::DatasetPtr& dataset) {
auto index_type = get_index_type();
if (is_in_nm_list(index_type)) {
auto tensor = dataset->Get<const void*>(knowhere::meta::TENSOR);
auto row_num = dataset->Get<int64_t>(knowhere::meta::ROWS);
auto dim = dataset->Get<int64_t>(knowhere::meta::DIM);
auto tensor = knowhere::GetDatasetTensor(dataset);
auto row_num = knowhere::GetDatasetRows(dataset);
auto dim = knowhere::GetDatasetDim(dataset);
int64_t data_size;
if (is_in_bin_list(index_type)) {
data_size = dim / 8 * row_num;
@ -263,20 +264,20 @@ VecIndexCreator::get_index_type() {
// knowhere bug here
// the index_type of all ivf-based index will change to ivf flat after loaded
auto type = get_config_by_name<std::string>("index_type");
return type.has_value() ? type.value() : knowhere::IndexEnum::INDEX_FAISS_IVFPQ;
return type.has_value() ? type.value() : std::string(knowhere::IndexEnum::INDEX_FAISS_IVFPQ);
}
std::string
VecIndexCreator::get_metric_type() {
auto type = get_config_by_name<std::string>(knowhere::Metric::TYPE);
auto type = get_config_by_name<std::string>(knowhere::meta::METRIC_TYPE);
if (type.has_value()) {
return type.value();
} else {
auto index_type = get_index_type();
if (is_in_bin_list(index_type)) {
return knowhere::Metric::JACCARD;
return std::string(knowhere::metric::JACCARD);
} else {
return knowhere::Metric::L2;
return std::string(knowhere::metric::L2);
}
}
}
@ -293,8 +294,8 @@ VecIndexCreator::get_index_mode() {
int64_t
VecIndexCreator::get_index_file_slice_size() {
if (config_.contains(knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE)) {
return config_[knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE].get<int64_t>();
if (knowhere::CheckKeyInConfig(config_, knowhere::meta::SLICE_SIZE)) {
return knowhere::GetMetaSliceSize(config_);
}
return knowhere::index_file_slice_size; // by default
}
@ -322,10 +323,10 @@ VecIndexCreator::QueryImpl(const knowhere::DatasetPtr& dataset, const knowhere::
}
auto res = index_->Query(dataset, conf, nullptr);
auto ids = res->Get<int64_t*>(knowhere::meta::IDS);
auto distances = res->Get<float*>(knowhere::meta::DISTANCE);
auto nq = dataset->Get<int64_t>(knowhere::meta::ROWS);
auto k = config_[knowhere::meta::TOPK].get<int64_t>();
auto ids = knowhere::GetDatasetIDs(res);
auto distances = knowhere::GetDatasetDistance(res);
auto nq = knowhere::GetDatasetRows(dataset);
auto k = knowhere::GetMetaTopk(config_);
auto query_res = std::make_unique<VecIndexCreator::QueryResult>();
query_res->nq = nq;

8
internal/core/src/indexbuilder/VecIndexCreator.h
View File

@ -16,9 +16,9 @@
#include <string>
#include <vector>
#include "knowhere/index/vector_index/VecIndex.h"
#include "knowhere/common/BinarySet.h"
#include "indexbuilder/IndexCreatorBase.h"
#include "knowhere/common/BinarySet.h"
#include "knowhere/index/VecIndex.h"
namespace milvus::indexbuilder {
@ -74,7 +74,7 @@ class VecIndexCreator : public IndexCreatorBase {
template <typename T>
std::optional<T>
get_config_by_name(std::string name);
get_config_by_name(std::string_view name);
void
StoreRawData(const knowhere::DatasetPtr& dataset);
@ -85,7 +85,7 @@ class VecIndexCreator : public IndexCreatorBase {
template <typename T>
void
check_parameter(knowhere::Config& conf,
const std::string& key,
const std::string_view& key,
std::function<T(std::string)> fn,
std::optional<T> default_v = std::nullopt);

37
internal/core/src/indexbuilder/utils.cpp
View File

@ -15,15 +15,12 @@
#include <string>
#include <tuple>
#include <vector>
#include <knowhere/index/vector_index/helpers/IndexParameter.h>
#include "knowhere/index/IndexType.h"
namespace milvus::indexbuilder {
std::vector<std::string>
std::vector<knowhere::IndexType>
NM_List() {
static std::vector<std::string> ret{
static std::vector<knowhere::IndexType> ret{
knowhere::IndexEnum::INDEX_FAISS_IVFFLAT,
#ifdef MILVUS_SUPPORT_NSG
knowhere::IndexEnum::INDEX_NSG,
@ -33,18 +30,18 @@ NM_List() {
return ret;
}
std::vector<std::string>
std::vector<knowhere::IndexType>
BIN_List() {
static std::vector<std::string> ret{
static std::vector<knowhere::IndexType> ret{
knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP,
knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT,
};
return ret;
}
std::vector<std::string>
std::vector<knowhere::IndexType>
Need_ID_List() {
static std::vector<std::string> ret{
static std::vector<knowhere::IndexType> ret{
// knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT,
// knowhere::IndexEnum::INDEX_NSG,
};
@ -52,9 +49,9 @@ Need_ID_List() {
return ret;
}
std::vector<std::string>
std::vector<knowhere::IndexType>
Need_BuildAll_list() {
static std::vector<std::string> ret{
static std::vector<knowhere::IndexType> ret{
#ifdef MILVUS_SUPPORT_NSG
knowhere::IndexEnum::INDEX_NSG,
#endif
@ -62,10 +59,10 @@ Need_BuildAll_list() {
return ret;
}
std::vector<std::tuple<std::string, std::string>>
std::vector<std::tuple<knowhere::IndexType, knowhere::MetricType>>
unsupported_index_combinations() {
static std::vector<std::tuple<std::string, std::string>> ret{
std::make_tuple(std::string(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT), std::string(knowhere::Metric::L2)),
static std::vector<std::tuple<knowhere::IndexType, knowhere::MetricType>> ret{
std::make_tuple(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, knowhere::metric::L2),
};
return ret;
}
@ -79,28 +76,28 @@ is_in_list(const T& t, std::function<std::vector<T>()> list_func) {
bool
is_in_bin_list(const knowhere::IndexType& index_type) {
return is_in_list<std::string>(index_type, BIN_List);
return is_in_list<knowhere::IndexType>(index_type, BIN_List);
}
bool
is_in_nm_list(const knowhere::IndexType& index_type) {
return is_in_list<std::string>(index_type, NM_List);
return is_in_list<knowhere::IndexType>(index_type, NM_List);
}
bool
is_in_need_build_all_list(const knowhere::IndexType& index_type) {
return is_in_list<std::string>(index_type, Need_BuildAll_list);
return is_in_list<knowhere::IndexType>(index_type, Need_BuildAll_list);
}
bool
is_in_need_id_list(const knowhere::IndexType& index_type) {
return is_in_list<std::string>(index_type, Need_ID_List);
return is_in_list<knowhere::IndexType>(index_type, Need_ID_List);
}
bool
is_unsupported(const knowhere::IndexType& index_type, const knowhere::MetricType& metric_type) {
return is_in_list<std::tuple<std::string, std::string>>(std::make_tuple(index_type, metric_type),
unsupported_index_combinations);
return is_in_list<std::tuple<knowhere::IndexType, knowhere::MetricType>>(std::make_tuple(index_type, metric_type),
unsupported_index_combinations);
}
} // namespace milvus::indexbuilder

13
internal/core/src/indexbuilder/utils.h
View File

@ -16,25 +16,26 @@
#include <tuple>
#include <vector>
#include <functional>
#include <knowhere/common/Typedef.h>
#include <knowhere/common/Typedef.h>
#include "knowhere/index/IndexType.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
namespace milvus::indexbuilder {
std::vector<std::string>
std::vector<knowhere::IndexType>
NM_List();
std::vector<std::string>
std::vector<knowhere::IndexType>
BIN_List();
std::vector<std::string>
std::vector<knowhere::IndexType>
Need_ID_List();
std::vector<std::string>
std::vector<knowhere::IndexType>
Need_BuildAll_list();
std::vector<std::tuple<std::string, std::string>>
std::vector<std::tuple<knowhere::IndexType, knowhere::MetricType>>
unsupported_index_combinations();
template <typename T>

4
internal/core/src/query/PlanNode.h
View File

@ -18,7 +18,7 @@
#include <string>
#include "Expr.h"
#include "utils/Json.h"
#include "knowhere/common/Config.h"
namespace milvus::query {
@ -39,7 +39,7 @@ struct SearchInfo {
int64_t round_decimal_;
FieldId field_id_;
MetricType metric_type_;
nlohmann::json search_params_;
knowhere::Config search_params_;
};
struct VectorPlanNode : PlanNode {

4
internal/core/src/query/SearchOnIndex.cpp
View File

@ -30,8 +30,8 @@ SearchOnIndex(const dataset::SearchDataset& search_dataset,
auto& indexing_nonconst = const_cast<knowhere::VecIndex&>(indexing);
auto ans = indexing_nonconst.Query(dataset, search_conf, bitset);
auto dis = ans->Get<float*>(knowhere::meta::DISTANCE);
auto uids = ans->Get<int64_t*>(knowhere::meta::IDS);
auto dis = knowhere::GetDatasetDistance(ans);
auto uids = knowhere::GetDatasetIDs(ans);
SubSearchResult sub_qr(num_queries, topK, metric_type, round_decimal);
std::copy_n(dis, num_queries * topK, sub_qr.get_distances());

2
internal/core/src/query/SearchOnIndex.h
View File

@ -12,7 +12,7 @@
#pragma once
#include "common/BitsetView.h"
#include "knowhere/index/vector_index/VecIndex.h"
#include "knowhere/index/VecIndex.h"
#include "query/SubSearchResult.h"
#include "query/helper.h"

17
internal/core/src/query/SearchOnSealed.cpp
View File

@ -11,7 +11,7 @@
#include <cmath>
#include "knowhere/index/vector_index/VecIndex.h"
#include "knowhere/index/VecIndex.h"
#include "knowhere/index/vector_index/ConfAdapter.h"
#include "knowhere/index/vector_index/ConfAdapterMgr.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
@ -46,17 +46,20 @@ SearchOnSealed(const Schema& schema,
auto ds = knowhere::GenDataset(num_queries, dim, query_data);
auto conf = search_info.search_params_;
conf[knowhere::meta::TOPK] = search_info.topk_;
conf[knowhere::Metric::TYPE] = MetricTypeToName(field_indexing->metric_type_);
knowhere::SetMetaTopk(conf, search_info.topk_);
knowhere::SetMetaMetricType(conf, MetricTypeToName(field_indexing->metric_type_));
auto index_type = field_indexing->indexing_->index_type();
auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_type);
AssertInfo(adapter->CheckSearch(conf, index_type, field_indexing->indexing_->index_mode()),
"[SearchOnSealed]Search params check failed");
try {
adapter->CheckSearch(conf, index_type, field_indexing->indexing_->index_mode());
} catch (std::exception& e) {
AssertInfo(false, e.what());
}
return field_indexing->indexing_->Query(ds, conf, bitset);
}();
auto ids = final->Get<idx_t*>(knowhere::meta::IDS);
auto distances = final->Get<float*>(knowhere::meta::DISTANCE);
auto ids = knowhere::GetDatasetIDs(final);
float* distances = (float*)knowhere::GetDatasetDistance(final);
auto total_num = num_queries * topk;

16
internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp
View File

@ -38,14 +38,14 @@ InferIndexType(const Json& search_params) {
// ngtpanng / ngtonng -> max_search_edges / epsilon
static const std::map<std::string, knowhere::IndexType> key_list = [] {
std::map<std::string, knowhere::IndexType> list;
namespace ip = knowhere::IndexParams;
namespace ip = knowhere::indexparam;
namespace ie = knowhere::IndexEnum;
list.emplace(ip::nprobe, ie::INDEX_FAISS_IVFFLAT);
list.emplace(ip::NPROBE, ie::INDEX_FAISS_IVFFLAT);
#ifdef MILVUS_SUPPORT_NSG
list.emplace(ip::search_length, ie::INDEX_NSG);
#endif
list.emplace(ip::ef, ie::INDEX_HNSW);
list.emplace(ip::search_k, ie::INDEX_ANNOY);
list.emplace(ip::EF, ie::INDEX_HNSW);
list.emplace(ip::SEARCH_K, ie::INDEX_ANNOY);
#ifdef MILVUS_SUPPORT_NGT
list.emplace(ip::max_search_edges, ie::INDEX_NGTONNG);
list.emplace(ip::epsilon, ie::INDEX_NGTONNG);
@ -64,9 +64,9 @@ InferIndexType(const Json& search_params) {
static knowhere::IndexType
InferBinaryIndexType(const Json& search_params) {
namespace ip = knowhere::IndexParams;
namespace ip = knowhere::indexparam;
namespace ie = knowhere::IndexEnum;
if (search_params.contains(ip::nprobe)) {
if (search_params.contains(ip::NPROBE)) {
return ie::INDEX_FAISS_BIN_IVFFLAT;
} else {
return ie::INDEX_FAISS_BIN_IDMAP;
@ -82,7 +82,7 @@ VerifyPlanNodeVisitor::visit(FloatVectorANNS& node) {
// mock the api, topk will be passed from placeholder
auto params_copy = search_params;
params_copy[knowhere::meta::TOPK] = 10;
knowhere::SetMetaTopk(params_copy, 10);
// NOTE: the second parameter is not checked in knowhere, may be redundant
auto passed = adapter->CheckSearch(params_copy, inferred_type, index_mode);
@ -100,7 +100,7 @@ VerifyPlanNodeVisitor::visit(BinaryVectorANNS& node) {
// mock the api, topk will be passed from placeholder
auto params_copy = search_params;
params_copy[knowhere::meta::TOPK] = 10;
knowhere::SetMetaTopk(params_copy, 10);
// NOTE: the second parameter is not checked in knowhere, may be redundant
auto passed = adapter->CheckSearch(params_copy, inferred_type, index_mode);

8
internal/core/src/segcore/FieldIndexing.cpp
View File

@ -54,8 +54,8 @@ VectorFieldIndexing::get_build_params() const {
auto base_params = config.build_params;
AssertInfo(base_params.count("nlist"), "Can't get nlist from index params");
base_params[knowhere::meta::DIM] = field_meta_.get_dim();
base_params[knowhere::Metric::TYPE] = type_name;
knowhere::SetMetaDim(base_params, field_meta_.get_dim());
knowhere::SetMetaMetricType(base_params, type_name);
return base_params;
}
@ -71,8 +71,8 @@ VectorFieldIndexing::get_search_params(int top_K) const {
auto base_params = config.search_params;
AssertInfo(base_params.count("nprobe"), "Can't get nprobe from base params");
base_params[knowhere::meta::TOPK] = top_K;
base_params[knowhere::Metric::TYPE] = type_name;
knowhere::SetMetaTopk(base_params, top_K);
knowhere::SetMetaMetricType(base_params, type_name);
return base_params;
}

2
internal/core/src/segcore/FieldIndexing.h
View File

@ -22,7 +22,7 @@
#include "AckResponder.h"
#include "InsertRecord.h"
#include "common/Schema.h"
#include "knowhere/index/vector_index/VecIndex.h"
#include "knowhere/index/VecIndex.h"
#include "segcore/SegcoreConfig.h"
namespace milvus::segcore {

2
internal/core/src/segcore/SealedIndexingRecord.h
View File

@ -20,7 +20,7 @@
#include "common/Types.h"
#include "exceptions/EasyAssert.h"
#include "knowhere/index/vector_index/VecIndex.h"
#include "knowhere/index/VecIndex.h"
namespace milvus::segcore {

1
internal/core/src/segcore/SegmentGrowingImpl.h
View File

@ -30,7 +30,6 @@
#include "SegmentGrowing.h"
#include "exceptions/EasyAssert.h"
#include "knowhere/index/vector_index/VecIndex.h"
#include "query/PlanNode.h"
#include "query/deprecated/GeneralQuery.h"
#include "utils/Status.h"

1
internal/core/src/segcore/SegmentInterface.h
View File

@ -27,7 +27,6 @@
#include "common/LoadInfo.h"
#include "common/BitsetView.h"
#include "common/QueryResult.h"
#include "knowhere/index/vector_index/VecIndex.h"
#include "query/Plan.h"
#include "query/PlanNode.h"
#include "pb/schema.pb.h"

8
internal/core/src/segcore/load_index_c.cpp
View File

@ -9,13 +9,13 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include "common/CDataType.h"
#include "common/LoadInfo.h"
#include "exceptions/EasyAssert.h"
#include "knowhere/common/BinarySet.h"
#include "knowhere/index/vector_index/VecIndexFactory.h"
#include "segcore/load_index_c.h"
#include "index/IndexFactory.h"
#include "common/CDataType.h"
#include "knowhere/common/BinarySet.h"
#include "knowhere/index/VecIndexFactory.h"
#include "segcore/load_index_c.h"
CStatus
NewLoadIndexInfo(CLoadIndexInfo* c_load_index_info) {

4
internal/core/thirdparty/knowhere/CMakeLists.txt vendored
View File

@ -11,8 +11,8 @@
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
set( KNOWHERE_VERSION v1.1.5 )
set( KNOWHERE_SOURCE_MD5 "5baa7ecaea0e9e5c9f3f440cf8757d3c")
set( KNOWHERE_VERSION v1.1.12 )
set( KNOWHERE_SOURCE_MD5 "816dadc4d41fc9a4eab1fb9c9a7c2736")
if ( DEFINED ENV{MILVUS_KNOWHERE_URL} )
set( KNOWHERE_SOURCE_URL "$ENV{MILVUS_KNOWHERE_URL}" )

2
internal/core/unittest/bench/bench_indexbuilder.cpp
View File

@ -36,7 +36,7 @@ auto index_type_collections = [] {
auto metric_type_collections = [] {
static std::map<int, knowhere::MetricType> collections{
{0, knowhere::Metric::L2},
{0, knowhere::metric::L2},
};
return collections;
}();

6
internal/core/unittest/test_bool_index.cpp
View File

@ -33,9 +33,9 @@ class BoolIndexTest : public ::testing::Test {
void
TearDown() override {
delete[](char*)(all_true_ds->Get<const void*>(knowhere::meta::TENSOR));
delete[](char*) all_false_ds->Get<const void*>(knowhere::meta::TENSOR);
delete[](char*) half_ds->Get<const void*>(knowhere::meta::TENSOR);
delete[](char*)(knowhere::GetDatasetTensor(all_true_ds));
delete[](char*)(knowhere::GetDatasetTensor(all_false_ds));
delete[](char*)(knowhere::GetDatasetTensor(half_ds));
}
protected:

293
internal/core/unittest/test_c_api.cpp
View File

@ -9,21 +9,21 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include <gtest/gtest.h>
#include <boost/format.hpp>
#include <chrono>
#include <google/protobuf/text_format.h>
#include <gtest/gtest.h>
#include <iostream>
#include <random>
#include <string>
#include <unordered_set>
#include <knowhere/index/vector_index/helpers/IndexParameter.h>
#include <knowhere/index/vector_index/adapter/VectorAdapter.h>
#include <knowhere/index/vector_index/VecIndexFactory.h>
#include <knowhere/index/vector_index/IndexIVFPQ.h>
#include <boost/format.hpp>
#include "common/LoadInfo.h"
#include "common/memory_c.h"
#include "knowhere/index/VecIndexFactory.h"
#include "knowhere/index/vector_index/IndexIVFPQ.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
#include "pb/plan.pb.h"
#include "query/ExprImpl.h"
#include "segcore/Collection.h"
@ -148,7 +148,7 @@ generate_collection_schema(std::string metric_type, int dim, bool is_binary) {
}
VecIndexPtr
generate_index(void* raw_data, knowhere::Config conf, int64_t dim, int64_t topK, int64_t N, std::string index_type) {
generate_index(void* raw_data, knowhere::Config conf, int64_t dim, int64_t topK, int64_t N, knowhere::IndexType index_type) {
auto indexing = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type, knowhere::IndexMode::MODE_CPU);
auto database = knowhere::GenDataset(N, dim, raw_data);
@ -1241,14 +1241,16 @@ TEST(CApiTest, LoadIndexInfo) {
auto N = 1024 * 10;
auto [raw_data, timestamps, uids] = generate_data(N);
auto indexing = std::make_shared<knowhere::IVFPQ>();
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 4},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::indexparam::NLIST, 100},
{knowhere::indexparam::NPROBE, 4},
{knowhere::indexparam::M, 4},
{knowhere::indexparam::NBITS, 8},
{knowhere::meta::DEVICE_ID, 0}
};
auto database = knowhere::GenDataset(N, DIM, raw_data.data());
indexing->Train(database, conf);
@ -1284,14 +1286,16 @@ TEST(CApiTest, LoadIndex_Search) {
auto num_query = 100;
auto [raw_data, timestamps, uids] = generate_data(N);
auto indexing = std::make_shared<knowhere::IVFPQ>();
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 4},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::indexparam::NLIST, 100},
{knowhere::indexparam::NPROBE, 4},
{knowhere::indexparam::M, 4},
{knowhere::indexparam::NBITS, 8},
{knowhere::meta::DEVICE_ID, 0}
};
auto database = knowhere::GenDataset(N, DIM, raw_data.data());
indexing->Train(database, conf);
@ -1317,8 +1321,8 @@ TEST(CApiTest, LoadIndex_Search) {
auto result = indexing->Query(query_dataset, conf, nullptr);
auto ids = result->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result->Get<float*>(knowhere::meta::DISTANCE);
auto ids = knowhere::GetDatasetIDs(result);
auto dis = knowhere::GetDatasetDistance(result);
// for (int i = 0; i < std::min(num_query * K, 100); ++i) {
// std::cout << ids[i] << "->" << dis[i] << std::endl;
//}
@ -1386,21 +1390,23 @@ TEST(CApiTest, Indexing_Without_Predicate) {
assert(res_before_load_index.error_code == Success);
// load index to segment
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::indexparam::NLIST, 100},
{knowhere::indexparam::NPROBE, 10},
{knowhere::indexparam::M, 4},
{knowhere::indexparam::NBITS, 8},
{knowhere::meta::DEVICE_ID, 0}
};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ);
// gen query dataset
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
auto ids = knowhere::GetDatasetIDs(result_on_index);
auto dis = knowhere::GetDatasetDistance(result_on_index);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -1511,21 +1517,23 @@ TEST(CApiTest, Indexing_Expr_Without_Predicate) {
assert(res_before_load_index.error_code == Success);
// load index to segment
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::indexparam::NLIST, 100},
{knowhere::indexparam::NPROBE, 10},
{knowhere::indexparam::M, 4},
{knowhere::indexparam::NBITS, 8},
{knowhere::meta::DEVICE_ID, 0}
};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ);
// gen query dataset
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
auto ids = knowhere::GetDatasetIDs(result_on_index);
auto dis = knowhere::GetDatasetDistance(result_on_index);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -1653,22 +1661,23 @@ TEST(CApiTest, Indexing_With_float_Predicate_Range) {
assert(res_before_load_index.error_code == Success);
// load index to segment
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::indexparam::NLIST, 100},
{knowhere::indexparam::NPROBE, 10},
{knowhere::indexparam::M, 4},
{knowhere::indexparam::NBITS, 8},
{knowhere::meta::DEVICE_ID, 0}};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ);
// gen query dataset
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
auto ids = knowhere::GetDatasetIDs(result_on_index);
auto dis = knowhere::GetDatasetDistance(result_on_index);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -1810,22 +1819,24 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Range) {
assert(res_before_load_index.error_code == Success);
// load index to segment
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::indexparam::NLIST, 100},
{knowhere::indexparam::NPROBE, 10},
{knowhere::indexparam::M, 4},
{knowhere::indexparam::NBITS, 8},
{knowhere::meta::DEVICE_ID, 0}
};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ);
// gen query dataset
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
auto ids = knowhere::GetDatasetIDs(result_on_index);
auto dis = knowhere::GetDatasetDistance(result_on_index);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -1951,22 +1962,24 @@ TEST(CApiTest, Indexing_With_float_Predicate_Term) {
assert(res_before_load_index.error_code == Success);
// load index to segment
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::indexparam::NLIST, 100},
{knowhere::indexparam::NPROBE, 10},
{knowhere::indexparam::M, 4},
{knowhere::indexparam::NBITS, 8},
{knowhere::meta::DEVICE_ID, 0}
};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ);
// gen query dataset
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
auto ids = knowhere::GetDatasetIDs(result_on_index);
auto dis = knowhere::GetDatasetDistance(result_on_index);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -2101,22 +2114,24 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Term) {
assert(res_before_load_index.error_code == Success);
// load index to segment
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::indexparam::NLIST, 100},
{knowhere::indexparam::NPROBE, 10},
{knowhere::indexparam::M, 4},
{knowhere::indexparam::NBITS, 8},
{knowhere::meta::DEVICE_ID, 0}
};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ);
// gen query dataset
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
auto ids = knowhere::GetDatasetIDs(result_on_index);
auto dis = knowhere::GetDatasetDistance(result_on_index);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -2244,13 +2259,13 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Range) {
// load index to segment
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::JACCARD},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::JACCARD},
{knowhere::indexparam::NPROBE, 10},
{knowhere::indexparam::NLIST, 100},
{knowhere::indexparam::M, 4},
{knowhere::indexparam::NBITS, 8},
};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_BIN_IVFFLAT);
@ -2258,8 +2273,8 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Range) {
// gen query dataset
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
auto ids = knowhere::GetDatasetIDs(result_on_index);
auto dis = knowhere::GetDatasetDistance(result_on_index);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -2400,13 +2415,13 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Range) {
// load index to segment
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::JACCARD},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::JACCARD},
{knowhere::indexparam::NPROBE, 10},
{knowhere::indexparam::NLIST, 100},
{knowhere::indexparam::M, 4},
{knowhere::indexparam::NBITS, 8},
};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_BIN_IVFFLAT);
@ -2414,8 +2429,8 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Range) {
// gen query dataset
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
auto ids = knowhere::GetDatasetIDs(result_on_index);
auto dis = knowhere::GetDatasetDistance(result_on_index);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -2543,13 +2558,13 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Term) {
// load index to segment
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::JACCARD},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::JACCARD},
{knowhere::indexparam::NPROBE, 10},
{knowhere::indexparam::NLIST, 100},
{knowhere::indexparam::M, 4},
{knowhere::indexparam::NBITS, 8},
};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_BIN_IVFFLAT);
@ -2557,8 +2572,8 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Term) {
// gen query dataset
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
auto ids = knowhere::GetDatasetIDs(result_on_index);
auto dis = knowhere::GetDatasetDistance(result_on_index);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -2708,13 +2723,13 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Term) {
// load index to segment
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::JACCARD},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::JACCARD},
{knowhere::indexparam::NPROBE, 10},
{knowhere::indexparam::NLIST, 100},
{knowhere::indexparam::M, 4},
{knowhere::indexparam::NBITS, 8},
};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_BIN_IVFFLAT);
@ -2722,8 +2737,8 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Term) {
// gen query dataset
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
auto ids = knowhere::GetDatasetIDs(result_on_index);
auto dis = knowhere::GetDatasetDistance(result_on_index);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -2890,22 +2905,24 @@ TEST(CApiTest, SealedSegment_search_float_Predicate_Range) {
Timestamp time = 10000000;
// load index to segment
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::indexparam::NLIST, 100},
{knowhere::indexparam::NPROBE, 10},
{knowhere::indexparam::M, 4},
{knowhere::indexparam::NBITS, 8},
{knowhere::meta::DEVICE_ID, 0}
};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ);
// gen query dataset
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
auto ids = knowhere::GetDatasetIDs(result_on_index);
auto dis = knowhere::GetDatasetDistance(result_on_index);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -2933,8 +2950,8 @@ TEST(CApiTest, SealedSegment_search_float_Predicate_Range) {
auto query_dataset2 = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto index = std::dynamic_pointer_cast<knowhere::VecIndex>(load_index_info->index);
auto result_on_index2 = index->Query(query_dataset2, conf, nullptr);
auto ids2 = result_on_index2->Get<int64_t*>(knowhere::meta::IDS);
auto dis2 = result_on_index2->Get<float*>(knowhere::meta::DISTANCE);
auto ids2 = knowhere::GetDatasetIDs(result_on_index2);
auto dis2 = knowhere::GetDatasetDistance(result_on_index2);
auto c_counter_field_data = CLoadFieldDataInfo{
101,
@ -3185,14 +3202,16 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) {
Timestamp time = 10000000;
// load index to segment
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::indexparam::NLIST, 100},
{knowhere::indexparam::NPROBE, 10},
{knowhere::indexparam::M, 4},
{knowhere::indexparam::NBITS, 8},
{knowhere::meta::DEVICE_ID, 0}
};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ);
@ -3218,8 +3237,8 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) {
auto query_dataset2 = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto index = std::dynamic_pointer_cast<knowhere::VecIndex>(load_index_info->index);
auto result_on_index2 = index->Query(query_dataset2, conf, nullptr);
auto ids2 = result_on_index2->Get<int64_t*>(knowhere::meta::IDS);
auto dis2 = result_on_index2->Get<float*>(knowhere::meta::DISTANCE);
auto ids2 = knowhere::GetDatasetIDs(result_on_index2);
auto dis2 = knowhere::GetDatasetDistance(result_on_index2);
status = UpdateSealedSegmentIndex(segment, c_load_index_info);
assert(status.error_code == Success);
@ -3254,8 +3273,8 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) {
// gen query dataset
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
auto ids = knowhere::GetDatasetIDs(result_on_index);
auto dis = knowhere::GetDatasetDistance(result_on_index);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {

14
internal/core/unittest/test_index_c_api.cpp
View File

@ -32,7 +32,7 @@ constexpr int NB = 10;
TEST(FloatVecIndex, All) {
auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFPQ;
auto metric_type = knowhere::Metric::L2;
auto metric_type = knowhere::metric::L2;
indexcgo::TypeParams type_params;
indexcgo::IndexParams index_params;
std::tie(type_params, index_params) = generate_params(index_type, metric_type);
@ -83,7 +83,7 @@ TEST(FloatVecIndex, All) {
TEST(BinaryVecIndex, All) {
auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT;
auto metric_type = knowhere::Metric::JACCARD;
auto metric_type = knowhere::metric::JACCARD;
indexcgo::TypeParams type_params;
indexcgo::IndexParams index_params;
std::tie(type_params, index_params) = generate_params(index_type, metric_type);
@ -159,8 +159,7 @@ TEST(CBoolIndexTest, All) {
ASSERT_EQ(Success, status.error_code);
}
{
status = BuildScalarIndex(index, half_ds->Get<int64_t>(knowhere::meta::ROWS),
half_ds->Get<const void*>(knowhere::meta::TENSOR));
status = BuildScalarIndex(index, knowhere::GetDatasetRows(half_ds), knowhere::GetDatasetTensor(half_ds));
ASSERT_EQ(Success, status.error_code);
}
{
@ -185,7 +184,7 @@ TEST(CBoolIndexTest, All) {
}
}
delete[](char*) half_ds->Get<const void*>(knowhere::meta::TENSOR);
delete[](char*) knowhere::GetDatasetTensor(half_ds);
}
// TODO: more scalar type.
@ -262,8 +261,7 @@ TEST(CStringIndexTest, All) {
ASSERT_EQ(Success, status.error_code);
}
{
status = BuildScalarIndex(index, str_ds->Get<int64_t>(knowhere::meta::ROWS),
str_ds->Get<const void*>(knowhere::meta::TENSOR));
status = BuildScalarIndex(index, knowhere::GetDatasetRows(str_ds), knowhere::GetDatasetTensor(str_ds));
ASSERT_EQ(Success, status.error_code);
}
{
@ -288,6 +286,6 @@ TEST(CStringIndexTest, All) {
}
}
delete[](char*) str_ds->Get<const void*>(knowhere::meta::TENSOR);
delete[](char*) knowhere::GetDatasetTensor(str_ds);
}
#endif

74
internal/core/unittest/test_index_wrapper.cpp
View File

@ -41,7 +41,7 @@ class IndexWrapperTest : public ::testing::TestWithParam<Param> {
metric_type = param.second;
std::tie(type_params, index_params) = generate_params(index_type, metric_type);
std::map<std::string, bool> is_binary_map = {
std::map<knowhere::MetricType, bool> is_binary_map = {
{knowhere::IndexEnum::INDEX_FAISS_IDMAP, false},
{knowhere::IndexEnum::INDEX_FAISS_IVFPQ, false},
{knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, false},
@ -109,7 +109,7 @@ class IndexWrapperTest : public ::testing::TestWithParam<Param> {
TEST(PQ, Build) {
auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFPQ;
auto metric_type = knowhere::Metric::L2;
auto metric_type = knowhere::metric::L2;
auto conf = generate_conf(index_type, metric_type);
auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto dataset = GenDataset(NB, metric_type, false);
@ -121,7 +121,7 @@ TEST(PQ, Build) {
TEST(IVFFLATNM, Build) {
auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT;
auto metric_type = knowhere::Metric::L2;
auto metric_type = knowhere::metric::L2;
auto conf = generate_conf(index_type, metric_type);
auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto dataset = GenDataset(NB, metric_type, false);
@ -135,7 +135,7 @@ TEST(IVFFLATNM, Query) {
knowhere::KnowhereConfig::SetStatisticsLevel(3);
auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT;
auto metric_type = knowhere::Metric::L2;
auto metric_type = knowhere::metric::L2;
auto conf = generate_conf(index_type, metric_type);
auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto dataset = GenDataset(NB, metric_type, false);
@ -153,8 +153,7 @@ TEST(IVFFLATNM, Query) {
auto xq_dataset = knowhere::GenDataset(NQ, DIM, xq_data.data());
auto result = index->Query(xq_dataset, conf, nullptr);
index->UpdateIndexSize();
ASSERT_GT(index->IndexSize(), 0);
ASSERT_GT(index->Size(), 0);
auto stats = index->GetStatistics();
ASSERT_TRUE(stats != nullptr);
@ -164,7 +163,7 @@ TEST(IVFFLATNM, Query) {
#ifdef MILVUS_SUPPORT_NSG
TEST(NSG, Query) {
auto index_type = knowhere::IndexEnum::INDEX_NSG;
auto metric_type = knowhere::Metric::L2;
auto metric_type = knowhere::metric::L2;
auto conf = generate_conf(index_type, metric_type);
auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto dataset = GenDataset(NB, metric_type, false);
@ -185,7 +184,7 @@ TEST(NSG, Query) {
TEST(BINFLAT, Build) {
auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT;
auto metric_type = knowhere::Metric::JACCARD;
auto metric_type = knowhere::metric::JACCARD;
auto conf = generate_conf(index_type, metric_type);
auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto dataset = GenDataset(NB, metric_type, true);
@ -212,11 +211,11 @@ TEST(BinIVFFlat, Build_and_Query) {
knowhere::KnowhereConfig::SetStatisticsLevel(2);
auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT;
auto metric_type = knowhere::Metric::TANIMOTO;
auto metric_type = knowhere::metric::TANIMOTO;
auto conf = generate_conf(index_type, metric_type);
auto topk = 10;
conf[knowhere::meta::TOPK] = topk;
conf[knowhere::IndexParams::nlist] = 1;
knowhere::SetMetaTopk(conf, topk);
knowhere::SetIndexParamNlist(conf, 1);
auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto nb = 2;
auto dim = 128;
@ -231,8 +230,8 @@ TEST(BinIVFFlat, Build_and_Query) {
auto xq_dataset = knowhere::GenDataset(nq, dim, xq_data.data());
auto result = index->Query(xq_dataset, conf, nullptr);
auto hit_ids = result->Get<int64_t*>(knowhere::meta::IDS);
auto distances = result->Get<float*>(knowhere::meta::DISTANCE);
auto hit_ids = knowhere::GetDatasetIDs(result);
auto distances = knowhere::GetDatasetDistance(result);
auto query_res = std::make_unique<milvus::indexbuilder::VecIndexCreator::QueryResult>();
query_res->nq = nq;
@ -244,8 +243,7 @@ TEST(BinIVFFlat, Build_and_Query) {
print_query_result(query_res);
index->UpdateIndexSize();
ASSERT_GT(index->IndexSize(), 0);
ASSERT_GT(index->Size(), 0);
auto stats = index->GetStatistics();
ASSERT_TRUE(stats != nullptr);
@ -254,7 +252,7 @@ TEST(BinIVFFlat, Build_and_Query) {
TEST(BINIDMAP, Build) {
auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP;
auto metric_type = knowhere::Metric::JACCARD;
auto metric_type = knowhere::metric::JACCARD;
auto conf = generate_conf(index_type, metric_type);
auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto dataset = GenDataset(NB, metric_type, true);
@ -267,7 +265,7 @@ TEST(BINIDMAP, Build) {
TEST(PQWrapper, Build) {
auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFPQ;
auto metric_type = knowhere::Metric::L2;
auto metric_type = knowhere::metric::L2;
indexcgo::TypeParams type_params;
indexcgo::IndexParams index_params;
std::tie(type_params, index_params) = generate_params(index_type, metric_type);
@ -287,7 +285,7 @@ TEST(PQWrapper, Build) {
TEST(IVFFLATNMWrapper, Build) {
auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT;
auto metric_type = knowhere::Metric::L2;
auto metric_type = knowhere::metric::L2;
indexcgo::TypeParams type_params;
indexcgo::IndexParams index_params;
std::tie(type_params, index_params) = generate_params(index_type, metric_type);
@ -308,7 +306,7 @@ TEST(IVFFLATNMWrapper, Build) {
TEST(IVFFLATNMWrapper, Codec) {
int64_t flat_nb = 100000;
auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT;
auto metric_type = knowhere::Metric::L2;
auto metric_type = knowhere::metric::L2;
indexcgo::TypeParams type_params;
indexcgo::IndexParams index_params;
std::tie(type_params, index_params) = generate_params(index_type, metric_type);
@ -342,7 +340,7 @@ TEST(IVFFLATNMWrapper, Codec) {
TEST(BinFlatWrapper, Build) {
auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT;
auto metric_type = knowhere::Metric::JACCARD;
auto metric_type = knowhere::metric::JACCARD;
indexcgo::TypeParams type_params;
indexcgo::IndexParams index_params;
std::tie(type_params, index_params) = generate_params(index_type, metric_type);
@ -365,7 +363,7 @@ TEST(BinFlatWrapper, Build) {
TEST(BinIdMapWrapper, Build) {
auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP;
auto metric_type = knowhere::Metric::JACCARD;
auto metric_type = knowhere::metric::JACCARD;
indexcgo::TypeParams type_params;
indexcgo::IndexParams index_params;
std::tie(type_params, index_params) = generate_params(index_type, metric_type);
@ -389,28 +387,28 @@ TEST(BinIdMapWrapper, Build) {
INSTANTIATE_TEST_CASE_P(
IndexTypeParameters,
IndexWrapperTest,
::testing::Values(std::pair(knowhere::IndexEnum::INDEX_FAISS_IDMAP, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_FAISS_IVFPQ, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, knowhere::Metric::JACCARD),
std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, knowhere::Metric::TANIMOTO),
std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, knowhere::Metric::JACCARD),
::testing::Values(std::pair(knowhere::IndexEnum::INDEX_FAISS_IDMAP, knowhere::metric::L2),
std::pair(knowhere::IndexEnum::INDEX_FAISS_IVFPQ, knowhere::metric::L2),
std::pair(knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, knowhere::metric::L2),
std::pair(knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, knowhere::metric::L2),
std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, knowhere::metric::JACCARD),
std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, knowhere::metric::TANIMOTO),
std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, knowhere::metric::JACCARD),
#ifdef MILVUS_SUPPORT_SPTAG
std::pair(knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, knowhere::metric::L2),
std::pair(knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, knowhere::metric::L2),
#endif
std::pair(knowhere::IndexEnum::INDEX_HNSW, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_ANNOY, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_RHNSWFlat, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_RHNSWPQ, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_RHNSWSQ, knowhere::Metric::L2)
std::pair(knowhere::IndexEnum::INDEX_HNSW, knowhere::metric::L2),
std::pair(knowhere::IndexEnum::INDEX_ANNOY, knowhere::metric::L2),
std::pair(knowhere::IndexEnum::INDEX_RHNSWFlat, knowhere::metric::L2),
std::pair(knowhere::IndexEnum::INDEX_RHNSWPQ, knowhere::metric::L2),
std::pair(knowhere::IndexEnum::INDEX_RHNSWSQ, knowhere::metric::L2)
#ifdef MILVUS_SUPPORT_NGT
std::pair(knowhere::IndexEnum::INDEX_NGTPANNG, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_NGTONNG, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_NGTPANNG, knowhere::metric::L2),
std::pair(knowhere::IndexEnum::INDEX_NGTONNG, knowhere::metric::L2),
#endif
#ifdef MILVUS_SUPPORT_NSG
std::pair(knowhere::IndexEnum::INDEX_NSG, knowhere::Metric::L2)
std::pair(knowhere::IndexEnum::INDEX_NSG, knowhere::metric::L2)
#endif
));

53
internal/core/unittest/test_indexing.cpp
View File

@ -144,14 +144,14 @@ TEST(Indexing, Naive) {
knowhere::IndexMode::MODE_CPU);
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 4},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0},
{knowhere::indexparam::NLIST, 100},
{knowhere::indexparam::NPROBE, 4},
{knowhere::indexparam::M, 4},
{knowhere::indexparam::NBITS, 8},
{knowhere::meta::DEVICE_ID, 0},
};
// auto ds = knowhere::GenDataset(N, DIM, raw_data.data());
@ -199,8 +199,8 @@ TEST(Indexing, Naive) {
BitsetView view = bitmap;
auto query_ds = knowhere::GenDataset(1, DIM, raw_data.data());
auto final = index->Query(query_ds, conf, view);
auto ids = final->Get<idx_t*>(knowhere::meta::IDS);
auto distances = final->Get<float*>(knowhere::meta::DISTANCE);
auto ids = knowhere::GetDatasetIDs(final);
auto distances = knowhere::GetDatasetDistance(final);
for (int i = 0; i < TOPK; ++i) {
if (ids[i] < N / 2) {
std::cout << "WRONG: ";
@ -221,12 +221,14 @@ TEST(Indexing, IVFFlat) {
auto [raw_data, timestamps, uids] = generate_data<DIM>(N);
std::cout << "generate data: " << timer.get_step_seconds() << " seconds" << std::endl;
auto indexing = std::make_shared<knowhere::IVF>();
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, NLIST},
{knowhere::IndexParams::nprobe, NPROBE},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::indexparam::NLIST, NLIST},
{knowhere::indexparam::NPROBE, NPROBE},
{knowhere::meta::DEVICE_ID, 0}
};
auto database = knowhere::GenDataset(N, DIM, raw_data.data());
std::cout << "init ivf " << timer.get_step_seconds() << " seconds" << std::endl;
@ -242,8 +244,8 @@ TEST(Indexing, IVFFlat) {
auto result = indexing->Query(dataset, conf, nullptr);
std::cout << "query ivf " << timer.get_step_seconds() << " seconds" << std::endl;
auto ids = result->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result->Get<float*>(knowhere::meta::DISTANCE);
auto ids = knowhere::GetDatasetIDs(result);
auto dis = knowhere::GetDatasetDistance(result);
for (int i = 0; i < std::min(NQ * TOPK, 100); ++i) {
std::cout << ids[i] << "->" << dis[i] << std::endl;
}
@ -261,12 +263,14 @@ TEST(Indexing, IVFFlatNM) {
auto [raw_data, timestamps, uids] = generate_data<DIM>(N);
std::cout << "generate data: " << timer.get_step_seconds() << " seconds" << std::endl;
auto indexing = std::make_shared<knowhere::IVF_NM>();
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, NLIST},
{knowhere::IndexParams::nprobe, NPROBE},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::indexparam::NLIST, NLIST},
{knowhere::indexparam::NPROBE, NPROBE},
{knowhere::meta::DEVICE_ID, 0}
};
auto database = knowhere::GenDataset(N, DIM, raw_data.data());
std::cout << "init ivf_nm " << timer.get_step_seconds() << " seconds" << std::endl;
@ -275,7 +279,6 @@ TEST(Indexing, IVFFlatNM) {
indexing->AddWithoutIds(database, conf);
std::cout << "insert ivf_nm " << timer.get_step_seconds() << " seconds" << std::endl;
indexing->SetIndexSize(NQ * DIM * sizeof(float));
knowhere::BinarySet bs = indexing->Serialize(conf);
knowhere::BinaryPtr bptr = std::make_shared<knowhere::Binary>();
@ -291,8 +294,8 @@ TEST(Indexing, IVFFlatNM) {
auto result = indexing->Query(dataset, conf, nullptr);
std::cout << "query ivf_nm " << timer.get_step_seconds() << " seconds" << std::endl;
auto ids = result->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result->Get<float*>(knowhere::meta::DISTANCE);
auto ids = knowhere::GetDatasetIDs(result);
auto dis = knowhere::GetDatasetDistance(result);
for (int i = 0; i < std::min(NQ * TOPK, 100); ++i) {
std::cout << ids[i] << "->" << dis[i] << std::endl;
}

4
internal/core/unittest/test_scalar_index_creator.cpp
View File

@ -54,7 +54,7 @@ build_index(const ScalarIndexCreatorPtr& creator, const std::vector<bool>& arr)
creator->Build(ds);
delete[](char*) ds->Get<const void*>(knowhere::meta::TENSOR);
delete[](char*) knowhere::GetDatasetTensor(ds);
}
template <>
@ -66,7 +66,7 @@ build_index(const ScalarIndexCreatorPtr& creator, const std::vector<std::string>
creator->Build(ds);
delete[](char*) ds->Get<const void*>(knowhere::meta::TENSOR);
delete[](char*) knowhere::GetDatasetTensor(ds);
}
} // namespace

38
internal/core/unittest/test_sealed.cpp
View File

@ -11,8 +11,8 @@
#include <gtest/gtest.h>
#include "knowhere/index/VecIndex.h"
#include "knowhere/index/vector_index/IndexIVF.h"
#include "knowhere/index/vector_index/VecIndex.h"
#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
#include "segcore/SegmentSealedImpl.h"
#include "test_utils/DataGen.h"
@ -78,12 +78,14 @@ TEST(Sealed, without_predicate) {
auto pre_result = SearchResultToJson(*sr);
auto indexing = std::make_shared<knowhere::IVF>();
auto conf = knowhere::Config{{knowhere::meta::DIM, dim},
{knowhere::meta::TOPK, topK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
{knowhere::meta::DIM, dim},
{knowhere::meta::TOPK, topK},
{knowhere::indexparam::NLIST, 100},
{knowhere::indexparam::NPROBE, 10},
{knowhere::meta::DEVICE_ID, 0}
};
auto database = knowhere::GenDataset(N, dim, vec_col.data() + 1000 * dim);
indexing->Train(database, conf);
@ -96,8 +98,8 @@ TEST(Sealed, without_predicate) {
auto result = indexing->Query(query_dataset, conf, nullptr);
auto ids = result->Get<int64_t*>(knowhere::meta::IDS); // for comparison
auto dis = result->Get<float*>(knowhere::meta::DISTANCE); // for comparison
auto ids = knowhere::GetDatasetIDs(result); // for comparison
auto dis = knowhere::GetDatasetDistance(result); // for comparison
std::vector<int64_t> vec_ids(ids, ids + topK * num_queries);
std::vector<float> vec_dis(dis, dis + topK * num_queries);
@ -183,12 +185,14 @@ TEST(Sealed, with_predicate) {
auto sr = segment->Search(plan.get(), ph_group.get(), time);
auto indexing = std::make_shared<knowhere::IVF>();
auto conf = knowhere::Config{{knowhere::meta::DIM, dim},
{knowhere::meta::TOPK, topK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
{knowhere::meta::DIM, dim},
{knowhere::meta::TOPK, topK},
{knowhere::indexparam::NLIST, 100},
{knowhere::indexparam::NPROBE, 10},
{knowhere::meta::DEVICE_ID, 0}
};
auto database = knowhere::GenDataset(N, dim, vec_col.data());
indexing->Train(database, conf);
@ -286,7 +290,7 @@ TEST(Sealed, LoadFieldData) {
LoadIndexInfo vec_info;
vec_info.field_id = fakevec_id.get();
vec_info.index = indexing;
vec_info.index_params["metric_type"] = knowhere::Metric::L2;
vec_info.index_params["metric_type"] = knowhere::metric::L2;
segment->LoadIndex(vec_info);
ASSERT_EQ(segment->num_chunk(), 1);
@ -414,7 +418,7 @@ TEST(Sealed, LoadScalarIndex) {
vec_info.field_id = fakevec_id.get();
vec_info.field_type = CDataType::FloatVector;
vec_info.index = indexing;
vec_info.index_params["metric_type"] = knowhere::Metric::L2;
vec_info.index_params["metric_type"] = knowhere::metric::L2;
segment->LoadIndex(vec_info);
LoadIndexInfo counter_index;

2
internal/core/unittest/test_string_index.cpp
View File

@ -39,7 +39,7 @@ class StringIndexBaseTest : public ::testing::Test {
void
TearDown() override {
delete[](char*)(str_ds->Get<const void*>(knowhere::meta::TENSOR));
delete[](char*)(knowhere::GetDatasetTensor(str_ds));
}
protected:

24
internal/core/unittest/test_utils/DataGen.h
View File

@ -15,20 +15,20 @@
#include <cstring>
#include <memory>
#include <random>
#include "google/protobuf/text_format.h"
#include <knowhere/index/vector_index/VecIndex.h>
#include <knowhere/index/vector_index/adapter/VectorAdapter.h>
#include <knowhere/index/vector_index/VecIndexFactory.h>
#include <knowhere/index/vector_index/IndexIVF.h>
#include <google/protobuf/text_format.h>
#include "Constants.h"
#include "common/Schema.h"
#include "index/ScalarIndexSort.h"
#include "index/StringIndexSort.h"
#include "knowhere/index/VecIndex.h"
#include "knowhere/index/VecIndexFactory.h"
#include "knowhere/index/vector_index/IndexIVF.h"
#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
#include "query/SearchOnIndex.h"
#include "segcore/SegmentGrowingImpl.h"
#include "segcore/SegmentSealedImpl.h"
#include "segcore/Utils.h"
#include "index/ScalarIndexSort.h"
#include "index/StringIndexSort.h"
using boost::algorithm::starts_with;
@ -396,10 +396,12 @@ SealedCreator(SchemaPtr schema, const GeneratedData& dataset) {
inline knowhere::VecIndexPtr
GenVecIndexing(int64_t N, int64_t dim, const float* vec) {
// {knowhere::IndexParams::nprobe, 10},
auto conf = knowhere::Config{{knowhere::meta::DIM, dim},
{knowhere::IndexParams::nlist, 1024},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{
{knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
{knowhere::meta::DIM, dim},
{knowhere::indexparam::NLIST, 1024},
{knowhere::meta::DEVICE_ID, 0}
};
auto database = knowhere::GenDataset(N, dim, vec);
auto indexing = std::make_shared<knowhere::IVF>();
indexing->Train(database, conf);

192
internal/core/unittest/test_utils/indexbuilder_test_utils.h
View File

@ -16,21 +16,20 @@
#include <limits>
#include <cmath>
#include <google/protobuf/text_format.h>
#include <knowhere/common/MetricType.h>
#include <knowhere/index/vector_index/helpers/IndexParameter.h>
#include <knowhere/index/vector_index/adapter/VectorAdapter.h>
#include <knowhere/index/vector_index/VecIndexFactory.h>
#include <index/ScalarIndex.h>
#include <index/StringIndex.h>
#include "pb/index_cgo_msg.pb.h"
#include "indexbuilder/VecIndexCreator.h"
#include "indexbuilder/index_c.h"
#include "DataGen.h"
#include "indexbuilder/utils.h"
#include "indexbuilder/helper.h"
#include "index/ScalarIndex.h"
#include "index/StringIndex.h"
#include "indexbuilder/ScalarIndexCreator.h"
#include "indexbuilder/VecIndexCreator.h"
#include "indexbuilder/helper.h"
#include "indexbuilder/index_c.h"
#include "indexbuilder/utils.h"
#include "knowhere/common/MetricType.h"
#include "knowhere/index/VecIndexFactory.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
#include "pb/index_cgo_msg.pb.h"
constexpr int64_t DIM = 8;
constexpr int64_t NQ = 10;
@ -52,164 +51,169 @@ auto
generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType& metric_type) {
if (index_type == knowhere::IndexEnum::INDEX_FAISS_IDMAP) {
return knowhere::Config{
{knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
{knowhere::meta::METRIC_TYPE, metric_type},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
};
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFPQ) {
return knowhere::Config{
{knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
{knowhere::meta::METRIC_TYPE, metric_type},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::IndexParams::nlist, 16},
{knowhere::IndexParams::nprobe, 4},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
{knowhere::indexparam::NLIST, 16},
{knowhere::indexparam::NPROBE, 4},
{knowhere::indexparam::M, 4},
{knowhere::indexparam::NBITS, 8},
};
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) {
return knowhere::Config{
{knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
{knowhere::meta::METRIC_TYPE, metric_type},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::IndexParams::nlist, 16},
{knowhere::IndexParams::nprobe, 4},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
{knowhere::indexparam::NLIST, 16},
{knowhere::indexparam::NPROBE, 4},
#ifdef MILVUS_GPU_VERSION
{knowhere::meta::DEVICEID, DEVICEID},
{knowhere::meta::DEVICE_ID, DEVICEID},
#endif
};
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFSQ8) {
return knowhere::Config{
{knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
{knowhere::meta::METRIC_TYPE, metric_type},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::IndexParams::nlist, 16},
{knowhere::IndexParams::nprobe, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
{knowhere::indexparam::NLIST, 16},
{knowhere::indexparam::NPROBE, 4},
{knowhere::indexparam::NBITS, 8},
#ifdef MILVUS_GPU_VERSION
{knowhere::meta::DEVICEID, DEVICEID},
{knowhere::meta::DEVICE_ID, DEVICEID},
#endif
};
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT) {
return knowhere::Config{
{knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
{knowhere::meta::METRIC_TYPE, metric_type},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::IndexParams::nlist, 16},
{knowhere::IndexParams::nprobe, 4},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
{knowhere::indexparam::NLIST, 16},
{knowhere::indexparam::NPROBE, 4},
{knowhere::indexparam::M, 4},
{knowhere::indexparam::NBITS, 8},
};
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP) {
return knowhere::Config{
{knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
{knowhere::meta::METRIC_TYPE, metric_type},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::Metric::TYPE, metric_type},
};
#ifdef MILVUS_SUPPORT_NSG
} else if (index_type == knowhere::IndexEnum::INDEX_NSG) {
return knowhere::Config{
{knowhere::meta::METRIC_TYPE, metric_type},
{knowhere::meta::DIM, DIM},
{knowhere::IndexParams::nlist, 163},
{knowhere::meta::TOPK, K},
{knowhere::IndexParams::nprobe, 8},
{knowhere::IndexParams::knng, 20},
{knowhere::IndexParams::search_length, 40},
{knowhere::IndexParams::out_degree, 30},
{knowhere::IndexParams::candidate, 100},
{knowhere::Metric::TYPE, metric_type},
{knowhere::indexparam::NLIST, 163},
{knowhere::indexparam::NPROBE, 8},
{knowhere::indexparam::KNNG, 20},
{knowhere::indexparam::SEARCH_LENGTH, 40},
{knowhere::indexparam::OUT_DEGREE, 30},
{knowhere::indexparam::CANDIDATE, 100},
};
#endif
#ifdef MILVUS_SUPPORT_SPTAG
} else if (index_type == knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT) {
return knowhere::Config{
{knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
{knowhere::meta::METRIC_TYPE, metric_type},
{knowhere::meta::DIM, DIM},
// {knowhere::meta::TOPK, 10},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
{knowhere::meta::TOPK, 10},
};
} else if (index_type == knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT) {
return knowhere::Config{
{knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
{knowhere::meta::METRIC_TYPE, metric_type},
{knowhere::meta::DIM, DIM},
// {knowhere::meta::TOPK, 10},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
{knowhere::meta::TOPK, 10},
};
#endif
} else if (index_type == knowhere::IndexEnum::INDEX_HNSW) {
return knowhere::Config{
{knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, K},
{knowhere::IndexParams::M, 16}, {knowhere::IndexParams::efConstruction, 200},
{knowhere::IndexParams::ef, 200}, {knowhere::Metric::TYPE, metric_type},
{knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
{knowhere::meta::METRIC_TYPE, metric_type},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::indexparam::HNSW_M, 16},
{knowhere::indexparam::EFCONSTRUCTION, 200},
{knowhere::indexparam::EF, 200},
};
} else if (index_type == knowhere::IndexEnum::INDEX_ANNOY) {
return knowhere::Config{
{knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
{knowhere::meta::METRIC_TYPE, metric_type},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::IndexParams::n_trees, 4},
{knowhere::IndexParams::search_k, 100},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
{knowhere::indexparam::N_TREES, 4},
{knowhere::indexparam::SEARCH_K, 100},
};
} else if (index_type == knowhere::IndexEnum::INDEX_RHNSWFlat) {
return knowhere::Config{
{knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
{knowhere::meta::METRIC_TYPE, metric_type},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::IndexParams::M, 16},
{knowhere::IndexParams::efConstruction, 200},
{knowhere::IndexParams::ef, 200},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
{knowhere::indexparam::HNSW_M, 16},
{knowhere::indexparam::EFCONSTRUCTION, 200},
{knowhere::indexparam::EF, 200},
};
} else if (index_type == knowhere::IndexEnum::INDEX_RHNSWPQ) {
return knowhere::Config{
{knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
{knowhere::meta::METRIC_TYPE, metric_type},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::IndexParams::M, 16},
{knowhere::IndexParams::efConstruction, 200},
{knowhere::IndexParams::ef, 200},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
{knowhere::IndexParams::PQM, 8},
{knowhere::indexparam::HNSW_M, 16},
{knowhere::indexparam::EFCONSTRUCTION, 200},
{knowhere::indexparam::EF, 200},
{knowhere::indexparam::PQ_M, 8},
};
} else if (index_type == knowhere::IndexEnum::INDEX_RHNSWSQ) {
return knowhere::Config{
{knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
{knowhere::meta::METRIC_TYPE, metric_type},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::IndexParams::M, 16},
{knowhere::IndexParams::efConstruction, 200},
{knowhere::IndexParams::ef, 200},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
{knowhere::indexparam::HNSW_M, 16},
{knowhere::indexparam::EFCONSTRUCTION, 200},
{knowhere::indexparam::EF, 200},
};
#ifdef MILVUS_SUPPORT_NGT
} else if (index_type == knowhere::IndexEnum::INDEX_NGTPANNG) {
return knowhere::Config{
{knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
{knowhere::meta::METRIC_TYPE, metric_type},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::Metric::TYPE, metric_type},
{knowhere::IndexParams::edge_size, 10},
{knowhere::IndexParams::epsilon, 0.1},
{knowhere::IndexParams::max_search_edges, 50},
{knowhere::IndexParams::forcedly_pruned_edge_size, 60},
{knowhere::IndexParams::selectively_pruned_edge_size, 30},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
{knowhere::indexparam::EDGE_SIZE, 10},
{knowhere::indexparam::EPSILON, 0.1},
{knowhere::indexparam::MAX_SEARCH_EDGES, 50},
{knowhere::indexparam::FORCEDLY_PRUNED_EDGE_SIZE, 60},
{knowhere::indexparam::SELECTIVELY_PRUNED_EDGE_SIZE, 30},
};
} else if (index_type == knowhere::IndexEnum::INDEX_NGTONNG) {
return knowhere::Config{
{knowhere::meta::SLICE_SIZE, knowhere::index_file_slice_size},
{knowhere::meta::METRIC_TYPE, metric_type},
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::Metric::TYPE, metric_type},
{knowhere::IndexParams::edge_size, 20},
{knowhere::IndexParams::epsilon, 0.1},
{knowhere::IndexParams::max_search_edges, 50},
{knowhere::IndexParams::outgoing_edge_size, 5},
{knowhere::IndexParams::incoming_edge_size, 40},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
{knowhere::indexparam::EDGE_SIZE, 20},
{knowhere::indexparam::EPSILON, 0.1},
{knowhere::indexparam::MAX_SEARCH_EDGES, 50},
{knowhere::indexparam::OUTGOING_EDGE_SIZE, 5},
{knowhere::indexparam::INCOMING_EDGE_SIZE, 40},
};
#endif
}
@ -241,7 +245,7 @@ generate_params(const knowhere::IndexType& index_type, const knowhere::MetricTyp
auto
GenDataset(int64_t N, const knowhere::MetricType& metric_type, bool is_binary, int64_t dim = DIM) {
auto schema = std::make_shared<milvus::Schema>();
auto faiss_metric_type = knowhere::GetMetricType(metric_type);
auto faiss_metric_type = knowhere::GetMetricType(std::string(metric_type));
if (!is_binary) {
schema->AddDebugField("fakevec", milvus::DataType::VECTOR_FLOAT, dim, faiss_metric_type);
return milvus::segcore::DataGen(schema, N);
@ -312,9 +316,9 @@ CountDistance(
if (point_a == nullptr || point_b == nullptr) {
return std::numeric_limits<float>::max();
}
if (metric == knowhere::Metric::L2) {
if (metric == knowhere::metric::L2) {
return L2(static_cast<const float*>(point_a), static_cast<const float*>(point_b), dim);
} else if (metric == knowhere::Metric::JACCARD) {
} else if (metric == knowhere::metric::JACCARD) {
return Jaccard(static_cast<const uint8_t*>(point_a), static_cast<const uint8_t*>(point_b), dim);
} else {
return std::numeric_limits<float>::max();
@ -327,9 +331,9 @@ CheckDistances(const QueryResultPtr& result,
const knowhere::DatasetPtr& query_dataset,
const knowhere::MetricType& metric,
const float threshold = 1.0e-5) {
auto base_vecs = base_dataset->Get<float*>(knowhere::meta::TENSOR);
auto query_vecs = query_dataset->Get<float*>(knowhere::meta::TENSOR);
auto dim = base_dataset->Get<int64_t>(knowhere::meta::DIM);
auto base_vecs = (float*)knowhere::GetDatasetTensor(base_dataset);
auto query_vecs = (float*)knowhere::GetDatasetTensor(query_dataset);
auto dim = knowhere::GetDatasetDim(base_dataset);
auto nq = result->nq;
auto k = result->topk;
for (auto i = 0; i < nq; i++) {