milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Upgrade to knowhere-v1.1.2 to support all index types for mac (#16416) 

Signed-off-by: Cai Yudong <yudong.cai@zilliz.com>
pull/16432/head
Cai Yudong 2022-04-08 15:29:31 +08:00 committed by GitHub
parent 9e590a788b
commit a37479d728
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 152 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/code-checker.yaml vendored
View File

@ -109,7 +109,7 @@ jobs:
if [[ ! -d "/var/tmp/ccache" ]];then
mkdir -p /var/tmp/ccache
fi
brew install boost libomp ninja tbb ccache
brew install boost libomp ninja tbb openblas ccache
make check-proto-product && make verifiers
centos:
name: Code Checker CentOS 7

21
internal/core/CMakeLists.txt
View File

@ -31,26 +31,7 @@ project(core)
include(CheckCXXCompilerFlag)
if ( APPLE )
message(STATUS "==============Darwin Environment==============")
check_cxx_compiler_flag(-std=c++11 HAS_STD_CPP11_FLAG)
if(HAS_STD_CPP11_FLAG)
add_compile_options(-std=c++11)
endif()
if(CMAKE_C_COMPILER_ID MATCHES "Clang")
set(OpenMP_C "${CMAKE_C_COMPILER}" CACHE STRING "" FORCE)
set(OpenMP_C_FLAGS "-fopenmp=libomp -Wno-unused-command-line-argument" CACHE STRING "" FORCE)
set(OpenMP_C_LIB_NAMES "libomp" "libgomp" "libiomp5" CACHE STRING "" FORCE)
set(OpenMP_libomp_LIBRARY ${OpenMP_C_LIB_NAMES} CACHE STRING "" FORCE)
set(OpenMP_libgomp_LIBRARY ${OpenMP_C_LIB_NAMES} CACHE STRING "" FORCE)
set(OpenMP_libiomp5_LIBRARY ${OpenMP_C_LIB_NAMES} CACHE STRING "" FORCE)
endif()
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
set(OpenMP_CXX "${CMAKE_CXX_COMPILER}" CACHE STRING "" FORCE)
set(OpenMP_CXX_FLAGS "-fopenmp=libomp -Wno-unused-command-line-argument" CACHE STRING "" FORCE)
set(OpenMP_CXX_LIB_NAMES "libomp" "libgomp" "libiomp5" CACHE STRING "" FORCE)
set(OpenMP_libomp_LIBRARY ${OpenMP_CXX_LIB_NAMES} CACHE STRING "" FORCE)
set(OpenMP_libgomp_LIBRARY ${OpenMP_CXX_LIB_NAMES} CACHE STRING "" FORCE)
set(OpenMP_libiomp5_LIBRARY ${OpenMP_CXX_LIB_NAMES} CACHE STRING "" FORCE)
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/usr/local/opt/llvm/include -I/usr/local/include -I/usr/local/opt/libomp/include -L/usr/local/opt/libomp/lib")
elseif (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
message(STATUS "==============Linux Environment===============")
set(LINUX TRUE)

3
internal/core/src/index/IndexFactory-inl.h
View File

@ -34,8 +34,9 @@ inline ScalarIndexPtr<std::string>
IndexFactory::CreateIndex(const std::string& index_type) {
#ifdef __linux__
return CreateStringIndexMarisa();
#endif
#else
throw std::runtime_error("unsupported platform");
#endif
}
} // namespace milvus::scalar

2
internal/core/src/index/StringIndexMarisa.cpp
View File

@ -80,7 +80,7 @@ StringIndexMarisa::Serialize(const Config& config) {
res_set.Append(MARISA_TRIE_INDEX, index_data, size);
res_set.Append(MARISA_STR_IDS, str_ids, str_ids_len);
knowhere::Disassemble(4 * 1024 * 1024, res_set);
knowhere::Disassemble(res_set, config);
return res_set;
}

5
internal/core/src/indexbuilder/VecIndexCreator.cpp
View File

@ -238,10 +238,9 @@ VecIndexCreator::Serialize() {
std::shared_ptr<uint8_t[]> raw_data(new uint8_t[raw_data_.size()], std::default_delete<uint8_t[]>());
memcpy(raw_data.get(), raw_data_.data(), raw_data_.size());
ret.Append(RAW_DATA, raw_data, raw_data_.size());
auto slice_size = get_index_file_slice_size();
// https://github.com/milvus-io/milvus/issues/6421
// Disassemble will only divide the raw vectors, other keys were already divided
knowhere::Disassemble(slice_size * 1024 * 1024, ret);
knowhere::Disassemble(ret, config_);
}
return ret;
}
@ -300,7 +299,7 @@ VecIndexCreator::get_index_file_slice_size() {
if (config_.contains(knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE)) {
return config_[knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE].get<int64_t>();
}
return 4; // by default
return knowhere::index_file_slice_size; // by default
}
std::unique_ptr<VecIndexCreator::QueryResult>

4
internal/core/src/indexbuilder/utils.cpp
View File

@ -25,7 +25,9 @@ std::vector<std::string>
NM_List() {
static std::vector<std::string> ret{
knowhere::IndexEnum::INDEX_FAISS_IVFFLAT,
#ifdef MILVUS_SUPPORT_NSG
knowhere::IndexEnum::INDEX_NSG,
#endif
knowhere::IndexEnum::INDEX_RHNSWFlat,
};
return ret;
@ -53,7 +55,9 @@ Need_ID_List() {
std::vector<std::string>
Need_BuildAll_list() {
static std::vector<std::string> ret{
#ifdef MILVUS_SUPPORT_NSG
knowhere::IndexEnum::INDEX_NSG,
#endif
};
return ret;
}

25
internal/core/src/query/SearchBruteForce.cpp
View File

@ -12,15 +12,8 @@
#include <string>
#include <vector>
#ifdef __APPLE__
#include "knowhere/index/vector_index/impl/bruteforce/BruteForce.h"
#include "knowhere/common/Heap.h"
#elif defined(__linux__) || defined(__MINGW64__)
#include <faiss/utils/BinaryDistance.h>
#include <faiss/utils/distances.h>
#else
#error "Unsupported OS environment.";
#endif
#include "SearchBruteForce.h"
#include "SubSearchResult.h"
@ -42,7 +35,6 @@ raw_search(MetricType metric_type,
float* D,
idx_t* labels,
const BitsetView bitset) {
#if defined(__linux__) || defined(__MINGW64__)
using namespace faiss; // NOLINT
if (metric_type == METRIC_Jaccard || metric_type == METRIC_Tanimoto) {
float_maxheap_array_t res = {size_t(n), size_t(k), labels, D};
@ -70,9 +62,6 @@ raw_search(MetricType metric_type,
std::string("binary search not support metric type: ") + segcore::MetricTypeToString(metric_type);
PanicInfo(msg);
}
#else
PanicInfo("Unsupported brute force for binary search on current OS environment!");
#endif
}
SubSearchResult
@ -111,17 +100,6 @@ FloatSearchBruteForce(const dataset::SearchDataset& dataset,
SubSearchResult sub_qr(num_queries, topk, metric_type, round_decimal);
auto query_data = reinterpret_cast<const float*>(dataset.query_data);
auto chunk_data = reinterpret_cast<const float*>(chunk_data_raw);
#ifdef __APPLE__
if (metric_type == MetricType::METRIC_L2) {
knowhere::float_maxheap_array_t buf{(size_t)num_queries, (size_t)topk, sub_qr.get_ids(),
sub_qr.get_distances()};
knowhere::knn_L2sqr_sse(query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, bitset);
} else {
knowhere::float_minheap_array_t buf{(size_t)num_queries, (size_t)topk, sub_qr.get_ids(),
sub_qr.get_distances()};
knowhere::knn_inner_product_sse(query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, bitset);
}
#elif defined(__linux__) || defined(__MINGW64__)
if (metric_type == MetricType::METRIC_L2) {
faiss::float_maxheap_array_t buf{(size_t)num_queries, (size_t)topk, sub_qr.get_ids(), sub_qr.get_distances()};
faiss::knn_L2sqr(query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, nullptr, bitset);
@ -129,9 +107,6 @@ FloatSearchBruteForce(const dataset::SearchDataset& dataset,
faiss::float_minheap_array_t buf{(size_t)num_queries, (size_t)topk, sub_qr.get_ids(), sub_qr.get_distances()};
faiss::knn_inner_product(query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, bitset);
}
#else
#error "Unsupported OS environment!";
#endif
sub_qr.round_values();
return sub_qr;
}

4
internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp
View File

@ -41,11 +41,15 @@ InferIndexType(const Json& search_params) {
namespace ip = knowhere::IndexParams;
namespace ie = knowhere::IndexEnum;
list.emplace(ip::nprobe, ie::INDEX_FAISS_IVFFLAT);
#ifdef MILVUS_SUPPORT_NSG
list.emplace(ip::search_length, ie::INDEX_NSG);
#endif
list.emplace(ip::ef, ie::INDEX_HNSW);
list.emplace(ip::search_k, ie::INDEX_ANNOY);
#ifdef MILVUS_SUPPORT_NGT
list.emplace(ip::max_search_edges, ie::INDEX_NGTONNG);
list.emplace(ip::epsilon, ie::INDEX_NGTONNG);
#endif
return list;
}();
auto dbg_str = search_params.dump();

6
internal/core/src/segcore/FieldIndexing.cpp
View File

@ -14,9 +14,7 @@
#include <index/ScalarIndexSort.h>
#include "common/SystemProperty.h"
#if defined(__linux__) || defined(__MINGW64__)
#include "knowhere/index/vector_index/IndexIVF.h"
#endif
#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
#include "segcore/FieldIndexing.h"
@ -24,7 +22,6 @@ namespace milvus::segcore {
void
VectorFieldIndexing::BuildIndexRange(int64_t ack_beg, int64_t ack_end, const VectorBase* vec_base) {
#if defined(__linux__) || defined(__MINGW64__)
AssertInfo(field_meta_.get_data_type() == DataType::VECTOR_FLOAT, "Data type of vector field is not VECTOR_FLOAT");
auto dim = field_meta_.get_dim();
@ -43,9 +40,6 @@ VectorFieldIndexing::BuildIndexRange(int64_t ack_beg, int64_t ack_end, const Vec
indexing->AddWithoutIds(dataset, conf);
data_[chunk_id] = std::move(indexing);
}
#else
throw std::runtime_error("Unsupported BuildIndexRange on current platform!");
#endif
}
knowhere::Config

4
internal/core/thirdparty/knowhere/CMakeLists.txt vendored
View File

@ -11,8 +11,8 @@
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
set( KNOWHERE_VERSION v1.1.1 )
set( KNOWHERE_SOURCE_MD5 "3f6dff91dc947a9b003434b8901e9130" )
set( KNOWHERE_VERSION v1.1.2 )
set( KNOWHERE_SOURCE_MD5 "7c6043bfaa073c6580a1c5b1e904c995" )
if ( DEFINED ENV{MILVUS_KNOWHERE_URL} )
set( KNOWHERE_SOURCE_URL "$ENV{MILVUS_KNOWHERE_URL}" )

76
internal/core/unittest/CMakeLists.txt
View File

@ -14,37 +14,45 @@ include_directories(${CMAKE_HOME_DIRECTORY}/src/thirdparty)
add_definitions(-DMILVUS_TEST_SEGCORE_YAML_PATH="${CMAKE_SOURCE_DIR}/unittest/test_utils/test_segcore.yaml")
# TODO: better to use ls/find pattern
set(MILVUS_TEST_FILES
init_gtest.cpp
test_binary.cpp
test_bitmap.cpp
test_bool_index.cpp
test_common.cpp
test_concurrent_vector.cpp
test_conf_adapter_mgr.cpp
test_c_api.cpp
test_expr.cpp
test_indexing.cpp
test_index_c_api.cpp
test_index_wrapper.cpp
test_init.cpp
test_plan_proto.cpp
test_query.cpp
test_reduce.cpp
test_reduce_c.cpp
test_retrieve.cpp
test_scalar_index.cpp
test_sealed.cpp
test_segcore.cpp
test_similarity_corelation.cpp
test_span.cpp
test_timestamp_index.cpp
test_utils.cpp
)
# These 2 cases cannot run in mac because of tbb build error
if (LINUX)
# TODO: better to use ls/find pattern
set(MILVUS_TEST_FILES
init_gtest.cpp
test_binary.cpp
test_bitmap.cpp
test_common.cpp
test_concurrent_vector.cpp
test_c_api.cpp
test_expr.cpp
test_retrieve.cpp
test_indexing.cpp
test_index_wrapper.cpp
test_init.cpp
test_plan_proto.cpp
test_query.cpp
test_reduce.cpp
test_sealed.cpp
test_segcore.cpp
test_span.cpp
test_timestamp_index.cpp
test_reduce_c.cpp
test_conf_adapter_mgr.cpp
test_similarity_corelation.cpp
test_utils.cpp
${MILVUS_TEST_FILES}
test_scalar_index_creator.cpp
test_index_c_api.cpp
test_scalar_index.cpp
test_string_index.cpp
test_bool_index.cpp
)
endif()
if (LINUX)
# check if memory leak exists in index builder
set(INDEX_BUILDER_TEST_FILES
test_index_wrapper.cpp
@ -68,22 +76,6 @@ if (LINUX)
install(TARGETS index_builder_test DESTINATION unittest)
endif()
if ( APPLE )
set(MILVUS_TEST_FILES
init_gtest.cpp
test_common.cpp
test_concurrent_vector.cpp
test_conf_adapter_mgr.cpp
test_plan_proto.cpp
test_reduce.cpp
test_reduce_c.cpp
test_segcore.cpp
test_timestamp_index.cpp
test_similarity_corelation.cpp
test_utils.cpp
)
endif ()
add_executable(all_tests
${MILVUS_TEST_FILES}
)

3
internal/core/unittest/test_index_c_api.cpp
View File

@ -236,6 +236,8 @@ TEST(CInt64IndexTest, All) {
}
}
// disable this case since marisa not supported in mac
#ifdef __linux__
TEST(CStringIndexTest, All) {
auto strs = GenStrArr(NB);
schemapb::StringArray str_arr;
@ -288,3 +290,4 @@ TEST(CStringIndexTest, All) {
delete[](char*) str_ds->Get<const void*>(knowhere::meta::TENSOR);
}
#endif

99
internal/core/unittest/test_indexing.cpp
View File

@ -298,82 +298,41 @@ TEST(Indexing, BinaryBruteForce) {
auto json = SearchResultToJson(sr);
std::cout << json.dump(2);
#ifdef __linux__
auto ref = json::parse(R"(
[
[
[
"1024->0.000000",
"48942->0.642000",
"18494->0.644000",
"68225->0.644000",
"93557->0.644000"
],
[
"1025->0.000000",
"73557->0.641000",
"53086->0.643000",
"9737->0.643000",
"62855->0.644000"
],
[
"1026->0.000000",
"62904->0.644000",
"46758->0.644000",
"57969->0.645000",
"98113->0.646000"
],
[
"1027->0.000000",
"92446->0.638000",
"96034->0.640000",
"92129->0.644000",
"45887->0.644000"
],
[
"1028->0.000000",
"22992->0.643000",
"73903->0.644000",
"19969->0.645000",
"65178->0.645000"
],
[
"1029->0.000000",
"19776->0.641000",
"15166->0.642000",
"85470->0.642000",
"16730->0.643000"
],
[
"1030->0.000000",
"55939->0.640000",
"84253->0.643000",
"31958->0.644000",
"11667->0.646000"
],
[
"1031->0.000000",
"89536->0.637000",
"61622->0.638000",
"9275->0.639000",
"91403->0.640000"
],
[
"1032->0.000000",
"69504->0.642000",
"23414->0.644000",
"48770->0.645000",
"23231->0.645000"
],
[
"1033->0.000000",
"33540->0.636000",
"25310->0.640000",
"18576->0.640000",
"73729->0.642000"
]
[ "1024->0.000000", "48942->0.642000", "18494->0.644000", "68225->0.644000", "93557->0.644000" ],
[ "1025->0.000000", "73557->0.641000", "53086->0.643000", "9737->0.643000", "62855->0.644000" ],
[ "1026->0.000000", "62904->0.644000", "46758->0.644000", "57969->0.645000", "98113->0.646000" ],
[ "1027->0.000000", "92446->0.638000", "96034->0.640000", "92129->0.644000", "45887->0.644000" ],
[ "1028->0.000000", "22992->0.643000", "73903->0.644000", "19969->0.645000", "65178->0.645000" ],
[ "1029->0.000000", "19776->0.641000", "15166->0.642000", "85470->0.642000", "16730->0.643000" ],
[ "1030->0.000000", "55939->0.640000", "84253->0.643000", "31958->0.644000", "11667->0.646000" ],
[ "1031->0.000000", "89536->0.637000", "61622->0.638000", "9275->0.639000", "91403->0.640000" ],
[ "1032->0.000000", "69504->0.642000", "23414->0.644000", "48770->0.645000", "23231->0.645000" ],
[ "1033->0.000000", "33540->0.636000", "25310->0.640000", "18576->0.640000", "73729->0.642000" ]
]
]
)");
#else // for mac
auto ref = json::parse(R"(
[
[
[ "1024->0.000000", "59169->0.645000", "98548->0.646000", "3356->0.646000", "90373->0.647000" ],
[ "1025->0.000000", "61245->0.638000", "95271->0.639000", "31087->0.639000", "31549->0.640000" ],
[ "1026->0.000000", "65225->0.648000", "35750->0.648000", "14971->0.649000", "75385->0.649000" ],
[ "1027->0.000000", "70158->0.640000", "27076->0.640000", "3407->0.641000", "59527->0.641000" ],
[ "1028->0.000000", "45757->0.645000", "3356->0.645000", "77230->0.646000", "28690->0.647000" ],
[ "1029->0.000000", "13291->0.642000", "24960->0.643000", "83770->0.643000", "88244->0.643000" ],
[ "1030->0.000000", "96807->0.641000", "39920->0.643000", "62943->0.644000", "12603->0.644000" ],
[ "1031->0.000000", "65769->0.648000", "60493->0.648000", "48738->0.648000", "4353->0.648000" ],
[ "1032->0.000000", "57827->0.637000", "8213->0.638000", "22221->0.639000", "23328->0.640000" ],
[ "1033->0.000000", "676->0.645000", "91430->0.646000", "85353->0.646000", "6014->0.646000" ]
]
]
)");
#endif
auto json_str = json.dump(2);
auto ref_str = ref.dump(2);
ASSERT_EQ(json_str, ref_str);

39
internal/core/unittest/test_query.cpp
View File

@ -186,6 +186,7 @@ TEST(Query, ExecWithPredicateLoader) {
int topk = 5;
Json json = SearchResultToJson(*sr);
#ifdef __linux__
auto ref = json::parse(R"(
[
[
@ -196,6 +197,18 @@ TEST(Query, ExecWithPredicateLoader) {
["66353->5.696000", "41087->5.917000", "97780->6.811000", "99239->7.562000", "86527->7.751000"]
]
])");
#else // for mac
auto ref = json::parse(R"(
[
[
["982->0.000000", "31864->4.270000", "18916->4.651000", "71547->5.125000", "13227->6.010000"],
["96984->4.192000", "65514->6.011000", "89328->6.138000", "80284->6.526000", "68218->6.563000"],
["30119->2.464000", "52595->4.323000", "32673->4.851000", "74834->5.009000", "50806->5.446000"],
["99625->6.129000", "86582->6.900000", "10069->7.388000", "89982->7.672000", "85934->7.792000"],
["37759->3.581000", "97019->5.557000", "53543->5.844000", "63535->6.439000", "94009->6.572000"]
]
])");
#endif
std::cout << json.dump(2);
ASSERT_EQ(json.dump(2), ref.dump(2));
}
@ -301,6 +314,7 @@ TEST(Query, ExecWithPredicate) {
int topk = 5;
Json json = SearchResultToJson(*sr);
#ifdef __linux__
auto ref = json::parse(R"(
[
[
@ -311,6 +325,18 @@ TEST(Query, ExecWithPredicate) {
["66353->5.696000", "41087->5.917000", "97780->6.811000", "99239->7.562000", "86527->7.751000"]
]
])");
#else // for mac
auto ref = json::parse(R"(
[
[
["982->0.000000", "31864->4.270000", "18916->4.651000", "71547->5.125000", "13227->6.010000"],
["96984->4.192000", "65514->6.011000", "89328->6.138000", "80284->6.526000", "68218->6.563000"],
["30119->2.464000", "52595->4.323000", "32673->4.851000", "74834->5.009000", "50806->5.446000"],
["99625->6.129000", "86582->6.900000", "10069->7.388000", "89982->7.672000", "85934->7.792000"],
["37759->3.581000", "97019->5.557000", "53543->5.844000", "63535->6.439000", "94009->6.572000"]
]
])");
#endif
std::cout << json.dump(2);
ASSERT_EQ(json.dump(2), ref.dump(2));
}
@ -498,6 +524,7 @@ TEST(Query, ExecWithoutPredicate) {
std::vector<std::vector<std::string>> results;
int topk = 5;
auto json = SearchResultToJson(*sr);
#ifdef __linux__
auto ref = json::parse(R"(
[
[
@ -508,6 +535,18 @@ TEST(Query, ExecWithoutPredicate) {
["66353->5.696000", "41087->5.917000", "24554->6.195000", "68019->6.654000", "97780->6.811000"]
]
])");
#else // for mac
auto ref = json::parse(R"(
[
[
["982->0.000000", "31864->4.270000", "18916->4.651000", "78227->4.808000", "71547->5.125000"],
["96984->4.192000", "45733->4.912000", "32891->5.016000", "65514->6.011000", "89328->6.138000"],
["30119->2.464000", "52595->4.323000", "32673->4.851000", "74834->5.009000", "76784->5.195000"],
["99625->6.129000", "86582->6.900000", "60608->7.285000", "10069->7.388000", "89982->7.672000"],
["37759->3.581000", "45814->4.872000", "97019->5.557000", "23626->5.839000", "53543->5.844000"]
]
])");
#endif
std::cout << json.dump(2);
ASSERT_EQ(json.dump(2), ref.dump(2));
}

13
internal/core/unittest/test_sealed.cpp
View File

@ -298,6 +298,7 @@ TEST(Sealed, LoadFieldData) {
ASSERT_EQ(json.dump(-2), json2.dump(-2));
segment->DropFieldData(double_id);
ASSERT_ANY_THROW(segment->Search(plan.get(), *ph_group, time));
#ifdef __linux__
auto std_json = Json::parse(R"(
[
[
@ -308,6 +309,18 @@ TEST(Sealed, LoadFieldData) {
["66353->5.696000", "30664->5.881000", "41087->5.917000", "10393->6.633000", "90215->7.202000"]
]
])");
#else // for mac
auto std_json = Json::parse(R"(
[
[
["982->0.000000", "31864->4.270000", "18916->4.651000", "71547->5.125000", "86706->5.991000"],
["96984->4.192000", "65514->6.011000", "89328->6.138000", "80284->6.526000", "68218->6.563000"],
["30119->2.464000", "82365->4.725000", "74834->5.009000", "79995->5.725000", "33359->5.816000"],
["99625->6.129000", "86582->6.900000", "85934->7.792000", "60450->8.087000", "19257->8.530000"],
["37759->3.581000", "31292->5.780000", "98124->6.216000", "63535->6.439000", "11707->6.553000"]
]
])");
#endif
ASSERT_EQ(std_json.dump(-2), json.dump(-2));
}

30
internal/core/unittest/test_utils/indexbuilder_test_utils.h
View File

@ -56,7 +56,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
};
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFPQ) {
return knowhere::Config{
@ -67,7 +67,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
};
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) {
return knowhere::Config{
@ -76,7 +76,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{knowhere::IndexParams::nlist, 16},
{knowhere::IndexParams::nprobe, 4},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
#ifdef MILVUS_GPU_VERSION
{knowhere::meta::DEVICEID, DEVICEID},
#endif
@ -89,7 +89,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{knowhere::IndexParams::nprobe, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
#ifdef MILVUS_GPU_VERSION
{knowhere::meta::DEVICEID, DEVICEID},
#endif
@ -103,7 +103,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
};
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP) {
return knowhere::Config{
@ -111,6 +111,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{knowhere::meta::TOPK, K},
{knowhere::Metric::TYPE, metric_type},
};
#ifdef MILVUS_SUPPORT_NSG
} else if (index_type == knowhere::IndexEnum::INDEX_NSG) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
@ -123,20 +124,21 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{knowhere::IndexParams::candidate, 100},
{knowhere::Metric::TYPE, metric_type},
};
#endif
#ifdef MILVUS_SUPPORT_SPTAG
} else if (index_type == knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
// {knowhere::meta::TOPK, 10},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
};
} else if (index_type == knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
// {knowhere::meta::TOPK, 10},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
};
#endif
} else if (index_type == knowhere::IndexEnum::INDEX_HNSW) {
@ -152,7 +154,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{knowhere::IndexParams::n_trees, 4},
{knowhere::IndexParams::search_k, 100},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
};
} else if (index_type == knowhere::IndexEnum::INDEX_RHNSWFlat) {
return knowhere::Config{
@ -162,7 +164,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{knowhere::IndexParams::efConstruction, 200},
{knowhere::IndexParams::ef, 200},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
};
} else if (index_type == knowhere::IndexEnum::INDEX_RHNSWPQ) {
return knowhere::Config{
@ -172,7 +174,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{knowhere::IndexParams::efConstruction, 200},
{knowhere::IndexParams::ef, 200},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
{knowhere::IndexParams::PQM, 8},
};
} else if (index_type == knowhere::IndexEnum::INDEX_RHNSWSQ) {
@ -183,8 +185,9 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{knowhere::IndexParams::efConstruction, 200},
{knowhere::IndexParams::ef, 200},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
};
#ifdef MILVUS_SUPPORT_NGT
} else if (index_type == knowhere::IndexEnum::INDEX_NGTPANNG) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
@ -195,7 +198,7 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{knowhere::IndexParams::max_search_edges, 50},
{knowhere::IndexParams::forcedly_pruned_edge_size, 60},
{knowhere::IndexParams::selectively_pruned_edge_size, 30},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
};
} else if (index_type == knowhere::IndexEnum::INDEX_NGTONNG) {
return knowhere::Config{
@ -207,8 +210,9 @@ generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType&
{knowhere::IndexParams::max_search_edges, 50},
{knowhere::IndexParams::outgoing_edge_size, 5},
{knowhere::IndexParams::incoming_edge_size, 40},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, knowhere::index_file_slice_size},
};
#endif
}
return knowhere::Config();
}