milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove ANNOY index type () 

Signed-off-by: Yudong Cai <yudong.cai@zilliz.com>
pull/23220/head
Cai Yudong 2023-04-04 16:30:27 +08:00 committed by GitHub
parent 940ead200a
commit ef63e64ded
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 13 additions and 108 deletions

7
internal/core/src/query/deprecated/ValidationUtil.cpp
View File

@ -201,7 +201,6 @@ ValidateVectorIndexType(std::string& index_type, bool is_binary) {
knowhere::IndexEnum::INDEX_FAISS_IVFPQ,
knowhere::IndexEnum::INDEX_FAISS_IVFSQ8,
knowhere::IndexEnum::INDEX_HNSW,
knowhere::IndexEnum::INDEX_ANNOY,
};
static std::set<std::string> s_binary_index_types = {
@ -335,12 +334,6 @@ ValidateIndexParams(const milvus::json& index_params,
if (!status.ok()) {
return status;
}
} else if (index_type == knowhere::IndexEnum::INDEX_ANNOY) {
auto status = CheckParameterRange(
index_params, knowhere::IndexParams::n_trees, 1, 1024);
if (!status.ok()) {
return status;
}
}
return Status::OK();

3
internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp
View File

@ -30,15 +30,12 @@ static IndexType
InferIndexType(const Json& search_params) {
// ivf -> nprobe
// hnsw -> ef
// annoy -> search_k
// ngtpanng / ngtonng -> max_search_edges / epsilon
static const std::map<std::string, IndexType> key_list = [] {
std::map<std::string, IndexType> list;
namespace ip = knowhere::indexparam;
namespace ie = knowhere::IndexEnum;
list.emplace(ip::NPROBE, ie::INDEX_FAISS_IVFFLAT);
list.emplace(ip::EF, ie::INDEX_HNSW);
list.emplace(ip::SEARCH_K, ie::INDEX_ANNOY);
return list;
}();
auto dbg_str = search_params.dump();

4
internal/core/unittest/test_index_wrapper.cpp
View File

@ -54,7 +54,6 @@ class IndexWrapperTest : public ::testing::TestWithParam<Param> {
{knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, true},
{knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, true},
{knowhere::IndexEnum::INDEX_HNSW, false},
{knowhere::IndexEnum::INDEX_ANNOY, false},
};
is_binary = is_binary_map[index_type];
@ -115,8 +114,7 @@ INSTANTIATE_TEST_CASE_P(
knowhere::metric::TANIMOTO),
std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP,
knowhere::metric::JACCARD),
std::pair(knowhere::IndexEnum::INDEX_HNSW, knowhere::metric::L2),
std::pair(knowhere::IndexEnum::INDEX_ANNOY, knowhere::metric::L2)));
std::pair(knowhere::IndexEnum::INDEX_HNSW, knowhere::metric::L2)));
TEST_P(IndexWrapperTest, BuildAndQuery) {
auto index = milvus::indexbuilder::IndexFactory::GetInstance().CreateIndex(

13
internal/core/unittest/test_indexing.cpp
View File

@ -312,7 +312,6 @@ class IndexTest : public ::testing::TestWithParam<Param> {
{knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, true},
{knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, true},
{knowhere::IndexEnum::INDEX_HNSW, false},
{knowhere::IndexEnum::INDEX_ANNOY, false},
{knowhere::IndexEnum::INDEX_DISKANN, false},
};
@ -374,12 +373,12 @@ INSTANTIATE_TEST_CASE_P(
knowhere::metric::TANIMOTO),
std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP,
knowhere::metric::JACCARD),
std::pair(knowhere::IndexEnum::INDEX_HNSW, knowhere::metric::L2),
std::pair(knowhere::IndexEnum::INDEX_HNSW, knowhere::metric::L2)
// ci ut not start minio, so not run ut about diskann index for now
// #ifdef BUILD_DISK_ANN
// std::pair(knowhere::IndexEnum::INDEX_DISKANN, knowhere::metric::L2),
// std::pair(knowhere::IndexEnum::INDEX_DISKANN, knowhere::metric::L2),
// #endif
std::pair(knowhere::IndexEnum::INDEX_ANNOY, knowhere::metric::L2)));
));
TEST_P(IndexTest, BuildAndQuery) {
milvus::index::CreateIndexInfo create_index_info;
@ -446,10 +445,8 @@ TEST_P(IndexTest, BuildAndQuery) {
if (!is_binary) {
EXPECT_EQ(result->seg_offsets_[0], query_offset);
}
if (index_type != knowhere::IndexEnum::INDEX_ANNOY) {
search_info.search_params_ = range_search_conf;
vec_index->Query(xq_dataset, search_info, nullptr);
}
search_info.search_params_ = range_search_conf;
vec_index->Query(xq_dataset, search_info, nullptr);
}
// #ifdef BUILD_DISK_ANN

10
internal/core/unittest/test_utils/indexbuilder_test_utils.h
View File

@ -241,12 +241,6 @@ generate_build_conf(const milvus::IndexType& index_type,
{knowhere::indexparam::HNSW_M, "16"},
{knowhere::indexparam::EFCONSTRUCTION, "200"},
};
} else if (index_type == knowhere::IndexEnum::INDEX_ANNOY) {
return knowhere::Json{
{knowhere::meta::METRIC_TYPE, metric_type},
{knowhere::meta::DIM, std::to_string(DIM)},
{knowhere::indexparam::N_TREES, "4"},
};
} else if (index_type == knowhere::IndexEnum::INDEX_DISKANN) {
return knowhere::Json{
{knowhere::meta::METRIC_TYPE, metric_type},
@ -298,8 +292,6 @@ generate_search_conf(const milvus::IndexType& index_type,
conf[knowhere::indexparam::NPROBE] = 4;
} else if (index_type == knowhere::IndexEnum::INDEX_HNSW) {
conf[knowhere::indexparam::EF] = 200;
} else if (index_type == knowhere::IndexEnum::INDEX_ANNOY) {
conf[knowhere::indexparam::SEARCH_K] = 100;
} else if (index_type == knowhere::IndexEnum::INDEX_DISKANN) {
conf[milvus::index::DISK_ANN_QUERY_LIST] = K * 2;
}
@ -326,8 +318,6 @@ generate_range_search_conf(const milvus::IndexType& index_type,
conf[knowhere::indexparam::NPROBE] = 4;
} else if (index_type == knowhere::IndexEnum::INDEX_HNSW) {
conf[knowhere::indexparam::EF] = 200;
} else if (index_type == knowhere::IndexEnum::INDEX_ANNOY) {
conf[knowhere::indexparam::SEARCH_K] = 100;
} else if (index_type == knowhere::IndexEnum::INDEX_DISKANN) {
conf[milvus::index::DISK_ANN_QUERY_LIST] = K * 2;
}

9
internal/querynode/mock_test.go
View File

@ -124,8 +124,7 @@ const (
IndexFaissBinIDMap = "BIN_FLAT"
IndexFaissBinIVFFlat = "BIN_IVF_FLAT"
IndexHNSW = "HNSW"
IndexANNOY = "ANNOY"
IndexHNSW = "HNSW"
// metric type
L2 = "L2"
@ -140,9 +139,6 @@ const (
nprobe = 8
efConstruction = 200
ef = 200
edgeSize = 10
epsilon = 0.1
maxSearchEdges = 50
)
// ---------- unittest util functions ----------
@ -463,9 +459,6 @@ func genIndexParams(indexType, metricType string) (map[string]string, map[string
indexParams["M"] = strconv.Itoa(16)
indexParams["efConstruction"] = strconv.Itoa(efConstruction)
//indexParams["ef"] = strconv.Itoa(ef)
} else if indexType == IndexANNOY {
indexParams["n_trees"] = strconv.Itoa(4)
indexParams["search_k"] = strconv.Itoa(100)
} else if indexType == IndexFaissBinIVFFlat { // binary vector
indexParams["nlist"] = strconv.Itoa(nlist)
indexParams["m"] = strconv.Itoa(m)

19
internal/util/indexparamcheck/conf_adapter.go
View File

@ -359,24 +359,7 @@ func newHNSWConfAdapter() *HNSWConfAdapter {
return &HNSWConfAdapter{}
}
// ANNOYConfAdapter checks if an ANNOY index can be built.
type ANNOYConfAdapter struct {
BaseConfAdapter
}
// CheckTrain checks if an annoy index can be built with specific parameters.
func (adapter *ANNOYConfAdapter) CheckTrain(params map[string]string) bool {
if !CheckIntByRange(params, NTREES, MinNTrees, MaxNTrees) {
return false
}
return adapter.BaseConfAdapter.CheckTrain(params)
}
func newANNOYConfAdapter() *ANNOYConfAdapter {
return &ANNOYConfAdapter{}
}
// DISKANNConfAdapter checks if an diskann index can be built.
type DISKANNConfAdapter struct {
BaseConfAdapter
}

1
internal/util/indexparamcheck/conf_adapter_mgr.go
View File

@ -55,7 +55,6 @@ func (mgr *ConfAdapterMgrImpl) registerConfAdapter() {
mgr.adapters[IndexFaissBinIDMap] = newBinIDMAPConfAdapter()
mgr.adapters[IndexFaissBinIvfFlat] = newBinIVFConfAdapter()
mgr.adapters[IndexHNSW] = newHNSWConfAdapter()
mgr.adapters[IndexANNOY] = newANNOYConfAdapter()
mgr.adapters[IndexDISKANN] = newDISKANNConfAdapter()
}

12
internal/util/indexparamcheck/conf_adapter_mgr_test.go
View File

@ -70,12 +70,6 @@ func Test_GetConfAdapterMgrInstance(t *testing.T) {
assert.NotEqual(t, nil, adapter)
_, ok = adapter.(*HNSWConfAdapter)
assert.Equal(t, true, ok)
adapter, err = adapterMgr.GetAdapter(IndexANNOY)
assert.Equal(t, nil, err)
assert.NotEqual(t, nil, adapter)
_, ok = adapter.(*ANNOYConfAdapter)
assert.Equal(t, true, ok)
}
func TestConfAdapterMgrImpl_GetAdapter(t *testing.T) {
@ -130,12 +124,6 @@ func TestConfAdapterMgrImpl_GetAdapter(t *testing.T) {
assert.NotEqual(t, nil, adapter)
_, ok = adapter.(*HNSWConfAdapter)
assert.Equal(t, true, ok)
adapter, err = adapterMgr.GetAdapter(IndexANNOY)
assert.Equal(t, nil, err)
assert.NotEqual(t, nil, adapter)
_, ok = adapter.(*ANNOYConfAdapter)
assert.Equal(t, true, ok)
}
func TestConfAdapterMgrImpl_GetAdapter_multiple_threads(t *testing.T) {

32
internal/util/indexparamcheck/conf_adapter_test.go
View File

@ -375,37 +375,7 @@ func TestHNSWConfAdapter_CheckTrain(t *testing.T) {
}
}
// ANNOYConfAdapter checks if an annoy index can be built
func TestANNOYConfAdapter_CheckTrain(t *testing.T) {
validParams := map[string]string{
DIM: strconv.Itoa(128),
NTREES: strconv.Itoa(4),
Metric: L2,
}
invalidTreeParamsMin := copyParams(validParams)
invalidTreeParamsMin[NTREES] = strconv.Itoa(MinNTrees - 1)
invalidTreeParamsMax := copyParams(validParams)
invalidTreeParamsMax[NTREES] = strconv.Itoa(MaxNTrees + 1)
cases := []struct {
params map[string]string
want bool
}{
{validParams, true},
{invalidTreeParamsMin, false},
{invalidTreeParamsMax, false},
}
adapter := newANNOYConfAdapter()
for _, test := range cases {
if got := adapter.CheckTrain(test.params); got != test.want {
t.Errorf("ANNOYConfAdapter.CheckTrain(%v) = %v", test.params, test.want)
}
}
}
// DISKANNConfAdapter checks if an diskann index can be built
func TestDiskAnnConfAdapter_CheckTrain(t *testing.T) {
validParams := map[string]string{
DIM: strconv.Itoa(128),

1
internal/util/indexparamcheck/index_type.go
View File

@ -25,6 +25,5 @@ const (
IndexFaissBinIDMap IndexType = "BIN_FLAT"
IndexFaissBinIvfFlat IndexType = "BIN_IVF_FLAT"
IndexHNSW IndexType = "HNSW"
IndexANNOY IndexType = "ANNOY"
IndexDISKANN IndexType = "DISKANN"
)

4
tests/python_client/deploy/common.py
View File

@ -1,10 +1,10 @@
import json
from utils.util_log import test_log as log
all_index_types = ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ", "HNSW", "ANNOY", "BIN_FLAT", "BIN_IVF_FLAT"]
all_index_types = ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ", "HNSW", "BIN_FLAT", "BIN_IVF_FLAT"]
default_index_params = [{"nlist": 128}, {"nlist": 128}, {"nlist": 128}, {"nlist": 128, "m": 16, "nbits": 8},
{"M": 48, "efConstruction": 500}, {"n_trees": 50}, {"nlist": 128}, {"nlist": 128}]
{"M": 48, "efConstruction": 500}, {"nlist": 128}, {"nlist": 128}]
index_params_map = dict(zip(all_index_types, default_index_params))

4
tests/python_client/deploy/scripts/utils.py
View File

@ -16,10 +16,10 @@ logger.add(sys.stderr, format= "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> |
pymilvus_version = pymilvus.__version__
all_index_types = ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ", "HNSW", "ANNOY"]
all_index_types = ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_PQ", "HNSW"]
default_index_params = [{}, {"nlist": 128}, {"nlist": 128}, {"nlist": 128, "m": 16, "nbits": 8},
{"M": 48, "efConstruction": 500}, {"n_trees": 50}]
{"M": 48, "efConstruction": 500}]
index_params_map = dict(zip(all_index_types, default_index_params))

2
tests/python_client/utils/util_pymilvus.py
View File

@ -39,7 +39,6 @@ all_index_types = [
"IVF_SQ8",
"IVF_PQ",
"HNSW",
"ANNOY",
"BIN_FLAT",
"BIN_IVF_FLAT"
]
@ -50,7 +49,6 @@ default_index_params = [
{"nlist": 128},
{"nlist": 128, "m": 16, "nbits": 8},
{"M": 48, "efConstruction": 500},
{"n_trees": 50},
{"nlist": 128},
{"nlist": 128}
]