milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add delete case for hnsw/pq in cpu-version (#1741) 

* add delete case for hnsw/pq in cpu-version

Signed-off-by: lizhenxiang <lizhenxiang@lianjia.com>

* Update compact timeout

Signed-off-by: zw <zw@milvus.io>

* add case: search with tag not existed

Signed-off-by: zw <zw@milvus.io>

* update compact timeout

Signed-off-by: zw <zw@milvus.io>

Co-authored-by: lizhenxiang <lizhenxiang@lianjia.com>
Co-authored-by: zw <zw@milvus.io>
pull/1773/head
del-zhenwu 2020-03-26 23:17:04 +08:00 committed by GitHub
parent 207b854c3f
commit 23e2780309
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 21 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
tests/milvus_python_test/test_compact.py
View File

@ -9,7 +9,7 @@ from utils import *
dim = 128
index_file_size = 10
COMPACT_TIMEOUT = 30
COMPACT_TIMEOUT = 180
nprobe = 1
top_k = 1
tag = "1970-01-01"
@ -81,8 +81,6 @@ class TestCompactBase:
size_before = info.partitions_stat[0].segments_stat[0].data_size
status = connect.compact(collection)
assert status.OK()
status = connect.flush([collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(collection)
assert status.OK()
@ -107,8 +105,6 @@ class TestCompactBase:
size_before = info.partitions_stat[0].segments_stat[0].data_size
status = connect.compact(collection)
assert status.OK()
status = connect.flush([collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(collection)
assert status.OK()
@ -140,8 +136,6 @@ class TestCompactBase:
logging.getLogger().info(size_before)
status = connect.compact(collection)
assert status.OK()
status = connect.flush([collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(collection)
assert status.OK()
@ -171,8 +165,6 @@ class TestCompactBase:
assert status.OK()
status = connect.compact(collection)
assert status.OK()
status = connect.flush([collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(collection)
assert status.OK()
@ -185,7 +177,7 @@ class TestCompactBase:
)
def get_simple_index(self, request, connect):
if str(connect._cmd("mode")[1]) == "CPU":
if request.param["index_type"] not in [IndexType.IVF_SQ8, IndexType.IVFLAT, IndexType.FLAT]:
if request.param["index_type"] not in [IndexType.IVF_SQ8, IndexType.IVFLAT, IndexType.FLAT, IndexType.IVF_PQ, IndexType.HNSW]:
pytest.skip("Only support index_type: flat/ivf_flat/ivf_sq8")
else:
pytest.skip("Only support CPU mode")
@ -221,8 +213,6 @@ class TestCompactBase:
assert status.OK()
status = connect.compact(collection)
assert status.OK()
status = connect.flush([collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(collection)
assert status.OK()
@ -257,8 +247,6 @@ class TestCompactBase:
assert(size_before == size_after)
status = connect.compact(collection)
assert status.OK()
status = connect.flush([collection])
assert status.OK()
# get collection info after compact twice
status, info = connect.collection_info(collection)
assert status.OK()
@ -288,8 +276,6 @@ class TestCompactBase:
size_before = info.partitions_stat[0].segments_stat[0].data_size
status = connect.compact(collection)
assert status.OK()
status = connect.flush([collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(collection)
assert status.OK()
@ -297,8 +283,6 @@ class TestCompactBase:
assert(size_before > size_after)
status = connect.compact(collection)
assert status.OK()
status = connect.flush([collection])
assert status.OK()
# get collection info after compact twice
status, info = connect.collection_info(collection)
assert status.OK()
@ -349,8 +333,6 @@ class TestCompactBase:
size_before = info.partitions_stat[0].segments_stat[0].data_size
status = connect.compact(collection)
assert status.OK()
status = connect.flush([collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(collection)
assert status.OK()
@ -372,10 +354,12 @@ class TestCompactBase:
assert status.OK()
status = connect.flush([collection])
assert status.OK()
status = connect.compact(collection)
status = connect.delete_by_id(collection, ids[:10])
assert status.OK()
status = connect.flush([collection])
assert status.OK()
status = connect.compact(collection)
assert status.OK()
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
status = connect.create_index(collection, index_type, index_param)
@ -419,14 +403,13 @@ class TestCompactBase:
assert status.OK()
status = connect.compact(collection)
assert status.OK()
status = connect.flush([collection])
assert status.OK()
query_vecs = [vectors[0]]
status, res = connect.search_vectors(collection, top_k, query_records=query_vecs)
logging.getLogger().info(res)
assert status.OK()
def test_compact_server_crashed_recovery(self, connect, collection):
# TODO: enable
def _test_compact_server_crashed_recovery(self, connect, collection):
'''
target: test compact when server crashed unexpectedly and restarted
method: add vectors, delete and compact collection; server stopped and restarted during compact
@ -447,8 +430,6 @@ class TestCompactBase:
status = connect.compact(collection)
# pdb.set_trace()
assert status.OK()
status = connect.flush([collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(collection)
assert status.OK()
@ -479,8 +460,6 @@ class TestCompactJAC:
size_before = info.partitions_stat[0].segments_stat[0].data_size
status = connect.compact(jac_collection)
assert status.OK()
status = connect.flush([jac_collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(jac_collection)
assert status.OK()
@ -505,8 +484,6 @@ class TestCompactJAC:
size_before = info.partitions_stat[0].segments_stat[0].data_size
status = connect.compact(jac_collection)
assert status.OK()
status = connect.flush([jac_collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(jac_collection)
assert status.OK()
@ -538,8 +515,6 @@ class TestCompactJAC:
logging.getLogger().info(size_before)
status = connect.compact(jac_collection)
assert status.OK()
status = connect.flush([jac_collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(jac_collection)
assert status.OK()
@ -569,8 +544,6 @@ class TestCompactJAC:
assert status.OK()
status = connect.compact(jac_collection)
assert status.OK()
status = connect.flush([jac_collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(jac_collection)
assert status.OK()
@ -595,8 +568,6 @@ class TestCompactJAC:
size_before = info.partitions_stat[0].segments_stat[0].data_size
status = connect.compact(jac_collection)
assert status.OK()
status = connect.flush([jac_collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(jac_collection)
assert status.OK()
@ -604,8 +575,6 @@ class TestCompactJAC:
assert(size_before == size_after)
status = connect.compact(jac_collection)
assert status.OK()
status = connect.flush([jac_collection])
assert status.OK()
# get collection info after compact twice
status, info = connect.collection_info(jac_collection)
assert status.OK()
@ -635,8 +604,6 @@ class TestCompactJAC:
size_before = info.partitions_stat[0].segments_stat[0].data_size
status = connect.compact(jac_collection)
assert status.OK()
status = connect.flush([jac_collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(jac_collection)
assert status.OK()
@ -644,8 +611,6 @@ class TestCompactJAC:
assert(size_before > size_after)
status = connect.compact(jac_collection)
assert status.OK()
status = connect.flush([jac_collection])
assert status.OK()
# get collection info after compact twice
status, info = connect.collection_info(jac_collection)
assert status.OK()
@ -700,8 +665,6 @@ class TestCompactJAC:
size_before = info.partitions_stat[0].segments_stat[0].data_size
status = connect.compact(jac_collection)
assert status.OK()
status = connect.flush([jac_collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(jac_collection)
assert status.OK()
@ -746,8 +709,6 @@ class TestCompactJAC:
assert status.OK()
status = connect.compact(jac_collection)
assert status.OK()
status = connect.flush([jac_collection])
assert status.OK()
query_vecs = [vectors[0]]
status, res = connect.search_vectors(jac_collection, top_k, query_records=query_vecs)
logging.getLogger().info(res)
@ -804,8 +765,6 @@ class TestCompactIP:
size_before = info.partitions_stat[0].segments_stat[0].data_size
status = connect.compact(ip_collection)
assert status.OK()
status = connect.flush([ip_collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(ip_collection)
assert status.OK()
@ -837,8 +796,6 @@ class TestCompactIP:
logging.getLogger().info(size_before)
status = connect.compact(ip_collection)
assert status.OK()
status = connect.flush([ip_collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(ip_collection)
assert status.OK()
@ -868,8 +825,6 @@ class TestCompactIP:
assert status.OK()
status = connect.compact(ip_collection)
assert status.OK()
status = connect.flush([ip_collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(ip_collection)
assert status.OK()
@ -894,8 +849,6 @@ class TestCompactIP:
size_before = info.partitions_stat[0].segments_stat[0].data_size
status = connect.compact(ip_collection)
assert status.OK()
status = connect.flush([ip_collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(ip_collection)
assert status.OK()
@ -903,8 +856,6 @@ class TestCompactIP:
assert(size_before == size_after)
status = connect.compact(ip_collection)
assert status.OK()
status = connect.flush([ip_collection])
assert status.OK()
# get collection info after compact twice
status, info = connect.collection_info(ip_collection)
assert status.OK()
@ -995,8 +946,6 @@ class TestCompactIP:
size_before = info.partitions_stat[0].segments_stat[0].data_size
status = connect.compact(ip_collection)
assert status.OK()
status = connect.flush([ip_collection])
assert status.OK()
# get collection info after compact
status, info = connect.collection_info(ip_collection)
assert status.OK()
@ -1020,8 +969,6 @@ class TestCompactIP:
assert status.OK()
status = connect.compact(ip_collection)
assert status.OK()
status = connect.flush([ip_collection])
assert status.OK()
status = connect.delete_by_id(ip_collection, ids)
assert status.OK()
status = connect.flush([ip_collection])
@ -1041,8 +988,6 @@ class TestCompactIP:
assert status.OK()
status = connect.compact(ip_collection)
assert status.OK()
status = connect.flush([ip_collection])
assert status.OK()
query_vecs = [vectors[0]]
status, res = connect.search_vectors(ip_collection, top_k, query_records=query_vecs)
logging.getLogger().info(res)

6
tests/milvus_python_test/test_delete_vectors.py
View File

@ -32,8 +32,8 @@ class TestDeleteBase:
)
def get_simple_index(self, request, connect):
if str(connect._cmd("mode")[1]) == "CPU":
if request.param["index_type"] not in [IndexType.IVF_SQ8, IndexType.IVFLAT, IndexType.FLAT]:
pytest.skip("Only support index_type: flat/ivf_flat/ivf_sq8")
if request.param["index_type"] not in [IndexType.IVF_SQ8, IndexType.IVFLAT, IndexType.FLAT, IndexType.IVF_PQ, IndexType.HNSW]:
pytest.skip("Only support index_type: flat/ivf_flat/ivf_sq8/hnsw/ivf_pq")
else:
pytest.skip("Only support CPU mode")
return request.param
@ -299,7 +299,7 @@ class TestDeleteIndexedVectors:
)
def get_simple_index(self, request, connect):
if str(connect._cmd("mode")[1]) == "CPU":
if request.param["index_type"] not in [IndexType.IVF_SQ8, IndexType.IVFLAT, IndexType.FLAT]:
if request.param["index_type"] not in [IndexType.IVF_SQ8, IndexType.IVFLAT, IndexType.FLAT, IndexType.IVF_PQ, IndexType.HNSW]:
pytest.skip("Only support index_type: flat/ivf_flat/ivf_sq8")
else:
pytest.skip("Only support CPU mode")

2
tests/milvus_python_test/test_flush.py
View File

@ -32,7 +32,7 @@ class TestFlushBase:
)
def get_simple_index(self, request, connect):
if str(connect._cmd("mode")[1]) == "CPU":
if request.param["index_type"] != IndexType.IVF_SQ8 or request.param["index_type"] != IndexType.IVFLAT or request.param["index_type"] != IndexType.FLAT:
if request.param["index_type"] not in [IndexType.IVF_SQ8, IndexType.IVFLAT, IndexType.FLAT, IndexType.IVF_PQ, IndexType.HNSW]:
pytest.skip("Only support index_type: flat/ivf_flat/ivf_sq8")
else:
pytest.skip("Only support CPU mode")

10
tests/milvus_python_test/test_search_vectors.py
View File

@ -1047,6 +1047,16 @@ class TestSearchParamsInvalid(object):
query_vecs = gen_vectors(1, dim)
with pytest.raises(Exception) as e:
status, result = connect.search_vectors(collection, top_k, query_vecs, partition_tags="tag")
logging.getLogger().debug(result)
@pytest.mark.level(1)
def test_search_with_tag_not_existed(self, connect, collection):
top_k = 1
nprobe = 1
query_vecs = gen_vectors(1, dim)
status, result = connect.search_vectors(collection, top_k, query_vecs, partition_tags=["tag"])
logging.getLogger().info(result)
assert not status.OK()
"""
Test search collection with invalid top-k