milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1618 from sahuang/master 

Fix python test cases
pull/1621/head
Jin Hai 2020-03-11 15:42:05 +08:00 committed by GitHub
parent 40c7e8baa2 a3b2842935
commit 6cd77b35df
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 5166 additions and 4263 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
tests/milvus_python_test/conftest.py
View File

@ -90,118 +90,118 @@ def milvus(request):
@pytest.fixture(scope="function")
def table(request, connect):
ori_table_name = getattr(request.module, "table_id", "test")
table_name = gen_unique_str(ori_table_name)
def collection(request, connect):
ori_collection_name = getattr(request.module, "collection_id", "test")
collection_name = gen_unique_str(ori_collection_name)
dim = getattr(request.module, "dim", "128")
param = {'table_name': table_name,
param = {'collection_name': collection_name,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.L2}
status = connect.create_table(param)
status = connect.create_collection(param)
# logging.getLogger().info(status)
if not status.OK():
pytest.exit("Table can not be created, exit pytest ...")
pytest.exit("collection can not be created, exit pytest ...")
def teardown():
status, table_names = connect.show_tables()
for table_name in table_names:
connect.delete_table(table_name)
status, collection_names = connect.show_collections()
for collection_name in collection_names:
connect.drop_collection(collection_name)
request.addfinalizer(teardown)
return table_name
return collection_name
@pytest.fixture(scope="function")
def ip_table(request, connect):
ori_table_name = getattr(request.module, "table_id", "test")
table_name = gen_unique_str(ori_table_name)
def ip_collection(request, connect):
ori_collection_name = getattr(request.module, "collection_id", "test")
collection_name = gen_unique_str(ori_collection_name)
dim = getattr(request.module, "dim", "128")
param = {'table_name': table_name,
param = {'collection_name': collection_name,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.IP}
status = connect.create_table(param)
status = connect.create_collection(param)
# logging.getLogger().info(status)
if not status.OK():
pytest.exit("Table can not be created, exit pytest ...")
pytest.exit("collection can not be created, exit pytest ...")
def teardown():
status, table_names = connect.show_tables()
for table_name in table_names:
connect.delete_table(table_name)
status, collection_names = connect.show_collections()
for collection_name in collection_names:
connect.drop_collection(collection_name)
request.addfinalizer(teardown)
return table_name
return collection_name
@pytest.fixture(scope="function")
def jac_table(request, connect):
ori_table_name = getattr(request.module, "table_id", "test")
table_name = gen_unique_str(ori_table_name)
def jac_collection(request, connect):
ori_collection_name = getattr(request.module, "collection_id", "test")
collection_name = gen_unique_str(ori_collection_name)
dim = getattr(request.module, "dim", "128")
param = {'table_name': table_name,
param = {'collection_name': collection_name,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.JACCARD}
status = connect.create_table(param)
status = connect.create_collection(param)
# logging.getLogger().info(status)
if not status.OK():
pytest.exit("Table can not be created, exit pytest ...")
pytest.exit("collection can not be created, exit pytest ...")
def teardown():
status, table_names = connect.show_tables()
for table_name in table_names:
connect.delete_table(table_name)
status, collection_names = connect.show_collections()
for collection_name in collection_names:
connect.drop_collection(collection_name)
request.addfinalizer(teardown)
return table_name
return collection_name
@pytest.fixture(scope="function")
def ham_table(request, connect):
ori_table_name = getattr(request.module, "table_id", "test")
table_name = gen_unique_str(ori_table_name)
def ham_collection(request, connect):
ori_collection_name = getattr(request.module, "collection_id", "test")
collection_name = gen_unique_str(ori_collection_name)
dim = getattr(request.module, "dim", "128")
param = {'table_name': table_name,
param = {'collection_name': collection_name,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.HAMMING}
status = connect.create_table(param)
status = connect.create_collection(param)
# logging.getLogger().info(status)
if not status.OK():
pytest.exit("Table can not be created, exit pytest ...")
pytest.exit("collection can not be created, exit pytest ...")
def teardown():
status, table_names = connect.show_tables()
for table_name in table_names:
connect.delete_table(table_name)
status, collection_names = connect.show_collections()
for collection_name in collection_names:
connect.drop_collection(collection_name)
request.addfinalizer(teardown)
return table_name
return collection_name
@pytest.fixture(scope="function")
def tanimoto_table(request, connect):
ori_table_name = getattr(request.module, "table_id", "test")
table_name = gen_unique_str(ori_table_name)
def tanimoto_collection(request, connect):
ori_collection_name = getattr(request.module, "collection_id", "test")
collection_name = gen_unique_str(ori_collection_name)
dim = getattr(request.module, "dim", "128")
param = {'table_name': table_name,
param = {'collection_name': collection_name,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.TANIMOTO}
status = connect.create_table(param)
status = connect.create_collection(param)
# logging.getLogger().info(status)
if not status.OK():
pytest.exit("Table can not be created, exit pytest ...")
pytest.exit("collection can not be created, exit pytest ...")
def teardown():
status, table_names = connect.show_tables()
for table_name in table_names:
connect.delete_table(table_name)
status, collection_names = connect.show_collections()
for collection_name in collection_names:
connect.drop_collection(collection_name)
request.addfinalizer(teardown)
return table_name
return collection_name

808
tests/milvus_python_test/test_add_vectors.py
View File

File diff suppressed because it is too large Load Diff

1051
tests/milvus_python_test/test_collection.py Normal file
View File

File diff suppressed because it is too large Load Diff

644
tests/milvus_python_test/test_collection_count.py Normal file
View File

@ -0,0 +1,644 @@
import pdb
import pytest
import logging
import itertools
from time import sleep
import threading
from multiprocessing import Process
from milvus import IndexType, MetricType
from utils import *
dim = 128
index_file_size = 10
add_time_interval = 3
tag = "1970-01-01"
nb = 6000
class TestCollectionCount:
"""
params means different nb, the nb value may trigger merge, or not
"""
@pytest.fixture(
scope="function",
params=[
1,
5000,
100000,
],
)
def add_vectors_nb(self, request):
yield request.param
"""
generate valid create_index params
"""
@pytest.fixture(
scope="function",
params=gen_simple_index()
)
def get_simple_index(self, request, connect):
if str(connect._cmd("mode")[1]) == "CPU":
if request.param["index_type"] == IndexType.IVF_SQ8H:
pytest.skip("sq8h not support in cpu mode")
if request.param["index_type"] == IndexType.IVF_PQ:
pytest.skip("Skip PQ Temporary")
return request.param
def test_collection_rows_count(self, connect, collection, add_vectors_nb):
'''
target: test collection rows_count is correct or not
method: create collection and add vectors in it,
assert the value returned by count_collection method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nb = add_vectors_nb
vectors = gen_vectors(nb, dim)
res = connect.add_vectors(collection_name=collection, records=vectors)
connect.flush([collection])
status, res = connect.count_collection(collection)
assert res == nb
def test_collection_rows_count_partition(self, connect, collection, add_vectors_nb):
'''
target: test collection rows_count is correct or not
method: create collection, create partition and add vectors in it,
assert the value returned by count_collection method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nb = add_vectors_nb
vectors = gen_vectors(nb, dim)
status = connect.create_partition(collection, tag)
assert status.OK()
res = connect.add_vectors(collection_name=collection, records=vectors, partition_tag=tag)
connect.flush([collection])
status, res = connect.count_collection(collection)
assert res == nb
def test_collection_rows_count_multi_partitions_A(self, connect, collection, add_vectors_nb):
'''
target: test collection rows_count is correct or not
method: create collection, create partitions and add vectors in it,
assert the value returned by count_collection method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
new_tag = "new_tag"
nb = add_vectors_nb
vectors = gen_vectors(nb, dim)
status = connect.create_partition(collection, tag)
status = connect.create_partition(collection, new_tag)
assert status.OK()
res = connect.add_vectors(collection_name=collection, records=vectors)
connect.flush([collection])
status, res = connect.count_collection(collection)
assert res == nb
def test_collection_rows_count_multi_partitions_B(self, connect, collection, add_vectors_nb):
'''
target: test collection rows_count is correct or not
method: create collection, create partitions and add vectors in one of the partitions,
assert the value returned by count_collection method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
new_tag = "new_tag"
nb = add_vectors_nb
vectors = gen_vectors(nb, dim)
status = connect.create_partition(collection, tag)
status = connect.create_partition(collection, new_tag)
assert status.OK()
res = connect.add_vectors(collection_name=collection, records=vectors, partition_tag=tag)
connect.flush([collection])
status, res = connect.count_collection(collection)
assert res == nb
def test_collection_rows_count_multi_partitions_C(self, connect, collection, add_vectors_nb):
'''
target: test collection rows_count is correct or not
method: create collection, create partitions and add vectors in one of the partitions,
assert the value returned by count_collection method is equal to length of vectors
expected: the collection count is equal to the length of vectors
'''
new_tag = "new_tag"
nb = add_vectors_nb
vectors = gen_vectors(nb, dim)
status = connect.create_partition(collection, tag)
status = connect.create_partition(collection, new_tag)
assert status.OK()
res = connect.add_vectors(collection_name=collection, records=vectors, partition_tag=tag)
res = connect.add_vectors(collection_name=collection, records=vectors, partition_tag=new_tag)
connect.flush([collection])
status, res = connect.count_collection(collection)
assert res == nb * 2
def test_collection_rows_count_after_index_created(self, connect, collection, get_simple_index):
'''
target: test count_collection, after index have been created
method: add vectors in db, and create index, then calling count_collection with correct params
expected: count_collection raise exception
'''
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
nb = 100
vectors = gen_vectors(nb, dim)
res = connect.add_vectors(collection_name=collection, records=vectors)
connect.flush([collection])
connect.create_index(collection, index_type, index_param)
status, res = connect.count_collection(collection)
assert res == nb
@pytest.mark.level(2)
def test_count_without_connection(self, collection, dis_connect):
'''
target: test count_collection, without connection
method: calling count_collection with correct params, with a disconnected instance
expected: count_collection raise exception
'''
with pytest.raises(Exception) as e:
status = dis_connect.count_collection(collection)
def test_collection_rows_count_no_vectors(self, connect, collection):
'''
target: test collection rows_count is correct or not, if collection is empty
method: create collection and no vectors in it,
assert the value returned by count_collection method is equal to 0
expected: the count is equal to 0
'''
collection_name = gen_unique_str()
param = {'collection_name': collection_name,
'dimension': dim,
'index_file_size': index_file_size}
connect.create_collection(param)
status, res = connect.count_collection(collection)
assert res == 0
# TODO: enable
@pytest.mark.level(2)
@pytest.mark.timeout(20)
def _test_collection_rows_count_multiprocessing(self, connect, collection, args):
'''
target: test collection rows_count is correct or not with multiprocess
method: create collection and add vectors in it,
assert the value returned by count_collection method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nq = 2
uri = "tcp://%s:%s" % (args["ip"], args["port"])
vectors = gen_vectors(nq, dim)
res = connect.add_vectors(collection_name=collection, records=vectors)
time.sleep(add_time_interval)
def rows_count(milvus):
status, res = milvus.count_collection(collection)
logging.getLogger().info(status)
assert res == nq
process_num = 8
processes = []
for i in range(process_num):
milvus = get_milvus(args["handler"])
milvus.connect(uri=uri)
p = Process(target=rows_count, args=(milvus, ))
processes.append(p)
p.start()
logging.getLogger().info(p)
for p in processes:
p.join()
def test_collection_rows_count_multi_collections(self, connect):
'''
target: test collection rows_count is correct or not with multiple collections of L2
method: create collection and add vectors in it,
assert the value returned by count_collection method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nq = 100
vectors = gen_vectors(nq, dim)
collection_list = []
for i in range(20):
collection_name = gen_unique_str()
collection_list.append(collection_name)
param = {'collection_name': collection_name,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.L2}
connect.create_collection(param)
res = connect.add_vectors(collection_name=collection_name, records=vectors)
connect.flush(collection_list)
for i in range(20):
status, res = connect.count_collection(collection_list[i])
assert status.OK()
assert res == nq
class TestCollectionCountIP:
"""
params means different nb, the nb value may trigger merge, or not
"""
@pytest.fixture(
scope="function",
params=[
1,
5000,
100000,
],
)
def add_vectors_nb(self, request):
yield request.param
"""
generate valid create_index params
"""
@pytest.fixture(
scope="function",
params=gen_simple_index()
)
def get_simple_index(self, request, connect):
if str(connect._cmd("mode")[1]) == "CPU":
if request.param["index_type"] == IndexType.IVF_SQ8H:
pytest.skip("sq8h not support in CPU mode")
if request.param["index_type"] == IndexType.IVF_PQ:
pytest.skip("Skip PQ Temporary")
return request.param
def test_collection_rows_count(self, connect, ip_collection, add_vectors_nb):
'''
target: test collection rows_count is correct or not
method: create collection and add vectors in it,
assert the value returned by count_collection method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nb = add_vectors_nb
vectors = gen_vectors(nb, dim)
res = connect.add_vectors(collection_name=ip_collection, records=vectors)
connect.flush([ip_collection])
status, res = connect.count_collection(ip_collection)
assert res == nb
def test_collection_rows_count_after_index_created(self, connect, ip_collection, get_simple_index):
'''
target: test count_collection, after index have been created
method: add vectors in db, and create index, then calling count_collection with correct params
expected: count_collection raise exception
'''
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
nb = 100
vectors = gen_vectors(nb, dim)
res = connect.add_vectors(collection_name=ip_collection, records=vectors)
connect.flush([ip_collection])
connect.create_index(ip_collection, index_type, index_param)
status, res = connect.count_collection(ip_collection)
assert res == nb
@pytest.mark.level(2)
def test_count_without_connection(self, ip_collection, dis_connect):
'''
target: test count_collection, without connection
method: calling count_collection with correct params, with a disconnected instance
expected: count_collection raise exception
'''
with pytest.raises(Exception) as e:
status = dis_connect.count_collection(ip_collection)
def test_collection_rows_count_no_vectors(self, connect, ip_collection):
'''
target: test collection rows_count is correct or not, if collection is empty
method: create collection and no vectors in it,
assert the value returned by count_collection method is equal to 0
expected: the count is equal to 0
'''
collection_name = gen_unique_str("test_collection")
param = {'collection_name': collection_name,
'dimension': dim,
'index_file_size': index_file_size}
connect.create_collection(param)
status, res = connect.count_collection(ip_collection)
assert res == 0
# TODO: enable
@pytest.mark.timeout(60)
def _test_collection_rows_count_multiprocessing(self, connect, ip_collection, args):
'''
target: test collection rows_count is correct or not with multiprocess
method: create collection and add vectors in it,
assert the value returned by count_collection method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nq = 2
uri = "tcp://%s:%s" % (args["ip"], args["port"])
vectors = gen_vectors(nq, dim)
res = connect.add_vectors(collection_name=ip_collection, records=vectors)
time.sleep(add_time_interval)
def rows_count(milvus):
status, res = milvus.count_collection(ip_collection)
logging.getLogger().info(status)
assert res == nq
process_num = 8
processes = []
for i in range(process_num):
milvus = get_milvus(args["handler"])
milvus.connect(uri=uri)
p = Process(target=rows_count, args=(milvus,))
processes.append(p)
p.start()
logging.getLogger().info(p)
for p in processes:
p.join()
def test_collection_rows_count_multi_collections(self, connect):
'''
target: test collection rows_count is correct or not with multiple collections of IP
method: create collection and add vectors in it,
assert the value returned by count_collection method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nq = 100
vectors = gen_vectors(nq, dim)
collection_list = []
for i in range(20):
collection_name = gen_unique_str('test_collection_rows_count_multi_collections')
collection_list.append(collection_name)
param = {'collection_name': collection_name,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.IP}
connect.create_collection(param)
res = connect.add_vectors(collection_name=collection_name, records=vectors)
connect.flush(collection_list)
for i in range(20):
status, res = connect.count_collection(collection_list[i])
assert status.OK()
assert res == nq
class TestCollectionCountJAC:
"""
params means different nb, the nb value may trigger merge, or not
"""
@pytest.fixture(
scope="function",
params=[
1,
5000,
100000,
],
)
def add_vectors_nb(self, request):
yield request.param
"""
generate valid create_index params
"""
@pytest.fixture(
scope="function",
params=gen_simple_index()
)
def get_jaccard_index(self, request, connect):
logging.getLogger().info(request.param)
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
return request.param
else:
pytest.skip("Skip index Temporary")
def test_collection_rows_count(self, connect, jac_collection, add_vectors_nb):
'''
target: test collection rows_count is correct or not
method: create collection and add vectors in it,
assert the value returned by count_collection method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nb = add_vectors_nb
tmp, vectors = gen_binary_vectors(nb, dim)
res = connect.add_vectors(collection_name=jac_collection, records=vectors)
connect.flush([jac_collection])
status, res = connect.count_collection(jac_collection)
assert res == nb
def test_collection_rows_count_after_index_created(self, connect, jac_collection, get_jaccard_index):
'''
target: test count_collection, after index have been created
method: add vectors in db, and create index, then calling count_collection with correct params
expected: count_collection raise exception
'''
nb = 100
index_param = get_jaccard_index["index_param"]
index_type = get_jaccard_index["index_type"]
tmp, vectors = gen_binary_vectors(nb, dim)
res = connect.add_vectors(collection_name=jac_collection, records=vectors)
connect.flush([jac_collection])
connect.create_index(jac_collection, index_type, index_param)
status, res = connect.count_collection(jac_collection)
assert res == nb
@pytest.mark.level(2)
def test_count_without_connection(self, jac_collection, dis_connect):
'''
target: test count_collection, without connection
method: calling count_collection with correct params, with a disconnected instance
expected: count_collection raise exception
'''
with pytest.raises(Exception) as e:
status = dis_connect.count_collection(jac_collection)
def test_collection_rows_count_no_vectors(self, connect, jac_collection):
'''
target: test collection rows_count is correct or not, if collection is empty
method: create collection and no vectors in it,
assert the value returned by count_collection method is equal to 0
expected: the count is equal to 0
'''
collection_name = gen_unique_str("test_collection")
param = {'collection_name': collection_name,
'dimension': dim,
'index_file_size': index_file_size}
connect.create_collection(param)
status, res = connect.count_collection(jac_collection)
assert res == 0
def test_collection_rows_count_multi_collections(self, connect):
'''
target: test collection rows_count is correct or not with multiple collections of IP
method: create collection and add vectors in it,
assert the value returned by count_collection method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nq = 100
tmp, vectors = gen_binary_vectors(nq, dim)
collection_list = []
for i in range(20):
collection_name = gen_unique_str('test_collection_rows_count_multi_collections')
collection_list.append(collection_name)
param = {'collection_name': collection_name,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.JACCARD}
connect.create_collection(param)
res = connect.add_vectors(collection_name=collection_name, records=vectors)
connect.flush(collection_list)
for i in range(20):
status, res = connect.count_collection(collection_list[i])
assert status.OK()
assert res == nq
class TestCollectionCountHAM:
"""
params means different nb, the nb value may trigger merge, or not
"""
@pytest.fixture(
scope="function",
params=[
1,
5000,
100000,
],
)
def add_vectors_nb(self, request):
yield request.param
"""
generate valid create_index params
"""
@pytest.fixture(
scope="function",
params=gen_simple_index()
)
def get_hamming_index(self, request, connect):
logging.getLogger().info(request.param)
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
return request.param
else:
pytest.skip("Skip index Temporary")
def test_collection_rows_count(self, connect, ham_collection, add_vectors_nb):
'''
target: test collection rows_count is correct or not
method: create collection and add vectors in it,
assert the value returned by count_collection method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nb = add_vectors_nb
tmp, vectors = gen_binary_vectors(nb, dim)
res = connect.add_vectors(collection_name=ham_collection, records=vectors)
connect.flush([ham_collection])
status, res = connect.count_collection(ham_collection)
assert res == nb
def test_collection_rows_count_after_index_created(self, connect, ham_collection, get_hamming_index):
'''
target: test count_collection, after index have been created
method: add vectors in db, and create index, then calling count_collection with correct params
expected: count_collection raise exception
'''
nb = 100
index_type = get_hamming_index["index_type"]
index_param = get_hamming_index["index_param"]
tmp, vectors = gen_binary_vectors(nb, dim)
res = connect.add_vectors(collection_name=ham_collection, records=vectors)
connect.flush([ham_collection])
connect.create_index(ham_collection, index_type, index_param)
status, res = connect.count_collection(ham_collection)
assert res == nb
@pytest.mark.level(2)
def test_count_without_connection(self, ham_collection, dis_connect):
'''
target: test count_collection, without connection
method: calling count_collection with correct params, with a disconnected instance
expected: count_collection raise exception
'''
with pytest.raises(Exception) as e:
status = dis_connect.count_collection(ham_collection)
def test_collection_rows_count_no_vectors(self, connect, ham_collection):
'''
target: test collection rows_count is correct or not, if collection is empty
method: create collection and no vectors in it,
assert the value returned by count_collection method is equal to 0
expected: the count is equal to 0
'''
collection_name = gen_unique_str("test_collection")
param = {'collection_name': collection_name,
'dimension': dim,
'index_file_size': index_file_size}
connect.create_collection(param)
status, res = connect.count_collection(ham_collection)
assert res == 0
def test_collection_rows_count_multi_collections(self, connect):
'''
target: test collection rows_count is correct or not with multiple collections of IP
method: create collection and add vectors in it,
assert the value returned by count_collection method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nq = 100
tmp, vectors = gen_binary_vectors(nq, dim)
collection_list = []
for i in range(20):
collection_name = gen_unique_str('test_collection_rows_count_multi_collections')
collection_list.append(collection_name)
param = {'collection_name': collection_name,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.HAMMING}
connect.create_collection(param)
res = connect.add_vectors(collection_name=collection_name, records=vectors)
connect.flush(collection_list)
for i in range(20):
status, res = connect.count_collection(collection_list[i])
assert status.OK()
assert res == nq
class TestCollectionCountTANIMOTO:
"""
params means different nb, the nb value may trigger merge, or not
"""
@pytest.fixture(
scope="function",
params=[
1,
5000,
100000,
],
)
def add_vectors_nb(self, request):
yield request.param
"""
generate valid create_index params
"""
@pytest.fixture(
scope="function",
params=gen_simple_index()
)
def get_tanimoto_index(self, request, connect):
logging.getLogger().info(request.param)
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
return request.param
else:
pytest.skip("Skip index Temporary")
def test_collection_rows_count(self, connect, tanimoto_collection, add_vectors_nb):
'''
target: test collection rows_count is correct or not
method: create collection and add vectors in it,
assert the value returned by count_collection method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nb = add_vectors_nb
tmp, vectors = gen_binary_vectors(nb, dim)
res = connect.add_vectors(collection_name=tanimoto_collection, records=vectors)
connect.flush([tanimoto_collection])
status, res = connect.count_collection(tanimoto_collection)
assert status.OK()
assert res == nb

164
tests/milvus_python_test/test_table_info.py → tests/milvus_python_test/test_collection_info.py
View File

@ -18,7 +18,7 @@ nb = 6000
nlist = 1024
class TestTableInfoBase:
class TestCollectionInfoBase:
def index_string_convert(self, index_string, index_type):
if index_string == "IDMAP" and index_type == IndexType.FLAT:
return True
@ -30,87 +30,87 @@ class TestTableInfoBase:
"""
******************************************************************
The following cases are used to test `table_info` function
The following cases are used to test `collection_info` function
******************************************************************
"""
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_name_None(self, connect, table):
def test_get_collection_info_name_None(self, connect, collection):
'''
target: get table info where table name is None
method: call table_info with the table_name: None
target: get collection info where collection name is None
method: call collection_info with the collection_name: None
expected: status not ok
'''
table_name = None
status, info = connect.table_info(table_name)
collection_name = None
status, info = connect.collection_info(collection_name)
assert not status.OK()
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_name_not_existed(self, connect, table):
def test_get_collection_info_name_not_existed(self, connect, collection):
'''
target: get table info where table name does not exist
method: call table_info with a random table_name, which is not in db
target: get collection info where collection name does not exist
method: call collection_info with a random collection_name, which is not in db
expected: status not ok
'''
table_name = gen_unique_str("not_existed_table")
status, info = connect.table_info(table_name)
collection_name = gen_unique_str("not_existed_collection")
status, info = connect.collection_info(collection_name)
assert not status.OK()
@pytest.fixture(
scope="function",
params=gen_invalid_table_names()
params=gen_invalid_collection_names()
)
def get_table_name(self, request):
def get_collection_name(self, request):
yield request.param
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_name_invalid(self, connect, get_table_name):
def test_get_collection_info_name_invalid(self, connect, get_collection_name):
'''
target: get table info where table name is invalid
method: call table_info with invalid table_name
target: get collection info where collection name is invalid
method: call collection_info with invalid collection_name
expected: status not ok
'''
table_name = get_table_name
status, info = connect.table_info(table_name)
collection_name = get_collection_name
status, info = connect.collection_info(collection_name)
assert not status.OK()
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_table_row_count(self, connect, table):
def test_get_collection_info_collection_row_count(self, connect, collection):
'''
target: get row count with table_info
method: add and delete vectors, check count in table info
target: get row count with collection_info
method: add and delete vectors, check count in collection info
expected: status ok, count as expected
'''
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
assert info.count == nb
# delete a few vectors
delete_ids = [ids[0], ids[-1]]
status = connect.delete_by_id(table, delete_ids)
status = connect.delete_by_id(collection, delete_ids)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
assert info.count == nb - 2
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_partition_stats_A(self, connect, table):
def test_get_collection_info_partition_stats_A(self, connect, collection):
'''
target: get partition info in a table
method: no partition, call table_info and check partition_stats
target: get partition info in a collection
method: no partition, call collection_info and check partition_stats
expected: status ok, "_default" partition is listed
'''
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
logging.getLogger().info(info)
assert len(info.partitions_stat) == 1
@ -119,19 +119,19 @@ class TestTableInfoBase:
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_partition_stats_B(self, connect, table):
def test_get_collection_info_partition_stats_B(self, connect, collection):
'''
target: get partition info in a table
method: call table_info after partition created and check partition_stats
target: get partition info in a collection
method: call collection_info after partition created and check partition_stats
expected: status ok, vectors added to partition
'''
vectors = gen_vectors(nb, dim)
status = connect.create_partition(table, tag)
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
status = connect.create_partition(collection, tag)
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
logging.getLogger().info(info)
assert len(info.partitions_stat) == 2
@ -139,23 +139,23 @@ class TestTableInfoBase:
assert info.partitions_stat[1].count == nb
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_partition_stats_C(self, connect, table):
def test_get_collection_info_partition_stats_C(self, connect, collection):
'''
target: get partition info in a table
method: create two partitions, add vectors in one of the partitions, call table_info and check
target: get partition info in a collection
method: create two partitions, add vectors in one of the partitions, call collection_info and check
expected: status ok, vectors added to one partition but not the other
'''
new_tag = "new_tag"
vectors = gen_vectors(nb, dim)
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
assert status.OK()
status = connect.create_partition(table, new_tag)
status = connect.create_partition(collection, new_tag)
assert status.OK()
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
logging.getLogger().info(info)
for partition in info.partitions_stat:
@ -165,25 +165,25 @@ class TestTableInfoBase:
assert partition.count == 0
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_partition_stats_D(self, connect, table):
def test_get_collection_info_partition_stats_D(self, connect, collection):
'''
target: get partition info in a table
method: create two partitions, add vectors in both partitions, call table_info and check
target: get partition info in a collection
method: create two partitions, add vectors in both partitions, call collection_info and check
expected: status ok, vectors added to both partitions
'''
new_tag = "new_tag"
vectors = gen_vectors(nb, dim)
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
assert status.OK()
status = connect.create_partition(table, new_tag)
status = connect.create_partition(collection, new_tag)
assert status.OK()
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
assert status.OK()
status, ids = connect.add_vectors(table, vectors, partition_tag=new_tag)
status, ids = connect.add_vectors(collection, vectors, partition_tag=new_tag)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
assert info.count == nb * 2
for partition in info.partitions_stat:
@ -205,24 +205,24 @@ class TestTableInfoBase:
return request.param
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_after_index_created(self, connect, table, get_simple_index):
def test_get_collection_info_after_index_created(self, connect, collection, get_simple_index):
'''
target: test table info after index created
method: create table, add vectors, create index and call table_info
target: test collection info after index created
method: create collection, add vectors, create index and call collection_info
expected: status ok, index created and shown in segments_stat
'''
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status = connect.create_index(table, index_type, index_param)
status = connect.create_index(collection, index_type, index_param)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
logging.getLogger().info(info)
index_string = info.partitions_stat[0].segments_stat[0].index_name
@ -231,24 +231,24 @@ class TestTableInfoBase:
assert nb == info.partitions_stat[0].segments_stat[0].count
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_after_create_same_index_repeatedly(self, connect, table, get_simple_index):
def test_get_collection_info_after_create_same_index_repeatedly(self, connect, collection, get_simple_index):
'''
target: test table info after index created repeatedly
method: create table, add vectors, create index and call table_info multiple times
target: test collection info after index created repeatedly
method: create collection, add vectors, create index and call collection_info multiple times
expected: status ok, index info shown in segments_stat
'''
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status = connect.create_index(table, index_type, index_param)
status = connect.create_index(table, index_type, index_param)
status = connect.create_index(table, index_type, index_param)
status = connect.create_index(collection, index_type, index_param)
status = connect.create_index(collection, index_type, index_param)
status = connect.create_index(collection, index_type, index_param)
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
logging.getLogger().info(info)
index_string = info.partitions_stat[0].segments_stat[0].index_name
@ -257,22 +257,22 @@ class TestTableInfoBase:
assert nb == info.partitions_stat[0].segments_stat[0].count
@pytest.mark.timeout(INFO_TIMEOUT)
def test_get_table_info_after_create_different_index_repeatedly(self, connect, table, get_simple_index):
def test_get_collection_info_after_create_different_index_repeatedly(self, connect, collection, get_simple_index):
'''
target: test table info after index created repeatedly
method: create table, add vectors, create index and call table_info multiple times
target: test collection info after index created repeatedly
method: create collection, add vectors, create index and call collection_info multiple times
expected: status ok, index info shown in segments_stat
'''
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
index_param = {"nlist": nlist}
for index_type in [IndexType.FLAT, IndexType.IVFLAT, IndexType.IVF_SQ8]:
status = connect.create_index(table, index_type, index_param)
status = connect.create_index(collection, index_type, index_param)
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
logging.getLogger().info(info)
index_string = info.partitions_stat[0].segments_stat[0].index_name

768
tests/milvus_python_test/test_compact.py
View File

File diff suppressed because it is too large Load Diff

1099
tests/milvus_python_test/test_config.py
View File

File diff suppressed because it is too large Load Diff

2
tests/milvus_python_test/test_connect.py
View File

@ -285,7 +285,7 @@ class TestConnect:
b. data_set not too large incase disconnection happens when data is underd-preparing
c. data_set not too small incase disconnection happens when data has already been transferred
d. make sure disconnection happens when data is in-transport
Expected: Failure, get_table_row_count == 0
Expected: Failure, count_collection == 0
'''
pass

278
tests/milvus_python_test/test_delete_vectors.py
View File

@ -11,7 +11,7 @@ from utils import *
dim = 128
index_file_size = 10
table_id = "test_delete"
collection_id = "test_delete"
DELETE_TIMEOUT = 60
nprobe = 1
epsilon = 0.001
@ -38,7 +38,7 @@ class TestDeleteBase:
pytest.skip("Only support CPU mode")
return request.param
def test_delete_vector_search(self, connect, table, get_simple_index):
def test_delete_vector_search(self, connect, collection, get_simple_index):
'''
target: test delete vector
method: add vector and delete
@ -47,22 +47,22 @@ class TestDeleteBase:
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
vector = gen_single_vector(dim)
status, ids = connect.add_vectors(table, vector)
status, ids = connect.add_vectors(collection, vector)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status = connect.delete_by_id(table, ids)
status = connect.delete_by_id(collection, ids)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
search_param = get_search_param(index_type)
status = connect.flush([table])
status = connect.flush([collection])
search_param = get_search_param(index_type)
status, res = connect.search_vectors(table, top_k, vector, params=search_param)
status, res = connect.search_vectors(collection, top_k, vector, params=search_param)
logging.getLogger().info(res)
assert status.OK()
assert len(res) == 0
def test_delete_vector_multi_same_ids(self, connect, table, get_simple_index):
def test_delete_vector_multi_same_ids(self, connect, collection, get_simple_index):
'''
target: test delete vector, with some same ids
method: add vector and delete
@ -71,58 +71,58 @@ class TestDeleteBase:
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
vectors = gen_vectors(nb, dim)
connect.add_vectors(table, vectors, ids=[1 for i in range(nb)])
status = connect.flush([table])
connect.add_vectors(collection, vectors, ids=[1 for i in range(nb)])
status = connect.flush([collection])
# Bloom filter error
assert status.OK()
status = connect.delete_by_id(table, [1])
status = connect.delete_by_id(collection, [1])
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
search_param = get_search_param(index_type)
status, res = connect.search_vectors(table, top_k, [vectors[0]], params=search_param)
status, res = connect.search_vectors(collection, top_k, [vectors[0]], params=search_param)
logging.getLogger().info(res)
assert status.OK()
assert len(res) == 0
def test_delete_vector_table_count(self, connect, table):
def test_delete_vector_collection_count(self, connect, collection):
'''
target: test delete vector
method: add vector and delete
expected: status ok, vector deleted
'''
vector = gen_single_vector(dim)
status, ids = connect.add_vectors(table, vector)
status, ids = connect.add_vectors(collection, vector)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status = connect.delete_by_id(table, ids)
status = connect.delete_by_id(collection, ids)
assert status.OK()
status = connect.flush([table])
status, res = connect.get_table_row_count(table)
status = connect.flush([collection])
status, res = connect.count_collection(collection)
assert status.OK()
assert res == 0
def test_delete_vector_table_count_no_flush(self, connect, table):
def test_delete_vector_collection_count_no_flush(self, connect, collection):
'''
target: test delete vector
method: add vector and delete, no flush(using auto flush)
expected: status ok, vector deleted
'''
vector = gen_single_vector(dim)
status, ids = connect.add_vectors(table, vector)
status, ids = connect.add_vectors(collection, vector)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status = connect.delete_by_id(table, ids)
status = connect.delete_by_id(collection, ids)
assert status.OK()
time.sleep(2)
status, res = connect.get_table_row_count(table)
status, res = connect.count_collection(collection)
assert status.OK()
assert res == 0
def test_delete_vector_id_not_exised(self, connect, table, get_simple_index):
def test_delete_vector_id_not_exised(self, connect, collection, get_simple_index):
'''
target: test delete vector, params vector_id not existed
method: add vector and delete
@ -131,34 +131,34 @@ class TestDeleteBase:
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
vector = gen_single_vector(dim)
status, ids = connect.add_vectors(table, vector)
status, ids = connect.add_vectors(collection, vector)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status = connect.delete_by_id(table, [0])
status = connect.delete_by_id(collection, [0])
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
search_param = get_search_param(index_type)
status, res = connect.search_vectors(table, top_k, vector, params=search_param)
status, res = connect.search_vectors(collection, top_k, vector, params=search_param)
assert status.OK()
assert res[0][0].id == ids[0]
def test_delete_vector_table_not_existed(self, connect, table):
def test_delete_vector_collection_not_existed(self, connect, collection):
'''
target: test delete vector, params table_name not existed
target: test delete vector, params collection_name not existed
method: add vector and delete
expected: status not ok
'''
vector = gen_single_vector(dim)
status, ids = connect.add_vectors(table, vector)
status, ids = connect.add_vectors(collection, vector)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
table_new = gen_unique_str()
status = connect.delete_by_id(table_new, [0])
collection_new = gen_unique_str()
status = connect.delete_by_id(collection_new, [0])
assert not status.OK()
def test_add_vectors_delete_vector(self, connect, table, get_simple_index):
def test_add_vectors_delete_vector(self, connect, collection, get_simple_index):
'''
method: add vectors and delete
expected: status ok, vectors deleted
@ -166,19 +166,19 @@ class TestDeleteBase:
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
delete_ids = [ids[0], ids[-1]]
query_vecs = [vectors[0], vectors[1], vectors[-1]]
status = connect.delete_by_id(table, delete_ids)
status = connect.delete_by_id(collection, delete_ids)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
search_param = get_search_param(index_type)
status, res = connect.search_vectors(table, top_k, query_vecs, params=search_param)
status, res = connect.search_vectors(collection, top_k, query_vecs, params=search_param)
assert status.OK()
logging.getLogger().info(res)
assert res[0][0].distance > epsilon
@ -186,7 +186,7 @@ class TestDeleteBase:
assert res[1][0].id == ids[1]
assert res[2][0].distance > epsilon
def test_create_index_after_delete(self, connect, table, get_simple_index):
def test_create_index_after_delete(self, connect, collection, get_simple_index):
'''
method: add vectors and delete, then create index
expected: status ok, vectors deleted, index created
@ -194,19 +194,19 @@ class TestDeleteBase:
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
delete_ids = [ids[0], ids[-1]]
query_vecs = [vectors[0], vectors[1], vectors[-1]]
status = connect.delete_by_id(table, delete_ids)
status = connect.delete_by_id(collection, delete_ids)
assert status.OK()
status = connect.flush([table])
status = connect.create_index(table, index_type, index_param)
status = connect.flush([collection])
status = connect.create_index(collection, index_type, index_param)
assert status.OK()
search_param = get_search_param(index_type)
status, res = connect.search_vectors(table, top_k, query_vecs, params=search_param)
status, res = connect.search_vectors(collection, top_k, query_vecs, params=search_param)
assert status.OK()
logging.getLogger().info(res)
assert res[0][0].distance > epsilon
@ -214,7 +214,7 @@ class TestDeleteBase:
assert res[1][0].id == ids[1]
assert res[2][0].distance > epsilon
def test_add_vector_after_delete(self, connect, table, get_simple_index):
def test_add_vector_after_delete(self, connect, collection, get_simple_index):
'''
method: add vectors and delete, then add vector
expected: status ok, vectors deleted, vector added
@ -222,22 +222,22 @@ class TestDeleteBase:
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
delete_ids = [ids[0], ids[-1]]
query_vecs = [vectors[0], vectors[1], vectors[-1]]
status = connect.delete_by_id(table, delete_ids)
status = connect.delete_by_id(collection, delete_ids)
assert status.OK()
status = connect.flush([table])
status, tmp_ids = connect.add_vectors(table, [vectors[0], vectors[-1]])
status = connect.flush([collection])
status, tmp_ids = connect.add_vectors(collection, [vectors[0], vectors[-1]])
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
search_param = get_search_param(index_type)
status, res = connect.search_vectors(table, top_k, query_vecs, params=search_param)
status, res = connect.search_vectors(collection, top_k, query_vecs, params=search_param)
assert status.OK()
logging.getLogger().info(res)
assert res[0][0].id == tmp_ids[0]
@ -246,41 +246,41 @@ class TestDeleteBase:
assert res[2][0].id == tmp_ids[-1]
assert res[2][0].distance < epsilon
def test_delete_multiable_times(self, connect, table):
def test_delete_multiable_times(self, connect, collection):
'''
method: add vectors and delete id serveral times
expected: status ok, vectors deleted, and status ok for next delete operation
'''
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
delete_ids = [ids[0], ids[-1]]
query_vecs = [vectors[0], vectors[1], vectors[-1]]
status = connect.delete_by_id(table, delete_ids)
status = connect.delete_by_id(collection, delete_ids)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
for i in range(10):
status = connect.delete_by_id(table, delete_ids)
status = connect.delete_by_id(collection, delete_ids)
assert status.OK()
def test_delete_no_flush_multiable_times(self, connect, table):
def test_delete_no_flush_multiable_times(self, connect, collection):
'''
method: add vectors and delete id serveral times
expected: status ok, vectors deleted, and status ok for next delete operation
'''
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
delete_ids = [ids[0], ids[-1]]
query_vecs = [vectors[0], vectors[1], vectors[-1]]
status = connect.delete_by_id(table, delete_ids)
status = connect.delete_by_id(collection, delete_ids)
assert status.OK()
for i in range(10):
status = connect.delete_by_id(table, delete_ids)
status = connect.delete_by_id(collection, delete_ids)
assert status.OK()
assert status.OK()
@ -303,7 +303,7 @@ class TestDeleteIndexedVectors:
pytest.skip("Only support CPU mode")
return request.param
def test_delete_vectors_after_index_created_search(self, connect, table, get_simple_index):
def test_delete_vectors_after_index_created_search(self, connect, collection, get_simple_index):
'''
target: test delete vector after index created
method: add vector, create index and delete vector
@ -312,22 +312,22 @@ class TestDeleteIndexedVectors:
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
vector = gen_single_vector(dim)
status, ids = connect.add_vectors(table, vector)
status, ids = connect.add_vectors(collection, vector)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status = connect.create_index(table, index_type, index_param)
status = connect.create_index(collection, index_type, index_param)
assert status.OK()
status = connect.delete_by_id(table, ids)
status = connect.delete_by_id(collection, ids)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
search_param = get_search_param(index_type)
status, res = connect.search_vectors(table, top_k, vector, params=search_param)
status, res = connect.search_vectors(collection, top_k, vector, params=search_param)
logging.getLogger().info(res)
assert status.OK()
assert len(res) == 0
def test_add_vectors_delete_vector(self, connect, table, get_simple_index):
def test_add_vectors_delete_vector(self, connect, collection, get_simple_index):
'''
method: add vectors and delete
expected: status ok, vectors deleted
@ -335,21 +335,21 @@ class TestDeleteIndexedVectors:
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status = connect.create_index(table, index_type, index_param)
status = connect.create_index(collection, index_type, index_param)
assert status.OK()
delete_ids = [ids[0], ids[-1]]
query_vecs = [vectors[0], vectors[1], vectors[-1]]
status = connect.delete_by_id(table, delete_ids)
status = connect.delete_by_id(collection, delete_ids)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
search_param = get_search_param(index_type)
status, res = connect.search_vectors(table, top_k, query_vecs, params=search_param)
status, res = connect.search_vectors(collection, top_k, query_vecs, params=search_param)
assert status.OK()
logging.getLogger().info(res)
assert res[0][0].distance > epsilon
@ -375,7 +375,7 @@ class TestDeleteBinary:
else:
pytest.skip("Skip index Temporary")
def test_delete_vector_search(self, connect, jac_table, get_simple_index):
def test_delete_vector_search(self, connect, jac_collection, get_simple_index):
'''
target: test delete vector
method: add vector and delete
@ -384,15 +384,15 @@ class TestDeleteBinary:
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
tmp, vector = gen_binary_vectors(1, dim)
status, ids = connect.add_vectors(jac_table, vector)
status, ids = connect.add_vectors(jac_collection, vector)
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
assert status.OK()
status = connect.delete_by_id(jac_table, ids)
status = connect.delete_by_id(jac_collection, ids)
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
search_param = get_search_param(index_type)
status, res = connect.search_vectors(jac_table, top_k, vector, params=search_param)
status, res = connect.search_vectors(jac_collection, top_k, vector, params=search_param)
logging.getLogger().info(res)
assert status.OK()
assert len(res) == 0
@ -400,25 +400,25 @@ class TestDeleteBinary:
assert len(res) == 0
# TODO: soft delete
def test_delete_vector_table_count(self, connect, jac_table):
def test_delete_vector_collection_count(self, connect, jac_collection):
'''
target: test delete vector
method: add vector and delete
expected: status ok, vector deleted
'''
tmp, vector = gen_binary_vectors(1, dim)
status, ids = connect.add_vectors(jac_table, vector)
status, ids = connect.add_vectors(jac_collection, vector)
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
assert status.OK()
status = connect.delete_by_id(jac_table, ids)
status = connect.delete_by_id(jac_collection, ids)
assert status.OK()
status = connect.flush([jac_table])
status, res = connect.get_table_row_count(jac_table)
status = connect.flush([jac_collection])
status, res = connect.count_collection(jac_collection)
assert status.OK()
assert res == 0
def test_delete_vector_id_not_exised(self, connect, jac_table, get_simple_index):
def test_delete_vector_id_not_exised(self, connect, jac_collection, get_simple_index):
'''
target: test delete vector, params vector_id not existed
method: add vector and delete
@ -427,37 +427,37 @@ class TestDeleteBinary:
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
tmp, vector = gen_binary_vectors(1, dim)
status, ids = connect.add_vectors(jac_table, vector)
status, ids = connect.add_vectors(jac_collection, vector)
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
assert status.OK()
status = connect.delete_by_id(jac_table, [0])
status = connect.delete_by_id(jac_collection, [0])
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
status = connect.flush([jac_collection])
search_param = get_search_param(index_type)
status, res = connect.search_vectors(jac_table, top_k, vector, params=search_param)
status, res = connect.search_vectors(jac_collection, top_k, vector, params=search_param)
assert status.OK()
assert res[0][0].id == ids[0]
def test_delete_vector_table_not_existed(self, connect, jac_table):
def test_delete_vector_collection_not_existed(self, connect, jac_collection):
'''
target: test delete vector, params table_name not existed
target: test delete vector, params collection_name not existed
method: add vector and delete
expected: status not ok
'''
tmp, vector = gen_binary_vectors(1, dim)
status, ids = connect.add_vectors(jac_table, vector)
status, ids = connect.add_vectors(jac_collection, vector)
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
assert status.OK()
table_new = gen_unique_str()
status = connect.delete_by_id(table_new, [0])
table_new = gen_unique_str()
status = connect.delete_by_id(table_new, [0])
collection_new = gen_unique_str()
status = connect.delete_by_id(collection_new, [0])
collection_new = gen_unique_str()
status = connect.delete_by_id(collection_new, [0])
assert not status.OK()
def test_add_vectors_delete_vector(self, connect, jac_table, get_simple_index):
def test_add_vectors_delete_vector(self, connect, jac_collection, get_simple_index):
'''
method: add vectors and delete
expected: status ok, vectors deleted
@ -465,24 +465,24 @@ class TestDeleteBinary:
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
tmp, vectors = gen_binary_vectors(nb, dim)
status, ids = connect.add_vectors(jac_table, vectors)
status, ids = connect.add_vectors(jac_collection, vectors)
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
assert status.OK()
delete_ids = [ids[0], ids[-1]]
query_vecs = [vectors[0], vectors[1], vectors[-1]]
status = connect.delete_by_id(jac_table, delete_ids)
status = connect.delete_by_id(jac_collection, delete_ids)
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
search_param = get_search_param(index_type)
status, res = connect.search_vectors(jac_table, top_k, query_vecs, params=search_param)
status, res = connect.search_vectors(jac_collection, top_k, query_vecs, params=search_param)
assert status.OK()
logging.getLogger().info(res)
assert res[0][0].id != ids[0]
assert res[1][0].id == ids[1]
assert res[2][0].id != ids[-1]
def test_add_after_delete_vector(self, connect, jac_table, get_simple_index):
def test_add_after_delete_vector(self, connect, jac_collection, get_simple_index):
'''
method: add vectors and delete, add
expected: status ok, vectors added
@ -490,20 +490,20 @@ class TestDeleteBinary:
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
tmp, vectors = gen_binary_vectors(nb, dim)
status, ids = connect.add_vectors(jac_table, vectors)
status, ids = connect.add_vectors(jac_collection, vectors)
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
assert status.OK()
delete_ids = [ids[0], ids[-1]]
query_vecs = [vectors[0], vectors[1], vectors[-1]]
status = connect.delete_by_id(jac_table, delete_ids)
status = connect.delete_by_id(jac_collection, delete_ids)
assert status.OK()
status = connect.flush([jac_table])
status, tmp_ids = connect.add_vectors(jac_table, [vectors[0], vectors[-1]])
status = connect.flush([jac_collection])
status, tmp_ids = connect.add_vectors(jac_collection, [vectors[0], vectors[-1]])
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
search_param = get_search_param(index_type)
status, res = connect.search_vectors(jac_table, top_k, query_vecs, params=search_param)
status, res = connect.search_vectors(jac_collection, top_k, query_vecs, params=search_param)
assert status.OK()
logging.getLogger().info(res)
assert res[0][0].id == tmp_ids[0]
@ -526,32 +526,32 @@ class TestDeleteIdsIngalid(object):
yield request.param
@pytest.mark.level(1)
def test_delete_vector_id_invalid(self, connect, table, gen_invalid_id):
def test_delete_vector_id_invalid(self, connect, collection, gen_invalid_id):
invalid_id = gen_invalid_id
with pytest.raises(Exception) as e:
status = connect.delete_by_id(table, [invalid_id])
status = connect.delete_by_id(collection, [invalid_id])
@pytest.mark.level(2)
def test_delete_vector_ids_invalid(self, connect, table, gen_invalid_id):
def test_delete_vector_ids_invalid(self, connect, collection, gen_invalid_id):
invalid_id = gen_invalid_id
with pytest.raises(Exception) as e:
status = connect.delete_by_id(table, [1, invalid_id])
status = connect.delete_by_id(collection, [1, invalid_id])
class TestTableNameInvalid(object):
class TestCollectionNameInvalid(object):
"""
Test adding vectors with invalid table names
Test adding vectors with invalid collection names
"""
@pytest.fixture(
scope="function",
params=gen_invalid_table_names()
params=gen_invalid_collection_names()
)
def get_table_name(self, request):
def get_collection_name(self, request):
yield request.param
@pytest.mark.level(2)
def test_delete_vectors_with_invalid_table_name(self, connect, get_table_name):
table_name = get_table_name
status = connect.delete_by_id(table_name, [1])
def test_delete_vectors_with_invalid_collection_name(self, connect, get_collection_name):
collection_name = get_collection_name
status = connect.delete_by_id(collection_name, [1])
assert not status.OK()

148
tests/milvus_python_test/test_flush.py
View File

@ -10,7 +10,7 @@ from utils import *
dim = 128
index_file_size = 10
table_id = "test_flush"
collection_id = "test_flush"
DELETE_TIMEOUT = 60
nprobe = 1
tag = "1970-01-01"
@ -38,117 +38,117 @@ class TestFlushBase:
pytest.skip("Only support CPU mode")
return request.param
def test_flush_table_not_existed(self, connect, table):
def test_flush_collection_not_existed(self, connect, collection):
'''
target: test delete vector, params table_name not existed
target: test delete vector, params collection_name not existed
method: add vector and delete
expected: status not ok
'''
table_new = gen_unique_str()
status = connect.flush([table_new])
collection_new = gen_unique_str()
status = connect.flush([collection_new])
assert not status.OK()
def test_flush_empty_table(self, connect, table):
def test_flush_empty_collection(self, connect, collection):
'''
method: flush table with no vectors
method: flush collection with no vectors
expected: status ok
'''
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
def test_add_partition_flush(self, connect, table):
def test_add_partition_flush(self, connect, collection):
'''
method: add vectors into partition in table, flush serveral times
method: add vectors into partition in collection, flush serveral times
expected: status ok
'''
vectors = gen_vector(nb, dim)
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
vectors = gen_vectors(nb, dim)
ids = [i for i in range(nb)]
status, ids = connect.insert(table, vectors, ids)
status = connect.flush([table])
result, res = connect.get_table_row_count(table)
status, ids = connect.insert(collection, vectors, ids)
status = connect.flush([collection])
result, res = connect.count_collection(collection)
assert res == nb
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
result, res = connect.get_table_row_count(table)
result, res = connect.count_collection(collection)
assert res == 2 * nb
def test_add_partitions_flush(self, connect, table):
def test_add_partitions_flush(self, connect, collection):
'''
method: add vectors into partitions in table, flush one
method: add vectors into partitions in collection, flush one
expected: status ok
'''
vectors = gen_vectors(nb, dim)
tag_new = gen_unique_str()
status = connect.create_partition(table, tag)
status = connect.create_partition(table, tag_new)
status = connect.create_partition(collection, tag)
status = connect.create_partition(collection, tag_new)
ids = [i for i in range(nb)]
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
status = connect.flush([table])
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
status = connect.flush([collection])
assert status.OK()
status, ids = connect.insert(table, vectors, ids, partition_tag=tag_new)
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag_new)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
result, res = connect.get_table_row_count(table)
result, res = connect.count_collection(collection)
assert res == 2 * nb
def test_add_tables_flush(self, connect, table):
def test_add_collections_flush(self, connect, collection):
'''
method: add vectors into tables, flush one
method: add vectors into collections, flush one
expected: status ok
'''
vectors = gen_vectors(nb, dim)
table_new = gen_unique_str()
param = {'table_name': table_new,
collection_new = gen_unique_str()
param = {'collection_name': collection_new,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.L2}
status = connect.create_table(param)
status = connect.create_partition(table, tag)
status = connect.create_partition(table_new, tag)
status = connect.create_collection(param)
status = connect.create_partition(collection, tag)
status = connect.create_partition(collection_new, tag)
vectors = gen_vectors(nb, dim)
ids = [i for i in range(nb)]
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
status, ids = connect.insert(table_new, vectors, ids, partition_tag=tag)
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
status, ids = connect.insert(collection_new, vectors, ids, partition_tag=tag)
assert status.OK()
status = connect.flush([table])
status = connect.flush([table_new])
status = connect.flush([collection])
status = connect.flush([collection_new])
assert status.OK()
result, res = connect.get_table_row_count(table)
result, res = connect.count_collection(collection)
assert res == nb
result, res = connect.get_table_row_count(table_new)
result, res = connect.count_collection(collection_new)
assert res == nb
def test_add_flush_multiable_times(self, connect, table):
def test_add_flush_multiable_times(self, connect, collection):
'''
method: add vectors, flush serveral times
expected: status ok
'''
vectors = gen_vectors(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
for i in range(10):
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
query_vecs = [vectors[0], vectors[1], vectors[-1]]
status, res = connect.search_vectors(table, top_k, query_records=query_vecs)
status, res = connect.search_vectors(collection, top_k, query_records=query_vecs)
assert status.OK()
def test_add_flush_auto(self, connect, table):
def test_add_flush_auto(self, connect, collection):
'''
method: add vectors
expected: status ok
'''
vectors = gen_vectors(nb, dim)
ids = [i for i in range(nb)]
status, ids = connect.add_vectors(table, vectors, ids)
status, ids = connect.add_vectors(collection, vectors, ids)
assert status.OK()
time.sleep(2)
status, res = connect.get_table_row_count(table)
status, res = connect.count_collection(collection)
assert status.OK()
assert res == nb
@ -163,7 +163,7 @@ class TestFlushBase:
yield request.param
# both autoflush / flush
def test_add_flush_same_ids(self, connect, table, same_ids):
def test_add_flush_same_ids(self, connect, collection, same_ids):
'''
method: add vectors, with same ids, count(same ids) < 15, > 15
expected: status ok
@ -173,79 +173,79 @@ class TestFlushBase:
for i, item in enumerate(ids):
if item <= same_ids:
ids[i] = 0
status, ids = connect.add_vectors(table, vectors, ids)
status, ids = connect.add_vectors(collection, vectors, ids)
time.sleep(2)
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, res = connect.get_table_row_count(table)
status, res = connect.count_collection(collection)
assert status.OK()
assert res == nb
def test_delete_flush_multiable_times(self, connect, table):
def test_delete_flush_multiable_times(self, connect, collection):
'''
method: delete vectors, flush serveral times
expected: status ok
'''
vectors = gen_vectors(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.delete_by_id(table, [ids[-1]])
status = connect.delete_by_id(collection, [ids[-1]])
assert status.OK()
for i in range(10):
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
query_vecs = [vectors[0], vectors[1], vectors[-1]]
status, res = connect.search_vectors(table, top_k, query_records=query_vecs)
status, res = connect.search_vectors(collection, top_k, query_records=query_vecs)
assert status.OK()
# TODO: CI fail, LOCAL pass
def _test_table_count_during_flush(self, connect, args):
def _test_collection_count_during_flush(self, connect, args):
'''
method: flush table at background, call `get_table_row_count`
method: flush collection at background, call `count_collection`
expected: status ok
'''
table = gen_unique_str()
collection = gen_unique_str()
uri = "tcp://%s:%s" % (args["ip"], args["port"])
param = {'table_name': table,
param = {'collection_name': collection,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.L2}
milvus = get_milvus()
milvus.connect(uri=uri)
milvus.create_table(param)
milvus.create_collection(param)
vectors = gen_vector(nb, dim)
status, ids = milvus.add_vectors(table, vectors, ids=[i for i in range(nb)])
def flush(table_name):
status, ids = milvus.add_vectors(collection, vectors, ids=[i for i in range(nb)])
def flush(collection_name):
milvus = get_milvus()
milvus.connect(uri=uri)
status = milvus.delete_by_id(table_name, [i for i in range(nb)])
status = milvus.delete_by_id(collection_name, [i for i in range(nb)])
assert status.OK()
status = milvus.flush([table_name])
status = milvus.flush([collection_name])
assert status.OK()
p = Process(target=flush, args=(table, ))
p = Process(target=flush, args=(collection, ))
p.start()
status, res = milvus.get_table_row_count(table)
status, res = milvus.count_collection(collection)
assert status.OK()
p.join()
status, res = milvus.get_table_row_count(table)
status, res = milvus.count_collection(collection)
assert status.OK()
logging.getLogger().info(res)
assert res == 0
class TestTableNameInvalid(object):
class TestCollectionNameInvalid(object):
"""
Test adding vectors with invalid table names
Test adding vectors with invalid collection names
"""
@pytest.fixture(
scope="function",
params=gen_invalid_table_names()
params=gen_invalid_collection_names()
)
def get_table_name(self, request):
def get_collection_name(self, request):
yield request.param
@pytest.mark.level(2)
def test_flush_with_invalid_table_name(self, connect, get_table_name):
table_name = get_table_name
def test_flush_with_invalid_collection_name(self, connect, get_collection_name):
collection_name = get_collection_name
with pytest.raises(Exception) as e:
status, result = connect.flush(table_name)
status, result = connect.flush(collection_name)

190
tests/milvus_python_test/test_get_vector_by_id.py
View File

@ -11,7 +11,7 @@ from utils import *
dim = 128
index_file_size = 10
table_id = "test_get_vector_by_id"
collection_id = "test_get_vector_by_id"
DELETE_TIMEOUT = 60
nprobe = 1
tag = "1970-01-01"
@ -25,54 +25,54 @@ class TestGetBase:
The following cases are used to test `get_vector_by_id` function
******************************************************************
"""
def test_get_vector_A(self, connect, table):
def test_get_vector_A(self, connect, collection):
'''
target: test get_vector_by_id
method: add vector, and get
expected: status ok, vector returned
'''
vector = gen_single_vector(dim)
status, ids = connect.add_vectors(table, vector)
status, ids = connect.add_vectors(collection, vector)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, res = connect.get_vector_by_id(table, ids[0])
status, res = connect.get_vector_by_id(collection, ids[0])
assert status.OK()
assert_equal_vector(res, vector[0])
def test_get_vector_B(self, connect, table):
def test_get_vector_B(self, connect, collection):
'''
target: test get_vector_by_id
method: add vector, and get
expected: status ok, vector returned
'''
vectors = gen_vectors(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, res = connect.get_vector_by_id(table, ids[0])
status, res = connect.get_vector_by_id(collection, ids[0])
assert status.OK()
assert_equal_vector(res, vectors[0])
def test_get_vector_partition(self, connect, table):
def test_get_vector_partition(self, connect, collection):
'''
target: test get_vector_by_id
method: add vector, and get
expected: status ok, vector returned
'''
vectors = gen_vectors(nb, dim)
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
assert status.OK()
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, res = connect.get_vector_by_id(table, ids[0])
status, res = connect.get_vector_by_id(collection, ids[0])
assert status.OK()
assert_equal_vector(res, vectors[0])
def test_get_vector_multi_same_ids(self, connect, table):
def test_get_vector_multi_same_ids(self, connect, collection):
'''
target: test get_vector_by_id
method: add vectors, with the same id, get vector by the given id
@ -81,10 +81,10 @@ class TestGetBase:
vectors = gen_vectors(nb, dim)
ids = [i for i in range(nb)]
ids[1] = 0; ids[-1] = 0
status, ids = connect.add_vectors(table, vectors, ids=ids)
status = connect.flush([table])
status, ids = connect.add_vectors(collection, vectors, ids=ids)
status = connect.flush([collection])
assert status.OK()
status, res = connect.get_vector_by_id(table, 0)
status, res = connect.get_vector_by_id(collection, 0)
assert status.OK()
assert_equal_vector(res, vectors[0])
@ -101,75 +101,75 @@ class TestGetBase:
def get_id(self, request):
yield request.param
def test_get_vector_after_delete(self, connect, table, get_id):
def test_get_vector_after_delete(self, connect, collection, get_id):
'''
target: test get_vector_by_id
method: add vectors, and delete, get vector by the given id
expected: status ok, get one vector
'''
vectors = gen_vectors(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
id = get_id
status = connect.delete_by_id(table, [ids[id]])
status = connect.delete_by_id(collection, [ids[id]])
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, res = connect.get_vector_by_id(table, ids[id])
status, res = connect.get_vector_by_id(collection, ids[id])
assert status.OK()
assert not res
def test_get_vector_after_delete_with_partition(self, connect, table, get_id):
def test_get_vector_after_delete_with_partition(self, connect, collection, get_id):
'''
target: test get_vector_by_id
method: add vectors into partition, and delete, get vector by the given id
expected: status ok, get one vector
'''
vectors = gen_vectors(nb, dim)
status = connect.create_partition(table, tag)
status, ids = connect.insert(table, vectors, partition_tag=tag)
status = connect.create_partition(collection, tag)
status, ids = connect.insert(collection, vectors, partition_tag=tag)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
id = get_id
status = connect.delete_by_id(table, [ids[id]])
status = connect.delete_by_id(collection, [ids[id]])
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, res = connect.get_vector_by_id(table, ids[id])
status, res = connect.get_vector_by_id(collection, ids[id])
assert status.OK()
assert not res
def test_get_vector_id_not_exised(self, connect, table):
def test_get_vector_id_not_exised(self, connect, collection):
'''
target: test get vector, params vector_id not existed
method: add vector and get
expected: status ok, empty result
'''
vector = gen_single_vector(dim)
status, ids = connect.add_vectors(table, vector)
status, ids = connect.add_vectors(collection, vector)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, res = connect.get_vector_by_id(table, 1)
status, res = connect.get_vector_by_id(collection, 1)
assert status.OK()
assert not res
def test_get_vector_table_not_existed(self, connect, table):
def test_get_vector_collection_not_existed(self, connect, collection):
'''
target: test get vector, params table_name not existed
target: test get vector, params collection_name not existed
method: add vector and get
expected: status not ok
'''
vector = gen_single_vector(dim)
status, ids = connect.add_vectors(table, vector)
status, ids = connect.add_vectors(collection, vector)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
table_new = gen_unique_str()
status, res = connect.get_vector_by_id(table_new, 1)
collection_new = gen_unique_str()
status, res = connect.get_vector_by_id(collection_new, 1)
assert not status.OK()
@ -205,7 +205,7 @@ class TestGetIndexedVectors:
def get_id(self, request):
yield request.param
def test_get_vectors_after_index_created(self, connect, table, get_simple_index, get_id):
def test_get_vectors_after_index_created(self, connect, collection, get_simple_index, get_id):
'''
target: test get vector after index created
method: add vector, create index and get vector
@ -214,20 +214,20 @@ class TestGetIndexedVectors:
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status = connect.create_index(table, index_type, index_param)
status = connect.create_index(collection, index_type, index_param)
assert status.OK()
id = get_id
status, res = connect.get_vector_by_id(table, ids[id])
status, res = connect.get_vector_by_id(collection, ids[id])
assert status.OK()
logging.getLogger().info(res)
assert status.OK()
assert_equal_vector(res, vectors[id])
def test_get_vector_after_delete(self, connect, table, get_simple_index, get_id):
def test_get_vector_after_delete(self, connect, collection, get_simple_index, get_id):
'''
target: test get_vector_by_id
method: add vectors, and delete, get vector by the given id
@ -236,22 +236,22 @@ class TestGetIndexedVectors:
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
vectors = gen_vectors(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status = connect.create_index(table, index_type, index_param)
status = connect.create_index(collection, index_type, index_param)
assert status.OK()
id = get_id
status = connect.delete_by_id(table, [ids[id]])
status = connect.delete_by_id(collection, [ids[id]])
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, res = connect.get_vector_by_id(table, ids[id])
status, res = connect.get_vector_by_id(collection, ids[id])
assert status.OK()
assert not res
def test_get_vector_partition(self, connect, table, get_simple_index, get_id):
def test_get_vector_partition(self, connect, collection, get_simple_index, get_id):
'''
target: test get_vector_by_id
method: add vector, and get
@ -260,16 +260,16 @@ class TestGetIndexedVectors:
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
vectors = gen_vectors(nb, dim)
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
ids = [i for i in range(nb)]
status, ids = connect.add_vectors(table, vectors, ids, partition_tag=tag)
status, ids = connect.add_vectors(collection, vectors, ids, partition_tag=tag)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status = connect.create_index(table, index_type, index_param)
status = connect.create_index(collection, index_type, index_param)
assert status.OK()
id = get_id
status, res = connect.get_vector_by_id(table, ids[id])
status, res = connect.get_vector_by_id(collection, ids[id])
assert status.OK()
assert_equal_vector(res, vectors[id])
@ -280,37 +280,37 @@ class TestGetBinary:
The following cases are used to test `get_vector_by_id` function
******************************************************************
"""
def test_get_vector_A(self, connect, jac_table):
def test_get_vector_A(self, connect, jac_collection):
'''
target: test get_vector_by_id
method: add vector, and get
expected: status ok, vector returned
'''
tmp, vector = gen_binary_vectors(1, dim)
status, ids = connect.add_vectors(jac_table, vector)
status, ids = connect.add_vectors(jac_collection, vector)
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
assert status.OK()
status, res = connect.get_vector_by_id(jac_table, ids[0])
status, res = connect.get_vector_by_id(jac_collection, ids[0])
assert status.OK()
assert res == vector[0]
def test_get_vector_B(self, connect, jac_table):
def test_get_vector_B(self, connect, jac_collection):
'''
target: test get_vector_by_id
method: add vector, and get
expected: status ok, vector returned
'''
tmp, vectors = gen_binary_vectors(nb, dim)
status, ids = connect.add_vectors(jac_table, vectors)
status, ids = connect.add_vectors(jac_collection, vectors)
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
assert status.OK()
status, res = connect.get_vector_by_id(jac_table, ids[0])
status, res = connect.get_vector_by_id(jac_collection, ids[0])
assert status.OK()
assert res == vectors[0]
def test_get_vector_multi_same_ids(self, connect, jac_table):
def test_get_vector_multi_same_ids(self, connect, jac_collection):
'''
target: test get_vector_by_id
method: add vectors, with the same id, get vector by the given id
@ -319,56 +319,56 @@ class TestGetBinary:
tmp, vectors = gen_binary_vectors(nb, dim)
ids = [i for i in range(nb)]
ids[0] = 0; ids[-1] = 0
status, ids = connect.add_vectors(jac_table, vectors, ids=ids)
status = connect.flush([jac_table])
status, ids = connect.add_vectors(jac_collection, vectors, ids=ids)
status = connect.flush([jac_collection])
assert status.OK()
status, res = connect.get_vector_by_id(jac_table, 0)
status, res = connect.get_vector_by_id(jac_collection, 0)
assert status.OK()
assert res == vectors[0]
def test_get_vector_id_not_exised(self, connect, jac_table):
def test_get_vector_id_not_exised(self, connect, jac_collection):
'''
target: test get vector, params vector_id not existed
method: add vector and get
expected: status ok, empty result
'''
tmp, vector = gen_binary_vectors(1, dim)
status, ids = connect.add_vectors(jac_table, vector)
status, ids = connect.add_vectors(jac_collection, vector)
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
assert status.OK()
status, res = connect.get_vector_by_id(jac_table, 1)
status, res = connect.get_vector_by_id(jac_collection, 1)
assert status.OK()
assert not res
def test_get_vector_table_not_existed(self, connect, jac_table):
def test_get_vector_collection_not_existed(self, connect, jac_collection):
'''
target: test get vector, params table_name not existed
target: test get vector, params collection_name not existed
method: add vector and get
expected: status not ok
'''
tmp, vector = gen_binary_vectors(1, dim)
status, ids = connect.add_vectors(jac_table, vector)
status, ids = connect.add_vectors(jac_collection, vector)
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
assert status.OK()
table_new = gen_unique_str()
status, res = connect.get_vector_by_id(table_new, 1)
collection_new = gen_unique_str()
status, res = connect.get_vector_by_id(collection_new, 1)
assert not status.OK()
def test_get_vector_partition(self, connect, jac_table):
def test_get_vector_partition(self, connect, jac_collection):
'''
target: test get_vector_by_id
method: add vector, and get
expected: status ok, vector returned
'''
tmp, vectors = gen_binary_vectors(nb, dim)
status = connect.create_partition(jac_table, tag)
status, ids = connect.add_vectors(jac_table, vectors, partition_tag=tag)
status = connect.create_partition(jac_collection, tag)
status, ids = connect.add_vectors(jac_collection, vectors, partition_tag=tag)
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
assert status.OK()
status, res = connect.get_vector_by_id(jac_table, ids[0])
status, res = connect.get_vector_by_id(jac_collection, ids[0])
assert status.OK()
assert res == vectors[0]
@ -387,26 +387,26 @@ class TestGetVectorIdIngalid(object):
yield request.param
@pytest.mark.level(2)
def test_get_vector_id_invalid(self, connect, table, gen_invalid_id):
def test_get_vector_id_invalid(self, connect, collection, gen_invalid_id):
invalid_id = gen_invalid_id
with pytest.raises(Exception) as e:
status = connect.get_vector_by_id(table, invalid_id)
status = connect.get_vector_by_id(collection, invalid_id)
class TestTableNameInvalid(object):
class TestCollectionNameInvalid(object):
"""
Test adding vectors with invalid table names
Test adding vectors with invalid collection names
"""
@pytest.fixture(
scope="function",
params=gen_invalid_table_names()
params=gen_invalid_collection_names()
)
def get_table_name(self, request):
def get_collection_name(self, request):
yield request.param
@pytest.mark.level(2)
def test_get_vectors_with_invalid_table_name(self, connect, get_table_name):
table_name = get_table_name
def test_get_vectors_with_invalid_collection_name(self, connect, get_collection_name):
collection_name = get_collection_name
vectors = gen_vectors(1, dim)
status, result = connect.get_vector_by_id(table_name, 1)
status, result = connect.get_vector_by_id(collection_name, 1)
assert not status.OK()

240
tests/milvus_python_test/test_get_vector_ids.py
View File

@ -19,13 +19,13 @@ nb = 6000
class TestGetVectorIdsBase:
def get_valid_segment_name(self, connect, table):
def get_valid_segment_name(self, connect, collection):
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
logging.getLogger().info(info.partitions_stat[0].segments_stat[0].segment_name)
return info.partitions_stat[0].segments_stat[0].segment_name
@ -36,88 +36,88 @@ class TestGetVectorIdsBase:
******************************************************************
"""
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_table_name_None(self, connect, table):
def test_get_vector_ids_collection_name_None(self, connect, collection):
'''
target: get vector ids where table name is None
method: call get_vector_ids with the table_name: None
target: get vector ids where collection name is None
method: call get_vector_ids with the collection_name: None
expected: exception raised
'''
table_name = None
segment_name = self.get_valid_segment_name(connect, table)
collection_name = None
segment_name = self.get_valid_segment_name(connect, collection)
with pytest.raises(Exception) as e:
status, vector_ids = connect.get_vector_ids(table_name, segment_name)
status, vector_ids = connect.get_vector_ids(collection_name, segment_name)
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_table_name_not_existed(self, connect, table):
def test_get_vector_ids_collection_name_not_existed(self, connect, collection):
'''
target: get vector ids where table name does not exist
method: call get_vector_ids with a random table_name, which is not in db
target: get vector ids where collection name does not exist
method: call get_vector_ids with a random collection_name, which is not in db
expected: status not ok
'''
table_name = gen_unique_str("not_existed_table")
segment_name = self.get_valid_segment_name(connect, table)
status, vector_ids = connect.get_vector_ids(table_name, segment_name)
collection_name = gen_unique_str("not_existed_collection")
segment_name = self.get_valid_segment_name(connect, collection)
status, vector_ids = connect.get_vector_ids(collection_name, segment_name)
assert not status.OK()
@pytest.fixture(
scope="function",
params=gen_invalid_table_names()
params=gen_invalid_collection_names()
)
def get_table_name(self, request):
def get_collection_name(self, request):
yield request.param
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_table_name_invalid(self, connect, table, get_table_name):
def test_get_vector_ids_collection_name_invalid(self, connect, collection, get_collection_name):
'''
target: get vector ids where table name is invalid
method: call get_vector_ids with invalid table_name
target: get vector ids where collection name is invalid
method: call get_vector_ids with invalid collection_name
expected: status not ok
'''
table_name = get_table_name
segment_name = self.get_valid_segment_name(connect, table)
status, vector_ids = connect.get_vector_ids(table_name, segment_name)
collection_name = get_collection_name
segment_name = self.get_valid_segment_name(connect, collection)
status, vector_ids = connect.get_vector_ids(collection_name, segment_name)
assert not status.OK()
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_segment_name_None(self, connect, table):
def test_get_vector_ids_segment_name_None(self, connect, collection):
'''
target: get vector ids where segment name is None
method: call get_vector_ids with the segment_name: None
expected: exception raised
'''
valid_segment_name = self.get_valid_segment_name(connect, table)
valid_segment_name = self.get_valid_segment_name(connect, collection)
segment = None
with pytest.raises(Exception) as e:
status, vector_ids = connect.get_vector_ids(table, segment)
status, vector_ids = connect.get_vector_ids(collection, segment)
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_segment_name_not_existed(self, connect, table):
def test_get_vector_ids_segment_name_not_existed(self, connect, collection):
'''
target: get vector ids where segment name does not exist
method: call get_vector_ids with a random segment name
expected: status not ok
'''
valid_segment_name = self.get_valid_segment_name(connect, table)
valid_segment_name = self.get_valid_segment_name(connect, collection)
segment = gen_unique_str("not_existed_segment")
status, vector_ids = connect.get_vector_ids(table, segment)
status, vector_ids = connect.get_vector_ids(collection, segment)
logging.getLogger().info(vector_ids)
assert not status.OK()
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_without_index_A(self, connect, table):
def test_get_vector_ids_without_index_A(self, connect, collection):
'''
target: get vector ids when there is no index
method: call get_vector_ids and check if the segment contains vectors
expected: status ok
'''
vectors = gen_vector(10, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[0].segments_stat[0].segment_name)
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[0].segments_stat[0].segment_name)
# vector_ids should match ids
assert len(vector_ids) == 10
for i in range(10):
@ -125,23 +125,23 @@ class TestGetVectorIdsBase:
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_without_index_B(self, connect, table):
def test_get_vector_ids_without_index_B(self, connect, collection):
'''
target: get vector ids when there is no index but with partition
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
expected: status ok
'''
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
assert status.OK()
vectors = gen_vector(10, dim)
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
assert info.partitions_stat[1].tag == tag
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[1].segments_stat[0].segment_name)
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[1].segments_stat[0].segment_name)
# vector_ids should match ids
assert len(vector_ids) == 10
for i in range(10):
@ -160,7 +160,7 @@ class TestGetVectorIdsBase:
return request.param
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_with_index_A(self, connect, table, get_simple_index):
def test_get_vector_ids_with_index_A(self, connect, collection, get_simple_index):
'''
target: get vector ids when there is index
method: call get_vector_ids and check if the segment contains vectors
@ -168,65 +168,65 @@ class TestGetVectorIdsBase:
'''
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
status = connect.create_index(table, index_type, index_param)
status = connect.create_index(collection, index_type, index_param)
assert status.OK()
vectors = gen_vector(10, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[0].segments_stat[0].segment_name)
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[0].segments_stat[0].segment_name)
# vector_ids should match ids
assert len(vector_ids) == 10
for i in range(10):
assert vector_ids[i] == ids[i]
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_with_index_B(self, connect, table, get_simple_index):
def test_get_vector_ids_with_index_B(self, connect, collection, get_simple_index):
'''
target: get vector ids when there is index and with partition
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
expected: status ok
'''
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
assert status.OK()
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
status = connect.create_index(table, index_type, index_param)
status = connect.create_index(collection, index_type, index_param)
assert status.OK()
vectors = gen_vector(10, dim)
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
assert info.partitions_stat[1].tag == tag
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[1].segments_stat[0].segment_name)
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[1].segments_stat[0].segment_name)
# vector_ids should match ids
assert len(vector_ids) == 10
for i in range(10):
assert vector_ids[i] == ids[i]
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_after_delete_vectors(self, connect, table):
def test_get_vector_ids_after_delete_vectors(self, connect, collection):
'''
target: get vector ids after vectors are deleted
method: add vectors and delete a few, call get_vector_ids
expected: status ok, vector_ids decreased after vectors deleted
'''
vectors = gen_vector(2, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
delete_ids = [ids[0]]
status = connect.delete_by_id(table, delete_ids)
status = connect.flush([table])
status = connect.delete_by_id(collection, delete_ids)
status = connect.flush([collection])
assert status.OK()
status, info = connect.table_info(table)
status, info = connect.collection_info(collection)
assert status.OK()
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[0].segments_stat[0].segment_name)
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[0].segments_stat[0].segment_name)
assert len(vector_ids) == 1
assert vector_ids[0] == ids[1]
@ -238,20 +238,20 @@ class TestGetVectorIdsIP:
******************************************************************
"""
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_without_index_A(self, connect, ip_table):
def test_get_vector_ids_without_index_A(self, connect, ip_collection):
'''
target: get vector ids when there is no index
method: call get_vector_ids and check if the segment contains vectors
expected: status ok
'''
vectors = gen_vector(10, dim)
status, ids = connect.add_vectors(ip_table, vectors)
status, ids = connect.add_vectors(ip_collection, vectors)
assert status.OK()
status = connect.flush([ip_table])
status = connect.flush([ip_collection])
assert status.OK()
status, info = connect.table_info(ip_table)
status, info = connect.collection_info(ip_collection)
assert status.OK()
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[0].segments_stat[0].segment_name)
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[0].segments_stat[0].segment_name)
# vector_ids should match ids
assert len(vector_ids) == 10
for i in range(10):
@ -259,23 +259,23 @@ class TestGetVectorIdsIP:
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_without_index_B(self, connect, ip_table):
def test_get_vector_ids_without_index_B(self, connect, ip_collection):
'''
target: get vector ids when there is no index but with partition
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
expected: status ok
'''
status = connect.create_partition(ip_table, tag)
status = connect.create_partition(ip_collection, tag)
assert status.OK()
vectors = gen_vector(10, dim)
status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag)
status, ids = connect.add_vectors(ip_collection, vectors, partition_tag=tag)
assert status.OK()
status = connect.flush([ip_table])
status = connect.flush([ip_collection])
assert status.OK()
status, info = connect.table_info(ip_table)
status, info = connect.collection_info(ip_collection)
assert status.OK()
assert info.partitions_stat[1].tag == tag
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[1].segments_stat[0].segment_name)
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[1].segments_stat[0].segment_name)
# vector_ids should match ids
assert len(vector_ids) == 10
for i in range(10):
@ -294,7 +294,7 @@ class TestGetVectorIdsIP:
return request.param
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_with_index_A(self, connect, ip_table, get_simple_index):
def test_get_vector_ids_with_index_A(self, connect, ip_collection, get_simple_index):
'''
target: get vector ids when there is index
method: call get_vector_ids and check if the segment contains vectors
@ -302,65 +302,65 @@ class TestGetVectorIdsIP:
'''
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
status = connect.create_index(ip_table, index_type, index_param)
status = connect.create_index(ip_collection, index_type, index_param)
assert status.OK()
vectors = gen_vector(10, dim)
status, ids = connect.add_vectors(ip_table, vectors)
status, ids = connect.add_vectors(ip_collection, vectors)
assert status.OK()
status = connect.flush([ip_table])
status = connect.flush([ip_collection])
assert status.OK()
status, info = connect.table_info(ip_table)
status, info = connect.collection_info(ip_collection)
assert status.OK()
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[0].segments_stat[0].segment_name)
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[0].segments_stat[0].segment_name)
# vector_ids should match ids
assert len(vector_ids) == 10
for i in range(10):
assert vector_ids[i] == ids[i]
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_with_index_B(self, connect, ip_table, get_simple_index):
def test_get_vector_ids_with_index_B(self, connect, ip_collection, get_simple_index):
'''
target: get vector ids when there is index and with partition
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
expected: status ok
'''
status = connect.create_partition(ip_table, tag)
status = connect.create_partition(ip_collection, tag)
assert status.OK()
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
status = connect.create_index(ip_table, index_type, index_param)
status = connect.create_index(ip_collection, index_type, index_param)
assert status.OK()
vectors = gen_vector(10, dim)
status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag)
status, ids = connect.add_vectors(ip_collection, vectors, partition_tag=tag)
assert status.OK()
status = connect.flush([ip_table])
status = connect.flush([ip_collection])
assert status.OK()
status, info = connect.table_info(ip_table)
status, info = connect.collection_info(ip_collection)
assert status.OK()
assert info.partitions_stat[1].tag == tag
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[1].segments_stat[0].segment_name)
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[1].segments_stat[0].segment_name)
# vector_ids should match ids
assert len(vector_ids) == 10
for i in range(10):
assert vector_ids[i] == ids[i]
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_after_delete_vectors(self, connect, ip_table):
def test_get_vector_ids_after_delete_vectors(self, connect, ip_collection):
'''
target: get vector ids after vectors are deleted
method: add vectors and delete a few, call get_vector_ids
expected: status ok, vector_ids decreased after vectors deleted
'''
vectors = gen_vector(2, dim)
status, ids = connect.add_vectors(ip_table, vectors)
status, ids = connect.add_vectors(ip_collection, vectors)
assert status.OK()
delete_ids = [ids[0]]
status = connect.delete_by_id(ip_table, delete_ids)
status = connect.flush([ip_table])
status = connect.delete_by_id(ip_collection, delete_ids)
status = connect.flush([ip_collection])
assert status.OK()
status, info = connect.table_info(ip_table)
status, info = connect.collection_info(ip_collection)
assert status.OK()
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[0].segments_stat[0].segment_name)
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[0].segments_stat[0].segment_name)
assert len(vector_ids) == 1
assert vector_ids[0] == ids[1]
@ -372,20 +372,20 @@ class TestGetVectorIdsJAC:
******************************************************************
"""
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_without_index_A(self, connect, jac_table):
def test_get_vector_ids_without_index_A(self, connect, jac_collection):
'''
target: get vector ids when there is no index
method: call get_vector_ids and check if the segment contains vectors
expected: status ok
'''
tmp, vectors = gen_binary_vectors(10, dim)
status, ids = connect.add_vectors(jac_table, vectors)
status, ids = connect.add_vectors(jac_collection, vectors)
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
assert status.OK()
status, info = connect.table_info(jac_table)
status, info = connect.collection_info(jac_collection)
assert status.OK()
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[0].segments_stat[0].segment_name)
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[0].segments_stat[0].segment_name)
# vector_ids should match ids
assert len(vector_ids) == 10
for i in range(10):
@ -393,23 +393,23 @@ class TestGetVectorIdsJAC:
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_without_index_B(self, connect, jac_table):
def test_get_vector_ids_without_index_B(self, connect, jac_collection):
'''
target: get vector ids when there is no index but with partition
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
expected: status ok
'''
status = connect.create_partition(jac_table, tag)
status = connect.create_partition(jac_collection, tag)
assert status.OK()
tmp, vectors = gen_binary_vectors(10, dim)
status, ids = connect.add_vectors(jac_table, vectors, partition_tag=tag)
status, ids = connect.add_vectors(jac_collection, vectors, partition_tag=tag)
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
assert status.OK()
status, info = connect.table_info(jac_table)
status, info = connect.collection_info(jac_collection)
assert status.OK()
assert info.partitions_stat[1].tag == tag
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[1].segments_stat[0].segment_name)
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[1].segments_stat[0].segment_name)
# vector_ids should match ids
assert len(vector_ids) == 10
for i in range(10):
@ -427,7 +427,7 @@ class TestGetVectorIdsJAC:
pytest.skip("Skip index Temporary")
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_with_index_A(self, connect, jac_table, get_jaccard_index):
def test_get_vector_ids_with_index_A(self, connect, jac_collection, get_jaccard_index):
'''
target: get vector ids when there is index
method: call get_vector_ids and check if the segment contains vectors
@ -435,64 +435,64 @@ class TestGetVectorIdsJAC:
'''
index_param = get_jaccard_index["index_param"]
index_type = get_jaccard_index["index_type"]
status = connect.create_index(jac_table, index_type, index_param)
status = connect.create_index(jac_collection, index_type, index_param)
assert status.OK()
tmp, vectors = gen_binary_vectors(10, dim)
status, ids = connect.add_vectors(jac_table, vectors)
status, ids = connect.add_vectors(jac_collection, vectors)
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
assert status.OK()
status, info = connect.table_info(jac_table)
status, info = connect.collection_info(jac_collection)
assert status.OK()
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[0].segments_stat[0].segment_name)
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[0].segments_stat[0].segment_name)
# vector_ids should match ids
assert len(vector_ids) == 10
for i in range(10):
assert vector_ids[i] == ids[i]
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_with_index_B(self, connect, jac_table, get_jaccard_index):
def test_get_vector_ids_with_index_B(self, connect, jac_collection, get_jaccard_index):
'''
target: get vector ids when there is index and with partition
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
expected: status ok
'''
status = connect.create_partition(jac_table, tag)
status = connect.create_partition(jac_collection, tag)
assert status.OK()
index_param = get_jaccard_index["index_param"]
index_type = get_jaccard_index["index_type"]
status = connect.create_index(jac_table, index_type, index_param)
status = connect.create_index(jac_collection, index_type, index_param)
assert status.OK()
tmp, vectors = gen_binary_vectors(10, dim)
status, ids = connect.add_vectors(jac_table, vectors, partition_tag=tag)
status, ids = connect.add_vectors(jac_collection, vectors, partition_tag=tag)
assert status.OK()
status = connect.flush([jac_table])
status = connect.flush([jac_collection])
assert status.OK()
status, info = connect.table_info(jac_table)
status, info = connect.collection_info(jac_collection)
assert status.OK()
assert info.partitions_stat[1].tag == tag
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[1].segments_stat[0].segment_name)
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[1].segments_stat[0].segment_name)
# vector_ids should match ids
assert len(vector_ids) == 10
for i in range(10):
assert vector_ids[i] == ids[i]
@pytest.mark.timeout(GET_TIMEOUT)
def test_get_vector_ids_after_delete_vectors(self, connect, jac_table):
def test_get_vector_ids_after_delete_vectors(self, connect, jac_collection):
'''
target: get vector ids after vectors are deleted
method: add vectors and delete a few, call get_vector_ids
expected: status ok, vector_ids decreased after vectors deleted
'''
tmp, vectors = gen_binary_vectors(2, dim)
status, ids = connect.add_vectors(jac_table, vectors)
status, ids = connect.add_vectors(jac_collection, vectors)
assert status.OK()
delete_ids = [ids[0]]
status = connect.delete_by_id(jac_table, delete_ids)
status = connect.flush([jac_table])
status = connect.delete_by_id(jac_collection, delete_ids)
status = connect.flush([jac_collection])
assert status.OK()
status, info = connect.table_info(jac_table)
status, info = connect.collection_info(jac_collection)
assert status.OK()
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[0].segments_stat[0].segment_name)
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[0].segments_stat[0].segment_name)
assert len(vector_ids) == 1
assert vector_ids[0] == ids[1]

904
tests/milvus_python_test/test_index.py
View File

File diff suppressed because it is too large Load Diff

76
tests/milvus_python_test/test_mix.py
View File

@ -12,7 +12,7 @@ from utils import *
dim = 128
index_file_size = 10
table_id = "test_mix"
collection_id = "test_mix"
add_interval_time = 2
vectors = gen_vectors(10000, dim)
vectors = sklearn.preprocessing.normalize(vectors, axis=1, norm='l2')
@ -29,35 +29,35 @@ class TestMixBase:
# disable
def _test_search_during_createIndex(self, args):
loops = 10000
table = gen_unique_str()
collection = gen_unique_str()
query_vecs = [vectors[0], vectors[1]]
uri = "tcp://%s:%s" % (args["ip"], args["port"])
id_0 = 0; id_1 = 0
milvus_instance = get_milvus(args["handler"])
milvus_instance.connect(uri=uri)
milvus_instance.create_table({'table_name': table,
milvus_instance.create_collection({'collection_name': collection,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.L2})
for i in range(10):
status, ids = milvus_instance.add_vectors(table, vectors)
status, ids = milvus_instance.add_vectors(collection, vectors)
# logging.getLogger().info(ids)
if i == 0:
id_0 = ids[0]; id_1 = ids[1]
def create_index(milvus_instance):
logging.getLogger().info("In create index")
status = milvus_instance.create_index(table, index_params)
status = milvus_instance.create_index(collection, index_params)
logging.getLogger().info(status)
status, result = milvus_instance.describe_index(table)
status, result = milvus_instance.describe_index(collection)
logging.getLogger().info(result)
def add_vectors(milvus_instance):
logging.getLogger().info("In add vectors")
status, ids = milvus_instance.add_vectors(table, vectors)
status, ids = milvus_instance.add_vectors(collection, vectors)
logging.getLogger().info(status)
def search(milvus_instance):
logging.getLogger().info("In search vectors")
for i in range(loops):
status, result = milvus_instance.search_vectors(table, top_k, nprobe, query_vecs)
status, result = milvus_instance.search_vectors(collection, top_k, nprobe, query_vecs)
logging.getLogger().info(status)
assert result[0][0].id == id_0
assert result[1][0].id == id_1
@ -72,82 +72,82 @@ class TestMixBase:
p_create.join()
@pytest.mark.level(2)
def test_mix_multi_tables(self, connect):
def test_mix_multi_collections(self, connect):
'''
target: test functions with multiple tables of different metric_types and index_types
method: create 60 tables which 30 are L2 and the other are IP, add vectors into them
target: test functions with multiple collections of different metric_types and index_types
method: create 60 collections which 30 are L2 and the other are IP, add vectors into them
and test describe index and search
expected: status ok
'''
nq = 10000
table_list = []
collection_list = []
idx = []
index_param = {'nlist': nlist}
#create table and add vectors
#create collection and add vectors
for i in range(30):
table_name = gen_unique_str('test_mix_multi_tables')
table_list.append(table_name)
param = {'table_name': table_name,
collection_name = gen_unique_str('test_mix_multi_collections')
collection_list.append(collection_name)
param = {'collection_name': collection_name,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.L2}
connect.create_table(param)
status, ids = connect.add_vectors(table_name=table_name, records=vectors)
connect.create_collection(param)
status, ids = connect.add_vectors(collection_name=collection_name, records=vectors)
idx.append(ids[0])
idx.append(ids[10])
idx.append(ids[20])
assert status.OK()
for i in range(30):
table_name = gen_unique_str('test_mix_multi_tables')
table_list.append(table_name)
param = {'table_name': table_name,
collection_name = gen_unique_str('test_mix_multi_collections')
collection_list.append(collection_name)
param = {'collection_name': collection_name,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.IP}
connect.create_table(param)
status, ids = connect.add_vectors(table_name=table_name, records=vectors)
connect.create_collection(param)
status, ids = connect.add_vectors(collection_name=collection_name, records=vectors)
assert status.OK()
status = connect.flush([table_name])
status = connect.flush([collection_name])
assert status.OK()
idx.append(ids[0])
idx.append(ids[10])
idx.append(ids[20])
assert status.OK()
for i in range(10):
status = connect.create_index(table_list[i], IndexType.FLAT, index_param)
status = connect.create_index(collection_list[i], IndexType.FLAT, index_param)
assert status.OK()
status = connect.create_index(table_list[30 + i], IndexType.FLAT, index_param)
status = connect.create_index(collection_list[30 + i], IndexType.FLAT, index_param)
assert status.OK()
status = connect.create_index(table_list[10 + i], IndexType.IVFLAT, index_param)
status = connect.create_index(collection_list[10 + i], IndexType.IVFLAT, index_param)
assert status.OK()
status = connect.create_index(table_list[40 + i], IndexType.IVFLAT, index_param)
status = connect.create_index(collection_list[40 + i], IndexType.IVFLAT, index_param)
assert status.OK()
status = connect.create_index(table_list[20 + i], IndexType.IVF_SQ8, index_param)
status = connect.create_index(collection_list[20 + i], IndexType.IVF_SQ8, index_param)
assert status.OK()
status = connect.create_index(table_list[50 + i], IndexType.IVF_SQ8, index_param)
status = connect.create_index(collection_list[50 + i], IndexType.IVF_SQ8, index_param)
assert status.OK()
#describe index
for i in range(10):
status, result = connect.describe_index(table_list[i])
status, result = connect.describe_index(collection_list[i])
assert result._index_type == IndexType.FLAT
status, result = connect.describe_index(table_list[10 + i])
status, result = connect.describe_index(collection_list[10 + i])
assert result._index_type == IndexType.IVFLAT
status, result = connect.describe_index(table_list[20 + i])
status, result = connect.describe_index(collection_list[20 + i])
assert result._index_type == IndexType.IVF_SQ8
status, result = connect.describe_index(table_list[30 + i])
status, result = connect.describe_index(collection_list[30 + i])
assert result._index_type == IndexType.FLAT
status, result = connect.describe_index(table_list[40 + i])
status, result = connect.describe_index(collection_list[40 + i])
assert result._index_type == IndexType.IVFLAT
status, result = connect.describe_index(table_list[50 + i])
status, result = connect.describe_index(collection_list[50 + i])
assert result._index_type == IndexType.IVF_SQ8
#search
query_vecs = [vectors[0], vectors[10], vectors[20]]
for i in range(60):
table = table_list[i]
status, result = connect.search_vectors(table, top_k, query_records=query_vecs, params={"nprobe": 1})
collection = collection_list[i]
status, result = connect.search_vectors(collection, top_k, query_records=query_vecs, params={"nprobe": 1})
assert status.OK()
assert len(result) == len(query_vecs)
logging.getLogger().info(i)

184
tests/milvus_python_test/test_partition.py
View File

@ -11,7 +11,7 @@ from utils import *
dim = 128
index_file_size = 10
table_id = "test_partition"
collection_id = "test_partition"
ADD_TIMEOUT = 60
nprobe = 1
tag = "1970-01-01"
@ -24,58 +24,58 @@ class TestCreateBase:
The following cases are used to test `create_partition` function
******************************************************************
"""
def test_create_partition(self, connect, table):
def test_create_partition(self, connect, collection):
'''
target: test create partition, check status returned
method: call function: create_partition
expected: status ok
'''
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
assert status.OK()
def test_create_partition_repeat(self, connect, table):
def test_create_partition_repeat(self, connect, collection):
'''
target: test create partition, check status returned
method: call function: create_partition
expected: status ok
'''
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
assert status.OK()
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
assert not status.OK()
def test_create_partition_table_not_existed(self, connect):
def test_create_partition_collection_not_existed(self, connect):
'''
target: test create partition, its owner table name not existed in db, check status returned
target: test create partition, its owner collection name not existed in db, check status returned
method: call function: create_partition
expected: status not ok
'''
table_name = gen_unique_str()
status = connect.create_partition(table_name, tag)
collection_name = gen_unique_str()
status = connect.create_partition(collection_name, tag)
assert not status.OK()
def test_create_partition_tag_name_None(self, connect, table):
def test_create_partition_tag_name_None(self, connect, collection):
'''
target: test create partition, tag name set None, check status returned
method: call function: create_partition
expected: status ok
'''
tag_name = None
status = connect.create_partition(table, tag_name)
status = connect.create_partition(collection, tag_name)
assert not status.OK()
def test_create_different_partition_tags(self, connect, table):
def test_create_different_partition_tags(self, connect, collection):
'''
target: test create partition twice with different names
method: call function: create_partition, and again
expected: status ok
'''
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
assert status.OK()
tag_name = gen_unique_str()
status = connect.create_partition(table, tag_name)
status = connect.create_partition(collection, tag_name)
assert status.OK()
status, res = connect.show_partitions(table)
status, res = connect.show_partitions(collection)
assert status.OK()
tag_list = []
for item in res:
@ -84,95 +84,95 @@ class TestCreateBase:
assert tag_name in tag_list
assert "_default" in tag_list
def test_create_partition_add_vectors_default(self, connect, table):
def test_create_partition_add_vectors_default(self, connect, collection):
'''
target: test create partition, and insert vectors, check status returned
method: call function: create_partition
expected: status ok
'''
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
assert status.OK()
nq = 100
vectors = gen_vectors(nq, dim)
ids = [i for i in range(nq)]
status, ids = connect.insert(table, vectors, ids)
status, ids = connect.insert(collection, vectors, ids)
assert status.OK()
def test_create_partition_insert_with_tag(self, connect, table):
def test_create_partition_insert_with_tag(self, connect, collection):
'''
target: test create partition, and insert vectors, check status returned
method: call function: create_partition
expected: status ok
'''
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
assert status.OK()
nq = 100
vectors = gen_vectors(nq, dim)
ids = [i for i in range(nq)]
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
assert status.OK()
def test_create_partition_insert_with_tag_not_existed(self, connect, table):
def test_create_partition_insert_with_tag_not_existed(self, connect, collection):
'''
target: test create partition, and insert vectors, check status returned
method: call function: create_partition
expected: status not ok
'''
tag_new = "tag_new"
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
assert status.OK()
nq = 100
vectors = gen_vectors(nq, dim)
ids = [i for i in range(nq)]
status, ids = connect.insert(table, vectors, ids, partition_tag=tag_new)
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag_new)
assert not status.OK()
def test_create_partition_insert_same_tags(self, connect, table):
def test_create_partition_insert_same_tags(self, connect, collection):
'''
target: test create partition, and insert vectors, check status returned
method: call function: create_partition
expected: status ok
'''
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
assert status.OK()
nq = 100
vectors = gen_vectors(nq, dim)
ids = [i for i in range(nq)]
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
ids = [(i+100) for i in range(nq)]
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, res = connect.get_table_row_count(table)
status, res = connect.count_collection(collection)
assert res == nq * 2
def test_create_partition_insert_same_tags_two_tables(self, connect, table):
def test_create_partition_insert_same_tags_two_collections(self, connect, collection):
'''
target: test create two partitions, and insert vectors with the same tag to each table, check status returned
target: test create two partitions, and insert vectors with the same tag to each collection, check status returned
method: call function: create_partition
expected: status ok, table length is correct
expected: status ok, collection length is correct
'''
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
assert status.OK()
table_new = gen_unique_str()
param = {'table_name': table_new,
collection_new = gen_unique_str()
param = {'collection_name': collection_new,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.L2}
status = connect.create_table(param)
status = connect.create_partition(table_new, tag)
status = connect.create_collection(param)
status = connect.create_partition(collection_new, tag)
nq = 100
vectors = gen_vectors(nq, dim)
ids = [i for i in range(nq)]
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
ids = [(i+100) for i in range(nq)]
status, ids = connect.insert(table_new, vectors, ids, partition_tag=tag)
status = connect.flush([table, table_new])
status, ids = connect.insert(collection_new, vectors, ids, partition_tag=tag)
status = connect.flush([collection, collection_new])
assert status.OK()
status, res = connect.get_table_row_count(table)
status, res = connect.count_collection(collection)
assert res == nq
status, res = connect.get_table_row_count(table_new)
status, res = connect.count_collection(collection_new)
assert res == nq
@ -183,35 +183,35 @@ class TestShowBase:
The following cases are used to test `show_partitions` function
******************************************************************
"""
def test_show_partitions(self, connect, table):
def test_show_partitions(self, connect, collection):
'''
target: test show partitions, check status and partitions returned
method: create partition first, then call function: show_partitions
expected: status ok, partition correct
'''
status = connect.create_partition(table, tag)
status, res = connect.show_partitions(table)
status = connect.create_partition(collection, tag)
status, res = connect.show_partitions(collection)
assert status.OK()
def test_show_partitions_no_partition(self, connect, table):
def test_show_partitions_no_partition(self, connect, collection):
'''
target: test show partitions with table name, check status and partitions returned
target: test show partitions with collection name, check status and partitions returned
method: call function: show_partitions
expected: status ok, partitions correct
'''
status, res = connect.show_partitions(table)
status, res = connect.show_partitions(collection)
assert status.OK()
def test_show_multi_partitions(self, connect, table):
def test_show_multi_partitions(self, connect, collection):
'''
target: test show partitions, check status and partitions returned
method: create partitions first, then call function: show_partitions
expected: status ok, partitions correct
'''
tag_new = gen_unique_str()
status = connect.create_partition(table, tag)
status = connect.create_partition(table, tag_new)
status, res = connect.show_partitions(table)
status = connect.create_partition(collection, tag)
status = connect.create_partition(collection, tag_new)
status, res = connect.show_partitions(collection)
assert status.OK()
@ -222,72 +222,72 @@ class TestDropBase:
The following cases are used to test `drop_partition` function
******************************************************************
"""
def test_drop_partition(self, connect, table):
def test_drop_partition(self, connect, collection):
'''
target: test drop partition, check status and partition if existed
method: create partitions first, then call function: drop_partition
expected: status ok, no partitions in db
'''
status = connect.create_partition(table, tag)
status = connect.drop_partition(table, tag)
status = connect.create_partition(collection, tag)
status = connect.drop_partition(collection, tag)
assert status.OK()
status, res = connect.show_partitions(table)
status, res = connect.show_partitions(collection)
tag_list = []
for item in res:
tag_list.append(item.tag)
assert tag not in tag_list
def test_drop_partition_tag_not_existed(self, connect, table):
def test_drop_partition_tag_not_existed(self, connect, collection):
'''
target: test drop partition, but tag not existed
method: create partitions first, then call function: drop_partition
expected: status not ok
'''
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
new_tag = "new_tag"
status = connect.drop_partition(table, new_tag)
status = connect.drop_partition(collection, new_tag)
assert not status.OK()
def test_drop_partition_tag_not_existed_A(self, connect, table):
def test_drop_partition_tag_not_existed_A(self, connect, collection):
'''
target: test drop partition, but table not existed
target: test drop partition, but collection not existed
method: create partitions first, then call function: drop_partition
expected: status not ok
'''
status = connect.create_partition(table, tag)
new_table = gen_unique_str()
status = connect.drop_partition(new_table, tag)
status = connect.create_partition(collection, tag)
new_collection = gen_unique_str()
status = connect.drop_partition(new_collection, tag)
assert not status.OK()
def test_drop_partition_repeatedly(self, connect, table):
def test_drop_partition_repeatedly(self, connect, collection):
'''
target: test drop partition twice, check status and partition if existed
method: create partitions first, then call function: drop_partition
expected: status not ok, no partitions in db
'''
status = connect.create_partition(table, tag)
status = connect.drop_partition(table, tag)
status = connect.drop_partition(table, tag)
status = connect.create_partition(collection, tag)
status = connect.drop_partition(collection, tag)
status = connect.drop_partition(collection, tag)
time.sleep(2)
assert not status.OK()
status, res = connect.show_partitions(table)
status, res = connect.show_partitions(collection)
tag_list = []
for item in res:
tag_list.append(item.tag)
assert tag not in tag_list
def test_drop_partition_create(self, connect, table):
def test_drop_partition_create(self, connect, collection):
'''
target: test drop partition, and create again, check status
method: create partitions first, then call function: drop_partition, create_partition
expected: status not ok, partition in db
'''
status = connect.create_partition(table, tag)
status = connect.drop_partition(table, tag)
status = connect.create_partition(collection, tag)
status = connect.drop_partition(collection, tag)
time.sleep(2)
status = connect.create_partition(table, tag)
status = connect.create_partition(collection, tag)
assert status.OK()
status, res = connect.show_partitions(table)
status, res = connect.show_partitions(collection)
tag_list = []
for item in res:
tag_list.append(item.tag)
@ -297,47 +297,47 @@ class TestDropBase:
class TestNameInvalid(object):
@pytest.fixture(
scope="function",
params=gen_invalid_table_names()
params=gen_invalid_collection_names()
)
def get_tag_name(self, request):
yield request.param
@pytest.fixture(
scope="function",
params=gen_invalid_table_names()
params=gen_invalid_collection_names()
)
def get_table_name(self, request):
def get_collection_name(self, request):
yield request.param
def test_drop_partition_with_invalid_table_name(self, connect, table, get_table_name):
def test_drop_partition_with_invalid_collection_name(self, connect, collection, get_collection_name):
'''
target: test drop partition, with invalid table name, check status returned
target: test drop partition, with invalid collection name, check status returned
method: call function: drop_partition
expected: status not ok
'''
table_name = get_table_name
status = connect.create_partition(table, tag)
status = connect.drop_partition(table_name, tag)
collection_name = get_collection_name
status = connect.create_partition(collection, tag)
status = connect.drop_partition(collection_name, tag)
assert not status.OK()
def test_drop_partition_with_invalid_tag_name(self, connect, table, get_tag_name):
def test_drop_partition_with_invalid_tag_name(self, connect, collection, get_tag_name):
'''
target: test drop partition, with invalid tag name, check status returned
method: call function: drop_partition
expected: status not ok
'''
tag_name = get_tag_name
status = connect.create_partition(table, tag)
status = connect.drop_partition(table, tag_name)
status = connect.create_partition(collection, tag)
status = connect.drop_partition(collection, tag_name)
assert not status.OK()
def test_show_partitions_with_invalid_table_name(self, connect, table, get_table_name):
def test_show_partitions_with_invalid_collection_name(self, connect, collection, get_collection_name):
'''
target: test show partitions, with invalid table name, check status returned
target: test show partitions, with invalid collection name, check status returned
method: call function: show_partitions
expected: status not ok
'''
table_name = get_table_name
status = connect.create_partition(table, tag)
status, res = connect.show_partitions(table_name)
collection_name = get_collection_name
status = connect.create_partition(collection, tag)
status, res = connect.show_partitions(collection_name)
assert not status.OK()

544
tests/milvus_python_test/test_search_by_id.py
View File

File diff suppressed because it is too large Load Diff

444
tests/milvus_python_test/test_search_vectors.py
View File

File diff suppressed because it is too large Load Diff

1051
tests/milvus_python_test/test_table.py
View File

File diff suppressed because it is too large Load Diff

644
tests/milvus_python_test/test_table_count.py
View File

@ -1,644 +0,0 @@
import pdb
import pytest
import logging
import itertools
from time import sleep
import threading
from multiprocessing import Process
from milvus import IndexType, MetricType
from utils import *
dim = 128
index_file_size = 10
add_time_interval = 3
tag = "1970-01-01"
nb = 6000
class TestTableCount:
"""
params means different nb, the nb value may trigger merge, or not
"""
@pytest.fixture(
scope="function",
params=[
1,
5000,
100000,
],
)
def add_vectors_nb(self, request):
yield request.param
"""
generate valid create_index params
"""
@pytest.fixture(
scope="function",
params=gen_simple_index()
)
def get_simple_index(self, request, connect):
if str(connect._cmd("mode")[1]) == "CPU":
if request.param["index_type"] == IndexType.IVF_SQ8H:
pytest.skip("sq8h not support in cpu mode")
if request.param["index_type"] == IndexType.IVF_PQ:
pytest.skip("Skip PQ Temporary")
return request.param
def test_table_rows_count(self, connect, table, add_vectors_nb):
'''
target: test table rows_count is correct or not
method: create table and add vectors in it,
assert the value returned by get_table_row_count method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nb = add_vectors_nb
vectors = gen_vectors(nb, dim)
res = connect.add_vectors(table_name=table, records=vectors)
connect.flush([table])
status, res = connect.get_table_row_count(table)
assert res == nb
def test_table_rows_count_partition(self, connect, table, add_vectors_nb):
'''
target: test table rows_count is correct or not
method: create table, create partition and add vectors in it,
assert the value returned by get_table_row_count method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nb = add_vectors_nb
vectors = gen_vectors(nb, dim)
status = connect.create_partition(table, tag)
assert status.OK()
res = connect.add_vectors(table_name=table, records=vectors, partition_tag=tag)
connect.flush([table])
status, res = connect.get_table_row_count(table)
assert res == nb
def test_table_rows_count_multi_partitions_A(self, connect, table, add_vectors_nb):
'''
target: test table rows_count is correct or not
method: create table, create partitions and add vectors in it,
assert the value returned by get_table_row_count method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
new_tag = "new_tag"
nb = add_vectors_nb
vectors = gen_vectors(nb, dim)
status = connect.create_partition(table, tag)
status = connect.create_partition(table, new_tag)
assert status.OK()
res = connect.add_vectors(table_name=table, records=vectors)
connect.flush([table])
status, res = connect.get_table_row_count(table)
assert res == nb
def test_table_rows_count_multi_partitions_B(self, connect, table, add_vectors_nb):
'''
target: test table rows_count is correct or not
method: create table, create partitions and add vectors in one of the partitions,
assert the value returned by get_table_row_count method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
new_tag = "new_tag"
nb = add_vectors_nb
vectors = gen_vectors(nb, dim)
status = connect.create_partition(table, tag)
status = connect.create_partition(table, new_tag)
assert status.OK()
res = connect.add_vectors(table_name=table, records=vectors, partition_tag=tag)
connect.flush([table])
status, res = connect.get_table_row_count(table)
assert res == nb
def test_table_rows_count_multi_partitions_C(self, connect, table, add_vectors_nb):
'''
target: test table rows_count is correct or not
method: create table, create partitions and add vectors in one of the partitions,
assert the value returned by get_table_row_count method is equal to length of vectors
expected: the table count is equal to the length of vectors
'''
new_tag = "new_tag"
nb = add_vectors_nb
vectors = gen_vectors(nb, dim)
status = connect.create_partition(table, tag)
status = connect.create_partition(table, new_tag)
assert status.OK()
res = connect.add_vectors(table_name=table, records=vectors, partition_tag=tag)
res = connect.add_vectors(table_name=table, records=vectors, partition_tag=new_tag)
connect.flush([table])
status, res = connect.get_table_row_count(table)
assert res == nb * 2
def test_table_rows_count_after_index_created(self, connect, table, get_simple_index):
'''
target: test get_table_row_count, after index have been created
method: add vectors in db, and create index, then calling get_table_row_count with correct params
expected: get_table_row_count raise exception
'''
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
nb = 100
vectors = gen_vectors(nb, dim)
res = connect.add_vectors(table_name=table, records=vectors)
connect.flush([table])
connect.create_index(table, index_type, index_param)
status, res = connect.get_table_row_count(table)
assert res == nb
@pytest.mark.level(2)
def test_count_without_connection(self, table, dis_connect):
'''
target: test get_table_row_count, without connection
method: calling get_table_row_count with correct params, with a disconnected instance
expected: get_table_row_count raise exception
'''
with pytest.raises(Exception) as e:
status = dis_connect.get_table_row_count(table)
def test_table_rows_count_no_vectors(self, connect, table):
'''
target: test table rows_count is correct or not, if table is empty
method: create table and no vectors in it,
assert the value returned by get_table_row_count method is equal to 0
expected: the count is equal to 0
'''
table_name = gen_unique_str()
param = {'table_name': table_name,
'dimension': dim,
'index_file_size': index_file_size}
connect.create_table(param)
status, res = connect.get_table_row_count(table)
assert res == 0
# TODO: enable
@pytest.mark.level(2)
@pytest.mark.timeout(20)
def _test_table_rows_count_multiprocessing(self, connect, table, args):
'''
target: test table rows_count is correct or not with multiprocess
method: create table and add vectors in it,
assert the value returned by get_table_row_count method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nq = 2
uri = "tcp://%s:%s" % (args["ip"], args["port"])
vectors = gen_vectors(nq, dim)
res = connect.add_vectors(table_name=table, records=vectors)
time.sleep(add_time_interval)
def rows_count(milvus):
status, res = milvus.get_table_row_count(table)
logging.getLogger().info(status)
assert res == nq
process_num = 8
processes = []
for i in range(process_num):
milvus = get_milvus(args["handler"])
milvus.connect(uri=uri)
p = Process(target=rows_count, args=(milvus, ))
processes.append(p)
p.start()
logging.getLogger().info(p)
for p in processes:
p.join()
def test_table_rows_count_multi_tables(self, connect):
'''
target: test table rows_count is correct or not with multiple tables of L2
method: create table and add vectors in it,
assert the value returned by get_table_row_count method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nq = 100
vectors = gen_vectors(nq, dim)
table_list = []
for i in range(20):
table_name = gen_unique_str()
table_list.append(table_name)
param = {'table_name': table_name,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.L2}
connect.create_table(param)
res = connect.add_vectors(table_name=table_name, records=vectors)
connect.flush(table_list)
for i in range(20):
status, res = connect.get_table_row_count(table_list[i])
assert status.OK()
assert res == nq
class TestTableCountIP:
"""
params means different nb, the nb value may trigger merge, or not
"""
@pytest.fixture(
scope="function",
params=[
1,
5000,
100000,
],
)
def add_vectors_nb(self, request):
yield request.param
"""
generate valid create_index params
"""
@pytest.fixture(
scope="function",
params=gen_simple_index()
)
def get_simple_index(self, request, connect):
if str(connect._cmd("mode")[1]) == "CPU":
if request.param["index_type"] == IndexType.IVF_SQ8H:
pytest.skip("sq8h not support in CPU mode")
if request.param["index_type"] == IndexType.IVF_PQ:
pytest.skip("Skip PQ Temporary")
return request.param
def test_table_rows_count(self, connect, ip_table, add_vectors_nb):
'''
target: test table rows_count is correct or not
method: create table and add vectors in it,
assert the value returned by get_table_row_count method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nb = add_vectors_nb
vectors = gen_vectors(nb, dim)
res = connect.add_vectors(table_name=ip_table, records=vectors)
connect.flush([ip_table])
status, res = connect.get_table_row_count(ip_table)
assert res == nb
def test_table_rows_count_after_index_created(self, connect, ip_table, get_simple_index):
'''
target: test get_table_row_count, after index have been created
method: add vectors in db, and create index, then calling get_table_row_count with correct params
expected: get_table_row_count raise exception
'''
index_param = get_simple_index["index_param"]
index_type = get_simple_index["index_type"]
nb = 100
vectors = gen_vectors(nb, dim)
res = connect.add_vectors(table_name=ip_table, records=vectors)
connect.flush([ip_table])
connect.create_index(ip_table, index_type, index_param)
status, res = connect.get_table_row_count(ip_table)
assert res == nb
@pytest.mark.level(2)
def test_count_without_connection(self, ip_table, dis_connect):
'''
target: test get_table_row_count, without connection
method: calling get_table_row_count with correct params, with a disconnected instance
expected: get_table_row_count raise exception
'''
with pytest.raises(Exception) as e:
status = dis_connect.get_table_row_count(ip_table)
def test_table_rows_count_no_vectors(self, connect, ip_table):
'''
target: test table rows_count is correct or not, if table is empty
method: create table and no vectors in it,
assert the value returned by get_table_row_count method is equal to 0
expected: the count is equal to 0
'''
table_name = gen_unique_str("test_table")
param = {'table_name': table_name,
'dimension': dim,
'index_file_size': index_file_size}
connect.create_table(param)
status, res = connect.get_table_row_count(ip_table)
assert res == 0
# TODO: enable
@pytest.mark.timeout(60)
def _test_table_rows_count_multiprocessing(self, connect, ip_table, args):
'''
target: test table rows_count is correct or not with multiprocess
method: create table and add vectors in it,
assert the value returned by get_table_row_count method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nq = 2
uri = "tcp://%s:%s" % (args["ip"], args["port"])
vectors = gen_vectors(nq, dim)
res = connect.add_vectors(table_name=ip_table, records=vectors)
time.sleep(add_time_interval)
def rows_count(milvus):
status, res = milvus.get_table_row_count(ip_table)
logging.getLogger().info(status)
assert res == nq
process_num = 8
processes = []
for i in range(process_num):
milvus = get_milvus(args["handler"])
milvus.connect(uri=uri)
p = Process(target=rows_count, args=(milvus,))
processes.append(p)
p.start()
logging.getLogger().info(p)
for p in processes:
p.join()
def test_table_rows_count_multi_tables(self, connect):
'''
target: test table rows_count is correct or not with multiple tables of IP
method: create table and add vectors in it,
assert the value returned by get_table_row_count method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nq = 100
vectors = gen_vectors(nq, dim)
table_list = []
for i in range(20):
table_name = gen_unique_str('test_table_rows_count_multi_tables')
table_list.append(table_name)
param = {'table_name': table_name,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.IP}
connect.create_table(param)
res = connect.add_vectors(table_name=table_name, records=vectors)
connect.flush(table_list)
for i in range(20):
status, res = connect.get_table_row_count(table_list[i])
assert status.OK()
assert res == nq
class TestTableCountJAC:
"""
params means different nb, the nb value may trigger merge, or not
"""
@pytest.fixture(
scope="function",
params=[
1,
5000,
100000,
],
)
def add_vectors_nb(self, request):
yield request.param
"""
generate valid create_index params
"""
@pytest.fixture(
scope="function",
params=gen_simple_index()
)
def get_jaccard_index(self, request, connect):
logging.getLogger().info(request.param)
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
return request.param
else:
pytest.skip("Skip index Temporary")
def test_table_rows_count(self, connect, jac_table, add_vectors_nb):
'''
target: test table rows_count is correct or not
method: create table and add vectors in it,
assert the value returned by get_table_row_count method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nb = add_vectors_nb
tmp, vectors = gen_binary_vectors(nb, dim)
res = connect.add_vectors(table_name=jac_table, records=vectors)
connect.flush([jac_table])
status, res = connect.get_table_row_count(jac_table)
assert res == nb
def test_table_rows_count_after_index_created(self, connect, jac_table, get_jaccard_index):
'''
target: test get_table_row_count, after index have been created
method: add vectors in db, and create index, then calling get_table_row_count with correct params
expected: get_table_row_count raise exception
'''
nb = 100
index_param = get_jaccard_index["index_param"]
index_type = get_jaccard_index["index_type"]
tmp, vectors = gen_binary_vectors(nb, dim)
res = connect.add_vectors(table_name=jac_table, records=vectors)
connect.flush([jac_table])
connect.create_index(jac_table, index_type, index_param)
status, res = connect.get_table_row_count(jac_table)
assert res == nb
@pytest.mark.level(2)
def test_count_without_connection(self, jac_table, dis_connect):
'''
target: test get_table_row_count, without connection
method: calling get_table_row_count with correct params, with a disconnected instance
expected: get_table_row_count raise exception
'''
with pytest.raises(Exception) as e:
status = dis_connect.get_table_row_count(jac_table)
def test_table_rows_count_no_vectors(self, connect, jac_table):
'''
target: test table rows_count is correct or not, if table is empty
method: create table and no vectors in it,
assert the value returned by get_table_row_count method is equal to 0
expected: the count is equal to 0
'''
table_name = gen_unique_str("test_table")
param = {'table_name': table_name,
'dimension': dim,
'index_file_size': index_file_size}
connect.create_table(param)
status, res = connect.get_table_row_count(jac_table)
assert res == 0
def test_table_rows_count_multi_tables(self, connect):
'''
target: test table rows_count is correct or not with multiple tables of IP
method: create table and add vectors in it,
assert the value returned by get_table_row_count method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nq = 100
tmp, vectors = gen_binary_vectors(nq, dim)
table_list = []
for i in range(20):
table_name = gen_unique_str('test_table_rows_count_multi_tables')
table_list.append(table_name)
param = {'table_name': table_name,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.JACCARD}
connect.create_table(param)
res = connect.add_vectors(table_name=table_name, records=vectors)
connect.flush(table_list)
for i in range(20):
status, res = connect.get_table_row_count(table_list[i])
assert status.OK()
assert res == nq
class TestTableCountHAM:
"""
params means different nb, the nb value may trigger merge, or not
"""
@pytest.fixture(
scope="function",
params=[
1,
5000,
100000,
],
)
def add_vectors_nb(self, request):
yield request.param
"""
generate valid create_index params
"""
@pytest.fixture(
scope="function",
params=gen_simple_index()
)
def get_hamming_index(self, request, connect):
logging.getLogger().info(request.param)
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
return request.param
else:
pytest.skip("Skip index Temporary")
def test_table_rows_count(self, connect, ham_table, add_vectors_nb):
'''
target: test table rows_count is correct or not
method: create table and add vectors in it,
assert the value returned by get_table_row_count method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nb = add_vectors_nb
tmp, vectors = gen_binary_vectors(nb, dim)
res = connect.add_vectors(table_name=ham_table, records=vectors)
connect.flush([ham_table])
status, res = connect.get_table_row_count(ham_table)
assert res == nb
def test_table_rows_count_after_index_created(self, connect, ham_table, get_hamming_index):
'''
target: test get_table_row_count, after index have been created
method: add vectors in db, and create index, then calling get_table_row_count with correct params
expected: get_table_row_count raise exception
'''
nb = 100
index_type = get_hamming_index["index_type"]
index_param = get_hamming_index["index_param"]
tmp, vectors = gen_binary_vectors(nb, dim)
res = connect.add_vectors(table_name=ham_table, records=vectors)
connect.flush([ham_table])
connect.create_index(ham_table, index_type, index_param)
status, res = connect.get_table_row_count(ham_table)
assert res == nb
@pytest.mark.level(2)
def test_count_without_connection(self, ham_table, dis_connect):
'''
target: test get_table_row_count, without connection
method: calling get_table_row_count with correct params, with a disconnected instance
expected: get_table_row_count raise exception
'''
with pytest.raises(Exception) as e:
status = dis_connect.get_table_row_count(ham_table)
def test_table_rows_count_no_vectors(self, connect, ham_table):
'''
target: test table rows_count is correct or not, if table is empty
method: create table and no vectors in it,
assert the value returned by get_table_row_count method is equal to 0
expected: the count is equal to 0
'''
table_name = gen_unique_str("test_table")
param = {'table_name': table_name,
'dimension': dim,
'index_file_size': index_file_size}
connect.create_table(param)
status, res = connect.get_table_row_count(ham_table)
assert res == 0
def test_table_rows_count_multi_tables(self, connect):
'''
target: test table rows_count is correct or not with multiple tables of IP
method: create table and add vectors in it,
assert the value returned by get_table_row_count method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nq = 100
tmp, vectors = gen_binary_vectors(nq, dim)
table_list = []
for i in range(20):
table_name = gen_unique_str('test_table_rows_count_multi_tables')
table_list.append(table_name)
param = {'table_name': table_name,
'dimension': dim,
'index_file_size': index_file_size,
'metric_type': MetricType.HAMMING}
connect.create_table(param)
res = connect.add_vectors(table_name=table_name, records=vectors)
connect.flush(table_list)
for i in range(20):
status, res = connect.get_table_row_count(table_list[i])
assert status.OK()
assert res == nq
class TestTableCountTANIMOTO:
"""
params means different nb, the nb value may trigger merge, or not
"""
@pytest.fixture(
scope="function",
params=[
1,
5000,
100000,
],
)
def add_vectors_nb(self, request):
yield request.param
"""
generate valid create_index params
"""
@pytest.fixture(
scope="function",
params=gen_simple_index()
)
def get_tanimoto_index(self, request, connect):
logging.getLogger().info(request.param)
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
return request.param
else:
pytest.skip("Skip index Temporary")
def test_table_rows_count(self, connect, tanimoto_table, add_vectors_nb):
'''
target: test table rows_count is correct or not
method: create table and add vectors in it,
assert the value returned by get_table_row_count method is equal to length of vectors
expected: the count is equal to the length of vectors
'''
nb = add_vectors_nb
tmp, vectors = gen_binary_vectors(nb, dim)
res = connect.add_vectors(table_name=tanimoto_table, records=vectors)
connect.flush([tanimoto_table])
status, res = connect.get_table_row_count(tanimoto_table)
assert status.OK()
assert res == nb

72
tests/milvus_python_test/test_wal.py
View File

@ -8,7 +8,7 @@ from milvus import IndexType, MetricType
from utils import *
dim = 128
table_id = "test_wal"
collection_id = "test_wal"
WAL_TIMEOUT = 30
nb = 6000
add_interval = 1.5
@ -21,109 +21,109 @@ class TestWalBase:
******************************************************************
"""
@pytest.mark.timeout(WAL_TIMEOUT)
def test_wal_add_vectors(self, connect, table):
def test_wal_add_vectors(self, connect, collection):
'''
target: add vectors in WAL
method: add vectors and flush when WAL is enabled
expected: status ok, vectors added
'''
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, res = connect.get_table_row_count(table)
status, res = connect.count_collection(collection)
assert status.OK()
assert res == nb
status, res = connect.get_vector_by_id(table, ids[0])
status, res = connect.get_vector_by_id(collection, ids[0])
logging.getLogger().info(res)
assert status.OK()
assert_equal_vector(res, vectors[0])
@pytest.mark.timeout(WAL_TIMEOUT)
def test_wal_delete_vectors(self, connect, table):
def test_wal_delete_vectors(self, connect, collection):
'''
target: delete vectors in WAL
method: delete vectors and flush when WAL is enabled
expected: status ok, vectors deleted
'''
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
connect.flush([table])
status, res = connect.get_table_row_count(table)
connect.flush([collection])
status, res = connect.count_collection(collection)
assert status.OK()
status = connect.delete_by_id(table, ids)
status = connect.delete_by_id(collection, ids)
assert status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, res = connect.get_table_row_count(table)
status, res = connect.count_collection(collection)
assert status.OK()
assert res == 0
@pytest.mark.timeout(WAL_TIMEOUT)
def test_wal_invalid_operation(self, connect, table):
def test_wal_invalid_operation(self, connect, collection):
'''
target: invalid operation in WAL
method: add vectors, delete with non-existent ids and flush when WAL is enabled
expected: status ok, search with vector have result
'''
vector = gen_single_vector(dim)
status, ids = connect.add_vectors(table, vector)
status, ids = connect.add_vectors(collection, vector)
assert status.OK()
connect.flush([table])
status = connect.delete_by_id(table, [0])
connect.flush([collection])
status = connect.delete_by_id(collection, [0])
assert status.OK()
status = connect.flush([table])
status, res = connect.get_table_row_count(table)
status = connect.flush([collection])
status, res = connect.count_collection(collection)
assert status.OK()
assert res == 1
@pytest.mark.timeout(WAL_TIMEOUT)
def test_wal_invalid_operation_B(self, connect, table):
def test_wal_invalid_operation_B(self, connect, collection):
'''
target: invalid operation in WAL
method: add vectors, delete with not existed table name when WAL is enabled
method: add vectors, delete with not existed collection name when WAL is enabled
expected: status not ok
'''
vectors = gen_vector(nb, dim)
status, ids = connect.add_vectors(table, vectors)
status, ids = connect.add_vectors(collection, vectors)
assert status.OK()
status = connect.flush([table])
status = connect.delete_by_id(table, [0])
connect.flush([table])
table_new = gen_unique_str()
status = connect.delete_by_id(table_new, ids)
status = connect.flush([collection])
status = connect.delete_by_id(collection, [0])
connect.flush([collection])
collection_new = gen_unique_str()
status = connect.delete_by_id(collection_new, ids)
assert not status.OK()
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, res = connect.get_table_row_count(table)
status, res = connect.count_collection(collection)
assert status.OK()
assert res == nb
@pytest.mark.timeout(WAL_TIMEOUT)
def test_wal_server_crashed_recovery(self, connect, table):
def test_wal_server_crashed_recovery(self, connect, collection):
'''
target: test wal when server crashed unexpectedly and restarted
method: add vectors, server killed before flush, restarted server and flush
expected: status ok, add request is recovered and vectors added
'''
vector = gen_single_vector(dim)
status, ids = connect.add_vectors(table, vector)
status, ids = connect.add_vectors(collection, vector)
assert status.OK()
status = connect.flush([table])
status, res = connect.get_table_row_count(table)
status = connect.flush([collection])
status, res = connect.count_collection(collection)
assert status.OK()
logging.getLogger().info(res) # should be 0 because no auto flush
logging.getLogger().info("Stop server and restart")
# kill server and restart. auto flush should be set to 15 seconds.
# time.sleep(15)
status = connect.flush([table])
status = connect.flush([collection])
assert status.OK()
status, res = connect.get_table_row_count(table)
status, res = connect.count_collection(collection)
assert status.OK()
assert res == 1
status, res = connect.get_vector_by_id(table, ids[0])
status, res = connect.get_vector_by_id(collection, ids[0])
logging.getLogger().info(res)
assert status.OK()
assert_equal_vector(res, vector[0])

18
tests/milvus_python_test/utils.py
View File

@ -146,8 +146,8 @@ def gen_invalid_uris():
return uris
def gen_invalid_table_names():
table_names = [
def gen_invalid_collection_names():
collection_names = [
"12-s",
"12/s",
" ",
@ -166,7 +166,7 @@ def gen_invalid_table_names():
"中文",
"a".join("a" for i in range(256))
]
return table_names
return collection_names
def gen_invalid_top_ks():
@ -588,7 +588,7 @@ def gen_simple_index():
IndexType.IVF_SQ8H,
IndexType.IVF_PQ,
IndexType.HNSW,
# IndexType.RNSG
IndexType.RNSG
]
params = [
{"nlist": 1024},
@ -597,7 +597,7 @@ def gen_simple_index():
{"nlist": 1024},
{"nlist": 1024, "m": 16},
{"M": 16, "efConstruction": 500},
# {"search_length": 100, "out_degree": 40, "candidate_pool_size": 66, "knng": 100}
{"search_length": 50, "out_degree": 40, "candidate_pool_size": 100, "knng": 50}
]
index_params = []
@ -611,14 +611,14 @@ def get_search_param(index_type):
return {"nprobe": 32}
elif index_type == IndexType.HNSW:
return {"ef": 64}
# elif index_type == IndexType.RNSG:
# return {"search_length": 100}
elif index_type == IndexType.RNSG:
return {"search_length": 50}
else:
logging.getLogger().info("Invalid index_type.")
def assert_has_table(conn, table_name):
status, ok = conn.has_table(table_name)
def assert_has_collection(conn, collection_name):
status, ok = conn.has_collection(collection_name)
return status.OK() and ok