mirror of https://github.com/milvus-io/milvus.git
commit
6cd77b35df
|
@ -90,118 +90,118 @@ def milvus(request):
|
|||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def table(request, connect):
|
||||
ori_table_name = getattr(request.module, "table_id", "test")
|
||||
table_name = gen_unique_str(ori_table_name)
|
||||
def collection(request, connect):
|
||||
ori_collection_name = getattr(request.module, "collection_id", "test")
|
||||
collection_name = gen_unique_str(ori_collection_name)
|
||||
dim = getattr(request.module, "dim", "128")
|
||||
param = {'table_name': table_name,
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.L2}
|
||||
status = connect.create_table(param)
|
||||
status = connect.create_collection(param)
|
||||
# logging.getLogger().info(status)
|
||||
if not status.OK():
|
||||
pytest.exit("Table can not be created, exit pytest ...")
|
||||
pytest.exit("collection can not be created, exit pytest ...")
|
||||
|
||||
def teardown():
|
||||
status, table_names = connect.show_tables()
|
||||
for table_name in table_names:
|
||||
connect.delete_table(table_name)
|
||||
status, collection_names = connect.show_collections()
|
||||
for collection_name in collection_names:
|
||||
connect.drop_collection(collection_name)
|
||||
|
||||
request.addfinalizer(teardown)
|
||||
|
||||
return table_name
|
||||
return collection_name
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def ip_table(request, connect):
|
||||
ori_table_name = getattr(request.module, "table_id", "test")
|
||||
table_name = gen_unique_str(ori_table_name)
|
||||
def ip_collection(request, connect):
|
||||
ori_collection_name = getattr(request.module, "collection_id", "test")
|
||||
collection_name = gen_unique_str(ori_collection_name)
|
||||
dim = getattr(request.module, "dim", "128")
|
||||
param = {'table_name': table_name,
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.IP}
|
||||
status = connect.create_table(param)
|
||||
status = connect.create_collection(param)
|
||||
# logging.getLogger().info(status)
|
||||
if not status.OK():
|
||||
pytest.exit("Table can not be created, exit pytest ...")
|
||||
pytest.exit("collection can not be created, exit pytest ...")
|
||||
|
||||
def teardown():
|
||||
status, table_names = connect.show_tables()
|
||||
for table_name in table_names:
|
||||
connect.delete_table(table_name)
|
||||
status, collection_names = connect.show_collections()
|
||||
for collection_name in collection_names:
|
||||
connect.drop_collection(collection_name)
|
||||
|
||||
request.addfinalizer(teardown)
|
||||
|
||||
return table_name
|
||||
return collection_name
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def jac_table(request, connect):
|
||||
ori_table_name = getattr(request.module, "table_id", "test")
|
||||
table_name = gen_unique_str(ori_table_name)
|
||||
def jac_collection(request, connect):
|
||||
ori_collection_name = getattr(request.module, "collection_id", "test")
|
||||
collection_name = gen_unique_str(ori_collection_name)
|
||||
dim = getattr(request.module, "dim", "128")
|
||||
param = {'table_name': table_name,
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.JACCARD}
|
||||
status = connect.create_table(param)
|
||||
status = connect.create_collection(param)
|
||||
# logging.getLogger().info(status)
|
||||
if not status.OK():
|
||||
pytest.exit("Table can not be created, exit pytest ...")
|
||||
pytest.exit("collection can not be created, exit pytest ...")
|
||||
|
||||
def teardown():
|
||||
status, table_names = connect.show_tables()
|
||||
for table_name in table_names:
|
||||
connect.delete_table(table_name)
|
||||
status, collection_names = connect.show_collections()
|
||||
for collection_name in collection_names:
|
||||
connect.drop_collection(collection_name)
|
||||
|
||||
request.addfinalizer(teardown)
|
||||
|
||||
return table_name
|
||||
return collection_name
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def ham_table(request, connect):
|
||||
ori_table_name = getattr(request.module, "table_id", "test")
|
||||
table_name = gen_unique_str(ori_table_name)
|
||||
def ham_collection(request, connect):
|
||||
ori_collection_name = getattr(request.module, "collection_id", "test")
|
||||
collection_name = gen_unique_str(ori_collection_name)
|
||||
dim = getattr(request.module, "dim", "128")
|
||||
param = {'table_name': table_name,
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.HAMMING}
|
||||
status = connect.create_table(param)
|
||||
status = connect.create_collection(param)
|
||||
# logging.getLogger().info(status)
|
||||
if not status.OK():
|
||||
pytest.exit("Table can not be created, exit pytest ...")
|
||||
pytest.exit("collection can not be created, exit pytest ...")
|
||||
|
||||
def teardown():
|
||||
status, table_names = connect.show_tables()
|
||||
for table_name in table_names:
|
||||
connect.delete_table(table_name)
|
||||
status, collection_names = connect.show_collections()
|
||||
for collection_name in collection_names:
|
||||
connect.drop_collection(collection_name)
|
||||
|
||||
request.addfinalizer(teardown)
|
||||
|
||||
return table_name
|
||||
return collection_name
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def tanimoto_table(request, connect):
|
||||
ori_table_name = getattr(request.module, "table_id", "test")
|
||||
table_name = gen_unique_str(ori_table_name)
|
||||
def tanimoto_collection(request, connect):
|
||||
ori_collection_name = getattr(request.module, "collection_id", "test")
|
||||
collection_name = gen_unique_str(ori_collection_name)
|
||||
dim = getattr(request.module, "dim", "128")
|
||||
param = {'table_name': table_name,
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.TANIMOTO}
|
||||
status = connect.create_table(param)
|
||||
status = connect.create_collection(param)
|
||||
# logging.getLogger().info(status)
|
||||
if not status.OK():
|
||||
pytest.exit("Table can not be created, exit pytest ...")
|
||||
pytest.exit("collection can not be created, exit pytest ...")
|
||||
|
||||
def teardown():
|
||||
status, table_names = connect.show_tables()
|
||||
for table_name in table_names:
|
||||
connect.delete_table(table_name)
|
||||
status, collection_names = connect.show_collections()
|
||||
for collection_name in collection_names:
|
||||
connect.drop_collection(collection_name)
|
||||
|
||||
request.addfinalizer(teardown)
|
||||
|
||||
return table_name
|
||||
return collection_name
|
||||
|
|
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,644 @@
|
|||
import pdb
|
||||
import pytest
|
||||
import logging
|
||||
import itertools
|
||||
from time import sleep
|
||||
import threading
|
||||
from multiprocessing import Process
|
||||
from milvus import IndexType, MetricType
|
||||
from utils import *
|
||||
|
||||
dim = 128
|
||||
index_file_size = 10
|
||||
add_time_interval = 3
|
||||
tag = "1970-01-01"
|
||||
nb = 6000
|
||||
|
||||
class TestCollectionCount:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_simple_index(self, request, connect):
|
||||
if str(connect._cmd("mode")[1]) == "CPU":
|
||||
if request.param["index_type"] == IndexType.IVF_SQ8H:
|
||||
pytest.skip("sq8h not support in cpu mode")
|
||||
if request.param["index_type"] == IndexType.IVF_PQ:
|
||||
pytest.skip("Skip PQ Temporary")
|
||||
return request.param
|
||||
|
||||
def test_collection_rows_count(self, connect, collection, add_vectors_nb):
|
||||
'''
|
||||
target: test collection rows_count is correct or not
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
res = connect.add_vectors(collection_name=collection, records=vectors)
|
||||
connect.flush([collection])
|
||||
status, res = connect.count_collection(collection)
|
||||
assert res == nb
|
||||
|
||||
def test_collection_rows_count_partition(self, connect, collection, add_vectors_nb):
|
||||
'''
|
||||
target: test collection rows_count is correct or not
|
||||
method: create collection, create partition and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
res = connect.add_vectors(collection_name=collection, records=vectors, partition_tag=tag)
|
||||
connect.flush([collection])
|
||||
status, res = connect.count_collection(collection)
|
||||
assert res == nb
|
||||
|
||||
def test_collection_rows_count_multi_partitions_A(self, connect, collection, add_vectors_nb):
|
||||
'''
|
||||
target: test collection rows_count is correct or not
|
||||
method: create collection, create partitions and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
new_tag = "new_tag"
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.create_partition(collection, new_tag)
|
||||
assert status.OK()
|
||||
res = connect.add_vectors(collection_name=collection, records=vectors)
|
||||
connect.flush([collection])
|
||||
status, res = connect.count_collection(collection)
|
||||
assert res == nb
|
||||
|
||||
def test_collection_rows_count_multi_partitions_B(self, connect, collection, add_vectors_nb):
|
||||
'''
|
||||
target: test collection rows_count is correct or not
|
||||
method: create collection, create partitions and add vectors in one of the partitions,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
new_tag = "new_tag"
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.create_partition(collection, new_tag)
|
||||
assert status.OK()
|
||||
res = connect.add_vectors(collection_name=collection, records=vectors, partition_tag=tag)
|
||||
connect.flush([collection])
|
||||
status, res = connect.count_collection(collection)
|
||||
assert res == nb
|
||||
|
||||
def test_collection_rows_count_multi_partitions_C(self, connect, collection, add_vectors_nb):
|
||||
'''
|
||||
target: test collection rows_count is correct or not
|
||||
method: create collection, create partitions and add vectors in one of the partitions,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the collection count is equal to the length of vectors
|
||||
'''
|
||||
new_tag = "new_tag"
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.create_partition(collection, new_tag)
|
||||
assert status.OK()
|
||||
res = connect.add_vectors(collection_name=collection, records=vectors, partition_tag=tag)
|
||||
res = connect.add_vectors(collection_name=collection, records=vectors, partition_tag=new_tag)
|
||||
connect.flush([collection])
|
||||
status, res = connect.count_collection(collection)
|
||||
assert res == nb * 2
|
||||
|
||||
def test_collection_rows_count_after_index_created(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: test count_collection, after index have been created
|
||||
method: add vectors in db, and create index, then calling count_collection with correct params
|
||||
expected: count_collection raise exception
|
||||
'''
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
nb = 100
|
||||
vectors = gen_vectors(nb, dim)
|
||||
res = connect.add_vectors(collection_name=collection, records=vectors)
|
||||
connect.flush([collection])
|
||||
connect.create_index(collection, index_type, index_param)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert res == nb
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_count_without_connection(self, collection, dis_connect):
|
||||
'''
|
||||
target: test count_collection, without connection
|
||||
method: calling count_collection with correct params, with a disconnected instance
|
||||
expected: count_collection raise exception
|
||||
'''
|
||||
with pytest.raises(Exception) as e:
|
||||
status = dis_connect.count_collection(collection)
|
||||
|
||||
def test_collection_rows_count_no_vectors(self, connect, collection):
|
||||
'''
|
||||
target: test collection rows_count is correct or not, if collection is empty
|
||||
method: create collection and no vectors in it,
|
||||
assert the value returned by count_collection method is equal to 0
|
||||
expected: the count is equal to 0
|
||||
'''
|
||||
collection_name = gen_unique_str()
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size}
|
||||
connect.create_collection(param)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert res == 0
|
||||
|
||||
# TODO: enable
|
||||
@pytest.mark.level(2)
|
||||
@pytest.mark.timeout(20)
|
||||
def _test_collection_rows_count_multiprocessing(self, connect, collection, args):
|
||||
'''
|
||||
target: test collection rows_count is correct or not with multiprocess
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 2
|
||||
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
||||
vectors = gen_vectors(nq, dim)
|
||||
res = connect.add_vectors(collection_name=collection, records=vectors)
|
||||
time.sleep(add_time_interval)
|
||||
|
||||
def rows_count(milvus):
|
||||
status, res = milvus.count_collection(collection)
|
||||
logging.getLogger().info(status)
|
||||
assert res == nq
|
||||
|
||||
process_num = 8
|
||||
processes = []
|
||||
for i in range(process_num):
|
||||
milvus = get_milvus(args["handler"])
|
||||
milvus.connect(uri=uri)
|
||||
p = Process(target=rows_count, args=(milvus, ))
|
||||
processes.append(p)
|
||||
p.start()
|
||||
logging.getLogger().info(p)
|
||||
for p in processes:
|
||||
p.join()
|
||||
|
||||
def test_collection_rows_count_multi_collections(self, connect):
|
||||
'''
|
||||
target: test collection rows_count is correct or not with multiple collections of L2
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 100
|
||||
vectors = gen_vectors(nq, dim)
|
||||
collection_list = []
|
||||
for i in range(20):
|
||||
collection_name = gen_unique_str()
|
||||
collection_list.append(collection_name)
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.L2}
|
||||
connect.create_collection(param)
|
||||
res = connect.add_vectors(collection_name=collection_name, records=vectors)
|
||||
connect.flush(collection_list)
|
||||
for i in range(20):
|
||||
status, res = connect.count_collection(collection_list[i])
|
||||
assert status.OK()
|
||||
assert res == nq
|
||||
|
||||
|
||||
class TestCollectionCountIP:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_simple_index(self, request, connect):
|
||||
if str(connect._cmd("mode")[1]) == "CPU":
|
||||
if request.param["index_type"] == IndexType.IVF_SQ8H:
|
||||
pytest.skip("sq8h not support in CPU mode")
|
||||
if request.param["index_type"] == IndexType.IVF_PQ:
|
||||
pytest.skip("Skip PQ Temporary")
|
||||
return request.param
|
||||
|
||||
def test_collection_rows_count(self, connect, ip_collection, add_vectors_nb):
|
||||
'''
|
||||
target: test collection rows_count is correct or not
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
res = connect.add_vectors(collection_name=ip_collection, records=vectors)
|
||||
connect.flush([ip_collection])
|
||||
status, res = connect.count_collection(ip_collection)
|
||||
assert res == nb
|
||||
|
||||
def test_collection_rows_count_after_index_created(self, connect, ip_collection, get_simple_index):
|
||||
'''
|
||||
target: test count_collection, after index have been created
|
||||
method: add vectors in db, and create index, then calling count_collection with correct params
|
||||
expected: count_collection raise exception
|
||||
'''
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
nb = 100
|
||||
vectors = gen_vectors(nb, dim)
|
||||
res = connect.add_vectors(collection_name=ip_collection, records=vectors)
|
||||
connect.flush([ip_collection])
|
||||
connect.create_index(ip_collection, index_type, index_param)
|
||||
status, res = connect.count_collection(ip_collection)
|
||||
assert res == nb
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_count_without_connection(self, ip_collection, dis_connect):
|
||||
'''
|
||||
target: test count_collection, without connection
|
||||
method: calling count_collection with correct params, with a disconnected instance
|
||||
expected: count_collection raise exception
|
||||
'''
|
||||
with pytest.raises(Exception) as e:
|
||||
status = dis_connect.count_collection(ip_collection)
|
||||
|
||||
def test_collection_rows_count_no_vectors(self, connect, ip_collection):
|
||||
'''
|
||||
target: test collection rows_count is correct or not, if collection is empty
|
||||
method: create collection and no vectors in it,
|
||||
assert the value returned by count_collection method is equal to 0
|
||||
expected: the count is equal to 0
|
||||
'''
|
||||
collection_name = gen_unique_str("test_collection")
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size}
|
||||
connect.create_collection(param)
|
||||
status, res = connect.count_collection(ip_collection)
|
||||
assert res == 0
|
||||
|
||||
# TODO: enable
|
||||
@pytest.mark.timeout(60)
|
||||
def _test_collection_rows_count_multiprocessing(self, connect, ip_collection, args):
|
||||
'''
|
||||
target: test collection rows_count is correct or not with multiprocess
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 2
|
||||
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
||||
vectors = gen_vectors(nq, dim)
|
||||
res = connect.add_vectors(collection_name=ip_collection, records=vectors)
|
||||
time.sleep(add_time_interval)
|
||||
|
||||
def rows_count(milvus):
|
||||
status, res = milvus.count_collection(ip_collection)
|
||||
logging.getLogger().info(status)
|
||||
assert res == nq
|
||||
|
||||
process_num = 8
|
||||
processes = []
|
||||
for i in range(process_num):
|
||||
milvus = get_milvus(args["handler"])
|
||||
milvus.connect(uri=uri)
|
||||
p = Process(target=rows_count, args=(milvus,))
|
||||
processes.append(p)
|
||||
p.start()
|
||||
logging.getLogger().info(p)
|
||||
for p in processes:
|
||||
p.join()
|
||||
|
||||
def test_collection_rows_count_multi_collections(self, connect):
|
||||
'''
|
||||
target: test collection rows_count is correct or not with multiple collections of IP
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 100
|
||||
vectors = gen_vectors(nq, dim)
|
||||
collection_list = []
|
||||
for i in range(20):
|
||||
collection_name = gen_unique_str('test_collection_rows_count_multi_collections')
|
||||
collection_list.append(collection_name)
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.IP}
|
||||
connect.create_collection(param)
|
||||
res = connect.add_vectors(collection_name=collection_name, records=vectors)
|
||||
connect.flush(collection_list)
|
||||
for i in range(20):
|
||||
status, res = connect.count_collection(collection_list[i])
|
||||
assert status.OK()
|
||||
assert res == nq
|
||||
|
||||
|
||||
class TestCollectionCountJAC:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_jaccard_index(self, request, connect):
|
||||
logging.getLogger().info(request.param)
|
||||
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
|
||||
return request.param
|
||||
else:
|
||||
pytest.skip("Skip index Temporary")
|
||||
|
||||
def test_collection_rows_count(self, connect, jac_collection, add_vectors_nb):
|
||||
'''
|
||||
target: test collection rows_count is correct or not
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(collection_name=jac_collection, records=vectors)
|
||||
connect.flush([jac_collection])
|
||||
status, res = connect.count_collection(jac_collection)
|
||||
assert res == nb
|
||||
|
||||
def test_collection_rows_count_after_index_created(self, connect, jac_collection, get_jaccard_index):
|
||||
'''
|
||||
target: test count_collection, after index have been created
|
||||
method: add vectors in db, and create index, then calling count_collection with correct params
|
||||
expected: count_collection raise exception
|
||||
'''
|
||||
nb = 100
|
||||
index_param = get_jaccard_index["index_param"]
|
||||
index_type = get_jaccard_index["index_type"]
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(collection_name=jac_collection, records=vectors)
|
||||
connect.flush([jac_collection])
|
||||
connect.create_index(jac_collection, index_type, index_param)
|
||||
status, res = connect.count_collection(jac_collection)
|
||||
assert res == nb
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_count_without_connection(self, jac_collection, dis_connect):
|
||||
'''
|
||||
target: test count_collection, without connection
|
||||
method: calling count_collection with correct params, with a disconnected instance
|
||||
expected: count_collection raise exception
|
||||
'''
|
||||
with pytest.raises(Exception) as e:
|
||||
status = dis_connect.count_collection(jac_collection)
|
||||
|
||||
def test_collection_rows_count_no_vectors(self, connect, jac_collection):
|
||||
'''
|
||||
target: test collection rows_count is correct or not, if collection is empty
|
||||
method: create collection and no vectors in it,
|
||||
assert the value returned by count_collection method is equal to 0
|
||||
expected: the count is equal to 0
|
||||
'''
|
||||
collection_name = gen_unique_str("test_collection")
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size}
|
||||
connect.create_collection(param)
|
||||
status, res = connect.count_collection(jac_collection)
|
||||
assert res == 0
|
||||
|
||||
def test_collection_rows_count_multi_collections(self, connect):
|
||||
'''
|
||||
target: test collection rows_count is correct or not with multiple collections of IP
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 100
|
||||
tmp, vectors = gen_binary_vectors(nq, dim)
|
||||
collection_list = []
|
||||
for i in range(20):
|
||||
collection_name = gen_unique_str('test_collection_rows_count_multi_collections')
|
||||
collection_list.append(collection_name)
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.JACCARD}
|
||||
connect.create_collection(param)
|
||||
res = connect.add_vectors(collection_name=collection_name, records=vectors)
|
||||
connect.flush(collection_list)
|
||||
for i in range(20):
|
||||
status, res = connect.count_collection(collection_list[i])
|
||||
assert status.OK()
|
||||
assert res == nq
|
||||
|
||||
class TestCollectionCountHAM:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_hamming_index(self, request, connect):
|
||||
logging.getLogger().info(request.param)
|
||||
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
|
||||
return request.param
|
||||
else:
|
||||
pytest.skip("Skip index Temporary")
|
||||
|
||||
def test_collection_rows_count(self, connect, ham_collection, add_vectors_nb):
|
||||
'''
|
||||
target: test collection rows_count is correct or not
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(collection_name=ham_collection, records=vectors)
|
||||
connect.flush([ham_collection])
|
||||
status, res = connect.count_collection(ham_collection)
|
||||
assert res == nb
|
||||
|
||||
def test_collection_rows_count_after_index_created(self, connect, ham_collection, get_hamming_index):
|
||||
'''
|
||||
target: test count_collection, after index have been created
|
||||
method: add vectors in db, and create index, then calling count_collection with correct params
|
||||
expected: count_collection raise exception
|
||||
'''
|
||||
nb = 100
|
||||
index_type = get_hamming_index["index_type"]
|
||||
index_param = get_hamming_index["index_param"]
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(collection_name=ham_collection, records=vectors)
|
||||
connect.flush([ham_collection])
|
||||
connect.create_index(ham_collection, index_type, index_param)
|
||||
status, res = connect.count_collection(ham_collection)
|
||||
assert res == nb
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_count_without_connection(self, ham_collection, dis_connect):
|
||||
'''
|
||||
target: test count_collection, without connection
|
||||
method: calling count_collection with correct params, with a disconnected instance
|
||||
expected: count_collection raise exception
|
||||
'''
|
||||
with pytest.raises(Exception) as e:
|
||||
status = dis_connect.count_collection(ham_collection)
|
||||
|
||||
def test_collection_rows_count_no_vectors(self, connect, ham_collection):
|
||||
'''
|
||||
target: test collection rows_count is correct or not, if collection is empty
|
||||
method: create collection and no vectors in it,
|
||||
assert the value returned by count_collection method is equal to 0
|
||||
expected: the count is equal to 0
|
||||
'''
|
||||
collection_name = gen_unique_str("test_collection")
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size}
|
||||
connect.create_collection(param)
|
||||
status, res = connect.count_collection(ham_collection)
|
||||
assert res == 0
|
||||
|
||||
def test_collection_rows_count_multi_collections(self, connect):
|
||||
'''
|
||||
target: test collection rows_count is correct or not with multiple collections of IP
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 100
|
||||
tmp, vectors = gen_binary_vectors(nq, dim)
|
||||
collection_list = []
|
||||
for i in range(20):
|
||||
collection_name = gen_unique_str('test_collection_rows_count_multi_collections')
|
||||
collection_list.append(collection_name)
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.HAMMING}
|
||||
connect.create_collection(param)
|
||||
res = connect.add_vectors(collection_name=collection_name, records=vectors)
|
||||
connect.flush(collection_list)
|
||||
for i in range(20):
|
||||
status, res = connect.count_collection(collection_list[i])
|
||||
assert status.OK()
|
||||
assert res == nq
|
||||
|
||||
|
||||
class TestCollectionCountTANIMOTO:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_tanimoto_index(self, request, connect):
|
||||
logging.getLogger().info(request.param)
|
||||
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
|
||||
return request.param
|
||||
else:
|
||||
pytest.skip("Skip index Temporary")
|
||||
|
||||
def test_collection_rows_count(self, connect, tanimoto_collection, add_vectors_nb):
|
||||
'''
|
||||
target: test collection rows_count is correct or not
|
||||
method: create collection and add vectors in it,
|
||||
assert the value returned by count_collection method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(collection_name=tanimoto_collection, records=vectors)
|
||||
connect.flush([tanimoto_collection])
|
||||
status, res = connect.count_collection(tanimoto_collection)
|
||||
assert status.OK()
|
||||
assert res == nb
|
|
@ -18,7 +18,7 @@ nb = 6000
|
|||
nlist = 1024
|
||||
|
||||
|
||||
class TestTableInfoBase:
|
||||
class TestCollectionInfoBase:
|
||||
def index_string_convert(self, index_string, index_type):
|
||||
if index_string == "IDMAP" and index_type == IndexType.FLAT:
|
||||
return True
|
||||
|
@ -30,87 +30,87 @@ class TestTableInfoBase:
|
|||
|
||||
"""
|
||||
******************************************************************
|
||||
The following cases are used to test `table_info` function
|
||||
The following cases are used to test `collection_info` function
|
||||
******************************************************************
|
||||
"""
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_name_None(self, connect, table):
|
||||
def test_get_collection_info_name_None(self, connect, collection):
|
||||
'''
|
||||
target: get table info where table name is None
|
||||
method: call table_info with the table_name: None
|
||||
target: get collection info where collection name is None
|
||||
method: call collection_info with the collection_name: None
|
||||
expected: status not ok
|
||||
'''
|
||||
table_name = None
|
||||
status, info = connect.table_info(table_name)
|
||||
collection_name = None
|
||||
status, info = connect.collection_info(collection_name)
|
||||
assert not status.OK()
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_name_not_existed(self, connect, table):
|
||||
def test_get_collection_info_name_not_existed(self, connect, collection):
|
||||
'''
|
||||
target: get table info where table name does not exist
|
||||
method: call table_info with a random table_name, which is not in db
|
||||
target: get collection info where collection name does not exist
|
||||
method: call collection_info with a random collection_name, which is not in db
|
||||
expected: status not ok
|
||||
'''
|
||||
table_name = gen_unique_str("not_existed_table")
|
||||
status, info = connect.table_info(table_name)
|
||||
collection_name = gen_unique_str("not_existed_collection")
|
||||
status, info = connect.collection_info(collection_name)
|
||||
assert not status.OK()
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_invalid_table_names()
|
||||
params=gen_invalid_collection_names()
|
||||
)
|
||||
def get_table_name(self, request):
|
||||
def get_collection_name(self, request):
|
||||
yield request.param
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_name_invalid(self, connect, get_table_name):
|
||||
def test_get_collection_info_name_invalid(self, connect, get_collection_name):
|
||||
'''
|
||||
target: get table info where table name is invalid
|
||||
method: call table_info with invalid table_name
|
||||
target: get collection info where collection name is invalid
|
||||
method: call collection_info with invalid collection_name
|
||||
expected: status not ok
|
||||
'''
|
||||
table_name = get_table_name
|
||||
status, info = connect.table_info(table_name)
|
||||
collection_name = get_collection_name
|
||||
status, info = connect.collection_info(collection_name)
|
||||
assert not status.OK()
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_table_row_count(self, connect, table):
|
||||
def test_get_collection_info_collection_row_count(self, connect, collection):
|
||||
'''
|
||||
target: get row count with table_info
|
||||
method: add and delete vectors, check count in table info
|
||||
target: get row count with collection_info
|
||||
method: add and delete vectors, check count in collection info
|
||||
expected: status ok, count as expected
|
||||
'''
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
assert info.count == nb
|
||||
# delete a few vectors
|
||||
delete_ids = [ids[0], ids[-1]]
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
assert info.count == nb - 2
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_partition_stats_A(self, connect, table):
|
||||
def test_get_collection_info_partition_stats_A(self, connect, collection):
|
||||
'''
|
||||
target: get partition info in a table
|
||||
method: no partition, call table_info and check partition_stats
|
||||
target: get partition info in a collection
|
||||
method: no partition, call collection_info and check partition_stats
|
||||
expected: status ok, "_default" partition is listed
|
||||
'''
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(info)
|
||||
assert len(info.partitions_stat) == 1
|
||||
|
@ -119,19 +119,19 @@ class TestTableInfoBase:
|
|||
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_partition_stats_B(self, connect, table):
|
||||
def test_get_collection_info_partition_stats_B(self, connect, collection):
|
||||
'''
|
||||
target: get partition info in a table
|
||||
method: call table_info after partition created and check partition_stats
|
||||
target: get partition info in a collection
|
||||
method: call collection_info after partition created and check partition_stats
|
||||
expected: status ok, vectors added to partition
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(info)
|
||||
assert len(info.partitions_stat) == 2
|
||||
|
@ -139,23 +139,23 @@ class TestTableInfoBase:
|
|||
assert info.partitions_stat[1].count == nb
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_partition_stats_C(self, connect, table):
|
||||
def test_get_collection_info_partition_stats_C(self, connect, collection):
|
||||
'''
|
||||
target: get partition info in a table
|
||||
method: create two partitions, add vectors in one of the partitions, call table_info and check
|
||||
target: get partition info in a collection
|
||||
method: create two partitions, add vectors in one of the partitions, call collection_info and check
|
||||
expected: status ok, vectors added to one partition but not the other
|
||||
'''
|
||||
new_tag = "new_tag"
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
status = connect.create_partition(table, new_tag)
|
||||
status = connect.create_partition(collection, new_tag)
|
||||
assert status.OK()
|
||||
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(info)
|
||||
for partition in info.partitions_stat:
|
||||
|
@ -165,25 +165,25 @@ class TestTableInfoBase:
|
|||
assert partition.count == 0
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_partition_stats_D(self, connect, table):
|
||||
def test_get_collection_info_partition_stats_D(self, connect, collection):
|
||||
'''
|
||||
target: get partition info in a table
|
||||
method: create two partitions, add vectors in both partitions, call table_info and check
|
||||
target: get partition info in a collection
|
||||
method: create two partitions, add vectors in both partitions, call collection_info and check
|
||||
expected: status ok, vectors added to both partitions
|
||||
'''
|
||||
new_tag = "new_tag"
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
status = connect.create_partition(table, new_tag)
|
||||
status = connect.create_partition(collection, new_tag)
|
||||
assert status.OK()
|
||||
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status, ids = connect.add_vectors(table, vectors, partition_tag=new_tag)
|
||||
status, ids = connect.add_vectors(collection, vectors, partition_tag=new_tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
assert info.count == nb * 2
|
||||
for partition in info.partitions_stat:
|
||||
|
@ -205,24 +205,24 @@ class TestTableInfoBase:
|
|||
return request.param
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_after_index_created(self, connect, table, get_simple_index):
|
||||
def test_get_collection_info_after_index_created(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: test table info after index created
|
||||
method: create table, add vectors, create index and call table_info
|
||||
target: test collection info after index created
|
||||
method: create collection, add vectors, create index and call collection_info
|
||||
expected: status ok, index created and shown in segments_stat
|
||||
'''
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(info)
|
||||
index_string = info.partitions_stat[0].segments_stat[0].index_name
|
||||
|
@ -231,24 +231,24 @@ class TestTableInfoBase:
|
|||
assert nb == info.partitions_stat[0].segments_stat[0].count
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_after_create_same_index_repeatedly(self, connect, table, get_simple_index):
|
||||
def test_get_collection_info_after_create_same_index_repeatedly(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: test table info after index created repeatedly
|
||||
method: create table, add vectors, create index and call table_info multiple times
|
||||
target: test collection info after index created repeatedly
|
||||
method: create collection, add vectors, create index and call collection_info multiple times
|
||||
expected: status ok, index info shown in segments_stat
|
||||
'''
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(info)
|
||||
index_string = info.partitions_stat[0].segments_stat[0].index_name
|
||||
|
@ -257,22 +257,22 @@ class TestTableInfoBase:
|
|||
assert nb == info.partitions_stat[0].segments_stat[0].count
|
||||
|
||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||
def test_get_table_info_after_create_different_index_repeatedly(self, connect, table, get_simple_index):
|
||||
def test_get_collection_info_after_create_different_index_repeatedly(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: test table info after index created repeatedly
|
||||
method: create table, add vectors, create index and call table_info multiple times
|
||||
target: test collection info after index created repeatedly
|
||||
method: create collection, add vectors, create index and call collection_info multiple times
|
||||
expected: status ok, index info shown in segments_stat
|
||||
'''
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
index_param = {"nlist": nlist}
|
||||
for index_type in [IndexType.FLAT, IndexType.IVFLAT, IndexType.IVF_SQ8]:
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(info)
|
||||
index_string = info.partitions_stat[0].segments_stat[0].index_name
|
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
@ -285,7 +285,7 @@ class TestConnect:
|
|||
b. data_set not too large incase disconnection happens when data is underd-preparing
|
||||
c. data_set not too small incase disconnection happens when data has already been transferred
|
||||
d. make sure disconnection happens when data is in-transport
|
||||
Expected: Failure, get_table_row_count == 0
|
||||
Expected: Failure, count_collection == 0
|
||||
|
||||
'''
|
||||
pass
|
||||
|
|
|
@ -11,7 +11,7 @@ from utils import *
|
|||
|
||||
dim = 128
|
||||
index_file_size = 10
|
||||
table_id = "test_delete"
|
||||
collection_id = "test_delete"
|
||||
DELETE_TIMEOUT = 60
|
||||
nprobe = 1
|
||||
epsilon = 0.001
|
||||
|
@ -38,7 +38,7 @@ class TestDeleteBase:
|
|||
pytest.skip("Only support CPU mode")
|
||||
return request.param
|
||||
|
||||
def test_delete_vector_search(self, connect, table, get_simple_index):
|
||||
def test_delete_vector_search(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: test delete vector
|
||||
method: add vector and delete
|
||||
|
@ -47,22 +47,22 @@ class TestDeleteBase:
|
|||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(table, ids)
|
||||
status = connect.delete_by_id(collection, ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(table, top_k, vector, params=search_param)
|
||||
status, res = connect.search_vectors(collection, top_k, vector, params=search_param)
|
||||
logging.getLogger().info(res)
|
||||
assert status.OK()
|
||||
assert len(res) == 0
|
||||
|
||||
def test_delete_vector_multi_same_ids(self, connect, table, get_simple_index):
|
||||
def test_delete_vector_multi_same_ids(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: test delete vector, with some same ids
|
||||
method: add vector and delete
|
||||
|
@ -71,58 +71,58 @@ class TestDeleteBase:
|
|||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vectors(nb, dim)
|
||||
connect.add_vectors(table, vectors, ids=[1 for i in range(nb)])
|
||||
status = connect.flush([table])
|
||||
connect.add_vectors(collection, vectors, ids=[1 for i in range(nb)])
|
||||
status = connect.flush([collection])
|
||||
# Bloom filter error
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(table, [1])
|
||||
status = connect.delete_by_id(collection, [1])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(table, top_k, [vectors[0]], params=search_param)
|
||||
status, res = connect.search_vectors(collection, top_k, [vectors[0]], params=search_param)
|
||||
logging.getLogger().info(res)
|
||||
assert status.OK()
|
||||
assert len(res) == 0
|
||||
|
||||
def test_delete_vector_table_count(self, connect, table):
|
||||
def test_delete_vector_collection_count(self, connect, collection):
|
||||
'''
|
||||
target: test delete vector
|
||||
method: add vector and delete
|
||||
expected: status ok, vector deleted
|
||||
'''
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(table, ids)
|
||||
status = connect.delete_by_id(collection, ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status = connect.flush([collection])
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
assert res == 0
|
||||
|
||||
def test_delete_vector_table_count_no_flush(self, connect, table):
|
||||
def test_delete_vector_collection_count_no_flush(self, connect, collection):
|
||||
'''
|
||||
target: test delete vector
|
||||
method: add vector and delete, no flush(using auto flush)
|
||||
expected: status ok, vector deleted
|
||||
'''
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(table, ids)
|
||||
status = connect.delete_by_id(collection, ids)
|
||||
assert status.OK()
|
||||
time.sleep(2)
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
assert res == 0
|
||||
|
||||
def test_delete_vector_id_not_exised(self, connect, table, get_simple_index):
|
||||
def test_delete_vector_id_not_exised(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: test delete vector, params vector_id not existed
|
||||
method: add vector and delete
|
||||
|
@ -131,34 +131,34 @@ class TestDeleteBase:
|
|||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(table, [0])
|
||||
status = connect.delete_by_id(collection, [0])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(table, top_k, vector, params=search_param)
|
||||
status, res = connect.search_vectors(collection, top_k, vector, params=search_param)
|
||||
assert status.OK()
|
||||
assert res[0][0].id == ids[0]
|
||||
|
||||
def test_delete_vector_table_not_existed(self, connect, table):
|
||||
def test_delete_vector_collection_not_existed(self, connect, collection):
|
||||
'''
|
||||
target: test delete vector, params table_name not existed
|
||||
target: test delete vector, params collection_name not existed
|
||||
method: add vector and delete
|
||||
expected: status not ok
|
||||
'''
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
table_new = gen_unique_str()
|
||||
status = connect.delete_by_id(table_new, [0])
|
||||
collection_new = gen_unique_str()
|
||||
status = connect.delete_by_id(collection_new, [0])
|
||||
assert not status.OK()
|
||||
|
||||
def test_add_vectors_delete_vector(self, connect, table, get_simple_index):
|
||||
def test_add_vectors_delete_vector(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
method: add vectors and delete
|
||||
expected: status ok, vectors deleted
|
||||
|
@ -166,19 +166,19 @@ class TestDeleteBase:
|
|||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0], ids[-1]]
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(table, top_k, query_vecs, params=search_param)
|
||||
status, res = connect.search_vectors(collection, top_k, query_vecs, params=search_param)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(res)
|
||||
assert res[0][0].distance > epsilon
|
||||
|
@ -186,7 +186,7 @@ class TestDeleteBase:
|
|||
assert res[1][0].id == ids[1]
|
||||
assert res[2][0].distance > epsilon
|
||||
|
||||
def test_create_index_after_delete(self, connect, table, get_simple_index):
|
||||
def test_create_index_after_delete(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
method: add vectors and delete, then create index
|
||||
expected: status ok, vectors deleted, index created
|
||||
|
@ -194,19 +194,19 @@ class TestDeleteBase:
|
|||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0], ids[-1]]
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.flush([collection])
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(table, top_k, query_vecs, params=search_param)
|
||||
status, res = connect.search_vectors(collection, top_k, query_vecs, params=search_param)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(res)
|
||||
assert res[0][0].distance > epsilon
|
||||
|
@ -214,7 +214,7 @@ class TestDeleteBase:
|
|||
assert res[1][0].id == ids[1]
|
||||
assert res[2][0].distance > epsilon
|
||||
|
||||
def test_add_vector_after_delete(self, connect, table, get_simple_index):
|
||||
def test_add_vector_after_delete(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
method: add vectors and delete, then add vector
|
||||
expected: status ok, vectors deleted, vector added
|
||||
|
@ -222,22 +222,22 @@ class TestDeleteBase:
|
|||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0], ids[-1]]
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status, tmp_ids = connect.add_vectors(table, [vectors[0], vectors[-1]])
|
||||
status = connect.flush([collection])
|
||||
status, tmp_ids = connect.add_vectors(collection, [vectors[0], vectors[-1]])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(table, top_k, query_vecs, params=search_param)
|
||||
status, res = connect.search_vectors(collection, top_k, query_vecs, params=search_param)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(res)
|
||||
assert res[0][0].id == tmp_ids[0]
|
||||
|
@ -246,41 +246,41 @@ class TestDeleteBase:
|
|||
assert res[2][0].id == tmp_ids[-1]
|
||||
assert res[2][0].distance < epsilon
|
||||
|
||||
def test_delete_multiable_times(self, connect, table):
|
||||
def test_delete_multiable_times(self, connect, collection):
|
||||
'''
|
||||
method: add vectors and delete id serveral times
|
||||
expected: status ok, vectors deleted, and status ok for next delete operation
|
||||
'''
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0], ids[-1]]
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
for i in range(10):
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
assert status.OK()
|
||||
|
||||
def test_delete_no_flush_multiable_times(self, connect, table):
|
||||
def test_delete_no_flush_multiable_times(self, connect, collection):
|
||||
'''
|
||||
method: add vectors and delete id serveral times
|
||||
expected: status ok, vectors deleted, and status ok for next delete operation
|
||||
'''
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0], ids[-1]]
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
assert status.OK()
|
||||
for i in range(10):
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
assert status.OK()
|
||||
assert status.OK()
|
||||
|
||||
|
@ -303,7 +303,7 @@ class TestDeleteIndexedVectors:
|
|||
pytest.skip("Only support CPU mode")
|
||||
return request.param
|
||||
|
||||
def test_delete_vectors_after_index_created_search(self, connect, table, get_simple_index):
|
||||
def test_delete_vectors_after_index_created_search(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: test delete vector after index created
|
||||
method: add vector, create index and delete vector
|
||||
|
@ -312,22 +312,22 @@ class TestDeleteIndexedVectors:
|
|||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(table, ids)
|
||||
status = connect.delete_by_id(collection, ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(table, top_k, vector, params=search_param)
|
||||
status, res = connect.search_vectors(collection, top_k, vector, params=search_param)
|
||||
logging.getLogger().info(res)
|
||||
assert status.OK()
|
||||
assert len(res) == 0
|
||||
|
||||
def test_add_vectors_delete_vector(self, connect, table, get_simple_index):
|
||||
def test_add_vectors_delete_vector(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
method: add vectors and delete
|
||||
expected: status ok, vectors deleted
|
||||
|
@ -335,21 +335,21 @@ class TestDeleteIndexedVectors:
|
|||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0], ids[-1]]
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(table, top_k, query_vecs, params=search_param)
|
||||
status, res = connect.search_vectors(collection, top_k, query_vecs, params=search_param)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(res)
|
||||
assert res[0][0].distance > epsilon
|
||||
|
@ -375,7 +375,7 @@ class TestDeleteBinary:
|
|||
else:
|
||||
pytest.skip("Skip index Temporary")
|
||||
|
||||
def test_delete_vector_search(self, connect, jac_table, get_simple_index):
|
||||
def test_delete_vector_search(self, connect, jac_collection, get_simple_index):
|
||||
'''
|
||||
target: test delete vector
|
||||
method: add vector and delete
|
||||
|
@ -384,15 +384,15 @@ class TestDeleteBinary:
|
|||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
tmp, vector = gen_binary_vectors(1, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vector)
|
||||
status, ids = connect.add_vectors(jac_collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(jac_table, ids)
|
||||
status = connect.delete_by_id(jac_collection, ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(jac_table, top_k, vector, params=search_param)
|
||||
status, res = connect.search_vectors(jac_collection, top_k, vector, params=search_param)
|
||||
logging.getLogger().info(res)
|
||||
assert status.OK()
|
||||
assert len(res) == 0
|
||||
|
@ -400,25 +400,25 @@ class TestDeleteBinary:
|
|||
assert len(res) == 0
|
||||
|
||||
# TODO: soft delete
|
||||
def test_delete_vector_table_count(self, connect, jac_table):
|
||||
def test_delete_vector_collection_count(self, connect, jac_collection):
|
||||
'''
|
||||
target: test delete vector
|
||||
method: add vector and delete
|
||||
expected: status ok, vector deleted
|
||||
'''
|
||||
tmp, vector = gen_binary_vectors(1, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vector)
|
||||
status, ids = connect.add_vectors(jac_collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(jac_table, ids)
|
||||
status = connect.delete_by_id(jac_collection, ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status, res = connect.get_table_row_count(jac_table)
|
||||
status = connect.flush([jac_collection])
|
||||
status, res = connect.count_collection(jac_collection)
|
||||
assert status.OK()
|
||||
assert res == 0
|
||||
|
||||
def test_delete_vector_id_not_exised(self, connect, jac_table, get_simple_index):
|
||||
def test_delete_vector_id_not_exised(self, connect, jac_collection, get_simple_index):
|
||||
'''
|
||||
target: test delete vector, params vector_id not existed
|
||||
method: add vector and delete
|
||||
|
@ -427,37 +427,37 @@ class TestDeleteBinary:
|
|||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
tmp, vector = gen_binary_vectors(1, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vector)
|
||||
status, ids = connect.add_vectors(jac_collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(jac_table, [0])
|
||||
status = connect.delete_by_id(jac_collection, [0])
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
status = connect.flush([jac_collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(jac_table, top_k, vector, params=search_param)
|
||||
status, res = connect.search_vectors(jac_collection, top_k, vector, params=search_param)
|
||||
assert status.OK()
|
||||
assert res[0][0].id == ids[0]
|
||||
|
||||
def test_delete_vector_table_not_existed(self, connect, jac_table):
|
||||
def test_delete_vector_collection_not_existed(self, connect, jac_collection):
|
||||
'''
|
||||
target: test delete vector, params table_name not existed
|
||||
target: test delete vector, params collection_name not existed
|
||||
method: add vector and delete
|
||||
expected: status not ok
|
||||
'''
|
||||
tmp, vector = gen_binary_vectors(1, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vector)
|
||||
status, ids = connect.add_vectors(jac_collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
table_new = gen_unique_str()
|
||||
status = connect.delete_by_id(table_new, [0])
|
||||
table_new = gen_unique_str()
|
||||
status = connect.delete_by_id(table_new, [0])
|
||||
collection_new = gen_unique_str()
|
||||
status = connect.delete_by_id(collection_new, [0])
|
||||
collection_new = gen_unique_str()
|
||||
status = connect.delete_by_id(collection_new, [0])
|
||||
assert not status.OK()
|
||||
|
||||
def test_add_vectors_delete_vector(self, connect, jac_table, get_simple_index):
|
||||
def test_add_vectors_delete_vector(self, connect, jac_collection, get_simple_index):
|
||||
'''
|
||||
method: add vectors and delete
|
||||
expected: status ok, vectors deleted
|
||||
|
@ -465,24 +465,24 @@ class TestDeleteBinary:
|
|||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vectors)
|
||||
status, ids = connect.add_vectors(jac_collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0], ids[-1]]
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status = connect.delete_by_id(jac_table, delete_ids)
|
||||
status = connect.delete_by_id(jac_collection, delete_ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(jac_table, top_k, query_vecs, params=search_param)
|
||||
status, res = connect.search_vectors(jac_collection, top_k, query_vecs, params=search_param)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(res)
|
||||
assert res[0][0].id != ids[0]
|
||||
assert res[1][0].id == ids[1]
|
||||
assert res[2][0].id != ids[-1]
|
||||
|
||||
def test_add_after_delete_vector(self, connect, jac_table, get_simple_index):
|
||||
def test_add_after_delete_vector(self, connect, jac_collection, get_simple_index):
|
||||
'''
|
||||
method: add vectors and delete, add
|
||||
expected: status ok, vectors added
|
||||
|
@ -490,20 +490,20 @@ class TestDeleteBinary:
|
|||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vectors)
|
||||
status, ids = connect.add_vectors(jac_collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0], ids[-1]]
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status = connect.delete_by_id(jac_table, delete_ids)
|
||||
status = connect.delete_by_id(jac_collection, delete_ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status, tmp_ids = connect.add_vectors(jac_table, [vectors[0], vectors[-1]])
|
||||
status = connect.flush([jac_collection])
|
||||
status, tmp_ids = connect.add_vectors(jac_collection, [vectors[0], vectors[-1]])
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
search_param = get_search_param(index_type)
|
||||
status, res = connect.search_vectors(jac_table, top_k, query_vecs, params=search_param)
|
||||
status, res = connect.search_vectors(jac_collection, top_k, query_vecs, params=search_param)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(res)
|
||||
assert res[0][0].id == tmp_ids[0]
|
||||
|
@ -526,32 +526,32 @@ class TestDeleteIdsIngalid(object):
|
|||
yield request.param
|
||||
|
||||
@pytest.mark.level(1)
|
||||
def test_delete_vector_id_invalid(self, connect, table, gen_invalid_id):
|
||||
def test_delete_vector_id_invalid(self, connect, collection, gen_invalid_id):
|
||||
invalid_id = gen_invalid_id
|
||||
with pytest.raises(Exception) as e:
|
||||
status = connect.delete_by_id(table, [invalid_id])
|
||||
status = connect.delete_by_id(collection, [invalid_id])
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_delete_vector_ids_invalid(self, connect, table, gen_invalid_id):
|
||||
def test_delete_vector_ids_invalid(self, connect, collection, gen_invalid_id):
|
||||
invalid_id = gen_invalid_id
|
||||
with pytest.raises(Exception) as e:
|
||||
status = connect.delete_by_id(table, [1, invalid_id])
|
||||
status = connect.delete_by_id(collection, [1, invalid_id])
|
||||
|
||||
|
||||
class TestTableNameInvalid(object):
|
||||
class TestCollectionNameInvalid(object):
|
||||
"""
|
||||
Test adding vectors with invalid table names
|
||||
Test adding vectors with invalid collection names
|
||||
"""
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_invalid_table_names()
|
||||
params=gen_invalid_collection_names()
|
||||
)
|
||||
def get_table_name(self, request):
|
||||
def get_collection_name(self, request):
|
||||
yield request.param
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_delete_vectors_with_invalid_table_name(self, connect, get_table_name):
|
||||
table_name = get_table_name
|
||||
status = connect.delete_by_id(table_name, [1])
|
||||
def test_delete_vectors_with_invalid_collection_name(self, connect, get_collection_name):
|
||||
collection_name = get_collection_name
|
||||
status = connect.delete_by_id(collection_name, [1])
|
||||
assert not status.OK()
|
||||
|
||||
|
|
|
@ -10,7 +10,7 @@ from utils import *
|
|||
|
||||
dim = 128
|
||||
index_file_size = 10
|
||||
table_id = "test_flush"
|
||||
collection_id = "test_flush"
|
||||
DELETE_TIMEOUT = 60
|
||||
nprobe = 1
|
||||
tag = "1970-01-01"
|
||||
|
@ -38,117 +38,117 @@ class TestFlushBase:
|
|||
pytest.skip("Only support CPU mode")
|
||||
return request.param
|
||||
|
||||
def test_flush_table_not_existed(self, connect, table):
|
||||
def test_flush_collection_not_existed(self, connect, collection):
|
||||
'''
|
||||
target: test delete vector, params table_name not existed
|
||||
target: test delete vector, params collection_name not existed
|
||||
method: add vector and delete
|
||||
expected: status not ok
|
||||
'''
|
||||
table_new = gen_unique_str()
|
||||
status = connect.flush([table_new])
|
||||
collection_new = gen_unique_str()
|
||||
status = connect.flush([collection_new])
|
||||
assert not status.OK()
|
||||
|
||||
def test_flush_empty_table(self, connect, table):
|
||||
def test_flush_empty_collection(self, connect, collection):
|
||||
'''
|
||||
method: flush table with no vectors
|
||||
method: flush collection with no vectors
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
|
||||
def test_add_partition_flush(self, connect, table):
|
||||
def test_add_partition_flush(self, connect, collection):
|
||||
'''
|
||||
method: add vectors into partition in table, flush serveral times
|
||||
method: add vectors into partition in collection, flush serveral times
|
||||
expected: status ok
|
||||
'''
|
||||
vectors = gen_vector(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
vectors = gen_vectors(nb, dim)
|
||||
ids = [i for i in range(nb)]
|
||||
status, ids = connect.insert(table, vectors, ids)
|
||||
status = connect.flush([table])
|
||||
result, res = connect.get_table_row_count(table)
|
||||
status, ids = connect.insert(collection, vectors, ids)
|
||||
status = connect.flush([collection])
|
||||
result, res = connect.count_collection(collection)
|
||||
assert res == nb
|
||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
||||
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
result, res = connect.get_table_row_count(table)
|
||||
result, res = connect.count_collection(collection)
|
||||
assert res == 2 * nb
|
||||
|
||||
def test_add_partitions_flush(self, connect, table):
|
||||
def test_add_partitions_flush(self, connect, collection):
|
||||
'''
|
||||
method: add vectors into partitions in table, flush one
|
||||
method: add vectors into partitions in collection, flush one
|
||||
expected: status ok
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
tag_new = gen_unique_str()
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(table, tag_new)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.create_partition(collection, tag_new)
|
||||
ids = [i for i in range(nb)]
|
||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
||||
status = connect.flush([table])
|
||||
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag_new)
|
||||
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag_new)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
result, res = connect.get_table_row_count(table)
|
||||
result, res = connect.count_collection(collection)
|
||||
assert res == 2 * nb
|
||||
|
||||
def test_add_tables_flush(self, connect, table):
|
||||
def test_add_collections_flush(self, connect, collection):
|
||||
'''
|
||||
method: add vectors into tables, flush one
|
||||
method: add vectors into collections, flush one
|
||||
expected: status ok
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
table_new = gen_unique_str()
|
||||
param = {'table_name': table_new,
|
||||
collection_new = gen_unique_str()
|
||||
param = {'collection_name': collection_new,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.L2}
|
||||
status = connect.create_table(param)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(table_new, tag)
|
||||
status = connect.create_collection(param)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.create_partition(collection_new, tag)
|
||||
vectors = gen_vectors(nb, dim)
|
||||
ids = [i for i in range(nb)]
|
||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
||||
status, ids = connect.insert(table_new, vectors, ids, partition_tag=tag)
|
||||
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||
status, ids = connect.insert(collection_new, vectors, ids, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([table_new])
|
||||
status = connect.flush([collection])
|
||||
status = connect.flush([collection_new])
|
||||
assert status.OK()
|
||||
result, res = connect.get_table_row_count(table)
|
||||
result, res = connect.count_collection(collection)
|
||||
assert res == nb
|
||||
result, res = connect.get_table_row_count(table_new)
|
||||
result, res = connect.count_collection(collection_new)
|
||||
assert res == nb
|
||||
|
||||
def test_add_flush_multiable_times(self, connect, table):
|
||||
def test_add_flush_multiable_times(self, connect, collection):
|
||||
'''
|
||||
method: add vectors, flush serveral times
|
||||
expected: status ok
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
for i in range(10):
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status, res = connect.search_vectors(table, top_k, query_records=query_vecs)
|
||||
status, res = connect.search_vectors(collection, top_k, query_records=query_vecs)
|
||||
assert status.OK()
|
||||
|
||||
def test_add_flush_auto(self, connect, table):
|
||||
def test_add_flush_auto(self, connect, collection):
|
||||
'''
|
||||
method: add vectors
|
||||
expected: status ok
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
ids = [i for i in range(nb)]
|
||||
status, ids = connect.add_vectors(table, vectors, ids)
|
||||
status, ids = connect.add_vectors(collection, vectors, ids)
|
||||
assert status.OK()
|
||||
time.sleep(2)
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
assert res == nb
|
||||
|
||||
|
@ -163,7 +163,7 @@ class TestFlushBase:
|
|||
yield request.param
|
||||
|
||||
# both autoflush / flush
|
||||
def test_add_flush_same_ids(self, connect, table, same_ids):
|
||||
def test_add_flush_same_ids(self, connect, collection, same_ids):
|
||||
'''
|
||||
method: add vectors, with same ids, count(same ids) < 15, > 15
|
||||
expected: status ok
|
||||
|
@ -173,79 +173,79 @@ class TestFlushBase:
|
|||
for i, item in enumerate(ids):
|
||||
if item <= same_ids:
|
||||
ids[i] = 0
|
||||
status, ids = connect.add_vectors(table, vectors, ids)
|
||||
status, ids = connect.add_vectors(collection, vectors, ids)
|
||||
time.sleep(2)
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
assert res == nb
|
||||
|
||||
def test_delete_flush_multiable_times(self, connect, table):
|
||||
def test_delete_flush_multiable_times(self, connect, collection):
|
||||
'''
|
||||
method: delete vectors, flush serveral times
|
||||
expected: status ok
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(table, [ids[-1]])
|
||||
status = connect.delete_by_id(collection, [ids[-1]])
|
||||
assert status.OK()
|
||||
for i in range(10):
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||
status, res = connect.search_vectors(table, top_k, query_records=query_vecs)
|
||||
status, res = connect.search_vectors(collection, top_k, query_records=query_vecs)
|
||||
assert status.OK()
|
||||
|
||||
# TODO: CI fail, LOCAL pass
|
||||
def _test_table_count_during_flush(self, connect, args):
|
||||
def _test_collection_count_during_flush(self, connect, args):
|
||||
'''
|
||||
method: flush table at background, call `get_table_row_count`
|
||||
method: flush collection at background, call `count_collection`
|
||||
expected: status ok
|
||||
'''
|
||||
table = gen_unique_str()
|
||||
collection = gen_unique_str()
|
||||
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
||||
param = {'table_name': table,
|
||||
param = {'collection_name': collection,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.L2}
|
||||
milvus = get_milvus()
|
||||
milvus.connect(uri=uri)
|
||||
milvus.create_table(param)
|
||||
milvus.create_collection(param)
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = milvus.add_vectors(table, vectors, ids=[i for i in range(nb)])
|
||||
def flush(table_name):
|
||||
status, ids = milvus.add_vectors(collection, vectors, ids=[i for i in range(nb)])
|
||||
def flush(collection_name):
|
||||
milvus = get_milvus()
|
||||
milvus.connect(uri=uri)
|
||||
status = milvus.delete_by_id(table_name, [i for i in range(nb)])
|
||||
status = milvus.delete_by_id(collection_name, [i for i in range(nb)])
|
||||
assert status.OK()
|
||||
status = milvus.flush([table_name])
|
||||
status = milvus.flush([collection_name])
|
||||
assert status.OK()
|
||||
p = Process(target=flush, args=(table, ))
|
||||
p = Process(target=flush, args=(collection, ))
|
||||
p.start()
|
||||
status, res = milvus.get_table_row_count(table)
|
||||
status, res = milvus.count_collection(collection)
|
||||
assert status.OK()
|
||||
p.join()
|
||||
status, res = milvus.get_table_row_count(table)
|
||||
status, res = milvus.count_collection(collection)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(res)
|
||||
assert res == 0
|
||||
|
||||
|
||||
class TestTableNameInvalid(object):
|
||||
class TestCollectionNameInvalid(object):
|
||||
"""
|
||||
Test adding vectors with invalid table names
|
||||
Test adding vectors with invalid collection names
|
||||
"""
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_invalid_table_names()
|
||||
params=gen_invalid_collection_names()
|
||||
)
|
||||
def get_table_name(self, request):
|
||||
def get_collection_name(self, request):
|
||||
yield request.param
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_flush_with_invalid_table_name(self, connect, get_table_name):
|
||||
table_name = get_table_name
|
||||
def test_flush_with_invalid_collection_name(self, connect, get_collection_name):
|
||||
collection_name = get_collection_name
|
||||
with pytest.raises(Exception) as e:
|
||||
status, result = connect.flush(table_name)
|
||||
status, result = connect.flush(collection_name)
|
||||
|
|
|
@ -11,7 +11,7 @@ from utils import *
|
|||
|
||||
dim = 128
|
||||
index_file_size = 10
|
||||
table_id = "test_get_vector_by_id"
|
||||
collection_id = "test_get_vector_by_id"
|
||||
DELETE_TIMEOUT = 60
|
||||
nprobe = 1
|
||||
tag = "1970-01-01"
|
||||
|
@ -25,54 +25,54 @@ class TestGetBase:
|
|||
The following cases are used to test `get_vector_by_id` function
|
||||
******************************************************************
|
||||
"""
|
||||
def test_get_vector_A(self, connect, table):
|
||||
def test_get_vector_A(self, connect, collection):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vector, and get
|
||||
expected: status ok, vector returned
|
||||
'''
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(table, ids[0])
|
||||
status, res = connect.get_vector_by_id(collection, ids[0])
|
||||
assert status.OK()
|
||||
assert_equal_vector(res, vector[0])
|
||||
|
||||
def test_get_vector_B(self, connect, table):
|
||||
def test_get_vector_B(self, connect, collection):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vector, and get
|
||||
expected: status ok, vector returned
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(table, ids[0])
|
||||
status, res = connect.get_vector_by_id(collection, ids[0])
|
||||
assert status.OK()
|
||||
assert_equal_vector(res, vectors[0])
|
||||
|
||||
def test_get_vector_partition(self, connect, table):
|
||||
def test_get_vector_partition(self, connect, collection):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vector, and get
|
||||
expected: status ok, vector returned
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(table, ids[0])
|
||||
status, res = connect.get_vector_by_id(collection, ids[0])
|
||||
assert status.OK()
|
||||
assert_equal_vector(res, vectors[0])
|
||||
|
||||
def test_get_vector_multi_same_ids(self, connect, table):
|
||||
def test_get_vector_multi_same_ids(self, connect, collection):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vectors, with the same id, get vector by the given id
|
||||
|
@ -81,10 +81,10 @@ class TestGetBase:
|
|||
vectors = gen_vectors(nb, dim)
|
||||
ids = [i for i in range(nb)]
|
||||
ids[1] = 0; ids[-1] = 0
|
||||
status, ids = connect.add_vectors(table, vectors, ids=ids)
|
||||
status = connect.flush([table])
|
||||
status, ids = connect.add_vectors(collection, vectors, ids=ids)
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(table, 0)
|
||||
status, res = connect.get_vector_by_id(collection, 0)
|
||||
assert status.OK()
|
||||
assert_equal_vector(res, vectors[0])
|
||||
|
||||
|
@ -101,75 +101,75 @@ class TestGetBase:
|
|||
def get_id(self, request):
|
||||
yield request.param
|
||||
|
||||
def test_get_vector_after_delete(self, connect, table, get_id):
|
||||
def test_get_vector_after_delete(self, connect, collection, get_id):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vectors, and delete, get vector by the given id
|
||||
expected: status ok, get one vector
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
id = get_id
|
||||
status = connect.delete_by_id(table, [ids[id]])
|
||||
status = connect.delete_by_id(collection, [ids[id]])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(table, ids[id])
|
||||
status, res = connect.get_vector_by_id(collection, ids[id])
|
||||
assert status.OK()
|
||||
assert not res
|
||||
|
||||
def test_get_vector_after_delete_with_partition(self, connect, table, get_id):
|
||||
def test_get_vector_after_delete_with_partition(self, connect, collection, get_id):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vectors into partition, and delete, get vector by the given id
|
||||
expected: status ok, get one vector
|
||||
'''
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status, ids = connect.insert(table, vectors, partition_tag=tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status, ids = connect.insert(collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
id = get_id
|
||||
status = connect.delete_by_id(table, [ids[id]])
|
||||
status = connect.delete_by_id(collection, [ids[id]])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(table, ids[id])
|
||||
status, res = connect.get_vector_by_id(collection, ids[id])
|
||||
assert status.OK()
|
||||
assert not res
|
||||
|
||||
def test_get_vector_id_not_exised(self, connect, table):
|
||||
def test_get_vector_id_not_exised(self, connect, collection):
|
||||
'''
|
||||
target: test get vector, params vector_id not existed
|
||||
method: add vector and get
|
||||
expected: status ok, empty result
|
||||
'''
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(table, 1)
|
||||
status, res = connect.get_vector_by_id(collection, 1)
|
||||
assert status.OK()
|
||||
assert not res
|
||||
|
||||
def test_get_vector_table_not_existed(self, connect, table):
|
||||
def test_get_vector_collection_not_existed(self, connect, collection):
|
||||
'''
|
||||
target: test get vector, params table_name not existed
|
||||
target: test get vector, params collection_name not existed
|
||||
method: add vector and get
|
||||
expected: status not ok
|
||||
'''
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
table_new = gen_unique_str()
|
||||
status, res = connect.get_vector_by_id(table_new, 1)
|
||||
collection_new = gen_unique_str()
|
||||
status, res = connect.get_vector_by_id(collection_new, 1)
|
||||
assert not status.OK()
|
||||
|
||||
|
||||
|
@ -205,7 +205,7 @@ class TestGetIndexedVectors:
|
|||
def get_id(self, request):
|
||||
yield request.param
|
||||
|
||||
def test_get_vectors_after_index_created(self, connect, table, get_simple_index, get_id):
|
||||
def test_get_vectors_after_index_created(self, connect, collection, get_simple_index, get_id):
|
||||
'''
|
||||
target: test get vector after index created
|
||||
method: add vector, create index and get vector
|
||||
|
@ -214,20 +214,20 @@ class TestGetIndexedVectors:
|
|||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
id = get_id
|
||||
status, res = connect.get_vector_by_id(table, ids[id])
|
||||
status, res = connect.get_vector_by_id(collection, ids[id])
|
||||
assert status.OK()
|
||||
logging.getLogger().info(res)
|
||||
assert status.OK()
|
||||
assert_equal_vector(res, vectors[id])
|
||||
|
||||
def test_get_vector_after_delete(self, connect, table, get_simple_index, get_id):
|
||||
def test_get_vector_after_delete(self, connect, collection, get_simple_index, get_id):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vectors, and delete, get vector by the given id
|
||||
|
@ -236,22 +236,22 @@ class TestGetIndexedVectors:
|
|||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
id = get_id
|
||||
status = connect.delete_by_id(table, [ids[id]])
|
||||
status = connect.delete_by_id(collection, [ids[id]])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(table, ids[id])
|
||||
status, res = connect.get_vector_by_id(collection, ids[id])
|
||||
assert status.OK()
|
||||
assert not res
|
||||
|
||||
def test_get_vector_partition(self, connect, table, get_simple_index, get_id):
|
||||
def test_get_vector_partition(self, connect, collection, get_simple_index, get_id):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vector, and get
|
||||
|
@ -260,16 +260,16 @@ class TestGetIndexedVectors:
|
|||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
ids = [i for i in range(nb)]
|
||||
status, ids = connect.add_vectors(table, vectors, ids, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(collection, vectors, ids, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
id = get_id
|
||||
status, res = connect.get_vector_by_id(table, ids[id])
|
||||
status, res = connect.get_vector_by_id(collection, ids[id])
|
||||
assert status.OK()
|
||||
assert_equal_vector(res, vectors[id])
|
||||
|
||||
|
@ -280,37 +280,37 @@ class TestGetBinary:
|
|||
The following cases are used to test `get_vector_by_id` function
|
||||
******************************************************************
|
||||
"""
|
||||
def test_get_vector_A(self, connect, jac_table):
|
||||
def test_get_vector_A(self, connect, jac_collection):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vector, and get
|
||||
expected: status ok, vector returned
|
||||
'''
|
||||
tmp, vector = gen_binary_vectors(1, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vector)
|
||||
status, ids = connect.add_vectors(jac_collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(jac_table, ids[0])
|
||||
status, res = connect.get_vector_by_id(jac_collection, ids[0])
|
||||
assert status.OK()
|
||||
assert res == vector[0]
|
||||
|
||||
def test_get_vector_B(self, connect, jac_table):
|
||||
def test_get_vector_B(self, connect, jac_collection):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vector, and get
|
||||
expected: status ok, vector returned
|
||||
'''
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vectors)
|
||||
status, ids = connect.add_vectors(jac_collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(jac_table, ids[0])
|
||||
status, res = connect.get_vector_by_id(jac_collection, ids[0])
|
||||
assert status.OK()
|
||||
assert res == vectors[0]
|
||||
|
||||
def test_get_vector_multi_same_ids(self, connect, jac_table):
|
||||
def test_get_vector_multi_same_ids(self, connect, jac_collection):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vectors, with the same id, get vector by the given id
|
||||
|
@ -319,56 +319,56 @@ class TestGetBinary:
|
|||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
ids = [i for i in range(nb)]
|
||||
ids[0] = 0; ids[-1] = 0
|
||||
status, ids = connect.add_vectors(jac_table, vectors, ids=ids)
|
||||
status = connect.flush([jac_table])
|
||||
status, ids = connect.add_vectors(jac_collection, vectors, ids=ids)
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(jac_table, 0)
|
||||
status, res = connect.get_vector_by_id(jac_collection, 0)
|
||||
assert status.OK()
|
||||
assert res == vectors[0]
|
||||
|
||||
def test_get_vector_id_not_exised(self, connect, jac_table):
|
||||
def test_get_vector_id_not_exised(self, connect, jac_collection):
|
||||
'''
|
||||
target: test get vector, params vector_id not existed
|
||||
method: add vector and get
|
||||
expected: status ok, empty result
|
||||
'''
|
||||
tmp, vector = gen_binary_vectors(1, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vector)
|
||||
status, ids = connect.add_vectors(jac_collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(jac_table, 1)
|
||||
status, res = connect.get_vector_by_id(jac_collection, 1)
|
||||
assert status.OK()
|
||||
assert not res
|
||||
|
||||
def test_get_vector_table_not_existed(self, connect, jac_table):
|
||||
def test_get_vector_collection_not_existed(self, connect, jac_collection):
|
||||
'''
|
||||
target: test get vector, params table_name not existed
|
||||
target: test get vector, params collection_name not existed
|
||||
method: add vector and get
|
||||
expected: status not ok
|
||||
'''
|
||||
tmp, vector = gen_binary_vectors(1, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vector)
|
||||
status, ids = connect.add_vectors(jac_collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
table_new = gen_unique_str()
|
||||
status, res = connect.get_vector_by_id(table_new, 1)
|
||||
collection_new = gen_unique_str()
|
||||
status, res = connect.get_vector_by_id(collection_new, 1)
|
||||
assert not status.OK()
|
||||
|
||||
def test_get_vector_partition(self, connect, jac_table):
|
||||
def test_get_vector_partition(self, connect, jac_collection):
|
||||
'''
|
||||
target: test get_vector_by_id
|
||||
method: add vector, and get
|
||||
expected: status ok, vector returned
|
||||
'''
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
status = connect.create_partition(jac_table, tag)
|
||||
status, ids = connect.add_vectors(jac_table, vectors, partition_tag=tag)
|
||||
status = connect.create_partition(jac_collection, tag)
|
||||
status, ids = connect.add_vectors(jac_collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_vector_by_id(jac_table, ids[0])
|
||||
status, res = connect.get_vector_by_id(jac_collection, ids[0])
|
||||
assert status.OK()
|
||||
assert res == vectors[0]
|
||||
|
||||
|
@ -387,26 +387,26 @@ class TestGetVectorIdIngalid(object):
|
|||
yield request.param
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_get_vector_id_invalid(self, connect, table, gen_invalid_id):
|
||||
def test_get_vector_id_invalid(self, connect, collection, gen_invalid_id):
|
||||
invalid_id = gen_invalid_id
|
||||
with pytest.raises(Exception) as e:
|
||||
status = connect.get_vector_by_id(table, invalid_id)
|
||||
status = connect.get_vector_by_id(collection, invalid_id)
|
||||
|
||||
|
||||
class TestTableNameInvalid(object):
|
||||
class TestCollectionNameInvalid(object):
|
||||
"""
|
||||
Test adding vectors with invalid table names
|
||||
Test adding vectors with invalid collection names
|
||||
"""
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_invalid_table_names()
|
||||
params=gen_invalid_collection_names()
|
||||
)
|
||||
def get_table_name(self, request):
|
||||
def get_collection_name(self, request):
|
||||
yield request.param
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_get_vectors_with_invalid_table_name(self, connect, get_table_name):
|
||||
table_name = get_table_name
|
||||
def test_get_vectors_with_invalid_collection_name(self, connect, get_collection_name):
|
||||
collection_name = get_collection_name
|
||||
vectors = gen_vectors(1, dim)
|
||||
status, result = connect.get_vector_by_id(table_name, 1)
|
||||
status, result = connect.get_vector_by_id(collection_name, 1)
|
||||
assert not status.OK()
|
||||
|
|
|
@ -19,13 +19,13 @@ nb = 6000
|
|||
|
||||
|
||||
class TestGetVectorIdsBase:
|
||||
def get_valid_segment_name(self, connect, table):
|
||||
def get_valid_segment_name(self, connect, collection):
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
return info.partitions_stat[0].segments_stat[0].segment_name
|
||||
|
@ -36,88 +36,88 @@ class TestGetVectorIdsBase:
|
|||
******************************************************************
|
||||
"""
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_table_name_None(self, connect, table):
|
||||
def test_get_vector_ids_collection_name_None(self, connect, collection):
|
||||
'''
|
||||
target: get vector ids where table name is None
|
||||
method: call get_vector_ids with the table_name: None
|
||||
target: get vector ids where collection name is None
|
||||
method: call get_vector_ids with the collection_name: None
|
||||
expected: exception raised
|
||||
'''
|
||||
table_name = None
|
||||
segment_name = self.get_valid_segment_name(connect, table)
|
||||
collection_name = None
|
||||
segment_name = self.get_valid_segment_name(connect, collection)
|
||||
with pytest.raises(Exception) as e:
|
||||
status, vector_ids = connect.get_vector_ids(table_name, segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(collection_name, segment_name)
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_table_name_not_existed(self, connect, table):
|
||||
def test_get_vector_ids_collection_name_not_existed(self, connect, collection):
|
||||
'''
|
||||
target: get vector ids where table name does not exist
|
||||
method: call get_vector_ids with a random table_name, which is not in db
|
||||
target: get vector ids where collection name does not exist
|
||||
method: call get_vector_ids with a random collection_name, which is not in db
|
||||
expected: status not ok
|
||||
'''
|
||||
table_name = gen_unique_str("not_existed_table")
|
||||
segment_name = self.get_valid_segment_name(connect, table)
|
||||
status, vector_ids = connect.get_vector_ids(table_name, segment_name)
|
||||
collection_name = gen_unique_str("not_existed_collection")
|
||||
segment_name = self.get_valid_segment_name(connect, collection)
|
||||
status, vector_ids = connect.get_vector_ids(collection_name, segment_name)
|
||||
assert not status.OK()
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_invalid_table_names()
|
||||
params=gen_invalid_collection_names()
|
||||
)
|
||||
def get_table_name(self, request):
|
||||
def get_collection_name(self, request):
|
||||
yield request.param
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_table_name_invalid(self, connect, table, get_table_name):
|
||||
def test_get_vector_ids_collection_name_invalid(self, connect, collection, get_collection_name):
|
||||
'''
|
||||
target: get vector ids where table name is invalid
|
||||
method: call get_vector_ids with invalid table_name
|
||||
target: get vector ids where collection name is invalid
|
||||
method: call get_vector_ids with invalid collection_name
|
||||
expected: status not ok
|
||||
'''
|
||||
table_name = get_table_name
|
||||
segment_name = self.get_valid_segment_name(connect, table)
|
||||
status, vector_ids = connect.get_vector_ids(table_name, segment_name)
|
||||
collection_name = get_collection_name
|
||||
segment_name = self.get_valid_segment_name(connect, collection)
|
||||
status, vector_ids = connect.get_vector_ids(collection_name, segment_name)
|
||||
assert not status.OK()
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_segment_name_None(self, connect, table):
|
||||
def test_get_vector_ids_segment_name_None(self, connect, collection):
|
||||
'''
|
||||
target: get vector ids where segment name is None
|
||||
method: call get_vector_ids with the segment_name: None
|
||||
expected: exception raised
|
||||
'''
|
||||
valid_segment_name = self.get_valid_segment_name(connect, table)
|
||||
valid_segment_name = self.get_valid_segment_name(connect, collection)
|
||||
segment = None
|
||||
with pytest.raises(Exception) as e:
|
||||
status, vector_ids = connect.get_vector_ids(table, segment)
|
||||
status, vector_ids = connect.get_vector_ids(collection, segment)
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_segment_name_not_existed(self, connect, table):
|
||||
def test_get_vector_ids_segment_name_not_existed(self, connect, collection):
|
||||
'''
|
||||
target: get vector ids where segment name does not exist
|
||||
method: call get_vector_ids with a random segment name
|
||||
expected: status not ok
|
||||
'''
|
||||
valid_segment_name = self.get_valid_segment_name(connect, table)
|
||||
valid_segment_name = self.get_valid_segment_name(connect, collection)
|
||||
segment = gen_unique_str("not_existed_segment")
|
||||
status, vector_ids = connect.get_vector_ids(table, segment)
|
||||
status, vector_ids = connect.get_vector_ids(collection, segment)
|
||||
logging.getLogger().info(vector_ids)
|
||||
assert not status.OK()
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_without_index_A(self, connect, table):
|
||||
def test_get_vector_ids_without_index_A(self, connect, collection):
|
||||
'''
|
||||
target: get vector ids when there is no index
|
||||
method: call get_vector_ids and check if the segment contains vectors
|
||||
expected: status ok
|
||||
'''
|
||||
vectors = gen_vector(10, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
|
@ -125,23 +125,23 @@ class TestGetVectorIdsBase:
|
|||
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_without_index_B(self, connect, table):
|
||||
def test_get_vector_ids_without_index_B(self, connect, collection):
|
||||
'''
|
||||
target: get vector ids when there is no index but with partition
|
||||
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
vectors = gen_vector(10, dim)
|
||||
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
assert info.partitions_stat[1].tag == tag
|
||||
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
|
@ -160,7 +160,7 @@ class TestGetVectorIdsBase:
|
|||
return request.param
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_with_index_A(self, connect, table, get_simple_index):
|
||||
def test_get_vector_ids_with_index_A(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: get vector ids when there is index
|
||||
method: call get_vector_ids and check if the segment contains vectors
|
||||
|
@ -168,65 +168,65 @@ class TestGetVectorIdsBase:
|
|||
'''
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
vectors = gen_vector(10, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
assert vector_ids[i] == ids[i]
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_with_index_B(self, connect, table, get_simple_index):
|
||||
def test_get_vector_ids_with_index_B(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
target: get vector ids when there is index and with partition
|
||||
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
status = connect.create_index(table, index_type, index_param)
|
||||
status = connect.create_index(collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
vectors = gen_vector(10, dim)
|
||||
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
assert info.partitions_stat[1].tag == tag
|
||||
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
assert vector_ids[i] == ids[i]
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_after_delete_vectors(self, connect, table):
|
||||
def test_get_vector_ids_after_delete_vectors(self, connect, collection):
|
||||
'''
|
||||
target: get vector ids after vectors are deleted
|
||||
method: add vectors and delete a few, call get_vector_ids
|
||||
expected: status ok, vector_ids decreased after vectors deleted
|
||||
'''
|
||||
vectors = gen_vector(2, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0]]
|
||||
status = connect.delete_by_id(table, delete_ids)
|
||||
status = connect.flush([table])
|
||||
status = connect.delete_by_id(collection, delete_ids)
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(table)
|
||||
status, info = connect.collection_info(collection)
|
||||
assert status.OK()
|
||||
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
assert len(vector_ids) == 1
|
||||
assert vector_ids[0] == ids[1]
|
||||
|
||||
|
@ -238,20 +238,20 @@ class TestGetVectorIdsIP:
|
|||
******************************************************************
|
||||
"""
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_without_index_A(self, connect, ip_table):
|
||||
def test_get_vector_ids_without_index_A(self, connect, ip_collection):
|
||||
'''
|
||||
target: get vector ids when there is no index
|
||||
method: call get_vector_ids and check if the segment contains vectors
|
||||
expected: status ok
|
||||
'''
|
||||
vectors = gen_vector(10, dim)
|
||||
status, ids = connect.add_vectors(ip_table, vectors)
|
||||
status, ids = connect.add_vectors(ip_collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([ip_table])
|
||||
status = connect.flush([ip_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(ip_table)
|
||||
status, info = connect.collection_info(ip_collection)
|
||||
assert status.OK()
|
||||
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
|
@ -259,23 +259,23 @@ class TestGetVectorIdsIP:
|
|||
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_without_index_B(self, connect, ip_table):
|
||||
def test_get_vector_ids_without_index_B(self, connect, ip_collection):
|
||||
'''
|
||||
target: get vector ids when there is no index but with partition
|
||||
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(ip_table, tag)
|
||||
status = connect.create_partition(ip_collection, tag)
|
||||
assert status.OK()
|
||||
vectors = gen_vector(10, dim)
|
||||
status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(ip_collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([ip_table])
|
||||
status = connect.flush([ip_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(ip_table)
|
||||
status, info = connect.collection_info(ip_collection)
|
||||
assert status.OK()
|
||||
assert info.partitions_stat[1].tag == tag
|
||||
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
|
@ -294,7 +294,7 @@ class TestGetVectorIdsIP:
|
|||
return request.param
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_with_index_A(self, connect, ip_table, get_simple_index):
|
||||
def test_get_vector_ids_with_index_A(self, connect, ip_collection, get_simple_index):
|
||||
'''
|
||||
target: get vector ids when there is index
|
||||
method: call get_vector_ids and check if the segment contains vectors
|
||||
|
@ -302,65 +302,65 @@ class TestGetVectorIdsIP:
|
|||
'''
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
status = connect.create_index(ip_table, index_type, index_param)
|
||||
status = connect.create_index(ip_collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
vectors = gen_vector(10, dim)
|
||||
status, ids = connect.add_vectors(ip_table, vectors)
|
||||
status, ids = connect.add_vectors(ip_collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([ip_table])
|
||||
status = connect.flush([ip_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(ip_table)
|
||||
status, info = connect.collection_info(ip_collection)
|
||||
assert status.OK()
|
||||
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
assert vector_ids[i] == ids[i]
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_with_index_B(self, connect, ip_table, get_simple_index):
|
||||
def test_get_vector_ids_with_index_B(self, connect, ip_collection, get_simple_index):
|
||||
'''
|
||||
target: get vector ids when there is index and with partition
|
||||
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(ip_table, tag)
|
||||
status = connect.create_partition(ip_collection, tag)
|
||||
assert status.OK()
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
status = connect.create_index(ip_table, index_type, index_param)
|
||||
status = connect.create_index(ip_collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
vectors = gen_vector(10, dim)
|
||||
status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(ip_collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([ip_table])
|
||||
status = connect.flush([ip_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(ip_table)
|
||||
status, info = connect.collection_info(ip_collection)
|
||||
assert status.OK()
|
||||
assert info.partitions_stat[1].tag == tag
|
||||
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
assert vector_ids[i] == ids[i]
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_after_delete_vectors(self, connect, ip_table):
|
||||
def test_get_vector_ids_after_delete_vectors(self, connect, ip_collection):
|
||||
'''
|
||||
target: get vector ids after vectors are deleted
|
||||
method: add vectors and delete a few, call get_vector_ids
|
||||
expected: status ok, vector_ids decreased after vectors deleted
|
||||
'''
|
||||
vectors = gen_vector(2, dim)
|
||||
status, ids = connect.add_vectors(ip_table, vectors)
|
||||
status, ids = connect.add_vectors(ip_collection, vectors)
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0]]
|
||||
status = connect.delete_by_id(ip_table, delete_ids)
|
||||
status = connect.flush([ip_table])
|
||||
status = connect.delete_by_id(ip_collection, delete_ids)
|
||||
status = connect.flush([ip_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(ip_table)
|
||||
status, info = connect.collection_info(ip_collection)
|
||||
assert status.OK()
|
||||
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
assert len(vector_ids) == 1
|
||||
assert vector_ids[0] == ids[1]
|
||||
|
||||
|
@ -372,20 +372,20 @@ class TestGetVectorIdsJAC:
|
|||
******************************************************************
|
||||
"""
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_without_index_A(self, connect, jac_table):
|
||||
def test_get_vector_ids_without_index_A(self, connect, jac_collection):
|
||||
'''
|
||||
target: get vector ids when there is no index
|
||||
method: call get_vector_ids and check if the segment contains vectors
|
||||
expected: status ok
|
||||
'''
|
||||
tmp, vectors = gen_binary_vectors(10, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vectors)
|
||||
status, ids = connect.add_vectors(jac_collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(jac_table)
|
||||
status, info = connect.collection_info(jac_collection)
|
||||
assert status.OK()
|
||||
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
|
@ -393,23 +393,23 @@ class TestGetVectorIdsJAC:
|
|||
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_without_index_B(self, connect, jac_table):
|
||||
def test_get_vector_ids_without_index_B(self, connect, jac_collection):
|
||||
'''
|
||||
target: get vector ids when there is no index but with partition
|
||||
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(jac_table, tag)
|
||||
status = connect.create_partition(jac_collection, tag)
|
||||
assert status.OK()
|
||||
tmp, vectors = gen_binary_vectors(10, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vectors, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(jac_collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(jac_table)
|
||||
status, info = connect.collection_info(jac_collection)
|
||||
assert status.OK()
|
||||
assert info.partitions_stat[1].tag == tag
|
||||
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
|
@ -427,7 +427,7 @@ class TestGetVectorIdsJAC:
|
|||
pytest.skip("Skip index Temporary")
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_with_index_A(self, connect, jac_table, get_jaccard_index):
|
||||
def test_get_vector_ids_with_index_A(self, connect, jac_collection, get_jaccard_index):
|
||||
'''
|
||||
target: get vector ids when there is index
|
||||
method: call get_vector_ids and check if the segment contains vectors
|
||||
|
@ -435,64 +435,64 @@ class TestGetVectorIdsJAC:
|
|||
'''
|
||||
index_param = get_jaccard_index["index_param"]
|
||||
index_type = get_jaccard_index["index_type"]
|
||||
status = connect.create_index(jac_table, index_type, index_param)
|
||||
status = connect.create_index(jac_collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
tmp, vectors = gen_binary_vectors(10, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vectors)
|
||||
status, ids = connect.add_vectors(jac_collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(jac_table)
|
||||
status, info = connect.collection_info(jac_collection)
|
||||
assert status.OK()
|
||||
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
assert vector_ids[i] == ids[i]
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_with_index_B(self, connect, jac_table, get_jaccard_index):
|
||||
def test_get_vector_ids_with_index_B(self, connect, jac_collection, get_jaccard_index):
|
||||
'''
|
||||
target: get vector ids when there is index and with partition
|
||||
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(jac_table, tag)
|
||||
status = connect.create_partition(jac_collection, tag)
|
||||
assert status.OK()
|
||||
index_param = get_jaccard_index["index_param"]
|
||||
index_type = get_jaccard_index["index_type"]
|
||||
status = connect.create_index(jac_table, index_type, index_param)
|
||||
status = connect.create_index(jac_collection, index_type, index_param)
|
||||
assert status.OK()
|
||||
tmp, vectors = gen_binary_vectors(10, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vectors, partition_tag=tag)
|
||||
status, ids = connect.add_vectors(jac_collection, vectors, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(jac_table)
|
||||
status, info = connect.collection_info(jac_collection)
|
||||
assert status.OK()
|
||||
assert info.partitions_stat[1].tag == tag
|
||||
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||
# vector_ids should match ids
|
||||
assert len(vector_ids) == 10
|
||||
for i in range(10):
|
||||
assert vector_ids[i] == ids[i]
|
||||
|
||||
@pytest.mark.timeout(GET_TIMEOUT)
|
||||
def test_get_vector_ids_after_delete_vectors(self, connect, jac_table):
|
||||
def test_get_vector_ids_after_delete_vectors(self, connect, jac_collection):
|
||||
'''
|
||||
target: get vector ids after vectors are deleted
|
||||
method: add vectors and delete a few, call get_vector_ids
|
||||
expected: status ok, vector_ids decreased after vectors deleted
|
||||
'''
|
||||
tmp, vectors = gen_binary_vectors(2, dim)
|
||||
status, ids = connect.add_vectors(jac_table, vectors)
|
||||
status, ids = connect.add_vectors(jac_collection, vectors)
|
||||
assert status.OK()
|
||||
delete_ids = [ids[0]]
|
||||
status = connect.delete_by_id(jac_table, delete_ids)
|
||||
status = connect.flush([jac_table])
|
||||
status = connect.delete_by_id(jac_collection, delete_ids)
|
||||
status = connect.flush([jac_collection])
|
||||
assert status.OK()
|
||||
status, info = connect.table_info(jac_table)
|
||||
status, info = connect.collection_info(jac_collection)
|
||||
assert status.OK()
|
||||
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||
assert len(vector_ids) == 1
|
||||
assert vector_ids[0] == ids[1]
|
File diff suppressed because it is too large
Load Diff
|
@ -12,7 +12,7 @@ from utils import *
|
|||
|
||||
dim = 128
|
||||
index_file_size = 10
|
||||
table_id = "test_mix"
|
||||
collection_id = "test_mix"
|
||||
add_interval_time = 2
|
||||
vectors = gen_vectors(10000, dim)
|
||||
vectors = sklearn.preprocessing.normalize(vectors, axis=1, norm='l2')
|
||||
|
@ -29,35 +29,35 @@ class TestMixBase:
|
|||
# disable
|
||||
def _test_search_during_createIndex(self, args):
|
||||
loops = 10000
|
||||
table = gen_unique_str()
|
||||
collection = gen_unique_str()
|
||||
query_vecs = [vectors[0], vectors[1]]
|
||||
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
||||
id_0 = 0; id_1 = 0
|
||||
milvus_instance = get_milvus(args["handler"])
|
||||
milvus_instance.connect(uri=uri)
|
||||
milvus_instance.create_table({'table_name': table,
|
||||
milvus_instance.create_collection({'collection_name': collection,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.L2})
|
||||
for i in range(10):
|
||||
status, ids = milvus_instance.add_vectors(table, vectors)
|
||||
status, ids = milvus_instance.add_vectors(collection, vectors)
|
||||
# logging.getLogger().info(ids)
|
||||
if i == 0:
|
||||
id_0 = ids[0]; id_1 = ids[1]
|
||||
def create_index(milvus_instance):
|
||||
logging.getLogger().info("In create index")
|
||||
status = milvus_instance.create_index(table, index_params)
|
||||
status = milvus_instance.create_index(collection, index_params)
|
||||
logging.getLogger().info(status)
|
||||
status, result = milvus_instance.describe_index(table)
|
||||
status, result = milvus_instance.describe_index(collection)
|
||||
logging.getLogger().info(result)
|
||||
def add_vectors(milvus_instance):
|
||||
logging.getLogger().info("In add vectors")
|
||||
status, ids = milvus_instance.add_vectors(table, vectors)
|
||||
status, ids = milvus_instance.add_vectors(collection, vectors)
|
||||
logging.getLogger().info(status)
|
||||
def search(milvus_instance):
|
||||
logging.getLogger().info("In search vectors")
|
||||
for i in range(loops):
|
||||
status, result = milvus_instance.search_vectors(table, top_k, nprobe, query_vecs)
|
||||
status, result = milvus_instance.search_vectors(collection, top_k, nprobe, query_vecs)
|
||||
logging.getLogger().info(status)
|
||||
assert result[0][0].id == id_0
|
||||
assert result[1][0].id == id_1
|
||||
|
@ -72,82 +72,82 @@ class TestMixBase:
|
|||
p_create.join()
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_mix_multi_tables(self, connect):
|
||||
def test_mix_multi_collections(self, connect):
|
||||
'''
|
||||
target: test functions with multiple tables of different metric_types and index_types
|
||||
method: create 60 tables which 30 are L2 and the other are IP, add vectors into them
|
||||
target: test functions with multiple collections of different metric_types and index_types
|
||||
method: create 60 collections which 30 are L2 and the other are IP, add vectors into them
|
||||
and test describe index and search
|
||||
expected: status ok
|
||||
'''
|
||||
nq = 10000
|
||||
table_list = []
|
||||
collection_list = []
|
||||
idx = []
|
||||
index_param = {'nlist': nlist}
|
||||
|
||||
#create table and add vectors
|
||||
#create collection and add vectors
|
||||
for i in range(30):
|
||||
table_name = gen_unique_str('test_mix_multi_tables')
|
||||
table_list.append(table_name)
|
||||
param = {'table_name': table_name,
|
||||
collection_name = gen_unique_str('test_mix_multi_collections')
|
||||
collection_list.append(collection_name)
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.L2}
|
||||
connect.create_table(param)
|
||||
status, ids = connect.add_vectors(table_name=table_name, records=vectors)
|
||||
connect.create_collection(param)
|
||||
status, ids = connect.add_vectors(collection_name=collection_name, records=vectors)
|
||||
idx.append(ids[0])
|
||||
idx.append(ids[10])
|
||||
idx.append(ids[20])
|
||||
assert status.OK()
|
||||
for i in range(30):
|
||||
table_name = gen_unique_str('test_mix_multi_tables')
|
||||
table_list.append(table_name)
|
||||
param = {'table_name': table_name,
|
||||
collection_name = gen_unique_str('test_mix_multi_collections')
|
||||
collection_list.append(collection_name)
|
||||
param = {'collection_name': collection_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.IP}
|
||||
connect.create_table(param)
|
||||
status, ids = connect.add_vectors(table_name=table_name, records=vectors)
|
||||
connect.create_collection(param)
|
||||
status, ids = connect.add_vectors(collection_name=collection_name, records=vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table_name])
|
||||
status = connect.flush([collection_name])
|
||||
assert status.OK()
|
||||
idx.append(ids[0])
|
||||
idx.append(ids[10])
|
||||
idx.append(ids[20])
|
||||
assert status.OK()
|
||||
for i in range(10):
|
||||
status = connect.create_index(table_list[i], IndexType.FLAT, index_param)
|
||||
status = connect.create_index(collection_list[i], IndexType.FLAT, index_param)
|
||||
assert status.OK()
|
||||
status = connect.create_index(table_list[30 + i], IndexType.FLAT, index_param)
|
||||
status = connect.create_index(collection_list[30 + i], IndexType.FLAT, index_param)
|
||||
assert status.OK()
|
||||
status = connect.create_index(table_list[10 + i], IndexType.IVFLAT, index_param)
|
||||
status = connect.create_index(collection_list[10 + i], IndexType.IVFLAT, index_param)
|
||||
assert status.OK()
|
||||
status = connect.create_index(table_list[40 + i], IndexType.IVFLAT, index_param)
|
||||
status = connect.create_index(collection_list[40 + i], IndexType.IVFLAT, index_param)
|
||||
assert status.OK()
|
||||
status = connect.create_index(table_list[20 + i], IndexType.IVF_SQ8, index_param)
|
||||
status = connect.create_index(collection_list[20 + i], IndexType.IVF_SQ8, index_param)
|
||||
assert status.OK()
|
||||
status = connect.create_index(table_list[50 + i], IndexType.IVF_SQ8, index_param)
|
||||
status = connect.create_index(collection_list[50 + i], IndexType.IVF_SQ8, index_param)
|
||||
assert status.OK()
|
||||
|
||||
#describe index
|
||||
for i in range(10):
|
||||
status, result = connect.describe_index(table_list[i])
|
||||
status, result = connect.describe_index(collection_list[i])
|
||||
assert result._index_type == IndexType.FLAT
|
||||
status, result = connect.describe_index(table_list[10 + i])
|
||||
status, result = connect.describe_index(collection_list[10 + i])
|
||||
assert result._index_type == IndexType.IVFLAT
|
||||
status, result = connect.describe_index(table_list[20 + i])
|
||||
status, result = connect.describe_index(collection_list[20 + i])
|
||||
assert result._index_type == IndexType.IVF_SQ8
|
||||
status, result = connect.describe_index(table_list[30 + i])
|
||||
status, result = connect.describe_index(collection_list[30 + i])
|
||||
assert result._index_type == IndexType.FLAT
|
||||
status, result = connect.describe_index(table_list[40 + i])
|
||||
status, result = connect.describe_index(collection_list[40 + i])
|
||||
assert result._index_type == IndexType.IVFLAT
|
||||
status, result = connect.describe_index(table_list[50 + i])
|
||||
status, result = connect.describe_index(collection_list[50 + i])
|
||||
assert result._index_type == IndexType.IVF_SQ8
|
||||
|
||||
#search
|
||||
query_vecs = [vectors[0], vectors[10], vectors[20]]
|
||||
for i in range(60):
|
||||
table = table_list[i]
|
||||
status, result = connect.search_vectors(table, top_k, query_records=query_vecs, params={"nprobe": 1})
|
||||
collection = collection_list[i]
|
||||
status, result = connect.search_vectors(collection, top_k, query_records=query_vecs, params={"nprobe": 1})
|
||||
assert status.OK()
|
||||
assert len(result) == len(query_vecs)
|
||||
logging.getLogger().info(i)
|
||||
|
|
|
@ -11,7 +11,7 @@ from utils import *
|
|||
|
||||
dim = 128
|
||||
index_file_size = 10
|
||||
table_id = "test_partition"
|
||||
collection_id = "test_partition"
|
||||
ADD_TIMEOUT = 60
|
||||
nprobe = 1
|
||||
tag = "1970-01-01"
|
||||
|
@ -24,58 +24,58 @@ class TestCreateBase:
|
|||
The following cases are used to test `create_partition` function
|
||||
******************************************************************
|
||||
"""
|
||||
def test_create_partition(self, connect, table):
|
||||
def test_create_partition(self, connect, collection):
|
||||
'''
|
||||
target: test create partition, check status returned
|
||||
method: call function: create_partition
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
|
||||
def test_create_partition_repeat(self, connect, table):
|
||||
def test_create_partition_repeat(self, connect, collection):
|
||||
'''
|
||||
target: test create partition, check status returned
|
||||
method: call function: create_partition
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert not status.OK()
|
||||
|
||||
def test_create_partition_table_not_existed(self, connect):
|
||||
def test_create_partition_collection_not_existed(self, connect):
|
||||
'''
|
||||
target: test create partition, its owner table name not existed in db, check status returned
|
||||
target: test create partition, its owner collection name not existed in db, check status returned
|
||||
method: call function: create_partition
|
||||
expected: status not ok
|
||||
'''
|
||||
table_name = gen_unique_str()
|
||||
status = connect.create_partition(table_name, tag)
|
||||
collection_name = gen_unique_str()
|
||||
status = connect.create_partition(collection_name, tag)
|
||||
assert not status.OK()
|
||||
|
||||
def test_create_partition_tag_name_None(self, connect, table):
|
||||
def test_create_partition_tag_name_None(self, connect, collection):
|
||||
'''
|
||||
target: test create partition, tag name set None, check status returned
|
||||
method: call function: create_partition
|
||||
expected: status ok
|
||||
'''
|
||||
tag_name = None
|
||||
status = connect.create_partition(table, tag_name)
|
||||
status = connect.create_partition(collection, tag_name)
|
||||
assert not status.OK()
|
||||
|
||||
def test_create_different_partition_tags(self, connect, table):
|
||||
def test_create_different_partition_tags(self, connect, collection):
|
||||
'''
|
||||
target: test create partition twice with different names
|
||||
method: call function: create_partition, and again
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
tag_name = gen_unique_str()
|
||||
status = connect.create_partition(table, tag_name)
|
||||
status = connect.create_partition(collection, tag_name)
|
||||
assert status.OK()
|
||||
status, res = connect.show_partitions(table)
|
||||
status, res = connect.show_partitions(collection)
|
||||
assert status.OK()
|
||||
tag_list = []
|
||||
for item in res:
|
||||
|
@ -84,95 +84,95 @@ class TestCreateBase:
|
|||
assert tag_name in tag_list
|
||||
assert "_default" in tag_list
|
||||
|
||||
def test_create_partition_add_vectors_default(self, connect, table):
|
||||
def test_create_partition_add_vectors_default(self, connect, collection):
|
||||
'''
|
||||
target: test create partition, and insert vectors, check status returned
|
||||
method: call function: create_partition
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
nq = 100
|
||||
vectors = gen_vectors(nq, dim)
|
||||
ids = [i for i in range(nq)]
|
||||
status, ids = connect.insert(table, vectors, ids)
|
||||
status, ids = connect.insert(collection, vectors, ids)
|
||||
assert status.OK()
|
||||
|
||||
def test_create_partition_insert_with_tag(self, connect, table):
|
||||
def test_create_partition_insert_with_tag(self, connect, collection):
|
||||
'''
|
||||
target: test create partition, and insert vectors, check status returned
|
||||
method: call function: create_partition
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
nq = 100
|
||||
vectors = gen_vectors(nq, dim)
|
||||
ids = [i for i in range(nq)]
|
||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
||||
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||
assert status.OK()
|
||||
|
||||
def test_create_partition_insert_with_tag_not_existed(self, connect, table):
|
||||
def test_create_partition_insert_with_tag_not_existed(self, connect, collection):
|
||||
'''
|
||||
target: test create partition, and insert vectors, check status returned
|
||||
method: call function: create_partition
|
||||
expected: status not ok
|
||||
'''
|
||||
tag_new = "tag_new"
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
nq = 100
|
||||
vectors = gen_vectors(nq, dim)
|
||||
ids = [i for i in range(nq)]
|
||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag_new)
|
||||
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag_new)
|
||||
assert not status.OK()
|
||||
|
||||
def test_create_partition_insert_same_tags(self, connect, table):
|
||||
def test_create_partition_insert_same_tags(self, connect, collection):
|
||||
'''
|
||||
target: test create partition, and insert vectors, check status returned
|
||||
method: call function: create_partition
|
||||
expected: status ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
nq = 100
|
||||
vectors = gen_vectors(nq, dim)
|
||||
ids = [i for i in range(nq)]
|
||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
||||
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||
ids = [(i+100) for i in range(nq)]
|
||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
||||
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert res == nq * 2
|
||||
|
||||
def test_create_partition_insert_same_tags_two_tables(self, connect, table):
|
||||
def test_create_partition_insert_same_tags_two_collections(self, connect, collection):
|
||||
'''
|
||||
target: test create two partitions, and insert vectors with the same tag to each table, check status returned
|
||||
target: test create two partitions, and insert vectors with the same tag to each collection, check status returned
|
||||
method: call function: create_partition
|
||||
expected: status ok, table length is correct
|
||||
expected: status ok, collection length is correct
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
table_new = gen_unique_str()
|
||||
param = {'table_name': table_new,
|
||||
collection_new = gen_unique_str()
|
||||
param = {'collection_name': collection_new,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.L2}
|
||||
status = connect.create_table(param)
|
||||
status = connect.create_partition(table_new, tag)
|
||||
status = connect.create_collection(param)
|
||||
status = connect.create_partition(collection_new, tag)
|
||||
nq = 100
|
||||
vectors = gen_vectors(nq, dim)
|
||||
ids = [i for i in range(nq)]
|
||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
||||
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||
ids = [(i+100) for i in range(nq)]
|
||||
status, ids = connect.insert(table_new, vectors, ids, partition_tag=tag)
|
||||
status = connect.flush([table, table_new])
|
||||
status, ids = connect.insert(collection_new, vectors, ids, partition_tag=tag)
|
||||
status = connect.flush([collection, collection_new])
|
||||
assert status.OK()
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert res == nq
|
||||
status, res = connect.get_table_row_count(table_new)
|
||||
status, res = connect.count_collection(collection_new)
|
||||
assert res == nq
|
||||
|
||||
|
||||
|
@ -183,35 +183,35 @@ class TestShowBase:
|
|||
The following cases are used to test `show_partitions` function
|
||||
******************************************************************
|
||||
"""
|
||||
def test_show_partitions(self, connect, table):
|
||||
def test_show_partitions(self, connect, collection):
|
||||
'''
|
||||
target: test show partitions, check status and partitions returned
|
||||
method: create partition first, then call function: show_partitions
|
||||
expected: status ok, partition correct
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status, res = connect.show_partitions(table)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status, res = connect.show_partitions(collection)
|
||||
assert status.OK()
|
||||
|
||||
def test_show_partitions_no_partition(self, connect, table):
|
||||
def test_show_partitions_no_partition(self, connect, collection):
|
||||
'''
|
||||
target: test show partitions with table name, check status and partitions returned
|
||||
target: test show partitions with collection name, check status and partitions returned
|
||||
method: call function: show_partitions
|
||||
expected: status ok, partitions correct
|
||||
'''
|
||||
status, res = connect.show_partitions(table)
|
||||
status, res = connect.show_partitions(collection)
|
||||
assert status.OK()
|
||||
|
||||
def test_show_multi_partitions(self, connect, table):
|
||||
def test_show_multi_partitions(self, connect, collection):
|
||||
'''
|
||||
target: test show partitions, check status and partitions returned
|
||||
method: create partitions first, then call function: show_partitions
|
||||
expected: status ok, partitions correct
|
||||
'''
|
||||
tag_new = gen_unique_str()
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(table, tag_new)
|
||||
status, res = connect.show_partitions(table)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.create_partition(collection, tag_new)
|
||||
status, res = connect.show_partitions(collection)
|
||||
assert status.OK()
|
||||
|
||||
|
||||
|
@ -222,72 +222,72 @@ class TestDropBase:
|
|||
The following cases are used to test `drop_partition` function
|
||||
******************************************************************
|
||||
"""
|
||||
def test_drop_partition(self, connect, table):
|
||||
def test_drop_partition(self, connect, collection):
|
||||
'''
|
||||
target: test drop partition, check status and partition if existed
|
||||
method: create partitions first, then call function: drop_partition
|
||||
expected: status ok, no partitions in db
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.drop_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.drop_partition(collection, tag)
|
||||
assert status.OK()
|
||||
status, res = connect.show_partitions(table)
|
||||
status, res = connect.show_partitions(collection)
|
||||
tag_list = []
|
||||
for item in res:
|
||||
tag_list.append(item.tag)
|
||||
assert tag not in tag_list
|
||||
|
||||
def test_drop_partition_tag_not_existed(self, connect, table):
|
||||
def test_drop_partition_tag_not_existed(self, connect, collection):
|
||||
'''
|
||||
target: test drop partition, but tag not existed
|
||||
method: create partitions first, then call function: drop_partition
|
||||
expected: status not ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
new_tag = "new_tag"
|
||||
status = connect.drop_partition(table, new_tag)
|
||||
status = connect.drop_partition(collection, new_tag)
|
||||
assert not status.OK()
|
||||
|
||||
def test_drop_partition_tag_not_existed_A(self, connect, table):
|
||||
def test_drop_partition_tag_not_existed_A(self, connect, collection):
|
||||
'''
|
||||
target: test drop partition, but table not existed
|
||||
target: test drop partition, but collection not existed
|
||||
method: create partitions first, then call function: drop_partition
|
||||
expected: status not ok
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
new_table = gen_unique_str()
|
||||
status = connect.drop_partition(new_table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
new_collection = gen_unique_str()
|
||||
status = connect.drop_partition(new_collection, tag)
|
||||
assert not status.OK()
|
||||
|
||||
def test_drop_partition_repeatedly(self, connect, table):
|
||||
def test_drop_partition_repeatedly(self, connect, collection):
|
||||
'''
|
||||
target: test drop partition twice, check status and partition if existed
|
||||
method: create partitions first, then call function: drop_partition
|
||||
expected: status not ok, no partitions in db
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.drop_partition(table, tag)
|
||||
status = connect.drop_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.drop_partition(collection, tag)
|
||||
status = connect.drop_partition(collection, tag)
|
||||
time.sleep(2)
|
||||
assert not status.OK()
|
||||
status, res = connect.show_partitions(table)
|
||||
status, res = connect.show_partitions(collection)
|
||||
tag_list = []
|
||||
for item in res:
|
||||
tag_list.append(item.tag)
|
||||
assert tag not in tag_list
|
||||
|
||||
def test_drop_partition_create(self, connect, table):
|
||||
def test_drop_partition_create(self, connect, collection):
|
||||
'''
|
||||
target: test drop partition, and create again, check status
|
||||
method: create partitions first, then call function: drop_partition, create_partition
|
||||
expected: status not ok, partition in db
|
||||
'''
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.drop_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.drop_partition(collection, tag)
|
||||
time.sleep(2)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(collection, tag)
|
||||
assert status.OK()
|
||||
status, res = connect.show_partitions(table)
|
||||
status, res = connect.show_partitions(collection)
|
||||
tag_list = []
|
||||
for item in res:
|
||||
tag_list.append(item.tag)
|
||||
|
@ -297,47 +297,47 @@ class TestDropBase:
|
|||
class TestNameInvalid(object):
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_invalid_table_names()
|
||||
params=gen_invalid_collection_names()
|
||||
)
|
||||
def get_tag_name(self, request):
|
||||
yield request.param
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_invalid_table_names()
|
||||
params=gen_invalid_collection_names()
|
||||
)
|
||||
def get_table_name(self, request):
|
||||
def get_collection_name(self, request):
|
||||
yield request.param
|
||||
|
||||
def test_drop_partition_with_invalid_table_name(self, connect, table, get_table_name):
|
||||
def test_drop_partition_with_invalid_collection_name(self, connect, collection, get_collection_name):
|
||||
'''
|
||||
target: test drop partition, with invalid table name, check status returned
|
||||
target: test drop partition, with invalid collection name, check status returned
|
||||
method: call function: drop_partition
|
||||
expected: status not ok
|
||||
'''
|
||||
table_name = get_table_name
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.drop_partition(table_name, tag)
|
||||
collection_name = get_collection_name
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.drop_partition(collection_name, tag)
|
||||
assert not status.OK()
|
||||
|
||||
def test_drop_partition_with_invalid_tag_name(self, connect, table, get_tag_name):
|
||||
def test_drop_partition_with_invalid_tag_name(self, connect, collection, get_tag_name):
|
||||
'''
|
||||
target: test drop partition, with invalid tag name, check status returned
|
||||
method: call function: drop_partition
|
||||
expected: status not ok
|
||||
'''
|
||||
tag_name = get_tag_name
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.drop_partition(table, tag_name)
|
||||
status = connect.create_partition(collection, tag)
|
||||
status = connect.drop_partition(collection, tag_name)
|
||||
assert not status.OK()
|
||||
|
||||
def test_show_partitions_with_invalid_table_name(self, connect, table, get_table_name):
|
||||
def test_show_partitions_with_invalid_collection_name(self, connect, collection, get_collection_name):
|
||||
'''
|
||||
target: test show partitions, with invalid table name, check status returned
|
||||
target: test show partitions, with invalid collection name, check status returned
|
||||
method: call function: show_partitions
|
||||
expected: status not ok
|
||||
'''
|
||||
table_name = get_table_name
|
||||
status = connect.create_partition(table, tag)
|
||||
status, res = connect.show_partitions(table_name)
|
||||
collection_name = get_collection_name
|
||||
status = connect.create_partition(collection, tag)
|
||||
status, res = connect.show_partitions(collection_name)
|
||||
assert not status.OK()
|
||||
|
|
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
@ -1,644 +0,0 @@
|
|||
import pdb
|
||||
import pytest
|
||||
import logging
|
||||
import itertools
|
||||
from time import sleep
|
||||
import threading
|
||||
from multiprocessing import Process
|
||||
from milvus import IndexType, MetricType
|
||||
from utils import *
|
||||
|
||||
dim = 128
|
||||
index_file_size = 10
|
||||
add_time_interval = 3
|
||||
tag = "1970-01-01"
|
||||
nb = 6000
|
||||
|
||||
class TestTableCount:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_simple_index(self, request, connect):
|
||||
if str(connect._cmd("mode")[1]) == "CPU":
|
||||
if request.param["index_type"] == IndexType.IVF_SQ8H:
|
||||
pytest.skip("sq8h not support in cpu mode")
|
||||
if request.param["index_type"] == IndexType.IVF_PQ:
|
||||
pytest.skip("Skip PQ Temporary")
|
||||
return request.param
|
||||
|
||||
def test_table_rows_count(self, connect, table, add_vectors_nb):
|
||||
'''
|
||||
target: test table rows_count is correct or not
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
res = connect.add_vectors(table_name=table, records=vectors)
|
||||
connect.flush([table])
|
||||
status, res = connect.get_table_row_count(table)
|
||||
assert res == nb
|
||||
|
||||
def test_table_rows_count_partition(self, connect, table, add_vectors_nb):
|
||||
'''
|
||||
target: test table rows_count is correct or not
|
||||
method: create table, create partition and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
assert status.OK()
|
||||
res = connect.add_vectors(table_name=table, records=vectors, partition_tag=tag)
|
||||
connect.flush([table])
|
||||
status, res = connect.get_table_row_count(table)
|
||||
assert res == nb
|
||||
|
||||
def test_table_rows_count_multi_partitions_A(self, connect, table, add_vectors_nb):
|
||||
'''
|
||||
target: test table rows_count is correct or not
|
||||
method: create table, create partitions and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
new_tag = "new_tag"
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(table, new_tag)
|
||||
assert status.OK()
|
||||
res = connect.add_vectors(table_name=table, records=vectors)
|
||||
connect.flush([table])
|
||||
status, res = connect.get_table_row_count(table)
|
||||
assert res == nb
|
||||
|
||||
def test_table_rows_count_multi_partitions_B(self, connect, table, add_vectors_nb):
|
||||
'''
|
||||
target: test table rows_count is correct or not
|
||||
method: create table, create partitions and add vectors in one of the partitions,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
new_tag = "new_tag"
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(table, new_tag)
|
||||
assert status.OK()
|
||||
res = connect.add_vectors(table_name=table, records=vectors, partition_tag=tag)
|
||||
connect.flush([table])
|
||||
status, res = connect.get_table_row_count(table)
|
||||
assert res == nb
|
||||
|
||||
def test_table_rows_count_multi_partitions_C(self, connect, table, add_vectors_nb):
|
||||
'''
|
||||
target: test table rows_count is correct or not
|
||||
method: create table, create partitions and add vectors in one of the partitions,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the table count is equal to the length of vectors
|
||||
'''
|
||||
new_tag = "new_tag"
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
status = connect.create_partition(table, tag)
|
||||
status = connect.create_partition(table, new_tag)
|
||||
assert status.OK()
|
||||
res = connect.add_vectors(table_name=table, records=vectors, partition_tag=tag)
|
||||
res = connect.add_vectors(table_name=table, records=vectors, partition_tag=new_tag)
|
||||
connect.flush([table])
|
||||
status, res = connect.get_table_row_count(table)
|
||||
assert res == nb * 2
|
||||
|
||||
def test_table_rows_count_after_index_created(self, connect, table, get_simple_index):
|
||||
'''
|
||||
target: test get_table_row_count, after index have been created
|
||||
method: add vectors in db, and create index, then calling get_table_row_count with correct params
|
||||
expected: get_table_row_count raise exception
|
||||
'''
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
nb = 100
|
||||
vectors = gen_vectors(nb, dim)
|
||||
res = connect.add_vectors(table_name=table, records=vectors)
|
||||
connect.flush([table])
|
||||
connect.create_index(table, index_type, index_param)
|
||||
status, res = connect.get_table_row_count(table)
|
||||
assert res == nb
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_count_without_connection(self, table, dis_connect):
|
||||
'''
|
||||
target: test get_table_row_count, without connection
|
||||
method: calling get_table_row_count with correct params, with a disconnected instance
|
||||
expected: get_table_row_count raise exception
|
||||
'''
|
||||
with pytest.raises(Exception) as e:
|
||||
status = dis_connect.get_table_row_count(table)
|
||||
|
||||
def test_table_rows_count_no_vectors(self, connect, table):
|
||||
'''
|
||||
target: test table rows_count is correct or not, if table is empty
|
||||
method: create table and no vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to 0
|
||||
expected: the count is equal to 0
|
||||
'''
|
||||
table_name = gen_unique_str()
|
||||
param = {'table_name': table_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size}
|
||||
connect.create_table(param)
|
||||
status, res = connect.get_table_row_count(table)
|
||||
assert res == 0
|
||||
|
||||
# TODO: enable
|
||||
@pytest.mark.level(2)
|
||||
@pytest.mark.timeout(20)
|
||||
def _test_table_rows_count_multiprocessing(self, connect, table, args):
|
||||
'''
|
||||
target: test table rows_count is correct or not with multiprocess
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 2
|
||||
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
||||
vectors = gen_vectors(nq, dim)
|
||||
res = connect.add_vectors(table_name=table, records=vectors)
|
||||
time.sleep(add_time_interval)
|
||||
|
||||
def rows_count(milvus):
|
||||
status, res = milvus.get_table_row_count(table)
|
||||
logging.getLogger().info(status)
|
||||
assert res == nq
|
||||
|
||||
process_num = 8
|
||||
processes = []
|
||||
for i in range(process_num):
|
||||
milvus = get_milvus(args["handler"])
|
||||
milvus.connect(uri=uri)
|
||||
p = Process(target=rows_count, args=(milvus, ))
|
||||
processes.append(p)
|
||||
p.start()
|
||||
logging.getLogger().info(p)
|
||||
for p in processes:
|
||||
p.join()
|
||||
|
||||
def test_table_rows_count_multi_tables(self, connect):
|
||||
'''
|
||||
target: test table rows_count is correct or not with multiple tables of L2
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 100
|
||||
vectors = gen_vectors(nq, dim)
|
||||
table_list = []
|
||||
for i in range(20):
|
||||
table_name = gen_unique_str()
|
||||
table_list.append(table_name)
|
||||
param = {'table_name': table_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.L2}
|
||||
connect.create_table(param)
|
||||
res = connect.add_vectors(table_name=table_name, records=vectors)
|
||||
connect.flush(table_list)
|
||||
for i in range(20):
|
||||
status, res = connect.get_table_row_count(table_list[i])
|
||||
assert status.OK()
|
||||
assert res == nq
|
||||
|
||||
|
||||
class TestTableCountIP:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_simple_index(self, request, connect):
|
||||
if str(connect._cmd("mode")[1]) == "CPU":
|
||||
if request.param["index_type"] == IndexType.IVF_SQ8H:
|
||||
pytest.skip("sq8h not support in CPU mode")
|
||||
if request.param["index_type"] == IndexType.IVF_PQ:
|
||||
pytest.skip("Skip PQ Temporary")
|
||||
return request.param
|
||||
|
||||
def test_table_rows_count(self, connect, ip_table, add_vectors_nb):
|
||||
'''
|
||||
target: test table rows_count is correct or not
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
vectors = gen_vectors(nb, dim)
|
||||
res = connect.add_vectors(table_name=ip_table, records=vectors)
|
||||
connect.flush([ip_table])
|
||||
status, res = connect.get_table_row_count(ip_table)
|
||||
assert res == nb
|
||||
|
||||
def test_table_rows_count_after_index_created(self, connect, ip_table, get_simple_index):
|
||||
'''
|
||||
target: test get_table_row_count, after index have been created
|
||||
method: add vectors in db, and create index, then calling get_table_row_count with correct params
|
||||
expected: get_table_row_count raise exception
|
||||
'''
|
||||
index_param = get_simple_index["index_param"]
|
||||
index_type = get_simple_index["index_type"]
|
||||
nb = 100
|
||||
vectors = gen_vectors(nb, dim)
|
||||
res = connect.add_vectors(table_name=ip_table, records=vectors)
|
||||
connect.flush([ip_table])
|
||||
connect.create_index(ip_table, index_type, index_param)
|
||||
status, res = connect.get_table_row_count(ip_table)
|
||||
assert res == nb
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_count_without_connection(self, ip_table, dis_connect):
|
||||
'''
|
||||
target: test get_table_row_count, without connection
|
||||
method: calling get_table_row_count with correct params, with a disconnected instance
|
||||
expected: get_table_row_count raise exception
|
||||
'''
|
||||
with pytest.raises(Exception) as e:
|
||||
status = dis_connect.get_table_row_count(ip_table)
|
||||
|
||||
def test_table_rows_count_no_vectors(self, connect, ip_table):
|
||||
'''
|
||||
target: test table rows_count is correct or not, if table is empty
|
||||
method: create table and no vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to 0
|
||||
expected: the count is equal to 0
|
||||
'''
|
||||
table_name = gen_unique_str("test_table")
|
||||
param = {'table_name': table_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size}
|
||||
connect.create_table(param)
|
||||
status, res = connect.get_table_row_count(ip_table)
|
||||
assert res == 0
|
||||
|
||||
# TODO: enable
|
||||
@pytest.mark.timeout(60)
|
||||
def _test_table_rows_count_multiprocessing(self, connect, ip_table, args):
|
||||
'''
|
||||
target: test table rows_count is correct or not with multiprocess
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 2
|
||||
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
||||
vectors = gen_vectors(nq, dim)
|
||||
res = connect.add_vectors(table_name=ip_table, records=vectors)
|
||||
time.sleep(add_time_interval)
|
||||
|
||||
def rows_count(milvus):
|
||||
status, res = milvus.get_table_row_count(ip_table)
|
||||
logging.getLogger().info(status)
|
||||
assert res == nq
|
||||
|
||||
process_num = 8
|
||||
processes = []
|
||||
for i in range(process_num):
|
||||
milvus = get_milvus(args["handler"])
|
||||
milvus.connect(uri=uri)
|
||||
p = Process(target=rows_count, args=(milvus,))
|
||||
processes.append(p)
|
||||
p.start()
|
||||
logging.getLogger().info(p)
|
||||
for p in processes:
|
||||
p.join()
|
||||
|
||||
def test_table_rows_count_multi_tables(self, connect):
|
||||
'''
|
||||
target: test table rows_count is correct or not with multiple tables of IP
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 100
|
||||
vectors = gen_vectors(nq, dim)
|
||||
table_list = []
|
||||
for i in range(20):
|
||||
table_name = gen_unique_str('test_table_rows_count_multi_tables')
|
||||
table_list.append(table_name)
|
||||
param = {'table_name': table_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.IP}
|
||||
connect.create_table(param)
|
||||
res = connect.add_vectors(table_name=table_name, records=vectors)
|
||||
connect.flush(table_list)
|
||||
for i in range(20):
|
||||
status, res = connect.get_table_row_count(table_list[i])
|
||||
assert status.OK()
|
||||
assert res == nq
|
||||
|
||||
|
||||
class TestTableCountJAC:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_jaccard_index(self, request, connect):
|
||||
logging.getLogger().info(request.param)
|
||||
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
|
||||
return request.param
|
||||
else:
|
||||
pytest.skip("Skip index Temporary")
|
||||
|
||||
def test_table_rows_count(self, connect, jac_table, add_vectors_nb):
|
||||
'''
|
||||
target: test table rows_count is correct or not
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(table_name=jac_table, records=vectors)
|
||||
connect.flush([jac_table])
|
||||
status, res = connect.get_table_row_count(jac_table)
|
||||
assert res == nb
|
||||
|
||||
def test_table_rows_count_after_index_created(self, connect, jac_table, get_jaccard_index):
|
||||
'''
|
||||
target: test get_table_row_count, after index have been created
|
||||
method: add vectors in db, and create index, then calling get_table_row_count with correct params
|
||||
expected: get_table_row_count raise exception
|
||||
'''
|
||||
nb = 100
|
||||
index_param = get_jaccard_index["index_param"]
|
||||
index_type = get_jaccard_index["index_type"]
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(table_name=jac_table, records=vectors)
|
||||
connect.flush([jac_table])
|
||||
connect.create_index(jac_table, index_type, index_param)
|
||||
status, res = connect.get_table_row_count(jac_table)
|
||||
assert res == nb
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_count_without_connection(self, jac_table, dis_connect):
|
||||
'''
|
||||
target: test get_table_row_count, without connection
|
||||
method: calling get_table_row_count with correct params, with a disconnected instance
|
||||
expected: get_table_row_count raise exception
|
||||
'''
|
||||
with pytest.raises(Exception) as e:
|
||||
status = dis_connect.get_table_row_count(jac_table)
|
||||
|
||||
def test_table_rows_count_no_vectors(self, connect, jac_table):
|
||||
'''
|
||||
target: test table rows_count is correct or not, if table is empty
|
||||
method: create table and no vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to 0
|
||||
expected: the count is equal to 0
|
||||
'''
|
||||
table_name = gen_unique_str("test_table")
|
||||
param = {'table_name': table_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size}
|
||||
connect.create_table(param)
|
||||
status, res = connect.get_table_row_count(jac_table)
|
||||
assert res == 0
|
||||
|
||||
def test_table_rows_count_multi_tables(self, connect):
|
||||
'''
|
||||
target: test table rows_count is correct or not with multiple tables of IP
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 100
|
||||
tmp, vectors = gen_binary_vectors(nq, dim)
|
||||
table_list = []
|
||||
for i in range(20):
|
||||
table_name = gen_unique_str('test_table_rows_count_multi_tables')
|
||||
table_list.append(table_name)
|
||||
param = {'table_name': table_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.JACCARD}
|
||||
connect.create_table(param)
|
||||
res = connect.add_vectors(table_name=table_name, records=vectors)
|
||||
connect.flush(table_list)
|
||||
for i in range(20):
|
||||
status, res = connect.get_table_row_count(table_list[i])
|
||||
assert status.OK()
|
||||
assert res == nq
|
||||
|
||||
class TestTableCountHAM:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_hamming_index(self, request, connect):
|
||||
logging.getLogger().info(request.param)
|
||||
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
|
||||
return request.param
|
||||
else:
|
||||
pytest.skip("Skip index Temporary")
|
||||
|
||||
def test_table_rows_count(self, connect, ham_table, add_vectors_nb):
|
||||
'''
|
||||
target: test table rows_count is correct or not
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(table_name=ham_table, records=vectors)
|
||||
connect.flush([ham_table])
|
||||
status, res = connect.get_table_row_count(ham_table)
|
||||
assert res == nb
|
||||
|
||||
def test_table_rows_count_after_index_created(self, connect, ham_table, get_hamming_index):
|
||||
'''
|
||||
target: test get_table_row_count, after index have been created
|
||||
method: add vectors in db, and create index, then calling get_table_row_count with correct params
|
||||
expected: get_table_row_count raise exception
|
||||
'''
|
||||
nb = 100
|
||||
index_type = get_hamming_index["index_type"]
|
||||
index_param = get_hamming_index["index_param"]
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(table_name=ham_table, records=vectors)
|
||||
connect.flush([ham_table])
|
||||
connect.create_index(ham_table, index_type, index_param)
|
||||
status, res = connect.get_table_row_count(ham_table)
|
||||
assert res == nb
|
||||
|
||||
@pytest.mark.level(2)
|
||||
def test_count_without_connection(self, ham_table, dis_connect):
|
||||
'''
|
||||
target: test get_table_row_count, without connection
|
||||
method: calling get_table_row_count with correct params, with a disconnected instance
|
||||
expected: get_table_row_count raise exception
|
||||
'''
|
||||
with pytest.raises(Exception) as e:
|
||||
status = dis_connect.get_table_row_count(ham_table)
|
||||
|
||||
def test_table_rows_count_no_vectors(self, connect, ham_table):
|
||||
'''
|
||||
target: test table rows_count is correct or not, if table is empty
|
||||
method: create table and no vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to 0
|
||||
expected: the count is equal to 0
|
||||
'''
|
||||
table_name = gen_unique_str("test_table")
|
||||
param = {'table_name': table_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size}
|
||||
connect.create_table(param)
|
||||
status, res = connect.get_table_row_count(ham_table)
|
||||
assert res == 0
|
||||
|
||||
def test_table_rows_count_multi_tables(self, connect):
|
||||
'''
|
||||
target: test table rows_count is correct or not with multiple tables of IP
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nq = 100
|
||||
tmp, vectors = gen_binary_vectors(nq, dim)
|
||||
table_list = []
|
||||
for i in range(20):
|
||||
table_name = gen_unique_str('test_table_rows_count_multi_tables')
|
||||
table_list.append(table_name)
|
||||
param = {'table_name': table_name,
|
||||
'dimension': dim,
|
||||
'index_file_size': index_file_size,
|
||||
'metric_type': MetricType.HAMMING}
|
||||
connect.create_table(param)
|
||||
res = connect.add_vectors(table_name=table_name, records=vectors)
|
||||
connect.flush(table_list)
|
||||
for i in range(20):
|
||||
status, res = connect.get_table_row_count(table_list[i])
|
||||
assert status.OK()
|
||||
assert res == nq
|
||||
|
||||
|
||||
class TestTableCountTANIMOTO:
|
||||
"""
|
||||
params means different nb, the nb value may trigger merge, or not
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
1,
|
||||
5000,
|
||||
100000,
|
||||
],
|
||||
)
|
||||
def add_vectors_nb(self, request):
|
||||
yield request.param
|
||||
|
||||
"""
|
||||
generate valid create_index params
|
||||
"""
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=gen_simple_index()
|
||||
)
|
||||
def get_tanimoto_index(self, request, connect):
|
||||
logging.getLogger().info(request.param)
|
||||
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
|
||||
return request.param
|
||||
else:
|
||||
pytest.skip("Skip index Temporary")
|
||||
|
||||
def test_table_rows_count(self, connect, tanimoto_table, add_vectors_nb):
|
||||
'''
|
||||
target: test table rows_count is correct or not
|
||||
method: create table and add vectors in it,
|
||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
||||
expected: the count is equal to the length of vectors
|
||||
'''
|
||||
nb = add_vectors_nb
|
||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||
res = connect.add_vectors(table_name=tanimoto_table, records=vectors)
|
||||
connect.flush([tanimoto_table])
|
||||
status, res = connect.get_table_row_count(tanimoto_table)
|
||||
assert status.OK()
|
||||
assert res == nb
|
|
@ -8,7 +8,7 @@ from milvus import IndexType, MetricType
|
|||
from utils import *
|
||||
|
||||
dim = 128
|
||||
table_id = "test_wal"
|
||||
collection_id = "test_wal"
|
||||
WAL_TIMEOUT = 30
|
||||
nb = 6000
|
||||
add_interval = 1.5
|
||||
|
@ -21,109 +21,109 @@ class TestWalBase:
|
|||
******************************************************************
|
||||
"""
|
||||
@pytest.mark.timeout(WAL_TIMEOUT)
|
||||
def test_wal_add_vectors(self, connect, table):
|
||||
def test_wal_add_vectors(self, connect, collection):
|
||||
'''
|
||||
target: add vectors in WAL
|
||||
method: add vectors and flush when WAL is enabled
|
||||
expected: status ok, vectors added
|
||||
'''
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
assert res == nb
|
||||
status, res = connect.get_vector_by_id(table, ids[0])
|
||||
status, res = connect.get_vector_by_id(collection, ids[0])
|
||||
logging.getLogger().info(res)
|
||||
assert status.OK()
|
||||
assert_equal_vector(res, vectors[0])
|
||||
|
||||
@pytest.mark.timeout(WAL_TIMEOUT)
|
||||
def test_wal_delete_vectors(self, connect, table):
|
||||
def test_wal_delete_vectors(self, connect, collection):
|
||||
'''
|
||||
target: delete vectors in WAL
|
||||
method: delete vectors and flush when WAL is enabled
|
||||
expected: status ok, vectors deleted
|
||||
'''
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
connect.flush([table])
|
||||
status, res = connect.get_table_row_count(table)
|
||||
connect.flush([collection])
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
status = connect.delete_by_id(table, ids)
|
||||
status = connect.delete_by_id(collection, ids)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
assert res == 0
|
||||
|
||||
@pytest.mark.timeout(WAL_TIMEOUT)
|
||||
def test_wal_invalid_operation(self, connect, table):
|
||||
def test_wal_invalid_operation(self, connect, collection):
|
||||
'''
|
||||
target: invalid operation in WAL
|
||||
method: add vectors, delete with non-existent ids and flush when WAL is enabled
|
||||
expected: status ok, search with vector have result
|
||||
'''
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
connect.flush([table])
|
||||
status = connect.delete_by_id(table, [0])
|
||||
connect.flush([collection])
|
||||
status = connect.delete_by_id(collection, [0])
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status = connect.flush([collection])
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
assert res == 1
|
||||
|
||||
@pytest.mark.timeout(WAL_TIMEOUT)
|
||||
def test_wal_invalid_operation_B(self, connect, table):
|
||||
def test_wal_invalid_operation_B(self, connect, collection):
|
||||
'''
|
||||
target: invalid operation in WAL
|
||||
method: add vectors, delete with not existed table name when WAL is enabled
|
||||
method: add vectors, delete with not existed collection name when WAL is enabled
|
||||
expected: status not ok
|
||||
'''
|
||||
vectors = gen_vector(nb, dim)
|
||||
status, ids = connect.add_vectors(table, vectors)
|
||||
status, ids = connect.add_vectors(collection, vectors)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.delete_by_id(table, [0])
|
||||
connect.flush([table])
|
||||
table_new = gen_unique_str()
|
||||
status = connect.delete_by_id(table_new, ids)
|
||||
status = connect.flush([collection])
|
||||
status = connect.delete_by_id(collection, [0])
|
||||
connect.flush([collection])
|
||||
collection_new = gen_unique_str()
|
||||
status = connect.delete_by_id(collection_new, ids)
|
||||
assert not status.OK()
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
assert res == nb
|
||||
|
||||
@pytest.mark.timeout(WAL_TIMEOUT)
|
||||
def test_wal_server_crashed_recovery(self, connect, table):
|
||||
def test_wal_server_crashed_recovery(self, connect, collection):
|
||||
'''
|
||||
target: test wal when server crashed unexpectedly and restarted
|
||||
method: add vectors, server killed before flush, restarted server and flush
|
||||
expected: status ok, add request is recovered and vectors added
|
||||
'''
|
||||
vector = gen_single_vector(dim)
|
||||
status, ids = connect.add_vectors(table, vector)
|
||||
status, ids = connect.add_vectors(collection, vector)
|
||||
assert status.OK()
|
||||
status = connect.flush([table])
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status = connect.flush([collection])
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
logging.getLogger().info(res) # should be 0 because no auto flush
|
||||
logging.getLogger().info("Stop server and restart")
|
||||
# kill server and restart. auto flush should be set to 15 seconds.
|
||||
# time.sleep(15)
|
||||
status = connect.flush([table])
|
||||
status = connect.flush([collection])
|
||||
assert status.OK()
|
||||
status, res = connect.get_table_row_count(table)
|
||||
status, res = connect.count_collection(collection)
|
||||
assert status.OK()
|
||||
assert res == 1
|
||||
status, res = connect.get_vector_by_id(table, ids[0])
|
||||
status, res = connect.get_vector_by_id(collection, ids[0])
|
||||
logging.getLogger().info(res)
|
||||
assert status.OK()
|
||||
assert_equal_vector(res, vector[0])
|
||||
|
|
|
@ -146,8 +146,8 @@ def gen_invalid_uris():
|
|||
return uris
|
||||
|
||||
|
||||
def gen_invalid_table_names():
|
||||
table_names = [
|
||||
def gen_invalid_collection_names():
|
||||
collection_names = [
|
||||
"12-s",
|
||||
"12/s",
|
||||
" ",
|
||||
|
@ -166,7 +166,7 @@ def gen_invalid_table_names():
|
|||
"中文",
|
||||
"a".join("a" for i in range(256))
|
||||
]
|
||||
return table_names
|
||||
return collection_names
|
||||
|
||||
|
||||
def gen_invalid_top_ks():
|
||||
|
@ -588,7 +588,7 @@ def gen_simple_index():
|
|||
IndexType.IVF_SQ8H,
|
||||
IndexType.IVF_PQ,
|
||||
IndexType.HNSW,
|
||||
# IndexType.RNSG
|
||||
IndexType.RNSG
|
||||
]
|
||||
params = [
|
||||
{"nlist": 1024},
|
||||
|
@ -597,7 +597,7 @@ def gen_simple_index():
|
|||
{"nlist": 1024},
|
||||
{"nlist": 1024, "m": 16},
|
||||
{"M": 16, "efConstruction": 500},
|
||||
# {"search_length": 100, "out_degree": 40, "candidate_pool_size": 66, "knng": 100}
|
||||
{"search_length": 50, "out_degree": 40, "candidate_pool_size": 100, "knng": 50}
|
||||
]
|
||||
|
||||
index_params = []
|
||||
|
@ -611,14 +611,14 @@ def get_search_param(index_type):
|
|||
return {"nprobe": 32}
|
||||
elif index_type == IndexType.HNSW:
|
||||
return {"ef": 64}
|
||||
# elif index_type == IndexType.RNSG:
|
||||
# return {"search_length": 100}
|
||||
elif index_type == IndexType.RNSG:
|
||||
return {"search_length": 50}
|
||||
else:
|
||||
logging.getLogger().info("Invalid index_type.")
|
||||
|
||||
|
||||
def assert_has_table(conn, table_name):
|
||||
status, ok = conn.has_table(table_name)
|
||||
def assert_has_collection(conn, collection_name):
|
||||
status, ok = conn.has_collection(collection_name)
|
||||
return status.OK() and ok
|
||||
|
||||
|
||||
|
|
Loading…
Reference in New Issue