mirror of https://github.com/milvus-io/milvus.git
commit
6cd77b35df
|
@ -90,118 +90,118 @@ def milvus(request):
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="function")
|
@pytest.fixture(scope="function")
|
||||||
def table(request, connect):
|
def collection(request, connect):
|
||||||
ori_table_name = getattr(request.module, "table_id", "test")
|
ori_collection_name = getattr(request.module, "collection_id", "test")
|
||||||
table_name = gen_unique_str(ori_table_name)
|
collection_name = gen_unique_str(ori_collection_name)
|
||||||
dim = getattr(request.module, "dim", "128")
|
dim = getattr(request.module, "dim", "128")
|
||||||
param = {'table_name': table_name,
|
param = {'collection_name': collection_name,
|
||||||
'dimension': dim,
|
'dimension': dim,
|
||||||
'index_file_size': index_file_size,
|
'index_file_size': index_file_size,
|
||||||
'metric_type': MetricType.L2}
|
'metric_type': MetricType.L2}
|
||||||
status = connect.create_table(param)
|
status = connect.create_collection(param)
|
||||||
# logging.getLogger().info(status)
|
# logging.getLogger().info(status)
|
||||||
if not status.OK():
|
if not status.OK():
|
||||||
pytest.exit("Table can not be created, exit pytest ...")
|
pytest.exit("collection can not be created, exit pytest ...")
|
||||||
|
|
||||||
def teardown():
|
def teardown():
|
||||||
status, table_names = connect.show_tables()
|
status, collection_names = connect.show_collections()
|
||||||
for table_name in table_names:
|
for collection_name in collection_names:
|
||||||
connect.delete_table(table_name)
|
connect.drop_collection(collection_name)
|
||||||
|
|
||||||
request.addfinalizer(teardown)
|
request.addfinalizer(teardown)
|
||||||
|
|
||||||
return table_name
|
return collection_name
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="function")
|
@pytest.fixture(scope="function")
|
||||||
def ip_table(request, connect):
|
def ip_collection(request, connect):
|
||||||
ori_table_name = getattr(request.module, "table_id", "test")
|
ori_collection_name = getattr(request.module, "collection_id", "test")
|
||||||
table_name = gen_unique_str(ori_table_name)
|
collection_name = gen_unique_str(ori_collection_name)
|
||||||
dim = getattr(request.module, "dim", "128")
|
dim = getattr(request.module, "dim", "128")
|
||||||
param = {'table_name': table_name,
|
param = {'collection_name': collection_name,
|
||||||
'dimension': dim,
|
'dimension': dim,
|
||||||
'index_file_size': index_file_size,
|
'index_file_size': index_file_size,
|
||||||
'metric_type': MetricType.IP}
|
'metric_type': MetricType.IP}
|
||||||
status = connect.create_table(param)
|
status = connect.create_collection(param)
|
||||||
# logging.getLogger().info(status)
|
# logging.getLogger().info(status)
|
||||||
if not status.OK():
|
if not status.OK():
|
||||||
pytest.exit("Table can not be created, exit pytest ...")
|
pytest.exit("collection can not be created, exit pytest ...")
|
||||||
|
|
||||||
def teardown():
|
def teardown():
|
||||||
status, table_names = connect.show_tables()
|
status, collection_names = connect.show_collections()
|
||||||
for table_name in table_names:
|
for collection_name in collection_names:
|
||||||
connect.delete_table(table_name)
|
connect.drop_collection(collection_name)
|
||||||
|
|
||||||
request.addfinalizer(teardown)
|
request.addfinalizer(teardown)
|
||||||
|
|
||||||
return table_name
|
return collection_name
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="function")
|
@pytest.fixture(scope="function")
|
||||||
def jac_table(request, connect):
|
def jac_collection(request, connect):
|
||||||
ori_table_name = getattr(request.module, "table_id", "test")
|
ori_collection_name = getattr(request.module, "collection_id", "test")
|
||||||
table_name = gen_unique_str(ori_table_name)
|
collection_name = gen_unique_str(ori_collection_name)
|
||||||
dim = getattr(request.module, "dim", "128")
|
dim = getattr(request.module, "dim", "128")
|
||||||
param = {'table_name': table_name,
|
param = {'collection_name': collection_name,
|
||||||
'dimension': dim,
|
'dimension': dim,
|
||||||
'index_file_size': index_file_size,
|
'index_file_size': index_file_size,
|
||||||
'metric_type': MetricType.JACCARD}
|
'metric_type': MetricType.JACCARD}
|
||||||
status = connect.create_table(param)
|
status = connect.create_collection(param)
|
||||||
# logging.getLogger().info(status)
|
# logging.getLogger().info(status)
|
||||||
if not status.OK():
|
if not status.OK():
|
||||||
pytest.exit("Table can not be created, exit pytest ...")
|
pytest.exit("collection can not be created, exit pytest ...")
|
||||||
|
|
||||||
def teardown():
|
def teardown():
|
||||||
status, table_names = connect.show_tables()
|
status, collection_names = connect.show_collections()
|
||||||
for table_name in table_names:
|
for collection_name in collection_names:
|
||||||
connect.delete_table(table_name)
|
connect.drop_collection(collection_name)
|
||||||
|
|
||||||
request.addfinalizer(teardown)
|
request.addfinalizer(teardown)
|
||||||
|
|
||||||
return table_name
|
return collection_name
|
||||||
|
|
||||||
@pytest.fixture(scope="function")
|
@pytest.fixture(scope="function")
|
||||||
def ham_table(request, connect):
|
def ham_collection(request, connect):
|
||||||
ori_table_name = getattr(request.module, "table_id", "test")
|
ori_collection_name = getattr(request.module, "collection_id", "test")
|
||||||
table_name = gen_unique_str(ori_table_name)
|
collection_name = gen_unique_str(ori_collection_name)
|
||||||
dim = getattr(request.module, "dim", "128")
|
dim = getattr(request.module, "dim", "128")
|
||||||
param = {'table_name': table_name,
|
param = {'collection_name': collection_name,
|
||||||
'dimension': dim,
|
'dimension': dim,
|
||||||
'index_file_size': index_file_size,
|
'index_file_size': index_file_size,
|
||||||
'metric_type': MetricType.HAMMING}
|
'metric_type': MetricType.HAMMING}
|
||||||
status = connect.create_table(param)
|
status = connect.create_collection(param)
|
||||||
# logging.getLogger().info(status)
|
# logging.getLogger().info(status)
|
||||||
if not status.OK():
|
if not status.OK():
|
||||||
pytest.exit("Table can not be created, exit pytest ...")
|
pytest.exit("collection can not be created, exit pytest ...")
|
||||||
|
|
||||||
def teardown():
|
def teardown():
|
||||||
status, table_names = connect.show_tables()
|
status, collection_names = connect.show_collections()
|
||||||
for table_name in table_names:
|
for collection_name in collection_names:
|
||||||
connect.delete_table(table_name)
|
connect.drop_collection(collection_name)
|
||||||
|
|
||||||
request.addfinalizer(teardown)
|
request.addfinalizer(teardown)
|
||||||
|
|
||||||
return table_name
|
return collection_name
|
||||||
|
|
||||||
@pytest.fixture(scope="function")
|
@pytest.fixture(scope="function")
|
||||||
def tanimoto_table(request, connect):
|
def tanimoto_collection(request, connect):
|
||||||
ori_table_name = getattr(request.module, "table_id", "test")
|
ori_collection_name = getattr(request.module, "collection_id", "test")
|
||||||
table_name = gen_unique_str(ori_table_name)
|
collection_name = gen_unique_str(ori_collection_name)
|
||||||
dim = getattr(request.module, "dim", "128")
|
dim = getattr(request.module, "dim", "128")
|
||||||
param = {'table_name': table_name,
|
param = {'collection_name': collection_name,
|
||||||
'dimension': dim,
|
'dimension': dim,
|
||||||
'index_file_size': index_file_size,
|
'index_file_size': index_file_size,
|
||||||
'metric_type': MetricType.TANIMOTO}
|
'metric_type': MetricType.TANIMOTO}
|
||||||
status = connect.create_table(param)
|
status = connect.create_collection(param)
|
||||||
# logging.getLogger().info(status)
|
# logging.getLogger().info(status)
|
||||||
if not status.OK():
|
if not status.OK():
|
||||||
pytest.exit("Table can not be created, exit pytest ...")
|
pytest.exit("collection can not be created, exit pytest ...")
|
||||||
|
|
||||||
def teardown():
|
def teardown():
|
||||||
status, table_names = connect.show_tables()
|
status, collection_names = connect.show_collections()
|
||||||
for table_name in table_names:
|
for collection_name in collection_names:
|
||||||
connect.delete_table(table_name)
|
connect.drop_collection(collection_name)
|
||||||
|
|
||||||
request.addfinalizer(teardown)
|
request.addfinalizer(teardown)
|
||||||
|
|
||||||
return table_name
|
return collection_name
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,644 @@
|
||||||
|
import pdb
|
||||||
|
import pytest
|
||||||
|
import logging
|
||||||
|
import itertools
|
||||||
|
from time import sleep
|
||||||
|
import threading
|
||||||
|
from multiprocessing import Process
|
||||||
|
from milvus import IndexType, MetricType
|
||||||
|
from utils import *
|
||||||
|
|
||||||
|
dim = 128
|
||||||
|
index_file_size = 10
|
||||||
|
add_time_interval = 3
|
||||||
|
tag = "1970-01-01"
|
||||||
|
nb = 6000
|
||||||
|
|
||||||
|
class TestCollectionCount:
|
||||||
|
"""
|
||||||
|
params means different nb, the nb value may trigger merge, or not
|
||||||
|
"""
|
||||||
|
@pytest.fixture(
|
||||||
|
scope="function",
|
||||||
|
params=[
|
||||||
|
1,
|
||||||
|
5000,
|
||||||
|
100000,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def add_vectors_nb(self, request):
|
||||||
|
yield request.param
|
||||||
|
|
||||||
|
"""
|
||||||
|
generate valid create_index params
|
||||||
|
"""
|
||||||
|
@pytest.fixture(
|
||||||
|
scope="function",
|
||||||
|
params=gen_simple_index()
|
||||||
|
)
|
||||||
|
def get_simple_index(self, request, connect):
|
||||||
|
if str(connect._cmd("mode")[1]) == "CPU":
|
||||||
|
if request.param["index_type"] == IndexType.IVF_SQ8H:
|
||||||
|
pytest.skip("sq8h not support in cpu mode")
|
||||||
|
if request.param["index_type"] == IndexType.IVF_PQ:
|
||||||
|
pytest.skip("Skip PQ Temporary")
|
||||||
|
return request.param
|
||||||
|
|
||||||
|
def test_collection_rows_count(self, connect, collection, add_vectors_nb):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not
|
||||||
|
method: create collection and add vectors in it,
|
||||||
|
assert the value returned by count_collection method is equal to length of vectors
|
||||||
|
expected: the count is equal to the length of vectors
|
||||||
|
'''
|
||||||
|
nb = add_vectors_nb
|
||||||
|
vectors = gen_vectors(nb, dim)
|
||||||
|
res = connect.add_vectors(collection_name=collection, records=vectors)
|
||||||
|
connect.flush([collection])
|
||||||
|
status, res = connect.count_collection(collection)
|
||||||
|
assert res == nb
|
||||||
|
|
||||||
|
def test_collection_rows_count_partition(self, connect, collection, add_vectors_nb):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not
|
||||||
|
method: create collection, create partition and add vectors in it,
|
||||||
|
assert the value returned by count_collection method is equal to length of vectors
|
||||||
|
expected: the count is equal to the length of vectors
|
||||||
|
'''
|
||||||
|
nb = add_vectors_nb
|
||||||
|
vectors = gen_vectors(nb, dim)
|
||||||
|
status = connect.create_partition(collection, tag)
|
||||||
|
assert status.OK()
|
||||||
|
res = connect.add_vectors(collection_name=collection, records=vectors, partition_tag=tag)
|
||||||
|
connect.flush([collection])
|
||||||
|
status, res = connect.count_collection(collection)
|
||||||
|
assert res == nb
|
||||||
|
|
||||||
|
def test_collection_rows_count_multi_partitions_A(self, connect, collection, add_vectors_nb):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not
|
||||||
|
method: create collection, create partitions and add vectors in it,
|
||||||
|
assert the value returned by count_collection method is equal to length of vectors
|
||||||
|
expected: the count is equal to the length of vectors
|
||||||
|
'''
|
||||||
|
new_tag = "new_tag"
|
||||||
|
nb = add_vectors_nb
|
||||||
|
vectors = gen_vectors(nb, dim)
|
||||||
|
status = connect.create_partition(collection, tag)
|
||||||
|
status = connect.create_partition(collection, new_tag)
|
||||||
|
assert status.OK()
|
||||||
|
res = connect.add_vectors(collection_name=collection, records=vectors)
|
||||||
|
connect.flush([collection])
|
||||||
|
status, res = connect.count_collection(collection)
|
||||||
|
assert res == nb
|
||||||
|
|
||||||
|
def test_collection_rows_count_multi_partitions_B(self, connect, collection, add_vectors_nb):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not
|
||||||
|
method: create collection, create partitions and add vectors in one of the partitions,
|
||||||
|
assert the value returned by count_collection method is equal to length of vectors
|
||||||
|
expected: the count is equal to the length of vectors
|
||||||
|
'''
|
||||||
|
new_tag = "new_tag"
|
||||||
|
nb = add_vectors_nb
|
||||||
|
vectors = gen_vectors(nb, dim)
|
||||||
|
status = connect.create_partition(collection, tag)
|
||||||
|
status = connect.create_partition(collection, new_tag)
|
||||||
|
assert status.OK()
|
||||||
|
res = connect.add_vectors(collection_name=collection, records=vectors, partition_tag=tag)
|
||||||
|
connect.flush([collection])
|
||||||
|
status, res = connect.count_collection(collection)
|
||||||
|
assert res == nb
|
||||||
|
|
||||||
|
def test_collection_rows_count_multi_partitions_C(self, connect, collection, add_vectors_nb):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not
|
||||||
|
method: create collection, create partitions and add vectors in one of the partitions,
|
||||||
|
assert the value returned by count_collection method is equal to length of vectors
|
||||||
|
expected: the collection count is equal to the length of vectors
|
||||||
|
'''
|
||||||
|
new_tag = "new_tag"
|
||||||
|
nb = add_vectors_nb
|
||||||
|
vectors = gen_vectors(nb, dim)
|
||||||
|
status = connect.create_partition(collection, tag)
|
||||||
|
status = connect.create_partition(collection, new_tag)
|
||||||
|
assert status.OK()
|
||||||
|
res = connect.add_vectors(collection_name=collection, records=vectors, partition_tag=tag)
|
||||||
|
res = connect.add_vectors(collection_name=collection, records=vectors, partition_tag=new_tag)
|
||||||
|
connect.flush([collection])
|
||||||
|
status, res = connect.count_collection(collection)
|
||||||
|
assert res == nb * 2
|
||||||
|
|
||||||
|
def test_collection_rows_count_after_index_created(self, connect, collection, get_simple_index):
|
||||||
|
'''
|
||||||
|
target: test count_collection, after index have been created
|
||||||
|
method: add vectors in db, and create index, then calling count_collection with correct params
|
||||||
|
expected: count_collection raise exception
|
||||||
|
'''
|
||||||
|
index_param = get_simple_index["index_param"]
|
||||||
|
index_type = get_simple_index["index_type"]
|
||||||
|
nb = 100
|
||||||
|
vectors = gen_vectors(nb, dim)
|
||||||
|
res = connect.add_vectors(collection_name=collection, records=vectors)
|
||||||
|
connect.flush([collection])
|
||||||
|
connect.create_index(collection, index_type, index_param)
|
||||||
|
status, res = connect.count_collection(collection)
|
||||||
|
assert res == nb
|
||||||
|
|
||||||
|
@pytest.mark.level(2)
|
||||||
|
def test_count_without_connection(self, collection, dis_connect):
|
||||||
|
'''
|
||||||
|
target: test count_collection, without connection
|
||||||
|
method: calling count_collection with correct params, with a disconnected instance
|
||||||
|
expected: count_collection raise exception
|
||||||
|
'''
|
||||||
|
with pytest.raises(Exception) as e:
|
||||||
|
status = dis_connect.count_collection(collection)
|
||||||
|
|
||||||
|
def test_collection_rows_count_no_vectors(self, connect, collection):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not, if collection is empty
|
||||||
|
method: create collection and no vectors in it,
|
||||||
|
assert the value returned by count_collection method is equal to 0
|
||||||
|
expected: the count is equal to 0
|
||||||
|
'''
|
||||||
|
collection_name = gen_unique_str()
|
||||||
|
param = {'collection_name': collection_name,
|
||||||
|
'dimension': dim,
|
||||||
|
'index_file_size': index_file_size}
|
||||||
|
connect.create_collection(param)
|
||||||
|
status, res = connect.count_collection(collection)
|
||||||
|
assert res == 0
|
||||||
|
|
||||||
|
# TODO: enable
|
||||||
|
@pytest.mark.level(2)
|
||||||
|
@pytest.mark.timeout(20)
|
||||||
|
def _test_collection_rows_count_multiprocessing(self, connect, collection, args):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not with multiprocess
|
||||||
|
method: create collection and add vectors in it,
|
||||||
|
assert the value returned by count_collection method is equal to length of vectors
|
||||||
|
expected: the count is equal to the length of vectors
|
||||||
|
'''
|
||||||
|
nq = 2
|
||||||
|
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
||||||
|
vectors = gen_vectors(nq, dim)
|
||||||
|
res = connect.add_vectors(collection_name=collection, records=vectors)
|
||||||
|
time.sleep(add_time_interval)
|
||||||
|
|
||||||
|
def rows_count(milvus):
|
||||||
|
status, res = milvus.count_collection(collection)
|
||||||
|
logging.getLogger().info(status)
|
||||||
|
assert res == nq
|
||||||
|
|
||||||
|
process_num = 8
|
||||||
|
processes = []
|
||||||
|
for i in range(process_num):
|
||||||
|
milvus = get_milvus(args["handler"])
|
||||||
|
milvus.connect(uri=uri)
|
||||||
|
p = Process(target=rows_count, args=(milvus, ))
|
||||||
|
processes.append(p)
|
||||||
|
p.start()
|
||||||
|
logging.getLogger().info(p)
|
||||||
|
for p in processes:
|
||||||
|
p.join()
|
||||||
|
|
||||||
|
def test_collection_rows_count_multi_collections(self, connect):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not with multiple collections of L2
|
||||||
|
method: create collection and add vectors in it,
|
||||||
|
assert the value returned by count_collection method is equal to length of vectors
|
||||||
|
expected: the count is equal to the length of vectors
|
||||||
|
'''
|
||||||
|
nq = 100
|
||||||
|
vectors = gen_vectors(nq, dim)
|
||||||
|
collection_list = []
|
||||||
|
for i in range(20):
|
||||||
|
collection_name = gen_unique_str()
|
||||||
|
collection_list.append(collection_name)
|
||||||
|
param = {'collection_name': collection_name,
|
||||||
|
'dimension': dim,
|
||||||
|
'index_file_size': index_file_size,
|
||||||
|
'metric_type': MetricType.L2}
|
||||||
|
connect.create_collection(param)
|
||||||
|
res = connect.add_vectors(collection_name=collection_name, records=vectors)
|
||||||
|
connect.flush(collection_list)
|
||||||
|
for i in range(20):
|
||||||
|
status, res = connect.count_collection(collection_list[i])
|
||||||
|
assert status.OK()
|
||||||
|
assert res == nq
|
||||||
|
|
||||||
|
|
||||||
|
class TestCollectionCountIP:
|
||||||
|
"""
|
||||||
|
params means different nb, the nb value may trigger merge, or not
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.fixture(
|
||||||
|
scope="function",
|
||||||
|
params=[
|
||||||
|
1,
|
||||||
|
5000,
|
||||||
|
100000,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def add_vectors_nb(self, request):
|
||||||
|
yield request.param
|
||||||
|
|
||||||
|
"""
|
||||||
|
generate valid create_index params
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.fixture(
|
||||||
|
scope="function",
|
||||||
|
params=gen_simple_index()
|
||||||
|
)
|
||||||
|
def get_simple_index(self, request, connect):
|
||||||
|
if str(connect._cmd("mode")[1]) == "CPU":
|
||||||
|
if request.param["index_type"] == IndexType.IVF_SQ8H:
|
||||||
|
pytest.skip("sq8h not support in CPU mode")
|
||||||
|
if request.param["index_type"] == IndexType.IVF_PQ:
|
||||||
|
pytest.skip("Skip PQ Temporary")
|
||||||
|
return request.param
|
||||||
|
|
||||||
|
def test_collection_rows_count(self, connect, ip_collection, add_vectors_nb):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not
|
||||||
|
method: create collection and add vectors in it,
|
||||||
|
assert the value returned by count_collection method is equal to length of vectors
|
||||||
|
expected: the count is equal to the length of vectors
|
||||||
|
'''
|
||||||
|
nb = add_vectors_nb
|
||||||
|
vectors = gen_vectors(nb, dim)
|
||||||
|
res = connect.add_vectors(collection_name=ip_collection, records=vectors)
|
||||||
|
connect.flush([ip_collection])
|
||||||
|
status, res = connect.count_collection(ip_collection)
|
||||||
|
assert res == nb
|
||||||
|
|
||||||
|
def test_collection_rows_count_after_index_created(self, connect, ip_collection, get_simple_index):
|
||||||
|
'''
|
||||||
|
target: test count_collection, after index have been created
|
||||||
|
method: add vectors in db, and create index, then calling count_collection with correct params
|
||||||
|
expected: count_collection raise exception
|
||||||
|
'''
|
||||||
|
index_param = get_simple_index["index_param"]
|
||||||
|
index_type = get_simple_index["index_type"]
|
||||||
|
nb = 100
|
||||||
|
vectors = gen_vectors(nb, dim)
|
||||||
|
res = connect.add_vectors(collection_name=ip_collection, records=vectors)
|
||||||
|
connect.flush([ip_collection])
|
||||||
|
connect.create_index(ip_collection, index_type, index_param)
|
||||||
|
status, res = connect.count_collection(ip_collection)
|
||||||
|
assert res == nb
|
||||||
|
|
||||||
|
@pytest.mark.level(2)
|
||||||
|
def test_count_without_connection(self, ip_collection, dis_connect):
|
||||||
|
'''
|
||||||
|
target: test count_collection, without connection
|
||||||
|
method: calling count_collection with correct params, with a disconnected instance
|
||||||
|
expected: count_collection raise exception
|
||||||
|
'''
|
||||||
|
with pytest.raises(Exception) as e:
|
||||||
|
status = dis_connect.count_collection(ip_collection)
|
||||||
|
|
||||||
|
def test_collection_rows_count_no_vectors(self, connect, ip_collection):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not, if collection is empty
|
||||||
|
method: create collection and no vectors in it,
|
||||||
|
assert the value returned by count_collection method is equal to 0
|
||||||
|
expected: the count is equal to 0
|
||||||
|
'''
|
||||||
|
collection_name = gen_unique_str("test_collection")
|
||||||
|
param = {'collection_name': collection_name,
|
||||||
|
'dimension': dim,
|
||||||
|
'index_file_size': index_file_size}
|
||||||
|
connect.create_collection(param)
|
||||||
|
status, res = connect.count_collection(ip_collection)
|
||||||
|
assert res == 0
|
||||||
|
|
||||||
|
# TODO: enable
|
||||||
|
@pytest.mark.timeout(60)
|
||||||
|
def _test_collection_rows_count_multiprocessing(self, connect, ip_collection, args):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not with multiprocess
|
||||||
|
method: create collection and add vectors in it,
|
||||||
|
assert the value returned by count_collection method is equal to length of vectors
|
||||||
|
expected: the count is equal to the length of vectors
|
||||||
|
'''
|
||||||
|
nq = 2
|
||||||
|
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
||||||
|
vectors = gen_vectors(nq, dim)
|
||||||
|
res = connect.add_vectors(collection_name=ip_collection, records=vectors)
|
||||||
|
time.sleep(add_time_interval)
|
||||||
|
|
||||||
|
def rows_count(milvus):
|
||||||
|
status, res = milvus.count_collection(ip_collection)
|
||||||
|
logging.getLogger().info(status)
|
||||||
|
assert res == nq
|
||||||
|
|
||||||
|
process_num = 8
|
||||||
|
processes = []
|
||||||
|
for i in range(process_num):
|
||||||
|
milvus = get_milvus(args["handler"])
|
||||||
|
milvus.connect(uri=uri)
|
||||||
|
p = Process(target=rows_count, args=(milvus,))
|
||||||
|
processes.append(p)
|
||||||
|
p.start()
|
||||||
|
logging.getLogger().info(p)
|
||||||
|
for p in processes:
|
||||||
|
p.join()
|
||||||
|
|
||||||
|
def test_collection_rows_count_multi_collections(self, connect):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not with multiple collections of IP
|
||||||
|
method: create collection and add vectors in it,
|
||||||
|
assert the value returned by count_collection method is equal to length of vectors
|
||||||
|
expected: the count is equal to the length of vectors
|
||||||
|
'''
|
||||||
|
nq = 100
|
||||||
|
vectors = gen_vectors(nq, dim)
|
||||||
|
collection_list = []
|
||||||
|
for i in range(20):
|
||||||
|
collection_name = gen_unique_str('test_collection_rows_count_multi_collections')
|
||||||
|
collection_list.append(collection_name)
|
||||||
|
param = {'collection_name': collection_name,
|
||||||
|
'dimension': dim,
|
||||||
|
'index_file_size': index_file_size,
|
||||||
|
'metric_type': MetricType.IP}
|
||||||
|
connect.create_collection(param)
|
||||||
|
res = connect.add_vectors(collection_name=collection_name, records=vectors)
|
||||||
|
connect.flush(collection_list)
|
||||||
|
for i in range(20):
|
||||||
|
status, res = connect.count_collection(collection_list[i])
|
||||||
|
assert status.OK()
|
||||||
|
assert res == nq
|
||||||
|
|
||||||
|
|
||||||
|
class TestCollectionCountJAC:
|
||||||
|
"""
|
||||||
|
params means different nb, the nb value may trigger merge, or not
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.fixture(
|
||||||
|
scope="function",
|
||||||
|
params=[
|
||||||
|
1,
|
||||||
|
5000,
|
||||||
|
100000,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def add_vectors_nb(self, request):
|
||||||
|
yield request.param
|
||||||
|
|
||||||
|
"""
|
||||||
|
generate valid create_index params
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.fixture(
|
||||||
|
scope="function",
|
||||||
|
params=gen_simple_index()
|
||||||
|
)
|
||||||
|
def get_jaccard_index(self, request, connect):
|
||||||
|
logging.getLogger().info(request.param)
|
||||||
|
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
|
||||||
|
return request.param
|
||||||
|
else:
|
||||||
|
pytest.skip("Skip index Temporary")
|
||||||
|
|
||||||
|
def test_collection_rows_count(self, connect, jac_collection, add_vectors_nb):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not
|
||||||
|
method: create collection and add vectors in it,
|
||||||
|
assert the value returned by count_collection method is equal to length of vectors
|
||||||
|
expected: the count is equal to the length of vectors
|
||||||
|
'''
|
||||||
|
nb = add_vectors_nb
|
||||||
|
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||||
|
res = connect.add_vectors(collection_name=jac_collection, records=vectors)
|
||||||
|
connect.flush([jac_collection])
|
||||||
|
status, res = connect.count_collection(jac_collection)
|
||||||
|
assert res == nb
|
||||||
|
|
||||||
|
def test_collection_rows_count_after_index_created(self, connect, jac_collection, get_jaccard_index):
|
||||||
|
'''
|
||||||
|
target: test count_collection, after index have been created
|
||||||
|
method: add vectors in db, and create index, then calling count_collection with correct params
|
||||||
|
expected: count_collection raise exception
|
||||||
|
'''
|
||||||
|
nb = 100
|
||||||
|
index_param = get_jaccard_index["index_param"]
|
||||||
|
index_type = get_jaccard_index["index_type"]
|
||||||
|
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||||
|
res = connect.add_vectors(collection_name=jac_collection, records=vectors)
|
||||||
|
connect.flush([jac_collection])
|
||||||
|
connect.create_index(jac_collection, index_type, index_param)
|
||||||
|
status, res = connect.count_collection(jac_collection)
|
||||||
|
assert res == nb
|
||||||
|
|
||||||
|
@pytest.mark.level(2)
|
||||||
|
def test_count_without_connection(self, jac_collection, dis_connect):
|
||||||
|
'''
|
||||||
|
target: test count_collection, without connection
|
||||||
|
method: calling count_collection with correct params, with a disconnected instance
|
||||||
|
expected: count_collection raise exception
|
||||||
|
'''
|
||||||
|
with pytest.raises(Exception) as e:
|
||||||
|
status = dis_connect.count_collection(jac_collection)
|
||||||
|
|
||||||
|
def test_collection_rows_count_no_vectors(self, connect, jac_collection):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not, if collection is empty
|
||||||
|
method: create collection and no vectors in it,
|
||||||
|
assert the value returned by count_collection method is equal to 0
|
||||||
|
expected: the count is equal to 0
|
||||||
|
'''
|
||||||
|
collection_name = gen_unique_str("test_collection")
|
||||||
|
param = {'collection_name': collection_name,
|
||||||
|
'dimension': dim,
|
||||||
|
'index_file_size': index_file_size}
|
||||||
|
connect.create_collection(param)
|
||||||
|
status, res = connect.count_collection(jac_collection)
|
||||||
|
assert res == 0
|
||||||
|
|
||||||
|
def test_collection_rows_count_multi_collections(self, connect):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not with multiple collections of IP
|
||||||
|
method: create collection and add vectors in it,
|
||||||
|
assert the value returned by count_collection method is equal to length of vectors
|
||||||
|
expected: the count is equal to the length of vectors
|
||||||
|
'''
|
||||||
|
nq = 100
|
||||||
|
tmp, vectors = gen_binary_vectors(nq, dim)
|
||||||
|
collection_list = []
|
||||||
|
for i in range(20):
|
||||||
|
collection_name = gen_unique_str('test_collection_rows_count_multi_collections')
|
||||||
|
collection_list.append(collection_name)
|
||||||
|
param = {'collection_name': collection_name,
|
||||||
|
'dimension': dim,
|
||||||
|
'index_file_size': index_file_size,
|
||||||
|
'metric_type': MetricType.JACCARD}
|
||||||
|
connect.create_collection(param)
|
||||||
|
res = connect.add_vectors(collection_name=collection_name, records=vectors)
|
||||||
|
connect.flush(collection_list)
|
||||||
|
for i in range(20):
|
||||||
|
status, res = connect.count_collection(collection_list[i])
|
||||||
|
assert status.OK()
|
||||||
|
assert res == nq
|
||||||
|
|
||||||
|
class TestCollectionCountHAM:
|
||||||
|
"""
|
||||||
|
params means different nb, the nb value may trigger merge, or not
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.fixture(
|
||||||
|
scope="function",
|
||||||
|
params=[
|
||||||
|
1,
|
||||||
|
5000,
|
||||||
|
100000,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def add_vectors_nb(self, request):
|
||||||
|
yield request.param
|
||||||
|
|
||||||
|
"""
|
||||||
|
generate valid create_index params
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.fixture(
|
||||||
|
scope="function",
|
||||||
|
params=gen_simple_index()
|
||||||
|
)
|
||||||
|
def get_hamming_index(self, request, connect):
|
||||||
|
logging.getLogger().info(request.param)
|
||||||
|
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
|
||||||
|
return request.param
|
||||||
|
else:
|
||||||
|
pytest.skip("Skip index Temporary")
|
||||||
|
|
||||||
|
def test_collection_rows_count(self, connect, ham_collection, add_vectors_nb):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not
|
||||||
|
method: create collection and add vectors in it,
|
||||||
|
assert the value returned by count_collection method is equal to length of vectors
|
||||||
|
expected: the count is equal to the length of vectors
|
||||||
|
'''
|
||||||
|
nb = add_vectors_nb
|
||||||
|
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||||
|
res = connect.add_vectors(collection_name=ham_collection, records=vectors)
|
||||||
|
connect.flush([ham_collection])
|
||||||
|
status, res = connect.count_collection(ham_collection)
|
||||||
|
assert res == nb
|
||||||
|
|
||||||
|
def test_collection_rows_count_after_index_created(self, connect, ham_collection, get_hamming_index):
|
||||||
|
'''
|
||||||
|
target: test count_collection, after index have been created
|
||||||
|
method: add vectors in db, and create index, then calling count_collection with correct params
|
||||||
|
expected: count_collection raise exception
|
||||||
|
'''
|
||||||
|
nb = 100
|
||||||
|
index_type = get_hamming_index["index_type"]
|
||||||
|
index_param = get_hamming_index["index_param"]
|
||||||
|
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||||
|
res = connect.add_vectors(collection_name=ham_collection, records=vectors)
|
||||||
|
connect.flush([ham_collection])
|
||||||
|
connect.create_index(ham_collection, index_type, index_param)
|
||||||
|
status, res = connect.count_collection(ham_collection)
|
||||||
|
assert res == nb
|
||||||
|
|
||||||
|
@pytest.mark.level(2)
|
||||||
|
def test_count_without_connection(self, ham_collection, dis_connect):
|
||||||
|
'''
|
||||||
|
target: test count_collection, without connection
|
||||||
|
method: calling count_collection with correct params, with a disconnected instance
|
||||||
|
expected: count_collection raise exception
|
||||||
|
'''
|
||||||
|
with pytest.raises(Exception) as e:
|
||||||
|
status = dis_connect.count_collection(ham_collection)
|
||||||
|
|
||||||
|
def test_collection_rows_count_no_vectors(self, connect, ham_collection):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not, if collection is empty
|
||||||
|
method: create collection and no vectors in it,
|
||||||
|
assert the value returned by count_collection method is equal to 0
|
||||||
|
expected: the count is equal to 0
|
||||||
|
'''
|
||||||
|
collection_name = gen_unique_str("test_collection")
|
||||||
|
param = {'collection_name': collection_name,
|
||||||
|
'dimension': dim,
|
||||||
|
'index_file_size': index_file_size}
|
||||||
|
connect.create_collection(param)
|
||||||
|
status, res = connect.count_collection(ham_collection)
|
||||||
|
assert res == 0
|
||||||
|
|
||||||
|
def test_collection_rows_count_multi_collections(self, connect):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not with multiple collections of IP
|
||||||
|
method: create collection and add vectors in it,
|
||||||
|
assert the value returned by count_collection method is equal to length of vectors
|
||||||
|
expected: the count is equal to the length of vectors
|
||||||
|
'''
|
||||||
|
nq = 100
|
||||||
|
tmp, vectors = gen_binary_vectors(nq, dim)
|
||||||
|
collection_list = []
|
||||||
|
for i in range(20):
|
||||||
|
collection_name = gen_unique_str('test_collection_rows_count_multi_collections')
|
||||||
|
collection_list.append(collection_name)
|
||||||
|
param = {'collection_name': collection_name,
|
||||||
|
'dimension': dim,
|
||||||
|
'index_file_size': index_file_size,
|
||||||
|
'metric_type': MetricType.HAMMING}
|
||||||
|
connect.create_collection(param)
|
||||||
|
res = connect.add_vectors(collection_name=collection_name, records=vectors)
|
||||||
|
connect.flush(collection_list)
|
||||||
|
for i in range(20):
|
||||||
|
status, res = connect.count_collection(collection_list[i])
|
||||||
|
assert status.OK()
|
||||||
|
assert res == nq
|
||||||
|
|
||||||
|
|
||||||
|
class TestCollectionCountTANIMOTO:
|
||||||
|
"""
|
||||||
|
params means different nb, the nb value may trigger merge, or not
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.fixture(
|
||||||
|
scope="function",
|
||||||
|
params=[
|
||||||
|
1,
|
||||||
|
5000,
|
||||||
|
100000,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def add_vectors_nb(self, request):
|
||||||
|
yield request.param
|
||||||
|
|
||||||
|
"""
|
||||||
|
generate valid create_index params
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.fixture(
|
||||||
|
scope="function",
|
||||||
|
params=gen_simple_index()
|
||||||
|
)
|
||||||
|
def get_tanimoto_index(self, request, connect):
|
||||||
|
logging.getLogger().info(request.param)
|
||||||
|
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
|
||||||
|
return request.param
|
||||||
|
else:
|
||||||
|
pytest.skip("Skip index Temporary")
|
||||||
|
|
||||||
|
def test_collection_rows_count(self, connect, tanimoto_collection, add_vectors_nb):
|
||||||
|
'''
|
||||||
|
target: test collection rows_count is correct or not
|
||||||
|
method: create collection and add vectors in it,
|
||||||
|
assert the value returned by count_collection method is equal to length of vectors
|
||||||
|
expected: the count is equal to the length of vectors
|
||||||
|
'''
|
||||||
|
nb = add_vectors_nb
|
||||||
|
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||||
|
res = connect.add_vectors(collection_name=tanimoto_collection, records=vectors)
|
||||||
|
connect.flush([tanimoto_collection])
|
||||||
|
status, res = connect.count_collection(tanimoto_collection)
|
||||||
|
assert status.OK()
|
||||||
|
assert res == nb
|
|
@ -18,7 +18,7 @@ nb = 6000
|
||||||
nlist = 1024
|
nlist = 1024
|
||||||
|
|
||||||
|
|
||||||
class TestTableInfoBase:
|
class TestCollectionInfoBase:
|
||||||
def index_string_convert(self, index_string, index_type):
|
def index_string_convert(self, index_string, index_type):
|
||||||
if index_string == "IDMAP" and index_type == IndexType.FLAT:
|
if index_string == "IDMAP" and index_type == IndexType.FLAT:
|
||||||
return True
|
return True
|
||||||
|
@ -30,87 +30,87 @@ class TestTableInfoBase:
|
||||||
|
|
||||||
"""
|
"""
|
||||||
******************************************************************
|
******************************************************************
|
||||||
The following cases are used to test `table_info` function
|
The following cases are used to test `collection_info` function
|
||||||
******************************************************************
|
******************************************************************
|
||||||
"""
|
"""
|
||||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||||
def test_get_table_info_name_None(self, connect, table):
|
def test_get_collection_info_name_None(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: get table info where table name is None
|
target: get collection info where collection name is None
|
||||||
method: call table_info with the table_name: None
|
method: call collection_info with the collection_name: None
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
table_name = None
|
collection_name = None
|
||||||
status, info = connect.table_info(table_name)
|
status, info = connect.collection_info(collection_name)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||||
def test_get_table_info_name_not_existed(self, connect, table):
|
def test_get_collection_info_name_not_existed(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: get table info where table name does not exist
|
target: get collection info where collection name does not exist
|
||||||
method: call table_info with a random table_name, which is not in db
|
method: call collection_info with a random collection_name, which is not in db
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
table_name = gen_unique_str("not_existed_table")
|
collection_name = gen_unique_str("not_existed_collection")
|
||||||
status, info = connect.table_info(table_name)
|
status, info = connect.collection_info(collection_name)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
@pytest.fixture(
|
@pytest.fixture(
|
||||||
scope="function",
|
scope="function",
|
||||||
params=gen_invalid_table_names()
|
params=gen_invalid_collection_names()
|
||||||
)
|
)
|
||||||
def get_table_name(self, request):
|
def get_collection_name(self, request):
|
||||||
yield request.param
|
yield request.param
|
||||||
|
|
||||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||||
def test_get_table_info_name_invalid(self, connect, get_table_name):
|
def test_get_collection_info_name_invalid(self, connect, get_collection_name):
|
||||||
'''
|
'''
|
||||||
target: get table info where table name is invalid
|
target: get collection info where collection name is invalid
|
||||||
method: call table_info with invalid table_name
|
method: call collection_info with invalid collection_name
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
table_name = get_table_name
|
collection_name = get_collection_name
|
||||||
status, info = connect.table_info(table_name)
|
status, info = connect.collection_info(collection_name)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||||
def test_get_table_info_table_row_count(self, connect, table):
|
def test_get_collection_info_collection_row_count(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: get row count with table_info
|
target: get row count with collection_info
|
||||||
method: add and delete vectors, check count in table info
|
method: add and delete vectors, check count in collection info
|
||||||
expected: status ok, count as expected
|
expected: status ok, count as expected
|
||||||
'''
|
'''
|
||||||
vectors = gen_vector(nb, dim)
|
vectors = gen_vector(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(table)
|
status, info = connect.collection_info(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert info.count == nb
|
assert info.count == nb
|
||||||
# delete a few vectors
|
# delete a few vectors
|
||||||
delete_ids = [ids[0], ids[-1]]
|
delete_ids = [ids[0], ids[-1]]
|
||||||
status = connect.delete_by_id(table, delete_ids)
|
status = connect.delete_by_id(collection, delete_ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(table)
|
status, info = connect.collection_info(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert info.count == nb - 2
|
assert info.count == nb - 2
|
||||||
|
|
||||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||||
def test_get_table_info_partition_stats_A(self, connect, table):
|
def test_get_collection_info_partition_stats_A(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: get partition info in a table
|
target: get partition info in a collection
|
||||||
method: no partition, call table_info and check partition_stats
|
method: no partition, call collection_info and check partition_stats
|
||||||
expected: status ok, "_default" partition is listed
|
expected: status ok, "_default" partition is listed
|
||||||
'''
|
'''
|
||||||
vectors = gen_vector(nb, dim)
|
vectors = gen_vector(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(table)
|
status, info = connect.collection_info(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
logging.getLogger().info(info)
|
logging.getLogger().info(info)
|
||||||
assert len(info.partitions_stat) == 1
|
assert len(info.partitions_stat) == 1
|
||||||
|
@ -119,19 +119,19 @@ class TestTableInfoBase:
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||||
def test_get_table_info_partition_stats_B(self, connect, table):
|
def test_get_collection_info_partition_stats_B(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: get partition info in a table
|
target: get partition info in a collection
|
||||||
method: call table_info after partition created and check partition_stats
|
method: call collection_info after partition created and check partition_stats
|
||||||
expected: status ok, vectors added to partition
|
expected: status ok, vectors added to partition
|
||||||
'''
|
'''
|
||||||
vectors = gen_vectors(nb, dim)
|
vectors = gen_vectors(nb, dim)
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
|
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(table)
|
status, info = connect.collection_info(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
logging.getLogger().info(info)
|
logging.getLogger().info(info)
|
||||||
assert len(info.partitions_stat) == 2
|
assert len(info.partitions_stat) == 2
|
||||||
|
@ -139,23 +139,23 @@ class TestTableInfoBase:
|
||||||
assert info.partitions_stat[1].count == nb
|
assert info.partitions_stat[1].count == nb
|
||||||
|
|
||||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||||
def test_get_table_info_partition_stats_C(self, connect, table):
|
def test_get_collection_info_partition_stats_C(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: get partition info in a table
|
target: get partition info in a collection
|
||||||
method: create two partitions, add vectors in one of the partitions, call table_info and check
|
method: create two partitions, add vectors in one of the partitions, call collection_info and check
|
||||||
expected: status ok, vectors added to one partition but not the other
|
expected: status ok, vectors added to one partition but not the other
|
||||||
'''
|
'''
|
||||||
new_tag = "new_tag"
|
new_tag = "new_tag"
|
||||||
vectors = gen_vectors(nb, dim)
|
vectors = gen_vectors(nb, dim)
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.create_partition(table, new_tag)
|
status = connect.create_partition(collection, new_tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
|
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(table)
|
status, info = connect.collection_info(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
logging.getLogger().info(info)
|
logging.getLogger().info(info)
|
||||||
for partition in info.partitions_stat:
|
for partition in info.partitions_stat:
|
||||||
|
@ -165,25 +165,25 @@ class TestTableInfoBase:
|
||||||
assert partition.count == 0
|
assert partition.count == 0
|
||||||
|
|
||||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||||
def test_get_table_info_partition_stats_D(self, connect, table):
|
def test_get_collection_info_partition_stats_D(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: get partition info in a table
|
target: get partition info in a collection
|
||||||
method: create two partitions, add vectors in both partitions, call table_info and check
|
method: create two partitions, add vectors in both partitions, call collection_info and check
|
||||||
expected: status ok, vectors added to both partitions
|
expected: status ok, vectors added to both partitions
|
||||||
'''
|
'''
|
||||||
new_tag = "new_tag"
|
new_tag = "new_tag"
|
||||||
vectors = gen_vectors(nb, dim)
|
vectors = gen_vectors(nb, dim)
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.create_partition(table, new_tag)
|
status = connect.create_partition(collection, new_tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
|
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, ids = connect.add_vectors(table, vectors, partition_tag=new_tag)
|
status, ids = connect.add_vectors(collection, vectors, partition_tag=new_tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(table)
|
status, info = connect.collection_info(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert info.count == nb * 2
|
assert info.count == nb * 2
|
||||||
for partition in info.partitions_stat:
|
for partition in info.partitions_stat:
|
||||||
|
@ -205,24 +205,24 @@ class TestTableInfoBase:
|
||||||
return request.param
|
return request.param
|
||||||
|
|
||||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||||
def test_get_table_info_after_index_created(self, connect, table, get_simple_index):
|
def test_get_collection_info_after_index_created(self, connect, collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
target: test table info after index created
|
target: test collection info after index created
|
||||||
method: create table, add vectors, create index and call table_info
|
method: create collection, add vectors, create index and call collection_info
|
||||||
expected: status ok, index created and shown in segments_stat
|
expected: status ok, index created and shown in segments_stat
|
||||||
'''
|
'''
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
vectors = gen_vector(nb, dim)
|
vectors = gen_vector(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.create_index(table, index_type, index_param)
|
status = connect.create_index(collection, index_type, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(table)
|
status, info = connect.collection_info(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
logging.getLogger().info(info)
|
logging.getLogger().info(info)
|
||||||
index_string = info.partitions_stat[0].segments_stat[0].index_name
|
index_string = info.partitions_stat[0].segments_stat[0].index_name
|
||||||
|
@ -231,24 +231,24 @@ class TestTableInfoBase:
|
||||||
assert nb == info.partitions_stat[0].segments_stat[0].count
|
assert nb == info.partitions_stat[0].segments_stat[0].count
|
||||||
|
|
||||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||||
def test_get_table_info_after_create_same_index_repeatedly(self, connect, table, get_simple_index):
|
def test_get_collection_info_after_create_same_index_repeatedly(self, connect, collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
target: test table info after index created repeatedly
|
target: test collection info after index created repeatedly
|
||||||
method: create table, add vectors, create index and call table_info multiple times
|
method: create collection, add vectors, create index and call collection_info multiple times
|
||||||
expected: status ok, index info shown in segments_stat
|
expected: status ok, index info shown in segments_stat
|
||||||
'''
|
'''
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
vectors = gen_vector(nb, dim)
|
vectors = gen_vector(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.create_index(table, index_type, index_param)
|
status = connect.create_index(collection, index_type, index_param)
|
||||||
status = connect.create_index(table, index_type, index_param)
|
status = connect.create_index(collection, index_type, index_param)
|
||||||
status = connect.create_index(table, index_type, index_param)
|
status = connect.create_index(collection, index_type, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(table)
|
status, info = connect.collection_info(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
logging.getLogger().info(info)
|
logging.getLogger().info(info)
|
||||||
index_string = info.partitions_stat[0].segments_stat[0].index_name
|
index_string = info.partitions_stat[0].segments_stat[0].index_name
|
||||||
|
@ -257,22 +257,22 @@ class TestTableInfoBase:
|
||||||
assert nb == info.partitions_stat[0].segments_stat[0].count
|
assert nb == info.partitions_stat[0].segments_stat[0].count
|
||||||
|
|
||||||
@pytest.mark.timeout(INFO_TIMEOUT)
|
@pytest.mark.timeout(INFO_TIMEOUT)
|
||||||
def test_get_table_info_after_create_different_index_repeatedly(self, connect, table, get_simple_index):
|
def test_get_collection_info_after_create_different_index_repeatedly(self, connect, collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
target: test table info after index created repeatedly
|
target: test collection info after index created repeatedly
|
||||||
method: create table, add vectors, create index and call table_info multiple times
|
method: create collection, add vectors, create index and call collection_info multiple times
|
||||||
expected: status ok, index info shown in segments_stat
|
expected: status ok, index info shown in segments_stat
|
||||||
'''
|
'''
|
||||||
vectors = gen_vector(nb, dim)
|
vectors = gen_vector(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
index_param = {"nlist": nlist}
|
index_param = {"nlist": nlist}
|
||||||
for index_type in [IndexType.FLAT, IndexType.IVFLAT, IndexType.IVF_SQ8]:
|
for index_type in [IndexType.FLAT, IndexType.IVFLAT, IndexType.IVF_SQ8]:
|
||||||
status = connect.create_index(table, index_type, index_param)
|
status = connect.create_index(collection, index_type, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(table)
|
status, info = connect.collection_info(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
logging.getLogger().info(info)
|
logging.getLogger().info(info)
|
||||||
index_string = info.partitions_stat[0].segments_stat[0].index_name
|
index_string = info.partitions_stat[0].segments_stat[0].index_name
|
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
@ -285,7 +285,7 @@ class TestConnect:
|
||||||
b. data_set not too large incase disconnection happens when data is underd-preparing
|
b. data_set not too large incase disconnection happens when data is underd-preparing
|
||||||
c. data_set not too small incase disconnection happens when data has already been transferred
|
c. data_set not too small incase disconnection happens when data has already been transferred
|
||||||
d. make sure disconnection happens when data is in-transport
|
d. make sure disconnection happens when data is in-transport
|
||||||
Expected: Failure, get_table_row_count == 0
|
Expected: Failure, count_collection == 0
|
||||||
|
|
||||||
'''
|
'''
|
||||||
pass
|
pass
|
||||||
|
|
|
@ -11,7 +11,7 @@ from utils import *
|
||||||
|
|
||||||
dim = 128
|
dim = 128
|
||||||
index_file_size = 10
|
index_file_size = 10
|
||||||
table_id = "test_delete"
|
collection_id = "test_delete"
|
||||||
DELETE_TIMEOUT = 60
|
DELETE_TIMEOUT = 60
|
||||||
nprobe = 1
|
nprobe = 1
|
||||||
epsilon = 0.001
|
epsilon = 0.001
|
||||||
|
@ -38,7 +38,7 @@ class TestDeleteBase:
|
||||||
pytest.skip("Only support CPU mode")
|
pytest.skip("Only support CPU mode")
|
||||||
return request.param
|
return request.param
|
||||||
|
|
||||||
def test_delete_vector_search(self, connect, table, get_simple_index):
|
def test_delete_vector_search(self, connect, collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
target: test delete vector
|
target: test delete vector
|
||||||
method: add vector and delete
|
method: add vector and delete
|
||||||
|
@ -47,22 +47,22 @@ class TestDeleteBase:
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
vector = gen_single_vector(dim)
|
vector = gen_single_vector(dim)
|
||||||
status, ids = connect.add_vectors(table, vector)
|
status, ids = connect.add_vectors(collection, vector)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.delete_by_id(table, ids)
|
status = connect.delete_by_id(collection, ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
search_param = get_search_param(index_type)
|
search_param = get_search_param(index_type)
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
search_param = get_search_param(index_type)
|
search_param = get_search_param(index_type)
|
||||||
status, res = connect.search_vectors(table, top_k, vector, params=search_param)
|
status, res = connect.search_vectors(collection, top_k, vector, params=search_param)
|
||||||
logging.getLogger().info(res)
|
logging.getLogger().info(res)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert len(res) == 0
|
assert len(res) == 0
|
||||||
|
|
||||||
def test_delete_vector_multi_same_ids(self, connect, table, get_simple_index):
|
def test_delete_vector_multi_same_ids(self, connect, collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
target: test delete vector, with some same ids
|
target: test delete vector, with some same ids
|
||||||
method: add vector and delete
|
method: add vector and delete
|
||||||
|
@ -71,58 +71,58 @@ class TestDeleteBase:
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
vectors = gen_vectors(nb, dim)
|
vectors = gen_vectors(nb, dim)
|
||||||
connect.add_vectors(table, vectors, ids=[1 for i in range(nb)])
|
connect.add_vectors(collection, vectors, ids=[1 for i in range(nb)])
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
# Bloom filter error
|
# Bloom filter error
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.delete_by_id(table, [1])
|
status = connect.delete_by_id(collection, [1])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
search_param = get_search_param(index_type)
|
search_param = get_search_param(index_type)
|
||||||
status, res = connect.search_vectors(table, top_k, [vectors[0]], params=search_param)
|
status, res = connect.search_vectors(collection, top_k, [vectors[0]], params=search_param)
|
||||||
logging.getLogger().info(res)
|
logging.getLogger().info(res)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert len(res) == 0
|
assert len(res) == 0
|
||||||
|
|
||||||
def test_delete_vector_table_count(self, connect, table):
|
def test_delete_vector_collection_count(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test delete vector
|
target: test delete vector
|
||||||
method: add vector and delete
|
method: add vector and delete
|
||||||
expected: status ok, vector deleted
|
expected: status ok, vector deleted
|
||||||
'''
|
'''
|
||||||
vector = gen_single_vector(dim)
|
vector = gen_single_vector(dim)
|
||||||
status, ids = connect.add_vectors(table, vector)
|
status, ids = connect.add_vectors(collection, vector)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.delete_by_id(table, ids)
|
status = connect.delete_by_id(collection, ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
status, res = connect.get_table_row_count(table)
|
status, res = connect.count_collection(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert res == 0
|
assert res == 0
|
||||||
|
|
||||||
def test_delete_vector_table_count_no_flush(self, connect, table):
|
def test_delete_vector_collection_count_no_flush(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test delete vector
|
target: test delete vector
|
||||||
method: add vector and delete, no flush(using auto flush)
|
method: add vector and delete, no flush(using auto flush)
|
||||||
expected: status ok, vector deleted
|
expected: status ok, vector deleted
|
||||||
'''
|
'''
|
||||||
vector = gen_single_vector(dim)
|
vector = gen_single_vector(dim)
|
||||||
status, ids = connect.add_vectors(table, vector)
|
status, ids = connect.add_vectors(collection, vector)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.delete_by_id(table, ids)
|
status = connect.delete_by_id(collection, ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
time.sleep(2)
|
time.sleep(2)
|
||||||
status, res = connect.get_table_row_count(table)
|
status, res = connect.count_collection(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert res == 0
|
assert res == 0
|
||||||
|
|
||||||
def test_delete_vector_id_not_exised(self, connect, table, get_simple_index):
|
def test_delete_vector_id_not_exised(self, connect, collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
target: test delete vector, params vector_id not existed
|
target: test delete vector, params vector_id not existed
|
||||||
method: add vector and delete
|
method: add vector and delete
|
||||||
|
@ -131,34 +131,34 @@ class TestDeleteBase:
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
vector = gen_single_vector(dim)
|
vector = gen_single_vector(dim)
|
||||||
status, ids = connect.add_vectors(table, vector)
|
status, ids = connect.add_vectors(collection, vector)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.delete_by_id(table, [0])
|
status = connect.delete_by_id(collection, [0])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
search_param = get_search_param(index_type)
|
search_param = get_search_param(index_type)
|
||||||
status, res = connect.search_vectors(table, top_k, vector, params=search_param)
|
status, res = connect.search_vectors(collection, top_k, vector, params=search_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert res[0][0].id == ids[0]
|
assert res[0][0].id == ids[0]
|
||||||
|
|
||||||
def test_delete_vector_table_not_existed(self, connect, table):
|
def test_delete_vector_collection_not_existed(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test delete vector, params table_name not existed
|
target: test delete vector, params collection_name not existed
|
||||||
method: add vector and delete
|
method: add vector and delete
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
vector = gen_single_vector(dim)
|
vector = gen_single_vector(dim)
|
||||||
status, ids = connect.add_vectors(table, vector)
|
status, ids = connect.add_vectors(collection, vector)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
table_new = gen_unique_str()
|
collection_new = gen_unique_str()
|
||||||
status = connect.delete_by_id(table_new, [0])
|
status = connect.delete_by_id(collection_new, [0])
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
def test_add_vectors_delete_vector(self, connect, table, get_simple_index):
|
def test_add_vectors_delete_vector(self, connect, collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
method: add vectors and delete
|
method: add vectors and delete
|
||||||
expected: status ok, vectors deleted
|
expected: status ok, vectors deleted
|
||||||
|
@ -166,19 +166,19 @@ class TestDeleteBase:
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
vectors = gen_vector(nb, dim)
|
vectors = gen_vector(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
delete_ids = [ids[0], ids[-1]]
|
delete_ids = [ids[0], ids[-1]]
|
||||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||||
status = connect.delete_by_id(table, delete_ids)
|
status = connect.delete_by_id(collection, delete_ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
search_param = get_search_param(index_type)
|
search_param = get_search_param(index_type)
|
||||||
status, res = connect.search_vectors(table, top_k, query_vecs, params=search_param)
|
status, res = connect.search_vectors(collection, top_k, query_vecs, params=search_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
logging.getLogger().info(res)
|
logging.getLogger().info(res)
|
||||||
assert res[0][0].distance > epsilon
|
assert res[0][0].distance > epsilon
|
||||||
|
@ -186,7 +186,7 @@ class TestDeleteBase:
|
||||||
assert res[1][0].id == ids[1]
|
assert res[1][0].id == ids[1]
|
||||||
assert res[2][0].distance > epsilon
|
assert res[2][0].distance > epsilon
|
||||||
|
|
||||||
def test_create_index_after_delete(self, connect, table, get_simple_index):
|
def test_create_index_after_delete(self, connect, collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
method: add vectors and delete, then create index
|
method: add vectors and delete, then create index
|
||||||
expected: status ok, vectors deleted, index created
|
expected: status ok, vectors deleted, index created
|
||||||
|
@ -194,19 +194,19 @@ class TestDeleteBase:
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
vectors = gen_vector(nb, dim)
|
vectors = gen_vector(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
delete_ids = [ids[0], ids[-1]]
|
delete_ids = [ids[0], ids[-1]]
|
||||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||||
status = connect.delete_by_id(table, delete_ids)
|
status = connect.delete_by_id(collection, delete_ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
status = connect.create_index(table, index_type, index_param)
|
status = connect.create_index(collection, index_type, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
search_param = get_search_param(index_type)
|
search_param = get_search_param(index_type)
|
||||||
status, res = connect.search_vectors(table, top_k, query_vecs, params=search_param)
|
status, res = connect.search_vectors(collection, top_k, query_vecs, params=search_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
logging.getLogger().info(res)
|
logging.getLogger().info(res)
|
||||||
assert res[0][0].distance > epsilon
|
assert res[0][0].distance > epsilon
|
||||||
|
@ -214,7 +214,7 @@ class TestDeleteBase:
|
||||||
assert res[1][0].id == ids[1]
|
assert res[1][0].id == ids[1]
|
||||||
assert res[2][0].distance > epsilon
|
assert res[2][0].distance > epsilon
|
||||||
|
|
||||||
def test_add_vector_after_delete(self, connect, table, get_simple_index):
|
def test_add_vector_after_delete(self, connect, collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
method: add vectors and delete, then add vector
|
method: add vectors and delete, then add vector
|
||||||
expected: status ok, vectors deleted, vector added
|
expected: status ok, vectors deleted, vector added
|
||||||
|
@ -222,22 +222,22 @@ class TestDeleteBase:
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
vectors = gen_vector(nb, dim)
|
vectors = gen_vector(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
delete_ids = [ids[0], ids[-1]]
|
delete_ids = [ids[0], ids[-1]]
|
||||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||||
status = connect.delete_by_id(table, delete_ids)
|
status = connect.delete_by_id(collection, delete_ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
status, tmp_ids = connect.add_vectors(table, [vectors[0], vectors[-1]])
|
status, tmp_ids = connect.add_vectors(collection, [vectors[0], vectors[-1]])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
search_param = get_search_param(index_type)
|
search_param = get_search_param(index_type)
|
||||||
status, res = connect.search_vectors(table, top_k, query_vecs, params=search_param)
|
status, res = connect.search_vectors(collection, top_k, query_vecs, params=search_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
logging.getLogger().info(res)
|
logging.getLogger().info(res)
|
||||||
assert res[0][0].id == tmp_ids[0]
|
assert res[0][0].id == tmp_ids[0]
|
||||||
|
@ -246,41 +246,41 @@ class TestDeleteBase:
|
||||||
assert res[2][0].id == tmp_ids[-1]
|
assert res[2][0].id == tmp_ids[-1]
|
||||||
assert res[2][0].distance < epsilon
|
assert res[2][0].distance < epsilon
|
||||||
|
|
||||||
def test_delete_multiable_times(self, connect, table):
|
def test_delete_multiable_times(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
method: add vectors and delete id serveral times
|
method: add vectors and delete id serveral times
|
||||||
expected: status ok, vectors deleted, and status ok for next delete operation
|
expected: status ok, vectors deleted, and status ok for next delete operation
|
||||||
'''
|
'''
|
||||||
vectors = gen_vector(nb, dim)
|
vectors = gen_vector(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
delete_ids = [ids[0], ids[-1]]
|
delete_ids = [ids[0], ids[-1]]
|
||||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||||
status = connect.delete_by_id(table, delete_ids)
|
status = connect.delete_by_id(collection, delete_ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
status = connect.delete_by_id(table, delete_ids)
|
status = connect.delete_by_id(collection, delete_ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
|
|
||||||
def test_delete_no_flush_multiable_times(self, connect, table):
|
def test_delete_no_flush_multiable_times(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
method: add vectors and delete id serveral times
|
method: add vectors and delete id serveral times
|
||||||
expected: status ok, vectors deleted, and status ok for next delete operation
|
expected: status ok, vectors deleted, and status ok for next delete operation
|
||||||
'''
|
'''
|
||||||
vectors = gen_vector(nb, dim)
|
vectors = gen_vector(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
delete_ids = [ids[0], ids[-1]]
|
delete_ids = [ids[0], ids[-1]]
|
||||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||||
status = connect.delete_by_id(table, delete_ids)
|
status = connect.delete_by_id(collection, delete_ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
status = connect.delete_by_id(table, delete_ids)
|
status = connect.delete_by_id(collection, delete_ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
|
|
||||||
|
@ -303,7 +303,7 @@ class TestDeleteIndexedVectors:
|
||||||
pytest.skip("Only support CPU mode")
|
pytest.skip("Only support CPU mode")
|
||||||
return request.param
|
return request.param
|
||||||
|
|
||||||
def test_delete_vectors_after_index_created_search(self, connect, table, get_simple_index):
|
def test_delete_vectors_after_index_created_search(self, connect, collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
target: test delete vector after index created
|
target: test delete vector after index created
|
||||||
method: add vector, create index and delete vector
|
method: add vector, create index and delete vector
|
||||||
|
@ -312,22 +312,22 @@ class TestDeleteIndexedVectors:
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
vector = gen_single_vector(dim)
|
vector = gen_single_vector(dim)
|
||||||
status, ids = connect.add_vectors(table, vector)
|
status, ids = connect.add_vectors(collection, vector)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.create_index(table, index_type, index_param)
|
status = connect.create_index(collection, index_type, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.delete_by_id(table, ids)
|
status = connect.delete_by_id(collection, ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
search_param = get_search_param(index_type)
|
search_param = get_search_param(index_type)
|
||||||
status, res = connect.search_vectors(table, top_k, vector, params=search_param)
|
status, res = connect.search_vectors(collection, top_k, vector, params=search_param)
|
||||||
logging.getLogger().info(res)
|
logging.getLogger().info(res)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert len(res) == 0
|
assert len(res) == 0
|
||||||
|
|
||||||
def test_add_vectors_delete_vector(self, connect, table, get_simple_index):
|
def test_add_vectors_delete_vector(self, connect, collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
method: add vectors and delete
|
method: add vectors and delete
|
||||||
expected: status ok, vectors deleted
|
expected: status ok, vectors deleted
|
||||||
|
@ -335,21 +335,21 @@ class TestDeleteIndexedVectors:
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
vectors = gen_vector(nb, dim)
|
vectors = gen_vector(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.create_index(table, index_type, index_param)
|
status = connect.create_index(collection, index_type, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
delete_ids = [ids[0], ids[-1]]
|
delete_ids = [ids[0], ids[-1]]
|
||||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||||
status = connect.delete_by_id(table, delete_ids)
|
status = connect.delete_by_id(collection, delete_ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
search_param = get_search_param(index_type)
|
search_param = get_search_param(index_type)
|
||||||
status, res = connect.search_vectors(table, top_k, query_vecs, params=search_param)
|
status, res = connect.search_vectors(collection, top_k, query_vecs, params=search_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
logging.getLogger().info(res)
|
logging.getLogger().info(res)
|
||||||
assert res[0][0].distance > epsilon
|
assert res[0][0].distance > epsilon
|
||||||
|
@ -375,7 +375,7 @@ class TestDeleteBinary:
|
||||||
else:
|
else:
|
||||||
pytest.skip("Skip index Temporary")
|
pytest.skip("Skip index Temporary")
|
||||||
|
|
||||||
def test_delete_vector_search(self, connect, jac_table, get_simple_index):
|
def test_delete_vector_search(self, connect, jac_collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
target: test delete vector
|
target: test delete vector
|
||||||
method: add vector and delete
|
method: add vector and delete
|
||||||
|
@ -384,15 +384,15 @@ class TestDeleteBinary:
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
tmp, vector = gen_binary_vectors(1, dim)
|
tmp, vector = gen_binary_vectors(1, dim)
|
||||||
status, ids = connect.add_vectors(jac_table, vector)
|
status, ids = connect.add_vectors(jac_collection, vector)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.delete_by_id(jac_table, ids)
|
status = connect.delete_by_id(jac_collection, ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
search_param = get_search_param(index_type)
|
search_param = get_search_param(index_type)
|
||||||
status, res = connect.search_vectors(jac_table, top_k, vector, params=search_param)
|
status, res = connect.search_vectors(jac_collection, top_k, vector, params=search_param)
|
||||||
logging.getLogger().info(res)
|
logging.getLogger().info(res)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert len(res) == 0
|
assert len(res) == 0
|
||||||
|
@ -400,25 +400,25 @@ class TestDeleteBinary:
|
||||||
assert len(res) == 0
|
assert len(res) == 0
|
||||||
|
|
||||||
# TODO: soft delete
|
# TODO: soft delete
|
||||||
def test_delete_vector_table_count(self, connect, jac_table):
|
def test_delete_vector_collection_count(self, connect, jac_collection):
|
||||||
'''
|
'''
|
||||||
target: test delete vector
|
target: test delete vector
|
||||||
method: add vector and delete
|
method: add vector and delete
|
||||||
expected: status ok, vector deleted
|
expected: status ok, vector deleted
|
||||||
'''
|
'''
|
||||||
tmp, vector = gen_binary_vectors(1, dim)
|
tmp, vector = gen_binary_vectors(1, dim)
|
||||||
status, ids = connect.add_vectors(jac_table, vector)
|
status, ids = connect.add_vectors(jac_collection, vector)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.delete_by_id(jac_table, ids)
|
status = connect.delete_by_id(jac_collection, ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
status, res = connect.get_table_row_count(jac_table)
|
status, res = connect.count_collection(jac_collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert res == 0
|
assert res == 0
|
||||||
|
|
||||||
def test_delete_vector_id_not_exised(self, connect, jac_table, get_simple_index):
|
def test_delete_vector_id_not_exised(self, connect, jac_collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
target: test delete vector, params vector_id not existed
|
target: test delete vector, params vector_id not existed
|
||||||
method: add vector and delete
|
method: add vector and delete
|
||||||
|
@ -427,37 +427,37 @@ class TestDeleteBinary:
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
tmp, vector = gen_binary_vectors(1, dim)
|
tmp, vector = gen_binary_vectors(1, dim)
|
||||||
status, ids = connect.add_vectors(jac_table, vector)
|
status, ids = connect.add_vectors(jac_collection, vector)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.delete_by_id(jac_table, [0])
|
status = connect.delete_by_id(jac_collection, [0])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
search_param = get_search_param(index_type)
|
search_param = get_search_param(index_type)
|
||||||
status, res = connect.search_vectors(jac_table, top_k, vector, params=search_param)
|
status, res = connect.search_vectors(jac_collection, top_k, vector, params=search_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert res[0][0].id == ids[0]
|
assert res[0][0].id == ids[0]
|
||||||
|
|
||||||
def test_delete_vector_table_not_existed(self, connect, jac_table):
|
def test_delete_vector_collection_not_existed(self, connect, jac_collection):
|
||||||
'''
|
'''
|
||||||
target: test delete vector, params table_name not existed
|
target: test delete vector, params collection_name not existed
|
||||||
method: add vector and delete
|
method: add vector and delete
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
tmp, vector = gen_binary_vectors(1, dim)
|
tmp, vector = gen_binary_vectors(1, dim)
|
||||||
status, ids = connect.add_vectors(jac_table, vector)
|
status, ids = connect.add_vectors(jac_collection, vector)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
table_new = gen_unique_str()
|
collection_new = gen_unique_str()
|
||||||
status = connect.delete_by_id(table_new, [0])
|
status = connect.delete_by_id(collection_new, [0])
|
||||||
table_new = gen_unique_str()
|
collection_new = gen_unique_str()
|
||||||
status = connect.delete_by_id(table_new, [0])
|
status = connect.delete_by_id(collection_new, [0])
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
def test_add_vectors_delete_vector(self, connect, jac_table, get_simple_index):
|
def test_add_vectors_delete_vector(self, connect, jac_collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
method: add vectors and delete
|
method: add vectors and delete
|
||||||
expected: status ok, vectors deleted
|
expected: status ok, vectors deleted
|
||||||
|
@ -465,24 +465,24 @@ class TestDeleteBinary:
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||||
status, ids = connect.add_vectors(jac_table, vectors)
|
status, ids = connect.add_vectors(jac_collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
delete_ids = [ids[0], ids[-1]]
|
delete_ids = [ids[0], ids[-1]]
|
||||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||||
status = connect.delete_by_id(jac_table, delete_ids)
|
status = connect.delete_by_id(jac_collection, delete_ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
search_param = get_search_param(index_type)
|
search_param = get_search_param(index_type)
|
||||||
status, res = connect.search_vectors(jac_table, top_k, query_vecs, params=search_param)
|
status, res = connect.search_vectors(jac_collection, top_k, query_vecs, params=search_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
logging.getLogger().info(res)
|
logging.getLogger().info(res)
|
||||||
assert res[0][0].id != ids[0]
|
assert res[0][0].id != ids[0]
|
||||||
assert res[1][0].id == ids[1]
|
assert res[1][0].id == ids[1]
|
||||||
assert res[2][0].id != ids[-1]
|
assert res[2][0].id != ids[-1]
|
||||||
|
|
||||||
def test_add_after_delete_vector(self, connect, jac_table, get_simple_index):
|
def test_add_after_delete_vector(self, connect, jac_collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
method: add vectors and delete, add
|
method: add vectors and delete, add
|
||||||
expected: status ok, vectors added
|
expected: status ok, vectors added
|
||||||
|
@ -490,20 +490,20 @@ class TestDeleteBinary:
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||||
status, ids = connect.add_vectors(jac_table, vectors)
|
status, ids = connect.add_vectors(jac_collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
delete_ids = [ids[0], ids[-1]]
|
delete_ids = [ids[0], ids[-1]]
|
||||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||||
status = connect.delete_by_id(jac_table, delete_ids)
|
status = connect.delete_by_id(jac_collection, delete_ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
status, tmp_ids = connect.add_vectors(jac_table, [vectors[0], vectors[-1]])
|
status, tmp_ids = connect.add_vectors(jac_collection, [vectors[0], vectors[-1]])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
search_param = get_search_param(index_type)
|
search_param = get_search_param(index_type)
|
||||||
status, res = connect.search_vectors(jac_table, top_k, query_vecs, params=search_param)
|
status, res = connect.search_vectors(jac_collection, top_k, query_vecs, params=search_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
logging.getLogger().info(res)
|
logging.getLogger().info(res)
|
||||||
assert res[0][0].id == tmp_ids[0]
|
assert res[0][0].id == tmp_ids[0]
|
||||||
|
@ -526,32 +526,32 @@ class TestDeleteIdsIngalid(object):
|
||||||
yield request.param
|
yield request.param
|
||||||
|
|
||||||
@pytest.mark.level(1)
|
@pytest.mark.level(1)
|
||||||
def test_delete_vector_id_invalid(self, connect, table, gen_invalid_id):
|
def test_delete_vector_id_invalid(self, connect, collection, gen_invalid_id):
|
||||||
invalid_id = gen_invalid_id
|
invalid_id = gen_invalid_id
|
||||||
with pytest.raises(Exception) as e:
|
with pytest.raises(Exception) as e:
|
||||||
status = connect.delete_by_id(table, [invalid_id])
|
status = connect.delete_by_id(collection, [invalid_id])
|
||||||
|
|
||||||
@pytest.mark.level(2)
|
@pytest.mark.level(2)
|
||||||
def test_delete_vector_ids_invalid(self, connect, table, gen_invalid_id):
|
def test_delete_vector_ids_invalid(self, connect, collection, gen_invalid_id):
|
||||||
invalid_id = gen_invalid_id
|
invalid_id = gen_invalid_id
|
||||||
with pytest.raises(Exception) as e:
|
with pytest.raises(Exception) as e:
|
||||||
status = connect.delete_by_id(table, [1, invalid_id])
|
status = connect.delete_by_id(collection, [1, invalid_id])
|
||||||
|
|
||||||
|
|
||||||
class TestTableNameInvalid(object):
|
class TestCollectionNameInvalid(object):
|
||||||
"""
|
"""
|
||||||
Test adding vectors with invalid table names
|
Test adding vectors with invalid collection names
|
||||||
"""
|
"""
|
||||||
@pytest.fixture(
|
@pytest.fixture(
|
||||||
scope="function",
|
scope="function",
|
||||||
params=gen_invalid_table_names()
|
params=gen_invalid_collection_names()
|
||||||
)
|
)
|
||||||
def get_table_name(self, request):
|
def get_collection_name(self, request):
|
||||||
yield request.param
|
yield request.param
|
||||||
|
|
||||||
@pytest.mark.level(2)
|
@pytest.mark.level(2)
|
||||||
def test_delete_vectors_with_invalid_table_name(self, connect, get_table_name):
|
def test_delete_vectors_with_invalid_collection_name(self, connect, get_collection_name):
|
||||||
table_name = get_table_name
|
collection_name = get_collection_name
|
||||||
status = connect.delete_by_id(table_name, [1])
|
status = connect.delete_by_id(collection_name, [1])
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
|
|
|
@ -10,7 +10,7 @@ from utils import *
|
||||||
|
|
||||||
dim = 128
|
dim = 128
|
||||||
index_file_size = 10
|
index_file_size = 10
|
||||||
table_id = "test_flush"
|
collection_id = "test_flush"
|
||||||
DELETE_TIMEOUT = 60
|
DELETE_TIMEOUT = 60
|
||||||
nprobe = 1
|
nprobe = 1
|
||||||
tag = "1970-01-01"
|
tag = "1970-01-01"
|
||||||
|
@ -38,117 +38,117 @@ class TestFlushBase:
|
||||||
pytest.skip("Only support CPU mode")
|
pytest.skip("Only support CPU mode")
|
||||||
return request.param
|
return request.param
|
||||||
|
|
||||||
def test_flush_table_not_existed(self, connect, table):
|
def test_flush_collection_not_existed(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test delete vector, params table_name not existed
|
target: test delete vector, params collection_name not existed
|
||||||
method: add vector and delete
|
method: add vector and delete
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
table_new = gen_unique_str()
|
collection_new = gen_unique_str()
|
||||||
status = connect.flush([table_new])
|
status = connect.flush([collection_new])
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
def test_flush_empty_table(self, connect, table):
|
def test_flush_empty_collection(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
method: flush table with no vectors
|
method: flush collection with no vectors
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
|
|
||||||
def test_add_partition_flush(self, connect, table):
|
def test_add_partition_flush(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
method: add vectors into partition in table, flush serveral times
|
method: add vectors into partition in collection, flush serveral times
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
vectors = gen_vector(nb, dim)
|
vectors = gen_vector(nb, dim)
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
vectors = gen_vectors(nb, dim)
|
vectors = gen_vectors(nb, dim)
|
||||||
ids = [i for i in range(nb)]
|
ids = [i for i in range(nb)]
|
||||||
status, ids = connect.insert(table, vectors, ids)
|
status, ids = connect.insert(collection, vectors, ids)
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
result, res = connect.get_table_row_count(table)
|
result, res = connect.count_collection(collection)
|
||||||
assert res == nb
|
assert res == nb
|
||||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
result, res = connect.get_table_row_count(table)
|
result, res = connect.count_collection(collection)
|
||||||
assert res == 2 * nb
|
assert res == 2 * nb
|
||||||
|
|
||||||
def test_add_partitions_flush(self, connect, table):
|
def test_add_partitions_flush(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
method: add vectors into partitions in table, flush one
|
method: add vectors into partitions in collection, flush one
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
vectors = gen_vectors(nb, dim)
|
vectors = gen_vectors(nb, dim)
|
||||||
tag_new = gen_unique_str()
|
tag_new = gen_unique_str()
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
status = connect.create_partition(table, tag_new)
|
status = connect.create_partition(collection, tag_new)
|
||||||
ids = [i for i in range(nb)]
|
ids = [i for i in range(nb)]
|
||||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag_new)
|
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag_new)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
result, res = connect.get_table_row_count(table)
|
result, res = connect.count_collection(collection)
|
||||||
assert res == 2 * nb
|
assert res == 2 * nb
|
||||||
|
|
||||||
def test_add_tables_flush(self, connect, table):
|
def test_add_collections_flush(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
method: add vectors into tables, flush one
|
method: add vectors into collections, flush one
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
vectors = gen_vectors(nb, dim)
|
vectors = gen_vectors(nb, dim)
|
||||||
table_new = gen_unique_str()
|
collection_new = gen_unique_str()
|
||||||
param = {'table_name': table_new,
|
param = {'collection_name': collection_new,
|
||||||
'dimension': dim,
|
'dimension': dim,
|
||||||
'index_file_size': index_file_size,
|
'index_file_size': index_file_size,
|
||||||
'metric_type': MetricType.L2}
|
'metric_type': MetricType.L2}
|
||||||
status = connect.create_table(param)
|
status = connect.create_collection(param)
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
status = connect.create_partition(table_new, tag)
|
status = connect.create_partition(collection_new, tag)
|
||||||
vectors = gen_vectors(nb, dim)
|
vectors = gen_vectors(nb, dim)
|
||||||
ids = [i for i in range(nb)]
|
ids = [i for i in range(nb)]
|
||||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||||
status, ids = connect.insert(table_new, vectors, ids, partition_tag=tag)
|
status, ids = connect.insert(collection_new, vectors, ids, partition_tag=tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
status = connect.flush([table_new])
|
status = connect.flush([collection_new])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
result, res = connect.get_table_row_count(table)
|
result, res = connect.count_collection(collection)
|
||||||
assert res == nb
|
assert res == nb
|
||||||
result, res = connect.get_table_row_count(table_new)
|
result, res = connect.count_collection(collection_new)
|
||||||
assert res == nb
|
assert res == nb
|
||||||
|
|
||||||
def test_add_flush_multiable_times(self, connect, table):
|
def test_add_flush_multiable_times(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
method: add vectors, flush serveral times
|
method: add vectors, flush serveral times
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
vectors = gen_vectors(nb, dim)
|
vectors = gen_vectors(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||||
status, res = connect.search_vectors(table, top_k, query_records=query_vecs)
|
status, res = connect.search_vectors(collection, top_k, query_records=query_vecs)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
|
|
||||||
def test_add_flush_auto(self, connect, table):
|
def test_add_flush_auto(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
method: add vectors
|
method: add vectors
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
vectors = gen_vectors(nb, dim)
|
vectors = gen_vectors(nb, dim)
|
||||||
ids = [i for i in range(nb)]
|
ids = [i for i in range(nb)]
|
||||||
status, ids = connect.add_vectors(table, vectors, ids)
|
status, ids = connect.add_vectors(collection, vectors, ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
time.sleep(2)
|
time.sleep(2)
|
||||||
status, res = connect.get_table_row_count(table)
|
status, res = connect.count_collection(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert res == nb
|
assert res == nb
|
||||||
|
|
||||||
|
@ -163,7 +163,7 @@ class TestFlushBase:
|
||||||
yield request.param
|
yield request.param
|
||||||
|
|
||||||
# both autoflush / flush
|
# both autoflush / flush
|
||||||
def test_add_flush_same_ids(self, connect, table, same_ids):
|
def test_add_flush_same_ids(self, connect, collection, same_ids):
|
||||||
'''
|
'''
|
||||||
method: add vectors, with same ids, count(same ids) < 15, > 15
|
method: add vectors, with same ids, count(same ids) < 15, > 15
|
||||||
expected: status ok
|
expected: status ok
|
||||||
|
@ -173,79 +173,79 @@ class TestFlushBase:
|
||||||
for i, item in enumerate(ids):
|
for i, item in enumerate(ids):
|
||||||
if item <= same_ids:
|
if item <= same_ids:
|
||||||
ids[i] = 0
|
ids[i] = 0
|
||||||
status, ids = connect.add_vectors(table, vectors, ids)
|
status, ids = connect.add_vectors(collection, vectors, ids)
|
||||||
time.sleep(2)
|
time.sleep(2)
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_table_row_count(table)
|
status, res = connect.count_collection(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert res == nb
|
assert res == nb
|
||||||
|
|
||||||
def test_delete_flush_multiable_times(self, connect, table):
|
def test_delete_flush_multiable_times(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
method: delete vectors, flush serveral times
|
method: delete vectors, flush serveral times
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
vectors = gen_vectors(nb, dim)
|
vectors = gen_vectors(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.delete_by_id(table, [ids[-1]])
|
status = connect.delete_by_id(collection, [ids[-1]])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
query_vecs = [vectors[0], vectors[1], vectors[-1]]
|
||||||
status, res = connect.search_vectors(table, top_k, query_records=query_vecs)
|
status, res = connect.search_vectors(collection, top_k, query_records=query_vecs)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
|
|
||||||
# TODO: CI fail, LOCAL pass
|
# TODO: CI fail, LOCAL pass
|
||||||
def _test_table_count_during_flush(self, connect, args):
|
def _test_collection_count_during_flush(self, connect, args):
|
||||||
'''
|
'''
|
||||||
method: flush table at background, call `get_table_row_count`
|
method: flush collection at background, call `count_collection`
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
table = gen_unique_str()
|
collection = gen_unique_str()
|
||||||
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
||||||
param = {'table_name': table,
|
param = {'collection_name': collection,
|
||||||
'dimension': dim,
|
'dimension': dim,
|
||||||
'index_file_size': index_file_size,
|
'index_file_size': index_file_size,
|
||||||
'metric_type': MetricType.L2}
|
'metric_type': MetricType.L2}
|
||||||
milvus = get_milvus()
|
milvus = get_milvus()
|
||||||
milvus.connect(uri=uri)
|
milvus.connect(uri=uri)
|
||||||
milvus.create_table(param)
|
milvus.create_collection(param)
|
||||||
vectors = gen_vector(nb, dim)
|
vectors = gen_vector(nb, dim)
|
||||||
status, ids = milvus.add_vectors(table, vectors, ids=[i for i in range(nb)])
|
status, ids = milvus.add_vectors(collection, vectors, ids=[i for i in range(nb)])
|
||||||
def flush(table_name):
|
def flush(collection_name):
|
||||||
milvus = get_milvus()
|
milvus = get_milvus()
|
||||||
milvus.connect(uri=uri)
|
milvus.connect(uri=uri)
|
||||||
status = milvus.delete_by_id(table_name, [i for i in range(nb)])
|
status = milvus.delete_by_id(collection_name, [i for i in range(nb)])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = milvus.flush([table_name])
|
status = milvus.flush([collection_name])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
p = Process(target=flush, args=(table, ))
|
p = Process(target=flush, args=(collection, ))
|
||||||
p.start()
|
p.start()
|
||||||
status, res = milvus.get_table_row_count(table)
|
status, res = milvus.count_collection(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
p.join()
|
p.join()
|
||||||
status, res = milvus.get_table_row_count(table)
|
status, res = milvus.count_collection(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
logging.getLogger().info(res)
|
logging.getLogger().info(res)
|
||||||
assert res == 0
|
assert res == 0
|
||||||
|
|
||||||
|
|
||||||
class TestTableNameInvalid(object):
|
class TestCollectionNameInvalid(object):
|
||||||
"""
|
"""
|
||||||
Test adding vectors with invalid table names
|
Test adding vectors with invalid collection names
|
||||||
"""
|
"""
|
||||||
@pytest.fixture(
|
@pytest.fixture(
|
||||||
scope="function",
|
scope="function",
|
||||||
params=gen_invalid_table_names()
|
params=gen_invalid_collection_names()
|
||||||
)
|
)
|
||||||
def get_table_name(self, request):
|
def get_collection_name(self, request):
|
||||||
yield request.param
|
yield request.param
|
||||||
|
|
||||||
@pytest.mark.level(2)
|
@pytest.mark.level(2)
|
||||||
def test_flush_with_invalid_table_name(self, connect, get_table_name):
|
def test_flush_with_invalid_collection_name(self, connect, get_collection_name):
|
||||||
table_name = get_table_name
|
collection_name = get_collection_name
|
||||||
with pytest.raises(Exception) as e:
|
with pytest.raises(Exception) as e:
|
||||||
status, result = connect.flush(table_name)
|
status, result = connect.flush(collection_name)
|
||||||
|
|
|
@ -11,7 +11,7 @@ from utils import *
|
||||||
|
|
||||||
dim = 128
|
dim = 128
|
||||||
index_file_size = 10
|
index_file_size = 10
|
||||||
table_id = "test_get_vector_by_id"
|
collection_id = "test_get_vector_by_id"
|
||||||
DELETE_TIMEOUT = 60
|
DELETE_TIMEOUT = 60
|
||||||
nprobe = 1
|
nprobe = 1
|
||||||
tag = "1970-01-01"
|
tag = "1970-01-01"
|
||||||
|
@ -25,54 +25,54 @@ class TestGetBase:
|
||||||
The following cases are used to test `get_vector_by_id` function
|
The following cases are used to test `get_vector_by_id` function
|
||||||
******************************************************************
|
******************************************************************
|
||||||
"""
|
"""
|
||||||
def test_get_vector_A(self, connect, table):
|
def test_get_vector_A(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test get_vector_by_id
|
target: test get_vector_by_id
|
||||||
method: add vector, and get
|
method: add vector, and get
|
||||||
expected: status ok, vector returned
|
expected: status ok, vector returned
|
||||||
'''
|
'''
|
||||||
vector = gen_single_vector(dim)
|
vector = gen_single_vector(dim)
|
||||||
status, ids = connect.add_vectors(table, vector)
|
status, ids = connect.add_vectors(collection, vector)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_vector_by_id(table, ids[0])
|
status, res = connect.get_vector_by_id(collection, ids[0])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert_equal_vector(res, vector[0])
|
assert_equal_vector(res, vector[0])
|
||||||
|
|
||||||
def test_get_vector_B(self, connect, table):
|
def test_get_vector_B(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test get_vector_by_id
|
target: test get_vector_by_id
|
||||||
method: add vector, and get
|
method: add vector, and get
|
||||||
expected: status ok, vector returned
|
expected: status ok, vector returned
|
||||||
'''
|
'''
|
||||||
vectors = gen_vectors(nb, dim)
|
vectors = gen_vectors(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_vector_by_id(table, ids[0])
|
status, res = connect.get_vector_by_id(collection, ids[0])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert_equal_vector(res, vectors[0])
|
assert_equal_vector(res, vectors[0])
|
||||||
|
|
||||||
def test_get_vector_partition(self, connect, table):
|
def test_get_vector_partition(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test get_vector_by_id
|
target: test get_vector_by_id
|
||||||
method: add vector, and get
|
method: add vector, and get
|
||||||
expected: status ok, vector returned
|
expected: status ok, vector returned
|
||||||
'''
|
'''
|
||||||
vectors = gen_vectors(nb, dim)
|
vectors = gen_vectors(nb, dim)
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
|
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_vector_by_id(table, ids[0])
|
status, res = connect.get_vector_by_id(collection, ids[0])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert_equal_vector(res, vectors[0])
|
assert_equal_vector(res, vectors[0])
|
||||||
|
|
||||||
def test_get_vector_multi_same_ids(self, connect, table):
|
def test_get_vector_multi_same_ids(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test get_vector_by_id
|
target: test get_vector_by_id
|
||||||
method: add vectors, with the same id, get vector by the given id
|
method: add vectors, with the same id, get vector by the given id
|
||||||
|
@ -81,10 +81,10 @@ class TestGetBase:
|
||||||
vectors = gen_vectors(nb, dim)
|
vectors = gen_vectors(nb, dim)
|
||||||
ids = [i for i in range(nb)]
|
ids = [i for i in range(nb)]
|
||||||
ids[1] = 0; ids[-1] = 0
|
ids[1] = 0; ids[-1] = 0
|
||||||
status, ids = connect.add_vectors(table, vectors, ids=ids)
|
status, ids = connect.add_vectors(collection, vectors, ids=ids)
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_vector_by_id(table, 0)
|
status, res = connect.get_vector_by_id(collection, 0)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert_equal_vector(res, vectors[0])
|
assert_equal_vector(res, vectors[0])
|
||||||
|
|
||||||
|
@ -101,75 +101,75 @@ class TestGetBase:
|
||||||
def get_id(self, request):
|
def get_id(self, request):
|
||||||
yield request.param
|
yield request.param
|
||||||
|
|
||||||
def test_get_vector_after_delete(self, connect, table, get_id):
|
def test_get_vector_after_delete(self, connect, collection, get_id):
|
||||||
'''
|
'''
|
||||||
target: test get_vector_by_id
|
target: test get_vector_by_id
|
||||||
method: add vectors, and delete, get vector by the given id
|
method: add vectors, and delete, get vector by the given id
|
||||||
expected: status ok, get one vector
|
expected: status ok, get one vector
|
||||||
'''
|
'''
|
||||||
vectors = gen_vectors(nb, dim)
|
vectors = gen_vectors(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
id = get_id
|
id = get_id
|
||||||
status = connect.delete_by_id(table, [ids[id]])
|
status = connect.delete_by_id(collection, [ids[id]])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_vector_by_id(table, ids[id])
|
status, res = connect.get_vector_by_id(collection, ids[id])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert not res
|
assert not res
|
||||||
|
|
||||||
def test_get_vector_after_delete_with_partition(self, connect, table, get_id):
|
def test_get_vector_after_delete_with_partition(self, connect, collection, get_id):
|
||||||
'''
|
'''
|
||||||
target: test get_vector_by_id
|
target: test get_vector_by_id
|
||||||
method: add vectors into partition, and delete, get vector by the given id
|
method: add vectors into partition, and delete, get vector by the given id
|
||||||
expected: status ok, get one vector
|
expected: status ok, get one vector
|
||||||
'''
|
'''
|
||||||
vectors = gen_vectors(nb, dim)
|
vectors = gen_vectors(nb, dim)
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
status, ids = connect.insert(table, vectors, partition_tag=tag)
|
status, ids = connect.insert(collection, vectors, partition_tag=tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
id = get_id
|
id = get_id
|
||||||
status = connect.delete_by_id(table, [ids[id]])
|
status = connect.delete_by_id(collection, [ids[id]])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_vector_by_id(table, ids[id])
|
status, res = connect.get_vector_by_id(collection, ids[id])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert not res
|
assert not res
|
||||||
|
|
||||||
def test_get_vector_id_not_exised(self, connect, table):
|
def test_get_vector_id_not_exised(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test get vector, params vector_id not existed
|
target: test get vector, params vector_id not existed
|
||||||
method: add vector and get
|
method: add vector and get
|
||||||
expected: status ok, empty result
|
expected: status ok, empty result
|
||||||
'''
|
'''
|
||||||
vector = gen_single_vector(dim)
|
vector = gen_single_vector(dim)
|
||||||
status, ids = connect.add_vectors(table, vector)
|
status, ids = connect.add_vectors(collection, vector)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_vector_by_id(table, 1)
|
status, res = connect.get_vector_by_id(collection, 1)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert not res
|
assert not res
|
||||||
|
|
||||||
def test_get_vector_table_not_existed(self, connect, table):
|
def test_get_vector_collection_not_existed(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test get vector, params table_name not existed
|
target: test get vector, params collection_name not existed
|
||||||
method: add vector and get
|
method: add vector and get
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
vector = gen_single_vector(dim)
|
vector = gen_single_vector(dim)
|
||||||
status, ids = connect.add_vectors(table, vector)
|
status, ids = connect.add_vectors(collection, vector)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
table_new = gen_unique_str()
|
collection_new = gen_unique_str()
|
||||||
status, res = connect.get_vector_by_id(table_new, 1)
|
status, res = connect.get_vector_by_id(collection_new, 1)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
|
|
||||||
|
@ -205,7 +205,7 @@ class TestGetIndexedVectors:
|
||||||
def get_id(self, request):
|
def get_id(self, request):
|
||||||
yield request.param
|
yield request.param
|
||||||
|
|
||||||
def test_get_vectors_after_index_created(self, connect, table, get_simple_index, get_id):
|
def test_get_vectors_after_index_created(self, connect, collection, get_simple_index, get_id):
|
||||||
'''
|
'''
|
||||||
target: test get vector after index created
|
target: test get vector after index created
|
||||||
method: add vector, create index and get vector
|
method: add vector, create index and get vector
|
||||||
|
@ -214,20 +214,20 @@ class TestGetIndexedVectors:
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
vectors = gen_vector(nb, dim)
|
vectors = gen_vector(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.create_index(table, index_type, index_param)
|
status = connect.create_index(collection, index_type, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
id = get_id
|
id = get_id
|
||||||
status, res = connect.get_vector_by_id(table, ids[id])
|
status, res = connect.get_vector_by_id(collection, ids[id])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
logging.getLogger().info(res)
|
logging.getLogger().info(res)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert_equal_vector(res, vectors[id])
|
assert_equal_vector(res, vectors[id])
|
||||||
|
|
||||||
def test_get_vector_after_delete(self, connect, table, get_simple_index, get_id):
|
def test_get_vector_after_delete(self, connect, collection, get_simple_index, get_id):
|
||||||
'''
|
'''
|
||||||
target: test get_vector_by_id
|
target: test get_vector_by_id
|
||||||
method: add vectors, and delete, get vector by the given id
|
method: add vectors, and delete, get vector by the given id
|
||||||
|
@ -236,22 +236,22 @@ class TestGetIndexedVectors:
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
vectors = gen_vectors(nb, dim)
|
vectors = gen_vectors(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.create_index(table, index_type, index_param)
|
status = connect.create_index(collection, index_type, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
id = get_id
|
id = get_id
|
||||||
status = connect.delete_by_id(table, [ids[id]])
|
status = connect.delete_by_id(collection, [ids[id]])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_vector_by_id(table, ids[id])
|
status, res = connect.get_vector_by_id(collection, ids[id])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert not res
|
assert not res
|
||||||
|
|
||||||
def test_get_vector_partition(self, connect, table, get_simple_index, get_id):
|
def test_get_vector_partition(self, connect, collection, get_simple_index, get_id):
|
||||||
'''
|
'''
|
||||||
target: test get_vector_by_id
|
target: test get_vector_by_id
|
||||||
method: add vector, and get
|
method: add vector, and get
|
||||||
|
@ -260,16 +260,16 @@ class TestGetIndexedVectors:
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
vectors = gen_vectors(nb, dim)
|
vectors = gen_vectors(nb, dim)
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
ids = [i for i in range(nb)]
|
ids = [i for i in range(nb)]
|
||||||
status, ids = connect.add_vectors(table, vectors, ids, partition_tag=tag)
|
status, ids = connect.add_vectors(collection, vectors, ids, partition_tag=tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.create_index(table, index_type, index_param)
|
status = connect.create_index(collection, index_type, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
id = get_id
|
id = get_id
|
||||||
status, res = connect.get_vector_by_id(table, ids[id])
|
status, res = connect.get_vector_by_id(collection, ids[id])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert_equal_vector(res, vectors[id])
|
assert_equal_vector(res, vectors[id])
|
||||||
|
|
||||||
|
@ -280,37 +280,37 @@ class TestGetBinary:
|
||||||
The following cases are used to test `get_vector_by_id` function
|
The following cases are used to test `get_vector_by_id` function
|
||||||
******************************************************************
|
******************************************************************
|
||||||
"""
|
"""
|
||||||
def test_get_vector_A(self, connect, jac_table):
|
def test_get_vector_A(self, connect, jac_collection):
|
||||||
'''
|
'''
|
||||||
target: test get_vector_by_id
|
target: test get_vector_by_id
|
||||||
method: add vector, and get
|
method: add vector, and get
|
||||||
expected: status ok, vector returned
|
expected: status ok, vector returned
|
||||||
'''
|
'''
|
||||||
tmp, vector = gen_binary_vectors(1, dim)
|
tmp, vector = gen_binary_vectors(1, dim)
|
||||||
status, ids = connect.add_vectors(jac_table, vector)
|
status, ids = connect.add_vectors(jac_collection, vector)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_vector_by_id(jac_table, ids[0])
|
status, res = connect.get_vector_by_id(jac_collection, ids[0])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert res == vector[0]
|
assert res == vector[0]
|
||||||
|
|
||||||
def test_get_vector_B(self, connect, jac_table):
|
def test_get_vector_B(self, connect, jac_collection):
|
||||||
'''
|
'''
|
||||||
target: test get_vector_by_id
|
target: test get_vector_by_id
|
||||||
method: add vector, and get
|
method: add vector, and get
|
||||||
expected: status ok, vector returned
|
expected: status ok, vector returned
|
||||||
'''
|
'''
|
||||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||||
status, ids = connect.add_vectors(jac_table, vectors)
|
status, ids = connect.add_vectors(jac_collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_vector_by_id(jac_table, ids[0])
|
status, res = connect.get_vector_by_id(jac_collection, ids[0])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert res == vectors[0]
|
assert res == vectors[0]
|
||||||
|
|
||||||
def test_get_vector_multi_same_ids(self, connect, jac_table):
|
def test_get_vector_multi_same_ids(self, connect, jac_collection):
|
||||||
'''
|
'''
|
||||||
target: test get_vector_by_id
|
target: test get_vector_by_id
|
||||||
method: add vectors, with the same id, get vector by the given id
|
method: add vectors, with the same id, get vector by the given id
|
||||||
|
@ -319,56 +319,56 @@ class TestGetBinary:
|
||||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||||
ids = [i for i in range(nb)]
|
ids = [i for i in range(nb)]
|
||||||
ids[0] = 0; ids[-1] = 0
|
ids[0] = 0; ids[-1] = 0
|
||||||
status, ids = connect.add_vectors(jac_table, vectors, ids=ids)
|
status, ids = connect.add_vectors(jac_collection, vectors, ids=ids)
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_vector_by_id(jac_table, 0)
|
status, res = connect.get_vector_by_id(jac_collection, 0)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert res == vectors[0]
|
assert res == vectors[0]
|
||||||
|
|
||||||
def test_get_vector_id_not_exised(self, connect, jac_table):
|
def test_get_vector_id_not_exised(self, connect, jac_collection):
|
||||||
'''
|
'''
|
||||||
target: test get vector, params vector_id not existed
|
target: test get vector, params vector_id not existed
|
||||||
method: add vector and get
|
method: add vector and get
|
||||||
expected: status ok, empty result
|
expected: status ok, empty result
|
||||||
'''
|
'''
|
||||||
tmp, vector = gen_binary_vectors(1, dim)
|
tmp, vector = gen_binary_vectors(1, dim)
|
||||||
status, ids = connect.add_vectors(jac_table, vector)
|
status, ids = connect.add_vectors(jac_collection, vector)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_vector_by_id(jac_table, 1)
|
status, res = connect.get_vector_by_id(jac_collection, 1)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert not res
|
assert not res
|
||||||
|
|
||||||
def test_get_vector_table_not_existed(self, connect, jac_table):
|
def test_get_vector_collection_not_existed(self, connect, jac_collection):
|
||||||
'''
|
'''
|
||||||
target: test get vector, params table_name not existed
|
target: test get vector, params collection_name not existed
|
||||||
method: add vector and get
|
method: add vector and get
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
tmp, vector = gen_binary_vectors(1, dim)
|
tmp, vector = gen_binary_vectors(1, dim)
|
||||||
status, ids = connect.add_vectors(jac_table, vector)
|
status, ids = connect.add_vectors(jac_collection, vector)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
table_new = gen_unique_str()
|
collection_new = gen_unique_str()
|
||||||
status, res = connect.get_vector_by_id(table_new, 1)
|
status, res = connect.get_vector_by_id(collection_new, 1)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
def test_get_vector_partition(self, connect, jac_table):
|
def test_get_vector_partition(self, connect, jac_collection):
|
||||||
'''
|
'''
|
||||||
target: test get_vector_by_id
|
target: test get_vector_by_id
|
||||||
method: add vector, and get
|
method: add vector, and get
|
||||||
expected: status ok, vector returned
|
expected: status ok, vector returned
|
||||||
'''
|
'''
|
||||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
tmp, vectors = gen_binary_vectors(nb, dim)
|
||||||
status = connect.create_partition(jac_table, tag)
|
status = connect.create_partition(jac_collection, tag)
|
||||||
status, ids = connect.add_vectors(jac_table, vectors, partition_tag=tag)
|
status, ids = connect.add_vectors(jac_collection, vectors, partition_tag=tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_vector_by_id(jac_table, ids[0])
|
status, res = connect.get_vector_by_id(jac_collection, ids[0])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert res == vectors[0]
|
assert res == vectors[0]
|
||||||
|
|
||||||
|
@ -387,26 +387,26 @@ class TestGetVectorIdIngalid(object):
|
||||||
yield request.param
|
yield request.param
|
||||||
|
|
||||||
@pytest.mark.level(2)
|
@pytest.mark.level(2)
|
||||||
def test_get_vector_id_invalid(self, connect, table, gen_invalid_id):
|
def test_get_vector_id_invalid(self, connect, collection, gen_invalid_id):
|
||||||
invalid_id = gen_invalid_id
|
invalid_id = gen_invalid_id
|
||||||
with pytest.raises(Exception) as e:
|
with pytest.raises(Exception) as e:
|
||||||
status = connect.get_vector_by_id(table, invalid_id)
|
status = connect.get_vector_by_id(collection, invalid_id)
|
||||||
|
|
||||||
|
|
||||||
class TestTableNameInvalid(object):
|
class TestCollectionNameInvalid(object):
|
||||||
"""
|
"""
|
||||||
Test adding vectors with invalid table names
|
Test adding vectors with invalid collection names
|
||||||
"""
|
"""
|
||||||
@pytest.fixture(
|
@pytest.fixture(
|
||||||
scope="function",
|
scope="function",
|
||||||
params=gen_invalid_table_names()
|
params=gen_invalid_collection_names()
|
||||||
)
|
)
|
||||||
def get_table_name(self, request):
|
def get_collection_name(self, request):
|
||||||
yield request.param
|
yield request.param
|
||||||
|
|
||||||
@pytest.mark.level(2)
|
@pytest.mark.level(2)
|
||||||
def test_get_vectors_with_invalid_table_name(self, connect, get_table_name):
|
def test_get_vectors_with_invalid_collection_name(self, connect, get_collection_name):
|
||||||
table_name = get_table_name
|
collection_name = get_collection_name
|
||||||
vectors = gen_vectors(1, dim)
|
vectors = gen_vectors(1, dim)
|
||||||
status, result = connect.get_vector_by_id(table_name, 1)
|
status, result = connect.get_vector_by_id(collection_name, 1)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
|
@ -19,13 +19,13 @@ nb = 6000
|
||||||
|
|
||||||
|
|
||||||
class TestGetVectorIdsBase:
|
class TestGetVectorIdsBase:
|
||||||
def get_valid_segment_name(self, connect, table):
|
def get_valid_segment_name(self, connect, collection):
|
||||||
vectors = gen_vector(nb, dim)
|
vectors = gen_vector(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(table)
|
status, info = connect.collection_info(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
logging.getLogger().info(info.partitions_stat[0].segments_stat[0].segment_name)
|
logging.getLogger().info(info.partitions_stat[0].segments_stat[0].segment_name)
|
||||||
return info.partitions_stat[0].segments_stat[0].segment_name
|
return info.partitions_stat[0].segments_stat[0].segment_name
|
||||||
|
@ -36,88 +36,88 @@ class TestGetVectorIdsBase:
|
||||||
******************************************************************
|
******************************************************************
|
||||||
"""
|
"""
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_table_name_None(self, connect, table):
|
def test_get_vector_ids_collection_name_None(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: get vector ids where table name is None
|
target: get vector ids where collection name is None
|
||||||
method: call get_vector_ids with the table_name: None
|
method: call get_vector_ids with the collection_name: None
|
||||||
expected: exception raised
|
expected: exception raised
|
||||||
'''
|
'''
|
||||||
table_name = None
|
collection_name = None
|
||||||
segment_name = self.get_valid_segment_name(connect, table)
|
segment_name = self.get_valid_segment_name(connect, collection)
|
||||||
with pytest.raises(Exception) as e:
|
with pytest.raises(Exception) as e:
|
||||||
status, vector_ids = connect.get_vector_ids(table_name, segment_name)
|
status, vector_ids = connect.get_vector_ids(collection_name, segment_name)
|
||||||
|
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_table_name_not_existed(self, connect, table):
|
def test_get_vector_ids_collection_name_not_existed(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: get vector ids where table name does not exist
|
target: get vector ids where collection name does not exist
|
||||||
method: call get_vector_ids with a random table_name, which is not in db
|
method: call get_vector_ids with a random collection_name, which is not in db
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
table_name = gen_unique_str("not_existed_table")
|
collection_name = gen_unique_str("not_existed_collection")
|
||||||
segment_name = self.get_valid_segment_name(connect, table)
|
segment_name = self.get_valid_segment_name(connect, collection)
|
||||||
status, vector_ids = connect.get_vector_ids(table_name, segment_name)
|
status, vector_ids = connect.get_vector_ids(collection_name, segment_name)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
@pytest.fixture(
|
@pytest.fixture(
|
||||||
scope="function",
|
scope="function",
|
||||||
params=gen_invalid_table_names()
|
params=gen_invalid_collection_names()
|
||||||
)
|
)
|
||||||
def get_table_name(self, request):
|
def get_collection_name(self, request):
|
||||||
yield request.param
|
yield request.param
|
||||||
|
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_table_name_invalid(self, connect, table, get_table_name):
|
def test_get_vector_ids_collection_name_invalid(self, connect, collection, get_collection_name):
|
||||||
'''
|
'''
|
||||||
target: get vector ids where table name is invalid
|
target: get vector ids where collection name is invalid
|
||||||
method: call get_vector_ids with invalid table_name
|
method: call get_vector_ids with invalid collection_name
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
table_name = get_table_name
|
collection_name = get_collection_name
|
||||||
segment_name = self.get_valid_segment_name(connect, table)
|
segment_name = self.get_valid_segment_name(connect, collection)
|
||||||
status, vector_ids = connect.get_vector_ids(table_name, segment_name)
|
status, vector_ids = connect.get_vector_ids(collection_name, segment_name)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_segment_name_None(self, connect, table):
|
def test_get_vector_ids_segment_name_None(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: get vector ids where segment name is None
|
target: get vector ids where segment name is None
|
||||||
method: call get_vector_ids with the segment_name: None
|
method: call get_vector_ids with the segment_name: None
|
||||||
expected: exception raised
|
expected: exception raised
|
||||||
'''
|
'''
|
||||||
valid_segment_name = self.get_valid_segment_name(connect, table)
|
valid_segment_name = self.get_valid_segment_name(connect, collection)
|
||||||
segment = None
|
segment = None
|
||||||
with pytest.raises(Exception) as e:
|
with pytest.raises(Exception) as e:
|
||||||
status, vector_ids = connect.get_vector_ids(table, segment)
|
status, vector_ids = connect.get_vector_ids(collection, segment)
|
||||||
|
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_segment_name_not_existed(self, connect, table):
|
def test_get_vector_ids_segment_name_not_existed(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: get vector ids where segment name does not exist
|
target: get vector ids where segment name does not exist
|
||||||
method: call get_vector_ids with a random segment name
|
method: call get_vector_ids with a random segment name
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
valid_segment_name = self.get_valid_segment_name(connect, table)
|
valid_segment_name = self.get_valid_segment_name(connect, collection)
|
||||||
segment = gen_unique_str("not_existed_segment")
|
segment = gen_unique_str("not_existed_segment")
|
||||||
status, vector_ids = connect.get_vector_ids(table, segment)
|
status, vector_ids = connect.get_vector_ids(collection, segment)
|
||||||
logging.getLogger().info(vector_ids)
|
logging.getLogger().info(vector_ids)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_without_index_A(self, connect, table):
|
def test_get_vector_ids_without_index_A(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: get vector ids when there is no index
|
target: get vector ids when there is no index
|
||||||
method: call get_vector_ids and check if the segment contains vectors
|
method: call get_vector_ids and check if the segment contains vectors
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
vectors = gen_vector(10, dim)
|
vectors = gen_vector(10, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(table)
|
status, info = connect.collection_info(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[0].segments_stat[0].segment_name)
|
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||||
# vector_ids should match ids
|
# vector_ids should match ids
|
||||||
assert len(vector_ids) == 10
|
assert len(vector_ids) == 10
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
|
@ -125,23 +125,23 @@ class TestGetVectorIdsBase:
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_without_index_B(self, connect, table):
|
def test_get_vector_ids_without_index_B(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: get vector ids when there is no index but with partition
|
target: get vector ids when there is no index but with partition
|
||||||
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
vectors = gen_vector(10, dim)
|
vectors = gen_vector(10, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
|
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(table)
|
status, info = connect.collection_info(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert info.partitions_stat[1].tag == tag
|
assert info.partitions_stat[1].tag == tag
|
||||||
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[1].segments_stat[0].segment_name)
|
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||||
# vector_ids should match ids
|
# vector_ids should match ids
|
||||||
assert len(vector_ids) == 10
|
assert len(vector_ids) == 10
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
|
@ -160,7 +160,7 @@ class TestGetVectorIdsBase:
|
||||||
return request.param
|
return request.param
|
||||||
|
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_with_index_A(self, connect, table, get_simple_index):
|
def test_get_vector_ids_with_index_A(self, connect, collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
target: get vector ids when there is index
|
target: get vector ids when there is index
|
||||||
method: call get_vector_ids and check if the segment contains vectors
|
method: call get_vector_ids and check if the segment contains vectors
|
||||||
|
@ -168,65 +168,65 @@ class TestGetVectorIdsBase:
|
||||||
'''
|
'''
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
status = connect.create_index(table, index_type, index_param)
|
status = connect.create_index(collection, index_type, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
vectors = gen_vector(10, dim)
|
vectors = gen_vector(10, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(table)
|
status, info = connect.collection_info(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[0].segments_stat[0].segment_name)
|
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||||
# vector_ids should match ids
|
# vector_ids should match ids
|
||||||
assert len(vector_ids) == 10
|
assert len(vector_ids) == 10
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
assert vector_ids[i] == ids[i]
|
assert vector_ids[i] == ids[i]
|
||||||
|
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_with_index_B(self, connect, table, get_simple_index):
|
def test_get_vector_ids_with_index_B(self, connect, collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
target: get vector ids when there is index and with partition
|
target: get vector ids when there is index and with partition
|
||||||
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
status = connect.create_index(table, index_type, index_param)
|
status = connect.create_index(collection, index_type, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
vectors = gen_vector(10, dim)
|
vectors = gen_vector(10, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
|
status, ids = connect.add_vectors(collection, vectors, partition_tag=tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(table)
|
status, info = connect.collection_info(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert info.partitions_stat[1].tag == tag
|
assert info.partitions_stat[1].tag == tag
|
||||||
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[1].segments_stat[0].segment_name)
|
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||||
# vector_ids should match ids
|
# vector_ids should match ids
|
||||||
assert len(vector_ids) == 10
|
assert len(vector_ids) == 10
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
assert vector_ids[i] == ids[i]
|
assert vector_ids[i] == ids[i]
|
||||||
|
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_after_delete_vectors(self, connect, table):
|
def test_get_vector_ids_after_delete_vectors(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: get vector ids after vectors are deleted
|
target: get vector ids after vectors are deleted
|
||||||
method: add vectors and delete a few, call get_vector_ids
|
method: add vectors and delete a few, call get_vector_ids
|
||||||
expected: status ok, vector_ids decreased after vectors deleted
|
expected: status ok, vector_ids decreased after vectors deleted
|
||||||
'''
|
'''
|
||||||
vectors = gen_vector(2, dim)
|
vectors = gen_vector(2, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
delete_ids = [ids[0]]
|
delete_ids = [ids[0]]
|
||||||
status = connect.delete_by_id(table, delete_ids)
|
status = connect.delete_by_id(collection, delete_ids)
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(table)
|
status, info = connect.collection_info(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, vector_ids = connect.get_vector_ids(table, info.partitions_stat[0].segments_stat[0].segment_name)
|
status, vector_ids = connect.get_vector_ids(collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||||
assert len(vector_ids) == 1
|
assert len(vector_ids) == 1
|
||||||
assert vector_ids[0] == ids[1]
|
assert vector_ids[0] == ids[1]
|
||||||
|
|
||||||
|
@ -238,20 +238,20 @@ class TestGetVectorIdsIP:
|
||||||
******************************************************************
|
******************************************************************
|
||||||
"""
|
"""
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_without_index_A(self, connect, ip_table):
|
def test_get_vector_ids_without_index_A(self, connect, ip_collection):
|
||||||
'''
|
'''
|
||||||
target: get vector ids when there is no index
|
target: get vector ids when there is no index
|
||||||
method: call get_vector_ids and check if the segment contains vectors
|
method: call get_vector_ids and check if the segment contains vectors
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
vectors = gen_vector(10, dim)
|
vectors = gen_vector(10, dim)
|
||||||
status, ids = connect.add_vectors(ip_table, vectors)
|
status, ids = connect.add_vectors(ip_collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([ip_table])
|
status = connect.flush([ip_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(ip_table)
|
status, info = connect.collection_info(ip_collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[0].segments_stat[0].segment_name)
|
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||||
# vector_ids should match ids
|
# vector_ids should match ids
|
||||||
assert len(vector_ids) == 10
|
assert len(vector_ids) == 10
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
|
@ -259,23 +259,23 @@ class TestGetVectorIdsIP:
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_without_index_B(self, connect, ip_table):
|
def test_get_vector_ids_without_index_B(self, connect, ip_collection):
|
||||||
'''
|
'''
|
||||||
target: get vector ids when there is no index but with partition
|
target: get vector ids when there is no index but with partition
|
||||||
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(ip_table, tag)
|
status = connect.create_partition(ip_collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
vectors = gen_vector(10, dim)
|
vectors = gen_vector(10, dim)
|
||||||
status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag)
|
status, ids = connect.add_vectors(ip_collection, vectors, partition_tag=tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([ip_table])
|
status = connect.flush([ip_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(ip_table)
|
status, info = connect.collection_info(ip_collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert info.partitions_stat[1].tag == tag
|
assert info.partitions_stat[1].tag == tag
|
||||||
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[1].segments_stat[0].segment_name)
|
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||||
# vector_ids should match ids
|
# vector_ids should match ids
|
||||||
assert len(vector_ids) == 10
|
assert len(vector_ids) == 10
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
|
@ -294,7 +294,7 @@ class TestGetVectorIdsIP:
|
||||||
return request.param
|
return request.param
|
||||||
|
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_with_index_A(self, connect, ip_table, get_simple_index):
|
def test_get_vector_ids_with_index_A(self, connect, ip_collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
target: get vector ids when there is index
|
target: get vector ids when there is index
|
||||||
method: call get_vector_ids and check if the segment contains vectors
|
method: call get_vector_ids and check if the segment contains vectors
|
||||||
|
@ -302,65 +302,65 @@ class TestGetVectorIdsIP:
|
||||||
'''
|
'''
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
status = connect.create_index(ip_table, index_type, index_param)
|
status = connect.create_index(ip_collection, index_type, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
vectors = gen_vector(10, dim)
|
vectors = gen_vector(10, dim)
|
||||||
status, ids = connect.add_vectors(ip_table, vectors)
|
status, ids = connect.add_vectors(ip_collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([ip_table])
|
status = connect.flush([ip_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(ip_table)
|
status, info = connect.collection_info(ip_collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[0].segments_stat[0].segment_name)
|
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||||
# vector_ids should match ids
|
# vector_ids should match ids
|
||||||
assert len(vector_ids) == 10
|
assert len(vector_ids) == 10
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
assert vector_ids[i] == ids[i]
|
assert vector_ids[i] == ids[i]
|
||||||
|
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_with_index_B(self, connect, ip_table, get_simple_index):
|
def test_get_vector_ids_with_index_B(self, connect, ip_collection, get_simple_index):
|
||||||
'''
|
'''
|
||||||
target: get vector ids when there is index and with partition
|
target: get vector ids when there is index and with partition
|
||||||
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(ip_table, tag)
|
status = connect.create_partition(ip_collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
index_param = get_simple_index["index_param"]
|
index_param = get_simple_index["index_param"]
|
||||||
index_type = get_simple_index["index_type"]
|
index_type = get_simple_index["index_type"]
|
||||||
status = connect.create_index(ip_table, index_type, index_param)
|
status = connect.create_index(ip_collection, index_type, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
vectors = gen_vector(10, dim)
|
vectors = gen_vector(10, dim)
|
||||||
status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag)
|
status, ids = connect.add_vectors(ip_collection, vectors, partition_tag=tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([ip_table])
|
status = connect.flush([ip_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(ip_table)
|
status, info = connect.collection_info(ip_collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert info.partitions_stat[1].tag == tag
|
assert info.partitions_stat[1].tag == tag
|
||||||
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[1].segments_stat[0].segment_name)
|
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||||
# vector_ids should match ids
|
# vector_ids should match ids
|
||||||
assert len(vector_ids) == 10
|
assert len(vector_ids) == 10
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
assert vector_ids[i] == ids[i]
|
assert vector_ids[i] == ids[i]
|
||||||
|
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_after_delete_vectors(self, connect, ip_table):
|
def test_get_vector_ids_after_delete_vectors(self, connect, ip_collection):
|
||||||
'''
|
'''
|
||||||
target: get vector ids after vectors are deleted
|
target: get vector ids after vectors are deleted
|
||||||
method: add vectors and delete a few, call get_vector_ids
|
method: add vectors and delete a few, call get_vector_ids
|
||||||
expected: status ok, vector_ids decreased after vectors deleted
|
expected: status ok, vector_ids decreased after vectors deleted
|
||||||
'''
|
'''
|
||||||
vectors = gen_vector(2, dim)
|
vectors = gen_vector(2, dim)
|
||||||
status, ids = connect.add_vectors(ip_table, vectors)
|
status, ids = connect.add_vectors(ip_collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
delete_ids = [ids[0]]
|
delete_ids = [ids[0]]
|
||||||
status = connect.delete_by_id(ip_table, delete_ids)
|
status = connect.delete_by_id(ip_collection, delete_ids)
|
||||||
status = connect.flush([ip_table])
|
status = connect.flush([ip_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(ip_table)
|
status, info = connect.collection_info(ip_collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, vector_ids = connect.get_vector_ids(ip_table, info.partitions_stat[0].segments_stat[0].segment_name)
|
status, vector_ids = connect.get_vector_ids(ip_collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||||
assert len(vector_ids) == 1
|
assert len(vector_ids) == 1
|
||||||
assert vector_ids[0] == ids[1]
|
assert vector_ids[0] == ids[1]
|
||||||
|
|
||||||
|
@ -372,20 +372,20 @@ class TestGetVectorIdsJAC:
|
||||||
******************************************************************
|
******************************************************************
|
||||||
"""
|
"""
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_without_index_A(self, connect, jac_table):
|
def test_get_vector_ids_without_index_A(self, connect, jac_collection):
|
||||||
'''
|
'''
|
||||||
target: get vector ids when there is no index
|
target: get vector ids when there is no index
|
||||||
method: call get_vector_ids and check if the segment contains vectors
|
method: call get_vector_ids and check if the segment contains vectors
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
tmp, vectors = gen_binary_vectors(10, dim)
|
tmp, vectors = gen_binary_vectors(10, dim)
|
||||||
status, ids = connect.add_vectors(jac_table, vectors)
|
status, ids = connect.add_vectors(jac_collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(jac_table)
|
status, info = connect.collection_info(jac_collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[0].segments_stat[0].segment_name)
|
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||||
# vector_ids should match ids
|
# vector_ids should match ids
|
||||||
assert len(vector_ids) == 10
|
assert len(vector_ids) == 10
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
|
@ -393,23 +393,23 @@ class TestGetVectorIdsJAC:
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_without_index_B(self, connect, jac_table):
|
def test_get_vector_ids_without_index_B(self, connect, jac_collection):
|
||||||
'''
|
'''
|
||||||
target: get vector ids when there is no index but with partition
|
target: get vector ids when there is no index but with partition
|
||||||
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(jac_table, tag)
|
status = connect.create_partition(jac_collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
tmp, vectors = gen_binary_vectors(10, dim)
|
tmp, vectors = gen_binary_vectors(10, dim)
|
||||||
status, ids = connect.add_vectors(jac_table, vectors, partition_tag=tag)
|
status, ids = connect.add_vectors(jac_collection, vectors, partition_tag=tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(jac_table)
|
status, info = connect.collection_info(jac_collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert info.partitions_stat[1].tag == tag
|
assert info.partitions_stat[1].tag == tag
|
||||||
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[1].segments_stat[0].segment_name)
|
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||||
# vector_ids should match ids
|
# vector_ids should match ids
|
||||||
assert len(vector_ids) == 10
|
assert len(vector_ids) == 10
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
|
@ -427,7 +427,7 @@ class TestGetVectorIdsJAC:
|
||||||
pytest.skip("Skip index Temporary")
|
pytest.skip("Skip index Temporary")
|
||||||
|
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_with_index_A(self, connect, jac_table, get_jaccard_index):
|
def test_get_vector_ids_with_index_A(self, connect, jac_collection, get_jaccard_index):
|
||||||
'''
|
'''
|
||||||
target: get vector ids when there is index
|
target: get vector ids when there is index
|
||||||
method: call get_vector_ids and check if the segment contains vectors
|
method: call get_vector_ids and check if the segment contains vectors
|
||||||
|
@ -435,64 +435,64 @@ class TestGetVectorIdsJAC:
|
||||||
'''
|
'''
|
||||||
index_param = get_jaccard_index["index_param"]
|
index_param = get_jaccard_index["index_param"]
|
||||||
index_type = get_jaccard_index["index_type"]
|
index_type = get_jaccard_index["index_type"]
|
||||||
status = connect.create_index(jac_table, index_type, index_param)
|
status = connect.create_index(jac_collection, index_type, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
tmp, vectors = gen_binary_vectors(10, dim)
|
tmp, vectors = gen_binary_vectors(10, dim)
|
||||||
status, ids = connect.add_vectors(jac_table, vectors)
|
status, ids = connect.add_vectors(jac_collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(jac_table)
|
status, info = connect.collection_info(jac_collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[0].segments_stat[0].segment_name)
|
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||||
# vector_ids should match ids
|
# vector_ids should match ids
|
||||||
assert len(vector_ids) == 10
|
assert len(vector_ids) == 10
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
assert vector_ids[i] == ids[i]
|
assert vector_ids[i] == ids[i]
|
||||||
|
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_with_index_B(self, connect, jac_table, get_jaccard_index):
|
def test_get_vector_ids_with_index_B(self, connect, jac_collection, get_jaccard_index):
|
||||||
'''
|
'''
|
||||||
target: get vector ids when there is index and with partition
|
target: get vector ids when there is index and with partition
|
||||||
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
method: create partition, add vectors to it and call get_vector_ids, check if the segment contains vectors
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(jac_table, tag)
|
status = connect.create_partition(jac_collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
index_param = get_jaccard_index["index_param"]
|
index_param = get_jaccard_index["index_param"]
|
||||||
index_type = get_jaccard_index["index_type"]
|
index_type = get_jaccard_index["index_type"]
|
||||||
status = connect.create_index(jac_table, index_type, index_param)
|
status = connect.create_index(jac_collection, index_type, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
tmp, vectors = gen_binary_vectors(10, dim)
|
tmp, vectors = gen_binary_vectors(10, dim)
|
||||||
status, ids = connect.add_vectors(jac_table, vectors, partition_tag=tag)
|
status, ids = connect.add_vectors(jac_collection, vectors, partition_tag=tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(jac_table)
|
status, info = connect.collection_info(jac_collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert info.partitions_stat[1].tag == tag
|
assert info.partitions_stat[1].tag == tag
|
||||||
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[1].segments_stat[0].segment_name)
|
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[1].segments_stat[0].segment_name)
|
||||||
# vector_ids should match ids
|
# vector_ids should match ids
|
||||||
assert len(vector_ids) == 10
|
assert len(vector_ids) == 10
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
assert vector_ids[i] == ids[i]
|
assert vector_ids[i] == ids[i]
|
||||||
|
|
||||||
@pytest.mark.timeout(GET_TIMEOUT)
|
@pytest.mark.timeout(GET_TIMEOUT)
|
||||||
def test_get_vector_ids_after_delete_vectors(self, connect, jac_table):
|
def test_get_vector_ids_after_delete_vectors(self, connect, jac_collection):
|
||||||
'''
|
'''
|
||||||
target: get vector ids after vectors are deleted
|
target: get vector ids after vectors are deleted
|
||||||
method: add vectors and delete a few, call get_vector_ids
|
method: add vectors and delete a few, call get_vector_ids
|
||||||
expected: status ok, vector_ids decreased after vectors deleted
|
expected: status ok, vector_ids decreased after vectors deleted
|
||||||
'''
|
'''
|
||||||
tmp, vectors = gen_binary_vectors(2, dim)
|
tmp, vectors = gen_binary_vectors(2, dim)
|
||||||
status, ids = connect.add_vectors(jac_table, vectors)
|
status, ids = connect.add_vectors(jac_collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
delete_ids = [ids[0]]
|
delete_ids = [ids[0]]
|
||||||
status = connect.delete_by_id(jac_table, delete_ids)
|
status = connect.delete_by_id(jac_collection, delete_ids)
|
||||||
status = connect.flush([jac_table])
|
status = connect.flush([jac_collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, info = connect.table_info(jac_table)
|
status, info = connect.collection_info(jac_collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, vector_ids = connect.get_vector_ids(jac_table, info.partitions_stat[0].segments_stat[0].segment_name)
|
status, vector_ids = connect.get_vector_ids(jac_collection, info.partitions_stat[0].segments_stat[0].segment_name)
|
||||||
assert len(vector_ids) == 1
|
assert len(vector_ids) == 1
|
||||||
assert vector_ids[0] == ids[1]
|
assert vector_ids[0] == ids[1]
|
File diff suppressed because it is too large
Load Diff
|
@ -12,7 +12,7 @@ from utils import *
|
||||||
|
|
||||||
dim = 128
|
dim = 128
|
||||||
index_file_size = 10
|
index_file_size = 10
|
||||||
table_id = "test_mix"
|
collection_id = "test_mix"
|
||||||
add_interval_time = 2
|
add_interval_time = 2
|
||||||
vectors = gen_vectors(10000, dim)
|
vectors = gen_vectors(10000, dim)
|
||||||
vectors = sklearn.preprocessing.normalize(vectors, axis=1, norm='l2')
|
vectors = sklearn.preprocessing.normalize(vectors, axis=1, norm='l2')
|
||||||
|
@ -29,35 +29,35 @@ class TestMixBase:
|
||||||
# disable
|
# disable
|
||||||
def _test_search_during_createIndex(self, args):
|
def _test_search_during_createIndex(self, args):
|
||||||
loops = 10000
|
loops = 10000
|
||||||
table = gen_unique_str()
|
collection = gen_unique_str()
|
||||||
query_vecs = [vectors[0], vectors[1]]
|
query_vecs = [vectors[0], vectors[1]]
|
||||||
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
||||||
id_0 = 0; id_1 = 0
|
id_0 = 0; id_1 = 0
|
||||||
milvus_instance = get_milvus(args["handler"])
|
milvus_instance = get_milvus(args["handler"])
|
||||||
milvus_instance.connect(uri=uri)
|
milvus_instance.connect(uri=uri)
|
||||||
milvus_instance.create_table({'table_name': table,
|
milvus_instance.create_collection({'collection_name': collection,
|
||||||
'dimension': dim,
|
'dimension': dim,
|
||||||
'index_file_size': index_file_size,
|
'index_file_size': index_file_size,
|
||||||
'metric_type': MetricType.L2})
|
'metric_type': MetricType.L2})
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
status, ids = milvus_instance.add_vectors(table, vectors)
|
status, ids = milvus_instance.add_vectors(collection, vectors)
|
||||||
# logging.getLogger().info(ids)
|
# logging.getLogger().info(ids)
|
||||||
if i == 0:
|
if i == 0:
|
||||||
id_0 = ids[0]; id_1 = ids[1]
|
id_0 = ids[0]; id_1 = ids[1]
|
||||||
def create_index(milvus_instance):
|
def create_index(milvus_instance):
|
||||||
logging.getLogger().info("In create index")
|
logging.getLogger().info("In create index")
|
||||||
status = milvus_instance.create_index(table, index_params)
|
status = milvus_instance.create_index(collection, index_params)
|
||||||
logging.getLogger().info(status)
|
logging.getLogger().info(status)
|
||||||
status, result = milvus_instance.describe_index(table)
|
status, result = milvus_instance.describe_index(collection)
|
||||||
logging.getLogger().info(result)
|
logging.getLogger().info(result)
|
||||||
def add_vectors(milvus_instance):
|
def add_vectors(milvus_instance):
|
||||||
logging.getLogger().info("In add vectors")
|
logging.getLogger().info("In add vectors")
|
||||||
status, ids = milvus_instance.add_vectors(table, vectors)
|
status, ids = milvus_instance.add_vectors(collection, vectors)
|
||||||
logging.getLogger().info(status)
|
logging.getLogger().info(status)
|
||||||
def search(milvus_instance):
|
def search(milvus_instance):
|
||||||
logging.getLogger().info("In search vectors")
|
logging.getLogger().info("In search vectors")
|
||||||
for i in range(loops):
|
for i in range(loops):
|
||||||
status, result = milvus_instance.search_vectors(table, top_k, nprobe, query_vecs)
|
status, result = milvus_instance.search_vectors(collection, top_k, nprobe, query_vecs)
|
||||||
logging.getLogger().info(status)
|
logging.getLogger().info(status)
|
||||||
assert result[0][0].id == id_0
|
assert result[0][0].id == id_0
|
||||||
assert result[1][0].id == id_1
|
assert result[1][0].id == id_1
|
||||||
|
@ -72,82 +72,82 @@ class TestMixBase:
|
||||||
p_create.join()
|
p_create.join()
|
||||||
|
|
||||||
@pytest.mark.level(2)
|
@pytest.mark.level(2)
|
||||||
def test_mix_multi_tables(self, connect):
|
def test_mix_multi_collections(self, connect):
|
||||||
'''
|
'''
|
||||||
target: test functions with multiple tables of different metric_types and index_types
|
target: test functions with multiple collections of different metric_types and index_types
|
||||||
method: create 60 tables which 30 are L2 and the other are IP, add vectors into them
|
method: create 60 collections which 30 are L2 and the other are IP, add vectors into them
|
||||||
and test describe index and search
|
and test describe index and search
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
nq = 10000
|
nq = 10000
|
||||||
table_list = []
|
collection_list = []
|
||||||
idx = []
|
idx = []
|
||||||
index_param = {'nlist': nlist}
|
index_param = {'nlist': nlist}
|
||||||
|
|
||||||
#create table and add vectors
|
#create collection and add vectors
|
||||||
for i in range(30):
|
for i in range(30):
|
||||||
table_name = gen_unique_str('test_mix_multi_tables')
|
collection_name = gen_unique_str('test_mix_multi_collections')
|
||||||
table_list.append(table_name)
|
collection_list.append(collection_name)
|
||||||
param = {'table_name': table_name,
|
param = {'collection_name': collection_name,
|
||||||
'dimension': dim,
|
'dimension': dim,
|
||||||
'index_file_size': index_file_size,
|
'index_file_size': index_file_size,
|
||||||
'metric_type': MetricType.L2}
|
'metric_type': MetricType.L2}
|
||||||
connect.create_table(param)
|
connect.create_collection(param)
|
||||||
status, ids = connect.add_vectors(table_name=table_name, records=vectors)
|
status, ids = connect.add_vectors(collection_name=collection_name, records=vectors)
|
||||||
idx.append(ids[0])
|
idx.append(ids[0])
|
||||||
idx.append(ids[10])
|
idx.append(ids[10])
|
||||||
idx.append(ids[20])
|
idx.append(ids[20])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
for i in range(30):
|
for i in range(30):
|
||||||
table_name = gen_unique_str('test_mix_multi_tables')
|
collection_name = gen_unique_str('test_mix_multi_collections')
|
||||||
table_list.append(table_name)
|
collection_list.append(collection_name)
|
||||||
param = {'table_name': table_name,
|
param = {'collection_name': collection_name,
|
||||||
'dimension': dim,
|
'dimension': dim,
|
||||||
'index_file_size': index_file_size,
|
'index_file_size': index_file_size,
|
||||||
'metric_type': MetricType.IP}
|
'metric_type': MetricType.IP}
|
||||||
connect.create_table(param)
|
connect.create_collection(param)
|
||||||
status, ids = connect.add_vectors(table_name=table_name, records=vectors)
|
status, ids = connect.add_vectors(collection_name=collection_name, records=vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table_name])
|
status = connect.flush([collection_name])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
idx.append(ids[0])
|
idx.append(ids[0])
|
||||||
idx.append(ids[10])
|
idx.append(ids[10])
|
||||||
idx.append(ids[20])
|
idx.append(ids[20])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
status = connect.create_index(table_list[i], IndexType.FLAT, index_param)
|
status = connect.create_index(collection_list[i], IndexType.FLAT, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.create_index(table_list[30 + i], IndexType.FLAT, index_param)
|
status = connect.create_index(collection_list[30 + i], IndexType.FLAT, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.create_index(table_list[10 + i], IndexType.IVFLAT, index_param)
|
status = connect.create_index(collection_list[10 + i], IndexType.IVFLAT, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.create_index(table_list[40 + i], IndexType.IVFLAT, index_param)
|
status = connect.create_index(collection_list[40 + i], IndexType.IVFLAT, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.create_index(table_list[20 + i], IndexType.IVF_SQ8, index_param)
|
status = connect.create_index(collection_list[20 + i], IndexType.IVF_SQ8, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.create_index(table_list[50 + i], IndexType.IVF_SQ8, index_param)
|
status = connect.create_index(collection_list[50 + i], IndexType.IVF_SQ8, index_param)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
|
|
||||||
#describe index
|
#describe index
|
||||||
for i in range(10):
|
for i in range(10):
|
||||||
status, result = connect.describe_index(table_list[i])
|
status, result = connect.describe_index(collection_list[i])
|
||||||
assert result._index_type == IndexType.FLAT
|
assert result._index_type == IndexType.FLAT
|
||||||
status, result = connect.describe_index(table_list[10 + i])
|
status, result = connect.describe_index(collection_list[10 + i])
|
||||||
assert result._index_type == IndexType.IVFLAT
|
assert result._index_type == IndexType.IVFLAT
|
||||||
status, result = connect.describe_index(table_list[20 + i])
|
status, result = connect.describe_index(collection_list[20 + i])
|
||||||
assert result._index_type == IndexType.IVF_SQ8
|
assert result._index_type == IndexType.IVF_SQ8
|
||||||
status, result = connect.describe_index(table_list[30 + i])
|
status, result = connect.describe_index(collection_list[30 + i])
|
||||||
assert result._index_type == IndexType.FLAT
|
assert result._index_type == IndexType.FLAT
|
||||||
status, result = connect.describe_index(table_list[40 + i])
|
status, result = connect.describe_index(collection_list[40 + i])
|
||||||
assert result._index_type == IndexType.IVFLAT
|
assert result._index_type == IndexType.IVFLAT
|
||||||
status, result = connect.describe_index(table_list[50 + i])
|
status, result = connect.describe_index(collection_list[50 + i])
|
||||||
assert result._index_type == IndexType.IVF_SQ8
|
assert result._index_type == IndexType.IVF_SQ8
|
||||||
|
|
||||||
#search
|
#search
|
||||||
query_vecs = [vectors[0], vectors[10], vectors[20]]
|
query_vecs = [vectors[0], vectors[10], vectors[20]]
|
||||||
for i in range(60):
|
for i in range(60):
|
||||||
table = table_list[i]
|
collection = collection_list[i]
|
||||||
status, result = connect.search_vectors(table, top_k, query_records=query_vecs, params={"nprobe": 1})
|
status, result = connect.search_vectors(collection, top_k, query_records=query_vecs, params={"nprobe": 1})
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert len(result) == len(query_vecs)
|
assert len(result) == len(query_vecs)
|
||||||
logging.getLogger().info(i)
|
logging.getLogger().info(i)
|
||||||
|
|
|
@ -11,7 +11,7 @@ from utils import *
|
||||||
|
|
||||||
dim = 128
|
dim = 128
|
||||||
index_file_size = 10
|
index_file_size = 10
|
||||||
table_id = "test_partition"
|
collection_id = "test_partition"
|
||||||
ADD_TIMEOUT = 60
|
ADD_TIMEOUT = 60
|
||||||
nprobe = 1
|
nprobe = 1
|
||||||
tag = "1970-01-01"
|
tag = "1970-01-01"
|
||||||
|
@ -24,58 +24,58 @@ class TestCreateBase:
|
||||||
The following cases are used to test `create_partition` function
|
The following cases are used to test `create_partition` function
|
||||||
******************************************************************
|
******************************************************************
|
||||||
"""
|
"""
|
||||||
def test_create_partition(self, connect, table):
|
def test_create_partition(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test create partition, check status returned
|
target: test create partition, check status returned
|
||||||
method: call function: create_partition
|
method: call function: create_partition
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
|
|
||||||
def test_create_partition_repeat(self, connect, table):
|
def test_create_partition_repeat(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test create partition, check status returned
|
target: test create partition, check status returned
|
||||||
method: call function: create_partition
|
method: call function: create_partition
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
def test_create_partition_table_not_existed(self, connect):
|
def test_create_partition_collection_not_existed(self, connect):
|
||||||
'''
|
'''
|
||||||
target: test create partition, its owner table name not existed in db, check status returned
|
target: test create partition, its owner collection name not existed in db, check status returned
|
||||||
method: call function: create_partition
|
method: call function: create_partition
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
table_name = gen_unique_str()
|
collection_name = gen_unique_str()
|
||||||
status = connect.create_partition(table_name, tag)
|
status = connect.create_partition(collection_name, tag)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
def test_create_partition_tag_name_None(self, connect, table):
|
def test_create_partition_tag_name_None(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test create partition, tag name set None, check status returned
|
target: test create partition, tag name set None, check status returned
|
||||||
method: call function: create_partition
|
method: call function: create_partition
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
tag_name = None
|
tag_name = None
|
||||||
status = connect.create_partition(table, tag_name)
|
status = connect.create_partition(collection, tag_name)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
def test_create_different_partition_tags(self, connect, table):
|
def test_create_different_partition_tags(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test create partition twice with different names
|
target: test create partition twice with different names
|
||||||
method: call function: create_partition, and again
|
method: call function: create_partition, and again
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
tag_name = gen_unique_str()
|
tag_name = gen_unique_str()
|
||||||
status = connect.create_partition(table, tag_name)
|
status = connect.create_partition(collection, tag_name)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.show_partitions(table)
|
status, res = connect.show_partitions(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
tag_list = []
|
tag_list = []
|
||||||
for item in res:
|
for item in res:
|
||||||
|
@ -84,95 +84,95 @@ class TestCreateBase:
|
||||||
assert tag_name in tag_list
|
assert tag_name in tag_list
|
||||||
assert "_default" in tag_list
|
assert "_default" in tag_list
|
||||||
|
|
||||||
def test_create_partition_add_vectors_default(self, connect, table):
|
def test_create_partition_add_vectors_default(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test create partition, and insert vectors, check status returned
|
target: test create partition, and insert vectors, check status returned
|
||||||
method: call function: create_partition
|
method: call function: create_partition
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
nq = 100
|
nq = 100
|
||||||
vectors = gen_vectors(nq, dim)
|
vectors = gen_vectors(nq, dim)
|
||||||
ids = [i for i in range(nq)]
|
ids = [i for i in range(nq)]
|
||||||
status, ids = connect.insert(table, vectors, ids)
|
status, ids = connect.insert(collection, vectors, ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
|
|
||||||
def test_create_partition_insert_with_tag(self, connect, table):
|
def test_create_partition_insert_with_tag(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test create partition, and insert vectors, check status returned
|
target: test create partition, and insert vectors, check status returned
|
||||||
method: call function: create_partition
|
method: call function: create_partition
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
nq = 100
|
nq = 100
|
||||||
vectors = gen_vectors(nq, dim)
|
vectors = gen_vectors(nq, dim)
|
||||||
ids = [i for i in range(nq)]
|
ids = [i for i in range(nq)]
|
||||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
|
|
||||||
def test_create_partition_insert_with_tag_not_existed(self, connect, table):
|
def test_create_partition_insert_with_tag_not_existed(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test create partition, and insert vectors, check status returned
|
target: test create partition, and insert vectors, check status returned
|
||||||
method: call function: create_partition
|
method: call function: create_partition
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
tag_new = "tag_new"
|
tag_new = "tag_new"
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
nq = 100
|
nq = 100
|
||||||
vectors = gen_vectors(nq, dim)
|
vectors = gen_vectors(nq, dim)
|
||||||
ids = [i for i in range(nq)]
|
ids = [i for i in range(nq)]
|
||||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag_new)
|
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag_new)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
def test_create_partition_insert_same_tags(self, connect, table):
|
def test_create_partition_insert_same_tags(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test create partition, and insert vectors, check status returned
|
target: test create partition, and insert vectors, check status returned
|
||||||
method: call function: create_partition
|
method: call function: create_partition
|
||||||
expected: status ok
|
expected: status ok
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
nq = 100
|
nq = 100
|
||||||
vectors = gen_vectors(nq, dim)
|
vectors = gen_vectors(nq, dim)
|
||||||
ids = [i for i in range(nq)]
|
ids = [i for i in range(nq)]
|
||||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||||
ids = [(i+100) for i in range(nq)]
|
ids = [(i+100) for i in range(nq)]
|
||||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_table_row_count(table)
|
status, res = connect.count_collection(collection)
|
||||||
assert res == nq * 2
|
assert res == nq * 2
|
||||||
|
|
||||||
def test_create_partition_insert_same_tags_two_tables(self, connect, table):
|
def test_create_partition_insert_same_tags_two_collections(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test create two partitions, and insert vectors with the same tag to each table, check status returned
|
target: test create two partitions, and insert vectors with the same tag to each collection, check status returned
|
||||||
method: call function: create_partition
|
method: call function: create_partition
|
||||||
expected: status ok, table length is correct
|
expected: status ok, collection length is correct
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
table_new = gen_unique_str()
|
collection_new = gen_unique_str()
|
||||||
param = {'table_name': table_new,
|
param = {'collection_name': collection_new,
|
||||||
'dimension': dim,
|
'dimension': dim,
|
||||||
'index_file_size': index_file_size,
|
'index_file_size': index_file_size,
|
||||||
'metric_type': MetricType.L2}
|
'metric_type': MetricType.L2}
|
||||||
status = connect.create_table(param)
|
status = connect.create_collection(param)
|
||||||
status = connect.create_partition(table_new, tag)
|
status = connect.create_partition(collection_new, tag)
|
||||||
nq = 100
|
nq = 100
|
||||||
vectors = gen_vectors(nq, dim)
|
vectors = gen_vectors(nq, dim)
|
||||||
ids = [i for i in range(nq)]
|
ids = [i for i in range(nq)]
|
||||||
status, ids = connect.insert(table, vectors, ids, partition_tag=tag)
|
status, ids = connect.insert(collection, vectors, ids, partition_tag=tag)
|
||||||
ids = [(i+100) for i in range(nq)]
|
ids = [(i+100) for i in range(nq)]
|
||||||
status, ids = connect.insert(table_new, vectors, ids, partition_tag=tag)
|
status, ids = connect.insert(collection_new, vectors, ids, partition_tag=tag)
|
||||||
status = connect.flush([table, table_new])
|
status = connect.flush([collection, collection_new])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_table_row_count(table)
|
status, res = connect.count_collection(collection)
|
||||||
assert res == nq
|
assert res == nq
|
||||||
status, res = connect.get_table_row_count(table_new)
|
status, res = connect.count_collection(collection_new)
|
||||||
assert res == nq
|
assert res == nq
|
||||||
|
|
||||||
|
|
||||||
|
@ -183,35 +183,35 @@ class TestShowBase:
|
||||||
The following cases are used to test `show_partitions` function
|
The following cases are used to test `show_partitions` function
|
||||||
******************************************************************
|
******************************************************************
|
||||||
"""
|
"""
|
||||||
def test_show_partitions(self, connect, table):
|
def test_show_partitions(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test show partitions, check status and partitions returned
|
target: test show partitions, check status and partitions returned
|
||||||
method: create partition first, then call function: show_partitions
|
method: create partition first, then call function: show_partitions
|
||||||
expected: status ok, partition correct
|
expected: status ok, partition correct
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
status, res = connect.show_partitions(table)
|
status, res = connect.show_partitions(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
|
|
||||||
def test_show_partitions_no_partition(self, connect, table):
|
def test_show_partitions_no_partition(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test show partitions with table name, check status and partitions returned
|
target: test show partitions with collection name, check status and partitions returned
|
||||||
method: call function: show_partitions
|
method: call function: show_partitions
|
||||||
expected: status ok, partitions correct
|
expected: status ok, partitions correct
|
||||||
'''
|
'''
|
||||||
status, res = connect.show_partitions(table)
|
status, res = connect.show_partitions(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
|
|
||||||
def test_show_multi_partitions(self, connect, table):
|
def test_show_multi_partitions(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test show partitions, check status and partitions returned
|
target: test show partitions, check status and partitions returned
|
||||||
method: create partitions first, then call function: show_partitions
|
method: create partitions first, then call function: show_partitions
|
||||||
expected: status ok, partitions correct
|
expected: status ok, partitions correct
|
||||||
'''
|
'''
|
||||||
tag_new = gen_unique_str()
|
tag_new = gen_unique_str()
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
status = connect.create_partition(table, tag_new)
|
status = connect.create_partition(collection, tag_new)
|
||||||
status, res = connect.show_partitions(table)
|
status, res = connect.show_partitions(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
|
|
||||||
|
|
||||||
|
@ -222,72 +222,72 @@ class TestDropBase:
|
||||||
The following cases are used to test `drop_partition` function
|
The following cases are used to test `drop_partition` function
|
||||||
******************************************************************
|
******************************************************************
|
||||||
"""
|
"""
|
||||||
def test_drop_partition(self, connect, table):
|
def test_drop_partition(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test drop partition, check status and partition if existed
|
target: test drop partition, check status and partition if existed
|
||||||
method: create partitions first, then call function: drop_partition
|
method: create partitions first, then call function: drop_partition
|
||||||
expected: status ok, no partitions in db
|
expected: status ok, no partitions in db
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
status = connect.drop_partition(table, tag)
|
status = connect.drop_partition(collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.show_partitions(table)
|
status, res = connect.show_partitions(collection)
|
||||||
tag_list = []
|
tag_list = []
|
||||||
for item in res:
|
for item in res:
|
||||||
tag_list.append(item.tag)
|
tag_list.append(item.tag)
|
||||||
assert tag not in tag_list
|
assert tag not in tag_list
|
||||||
|
|
||||||
def test_drop_partition_tag_not_existed(self, connect, table):
|
def test_drop_partition_tag_not_existed(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test drop partition, but tag not existed
|
target: test drop partition, but tag not existed
|
||||||
method: create partitions first, then call function: drop_partition
|
method: create partitions first, then call function: drop_partition
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
new_tag = "new_tag"
|
new_tag = "new_tag"
|
||||||
status = connect.drop_partition(table, new_tag)
|
status = connect.drop_partition(collection, new_tag)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
def test_drop_partition_tag_not_existed_A(self, connect, table):
|
def test_drop_partition_tag_not_existed_A(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test drop partition, but table not existed
|
target: test drop partition, but collection not existed
|
||||||
method: create partitions first, then call function: drop_partition
|
method: create partitions first, then call function: drop_partition
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
new_table = gen_unique_str()
|
new_collection = gen_unique_str()
|
||||||
status = connect.drop_partition(new_table, tag)
|
status = connect.drop_partition(new_collection, tag)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
def test_drop_partition_repeatedly(self, connect, table):
|
def test_drop_partition_repeatedly(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test drop partition twice, check status and partition if existed
|
target: test drop partition twice, check status and partition if existed
|
||||||
method: create partitions first, then call function: drop_partition
|
method: create partitions first, then call function: drop_partition
|
||||||
expected: status not ok, no partitions in db
|
expected: status not ok, no partitions in db
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
status = connect.drop_partition(table, tag)
|
status = connect.drop_partition(collection, tag)
|
||||||
status = connect.drop_partition(table, tag)
|
status = connect.drop_partition(collection, tag)
|
||||||
time.sleep(2)
|
time.sleep(2)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
status, res = connect.show_partitions(table)
|
status, res = connect.show_partitions(collection)
|
||||||
tag_list = []
|
tag_list = []
|
||||||
for item in res:
|
for item in res:
|
||||||
tag_list.append(item.tag)
|
tag_list.append(item.tag)
|
||||||
assert tag not in tag_list
|
assert tag not in tag_list
|
||||||
|
|
||||||
def test_drop_partition_create(self, connect, table):
|
def test_drop_partition_create(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test drop partition, and create again, check status
|
target: test drop partition, and create again, check status
|
||||||
method: create partitions first, then call function: drop_partition, create_partition
|
method: create partitions first, then call function: drop_partition, create_partition
|
||||||
expected: status not ok, partition in db
|
expected: status not ok, partition in db
|
||||||
'''
|
'''
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
status = connect.drop_partition(table, tag)
|
status = connect.drop_partition(collection, tag)
|
||||||
time.sleep(2)
|
time.sleep(2)
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.show_partitions(table)
|
status, res = connect.show_partitions(collection)
|
||||||
tag_list = []
|
tag_list = []
|
||||||
for item in res:
|
for item in res:
|
||||||
tag_list.append(item.tag)
|
tag_list.append(item.tag)
|
||||||
|
@ -297,47 +297,47 @@ class TestDropBase:
|
||||||
class TestNameInvalid(object):
|
class TestNameInvalid(object):
|
||||||
@pytest.fixture(
|
@pytest.fixture(
|
||||||
scope="function",
|
scope="function",
|
||||||
params=gen_invalid_table_names()
|
params=gen_invalid_collection_names()
|
||||||
)
|
)
|
||||||
def get_tag_name(self, request):
|
def get_tag_name(self, request):
|
||||||
yield request.param
|
yield request.param
|
||||||
|
|
||||||
@pytest.fixture(
|
@pytest.fixture(
|
||||||
scope="function",
|
scope="function",
|
||||||
params=gen_invalid_table_names()
|
params=gen_invalid_collection_names()
|
||||||
)
|
)
|
||||||
def get_table_name(self, request):
|
def get_collection_name(self, request):
|
||||||
yield request.param
|
yield request.param
|
||||||
|
|
||||||
def test_drop_partition_with_invalid_table_name(self, connect, table, get_table_name):
|
def test_drop_partition_with_invalid_collection_name(self, connect, collection, get_collection_name):
|
||||||
'''
|
'''
|
||||||
target: test drop partition, with invalid table name, check status returned
|
target: test drop partition, with invalid collection name, check status returned
|
||||||
method: call function: drop_partition
|
method: call function: drop_partition
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
table_name = get_table_name
|
collection_name = get_collection_name
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
status = connect.drop_partition(table_name, tag)
|
status = connect.drop_partition(collection_name, tag)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
def test_drop_partition_with_invalid_tag_name(self, connect, table, get_tag_name):
|
def test_drop_partition_with_invalid_tag_name(self, connect, collection, get_tag_name):
|
||||||
'''
|
'''
|
||||||
target: test drop partition, with invalid tag name, check status returned
|
target: test drop partition, with invalid tag name, check status returned
|
||||||
method: call function: drop_partition
|
method: call function: drop_partition
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
tag_name = get_tag_name
|
tag_name = get_tag_name
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
status = connect.drop_partition(table, tag_name)
|
status = connect.drop_partition(collection, tag_name)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
||||||
def test_show_partitions_with_invalid_table_name(self, connect, table, get_table_name):
|
def test_show_partitions_with_invalid_collection_name(self, connect, collection, get_collection_name):
|
||||||
'''
|
'''
|
||||||
target: test show partitions, with invalid table name, check status returned
|
target: test show partitions, with invalid collection name, check status returned
|
||||||
method: call function: show_partitions
|
method: call function: show_partitions
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
table_name = get_table_name
|
collection_name = get_collection_name
|
||||||
status = connect.create_partition(table, tag)
|
status = connect.create_partition(collection, tag)
|
||||||
status, res = connect.show_partitions(table_name)
|
status, res = connect.show_partitions(collection_name)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
@ -1,644 +0,0 @@
|
||||||
import pdb
|
|
||||||
import pytest
|
|
||||||
import logging
|
|
||||||
import itertools
|
|
||||||
from time import sleep
|
|
||||||
import threading
|
|
||||||
from multiprocessing import Process
|
|
||||||
from milvus import IndexType, MetricType
|
|
||||||
from utils import *
|
|
||||||
|
|
||||||
dim = 128
|
|
||||||
index_file_size = 10
|
|
||||||
add_time_interval = 3
|
|
||||||
tag = "1970-01-01"
|
|
||||||
nb = 6000
|
|
||||||
|
|
||||||
class TestTableCount:
|
|
||||||
"""
|
|
||||||
params means different nb, the nb value may trigger merge, or not
|
|
||||||
"""
|
|
||||||
@pytest.fixture(
|
|
||||||
scope="function",
|
|
||||||
params=[
|
|
||||||
1,
|
|
||||||
5000,
|
|
||||||
100000,
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def add_vectors_nb(self, request):
|
|
||||||
yield request.param
|
|
||||||
|
|
||||||
"""
|
|
||||||
generate valid create_index params
|
|
||||||
"""
|
|
||||||
@pytest.fixture(
|
|
||||||
scope="function",
|
|
||||||
params=gen_simple_index()
|
|
||||||
)
|
|
||||||
def get_simple_index(self, request, connect):
|
|
||||||
if str(connect._cmd("mode")[1]) == "CPU":
|
|
||||||
if request.param["index_type"] == IndexType.IVF_SQ8H:
|
|
||||||
pytest.skip("sq8h not support in cpu mode")
|
|
||||||
if request.param["index_type"] == IndexType.IVF_PQ:
|
|
||||||
pytest.skip("Skip PQ Temporary")
|
|
||||||
return request.param
|
|
||||||
|
|
||||||
def test_table_rows_count(self, connect, table, add_vectors_nb):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not
|
|
||||||
method: create table and add vectors in it,
|
|
||||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
|
||||||
expected: the count is equal to the length of vectors
|
|
||||||
'''
|
|
||||||
nb = add_vectors_nb
|
|
||||||
vectors = gen_vectors(nb, dim)
|
|
||||||
res = connect.add_vectors(table_name=table, records=vectors)
|
|
||||||
connect.flush([table])
|
|
||||||
status, res = connect.get_table_row_count(table)
|
|
||||||
assert res == nb
|
|
||||||
|
|
||||||
def test_table_rows_count_partition(self, connect, table, add_vectors_nb):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not
|
|
||||||
method: create table, create partition and add vectors in it,
|
|
||||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
|
||||||
expected: the count is equal to the length of vectors
|
|
||||||
'''
|
|
||||||
nb = add_vectors_nb
|
|
||||||
vectors = gen_vectors(nb, dim)
|
|
||||||
status = connect.create_partition(table, tag)
|
|
||||||
assert status.OK()
|
|
||||||
res = connect.add_vectors(table_name=table, records=vectors, partition_tag=tag)
|
|
||||||
connect.flush([table])
|
|
||||||
status, res = connect.get_table_row_count(table)
|
|
||||||
assert res == nb
|
|
||||||
|
|
||||||
def test_table_rows_count_multi_partitions_A(self, connect, table, add_vectors_nb):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not
|
|
||||||
method: create table, create partitions and add vectors in it,
|
|
||||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
|
||||||
expected: the count is equal to the length of vectors
|
|
||||||
'''
|
|
||||||
new_tag = "new_tag"
|
|
||||||
nb = add_vectors_nb
|
|
||||||
vectors = gen_vectors(nb, dim)
|
|
||||||
status = connect.create_partition(table, tag)
|
|
||||||
status = connect.create_partition(table, new_tag)
|
|
||||||
assert status.OK()
|
|
||||||
res = connect.add_vectors(table_name=table, records=vectors)
|
|
||||||
connect.flush([table])
|
|
||||||
status, res = connect.get_table_row_count(table)
|
|
||||||
assert res == nb
|
|
||||||
|
|
||||||
def test_table_rows_count_multi_partitions_B(self, connect, table, add_vectors_nb):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not
|
|
||||||
method: create table, create partitions and add vectors in one of the partitions,
|
|
||||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
|
||||||
expected: the count is equal to the length of vectors
|
|
||||||
'''
|
|
||||||
new_tag = "new_tag"
|
|
||||||
nb = add_vectors_nb
|
|
||||||
vectors = gen_vectors(nb, dim)
|
|
||||||
status = connect.create_partition(table, tag)
|
|
||||||
status = connect.create_partition(table, new_tag)
|
|
||||||
assert status.OK()
|
|
||||||
res = connect.add_vectors(table_name=table, records=vectors, partition_tag=tag)
|
|
||||||
connect.flush([table])
|
|
||||||
status, res = connect.get_table_row_count(table)
|
|
||||||
assert res == nb
|
|
||||||
|
|
||||||
def test_table_rows_count_multi_partitions_C(self, connect, table, add_vectors_nb):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not
|
|
||||||
method: create table, create partitions and add vectors in one of the partitions,
|
|
||||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
|
||||||
expected: the table count is equal to the length of vectors
|
|
||||||
'''
|
|
||||||
new_tag = "new_tag"
|
|
||||||
nb = add_vectors_nb
|
|
||||||
vectors = gen_vectors(nb, dim)
|
|
||||||
status = connect.create_partition(table, tag)
|
|
||||||
status = connect.create_partition(table, new_tag)
|
|
||||||
assert status.OK()
|
|
||||||
res = connect.add_vectors(table_name=table, records=vectors, partition_tag=tag)
|
|
||||||
res = connect.add_vectors(table_name=table, records=vectors, partition_tag=new_tag)
|
|
||||||
connect.flush([table])
|
|
||||||
status, res = connect.get_table_row_count(table)
|
|
||||||
assert res == nb * 2
|
|
||||||
|
|
||||||
def test_table_rows_count_after_index_created(self, connect, table, get_simple_index):
|
|
||||||
'''
|
|
||||||
target: test get_table_row_count, after index have been created
|
|
||||||
method: add vectors in db, and create index, then calling get_table_row_count with correct params
|
|
||||||
expected: get_table_row_count raise exception
|
|
||||||
'''
|
|
||||||
index_param = get_simple_index["index_param"]
|
|
||||||
index_type = get_simple_index["index_type"]
|
|
||||||
nb = 100
|
|
||||||
vectors = gen_vectors(nb, dim)
|
|
||||||
res = connect.add_vectors(table_name=table, records=vectors)
|
|
||||||
connect.flush([table])
|
|
||||||
connect.create_index(table, index_type, index_param)
|
|
||||||
status, res = connect.get_table_row_count(table)
|
|
||||||
assert res == nb
|
|
||||||
|
|
||||||
@pytest.mark.level(2)
|
|
||||||
def test_count_without_connection(self, table, dis_connect):
|
|
||||||
'''
|
|
||||||
target: test get_table_row_count, without connection
|
|
||||||
method: calling get_table_row_count with correct params, with a disconnected instance
|
|
||||||
expected: get_table_row_count raise exception
|
|
||||||
'''
|
|
||||||
with pytest.raises(Exception) as e:
|
|
||||||
status = dis_connect.get_table_row_count(table)
|
|
||||||
|
|
||||||
def test_table_rows_count_no_vectors(self, connect, table):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not, if table is empty
|
|
||||||
method: create table and no vectors in it,
|
|
||||||
assert the value returned by get_table_row_count method is equal to 0
|
|
||||||
expected: the count is equal to 0
|
|
||||||
'''
|
|
||||||
table_name = gen_unique_str()
|
|
||||||
param = {'table_name': table_name,
|
|
||||||
'dimension': dim,
|
|
||||||
'index_file_size': index_file_size}
|
|
||||||
connect.create_table(param)
|
|
||||||
status, res = connect.get_table_row_count(table)
|
|
||||||
assert res == 0
|
|
||||||
|
|
||||||
# TODO: enable
|
|
||||||
@pytest.mark.level(2)
|
|
||||||
@pytest.mark.timeout(20)
|
|
||||||
def _test_table_rows_count_multiprocessing(self, connect, table, args):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not with multiprocess
|
|
||||||
method: create table and add vectors in it,
|
|
||||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
|
||||||
expected: the count is equal to the length of vectors
|
|
||||||
'''
|
|
||||||
nq = 2
|
|
||||||
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
|
||||||
vectors = gen_vectors(nq, dim)
|
|
||||||
res = connect.add_vectors(table_name=table, records=vectors)
|
|
||||||
time.sleep(add_time_interval)
|
|
||||||
|
|
||||||
def rows_count(milvus):
|
|
||||||
status, res = milvus.get_table_row_count(table)
|
|
||||||
logging.getLogger().info(status)
|
|
||||||
assert res == nq
|
|
||||||
|
|
||||||
process_num = 8
|
|
||||||
processes = []
|
|
||||||
for i in range(process_num):
|
|
||||||
milvus = get_milvus(args["handler"])
|
|
||||||
milvus.connect(uri=uri)
|
|
||||||
p = Process(target=rows_count, args=(milvus, ))
|
|
||||||
processes.append(p)
|
|
||||||
p.start()
|
|
||||||
logging.getLogger().info(p)
|
|
||||||
for p in processes:
|
|
||||||
p.join()
|
|
||||||
|
|
||||||
def test_table_rows_count_multi_tables(self, connect):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not with multiple tables of L2
|
|
||||||
method: create table and add vectors in it,
|
|
||||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
|
||||||
expected: the count is equal to the length of vectors
|
|
||||||
'''
|
|
||||||
nq = 100
|
|
||||||
vectors = gen_vectors(nq, dim)
|
|
||||||
table_list = []
|
|
||||||
for i in range(20):
|
|
||||||
table_name = gen_unique_str()
|
|
||||||
table_list.append(table_name)
|
|
||||||
param = {'table_name': table_name,
|
|
||||||
'dimension': dim,
|
|
||||||
'index_file_size': index_file_size,
|
|
||||||
'metric_type': MetricType.L2}
|
|
||||||
connect.create_table(param)
|
|
||||||
res = connect.add_vectors(table_name=table_name, records=vectors)
|
|
||||||
connect.flush(table_list)
|
|
||||||
for i in range(20):
|
|
||||||
status, res = connect.get_table_row_count(table_list[i])
|
|
||||||
assert status.OK()
|
|
||||||
assert res == nq
|
|
||||||
|
|
||||||
|
|
||||||
class TestTableCountIP:
|
|
||||||
"""
|
|
||||||
params means different nb, the nb value may trigger merge, or not
|
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.fixture(
|
|
||||||
scope="function",
|
|
||||||
params=[
|
|
||||||
1,
|
|
||||||
5000,
|
|
||||||
100000,
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def add_vectors_nb(self, request):
|
|
||||||
yield request.param
|
|
||||||
|
|
||||||
"""
|
|
||||||
generate valid create_index params
|
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.fixture(
|
|
||||||
scope="function",
|
|
||||||
params=gen_simple_index()
|
|
||||||
)
|
|
||||||
def get_simple_index(self, request, connect):
|
|
||||||
if str(connect._cmd("mode")[1]) == "CPU":
|
|
||||||
if request.param["index_type"] == IndexType.IVF_SQ8H:
|
|
||||||
pytest.skip("sq8h not support in CPU mode")
|
|
||||||
if request.param["index_type"] == IndexType.IVF_PQ:
|
|
||||||
pytest.skip("Skip PQ Temporary")
|
|
||||||
return request.param
|
|
||||||
|
|
||||||
def test_table_rows_count(self, connect, ip_table, add_vectors_nb):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not
|
|
||||||
method: create table and add vectors in it,
|
|
||||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
|
||||||
expected: the count is equal to the length of vectors
|
|
||||||
'''
|
|
||||||
nb = add_vectors_nb
|
|
||||||
vectors = gen_vectors(nb, dim)
|
|
||||||
res = connect.add_vectors(table_name=ip_table, records=vectors)
|
|
||||||
connect.flush([ip_table])
|
|
||||||
status, res = connect.get_table_row_count(ip_table)
|
|
||||||
assert res == nb
|
|
||||||
|
|
||||||
def test_table_rows_count_after_index_created(self, connect, ip_table, get_simple_index):
|
|
||||||
'''
|
|
||||||
target: test get_table_row_count, after index have been created
|
|
||||||
method: add vectors in db, and create index, then calling get_table_row_count with correct params
|
|
||||||
expected: get_table_row_count raise exception
|
|
||||||
'''
|
|
||||||
index_param = get_simple_index["index_param"]
|
|
||||||
index_type = get_simple_index["index_type"]
|
|
||||||
nb = 100
|
|
||||||
vectors = gen_vectors(nb, dim)
|
|
||||||
res = connect.add_vectors(table_name=ip_table, records=vectors)
|
|
||||||
connect.flush([ip_table])
|
|
||||||
connect.create_index(ip_table, index_type, index_param)
|
|
||||||
status, res = connect.get_table_row_count(ip_table)
|
|
||||||
assert res == nb
|
|
||||||
|
|
||||||
@pytest.mark.level(2)
|
|
||||||
def test_count_without_connection(self, ip_table, dis_connect):
|
|
||||||
'''
|
|
||||||
target: test get_table_row_count, without connection
|
|
||||||
method: calling get_table_row_count with correct params, with a disconnected instance
|
|
||||||
expected: get_table_row_count raise exception
|
|
||||||
'''
|
|
||||||
with pytest.raises(Exception) as e:
|
|
||||||
status = dis_connect.get_table_row_count(ip_table)
|
|
||||||
|
|
||||||
def test_table_rows_count_no_vectors(self, connect, ip_table):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not, if table is empty
|
|
||||||
method: create table and no vectors in it,
|
|
||||||
assert the value returned by get_table_row_count method is equal to 0
|
|
||||||
expected: the count is equal to 0
|
|
||||||
'''
|
|
||||||
table_name = gen_unique_str("test_table")
|
|
||||||
param = {'table_name': table_name,
|
|
||||||
'dimension': dim,
|
|
||||||
'index_file_size': index_file_size}
|
|
||||||
connect.create_table(param)
|
|
||||||
status, res = connect.get_table_row_count(ip_table)
|
|
||||||
assert res == 0
|
|
||||||
|
|
||||||
# TODO: enable
|
|
||||||
@pytest.mark.timeout(60)
|
|
||||||
def _test_table_rows_count_multiprocessing(self, connect, ip_table, args):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not with multiprocess
|
|
||||||
method: create table and add vectors in it,
|
|
||||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
|
||||||
expected: the count is equal to the length of vectors
|
|
||||||
'''
|
|
||||||
nq = 2
|
|
||||||
uri = "tcp://%s:%s" % (args["ip"], args["port"])
|
|
||||||
vectors = gen_vectors(nq, dim)
|
|
||||||
res = connect.add_vectors(table_name=ip_table, records=vectors)
|
|
||||||
time.sleep(add_time_interval)
|
|
||||||
|
|
||||||
def rows_count(milvus):
|
|
||||||
status, res = milvus.get_table_row_count(ip_table)
|
|
||||||
logging.getLogger().info(status)
|
|
||||||
assert res == nq
|
|
||||||
|
|
||||||
process_num = 8
|
|
||||||
processes = []
|
|
||||||
for i in range(process_num):
|
|
||||||
milvus = get_milvus(args["handler"])
|
|
||||||
milvus.connect(uri=uri)
|
|
||||||
p = Process(target=rows_count, args=(milvus,))
|
|
||||||
processes.append(p)
|
|
||||||
p.start()
|
|
||||||
logging.getLogger().info(p)
|
|
||||||
for p in processes:
|
|
||||||
p.join()
|
|
||||||
|
|
||||||
def test_table_rows_count_multi_tables(self, connect):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not with multiple tables of IP
|
|
||||||
method: create table and add vectors in it,
|
|
||||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
|
||||||
expected: the count is equal to the length of vectors
|
|
||||||
'''
|
|
||||||
nq = 100
|
|
||||||
vectors = gen_vectors(nq, dim)
|
|
||||||
table_list = []
|
|
||||||
for i in range(20):
|
|
||||||
table_name = gen_unique_str('test_table_rows_count_multi_tables')
|
|
||||||
table_list.append(table_name)
|
|
||||||
param = {'table_name': table_name,
|
|
||||||
'dimension': dim,
|
|
||||||
'index_file_size': index_file_size,
|
|
||||||
'metric_type': MetricType.IP}
|
|
||||||
connect.create_table(param)
|
|
||||||
res = connect.add_vectors(table_name=table_name, records=vectors)
|
|
||||||
connect.flush(table_list)
|
|
||||||
for i in range(20):
|
|
||||||
status, res = connect.get_table_row_count(table_list[i])
|
|
||||||
assert status.OK()
|
|
||||||
assert res == nq
|
|
||||||
|
|
||||||
|
|
||||||
class TestTableCountJAC:
|
|
||||||
"""
|
|
||||||
params means different nb, the nb value may trigger merge, or not
|
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.fixture(
|
|
||||||
scope="function",
|
|
||||||
params=[
|
|
||||||
1,
|
|
||||||
5000,
|
|
||||||
100000,
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def add_vectors_nb(self, request):
|
|
||||||
yield request.param
|
|
||||||
|
|
||||||
"""
|
|
||||||
generate valid create_index params
|
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.fixture(
|
|
||||||
scope="function",
|
|
||||||
params=gen_simple_index()
|
|
||||||
)
|
|
||||||
def get_jaccard_index(self, request, connect):
|
|
||||||
logging.getLogger().info(request.param)
|
|
||||||
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
|
|
||||||
return request.param
|
|
||||||
else:
|
|
||||||
pytest.skip("Skip index Temporary")
|
|
||||||
|
|
||||||
def test_table_rows_count(self, connect, jac_table, add_vectors_nb):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not
|
|
||||||
method: create table and add vectors in it,
|
|
||||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
|
||||||
expected: the count is equal to the length of vectors
|
|
||||||
'''
|
|
||||||
nb = add_vectors_nb
|
|
||||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
|
||||||
res = connect.add_vectors(table_name=jac_table, records=vectors)
|
|
||||||
connect.flush([jac_table])
|
|
||||||
status, res = connect.get_table_row_count(jac_table)
|
|
||||||
assert res == nb
|
|
||||||
|
|
||||||
def test_table_rows_count_after_index_created(self, connect, jac_table, get_jaccard_index):
|
|
||||||
'''
|
|
||||||
target: test get_table_row_count, after index have been created
|
|
||||||
method: add vectors in db, and create index, then calling get_table_row_count with correct params
|
|
||||||
expected: get_table_row_count raise exception
|
|
||||||
'''
|
|
||||||
nb = 100
|
|
||||||
index_param = get_jaccard_index["index_param"]
|
|
||||||
index_type = get_jaccard_index["index_type"]
|
|
||||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
|
||||||
res = connect.add_vectors(table_name=jac_table, records=vectors)
|
|
||||||
connect.flush([jac_table])
|
|
||||||
connect.create_index(jac_table, index_type, index_param)
|
|
||||||
status, res = connect.get_table_row_count(jac_table)
|
|
||||||
assert res == nb
|
|
||||||
|
|
||||||
@pytest.mark.level(2)
|
|
||||||
def test_count_without_connection(self, jac_table, dis_connect):
|
|
||||||
'''
|
|
||||||
target: test get_table_row_count, without connection
|
|
||||||
method: calling get_table_row_count with correct params, with a disconnected instance
|
|
||||||
expected: get_table_row_count raise exception
|
|
||||||
'''
|
|
||||||
with pytest.raises(Exception) as e:
|
|
||||||
status = dis_connect.get_table_row_count(jac_table)
|
|
||||||
|
|
||||||
def test_table_rows_count_no_vectors(self, connect, jac_table):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not, if table is empty
|
|
||||||
method: create table and no vectors in it,
|
|
||||||
assert the value returned by get_table_row_count method is equal to 0
|
|
||||||
expected: the count is equal to 0
|
|
||||||
'''
|
|
||||||
table_name = gen_unique_str("test_table")
|
|
||||||
param = {'table_name': table_name,
|
|
||||||
'dimension': dim,
|
|
||||||
'index_file_size': index_file_size}
|
|
||||||
connect.create_table(param)
|
|
||||||
status, res = connect.get_table_row_count(jac_table)
|
|
||||||
assert res == 0
|
|
||||||
|
|
||||||
def test_table_rows_count_multi_tables(self, connect):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not with multiple tables of IP
|
|
||||||
method: create table and add vectors in it,
|
|
||||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
|
||||||
expected: the count is equal to the length of vectors
|
|
||||||
'''
|
|
||||||
nq = 100
|
|
||||||
tmp, vectors = gen_binary_vectors(nq, dim)
|
|
||||||
table_list = []
|
|
||||||
for i in range(20):
|
|
||||||
table_name = gen_unique_str('test_table_rows_count_multi_tables')
|
|
||||||
table_list.append(table_name)
|
|
||||||
param = {'table_name': table_name,
|
|
||||||
'dimension': dim,
|
|
||||||
'index_file_size': index_file_size,
|
|
||||||
'metric_type': MetricType.JACCARD}
|
|
||||||
connect.create_table(param)
|
|
||||||
res = connect.add_vectors(table_name=table_name, records=vectors)
|
|
||||||
connect.flush(table_list)
|
|
||||||
for i in range(20):
|
|
||||||
status, res = connect.get_table_row_count(table_list[i])
|
|
||||||
assert status.OK()
|
|
||||||
assert res == nq
|
|
||||||
|
|
||||||
class TestTableCountHAM:
|
|
||||||
"""
|
|
||||||
params means different nb, the nb value may trigger merge, or not
|
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.fixture(
|
|
||||||
scope="function",
|
|
||||||
params=[
|
|
||||||
1,
|
|
||||||
5000,
|
|
||||||
100000,
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def add_vectors_nb(self, request):
|
|
||||||
yield request.param
|
|
||||||
|
|
||||||
"""
|
|
||||||
generate valid create_index params
|
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.fixture(
|
|
||||||
scope="function",
|
|
||||||
params=gen_simple_index()
|
|
||||||
)
|
|
||||||
def get_hamming_index(self, request, connect):
|
|
||||||
logging.getLogger().info(request.param)
|
|
||||||
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
|
|
||||||
return request.param
|
|
||||||
else:
|
|
||||||
pytest.skip("Skip index Temporary")
|
|
||||||
|
|
||||||
def test_table_rows_count(self, connect, ham_table, add_vectors_nb):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not
|
|
||||||
method: create table and add vectors in it,
|
|
||||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
|
||||||
expected: the count is equal to the length of vectors
|
|
||||||
'''
|
|
||||||
nb = add_vectors_nb
|
|
||||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
|
||||||
res = connect.add_vectors(table_name=ham_table, records=vectors)
|
|
||||||
connect.flush([ham_table])
|
|
||||||
status, res = connect.get_table_row_count(ham_table)
|
|
||||||
assert res == nb
|
|
||||||
|
|
||||||
def test_table_rows_count_after_index_created(self, connect, ham_table, get_hamming_index):
|
|
||||||
'''
|
|
||||||
target: test get_table_row_count, after index have been created
|
|
||||||
method: add vectors in db, and create index, then calling get_table_row_count with correct params
|
|
||||||
expected: get_table_row_count raise exception
|
|
||||||
'''
|
|
||||||
nb = 100
|
|
||||||
index_type = get_hamming_index["index_type"]
|
|
||||||
index_param = get_hamming_index["index_param"]
|
|
||||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
|
||||||
res = connect.add_vectors(table_name=ham_table, records=vectors)
|
|
||||||
connect.flush([ham_table])
|
|
||||||
connect.create_index(ham_table, index_type, index_param)
|
|
||||||
status, res = connect.get_table_row_count(ham_table)
|
|
||||||
assert res == nb
|
|
||||||
|
|
||||||
@pytest.mark.level(2)
|
|
||||||
def test_count_without_connection(self, ham_table, dis_connect):
|
|
||||||
'''
|
|
||||||
target: test get_table_row_count, without connection
|
|
||||||
method: calling get_table_row_count with correct params, with a disconnected instance
|
|
||||||
expected: get_table_row_count raise exception
|
|
||||||
'''
|
|
||||||
with pytest.raises(Exception) as e:
|
|
||||||
status = dis_connect.get_table_row_count(ham_table)
|
|
||||||
|
|
||||||
def test_table_rows_count_no_vectors(self, connect, ham_table):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not, if table is empty
|
|
||||||
method: create table and no vectors in it,
|
|
||||||
assert the value returned by get_table_row_count method is equal to 0
|
|
||||||
expected: the count is equal to 0
|
|
||||||
'''
|
|
||||||
table_name = gen_unique_str("test_table")
|
|
||||||
param = {'table_name': table_name,
|
|
||||||
'dimension': dim,
|
|
||||||
'index_file_size': index_file_size}
|
|
||||||
connect.create_table(param)
|
|
||||||
status, res = connect.get_table_row_count(ham_table)
|
|
||||||
assert res == 0
|
|
||||||
|
|
||||||
def test_table_rows_count_multi_tables(self, connect):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not with multiple tables of IP
|
|
||||||
method: create table and add vectors in it,
|
|
||||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
|
||||||
expected: the count is equal to the length of vectors
|
|
||||||
'''
|
|
||||||
nq = 100
|
|
||||||
tmp, vectors = gen_binary_vectors(nq, dim)
|
|
||||||
table_list = []
|
|
||||||
for i in range(20):
|
|
||||||
table_name = gen_unique_str('test_table_rows_count_multi_tables')
|
|
||||||
table_list.append(table_name)
|
|
||||||
param = {'table_name': table_name,
|
|
||||||
'dimension': dim,
|
|
||||||
'index_file_size': index_file_size,
|
|
||||||
'metric_type': MetricType.HAMMING}
|
|
||||||
connect.create_table(param)
|
|
||||||
res = connect.add_vectors(table_name=table_name, records=vectors)
|
|
||||||
connect.flush(table_list)
|
|
||||||
for i in range(20):
|
|
||||||
status, res = connect.get_table_row_count(table_list[i])
|
|
||||||
assert status.OK()
|
|
||||||
assert res == nq
|
|
||||||
|
|
||||||
|
|
||||||
class TestTableCountTANIMOTO:
|
|
||||||
"""
|
|
||||||
params means different nb, the nb value may trigger merge, or not
|
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.fixture(
|
|
||||||
scope="function",
|
|
||||||
params=[
|
|
||||||
1,
|
|
||||||
5000,
|
|
||||||
100000,
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def add_vectors_nb(self, request):
|
|
||||||
yield request.param
|
|
||||||
|
|
||||||
"""
|
|
||||||
generate valid create_index params
|
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.fixture(
|
|
||||||
scope="function",
|
|
||||||
params=gen_simple_index()
|
|
||||||
)
|
|
||||||
def get_tanimoto_index(self, request, connect):
|
|
||||||
logging.getLogger().info(request.param)
|
|
||||||
if request.param["index_type"] == IndexType.IVFLAT or request.param["index_type"] == IndexType.FLAT:
|
|
||||||
return request.param
|
|
||||||
else:
|
|
||||||
pytest.skip("Skip index Temporary")
|
|
||||||
|
|
||||||
def test_table_rows_count(self, connect, tanimoto_table, add_vectors_nb):
|
|
||||||
'''
|
|
||||||
target: test table rows_count is correct or not
|
|
||||||
method: create table and add vectors in it,
|
|
||||||
assert the value returned by get_table_row_count method is equal to length of vectors
|
|
||||||
expected: the count is equal to the length of vectors
|
|
||||||
'''
|
|
||||||
nb = add_vectors_nb
|
|
||||||
tmp, vectors = gen_binary_vectors(nb, dim)
|
|
||||||
res = connect.add_vectors(table_name=tanimoto_table, records=vectors)
|
|
||||||
connect.flush([tanimoto_table])
|
|
||||||
status, res = connect.get_table_row_count(tanimoto_table)
|
|
||||||
assert status.OK()
|
|
||||||
assert res == nb
|
|
|
@ -8,7 +8,7 @@ from milvus import IndexType, MetricType
|
||||||
from utils import *
|
from utils import *
|
||||||
|
|
||||||
dim = 128
|
dim = 128
|
||||||
table_id = "test_wal"
|
collection_id = "test_wal"
|
||||||
WAL_TIMEOUT = 30
|
WAL_TIMEOUT = 30
|
||||||
nb = 6000
|
nb = 6000
|
||||||
add_interval = 1.5
|
add_interval = 1.5
|
||||||
|
@ -21,109 +21,109 @@ class TestWalBase:
|
||||||
******************************************************************
|
******************************************************************
|
||||||
"""
|
"""
|
||||||
@pytest.mark.timeout(WAL_TIMEOUT)
|
@pytest.mark.timeout(WAL_TIMEOUT)
|
||||||
def test_wal_add_vectors(self, connect, table):
|
def test_wal_add_vectors(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: add vectors in WAL
|
target: add vectors in WAL
|
||||||
method: add vectors and flush when WAL is enabled
|
method: add vectors and flush when WAL is enabled
|
||||||
expected: status ok, vectors added
|
expected: status ok, vectors added
|
||||||
'''
|
'''
|
||||||
vectors = gen_vector(nb, dim)
|
vectors = gen_vector(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_table_row_count(table)
|
status, res = connect.count_collection(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert res == nb
|
assert res == nb
|
||||||
status, res = connect.get_vector_by_id(table, ids[0])
|
status, res = connect.get_vector_by_id(collection, ids[0])
|
||||||
logging.getLogger().info(res)
|
logging.getLogger().info(res)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert_equal_vector(res, vectors[0])
|
assert_equal_vector(res, vectors[0])
|
||||||
|
|
||||||
@pytest.mark.timeout(WAL_TIMEOUT)
|
@pytest.mark.timeout(WAL_TIMEOUT)
|
||||||
def test_wal_delete_vectors(self, connect, table):
|
def test_wal_delete_vectors(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: delete vectors in WAL
|
target: delete vectors in WAL
|
||||||
method: delete vectors and flush when WAL is enabled
|
method: delete vectors and flush when WAL is enabled
|
||||||
expected: status ok, vectors deleted
|
expected: status ok, vectors deleted
|
||||||
'''
|
'''
|
||||||
vectors = gen_vector(nb, dim)
|
vectors = gen_vector(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
connect.flush([table])
|
connect.flush([collection])
|
||||||
status, res = connect.get_table_row_count(table)
|
status, res = connect.count_collection(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.delete_by_id(table, ids)
|
status = connect.delete_by_id(collection, ids)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_table_row_count(table)
|
status, res = connect.count_collection(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert res == 0
|
assert res == 0
|
||||||
|
|
||||||
@pytest.mark.timeout(WAL_TIMEOUT)
|
@pytest.mark.timeout(WAL_TIMEOUT)
|
||||||
def test_wal_invalid_operation(self, connect, table):
|
def test_wal_invalid_operation(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: invalid operation in WAL
|
target: invalid operation in WAL
|
||||||
method: add vectors, delete with non-existent ids and flush when WAL is enabled
|
method: add vectors, delete with non-existent ids and flush when WAL is enabled
|
||||||
expected: status ok, search with vector have result
|
expected: status ok, search with vector have result
|
||||||
'''
|
'''
|
||||||
vector = gen_single_vector(dim)
|
vector = gen_single_vector(dim)
|
||||||
status, ids = connect.add_vectors(table, vector)
|
status, ids = connect.add_vectors(collection, vector)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
connect.flush([table])
|
connect.flush([collection])
|
||||||
status = connect.delete_by_id(table, [0])
|
status = connect.delete_by_id(collection, [0])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
status, res = connect.get_table_row_count(table)
|
status, res = connect.count_collection(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert res == 1
|
assert res == 1
|
||||||
|
|
||||||
@pytest.mark.timeout(WAL_TIMEOUT)
|
@pytest.mark.timeout(WAL_TIMEOUT)
|
||||||
def test_wal_invalid_operation_B(self, connect, table):
|
def test_wal_invalid_operation_B(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: invalid operation in WAL
|
target: invalid operation in WAL
|
||||||
method: add vectors, delete with not existed table name when WAL is enabled
|
method: add vectors, delete with not existed collection name when WAL is enabled
|
||||||
expected: status not ok
|
expected: status not ok
|
||||||
'''
|
'''
|
||||||
vectors = gen_vector(nb, dim)
|
vectors = gen_vector(nb, dim)
|
||||||
status, ids = connect.add_vectors(table, vectors)
|
status, ids = connect.add_vectors(collection, vectors)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
status = connect.delete_by_id(table, [0])
|
status = connect.delete_by_id(collection, [0])
|
||||||
connect.flush([table])
|
connect.flush([collection])
|
||||||
table_new = gen_unique_str()
|
collection_new = gen_unique_str()
|
||||||
status = connect.delete_by_id(table_new, ids)
|
status = connect.delete_by_id(collection_new, ids)
|
||||||
assert not status.OK()
|
assert not status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_table_row_count(table)
|
status, res = connect.count_collection(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert res == nb
|
assert res == nb
|
||||||
|
|
||||||
@pytest.mark.timeout(WAL_TIMEOUT)
|
@pytest.mark.timeout(WAL_TIMEOUT)
|
||||||
def test_wal_server_crashed_recovery(self, connect, table):
|
def test_wal_server_crashed_recovery(self, connect, collection):
|
||||||
'''
|
'''
|
||||||
target: test wal when server crashed unexpectedly and restarted
|
target: test wal when server crashed unexpectedly and restarted
|
||||||
method: add vectors, server killed before flush, restarted server and flush
|
method: add vectors, server killed before flush, restarted server and flush
|
||||||
expected: status ok, add request is recovered and vectors added
|
expected: status ok, add request is recovered and vectors added
|
||||||
'''
|
'''
|
||||||
vector = gen_single_vector(dim)
|
vector = gen_single_vector(dim)
|
||||||
status, ids = connect.add_vectors(table, vector)
|
status, ids = connect.add_vectors(collection, vector)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
status, res = connect.get_table_row_count(table)
|
status, res = connect.count_collection(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
logging.getLogger().info(res) # should be 0 because no auto flush
|
logging.getLogger().info(res) # should be 0 because no auto flush
|
||||||
logging.getLogger().info("Stop server and restart")
|
logging.getLogger().info("Stop server and restart")
|
||||||
# kill server and restart. auto flush should be set to 15 seconds.
|
# kill server and restart. auto flush should be set to 15 seconds.
|
||||||
# time.sleep(15)
|
# time.sleep(15)
|
||||||
status = connect.flush([table])
|
status = connect.flush([collection])
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
status, res = connect.get_table_row_count(table)
|
status, res = connect.count_collection(collection)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert res == 1
|
assert res == 1
|
||||||
status, res = connect.get_vector_by_id(table, ids[0])
|
status, res = connect.get_vector_by_id(collection, ids[0])
|
||||||
logging.getLogger().info(res)
|
logging.getLogger().info(res)
|
||||||
assert status.OK()
|
assert status.OK()
|
||||||
assert_equal_vector(res, vector[0])
|
assert_equal_vector(res, vector[0])
|
||||||
|
|
|
@ -146,8 +146,8 @@ def gen_invalid_uris():
|
||||||
return uris
|
return uris
|
||||||
|
|
||||||
|
|
||||||
def gen_invalid_table_names():
|
def gen_invalid_collection_names():
|
||||||
table_names = [
|
collection_names = [
|
||||||
"12-s",
|
"12-s",
|
||||||
"12/s",
|
"12/s",
|
||||||
" ",
|
" ",
|
||||||
|
@ -166,7 +166,7 @@ def gen_invalid_table_names():
|
||||||
"中文",
|
"中文",
|
||||||
"a".join("a" for i in range(256))
|
"a".join("a" for i in range(256))
|
||||||
]
|
]
|
||||||
return table_names
|
return collection_names
|
||||||
|
|
||||||
|
|
||||||
def gen_invalid_top_ks():
|
def gen_invalid_top_ks():
|
||||||
|
@ -588,7 +588,7 @@ def gen_simple_index():
|
||||||
IndexType.IVF_SQ8H,
|
IndexType.IVF_SQ8H,
|
||||||
IndexType.IVF_PQ,
|
IndexType.IVF_PQ,
|
||||||
IndexType.HNSW,
|
IndexType.HNSW,
|
||||||
# IndexType.RNSG
|
IndexType.RNSG
|
||||||
]
|
]
|
||||||
params = [
|
params = [
|
||||||
{"nlist": 1024},
|
{"nlist": 1024},
|
||||||
|
@ -597,7 +597,7 @@ def gen_simple_index():
|
||||||
{"nlist": 1024},
|
{"nlist": 1024},
|
||||||
{"nlist": 1024, "m": 16},
|
{"nlist": 1024, "m": 16},
|
||||||
{"M": 16, "efConstruction": 500},
|
{"M": 16, "efConstruction": 500},
|
||||||
# {"search_length": 100, "out_degree": 40, "candidate_pool_size": 66, "knng": 100}
|
{"search_length": 50, "out_degree": 40, "candidate_pool_size": 100, "knng": 50}
|
||||||
]
|
]
|
||||||
|
|
||||||
index_params = []
|
index_params = []
|
||||||
|
@ -611,14 +611,14 @@ def get_search_param(index_type):
|
||||||
return {"nprobe": 32}
|
return {"nprobe": 32}
|
||||||
elif index_type == IndexType.HNSW:
|
elif index_type == IndexType.HNSW:
|
||||||
return {"ef": 64}
|
return {"ef": 64}
|
||||||
# elif index_type == IndexType.RNSG:
|
elif index_type == IndexType.RNSG:
|
||||||
# return {"search_length": 100}
|
return {"search_length": 50}
|
||||||
else:
|
else:
|
||||||
logging.getLogger().info("Invalid index_type.")
|
logging.getLogger().info("Invalid index_type.")
|
||||||
|
|
||||||
|
|
||||||
def assert_has_table(conn, table_name):
|
def assert_has_collection(conn, collection_name):
|
||||||
status, ok = conn.has_table(table_name)
|
status, ok = conn.has_collection(collection_name)
|
||||||
return status.OK() and ok
|
return status.OK() and ok
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue