milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

test: modify search iterator V2 case about alias/db/rbac for milvus client (#39400) 

issue: #37548

Signed-off-by: qixuan <673771573@qq.com>
pull/38303/merge
qixuan 2025-01-20 10:25:03 +08:00 committed by GitHub
parent fdb968d0ea
commit 228bff5e7b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 110 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tests/python_client/base/client_v2_base.py
View File

@ -154,8 +154,8 @@ class TestMilvusClientV2Base(Base):
return res, check_result return res, check_result
@trace() @trace()
def search_interator(self, client, collection_name, data, batch_size=20, limit=100, filter=None, output_fields=None, def search_iterator(self, client, collection_name, data, batch_size, limit=-1, filter=None, output_fields=None,
search_params=None, timeout=None, check_task=None, check_items=None, **kwargs): search_params=None, timeout=None, check_task=None, check_items=None, **kwargs):
timeout = TIMEOUT if timeout is None else timeout timeout = TIMEOUT if timeout is None else timeout
kwargs.update({"timeout": timeout}) kwargs.update({"timeout": timeout})
func_name = sys._getframe().f_code.co_name func_name = sys._getframe().f_code.co_name

44
tests/python_client/check/func_check.py
View File

@ -429,26 +429,30 @@ class ResponseChecker:
search_iterator = search_res search_iterator = search_res
pk_list = [] pk_list = []
while True: while True:
res = search_iterator.next() try:
if len(res) == 0: res = search_iterator.next()
log.info("search iteration finished, close") if len(res) == 0:
search_iterator.close() log.info("search iteration finished, close")
break search_iterator.close()
if check_items.get("batch_size", None): break
assert len(res) <= check_items["batch_size"] if check_items.get("batch_size", None):
if check_items.get("radius", None): assert len(res) <= check_items["batch_size"]
for distance in res.distances(): if check_items.get("radius", None):
if check_items["metric_type"] == "L2": for distance in res.distances():
assert distance < check_items["radius"] if check_items["metric_type"] == "L2":
else: assert distance < check_items["radius"]
assert distance > check_items["radius"] else:
if check_items.get("range_filter", None): assert distance > check_items["radius"]
for distance in res.distances(): if check_items.get("range_filter", None):
if check_items["metric_type"] == "L2": for distance in res.distances():
assert distance >= check_items["range_filter"] if check_items["metric_type"] == "L2":
else: assert distance >= check_items["range_filter"]
assert distance <= check_items["range_filter"] else:
pk_list.extend(res.ids()) assert distance <= check_items["range_filter"]
pk_list.extend(res.ids())
except Exception as e:
assert check_items["err_msg"] in str(e)
return False
assert len(pk_list) == len(set(pk_list)) assert len(pk_list) == len(set(pk_list))
log.info("check: total %d results" % len(pk_list)) log.info("check: total %d results" % len(pk_list))

17
tests/python_client/milvus_client/test_milvus_client_rbac.py
View File

@ -597,9 +597,9 @@ class TestMilvusClientRbacAdvance(TestMilvusClientV2Base):
roles, _ = self.list_roles(client) roles, _ = self.list_roles(client)
for role in roles: for role in roles:
if role not in ['admin', 'public']: if role not in ['admin', 'public']:
res, _ = self.describe_role(client, role) role_info, _ = self.describe_role(client, role)
if res['privileges']: if role_info:
for privilege in res['privileges']: for privilege in role_info.get("privileges", []):
self.revoke_privilege(client, role, privilege["object_type"], self.revoke_privilege(client, role, privilege["object_type"],
privilege["privilege"], privilege["object_name"]) privilege["privilege"], privilege["object_name"])
self.drop_role(client, role) self.drop_role(client, role)
@ -613,8 +613,9 @@ class TestMilvusClientRbacAdvance(TestMilvusClientV2Base):
""" """
target: test search iterator(high level api) normal case about mul db by rbac target: test search iterator(high level api) normal case about mul db by rbac
method: create connection, collection, insert and search iterator method: create connection, collection, insert and search iterator
expected: search iterator permission deny after switch to no permission db expected: search iterator permission deny after switch to no permission db
""" """
batch_size = 20
uri = f"http://{cf.param_info.param_host}:{cf.param_info.param_port}" uri = f"http://{cf.param_info.param_host}:{cf.param_info.param_port}"
client, _ = self.init_milvus_client(uri=uri, token="root:Milvus") client, _ = self.init_milvus_client(uri=uri, token="root:Milvus")
my_db = cf.gen_unique_str(prefix) my_db = cf.gen_unique_str(prefix)
@ -647,14 +648,10 @@ class TestMilvusClientRbacAdvance(TestMilvusClientV2Base):
# 5. search_iterator # 5. search_iterator
vectors_to_search = rng.random((1, default_dim)) vectors_to_search = rng.random((1, default_dim))
self.search_interator(client, collection_name, vectors_to_search, use_rbac_mul_db=True, another_db=my_db, self.search_iterator(client, collection_name, vectors_to_search, batch_size, use_rbac_mul_db=True, another_db=my_db,
check_task=CheckTasks.check_permission_deny) check_task=CheckTasks.check_permission_deny)
client, _ = self.init_milvus_client(uri=uri, token="root:Milvus") client, _ = self.init_milvus_client(uri=uri, token="root:Milvus")
self.revoke_privilege(client, role_name, "Collection", "Search", collection_name, 'default')
self.revoke_privilege(client, role_name, "Collection", "Insert", collection_name, my_db) self.revoke_privilege(client, role_name, "Collection", "Insert", collection_name, my_db)
self.revoke_role(client, user_name=user_name, role_name=role_name)
self.release_collection(client, collection_name)
self.drop_collection(client, collection_name) self.drop_collection(client, collection_name)
self.using_database(client, 'default') self.using_database(client, 'default')
self.release_collection(client, collection_name)
self.drop_collection(client, collection_name) self.drop_collection(client, collection_name)

146
tests/python_client/milvus_client/test_milvus_client_search_interator.py
View File

@ -42,13 +42,14 @@ class TestMilvusClientSearchInteratorInValid(TestMilvusClientV2Base):
yield request.param yield request.param
@pytest.mark.tags(CaseLabel.L1) @pytest.mark.tags(CaseLabel.L1)
@pytest.mark.skip("https://github.com/milvus-io/milvus/issues/39045") # @pytest.mark.skip("https://github.com/milvus-io/milvus/issues/39045")
def test_milvus_client_search_iterator_using_mul_db(self, search_params): def test_milvus_client_search_iterator_using_mul_db(self, search_params):
""" """
target: test search iterator(high level api) case about mul db target: test search iterator(high level api) case about mul db
method: create connection, collection, insert and search iterator method: create connection, collection, insert and search iterator
expected: search iterator error after switch to another db expected: search iterator error after switch to another db
""" """
batch_size = 20
client = self._client() client = self._client()
collection_name = cf.gen_unique_str(prefix) collection_name = cf.gen_unique_str(prefix)
my_db = cf.gen_unique_str(prefix) my_db = cf.gen_unique_str(prefix)
@ -75,21 +76,23 @@ class TestMilvusClientSearchInteratorInValid(TestMilvusClientV2Base):
# 5. search_iterator # 5. search_iterator
vectors_to_search = rng.random((1, default_dim)) vectors_to_search = rng.random((1, default_dim))
search_params = {"params": search_params} search_params = {"params": search_params}
self.search_interator(client, collection_name, vectors_to_search, search_params=search_params, error_msg = "alias or database may have been changed"
use_mul_db=True, another_db=my_db, self.search_iterator(client, collection_name, vectors_to_search, batch_size, search_params=search_params,
check_task=CheckTasks.err_res, use_mul_db=True, another_db=my_db,
check_items={}) check_task=CheckTasks.check_search_iterator,
check_items={ct.err_code: 1, ct.err_msg: error_msg})
self.release_collection(client, collection_name) self.release_collection(client, collection_name)
self.drop_collection(client, collection_name) self.drop_collection(client, collection_name)
@pytest.mark.tags(CaseLabel.L1) @pytest.mark.tags(CaseLabel.L1)
@pytest.mark.skip("https://github.com/milvus-io/milvus/issues/39087") # @pytest.mark.skip("https://github.com/milvus-io/milvus/issues/39087")
def test_milvus_client_search_iterator_alias_different_col(self, search_params): def test_milvus_client_search_iterator_alias_different_col(self, search_params):
""" """
target: test search iterator(high level api) case about alias target: test search iterator(high level api) case about alias
method: create connection, collection, insert and search iterator method: create connection, collection, insert and search iterator
expected: search iterator error after alter alias expected: search iterator error after alter alias
""" """
batch_size = 20
client = self._client() client = self._client()
collection_name = cf.gen_unique_str(prefix) collection_name = cf.gen_unique_str(prefix)
collection_name_new = cf.gen_unique_str(prefix) collection_name_new = cf.gen_unique_str(prefix)
@ -114,11 +117,11 @@ class TestMilvusClientSearchInteratorInValid(TestMilvusClientV2Base):
# 3. search_iterator # 3. search_iterator
vectors_to_search = rng.random((1, default_dim)) vectors_to_search = rng.random((1, default_dim))
search_params = {"params": search_params} search_params = {"params": search_params}
error_msg = "" # todo error_msg = "alias or database may have been changed"
self.search_interator(client, alias, vectors_to_search, search_params=search_params, self.search_iterator(client, alias, vectors_to_search, batch_size, search_params=search_params,
use_alias=True, another_collection=collection_name_new, use_alias=True, another_collection=collection_name_new,
check_task=CheckTasks.err_res, check_task=CheckTasks.check_search_iterator,
check_items={ct.err_code: 1, ct.err_msg: error_msg}) check_items={ct.err_code: 1, ct.err_msg: error_msg})
self.release_collection(client, collection_name) self.release_collection(client, collection_name)
self.drop_collection(client, collection_name) self.drop_collection(client, collection_name)
self.release_collection(client, collection_name_new) self.release_collection(client, collection_name_new)
@ -153,6 +156,7 @@ class TestMilvusClientSearchInteratorValid(TestMilvusClientV2Base):
method: create connection, collection, insert and search iterator method: create connection, collection, insert and search iterator
expected: search iterator successfully expected: search iterator successfully
""" """
batch_size = 20
client = self._client() client = self._client()
collection_name = cf.gen_unique_str(prefix) collection_name = cf.gen_unique_str(prefix)
self.using_database(client, "default") self.using_database(client, "default")
@ -175,12 +179,12 @@ class TestMilvusClientSearchInteratorValid(TestMilvusClientV2Base):
vectors_to_search = rng.random((1, default_dim)) vectors_to_search = rng.random((1, default_dim))
insert_ids = [i for i in range(default_nb)] insert_ids = [i for i in range(default_nb)]
search_params = {"params": search_params} search_params = {"params": search_params}
self.search_interator(client, collection_name, vectors_to_search, search_params=search_params, self.search_iterator(client, collection_name, vectors_to_search, batch_size, search_params=search_params,
check_task=CheckTasks.check_search_iterator, check_task=CheckTasks.check_search_iterator,
check_items={"enable_milvus_client_api": True, check_items={"enable_milvus_client_api": True,
"nq": len(vectors_to_search), "nq": len(vectors_to_search),
"ids": insert_ids, "ids": insert_ids,
"limit": default_limit}) "limit": default_limit})
self.release_collection(client, collection_name) self.release_collection(client, collection_name)
self.drop_collection(client, collection_name) self.drop_collection(client, collection_name)
@ -192,6 +196,7 @@ class TestMilvusClientSearchInteratorValid(TestMilvusClientV2Base):
method: create connection, collection, insert and search iterator method: create connection, collection, insert and search iterator
expected: search iterator successfully expected: search iterator successfully
""" """
batch_size = 20
client = self._client() client = self._client()
collection_name = cf.gen_unique_str(prefix) collection_name = cf.gen_unique_str(prefix)
dim = 128 dim = 128
@ -218,13 +223,13 @@ class TestMilvusClientSearchInteratorValid(TestMilvusClientV2Base):
vectors_to_search = rng.random((1, default_dim)) vectors_to_search = rng.random((1, default_dim))
insert_ids = [i for i in range(default_nb)] insert_ids = [i for i in range(default_nb)]
search_params = {"params": search_params} search_params = {"params": search_params}
self.search_interator(client, collection_name, vectors_to_search, filter="nullable_field>=10", self.search_iterator(client, collection_name, vectors_to_search, batch_size, filter="nullable_field>=10",
search_params=search_params, search_params=search_params,
check_task=CheckTasks.check_search_iterator, check_task=CheckTasks.check_search_iterator,
check_items={"enable_milvus_client_api": True, check_items={"enable_milvus_client_api": True,
"nq": len(vectors_to_search), "nq": len(vectors_to_search),
"ids": insert_ids, "ids": insert_ids,
"limit": default_limit}) "limit": default_limit})
if self.has_collection(client, collection_name)[0]: if self.has_collection(client, collection_name)[0]:
self.drop_collection(client, collection_name) self.drop_collection(client, collection_name)
@ -235,6 +240,7 @@ class TestMilvusClientSearchInteratorValid(TestMilvusClientV2Base):
method: create connection, collection, insert and search iterator method: create connection, collection, insert and search iterator
expected: search iterator successfully expected: search iterator successfully
""" """
batch_size = 20
client = self._client() client = self._client()
collection_name = cf.gen_unique_str(prefix) collection_name = cf.gen_unique_str(prefix)
# 1. create collection # 1. create collection
@ -260,12 +266,12 @@ class TestMilvusClientSearchInteratorValid(TestMilvusClientV2Base):
vectors_to_search = rng.random((1, default_dim)) vectors_to_search = rng.random((1, default_dim))
insert_ids = [i for i in range(default_nb)] insert_ids = [i for i in range(default_nb)]
search_params = {"params": search_params} search_params = {"params": search_params}
self.search_interator(client, new_name, vectors_to_search, search_params=search_params, self.search_iterator(client, new_name, vectors_to_search, batch_size, search_params=search_params,
check_task=CheckTasks.check_search_iterator, check_task=CheckTasks.check_search_iterator,
check_items={"enable_milvus_client_api": True, check_items={"enable_milvus_client_api": True,
"nq": len(vectors_to_search), "nq": len(vectors_to_search),
"ids": insert_ids, "ids": insert_ids,
"limit": default_limit}) "limit": default_limit})
self.release_collection(client, new_name) self.release_collection(client, new_name)
self.drop_collection(client, new_name) self.drop_collection(client, new_name)
@ -276,6 +282,7 @@ class TestMilvusClientSearchInteratorValid(TestMilvusClientV2Base):
method: create connection, collection, insert and search iterator method: create connection, collection, insert and search iterator
expected: search iterator successfully expected: search iterator successfully
""" """
batch_size = 20
client = self._client() client = self._client()
collection_name = cf.gen_unique_str(prefix) collection_name = cf.gen_unique_str(prefix)
# 1. create collection # 1. create collection
@ -296,12 +303,12 @@ class TestMilvusClientSearchInteratorValid(TestMilvusClientV2Base):
vectors_to_search = rng.random((1, default_dim)) vectors_to_search = rng.random((1, default_dim))
insert_ids = [i for i in range(default_nb)] insert_ids = [i for i in range(default_nb)]
search_params = {"params": search_params} search_params = {"params": search_params}
self.search_interator(client, collection_name, vectors_to_search, search_params=search_params, self.search_iterator(client, collection_name, vectors_to_search, batch_size, search_params=search_params,
check_task=CheckTasks.check_search_iterator, check_task=CheckTasks.check_search_iterator,
check_items={"enable_milvus_client_api": True, check_items={"enable_milvus_client_api": True,
"nq": len(vectors_to_search), "nq": len(vectors_to_search),
"ids": insert_ids, "ids": insert_ids,
"limit": default_limit}) "limit": default_limit})
@pytest.mark.tags(CaseLabel.L2) @pytest.mark.tags(CaseLabel.L2)
def test_milvus_client_search_iterator_string(self, search_params): def test_milvus_client_search_iterator_string(self, search_params):
@ -310,6 +317,7 @@ class TestMilvusClientSearchInteratorValid(TestMilvusClientV2Base):
method: create connection, collection, insert and search iterator method: create connection, collection, insert and search iterator
expected: search iterator successfully expected: search iterator successfully
""" """
batch_size = 20
client = self._client() client = self._client()
collection_name = cf.gen_unique_str(prefix) collection_name = cf.gen_unique_str(prefix)
# 1. create collection # 1. create collection
@ -324,11 +332,11 @@ class TestMilvusClientSearchInteratorValid(TestMilvusClientV2Base):
# 3. search_iterator # 3. search_iterator
vectors_to_search = rng.random((1, default_dim)) vectors_to_search = rng.random((1, default_dim))
search_params = {"params": search_params} search_params = {"params": search_params}
self.search_interator(client, collection_name, vectors_to_search, search_params=search_params, self.search_iterator(client, collection_name, vectors_to_search, batch_size, search_params=search_params,
check_task=CheckTasks.check_search_iterator, check_task=CheckTasks.check_search_iterator,
check_items={"enable_milvus_client_api": True, check_items={"enable_milvus_client_api": True,
"nq": len(vectors_to_search), "nq": len(vectors_to_search),
"limit": default_limit}) "limit": default_limit})
self.drop_collection(client, collection_name) self.drop_collection(client, collection_name)
@pytest.mark.tags(CaseLabel.L2) @pytest.mark.tags(CaseLabel.L2)
@ -355,13 +363,13 @@ class TestMilvusClientSearchInteratorValid(TestMilvusClientV2Base):
# 3. search_iterator # 3. search_iterator
vectors_to_search = rng.random((1, default_dim)) vectors_to_search = rng.random((1, default_dim))
search_params = {"params": search_params} search_params = {"params": search_params}
self.search_interator(client, collection_name, vectors_to_search, batch_size=default_batch_size, self.search_iterator(client, collection_name, vectors_to_search, batch_size=default_batch_size,
limit=default_limit, search_params=search_params, limit=default_limit, search_params=search_params,
output_fields=[default_primary_key_field_name], output_fields=[default_primary_key_field_name],
check_task=CheckTasks.check_search_iterator, check_task=CheckTasks.check_search_iterator,
check_items={"enable_milvus_client_api": True, check_items={"enable_milvus_client_api": True,
"nq": len(vectors_to_search), "nq": len(vectors_to_search),
"limit": default_limit}) "limit": default_limit})
self.drop_collection(client, collection_name) self.drop_collection(client, collection_name)
@pytest.mark.tags(CaseLabel.L2) @pytest.mark.tags(CaseLabel.L2)
@ -389,13 +397,13 @@ class TestMilvusClientSearchInteratorValid(TestMilvusClientV2Base):
vectors_to_search = rng.random((1, default_dim)) vectors_to_search = rng.random((1, default_dim))
search_params = {"params": search_params} search_params = {"params": search_params}
search_params.update({"metric_type": metric_type}) search_params.update({"metric_type": metric_type})
self.search_interator(client, collection_name, vectors_to_search, batch_size=default_batch_size, self.search_iterator(client, collection_name, vectors_to_search, batch_size=default_batch_size,
limit=default_limit, search_params=search_params, limit=default_limit, search_params=search_params,
output_fields=[default_primary_key_field_name], output_fields=[default_primary_key_field_name],
check_task=CheckTasks.check_search_iterator, check_task=CheckTasks.check_search_iterator,
check_items={"enable_milvus_client_api": True, check_items={"enable_milvus_client_api": True,
"nq": len(vectors_to_search), "nq": len(vectors_to_search),
"limit": default_limit}) "limit": default_limit})
self.drop_collection(client, collection_name) self.drop_collection(client, collection_name)
@pytest.mark.tags(CaseLabel.L1) @pytest.mark.tags(CaseLabel.L1)
@ -426,13 +434,13 @@ class TestMilvusClientSearchInteratorValid(TestMilvusClientV2Base):
insert_ids.remove(insert_id) insert_ids.remove(insert_id)
limit = default_nb - delete_num limit = default_nb - delete_num
search_params = {"params": search_params} search_params = {"params": search_params}
self.search_interator(client, collection_name, vectors_to_search, batch_size=default_batch_size, self.search_iterator(client, collection_name, vectors_to_search, batch_size=default_batch_size,
search_params=search_params, limit=default_nb, search_params=search_params, limit=default_nb,
check_task=CheckTasks.check_search_iterator, check_task=CheckTasks.check_search_iterator,
check_items={"enable_milvus_client_api": True, check_items={"enable_milvus_client_api": True,
"nq": len(vectors_to_search), "nq": len(vectors_to_search),
"ids": insert_ids, "ids": insert_ids,
"limit": limit}) "limit": limit})
self.drop_collection(client, collection_name) self.drop_collection(client, collection_name)
@pytest.mark.tags(CaseLabel.L1) @pytest.mark.tags(CaseLabel.L1)
@ -463,13 +471,13 @@ class TestMilvusClientSearchInteratorValid(TestMilvusClientV2Base):
insert_ids.remove(insert_id) insert_ids.remove(insert_id)
limit = default_nb - delete_num limit = default_nb - delete_num
search_params = {"params": search_params} search_params = {"params": search_params}
self.search_interator(client, collection_name, vectors_to_search, batch_size=default_batch_size, self.search_iterator(client, collection_name, vectors_to_search, batch_size=default_batch_size,
search_params=search_params, limit=default_nb, search_params=search_params, limit=default_nb,
check_task=CheckTasks.check_search_iterator, check_task=CheckTasks.check_search_iterator,
check_items={"enable_milvus_client_api": True, check_items={"enable_milvus_client_api": True,
"nq": len(vectors_to_search), "nq": len(vectors_to_search),
"ids": insert_ids, "ids": insert_ids,
"limit": limit}) "limit": limit})
# 5. query # 5. query
self.query(client, collection_name, filter=default_search_exp, self.query(client, collection_name, filter=default_search_exp,
check_task=CheckTasks.check_query_results, check_task=CheckTasks.check_query_results,