mirror of https://github.com/milvus-io/milvus.git
add cases for collection with data (#5305)
- add cases for collection with data - cancel import * See also: #5302 #5304 Signed-off-by: ThreadDao yufen.zong@zilliz.compull/5313/head
ThreadDao
GitHub
parent
1c779f3efd
commit
613cf21950
|
|
@ -9,8 +9,8 @@ from base.index import ApiIndex
|
|||
from base.utility import ApiUtility
|
||||
|
||||
from config.test_info import test_info
|
||||
from common.common_func import *
|
||||
from check.func_check import *
|
||||
from utils.util_log import test_log as log
|
||||
from common import common_func as cf
|
||||
|
||||
|
||||
def request_catch():
|
||||
|
|
@ -82,7 +82,7 @@ class Base:
|
|||
@pytest.fixture(scope="module", autouse=True)
|
||||
def initialize_env(self, request):
|
||||
""" clean log before testing """
|
||||
modify_file([test_info.log_info, test_info.log_err])
|
||||
cf.modify_file([test_info.log_info, test_info.log_err])
|
||||
log.info("[initialize_milvus] Log cleaned up, start testing...")
|
||||
|
||||
host = request.config.getoption("--host")
|
||||
|
|
@ -103,8 +103,10 @@ class ApiReq(Base):
|
|||
res = self.connection.create_connection(alias='default')
|
||||
return res
|
||||
|
||||
def _collection(self, name=gen_unique_str(), data=None, schema=None, check_res=None, **kwargs):
|
||||
def _collection(self, name=None, data=None, schema=None, check_res=None, **kwargs):
|
||||
""" Testing func """
|
||||
self._connect()
|
||||
res = self.collection.collection_init(name=name, data=data, schema=schema, check_res=check_res, **kwargs)
|
||||
return res
|
||||
name = cf.gen_unique_str("ApiReq") if name is None else name
|
||||
schema = cf.gen_default_collection_schema() if schema is None else schema
|
||||
collection = self.collection.collection_init(name=name, data=data, schema=schema, check_res=check_res, **kwargs)
|
||||
return name, collection
|
||||
|
|
|
|||
|
|
@ -6,11 +6,7 @@ from sklearn import preprocessing
|
|||
|
||||
from pymilvus_orm.types import DataType
|
||||
from pymilvus_orm.schema import CollectionSchema, FieldSchema
|
||||
|
||||
from common.common_type import int_field_desc, float_field_desc, float_vec_field_desc, binary_vec_field_desc
|
||||
from common.common_type import default_nb, all_index_types, binary_support, default_index_params
|
||||
from common.common_type import default_int64_field, default_float_field, default_float_vec_field_name, default_binary_vec_field_name
|
||||
from common.common_type import default_dim, collection_desc, default_collection_desc, default_binary_desc
|
||||
from common import common_type as ct
|
||||
from utils.util_log import test_log as log
|
||||
|
||||
"""" Methods of processing data """
|
||||
|
|
@ -22,54 +18,81 @@ def gen_unique_str(str_value=None):
|
|||
return "test_" + prefix if str_value is None else str_value + "_" + prefix
|
||||
|
||||
|
||||
def gen_int64_field(is_primary=False, description=int_field_desc):
|
||||
int64_field = FieldSchema(name=default_int64_field, dtype=DataType.INT64, description=description,
|
||||
def gen_int64_field(is_primary=False, description=ct.int_field_desc):
|
||||
int64_field = FieldSchema(name=ct.default_int64_field, dtype=DataType.INT64, description=description,
|
||||
is_primary=is_primary)
|
||||
return int64_field
|
||||
|
||||
|
||||
def gen_float_field(is_primary=False, description=float_field_desc):
|
||||
float_field = FieldSchema(name=default_float_field, dtype=DataType.FLOAT, description=description,
|
||||
def gen_float_field(is_primary=False, description=ct.float_field_desc):
|
||||
float_field = FieldSchema(name=ct.default_float_field, dtype=DataType.FLOAT, description=description,
|
||||
is_primary=is_primary)
|
||||
return float_field
|
||||
|
||||
|
||||
def gen_float_vec_field(is_primary=False, dim=default_dim, description=float_vec_field_desc):
|
||||
float_vec_field = FieldSchema(name=default_float_vec_field_name, dtype=DataType.FLOAT_VECTOR,
|
||||
def gen_float_vec_field(is_primary=False, dim=ct.default_dim, description=ct.float_vec_field_desc):
|
||||
float_vec_field = FieldSchema(name=ct.default_float_vec_field_name, dtype=DataType.FLOAT_VECTOR,
|
||||
description=description, dim=dim, is_primary=is_primary)
|
||||
return float_vec_field
|
||||
|
||||
|
||||
def gen_binary_vec_field(is_primary=False, dim=default_dim, description=binary_vec_field_desc):
|
||||
binary_vec_field = FieldSchema(name=default_binary_vec_field_name, dtype=DataType.BINARY_VECTOR,
|
||||
def gen_binary_vec_field(is_primary=False, dim=ct.default_dim, description=ct.binary_vec_field_desc):
|
||||
binary_vec_field = FieldSchema(name=ct.default_binary_vec_field_name, dtype=DataType.BINARY_VECTOR,
|
||||
description=description, dim=dim, is_primary=is_primary)
|
||||
return binary_vec_field
|
||||
|
||||
|
||||
def gen_default_collection_schema(description=default_collection_desc, primary_field=None):
|
||||
def gen_default_collection_schema(description=ct.default_collection_desc, primary_field=None):
|
||||
fields = [gen_int64_field(), gen_float_field(), gen_float_vec_field()]
|
||||
schema = CollectionSchema(fields=fields, description=description, primary_field=primary_field)
|
||||
return schema
|
||||
|
||||
|
||||
def gen_collection_schema(fields, description=collection_desc, **kwargs):
|
||||
def gen_collection_schema(fields, description=ct.collection_desc, **kwargs):
|
||||
schema = CollectionSchema(fields=fields, description=description, **kwargs)
|
||||
return schema
|
||||
|
||||
|
||||
def gen_default_binary_collection_schema(description=default_binary_desc, primary_field=None):
|
||||
def gen_default_binary_collection_schema(description=ct.default_binary_desc, primary_field=None):
|
||||
fields = [gen_int64_field(), gen_float_field(), gen_binary_vec_field()]
|
||||
binary_schema = CollectionSchema(fields=fields, description=description, primary_field=primary_field)
|
||||
return binary_schema
|
||||
|
||||
|
||||
def gen_vectors(nb, dim):
|
||||
vectors = [[random.random() for _ in range(dim)] for _ in range(nb)]
|
||||
vectors = preprocessing.normalize(vectors, axis=1, norm='l2')
|
||||
return vectors.tolist()
|
||||
|
||||
|
||||
def gen_default_dataframe_data(nb=ct.default_nb):
|
||||
import pandas as pd
|
||||
int_values = pd.Series(data=[i for i in range(nb)])
|
||||
float_values = pd.Series(data=[float(i) for i in range(nb)], dtype="float32")
|
||||
float_vec_values = gen_vectors(nb, ct.default_dim)
|
||||
df = pd.DataFrame({
|
||||
ct.default_int64_field: int_values,
|
||||
ct.default_float_field: float_values,
|
||||
ct.default_float_vec_field_name: float_vec_values
|
||||
})
|
||||
return df
|
||||
|
||||
|
||||
def gen_default_list_data(nb=ct.default_nb):
|
||||
int_values = [i for i in range(nb)]
|
||||
float_values = [float(i) for i in range(nb)]
|
||||
float_vec_values = gen_vectors(nb, ct.default_dim)
|
||||
data = [int_values, float_values, float_vec_values]
|
||||
return data
|
||||
|
||||
|
||||
def gen_simple_index():
|
||||
index_params = []
|
||||
for i in range(len(all_index_types)):
|
||||
if all_index_types[i] in binary_support:
|
||||
for i in range(len(ct.all_index_types)):
|
||||
if ct.all_index_types[i] in ct.binary_support:
|
||||
continue
|
||||
dic = {"index_type": all_index_types[i], "metric_type": "L2"}
|
||||
dic.update({"params": default_index_params[i]})
|
||||
dic = {"index_type": ct.all_index_types[i], "metric_type": "L2"}
|
||||
dic.update({"params": ct.default_index_params[i]})
|
||||
index_params.append(dic)
|
||||
return index_params
|
||||
|
||||
|
|
@ -80,16 +103,6 @@ def get_vectors(num, dim, is_normal=True):
|
|||
return vectors.tolist()
|
||||
|
||||
|
||||
def get_entities(nb=default_nb, is_normal=False):
|
||||
vectors = get_vectors(nb, default_dim, is_normal)
|
||||
entities = [
|
||||
{"name": "int64", "type": DataType.INT64, "values": [i for i in range(nb)]},
|
||||
{"name": "float", "type": DataType.FLOAT, "values": [float(i) for i in range(nb)]},
|
||||
{"name": default_float_vec_field_name, "type": DataType.FLOAT_VECTOR, "values": vectors}
|
||||
]
|
||||
return entities
|
||||
|
||||
|
||||
def modify_file(file_name_list, input_content=""):
|
||||
if not isinstance(file_name_list, list):
|
||||
log.error("[modify_file] file is not a list.")
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ default_flush_interval = 1
|
|||
big_flush_interval = 1000
|
||||
default_drop_interval = 3
|
||||
default_dim = 128
|
||||
default_nb = 3000
|
||||
default_nb = 1200
|
||||
default_top_k = 10
|
||||
max_top_k = 16384
|
||||
max_partition_num = 4096 # 256
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
import pytest
|
||||
from base.client_request import ApiReq
|
||||
from common.common_type import get_invalid_strs, CaseLabel, default_int64_field, default_alias
|
||||
from utils.util_log import test_log as log
|
||||
from common.common_func import gen_default_collection_schema, gen_unique_str, gen_collection_schema
|
||||
from common.common_func import gen_int64_field, gen_float_vec_field
|
||||
from common import common_func as cf
|
||||
from common import common_type as ct
|
||||
from common.common_type import CaseLabel
|
||||
|
||||
prefix = "collection"
|
||||
default_schema = gen_default_collection_schema()
|
||||
default_schema = cf.gen_default_collection_schema()
|
||||
|
||||
|
||||
def assert_default_collection(collection, exp_name, exp_schema=default_schema, exp_num=0, exp_primary=None):
|
||||
|
|
@ -34,16 +34,16 @@ class TestCollectionParams(ApiReq):
|
|||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=get_invalid_strs
|
||||
params=ct.get_invalid_strs
|
||||
)
|
||||
def get_invalid_string(self, request):
|
||||
yield request.param
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=get_invalid_strs
|
||||
params=ct.get_invalid_strs
|
||||
)
|
||||
def get_invalid_schema(self, request):
|
||||
def get_invalid_schema_type(self, request):
|
||||
if request.param is None:
|
||||
pytest.skip("None schema is valid")
|
||||
yield request.param
|
||||
|
|
@ -57,7 +57,7 @@ class TestCollectionParams(ApiReq):
|
|||
expected: assert collection property
|
||||
"""
|
||||
self._connect()
|
||||
c_name = gen_unique_str(prefix)
|
||||
c_name = cf.gen_unique_str(prefix)
|
||||
collection, _ = self.collection.collection_init(c_name, data=None, schema=default_schema)
|
||||
assert_default_collection(collection, c_name)
|
||||
assert c_name in self.utility.list_collections()
|
||||
|
|
@ -95,8 +95,7 @@ class TestCollectionParams(ApiReq):
|
|||
expected: collection properties consistent
|
||||
"""
|
||||
self._connect()
|
||||
c_name = gen_unique_str(prefix)
|
||||
collection, _ = self.collection.collection_init(c_name, schema=default_schema)
|
||||
c_name, collection = self._collection()
|
||||
assert_default_collection(collection, c_name)
|
||||
dup_collection, _ = self.collection.collection_init(c_name)
|
||||
assert_default_collection(dup_collection, c_name)
|
||||
|
|
@ -114,11 +113,10 @@ class TestCollectionParams(ApiReq):
|
|||
expected: raise exception
|
||||
"""
|
||||
self._connect()
|
||||
c_name = gen_unique_str(prefix)
|
||||
collection, _ = self.collection.collection_init(c_name, data=None, schema=default_schema)
|
||||
c_name, collection = self._collection()
|
||||
assert_default_collection(collection, c_name)
|
||||
fields = [gen_int64_field()]
|
||||
schema = gen_collection_schema(fields=fields)
|
||||
fields = [cf.gen_int64_field()]
|
||||
schema = cf.gen_collection_schema(fields=fields)
|
||||
ex, _ = self.collection.collection_init(c_name, schema=schema)
|
||||
assert "The collection already exist, but the schema isnot the same as the passed in" in str(ex)
|
||||
|
||||
|
|
@ -131,10 +129,9 @@ class TestCollectionParams(ApiReq):
|
|||
expected: raise exception
|
||||
"""
|
||||
self._connect()
|
||||
c_name = gen_unique_str(prefix)
|
||||
collection, _ = self.collection.collection_init(c_name, schema=default_schema)
|
||||
c_name, collection = self._collection()
|
||||
assert_default_collection(collection, c_name)
|
||||
schema = gen_default_collection_schema(primary_field=default_int64_field)
|
||||
schema = cf.gen_default_collection_schema(primary_field=ct.default_int64_field)
|
||||
ex, _ = self.collection.collection_init(c_name, schema=schema)
|
||||
assert "The collection already exist, but the schema isnot the same as the passed in" in str(ex)
|
||||
assert collection.primary_field is None
|
||||
|
|
@ -147,29 +144,28 @@ class TestCollectionParams(ApiReq):
|
|||
expected: raise exception
|
||||
"""
|
||||
self._connect()
|
||||
c_name = gen_unique_str(prefix)
|
||||
new_dim = 120
|
||||
collection, _ = self.collection.collection_init(c_name, schema=default_schema)
|
||||
c_name, collection = self._collection()
|
||||
assert_default_collection(collection, c_name)
|
||||
schema = gen_default_collection_schema()
|
||||
new_fields = gen_float_vec_field(dim=new_dim)
|
||||
schema = cf.gen_default_collection_schema()
|
||||
new_fields = cf.gen_float_vec_field(dim=new_dim)
|
||||
schema.fields[-1] = new_fields
|
||||
ex, _ = self.collection.collection_init(c_name, schema=schema)
|
||||
assert "The collection already exist, but the schema isnot the same as the passed in" in str(ex)
|
||||
assert collection.primary_field is None
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L1)
|
||||
def test_collection_dup_name_invalid_schema(self, get_invalid_schema):
|
||||
@pytest.mark.xfail(reason="issue #5304")
|
||||
def test_collection_dup_name_invalid_schema_type(self, get_invalid_schema_type):
|
||||
"""
|
||||
target: test collection with dup name and invalid schema
|
||||
method: 1. default schema 2. invalid schema
|
||||
expected: raise exception and
|
||||
"""
|
||||
self._connect()
|
||||
c_name = gen_unique_str(prefix)
|
||||
collection, _ = self.collection.collection_init(c_name, schema=default_schema)
|
||||
c_name, collection = self._collection()
|
||||
assert_default_collection(collection, c_name)
|
||||
ex, _ = self.collection.collection_init(c_name, schema=get_invalid_schema)
|
||||
ex, _ = self.collection.collection_init(c_name, schema=get_invalid_schema_type)
|
||||
assert "schema type must be schema.CollectionSchema" in str(ex)
|
||||
assert_default_collection(collection, c_name)
|
||||
|
||||
|
|
@ -182,13 +178,46 @@ class TestCollectionParams(ApiReq):
|
|||
expected: two collection object is available
|
||||
"""
|
||||
self._connect()
|
||||
c_name = gen_unique_str(prefix)
|
||||
collection, _ = self.collection.collection_init(c_name, schema=default_schema)
|
||||
c_name, collection = self._collection()
|
||||
assert_default_collection(collection, c_name)
|
||||
dup_collection, _ = self.collection.collection_init(c_name, schema=default_schema)
|
||||
assert_default_collection(dup_collection, c_name)
|
||||
assert id(collection) == id(dup_collection)
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L1)
|
||||
@pytest.mark.xfail(reason="issue #5302")
|
||||
def test_collection_dup_name_none_schema_dataframe(self):
|
||||
"""
|
||||
target: test collection with dup name and insert dataframe
|
||||
method: create collection with dup name, none schema, dataframe
|
||||
expected: two collection object is correct
|
||||
"""
|
||||
self._connect()
|
||||
nb = ct.default_nb
|
||||
c_name, collection = self._collection()
|
||||
assert_default_collection(collection, c_name)
|
||||
df = cf.gen_default_dataframe_data(nb)
|
||||
dup_collection, _ = self.collection.collection_init(c_name, schema=None, data=df)
|
||||
assert_default_collection(dup_collection, c_name, exp_num=nb)
|
||||
assert collection.num_entities == nb
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L1)
|
||||
@pytest.mark.xfail(reason="issue #5302")
|
||||
def test_collection_dup_name_none_schema_data_list(self):
|
||||
"""
|
||||
target: test collection with dup name and insert data (list-like)
|
||||
method: create collection with dup name, none schema, data (list-like)
|
||||
expected: two collection object is correct
|
||||
"""
|
||||
self._connect()
|
||||
nb = ct.default_nb
|
||||
c_name, collection = self._collection()
|
||||
assert_default_collection(collection, c_name)
|
||||
data = cf.gen_default_dataframe_data(nb)
|
||||
dup_collection, _ = self.collection.collection_init(c_name, schema=None, data=data)
|
||||
assert_default_collection(dup_collection, c_name, exp_num=nb)
|
||||
assert collection.num_entities == nb
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L0)
|
||||
def test_collection_none_schema(self):
|
||||
"""
|
||||
|
|
@ -197,20 +226,20 @@ class TestCollectionParams(ApiReq):
|
|||
expected: raise exception
|
||||
"""
|
||||
self._connect()
|
||||
c_name = gen_unique_str(prefix)
|
||||
c_name = cf.gen_unique_str(prefix)
|
||||
ex, _ = self.collection.collection_init(c_name, schema=None)
|
||||
assert "Collection missing schema" in str(ex)
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L0)
|
||||
def test_collection_invalid_schema(self, get_invalid_schema):
|
||||
def test_collection_invalid_schema_type(self, get_invalid_schema_type):
|
||||
"""
|
||||
target: test collection with invalid schema
|
||||
method: create collection with non-CollectionSchema type schema
|
||||
expected: raise exception
|
||||
"""
|
||||
self._connect()
|
||||
c_name = gen_unique_str(prefix)
|
||||
ex, _ = self.collection.collection_init(c_name, schema=get_invalid_schema)
|
||||
c_name = cf.gen_unique_str(prefix)
|
||||
ex, _ = self.collection.collection_init(c_name, schema=get_invalid_schema_type)
|
||||
assert "schema type must be schema.CollectionSchema" in str(ex)
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L0)
|
||||
|
|
@ -222,11 +251,41 @@ class TestCollectionParams(ApiReq):
|
|||
expected: raise exception
|
||||
"""
|
||||
self._connect()
|
||||
c_name = gen_unique_str(prefix)
|
||||
schema = gen_collection_schema([gen_int64_field()])
|
||||
c_name = cf.gen_unique_str(prefix)
|
||||
schema = cf.gen_collection_schema([cf.gen_int64_field()])
|
||||
ex, _ = self.collection.collection_init(c_name, schema=schema)
|
||||
assert "must" in str(ex)
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L0)
|
||||
@pytest.mark.xfail(reason="issue #5302")
|
||||
def test_collection_with_dataframe(self):
|
||||
"""
|
||||
target: test collection with dataframe data
|
||||
method: create collection and insert with dataframe
|
||||
expected: collection num entities equal to nb
|
||||
"""
|
||||
self._connect()
|
||||
nb = ct.default_nb
|
||||
c_name = cf.gen_unique_str(prefix)
|
||||
data = cf.gen_default_dataframe_data(nb)
|
||||
collection, _ = self.collection.collection_init(c_name, schema=default_schema, data=data)
|
||||
assert_default_collection(collection, c_name, exp_num=nb)
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L0)
|
||||
@pytest.mark.xfail(reason="issue #5302")
|
||||
def test_collection_with_data_list(self):
|
||||
"""
|
||||
target: test collection with data (list-like)
|
||||
method: create collection with data (list-like)
|
||||
expected: collection num entities equal to nb
|
||||
"""
|
||||
self._connect()
|
||||
nb = ct.default_nb
|
||||
c_name = cf.gen_unique_str(prefix)
|
||||
data = cf.gen_default_list_data(nb)
|
||||
collection, _ = self.collection.collection_init(c_name, schema=default_schema, data=data)
|
||||
assert_default_collection(collection, c_name, exp_num=nb)
|
||||
|
||||
|
||||
class TestCollectionOperation(ApiReq):
|
||||
"""
|
||||
|
|
@ -244,12 +303,12 @@ class TestCollectionOperation(ApiReq):
|
|||
expected: raise exception
|
||||
"""
|
||||
self._connect()
|
||||
c_name = gen_unique_str(prefix)
|
||||
self.connection.remove_connection(default_alias)
|
||||
c_name = cf.gen_unique_str(prefix)
|
||||
self.connection.remove_connection(ct.default_alias)
|
||||
res_list = self.connection.list_connections()
|
||||
assert default_alias not in res_list
|
||||
assert ct.default_alias not in res_list
|
||||
ex, check = self.collection.collection_init(c_name, schema=default_schema)
|
||||
assert "There is no connection with alias '{}'".format(default_alias) in str(ex)
|
||||
assert "There is no connection with alias '{}'".format(ct.default_alias) in str(ex)
|
||||
assert self.collection.collection is None
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L1)
|
||||
|
|
@ -262,7 +321,7 @@ class TestCollectionOperation(ApiReq):
|
|||
self._connect()
|
||||
c_num = 20
|
||||
for _ in range(c_num):
|
||||
c_name = gen_unique_str(prefix)
|
||||
c_name = cf.gen_unique_str(prefix)
|
||||
collection, _ = self.collection.collection_init(c_name, schema=default_schema)
|
||||
assert_default_collection(collection, c_name)
|
||||
collection.drop()
|
||||
|
|
|
|||
Loading…
Reference in New Issue