milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[skip e2e] Optimize indexnode scale test case (#15016) 

Signed-off-by: ThreadDao <yufen.zong@zilliz.com>
pull/15024/merge
ThreadDao 2022-01-07 19:09:29 +08:00 committed by GitHub
parent e108a64135
commit 71ce45a407
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 42 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
tests/python_client/scale/test_index_node_scale.py
View File

@ -30,12 +30,14 @@ class TestIndexNodeScale:
"""
release_name = "expand-index"
image = f'{constants.IMAGE_REPOSITORY}:{constants.IMAGE_TAG}'
init_replicas = 1
expand_replicas = 2
data_config = {
'metadata.namespace': constants.NAMESPACE,
'metadata.name': release_name,
'spec.components.image': image,
'spec.components.proxy.serviceType': 'LoadBalancer',
'spec.components.indexNode.replicas': 1,
'spec.components.indexNode.replicas': init_replicas,
'spec.components.dataNode.replicas': 2,
'spec.config.dataCoord.enableCompaction': True,
'spec.config.dataCoord.enableGarbageCollection': True
@ -45,53 +47,55 @@ class TestIndexNodeScale:
healthy = mic.wait_for_healthy(release_name, constants.NAMESPACE, timeout=1200)
log.info(f"milvus healthy: {healthy}")
host = mic.endpoint(release_name, constants.NAMESPACE).split(':')[0]
# host = '10.98.0.8'
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
try:
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
data = cf.gen_default_dataframe_data(nb)
# create collection
c_name = "index_scale_one"
collection_w = ApiCollectionWrapper()
collection_w.init_collection(name=c_name, schema=cf.gen_default_collection_schema())
# create
c_name = "index_scale_one"
collection_w = ApiCollectionWrapper()
# collection_w.init_collection(name=c_name)
collection_w.init_collection(name=c_name, schema=cf.gen_default_collection_schema())
# insert data
data = cf.gen_default_dataframe_data(nb)
loop = 100
for i in range(loop):
collection_w.insert(data, timeout=60)
assert collection_w.num_entities == nb * loop
# insert
loop = 100
for i in range(loop):
collection_w.insert(data, timeout=60)
assert collection_w.num_entities == nb * loop
# create index
# Note that the num of segments and the num of indexNode are related to indexing time
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
assert collection_w.has_index()[0]
t0 = datetime.datetime.now() - start
log.info(f'Create index on {init_replicas} indexNode cost t0: {t0}')
# create index on collection
# note that the num of segments and the num of indexNode are related to indexing time
collection_w.drop_index()
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
assert collection_w.has_index()[0]
t0 = datetime.datetime.now() - start
# drop index
collection_w.drop_index()
assert not collection_w.has_index()[0]
log.debug(f't0: {t0}')
# expand indexNode
mic.upgrade(release_name, {'spec.components.indexNode.replicas': expand_replicas}, constants.NAMESPACE)
time.sleep(5)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
collection_w.drop_index()
assert not collection_w.has_index()[0]
# create index again
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
assert collection_w.has_index()[0]
t1 = datetime.datetime.now() - start
log.info(f'Create index on {expand_replicas} indexNode cost t1: {t1}')
# expand indexNode from 1 to 2
mic.upgrade(release_name, {'spec.components.indexNode.replicas': 2}, constants.NAMESPACE)
time.sleep(60)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
assert round(t0 / t1) == 2
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
assert collection_w.has_index()[0]
t1 = datetime.datetime.now() - start
except Exception as e:
raise Exception(str(e))
log.debug(f't1: {t1}')
assert round(t0 / t1) == 2
mic.uninstall(release_name, namespace=constants.NAMESPACE)
finally:
mic.uninstall(release_name, namespace=constants.NAMESPACE)
@pytest.mark.tags(CaseLabel.L3)
def test_shrink_index_node(self):