From 386954d9c3122b9e1b1deb531042c0dacda4caec Mon Sep 17 00:00:00 2001 From: ThreadDao Date: Mon, 26 Jul 2021 15:17:20 +0800 Subject: [PATCH] Add README of scale test (#6800) * add get_svc_ip func add scale case tag Signed-off-by: ThreadDao * add scale test README Signed-off-by: ThreadDao --- tests20/python_client/common/common_func.py | 1 + tests20/python_client/scale/README.md | 51 +++++++++++++++++++ tests20/python_client/scale/constants.py | 4 +- tests20/python_client/scale/helm_env.py | 21 ++++++-- tests20/python_client/scale/scale_common.py | 15 ++++++ .../scale/test_data_node_scale.py | 10 ++-- .../scale/test_index_node_scale.py | 5 +- .../python_client/scale/test_proxy_scale.py | 0 .../scale/test_query_node_scale.py | 5 ++ 9 files changed, 101 insertions(+), 11 deletions(-) create mode 100644 tests20/python_client/scale/README.md create mode 100644 tests20/python_client/scale/scale_common.py create mode 100644 tests20/python_client/scale/test_proxy_scale.py diff --git a/tests20/python_client/common/common_func.py b/tests20/python_client/common/common_func.py index 7603621c28..cad1555a14 100644 --- a/tests20/python_client/common/common_func.py +++ b/tests20/python_client/common/common_func.py @@ -137,6 +137,7 @@ def gen_binary_vectors(num, dim): for _ in range(num): raw_vector = [random.randint(0, 1) for _ in range(dim)] raw_vectors.append(raw_vector) + # packs a binary-valued array into bits in a unit8 array, and bytes array_of_ints binary_vectors.append(bytes(np.packbits(raw_vector, axis=-1).tolist())) return raw_vectors, binary_vectors diff --git a/tests20/python_client/scale/README.md b/tests20/python_client/scale/README.md new file mode 100644 index 0000000000..fa80bab415 --- /dev/null +++ b/tests20/python_client/scale/README.md @@ -0,0 +1,51 @@ +# Scale Tests +## Goal +Scale tests are designed to check the scalability of Milvus. + +For instance, if the dataNode pod expands from one to two: + - verify the consistency of existing data + + - verify that the DDL and DML operation is working + +## Prerequisite + - Milvus Helm Chart ( refer to [Milvus Helm Chart](https://github.com/milvus-io/milvus-helm/blob/master/charts/milvus/README.md) ) + +## Test Scenarios +### Milvus in cluster mode +- expand / shrink dataNode pod + +- expand / shrink indexNode pod + +- expand / shrink queryNode pod + +- expand / shrink proxy pod + +## How it works + +- Milvus scales the number of pods in a deployment based on the helm upgrade + +- Scale test decouple the milvus deployment from the test code + +- Each test scenario is carried out along the process: +
deploy milvus -> operate milvus -> scale milvus -> verify milvus + +- Milvus deployment and milvus scaling are designed in `helm_env.py` + +## Run +### Manually +Run a single test scenario manually(take scale dataNode as instance): + +- update milvus helm chart path (choose one of the following) + - `export MILVUS_CHART_ENV=/your/milvus-helm/charts/milvus` + + - update MILVUS_CHART = '/home/zong/milvus-helm/charts/milvus' in scale/constants.py + +- run the commands below: +```bash + cd /milvus/tests20/python_client/scale + + pytest test_data_node_scale.py::TestDataNodeScale::test_expand_data_node -v -s +``` + +### Nightly +still in planning diff --git a/tests20/python_client/scale/constants.py b/tests20/python_client/scale/constants.py index c70b3eedad..340b936c00 100644 --- a/tests20/python_client/scale/constants.py +++ b/tests20/python_client/scale/constants.py @@ -1,6 +1,7 @@ # scale object IMAGE_REPOSITORY = "milvusdb/milvus-dev" IMAGE_TAG = "master-latest" +NAMESPACE = "chaos-testing" IF_NOT_PRESENT = "IfNotPresent" ALWAYS = "Always" PROXY = "proxy" @@ -9,4 +10,5 @@ INDEX_NODE = "indexNode" QUERY_NODE = "queryNode" # my values.yaml path -HELM_VALUES_PATH = '/home/zong/milvus-helm/charts/milvus' +MILVUS_CHART_ENV = 'MILVUS_CHART_ENV' +MILVUS_CHART = '/home/zong/milvus-helm/charts/milvus' diff --git a/tests20/python_client/scale/helm_env.py b/tests20/python_client/scale/helm_env.py index 444b9110ad..72656a967b 100644 --- a/tests20/python_client/scale/helm_env.py +++ b/tests20/python_client/scale/helm_env.py @@ -4,6 +4,7 @@ from time import sleep from scale import constants from utils.util_log import test_log as log from common import common_func as cf +from scale import scale_common as sc class HelmEnv: @@ -35,8 +36,9 @@ class HelmEnv: f'--set indexNode.replicas={self.index_node} ' \ f'--set queryNode.replicas={self.query_node} ' \ f'{self.release_name} . ' - log.info(install_cmd) - os.system(f'cd {constants.HELM_VALUES_PATH} && {install_cmd}') + log.debug(f'install_cmd: {install_cmd}') + log.debug(f'MILVUS CHART: {sc.get_milvus_chart_env_var()}') + os.system(f'cd {sc.get_milvus_chart_env_var()} && {install_cmd}') # raise Exception("Failed to deploy cluster milvus") # todo # return svc ip @@ -62,8 +64,9 @@ class HelmEnv: f'--set indexNode.replicas={index_node} ' \ f'--set queryNode.replicas={query_node} ' \ f'{self.release_name} . ' - log.info(upgrade_cmd) - if os.system(f'cd {constants.HELM_VALUES_PATH} && {upgrade_cmd}'): + log.debug(f'upgrade_cmd: {upgrade_cmd}') + log.debug(f'MILVUS CHART: {sc.get_milvus_chart_env_var()}') + if os.system(f'cd {sc.get_milvus_chart_env_var()} && {upgrade_cmd}'): raise Exception(f'Failed to upgrade cluster milvus with {kwargs}') def helm_uninstall_cluster_milvus(self): @@ -81,11 +84,21 @@ class HelmEnv: # delete plusar # delete_pvc_plusar_cmd = "kubectl delete pvc scale-test-milvus-pulsar" + def get_svc_external_ip(self): + from kubernetes import client, config + # from kubernetes.client.rest import ApiException + config.load_kube_config() + v1 = client.CoreV1Api() + service = v1.read_namespaced_service(f'{self.release_name}-milvus', constants.NAMESPACE) + return service.status.load_balancer.ingress[0].ip + if __name__ == '__main__': # default deploy q replicas release_name = "scale-test" env = HelmEnv(release_name=release_name) + # host = env.get_svc_external_ip() + # log.debug(host) # env.helm_install_cluster_milvus() # env.helm_upgrade_cluster_milvus(queryNode=2) env.helm_uninstall_cluster_milvus() diff --git a/tests20/python_client/scale/scale_common.py b/tests20/python_client/scale/scale_common.py new file mode 100644 index 0000000000..a88e43251f --- /dev/null +++ b/tests20/python_client/scale/scale_common.py @@ -0,0 +1,15 @@ +import os + +from scale import constants +from utils.util_log import test_log as log + + +def get_milvus_chart_env_var(var=constants.MILVUS_CHART_ENV): + """ get log path for testing """ + try: + milvus_helm_chart = os.environ[var] + return str(milvus_helm_chart) + except Exception as e: + milvus_helm_chart = constants.MILVUS_CHART + log.warning(f"Failed to get environment variables: {var}, use default: {milvus_helm_chart}, error: {str(e)}") + return milvus_helm_chart diff --git a/tests20/python_client/scale/test_data_node_scale.py b/tests20/python_client/scale/test_data_node_scale.py index 8820fa3fcb..43dad68c93 100644 --- a/tests20/python_client/scale/test_data_node_scale.py +++ b/tests20/python_client/scale/test_data_node_scale.py @@ -1,8 +1,7 @@ -import random - import pytest from base.collection_wrapper import ApiCollectionWrapper +from common.common_type import CaseLabel from utils.util_log import test_log as log from common import common_func as cf from common import common_type as ct @@ -18,6 +17,7 @@ default_index_params = {"index_type": "IVF_SQ8", "metric_type": "L2", "params": class TestDataNodeScale: + @pytest.mark.tags(CaseLabel.L3) def test_expand_data_node(self): """ target: test create and insert api after expand dataNode pod @@ -30,9 +30,10 @@ class TestDataNodeScale: release_name = "scale-test" env = HelmEnv(release_name=release_name) env.helm_install_cluster_milvus() + host = env.get_svc_external_ip() # connect - connections.add_connection(default={"host": '10.98.0.8', "port": 19530}) + connections.add_connection(default={"host": host, "port": 19530}) connections.connect(alias='default') # create c_name = cf.gen_unique_str(prefix) @@ -63,6 +64,7 @@ class TestDataNodeScale: new_collection_w.drop() # env.helm_uninstall_cluster_milvus() + @pytest.mark.tags(CaseLabel.L3) def test_shrink_data_node(self): """ target: test shrink dataNode from 2 to 1 @@ -101,5 +103,3 @@ class TestDataNodeScale: collection_w2.drop() # env.helm_uninstall_cluster_milvus() - - diff --git a/tests20/python_client/scale/test_index_node_scale.py b/tests20/python_client/scale/test_index_node_scale.py index 6a0cdce544..d18c2dc277 100644 --- a/tests20/python_client/scale/test_index_node_scale.py +++ b/tests20/python_client/scale/test_index_node_scale.py @@ -1,9 +1,10 @@ import datetime # import pdb - +import pytest from pymilvus_orm import connections from base.collection_wrapper import ApiCollectionWrapper +from common.common_type import CaseLabel from scale.helm_env import HelmEnv from common import common_func as cf from common import common_type as ct @@ -15,6 +16,7 @@ default_index_params = {"index_type": "IVF_SQ8", "metric_type": "L2", "params": class TestIndexNodeScale: + @pytest.mark.tags(CaseLabel.L3) def test_expand_index_node(self): """ target: test expand indexNode from 1 to 2 @@ -67,6 +69,7 @@ class TestIndexNodeScale: log.debug(f't1: {t1}') assert round(t0 / t1) == 2 + @pytest.mark.tags(CaseLabel.L3) def test_shrink_index_node(self): """ target: test shrink indexNode from 2 to 1 diff --git a/tests20/python_client/scale/test_proxy_scale.py b/tests20/python_client/scale/test_proxy_scale.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests20/python_client/scale/test_query_node_scale.py b/tests20/python_client/scale/test_query_node_scale.py index dae957a708..47f1782bf1 100644 --- a/tests20/python_client/scale/test_query_node_scale.py +++ b/tests20/python_client/scale/test_query_node_scale.py @@ -1,7 +1,10 @@ import pdb import random +import pytest + from base.collection_wrapper import ApiCollectionWrapper +from common.common_type import CaseLabel from scale.helm_env import HelmEnv from utils.util_log import test_log as log from common import common_func as cf @@ -19,6 +22,7 @@ default_index_params = {"index_type": "IVF_SQ8", "metric_type": "L2", "params": class TestQueryNodeScale: + @pytest.mark.tags(CaseLabel.L3) def test_expand_query_node(self): release_name = "scale-query" env = HelmEnv(release_name=release_name) @@ -60,6 +64,7 @@ class TestQueryNodeScale: assert res1[0].ids == res2[0].ids + @pytest.mark.tags(CaseLabel.L3) def test_shrink_query_node(self): """ target: test shrink queryNode from 2 to 1