diff --git a/tests20/python_client/common/common_func.py b/tests20/python_client/common/common_func.py
index 7603621c28..cad1555a14 100644
--- a/tests20/python_client/common/common_func.py
+++ b/tests20/python_client/common/common_func.py
@@ -137,6 +137,7 @@ def gen_binary_vectors(num, dim):
for _ in range(num):
raw_vector = [random.randint(0, 1) for _ in range(dim)]
raw_vectors.append(raw_vector)
+ # packs a binary-valued array into bits in a unit8 array, and bytes array_of_ints
binary_vectors.append(bytes(np.packbits(raw_vector, axis=-1).tolist()))
return raw_vectors, binary_vectors
diff --git a/tests20/python_client/scale/README.md b/tests20/python_client/scale/README.md
new file mode 100644
index 0000000000..fa80bab415
--- /dev/null
+++ b/tests20/python_client/scale/README.md
@@ -0,0 +1,51 @@
+# Scale Tests
+## Goal
+Scale tests are designed to check the scalability of Milvus.
+
+For instance, if the dataNode pod expands from one to two:
+ - verify the consistency of existing data
+
+ - verify that the DDL and DML operation is working
+
+## Prerequisite
+ - Milvus Helm Chart ( refer to [Milvus Helm Chart](https://github.com/milvus-io/milvus-helm/blob/master/charts/milvus/README.md) )
+
+## Test Scenarios
+### Milvus in cluster mode
+- expand / shrink dataNode pod
+
+- expand / shrink indexNode pod
+
+- expand / shrink queryNode pod
+
+- expand / shrink proxy pod
+
+## How it works
+
+- Milvus scales the number of pods in a deployment based on the helm upgrade
+
+- Scale test decouple the milvus deployment from the test code
+
+- Each test scenario is carried out along the process:
+
deploy milvus -> operate milvus -> scale milvus -> verify milvus
+
+- Milvus deployment and milvus scaling are designed in `helm_env.py`
+
+## Run
+### Manually
+Run a single test scenario manually(take scale dataNode as instance):
+
+- update milvus helm chart path (choose one of the following)
+ - `export MILVUS_CHART_ENV=/your/milvus-helm/charts/milvus`
+
+ - update MILVUS_CHART = '/home/zong/milvus-helm/charts/milvus'
in scale/constants.py
+
+- run the commands below:
+```bash
+ cd /milvus/tests20/python_client/scale
+
+ pytest test_data_node_scale.py::TestDataNodeScale::test_expand_data_node -v -s
+```
+
+### Nightly
+still in planning
diff --git a/tests20/python_client/scale/constants.py b/tests20/python_client/scale/constants.py
index c70b3eedad..340b936c00 100644
--- a/tests20/python_client/scale/constants.py
+++ b/tests20/python_client/scale/constants.py
@@ -1,6 +1,7 @@
# scale object
IMAGE_REPOSITORY = "milvusdb/milvus-dev"
IMAGE_TAG = "master-latest"
+NAMESPACE = "chaos-testing"
IF_NOT_PRESENT = "IfNotPresent"
ALWAYS = "Always"
PROXY = "proxy"
@@ -9,4 +10,5 @@ INDEX_NODE = "indexNode"
QUERY_NODE = "queryNode"
# my values.yaml path
-HELM_VALUES_PATH = '/home/zong/milvus-helm/charts/milvus'
+MILVUS_CHART_ENV = 'MILVUS_CHART_ENV'
+MILVUS_CHART = '/home/zong/milvus-helm/charts/milvus'
diff --git a/tests20/python_client/scale/helm_env.py b/tests20/python_client/scale/helm_env.py
index 444b9110ad..72656a967b 100644
--- a/tests20/python_client/scale/helm_env.py
+++ b/tests20/python_client/scale/helm_env.py
@@ -4,6 +4,7 @@ from time import sleep
from scale import constants
from utils.util_log import test_log as log
from common import common_func as cf
+from scale import scale_common as sc
class HelmEnv:
@@ -35,8 +36,9 @@ class HelmEnv:
f'--set indexNode.replicas={self.index_node} ' \
f'--set queryNode.replicas={self.query_node} ' \
f'{self.release_name} . '
- log.info(install_cmd)
- os.system(f'cd {constants.HELM_VALUES_PATH} && {install_cmd}')
+ log.debug(f'install_cmd: {install_cmd}')
+ log.debug(f'MILVUS CHART: {sc.get_milvus_chart_env_var()}')
+ os.system(f'cd {sc.get_milvus_chart_env_var()} && {install_cmd}')
# raise Exception("Failed to deploy cluster milvus")
# todo
# return svc ip
@@ -62,8 +64,9 @@ class HelmEnv:
f'--set indexNode.replicas={index_node} ' \
f'--set queryNode.replicas={query_node} ' \
f'{self.release_name} . '
- log.info(upgrade_cmd)
- if os.system(f'cd {constants.HELM_VALUES_PATH} && {upgrade_cmd}'):
+ log.debug(f'upgrade_cmd: {upgrade_cmd}')
+ log.debug(f'MILVUS CHART: {sc.get_milvus_chart_env_var()}')
+ if os.system(f'cd {sc.get_milvus_chart_env_var()} && {upgrade_cmd}'):
raise Exception(f'Failed to upgrade cluster milvus with {kwargs}')
def helm_uninstall_cluster_milvus(self):
@@ -81,11 +84,21 @@ class HelmEnv:
# delete plusar
# delete_pvc_plusar_cmd = "kubectl delete pvc scale-test-milvus-pulsar"
+ def get_svc_external_ip(self):
+ from kubernetes import client, config
+ # from kubernetes.client.rest import ApiException
+ config.load_kube_config()
+ v1 = client.CoreV1Api()
+ service = v1.read_namespaced_service(f'{self.release_name}-milvus', constants.NAMESPACE)
+ return service.status.load_balancer.ingress[0].ip
+
if __name__ == '__main__':
# default deploy q replicas
release_name = "scale-test"
env = HelmEnv(release_name=release_name)
+ # host = env.get_svc_external_ip()
+ # log.debug(host)
# env.helm_install_cluster_milvus()
# env.helm_upgrade_cluster_milvus(queryNode=2)
env.helm_uninstall_cluster_milvus()
diff --git a/tests20/python_client/scale/scale_common.py b/tests20/python_client/scale/scale_common.py
new file mode 100644
index 0000000000..a88e43251f
--- /dev/null
+++ b/tests20/python_client/scale/scale_common.py
@@ -0,0 +1,15 @@
+import os
+
+from scale import constants
+from utils.util_log import test_log as log
+
+
+def get_milvus_chart_env_var(var=constants.MILVUS_CHART_ENV):
+ """ get log path for testing """
+ try:
+ milvus_helm_chart = os.environ[var]
+ return str(milvus_helm_chart)
+ except Exception as e:
+ milvus_helm_chart = constants.MILVUS_CHART
+ log.warning(f"Failed to get environment variables: {var}, use default: {milvus_helm_chart}, error: {str(e)}")
+ return milvus_helm_chart
diff --git a/tests20/python_client/scale/test_data_node_scale.py b/tests20/python_client/scale/test_data_node_scale.py
index 8820fa3fcb..43dad68c93 100644
--- a/tests20/python_client/scale/test_data_node_scale.py
+++ b/tests20/python_client/scale/test_data_node_scale.py
@@ -1,8 +1,7 @@
-import random
-
import pytest
from base.collection_wrapper import ApiCollectionWrapper
+from common.common_type import CaseLabel
from utils.util_log import test_log as log
from common import common_func as cf
from common import common_type as ct
@@ -18,6 +17,7 @@ default_index_params = {"index_type": "IVF_SQ8", "metric_type": "L2", "params":
class TestDataNodeScale:
+ @pytest.mark.tags(CaseLabel.L3)
def test_expand_data_node(self):
"""
target: test create and insert api after expand dataNode pod
@@ -30,9 +30,10 @@ class TestDataNodeScale:
release_name = "scale-test"
env = HelmEnv(release_name=release_name)
env.helm_install_cluster_milvus()
+ host = env.get_svc_external_ip()
# connect
- connections.add_connection(default={"host": '10.98.0.8', "port": 19530})
+ connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
# create
c_name = cf.gen_unique_str(prefix)
@@ -63,6 +64,7 @@ class TestDataNodeScale:
new_collection_w.drop()
# env.helm_uninstall_cluster_milvus()
+ @pytest.mark.tags(CaseLabel.L3)
def test_shrink_data_node(self):
"""
target: test shrink dataNode from 2 to 1
@@ -101,5 +103,3 @@ class TestDataNodeScale:
collection_w2.drop()
# env.helm_uninstall_cluster_milvus()
-
-
diff --git a/tests20/python_client/scale/test_index_node_scale.py b/tests20/python_client/scale/test_index_node_scale.py
index 6a0cdce544..d18c2dc277 100644
--- a/tests20/python_client/scale/test_index_node_scale.py
+++ b/tests20/python_client/scale/test_index_node_scale.py
@@ -1,9 +1,10 @@
import datetime
# import pdb
-
+import pytest
from pymilvus_orm import connections
from base.collection_wrapper import ApiCollectionWrapper
+from common.common_type import CaseLabel
from scale.helm_env import HelmEnv
from common import common_func as cf
from common import common_type as ct
@@ -15,6 +16,7 @@ default_index_params = {"index_type": "IVF_SQ8", "metric_type": "L2", "params":
class TestIndexNodeScale:
+ @pytest.mark.tags(CaseLabel.L3)
def test_expand_index_node(self):
"""
target: test expand indexNode from 1 to 2
@@ -67,6 +69,7 @@ class TestIndexNodeScale:
log.debug(f't1: {t1}')
assert round(t0 / t1) == 2
+ @pytest.mark.tags(CaseLabel.L3)
def test_shrink_index_node(self):
"""
target: test shrink indexNode from 2 to 1
diff --git a/tests20/python_client/scale/test_proxy_scale.py b/tests20/python_client/scale/test_proxy_scale.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests20/python_client/scale/test_query_node_scale.py b/tests20/python_client/scale/test_query_node_scale.py
index dae957a708..47f1782bf1 100644
--- a/tests20/python_client/scale/test_query_node_scale.py
+++ b/tests20/python_client/scale/test_query_node_scale.py
@@ -1,7 +1,10 @@
import pdb
import random
+import pytest
+
from base.collection_wrapper import ApiCollectionWrapper
+from common.common_type import CaseLabel
from scale.helm_env import HelmEnv
from utils.util_log import test_log as log
from common import common_func as cf
@@ -19,6 +22,7 @@ default_index_params = {"index_type": "IVF_SQ8", "metric_type": "L2", "params":
class TestQueryNodeScale:
+ @pytest.mark.tags(CaseLabel.L3)
def test_expand_query_node(self):
release_name = "scale-query"
env = HelmEnv(release_name=release_name)
@@ -60,6 +64,7 @@ class TestQueryNodeScale:
assert res1[0].ids == res2[0].ids
+ @pytest.mark.tags(CaseLabel.L3)
def test_shrink_query_node(self):
"""
target: test shrink queryNode from 2 to 1