Add README of scale test (#6800)

* add get_svc_ip func

add scale case tag

Signed-off-by: ThreadDao <yufen.zong@zilliz.com>

* add scale test README

Signed-off-by: ThreadDao <yufen.zong@zilliz.com>

tests20/python_client/common/common_func.py

@@ -137,6 +137,7 @@ def gen_binary_vectors(num, dim):
     for _ in range(num):
         raw_vector = [random.randint(0, 1) for _ in range(dim)]
         raw_vectors.append(raw_vector)
+        # packs a binary-valued array into bits in a unit8 array, and bytes array_of_ints
         binary_vectors.append(bytes(np.packbits(raw_vector, axis=-1).tolist()))
     return raw_vectors, binary_vectors
 

tests20/python_client/scale/README.md

@@ -0,0 +1,51 @@
+# Scale Tests
+## Goal
+Scale tests are designed to check the scalability of Milvus.
+
+For instance, if the dataNode pod expands from one to two:
+   - verify the consistency of existing data
+   
+   - verify that the DDL and DML operation is working
+
+## Prerequisite
+   - Milvus Helm Chart ( refer to [Milvus Helm Chart](https://github.com/milvus-io/milvus-helm/blob/master/charts/milvus/README.md) )
+
+## Test Scenarios
+### Milvus in cluster mode
+- expand / shrink dataNode pod
+   
+- expand / shrink indexNode pod
+
+- expand / shrink queryNode pod
+
+- expand / shrink proxy pod
+
+## How it works
+
+- Milvus scales the number of pods in a deployment based on the helm upgrade
+  
+- Scale test decouple the milvus deployment from the test code
+  
+- Each test scenario is carried out along the process:
+  <br> deploy milvus -> operate milvus -> scale milvus -> verify milvus 
+  
+- Milvus deployment and milvus scaling are designed in `helm_env.py`
+
+## Run
+### Manually
+Run a single test scenario manually(take scale dataNode as instance):  
+  
+- update milvus helm chart path (choose one of the following)  
+  - `export MILVUS_CHART_ENV=/your/milvus-helm/charts/milvus`  
+  
+  - update <code>MILVUS_CHART = '/home/zong/milvus-helm/charts/milvus'</code> in <code>scale/constants.py</code>  
+  
+- run the commands below:  
+```bash  
+    cd /milvus/tests20/python_client/scale  
+  
+    pytest test_data_node_scale.py::TestDataNodeScale::test_expand_data_node -v -s  
+```
+
+### Nightly 
+still in planning 

tests20/python_client/scale/constants.py

@@ -1,6 +1,7 @@
 # scale object
 IMAGE_REPOSITORY = "milvusdb/milvus-dev"
 IMAGE_TAG = "master-latest"
+NAMESPACE = "chaos-testing"
 IF_NOT_PRESENT = "IfNotPresent"
 ALWAYS = "Always"
 PROXY = "proxy"
@@ -9,4 +10,5 @@ INDEX_NODE = "indexNode"
 QUERY_NODE = "queryNode"
 
 # my values.yaml path
-HELM_VALUES_PATH = '/home/zong/milvus-helm/charts/milvus'
+MILVUS_CHART_ENV = 'MILVUS_CHART_ENV'
+MILVUS_CHART = '/home/zong/milvus-helm/charts/milvus'

tests20/python_client/scale/helm_env.py

@@ -4,6 +4,7 @@ from time import sleep
 from scale import constants
 from utils.util_log import test_log as log
 from common import common_func as cf
+from scale import scale_common as sc
 
 
 class HelmEnv:
@@ -35,8 +36,9 @@ class HelmEnv:
                       f'--set indexNode.replicas={self.index_node} ' \
                       f'--set queryNode.replicas={self.query_node} ' \
                       f'{self.release_name} . '
-        log.info(install_cmd)
+        log.debug(f'install_cmd: {install_cmd}')
-        os.system(f'cd {constants.HELM_VALUES_PATH} && {install_cmd}')
+        log.debug(f'MILVUS CHART: {sc.get_milvus_chart_env_var()}')
+        os.system(f'cd {sc.get_milvus_chart_env_var()} && {install_cmd}')
         # raise Exception("Failed to deploy cluster milvus")
         #     todo
         #   return svc ip
@@ -62,8 +64,9 @@ class HelmEnv:
                       f'--set indexNode.replicas={index_node} ' \
                       f'--set queryNode.replicas={query_node} ' \
                       f'{self.release_name} . '
-        log.info(upgrade_cmd)
+        log.debug(f'upgrade_cmd: {upgrade_cmd}')
-        if os.system(f'cd {constants.HELM_VALUES_PATH} && {upgrade_cmd}'):
+        log.debug(f'MILVUS CHART: {sc.get_milvus_chart_env_var()}')
+        if os.system(f'cd {sc.get_milvus_chart_env_var()} && {upgrade_cmd}'):
             raise Exception(f'Failed to upgrade cluster milvus with {kwargs}')
 
     def helm_uninstall_cluster_milvus(self):
@@ -81,11 +84,21 @@ class HelmEnv:
         # delete plusar
         # delete_pvc_plusar_cmd = "kubectl delete pvc scale-test-milvus-pulsar"
 
+    def get_svc_external_ip(self):
+        from kubernetes import client, config
+        # from kubernetes.client.rest import ApiException
+        config.load_kube_config()
+        v1 = client.CoreV1Api()
+        service = v1.read_namespaced_service(f'{self.release_name}-milvus', constants.NAMESPACE)
+        return service.status.load_balancer.ingress[0].ip
+
 
 if __name__ == '__main__':
     # default deploy q replicas
     release_name = "scale-test"
     env = HelmEnv(release_name=release_name)
+    # host = env.get_svc_external_ip()
+    # log.debug(host)
     # env.helm_install_cluster_milvus()
     # env.helm_upgrade_cluster_milvus(queryNode=2)
     env.helm_uninstall_cluster_milvus()

tests20/python_client/scale/scale_common.py

@@ -0,0 +1,15 @@
+import os
+
+from scale import constants
+from utils.util_log import test_log as log
+
+
+def get_milvus_chart_env_var(var=constants.MILVUS_CHART_ENV):
+    """ get log path for testing """
+    try:
+        milvus_helm_chart = os.environ[var]
+        return str(milvus_helm_chart)
+    except Exception as e:
+        milvus_helm_chart = constants.MILVUS_CHART
+        log.warning(f"Failed to get environment variables: {var}, use default: {milvus_helm_chart}, error: {str(e)}")
+        return milvus_helm_chart

tests20/python_client/scale/test_data_node_scale.py

@@ -1,8 +1,7 @@
-import random
-
 import pytest
 
 from base.collection_wrapper import ApiCollectionWrapper
+from common.common_type import CaseLabel
 from utils.util_log import test_log as log
 from common import common_func as cf
 from common import common_type as ct
@@ -18,6 +17,7 @@ default_index_params = {"index_type": "IVF_SQ8", "metric_type": "L2", "params":
 
 class TestDataNodeScale:
 
+    @pytest.mark.tags(CaseLabel.L3)
     def test_expand_data_node(self):
         """
         target: test create and insert api after expand dataNode pod
@@ -30,9 +30,10 @@ class TestDataNodeScale:
         release_name = "scale-test"
         env = HelmEnv(release_name=release_name)
         env.helm_install_cluster_milvus()
+        host = env.get_svc_external_ip()
 
         # connect
-        connections.add_connection(default={"host": '10.98.0.8', "port": 19530})
+        connections.add_connection(default={"host": host, "port": 19530})
         connections.connect(alias='default')
         # create
         c_name = cf.gen_unique_str(prefix)
@@ -63,6 +64,7 @@ class TestDataNodeScale:
         new_collection_w.drop()
         # env.helm_uninstall_cluster_milvus()
 
+    @pytest.mark.tags(CaseLabel.L3)
     def test_shrink_data_node(self):
         """
         target: test shrink dataNode from 2 to 1
@@ -101,5 +103,3 @@ class TestDataNodeScale:
         collection_w2.drop()
 
         # env.helm_uninstall_cluster_milvus()
-
-

tests20/python_client/scale/test_index_node_scale.py

@@ -1,9 +1,10 @@
 import datetime
 # import pdb
-
+import pytest
 from pymilvus_orm import connections
 
 from base.collection_wrapper import ApiCollectionWrapper
+from common.common_type import CaseLabel
 from scale.helm_env import HelmEnv
 from common import common_func as cf
 from common import common_type as ct
@@ -15,6 +16,7 @@ default_index_params = {"index_type": "IVF_SQ8", "metric_type": "L2", "params":
 
 class TestIndexNodeScale:
 
+    @pytest.mark.tags(CaseLabel.L3)
     def test_expand_index_node(self):
         """
         target: test expand indexNode from 1 to 2
@@ -67,6 +69,7 @@ class TestIndexNodeScale:
         log.debug(f't1: {t1}')
         assert round(t0 / t1) == 2
 
+    @pytest.mark.tags(CaseLabel.L3)
     def test_shrink_index_node(self):
         """
         target: test shrink indexNode from 2 to 1

tests20/python_client/scale/test_query_node_scale.py

@@ -1,7 +1,10 @@
 import pdb
 import random
 
+import pytest
+
 from base.collection_wrapper import ApiCollectionWrapper
+from common.common_type import CaseLabel
 from scale.helm_env import HelmEnv
 from utils.util_log import test_log as log
 from common import common_func as cf
@@ -19,6 +22,7 @@ default_index_params = {"index_type": "IVF_SQ8", "metric_type": "L2", "params":
 
 class TestQueryNodeScale:
 
+    @pytest.mark.tags(CaseLabel.L3)
     def test_expand_query_node(self):
         release_name = "scale-query"
         env = HelmEnv(release_name=release_name)
@@ -60,6 +64,7 @@ class TestQueryNodeScale:
 
         assert res1[0].ids == res2[0].ids
 
+    @pytest.mark.tags(CaseLabel.L3)
     def test_shrink_query_node(self):
         """
         target: test shrink queryNode from 2 to 1