[skip e2e] Fix index timeout after scale out (#15185)

Signed-off-by: ThreadDao <yufen.zong@zilliz.com>

tests/python_client/common/cus_resource_opts.py

@@ -38,9 +38,10 @@ class CustomResourceOperations(object):
         try:
             config.load_kube_config()
             api_instance = client.CustomObjectsApi()
-            data = api_instance.delete_namespaced_custom_object(self.group, self.version, self.namespace, self.plural,
-                                                                metadata_name)
-            log.debug(f"delete custom resource response: {data}")
+            api_response = api_instance.delete_namespaced_custom_object(self.group, self.version, self.namespace,
+                                                                        self.plural,
+                                                                        metadata_name)
+            log.debug(f"delete custom resource response: {api_response}")
         except ApiException as e:
             if raise_ex:
                 log.error("Exception when calling CustomObjectsApi->delete_namespaced_custom_object: %s\n" % e)
@@ -67,13 +68,13 @@ class CustomResourceOperations(object):
         try:
             config.load_kube_config()
             api_instance = client.CustomObjectsApi()
-            data = api_instance.list_namespaced_custom_object(self.group, self.version, self.namespace,
-                                                              plural=self.plural, pretty=pretty)
-            log.debug(f"list custom resource response: {data}")
+            api_response = api_instance.list_namespaced_custom_object(self.group, self.version, self.namespace,
+                                                                      plural=self.plural, pretty=pretty)
+            log.debug(f"list custom resource response: {api_response}")
         except ApiException as e:
             log.error("Exception when calling CustomObjectsApi->list_namespaced_custom_object: %s\n" % e)
             raise Exception(str(e))
-        return data
+        return api_response
 
     def get(self, metadata_name):
         """get a customer resources by name in k8s"""

tests/python_client/scale/test_index_node_scale.py

@@ -125,44 +125,51 @@ class TestIndexNodeScale:
         log.info(f"milvus healthy: {healthy}")
         host = mic.endpoint(release_name, constants.NAMESPACE).split(':')[0]
 
-        # connect
-        connections.add_connection(default={"host": host, "port": 19530})
-        connections.connect(alias='default')
+        try:
+            # connect
+            connections.add_connection(default={"host": host, "port": 19530})
+            connections.connect(alias='default')
 
-        data = cf.gen_default_dataframe_data(nb)
+            data = cf.gen_default_dataframe_data(nb)
 
-        # create
-        c_name = "index_scale_one"
-        collection_w = ApiCollectionWrapper()
-        # collection_w.init_collection(name=c_name)
-        collection_w.init_collection(name=c_name, schema=cf.gen_default_collection_schema())
-        # insert
-        loop = 10
-        for i in range(loop):
-            collection_w.insert(data)
-        assert collection_w.num_entities == nb * loop
+            # create
+            c_name = "index_scale_one"
+            collection_w = ApiCollectionWrapper()
+            # collection_w.init_collection(name=c_name)
+            collection_w.init_collection(name=c_name, schema=cf.gen_default_collection_schema())
+            # insert
+            loop = 10
+            for i in range(loop):
+                collection_w.insert(data)
+            assert collection_w.num_entities == nb * loop
 
-        # create index on collection one and two
-        start = datetime.datetime.now()
-        collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
-        assert collection_w.has_index()[0]
-        t0 = datetime.datetime.now() - start
+            # create index on collection one and two
+            start = datetime.datetime.now()
+            collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
+            assert collection_w.has_index()[0]
+            t0 = datetime.datetime.now() - start
 
-        log.debug(f'two indexNodes: {t0}')
+            log.debug(f'two indexNodes: {t0}')
 
-        collection_w.drop_index()
-        assert not collection_w.has_index()[0]
+            collection_w.drop_index()
+            assert not collection_w.has_index()[0]
 
-        # expand indexNode from 2 to 1
-        mic.upgrade(release_name, {'spec.components.indexNode.replicas': 1}, constants.NAMESPACE)
+            # expand indexNode from 2 to 1
+            mic.upgrade(release_name, {'spec.components.indexNode.replicas': 1}, constants.NAMESPACE)
+            time.sleep(5)
+            mic.wait_for_healthy(release_name, constants.NAMESPACE)
 
-        start = datetime.datetime.now()
-        collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
-        assert collection_w.has_index()[0]
-        t1 = datetime.datetime.now() - start
+            start = datetime.datetime.now()
+            collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
+            assert collection_w.has_index()[0]
+            t1 = datetime.datetime.now() - start
 
-        log.debug(f'one indexNode: {t1}')
-        log.debug(t1 / t0)
-        assert round(t1 / t0) == 2
+            log.debug(f'one indexNode: {t1}')
+            log.debug(t1 / t0)
+            assert round(t1 / t0) == 2
 
-        mic.uninstall(release_name, namespace=constants.NAMESPACE)
+        except Exception as e:
+            raise Exception(str(e))
+
+        finally:
+            mic.uninstall(release_name, namespace=constants.NAMESPACE)