[test]Fix apply chaos condition (#27625)

Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>

tests/python_client/chaos/checker.py

@@ -128,6 +128,9 @@ class ResultAnalyzer:
         df = df.sort_values(by='start_time')
         self.df = df
         self.chaos_info = get_chaos_info()
+        self.chaos_start_time = self.chaos_info['create_time'] if self.chaos_info is not None else None
+        self.chaos_end_time = self.chaos_info['delete_time'] if self.chaos_info is not None else None
+        self.recovery_time = self.chaos_info['recovery_time'] if self.chaos_info is not None else None
 
     def get_stage_success_rate(self):
         df = self.df
@@ -181,7 +184,9 @@ class ResultAnalyzer:
 
     def show_result_table(self):
         table = PrettyTable()
-        table.field_names = ['operation_name', 'before_chaos', 'during_chaos', 'after_chaos']
+        table.field_names = ['operation_name', 'before_chaos',
+                             f'during_chaos\n{self.chaos_start_time}~{self.recovery_time}',
+                             'after_chaos']
         data = self.get_stage_success_rate()
         for operation, values in data.items():
             row = [operation, values['before_chaos'], values['during_chaos'], values['after_chaos']]

tests/python_client/chaos/test_chaos_apply.py

@@ -63,8 +63,7 @@ class TestChaosApply:
             log.info("need wait signal to start chaos")
             ready_for_chaos = wait_signal_to_apply_chaos()
             if not ready_for_chaos:
-                log.info("did not get the signal to apply chaos")
-                raise Exception
+                log.info("get the signal to apply chaos timeout")
             else:
                 log.info("get the signal to apply chaos")
         log.info(connections.get_connection_addr('default'))

tests/python_client/utils/util_common.py

@@ -98,6 +98,7 @@ def wait_signal_to_apply_chaos():
         while True and (time.time() - t0 < timeout):
             try:
                 df = pd.read_parquet(f)
+                log.debug(f"read {f}:result\n {df}")
                 result = df[(df['event_name'] == 'init_chaos') & (df['event_status'] == 'ready')]
                 if len(result) > 0:
                     log.info(f"{f}: {result}")
@@ -108,6 +109,8 @@ def wait_signal_to_apply_chaos():
             except Exception as e:
                 log.error(f"read_parquet error: {e}")
                 ready_apply_chaos = False
+            time.sleep(10)
+
     return ready_apply_chaos
 
 

tests/python_client/utils/util_k8s.py

@@ -56,8 +56,8 @@ def wait_pods_ready(namespace, label_selector, expected_num=None, timeout=360):
     api_instance = client.CoreV1Api()
     try:
         all_pos_ready_flag = False
-        time_cnt = 0
-        while not all_pos_ready_flag and time_cnt < timeout:
+        t0 = time.time()
+        while not all_pos_ready_flag and time.time() - t0 < timeout:
             api_response = api_instance.list_namespaced_pod(namespace=namespace, label_selector=label_selector)
             all_pos_ready_flag = True
             if expected_num is not None and len(api_response.items) < expected_num:
@@ -74,8 +74,7 @@ def wait_pods_ready(namespace, label_selector, expected_num=None, timeout=360):
                             break
             if not all_pos_ready_flag:
                 log.debug("all pods are not ready, please wait")
-                time.sleep(30)
-                time_cnt += 30
+                time.sleep(5)
         if all_pos_ready_flag:
             log.info(f"all pods in namespace {namespace} with label {label_selector} are ready")
         else: