milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[skip e2e]Update multi replicas chaos test (#17084) 

Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>
pull/17064/head
zhuwenxing 2022-05-18 17:57:57 +08:00 committed by GitHub
parent 95bf91795d
commit 1cabaa16a1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
tests/python_client/chaos/test_chaos_multi_replicas.py
View File

@ -5,6 +5,7 @@ import pytest
import os import os
import time import time
import json import json
import random
from time import sleep from time import sleep
from pymilvus import connections from pymilvus import connections
@ -130,7 +131,7 @@ class TestChaos(TestChaosBase):
@pytest.mark.tags(CaseLabel.L3) @pytest.mark.tags(CaseLabel.L3)
@pytest.mark.parametrize("is_streaming", [False]) # [False, True] @pytest.mark.parametrize("is_streaming", [False]) # [False, True]
@pytest.mark.parametrize("failed_group_scope", ["one"]) # ["one", "all"] @pytest.mark.parametrize("failed_group_scope", ["one"]) # ["one", "except_one" "all"]
@pytest.mark.parametrize("failed_node_type", ["shard_leader"]) # ["non_shard_leader", "shard_leader"] @pytest.mark.parametrize("failed_node_type", ["shard_leader"]) # ["non_shard_leader", "shard_leader"]
@pytest.mark.parametrize("chaos_type", ["pod-failure"]) # ["pod-failure", "pod-kill"] @pytest.mark.parametrize("chaos_type", ["pod-failure"]) # ["pod-failure", "pod-kill"]
def test_multi_replicas_with_only_one_group_available(self, chaos_type, failed_node_type, failed_group_scope, is_streaming): def test_multi_replicas_with_only_one_group_available(self, chaos_type, failed_node_type, failed_group_scope, is_streaming):
@ -158,7 +159,9 @@ class TestChaos(TestChaosBase):
target_group = [] target_group = []
group_list = sorted(group_list, key=lambda x: -len(x)) group_list = sorted(group_list, key=lambda x: -len(x))
if failed_group_scope == "one": if failed_group_scope == "one":
target_group = group_list[:1] target_group = random.sample(group_list, 1)
if failed_group_scope == "except_one":
target_group = random.sample(group_list, len(group_list)-1)
if failed_group_scope == "all": if failed_group_scope == "all":
target_group = group_list[:] target_group = group_list[:]
for g in target_group: for g in target_group:
@ -170,6 +173,7 @@ class TestChaos(TestChaosBase):
for target_node in target_nodes: for target_node in target_nodes:
pod = querynode_id_pod_pair[target_node] pod = querynode_id_pod_pair[target_node]
target_pod_list.append(pod) target_pod_list.append(pod)
log.info(f"target_pod_list: {target_pod_list}")
chaos_config = cc.gen_experiment_config(f"chaos/chaos_objects/template/{chaos_type}-by-pod-list.yaml") chaos_config = cc.gen_experiment_config(f"chaos/chaos_objects/template/{chaos_type}-by-pod-list.yaml")
chaos_config['metadata']['name'] = f"test-multi-replicase-{int(time.time())}" chaos_config['metadata']['name'] = f"test-multi-replicase-{int(time.time())}"
meta_name = chaos_config.get('metadata', None).get('name', None) meta_name = chaos_config.get('metadata', None).get('name', None)
@ -191,6 +195,7 @@ class TestChaos(TestChaosBase):
group=constants.CHAOS_GROUP, group=constants.CHAOS_GROUP,
version=constants.CHAOS_VERSION, version=constants.CHAOS_VERSION,
namespace=constants.CHAOS_NAMESPACE) namespace=constants.CHAOS_NAMESPACE)
chaos_res.create(chaos_config) chaos_res.create(chaos_config)
log.info("chaos injected") log.info("chaos injected")
sleep(constants.WAIT_PER_OP * 2) sleep(constants.WAIT_PER_OP * 2)
@ -227,10 +232,11 @@ class TestChaos(TestChaosBase):
sleep(2) sleep(2)
# wait all pods ready # wait all pods ready
log.info(f"wait for pods in namespace {constants.CHAOS_NAMESPACE} with label app.kubernetes.io/instance={release_name}") log.info(f"wait for pods in namespace {constants.CHAOS_NAMESPACE} with label app.kubernetes.io/instance={release_name}")
wait_pods_ready(constants.CHAOS_NAMESPACE,f"app.kubernetes.io/instance={release_name}") ready_1 = wait_pods_ready(constants.CHAOS_NAMESPACE,f"app.kubernetes.io/instance={release_name}")
log.info(f"wait for pods in namespace {constants.CHAOS_NAMESPACE} with label release={release_name}") log.info(f"wait for pods in namespace {constants.CHAOS_NAMESPACE} with label release={release_name}")
wait_pods_ready(constants.CHAOS_NAMESPACE, f"release={release_name}") ready_2 = wait_pods_ready(constants.CHAOS_NAMESPACE, f"release={release_name}")
log.info("all pods are ready") if ready_1 and ready_2:
log.info("all pods are ready")
# reconnect if needed # reconnect if needed
sleep(constants.WAIT_PER_OP * 2) sleep(constants.WAIT_PER_OP * 2)
# cc.reconnect(connections, alias='default') # cc.reconnect(connections, alias='default')