diff --git a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_datacoord_podkill.yaml b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_datacoord_pod_kill.yaml similarity index 90% rename from tests/python_client/chaos/chaos_objects/pod_kill/chaos_datacoord_podkill.yaml rename to tests/python_client/chaos/chaos_objects/pod_kill/chaos_datacoord_pod_kill.yaml index 1c36b8bc13..f54a81fbaf 100644 --- a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_datacoord_podkill.yaml +++ b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_datacoord_pod_kill.yaml @@ -1,7 +1,7 @@ kind: Schedule apiVersion: chaos-mesh.org/v1alpha1 metadata: - name: test-datacoord-podkill + name: test-datacoord-pod-kill namespace: chaos-testing spec: schedule: '*/5 * * * * *' diff --git a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_datanode_podkill.yaml b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_datanode_pod_kill.yaml similarity index 90% rename from tests/python_client/chaos/chaos_objects/pod_kill/chaos_datanode_podkill.yaml rename to tests/python_client/chaos/chaos_objects/pod_kill/chaos_datanode_pod_kill.yaml index 7432e718d2..fbc598ae02 100644 --- a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_datanode_podkill.yaml +++ b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_datanode_pod_kill.yaml @@ -1,7 +1,7 @@ kind: Schedule apiVersion: chaos-mesh.org/v1alpha1 metadata: - name: test-datanode-podkill + name: test-datanode-pod-kill namespace: chaos-testing spec: schedule: '*/5 * * * * *' diff --git a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_etcd_podkill.yaml b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_etcd_pod_kill.yaml similarity index 90% rename from tests/python_client/chaos/chaos_objects/pod_kill/chaos_etcd_podkill.yaml rename to tests/python_client/chaos/chaos_objects/pod_kill/chaos_etcd_pod_kill.yaml index 38f7855eaa..f807aa176b 100644 --- a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_etcd_podkill.yaml +++ b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_etcd_pod_kill.yaml @@ -1,7 +1,7 @@ kind: Schedule apiVersion: chaos-mesh.org/v1alpha1 metadata: - name: test-etcd-podkill + name: test-etcd-pod-kill namespace: chaos-testing spec: schedule: '*/5 * * * * *' diff --git a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_indexcoord_podkill.yaml b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_indexcoord_pod_kill.yaml similarity index 89% rename from tests/python_client/chaos/chaos_objects/pod_kill/chaos_indexcoord_podkill.yaml rename to tests/python_client/chaos/chaos_objects/pod_kill/chaos_indexcoord_pod_kill.yaml index 3bf3c41373..6b9373d6f4 100644 --- a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_indexcoord_podkill.yaml +++ b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_indexcoord_pod_kill.yaml @@ -1,7 +1,7 @@ kind: Schedule apiVersion: chaos-mesh.org/v1alpha1 metadata: - name: test-indexcoord-podkill + name: test-indexcoord-pod-kill namespace: chaos-testing spec: schedule: '*/5 * * * * *' diff --git a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_indexnode_podkill.yaml b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_indexnode_pod_kill.yaml similarity index 90% rename from tests/python_client/chaos/chaos_objects/pod_kill/chaos_indexnode_podkill.yaml rename to tests/python_client/chaos/chaos_objects/pod_kill/chaos_indexnode_pod_kill.yaml index 8205628f9c..8fcfb2f490 100644 --- a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_indexnode_podkill.yaml +++ b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_indexnode_pod_kill.yaml @@ -1,7 +1,7 @@ kind: Schedule apiVersion: chaos-mesh.org/v1alpha1 metadata: - name: test-indexnode-podkill + name: test-indexnode-pod-kill namespace: chaos-testing spec: schedule: '*/5 * * * * *' diff --git a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_minio_podkill.yaml b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_minio_pod_kill.yaml similarity index 89% rename from tests/python_client/chaos/chaos_objects/pod_kill/chaos_minio_podkill.yaml rename to tests/python_client/chaos/chaos_objects/pod_kill/chaos_minio_pod_kill.yaml index c651b01404..18dbb1bdb4 100644 --- a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_minio_podkill.yaml +++ b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_minio_pod_kill.yaml @@ -1,7 +1,7 @@ kind: Schedule apiVersion: chaos-mesh.org/v1alpha1 metadata: - name: test-minio-podkill + name: test-minio-pod-kill namespace: chaos-testing spec: schedule: '*/5 * * * * *' diff --git a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_proxy_podkill.yaml b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_proxy_pod_kill.yaml similarity index 90% rename from tests/python_client/chaos/chaos_objects/pod_kill/chaos_proxy_podkill.yaml rename to tests/python_client/chaos/chaos_objects/pod_kill/chaos_proxy_pod_kill.yaml index f7a2bc204a..23fda791fd 100644 --- a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_proxy_podkill.yaml +++ b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_proxy_pod_kill.yaml @@ -1,7 +1,7 @@ kind: Schedule apiVersion: chaos-mesh.org/v1alpha1 metadata: - name: test-proxy-podkill + name: test-proxy-pod-kill namespace: chaos-testing spec: schedule: '*/5 * * * * *' diff --git a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_pulsar_podkill.yaml b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_pulsar_pod_kill.yaml similarity index 93% rename from tests/python_client/chaos/chaos_objects/pod_kill/chaos_pulsar_podkill.yaml rename to tests/python_client/chaos/chaos_objects/pod_kill/chaos_pulsar_pod_kill.yaml index ca5d3a72ce..0a9d6e826f 100644 --- a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_pulsar_podkill.yaml +++ b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_pulsar_pod_kill.yaml @@ -1,7 +1,7 @@ kind: Schedule apiVersion: chaos-mesh.org/v1alpha1 metadata: - name: test-pulsar-podkill + name: test-pulsar-pod-kill namespace: chaos-testing spec: schedule: '*/5 * * * * *' diff --git a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_querycoord_podkill.yaml b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_querycoord_pod_kill.yaml similarity index 89% rename from tests/python_client/chaos/chaos_objects/pod_kill/chaos_querycoord_podkill.yaml rename to tests/python_client/chaos/chaos_objects/pod_kill/chaos_querycoord_pod_kill.yaml index c35a0f9579..645a7d21cd 100644 --- a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_querycoord_podkill.yaml +++ b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_querycoord_pod_kill.yaml @@ -1,7 +1,7 @@ kind: Schedule apiVersion: chaos-mesh.org/v1alpha1 metadata: - name: test-querycoord-podkill + name: test-querycoord-pod-kill namespace: chaos-testing spec: schedule: '*/5 * * * * *' diff --git a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_querynode_podkill.yaml b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_querynode_pod_kill.yaml similarity index 91% rename from tests/python_client/chaos/chaos_objects/pod_kill/chaos_querynode_podkill.yaml rename to tests/python_client/chaos/chaos_objects/pod_kill/chaos_querynode_pod_kill.yaml index 6448a3446c..d7295510be 100644 --- a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_querynode_podkill.yaml +++ b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_querynode_pod_kill.yaml @@ -1,7 +1,7 @@ kind: Schedule apiVersion: chaos-mesh.org/v1alpha1 metadata: - name: test-querynode-podkill + name: test-querynode-pod-kill namespace: chaos-testing annotations: experiment.chaos-mesh.org/pause: 'false' diff --git a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_rootcoord_podkill.yaml b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_rootcoord_pod_kill.yaml similarity index 90% rename from tests/python_client/chaos/chaos_objects/pod_kill/chaos_rootcoord_podkill.yaml rename to tests/python_client/chaos/chaos_objects/pod_kill/chaos_rootcoord_pod_kill.yaml index b7ccefa389..25e2c6c94c 100644 --- a/tests/python_client/chaos/chaos_objects/pod_kill/chaos_rootcoord_podkill.yaml +++ b/tests/python_client/chaos/chaos_objects/pod_kill/chaos_rootcoord_pod_kill.yaml @@ -1,7 +1,7 @@ kind: Schedule apiVersion: chaos-mesh.org/v1alpha1 metadata: - name: test-rootcoord-podkill + name: test-rootcoord-pod-kill namespace: chaos-testing spec: schedule: '*/5 * * * * *' diff --git a/tests/python_client/chaos/chaos_objects/pod_kill/testcases.yaml b/tests/python_client/chaos/chaos_objects/pod_kill/testcases.yaml index 8794614e39..f4fb730bcb 100644 --- a/tests/python_client/chaos/chaos_objects/pod_kill/testcases.yaml +++ b/tests/python_client/chaos/chaos_objects/pod_kill/testcases.yaml @@ -11,8 +11,8 @@ Collections: - testcase: - name: test_querynode_podkill - chaos: chaos_querynode_podkill.yaml + name: test_querynode_pod_kill + chaos: chaos_querynode_pod_kill.yaml expectation: cluster_1_node: search: fail @@ -22,8 +22,8 @@ Collections: query: degrade - testcase: - name: test_querycoord_podkill - chaos: chaos_querycoord_podkill.yaml + name: test_querycoord_pod_kill + chaos: chaos_querycoord_pod_kill.yaml expectation: cluster_1_node: search: fail @@ -33,8 +33,8 @@ Collections: query: degrade - testcase: - name: test_datanode_podkill - chaos: chaos_datanode_podkill.yaml + name: test_datanode_pod_kill + chaos: chaos_datanode_pod_kill.yaml expectation: cluster_1_node: insert: succ @@ -43,8 +43,8 @@ Collections: insert: degrade - testcase: - name: test_datascoord_podkill - chaos: chaos_datacoord_podkill.yaml + name: test_datascoord_pod_kill + chaos: chaos_datacoord_pod_kill.yaml expectation: cluster_1_node: insert: succ @@ -53,8 +53,8 @@ Collections: insert: degrade - testcase: - name: test_indexnode_podkill - chaos: chaos_indexnode_podkill.yaml + name: test_indexnode_pod_kill + chaos: chaos_indexnode_pod_kill.yaml expectation: cluster_1_node: index: fail @@ -62,8 +62,8 @@ Collections: index: degrade - testcase: - name: test_indexcoord_podkill - chaos: chaos_indexcoord_podkill.yaml + name: test_indexcoord_pod_kill + chaos: chaos_indexcoord_pod_kill.yaml expectation: cluster_1_node: index: fail @@ -71,8 +71,8 @@ Collections: insert: degrade - testcase: - name: test_proxy_podkill - chaos: chaos_proxy_podkill.yaml + name: test_proxy_pod_kill + chaos: chaos_proxy_pod_kill.yaml expectation: cluster_1_node: create: fail @@ -85,8 +85,8 @@ Collections: insert: fail - testcase: - name: test_rootcoord_podkill - chaos: chaos_rootcoord_podkill.yaml + name: test_rootcoord_pod_kill + chaos: chaos_rootcoord_pod_kill.yaml expectation: cluster_1_node: create: fail @@ -99,8 +99,8 @@ Collections: insert: degrade - testcase: - name: test_etcd_podkill - chaos: chaos_etcd_podkill.yaml + name: test_etcd_pod_kill + chaos: chaos_etcd_pod_kill.yaml expectation: cluster_1_node: create: fail @@ -111,8 +111,8 @@ Collections: query: fail - testcase: - name: test_minio_podkill - chaos: chaos_minio_podkill.yaml + name: test_minio_pod_kill + chaos: chaos_minio_pod_kill.yaml expectation: cluster_1_node: create: fail @@ -123,8 +123,8 @@ Collections: query: fail - testcase: - name: test_pulsar_podkill - chaos: chaos_pulsar_podkill.yaml + name: test_pulsar_pod_kill + chaos: chaos_pulsar_pod_kill.yaml expectation: cluster_1_node: create: fail diff --git a/tests/python_client/chaos/chao_test.sh b/tests/python_client/chaos/chaos_test.sh similarity index 74% rename from tests/python_client/chaos/chao_test.sh rename to tests/python_client/chaos/chaos_test.sh index a982671d9a..3df68d468f 100644 --- a/tests/python_client/chaos/chao_test.sh +++ b/tests/python_client/chaos/chaos_test.sh @@ -14,8 +14,8 @@ echo "platform: $platform" # define chaos testing object release=${1:-"milvus-chaos"} -pod="pulsar" -chaos_type="pod_kill" +pod="proxy" +chaos_type="pod_failure" release="milvus-chaos" ns="chaos-testing" @@ -25,16 +25,24 @@ echo "uninstall milvus if exist" bash uninstall_milvus.sh ${release}|| true echo "install milvus" bash install_milvus.sh ${release} + +# if chaos_type is pod_failure, update replicas +if [ "$chaos_type" == "pod_failure" ]; +then + declare -A pod_map=(["querynode"]="queryNode" ["indexnode"]="indexNode" ["datanode"]="dataNode" ["proxy"]="proxy") + helm upgrade ${release} milvus/milvus --set ${pod_map[${pod}]}.replicas=2 --reuse-values +fi + popd # replace chaos object as defined if [ "$platform" == "Mac" ]; then sed -i "" "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/${chaos_type}\/'/g" constants.py - sed -i "" "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${pod}_podkill.yaml\'/g" constants.py + sed -i "" "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${pod}_${chaos_type}.yaml\'/g" constants.py else sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/${chaos_type}\/'/g" constants.py - sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${pod}_podkill.yaml\'/g" constants.py + sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${pod}_${chaos_type}.yaml\'/g" constants.py fi # run chaos testing