mirror of https://github.com/milvus-io/milvus.git
zhuwenxing
GitHub
parent
82fefd67ef
commit
8f86993569
|
|
@ -18,11 +18,11 @@ def assert_statistic(checkers, expectations={}):
|
|||
succ_rate = checkers[k].succ_rate()
|
||||
total = checkers[k].total()
|
||||
if expectations.get(k, '') == constants.FAIL:
|
||||
log.debug(f"Expect Fail: {str(k)} succ rate {succ_rate}, total: {total}")
|
||||
log.info(f"Expect Fail: {str(k)} succ rate {succ_rate}, total: {total}")
|
||||
expect(succ_rate < 0.49 or total < 2,
|
||||
f"Expect Fail: {str(k)} succ rate {succ_rate}, total: {total}")
|
||||
else:
|
||||
log.debug(f"Expect Succ: {str(k)} succ rate {succ_rate}, total: {total}")
|
||||
log.info(f"Expect Succ: {str(k)} succ rate {succ_rate}, total: {total}")
|
||||
expect(succ_rate > 0.90 or total > 2,
|
||||
f"Expect Succ: {str(k)} succ rate {succ_rate}, total: {total}")
|
||||
|
||||
|
|
@ -48,14 +48,14 @@ class TestChaosBase:
|
|||
test_chaos = t.get('testcase', {}).get('chaos', {})
|
||||
if test_chaos in chaos_yaml:
|
||||
expects = t.get('testcase', {}).get('expectation', {}).get('cluster_1_node', {})
|
||||
log.debug(f"yaml.expects: {expects}")
|
||||
log.info(f"yaml.expects: {expects}")
|
||||
self.expect_create = expects.get(Op.create.value, constants.SUCC)
|
||||
self.expect_insert = expects.get(Op.insert.value, constants.SUCC)
|
||||
self.expect_flush = expects.get(Op.flush.value, constants.SUCC)
|
||||
self.expect_index = expects.get(Op.index.value, constants.SUCC)
|
||||
self.expect_search = expects.get(Op.search.value, constants.SUCC)
|
||||
self.expect_query = expects.get(Op.query.value, constants.SUCC)
|
||||
log.debug(f"self.expects: create:{self.expect_create}, insert:{self.expect_insert}, "
|
||||
log.info(f"self.expects: create:{self.expect_create}, insert:{self.expect_insert}, "
|
||||
f"flush:{self.expect_flush}, index:{self.expect_index}, "
|
||||
f"search:{self.expect_search}, query:{self.expect_query}")
|
||||
return True
|
||||
|
|
@ -93,23 +93,23 @@ class TestChaos(TestChaosBase):
|
|||
chaos_opt.delete_chaos_object(meta_name, raise_ex=False)
|
||||
for k, ch in self.health_checkers.items():
|
||||
ch.terminate()
|
||||
log.debug(f"tear down: checker {k} terminated")
|
||||
log.info(f"tear down: checker {k} terminated")
|
||||
sleep(2)
|
||||
for k, t in self.checker_threads.items():
|
||||
log.debug(f"Thread {k} is_alive(): {t.is_alive()}")
|
||||
log.info(f"Thread {k} is_alive(): {t.is_alive()}")
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L3)
|
||||
@pytest.mark.parametrize('chaos_yaml', get_chaos_yamls())
|
||||
def test_chaos(self, chaos_yaml):
|
||||
# start the monitor threads to check the milvus ops
|
||||
log.debug("*********************Chaos Test Start**********************")
|
||||
log.debug(connections.get_connection_addr('default'))
|
||||
log.info("*********************Chaos Test Start**********************")
|
||||
log.info(connections.get_connection_addr('default'))
|
||||
self.checker_threads = start_monitor_threads(self.health_checkers)
|
||||
|
||||
# parse chaos object
|
||||
chaos_config = gen_experiment_config(chaos_yaml)
|
||||
self._chaos_config = chaos_config # cache the chaos config for tear down
|
||||
log.debug(chaos_config)
|
||||
log.info(chaos_config)
|
||||
|
||||
# parse the test expectations in testcases.yaml
|
||||
if self.parser_testcase_config(chaos_yaml) is False:
|
||||
|
|
@ -120,13 +120,13 @@ class TestChaos(TestChaosBase):
|
|||
sleep(constants.WAIT_PER_OP*2)
|
||||
|
||||
# assert statistic:all ops 100% succ
|
||||
log.debug("******1st assert before chaos: ")
|
||||
log.info("******1st assert before chaos: ")
|
||||
assert_statistic(self.health_checkers)
|
||||
|
||||
# apply chaos object
|
||||
chaos_opt = ChaosOpt(chaos_config['kind'])
|
||||
chaos_opt.create_chaos_object(chaos_config)
|
||||
log.debug("chaos injected")
|
||||
log.info("chaos injected")
|
||||
sleep(constants.WAIT_PER_OP * 2.1)
|
||||
# reset counting
|
||||
reset_counting(self.health_checkers)
|
||||
|
|
@ -135,10 +135,10 @@ class TestChaos(TestChaosBase):
|
|||
sleep(constants.WAIT_PER_OP*4)
|
||||
|
||||
for k, t in self.checker_threads.items():
|
||||
log.debug(f"10s later: Thread {k} is_alive(): {t.is_alive()}")
|
||||
log.info(f"10s later: Thread {k} is_alive(): {t.is_alive()}")
|
||||
|
||||
# assert statistic
|
||||
log.debug("******2nd assert after chaos injected: ")
|
||||
log.info("******2nd assert after chaos injected: ")
|
||||
assert_statistic(self.health_checkers,
|
||||
expectations={Op.create: self.expect_create,
|
||||
Op.insert: self.expect_insert,
|
||||
|
|
@ -151,9 +151,9 @@ class TestChaos(TestChaosBase):
|
|||
# delete chaos
|
||||
meta_name = chaos_config.get('metadata', None).get('name', None)
|
||||
chaos_opt.delete_chaos_object(meta_name)
|
||||
log.debug("chaos deleted")
|
||||
log.info("chaos deleted")
|
||||
for k, t in self.checker_threads.items():
|
||||
log.debug(f"Thread {k} is_alive(): {t.is_alive()}")
|
||||
log.info(f"Thread {k} is_alive(): {t.is_alive()}")
|
||||
sleep(2)
|
||||
|
||||
# reconnect if needed
|
||||
|
|
@ -167,10 +167,10 @@ class TestChaos(TestChaosBase):
|
|||
sleep(constants.WAIT_PER_OP*5)
|
||||
|
||||
# assert statistic: all ops success again
|
||||
log.debug("******3rd assert after chaos deleted: ")
|
||||
log.info("******3rd assert after chaos deleted: ")
|
||||
assert_statistic(self.health_checkers)
|
||||
|
||||
# assert all expectations
|
||||
assert_expectations()
|
||||
|
||||
log.debug("*********************Chaos Test Completed**********************")
|
||||
log.info("*********************Chaos Test Completed**********************")
|
||||
|
|
|
|||
Loading…
Reference in New Issue