mirror of https://github.com/milvus-io/milvus.git
138 lines
5.6 KiB
Python
138 lines
5.6 KiB
Python
import time
|
|
|
|
import pytest
|
|
from time import sleep
|
|
from pymilvus import connections, db
|
|
from chaos.checker import (
|
|
DatabaseCreateChecker,
|
|
DatabaseDropChecker,
|
|
CollectionCreateChecker,
|
|
CollectionDropChecker,
|
|
PartitionCreateChecker,
|
|
PartitionDropChecker,
|
|
CollectionLoadChecker,
|
|
CollectionReleaseChecker,
|
|
PartitionLoadChecker,
|
|
PartitionReleaseChecker,
|
|
IndexCreateChecker,
|
|
IndexDropChecker,
|
|
InsertChecker,
|
|
UpsertChecker,
|
|
DeleteChecker,
|
|
FlushChecker,
|
|
SearchChecker,
|
|
QueryChecker,
|
|
Op,
|
|
EventRecords,
|
|
ResultAnalyzer
|
|
)
|
|
from utils.util_log import test_log as log
|
|
from utils.util_k8s import wait_pods_ready, get_milvus_instance_name
|
|
from chaos import chaos_commons as cc
|
|
from common.common_type import CaseLabel
|
|
from common.milvus_sys import MilvusSys
|
|
from chaos.chaos_commons import assert_statistic
|
|
from chaos import constants
|
|
from delayed_assert import assert_expectations
|
|
|
|
|
|
class TestBase:
|
|
expect_create = constants.SUCC
|
|
expect_insert = constants.SUCC
|
|
expect_flush = constants.SUCC
|
|
expect_index = constants.SUCC
|
|
expect_search = constants.SUCC
|
|
expect_query = constants.SUCC
|
|
host = '127.0.0.1'
|
|
port = 19530
|
|
_chaos_config = None
|
|
health_checkers = {}
|
|
|
|
|
|
class TestOperations(TestBase):
|
|
|
|
@pytest.fixture(scope="function", autouse=True)
|
|
def connection(self, host, port, user, password, milvus_ns, database_name):
|
|
if user and password:
|
|
# log.info(f"connect to {host}:{port} with user {user} and password {password}")
|
|
connections.connect('default', host=host, port=port, user=user, password=password, secure=True)
|
|
else:
|
|
connections.connect('default', host=host, port=port)
|
|
if connections.has_connection("default") is False:
|
|
raise Exception("no connections")
|
|
all_dbs = db.list_database()
|
|
if database_name not in all_dbs:
|
|
db.create_database(database_name)
|
|
db.using_database(database_name)
|
|
log.info(f"connect to milvus {host}:{port}, db {database_name} successfully")
|
|
self.host = host
|
|
self.port = port
|
|
self.user = user
|
|
self.password = password
|
|
self.milvus_sys = MilvusSys(alias='default')
|
|
self.milvus_ns = milvus_ns
|
|
self.release_name = get_milvus_instance_name(self.milvus_ns, milvus_sys=self.milvus_sys)
|
|
|
|
def init_health_checkers(self, collection_name=None):
|
|
c_name = collection_name
|
|
checkers = {
|
|
Op.create_db: DatabaseCreateChecker(),
|
|
Op.create_collection: CollectionCreateChecker(collection_name=c_name),
|
|
Op.create_partition: PartitionCreateChecker(collection_name=c_name),
|
|
Op.drop_db: DatabaseDropChecker(),
|
|
Op.drop_collection: CollectionDropChecker(collection_name=c_name),
|
|
Op.drop_partition: PartitionDropChecker(collection_name=c_name),
|
|
Op.load_collection: CollectionLoadChecker(collection_name=c_name),
|
|
Op.load_partition: PartitionLoadChecker(collection_name=c_name),
|
|
Op.release_collection: CollectionReleaseChecker(collection_name=c_name),
|
|
Op.release_partition: PartitionReleaseChecker(collection_name=c_name),
|
|
Op.insert: InsertChecker(collection_name=c_name),
|
|
Op.upsert: UpsertChecker(collection_name=c_name),
|
|
Op.flush: FlushChecker(collection_name=c_name),
|
|
Op.create_index: IndexCreateChecker(collection_name=c_name),
|
|
Op.drop_index: IndexDropChecker(collection_name=c_name),
|
|
Op.search: SearchChecker(collection_name=c_name),
|
|
Op.query: QueryChecker(collection_name=c_name),
|
|
Op.delete: DeleteChecker(collection_name=c_name),
|
|
Op.drop: CollectionDropChecker(collection_name=c_name)
|
|
}
|
|
self.health_checkers = checkers
|
|
|
|
@pytest.mark.tags(CaseLabel.L3)
|
|
def test_operations(self, request_duration, is_check):
|
|
# start the monitor threads to check the milvus ops
|
|
log.info("*********************Test Start**********************")
|
|
log.info(connections.get_connection_addr('default'))
|
|
event_records = EventRecords()
|
|
c_name = None
|
|
event_records.insert("init_health_checkers", "start")
|
|
self.init_health_checkers(collection_name=c_name)
|
|
event_records.insert("init_health_checkers", "finished")
|
|
tasks = cc.start_monitor_threads(self.health_checkers)
|
|
log.info("*********************Load Start**********************")
|
|
# wait request_duration
|
|
request_duration = request_duration.replace("h", "*3600+").replace("m", "*60+").replace("s", "")
|
|
if request_duration[-1] == "+":
|
|
request_duration = request_duration[:-1]
|
|
request_duration = eval(request_duration)
|
|
for i in range(10):
|
|
sleep(request_duration // 10)
|
|
# add an event so that the chaos can start to apply
|
|
if i == 3:
|
|
event_records.insert("init_chaos", "ready")
|
|
for k, v in self.health_checkers.items():
|
|
v.check_result()
|
|
if is_check:
|
|
assert_statistic(self.health_checkers, succ_rate_threshold=0.98)
|
|
assert_expectations()
|
|
# wait all pod ready
|
|
wait_pods_ready(self.milvus_ns, f"app.kubernetes.io/instance={self.release_name}")
|
|
time.sleep(60)
|
|
cc.check_thread_status(tasks)
|
|
for k, v in self.health_checkers.items():
|
|
v.pause()
|
|
ra = ResultAnalyzer()
|
|
ra.get_stage_success_rate()
|
|
ra.show_result_table()
|
|
log.info("*********************Chaos Test Completed**********************")
|