Update simd compatibility test (#27880)

Signed-off-by: yanliang567 <yanliang.qiao@zilliz.com>

tests/python_client/customize/milvus_operator.py

@@ -58,7 +58,7 @@ class MilvusOperator(object):
         Return: custom resource object instance
         """
         new_configs = self._update_configs(configs, template)
-        print(new_configs)
+        log.debug(new_configs)
         namespace = new_configs['metadata'].get('namespace', 'default')
         # apply custom resource object to deploy milvus
         cus_res = CusResource(kind=self.plural, group=self.group,

tests/python_client/customize/template/default.yaml

@@ -1,29 +1,32 @@
-apiVersion: milvus.io/v1alpha1
-kind: MilvusCluster
+apiVersion: milvus.io/v1beta1
+kind: Milvus
 metadata:
   name: my-release
+  namespace: chaos-testing
   labels:
     app: milvus
 spec:
+  config:
+    log:
+      level: debug
+    common:
+      simdType: avx
+  components: {}
   dependencies:
     etcd:
-      inCluster:
-        deletionPolicy: Delete
-        pvcDeletion: true
-    pulsar:
       inCluster:
         deletionPolicy: Delete
         pvcDeletion: true
         values:
-          proxy:
-            configData:
-              httpNumThreads: "100"
+          metrics:
+            podMonitor:
+              enabled: true
     storage:
       inCluster:
         deletionPolicy: Delete
         pvcDeletion: true
-  config:
-    log:
-      level: debug
-  components: {}
+        values:
+          metrics:
+            podMonitor:
+              enabled: true
   

tests/python_client/customize/test_simd_compat.py

@@ -24,11 +24,10 @@ namespace = 'chaos-testing'
 
 def _install_milvus(simd):
     release_name = f"mil-{simd.replace('_','-')}-" + cf.gen_digits_by_length(6)
-    cus_configs = {'spec.components.image': 'milvusdb/milvus:master-latest',
+    cus_configs = {'spec.components.image': 'harbor.milvus.io/milvus/milvus:master-latest',
                    'metadata.namespace': namespace,
                    'metadata.name': release_name,
-                   'spec.components.proxy.serviceType': 'LoadBalancer',
-                   'spec.config.knowhere.simdType': simd
+                   'spec.config.common.simdType': simd
                    }
     milvus_op = MilvusOperator()
     log.info(f"install milvus with configs: {cus_configs}")
@@ -52,17 +51,17 @@ class TestSimdCompatibility:
         milvus_op.uninstall(self.release_name, namespace)
 
     @pytest.mark.tags(CaseLabel.L3)
-    @pytest.mark.parametrize('simd_id', [i for i in range(len(supported_simd_types))])
-    def test_simd_compat_e2e(self, simd_id):
+    @pytest.mark.parametrize('simd', supported_simd_types)
+    def test_simd_compat_e2e(self, simd):
         """
        steps
        1. [test_milvus_install]: set up milvus with customized simd configured
        2. [test_simd_compat_e2e]: verify milvus is working well
        4. [test_milvus_cleanup]: delete milvus instances in teardown
        """
-        simd = supported_simd_types[simd_id]
         log.info(f"start to install milvus with simd {simd}")
         release_name, host, port = _install_milvus(simd)
+        time.sleep(10)
         self.release_name = release_name
         assert host is not None
         conn = connections.connect("default", host=host, port=port)
@@ -70,38 +69,55 @@ class TestSimdCompatibility:
         mil = MilvusSys(alias="default")
         log.info(f"milvus build version: {mil.build_version}")
         log.info(f"milvus simdType: {mil.simd_type}")
-        assert str(mil.simd_type).lower() in [simd_type.lower() for simd_type in supported_simd_types[simd_id:]]
+        assert str(mil.simd_type).lower() == simd.lower()
 
         log.info(f"start to e2e verification: {simd}")
         # create
-        name = cf.gen_unique_str("compat")
+        prefix = "simd_"
+        name = cf.gen_unique_str(prefix)
         t0 = time.time()
         collection_w = ApiCollectionWrapper()
-        collection_w.init_collection(name=name,
-                                     schema=cf.gen_default_collection_schema(),
-                                     timeout=40)
+        collection_w.init_collection_wrap(name=name)
         tt = time.time() - t0
         assert collection_w.name == name
         entities = collection_w.num_entities
         log.info(f"assert create collection: {tt}, init_entities: {entities}")
 
         # insert
-        data = cf.gen_default_list_data()
-        t0 = time.time()
-        _, res = collection_w.insert(data)
-        tt = time.time() - t0
-        log.info(f"assert insert: {tt}")
-        assert res
+        for _ in range(10):
+            data = cf.gen_default_list_data(nb=300)
+            t0 = time.time()
+            _, res = collection_w.insert(data)
+            tt = time.time() - t0
+            log.info(f"assert insert: {tt}")
+            assert res
 
         # flush
         t0 = time.time()
+        _, check_result = collection_w.flush(timeout=180)
+        assert check_result
         assert collection_w.num_entities == len(data[0]) + entities
         tt = time.time() - t0
         entities = collection_w.num_entities
         log.info(f"assert flush: {tt}, entities: {entities}")
 
-        # search
+        # index
+        index_params = {"index_type": "IVF_SQ8", "params": {"nlist": 64}, "metric_type": "L2"}
+        t0 = time.time()
+        index, _ = collection_w.create_index(field_name=ct.default_float_vec_field_name,
+                                             index_params=index_params,
+                                             index_name=cf.gen_unique_str())
+        index, _ = collection_w.create_index(field_name=ct.default_string_field_name,
+                                             index_params={},
+                                             index_name=cf.gen_unique_str())
+        tt = time.time() - t0
+        log.info(f"assert index: {tt}")
+        assert len(collection_w.indexes) == 2
+
+        # load
         collection_w.load()
+
+        # search
         search_vectors = cf.gen_vectors(1, ct.default_dim)
         search_params = {"metric_type": "L2", "params": {"nprobe": 16}}
         t0 = time.time()
@@ -111,37 +127,35 @@ class TestSimdCompatibility:
         tt = time.time() - t0
         log.info(f"assert search: {tt}")
         assert len(res_1) == 1
+
+        # release
         collection_w.release()
 
-        # index
+        # insert
         d = cf.gen_default_list_data()
         collection_w.insert(d)
-        log.info(f"assert index entities: {collection_w.num_entities}")
-        _index_params = {"index_type": "IVF_SQ8", "params": {"nlist": 64}, "metric_type": "L2"}
-        t0 = time.time()
-        index, _ = collection_w.create_index(field_name=ct.default_float_vec_field_name,
-                                             index_params=_index_params,
-                                             name=cf.gen_unique_str())
-        tt = time.time() - t0
-        log.info(f"assert index: {tt}")
-        assert len(collection_w.indexes) == 1
 
         # search
         t0 = time.time()
         collection_w.load()
         tt = time.time() - t0
         log.info(f"assert load: {tt}")
-        search_vectors = cf.gen_vectors(1, ct.default_dim)
+        nq = 5
+        topk = 5
+        search_vectors = cf.gen_vectors(nq, ct.default_dim)
         t0 = time.time()
-        res_1, _ = collection_w.search(data=search_vectors,
-                                       anns_field=ct.default_float_vec_field_name,
-                                       param=search_params, limit=1)
+        res, _ = collection_w.search(data=search_vectors,
+                                     anns_field=ct.default_float_vec_field_name,
+                                     param=search_params, limit=topk)
         tt = time.time() - t0
         log.info(f"assert search: {tt}")
-
+        assert len(res) == nq
+        assert len(res[0]) <= topk
         # query
-        term_expr = f'{ct.default_int64_field_name} in [1001,1201,4999,2999]'
+        term_expr = f'{ct.default_int64_field_name} in [1, 2, 3, 4]'
         t0 = time.time()
         res, _ = collection_w.query(term_expr)
         tt = time.time() - t0
         log.info(f"assert query result {len(res)}: {tt}")
+        assert len(res) >= 4
+