Make CI stats testcases and remove skip label on some testcases

Signed-off-by: del-zhenwu <zhenxiang.li@zilliz.com>

tests/python_test/collection/test_collection_stats.py

@@ -123,8 +123,7 @@ class TestGetCollectionStats:
         stats = connect.get_collection_stats(collection)
         assert stats[row_count] == nb
 
-    @pytest.mark.skip("delete_by_id not support yet")
-    def test_get_collection_stats_after_delete(self, connect, collection):
+    def _test_get_collection_stats_after_delete(self, connect, collection):
         '''
         target: get row count with collection_stats
         method: add and delete entities, check count in collection info
@@ -142,8 +141,7 @@ class TestGetCollectionStats:
 
     # TODO: enable
     @pytest.mark.level(2)
-    @pytest.mark.skip("no compact")
-    def test_get_collection_stats_after_compact_parts(self, connect, collection):
+    def _test_get_collection_stats_after_compact_parts(self, connect, collection):
         '''
         target: get row count with collection_stats
         method: add and delete entities, and compact collection, check count in collection info
@@ -165,8 +163,7 @@ class TestGetCollectionStats:
         compact_after = stats["partitions"][0]["segments"][0]["data_size"]
         assert compact_before == compact_after
 
-    @pytest.mark.skip("no compact")
-    def test_get_collection_stats_after_compact_delete_one(self, connect, collection):
+    def _test_get_collection_stats_after_compact_delete_one(self, connect, collection):
         '''
         target: get row count with collection_stats
         method: add and delete one entity, and compact collection, check count in collection info

tests/python_test/collection/test_create_collection.py

@@ -60,8 +60,7 @@ class TestCreateCollection:
         connect.create_collection(collection_name, fields)
         assert connect.has_collection(collection_name)
 
-    @pytest.mark.skip("no segment_row_limit")
-    def test_create_collection_segment_row_limit(self, connect, get_segment_row_limit):
+    def _test_create_collection_segment_row_limit(self, connect, get_segment_row_limit):
         '''
         target: test create normal collection with different fields
         method: create collection with diff segment_row_limit
@@ -204,8 +203,7 @@ class TestCreateCollectionInvalid(object):
         yield request.param
 
     @pytest.mark.level(2)
-    @pytest.mark.skip("no segment_row_limit")
-    def test_create_collection_with_invalid_segment_row_limit(self, connect, get_segment_row_limit):
+    def _test_create_collection_with_invalid_segment_row_limit(self, connect, get_segment_row_limit):
         collection_name = gen_unique_str()
         fields = copy.deepcopy(default_fields)
         fields["segment_row_limit"] = get_segment_row_limit
@@ -256,8 +254,7 @@ class TestCreateCollectionInvalid(object):
         with pytest.raises(Exception) as e:
             connect.create_collection(collection_name, fields)
 
-    @pytest.mark.skip("no segment_row_limit")
-    def test_create_collection_no_segment_row_limit(self, connect):
+    def _test_create_collection_no_segment_row_limit(self, connect):
         '''
         target: test create collection with no segment_row_limit params
         method: create collection with correct params

tests/python_test/conftest.py

@@ -41,8 +41,22 @@ def pytest_runtest_setup(item):
 
 def pytest_runtestloop(session):
     if session.config.getoption('--dry-run'):
+        total_num = 0
+        file_num = 0
+        tags_num = 0
+        res = {"total_num": total_num, "tags_num": tags_num}
         for item in session.items:
             print(item.nodeid)
+            if item.fspath.basename not in res:
+                res.update({item.fspath.basename: {"total": 1, "tags": 0}})
+            else:
+                res[item.fspath.basename]["total"] += 1
+            res["total_num"] += 1
+            for marker in item.own_markers:
+                if "0331" in marker.args:
+                    res["tags_num"] += 1
+                    res[item.fspath.basename]["tags"] += 1
+        print(res)
         return True
 
 

tests/python_test/entity/test_insert.py

@@ -167,8 +167,7 @@ class TestInsertBase:
         res = connect.search(collection, default_single_query)
         assert res
 
-    @pytest.mark.skip("No segment_row_limit")
-    def test_insert_segment_row_count(self, connect, collection):
+    def _test_insert_segment_row_count(self, connect, collection):
         nb = default_segment_row_limit + 1
         res_ids = connect.insert(collection, gen_entities(nb))
         connect.flush([collection])

tests/python_test/entity/test_search.py

@@ -187,7 +187,6 @@ class TestSearchBase:
             with pytest.raises(Exception) as e:
                 res = connect.search(collection, query)
 
-    # @pytest.mark.skip("r0.3-test")
     def test_search_field(self, connect, collection, get_top_k, get_nq):
         '''
         target: test basic search function, all the search params is correct, change top-k value
@@ -211,8 +210,7 @@ class TestSearchBase:
             with pytest.raises(Exception):
                 connect.search(collection, query)
 
-    @pytest.mark.skip("delete_entity_by_id not ready")
-    def test_search_after_delete(self, connect, collection, get_top_k, get_nq):
+    def _test_search_after_delete(self, connect, collection, get_top_k, get_nq):
         '''
         target: test basic search function before and after deletion, all the search params is
                 correct, change top-k value.
@@ -287,7 +285,6 @@ class TestSearchBase:
             assert res[0]._distances[0] < epsilon
             assert check_id_result(res[0], ids[0])
 
-    # @pytest.mark.skip("r0.3-test")
     def test_search_after_index_different_metric_type(self, connect, collection, get_simple_index):
         '''
         target: test search with different metric_type
@@ -307,7 +304,6 @@ class TestSearchBase:
         assert len(res[0]) == default_top_k
         assert res[0]._distances[0] > res[0]._distances[default_top_k - 1]
 
-    # @pytest.mark.skip("r0.3-test")
     @pytest.mark.level(2)
     def test_search_index_empty_partition(self, connect, collection, get_simple_index, get_top_k, get_nq):
         '''
@@ -341,7 +337,6 @@ class TestSearchBase:
             res = connect.search(collection, query, partition_tags=[default_tag])
             assert len(res[0]) == 0
 
-    # @pytest.mark.skip("r0.3-test")
     @pytest.mark.level(2)
     def test_search_index_partition(self, connect, collection, get_simple_index, get_top_k, get_nq):
         '''
@@ -372,7 +367,6 @@ class TestSearchBase:
                 assert res[0]._distances[0] < epsilon
                 assert check_id_result(res[0], ids[0])
 
-    # @pytest.mark.skip("r0.3-test")
     @pytest.mark.level(2)
     def test_search_index_partition_not_existed(self, connect, collection, get_top_k, get_nq):
         '''
@@ -513,7 +507,6 @@ class TestSearchBase:
         assert check_id_result(res[0], ids[0])
         assert res[0]._distances[0] >= 1 - gen_inaccuracy(res[0]._distances[0])
 
-    # @pytest.mark.skip("r0.3-test")
     @pytest.mark.level(2)
     def test_search_ip_index_empty_partition(self, connect, collection, get_simple_index, get_top_k, get_nq):
         '''
@@ -957,7 +950,6 @@ class TestSearchBase:
                 assert res[i]._distances[0] < epsilon
                 assert res[i]._distances[1] > epsilon
 
-    # @pytest.mark.skip("r0.3-test")
     def test_query_entities_with_field_less_than_top_k(self, connect, id_collection):
         """
         target: test search with field, and let return entities less than topk
@@ -1064,7 +1056,6 @@ class TestSearchDSL(object):
     ******************************************************************
     """
 
-    # @pytest.mark.skip("r0.3-test")
     @pytest.mark.level(2)
     def test_query_term_value_not_in(self, connect, collection):
         '''
@@ -1082,7 +1073,6 @@ class TestSearchDSL(object):
         # TODO:
 
     # TODO:
-    # @pytest.mark.skip("r0.3-test")
     @pytest.mark.level(2)
     def test_query_term_value_all_in(self, connect, collection):
         '''
@@ -1099,7 +1089,6 @@ class TestSearchDSL(object):
         # TODO:
 
     # TODO:
-    # @pytest.mark.skip("r0.3-test")
     @pytest.mark.level(2)
     def test_query_term_values_not_in(self, connect, collection):
         '''
@@ -1116,7 +1105,6 @@ class TestSearchDSL(object):
         assert len(res[0]) == 0
         # TODO:
 
-    # @pytest.mark.skip("r0.3-test")
     def test_query_term_values_all_in(self, connect, collection):
         '''
         method: build query with vector and term expr, with all term can be filtered
@@ -1136,7 +1124,6 @@ class TestSearchDSL(object):
                 assert result.id in ids[:limit]
         # TODO:
 
-    # @pytest.mark.skip("r0.3-test")
     def test_query_term_values_parts_in(self, connect, collection):
         '''
         method: build query with vector and term expr, with parts of term can be filtered
@@ -1154,7 +1141,6 @@ class TestSearchDSL(object):
         # TODO:
 
     # TODO:
-    # @pytest.mark.skip("r0.3-test")
     @pytest.mark.level(2)
     def test_query_term_values_repeat(self, connect, collection):
         '''
@@ -1244,7 +1230,6 @@ class TestSearchDSL(object):
             res = connect.search(collection, query)
 
     # TODO
-    # @pytest.mark.skip("r0.3-test")
     @pytest.mark.level(2)
     def test_query_term_field_named_term(self, connect, collection):
         '''
@@ -1343,7 +1328,6 @@ class TestSearchDSL(object):
         with pytest.raises(Exception) as e:
             res = connect.search(collection, query)
 
-    # @pytest.mark.skip("r0.3-test")
     @pytest.mark.level(2)
     def test_query_range_invalid_ranges(self, connect, collection):
         '''
@@ -1366,7 +1350,6 @@ class TestSearchDSL(object):
     def get_valid_ranges(self, request):
         return request.param
 
-    # @pytest.mark.skip("r0.3-test")
     @pytest.mark.level(2)
     def test_query_range_valid_ranges(self, connect, collection, get_valid_ranges):
         '''
@@ -1404,7 +1387,6 @@ class TestSearchDSL(object):
     """
 
     # TODO
-    # @pytest.mark.skip("r0.3-test")
     @pytest.mark.level(2)
     def test_query_multi_term_has_common(self, connect, collection):
         '''
@@ -1422,7 +1404,6 @@ class TestSearchDSL(object):
         assert len(res[0]) == default_top_k
 
     # TODO
-    # @pytest.mark.skip("r0.3-test")
     @pytest.mark.level(2)
     def test_query_multi_term_no_common(self, connect, collection):
         '''
@@ -1440,7 +1421,6 @@ class TestSearchDSL(object):
         assert len(res[0]) == 0
 
     # TODO
-    # @pytest.mark.skip("r0.3-test")
     def test_query_multi_term_different_fields(self, connect, collection):
         '''
          method: build query with multi range with same field, and ranges no common
@@ -1475,7 +1455,6 @@ class TestSearchDSL(object):
             res = connect.search(collection, query)
 
     # TODO
-    # @pytest.mark.skip("r0.3-test")
     @pytest.mark.level(2)
     def test_query_multi_range_has_common(self, connect, collection):
         '''
@@ -1493,7 +1472,6 @@ class TestSearchDSL(object):
         assert len(res[0]) == default_top_k
 
     # TODO
-    # @pytest.mark.skip("r0.3-test")
     @pytest.mark.level(2)
     def test_query_multi_range_no_common(self, connect, collection):
         '''
@@ -1511,7 +1489,6 @@ class TestSearchDSL(object):
         assert len(res[0]) == 0
 
     # TODO
-    # @pytest.mark.skip("r0.3-test")
     @pytest.mark.level(2)
     def test_query_multi_range_different_fields(self, connect, collection):
         '''
@@ -1552,7 +1529,6 @@ class TestSearchDSL(object):
     """
 
     # TODO
-    # @pytest.mark.skip("r0.3-test")
     @pytest.mark.level(2)
     def test_query_single_term_range_has_common(self, connect, collection):
         '''
@@ -1570,7 +1546,6 @@ class TestSearchDSL(object):
         assert len(res[0]) == default_top_k
 
     # TODO
-    # @pytest.mark.skip("r0.3-test")
     def test_query_single_term_range_no_common(self, connect, collection):
         '''
         method: build query with single term single range

tests/python_test/test_index.py

@@ -650,7 +650,6 @@ class TestIndexBinary:
       The following cases are used to test `describe_index` function
     ***************************************************************
     """
-    @pytest.mark.skip("repeat with test_create_index binary")
     def test_get_index_info(self, connect, binary_collection, get_jaccard_index):
         '''
         target: test describe index interface
@@ -670,7 +669,6 @@ class TestIndexBinary:
                         if "index_type" in file:
                             assert file["index_type"] == get_jaccard_index["index_type"]
 
-    @pytest.mark.skip("repeat with test_create_index_partition binary")
     def test_get_index_info_partition(self, connect, binary_collection, get_jaccard_index):
         '''
         target: test describe index interface

tests/python_test/test_partition.py

@@ -157,7 +157,6 @@ class TestCreateBase:
             message = getattr(e, 'message', "The exception does not contain the field of message.")
             assert message == "partitionID of partitionName:%s can not be find" % tag_new
 
-    @pytest.mark.tags("fail")
     def test_create_partition_insert_same_tags(self, connect, id_collection):
         '''
         target: test create partition, and insert vectors, check status returned
@@ -175,7 +174,6 @@ class TestCreateBase:
         res = connect.get_collection_stats(id_collection)
         assert res["row_count"] == default_nb * 2
 
-    @pytest.mark.tags("fail")
     @pytest.mark.level(2)
     def test_create_partition_insert_same_tags_two_collections(self, connect, collection):
         '''