Test suite runner enhancements:

To chase down a set of tests that were failing together, we wanted to run
deeper trees of tests. We changed the behavior of the --pkg argument to
runtests.py so it runs tests under a package and its subpackages, instead
of for a single node.

It gave us more flexibility in what tests to run. This also includes the
--exclude option.

To run tests for a single node, ".tests" can be appended to the package path.

Tira & George.

web/pgadmin/utils/route.py

@@ -54,27 +54,23 @@ class TestsGeneratorRegistry(ABCMeta):
         ABCMeta.__init__(cls, name, bases, d)
 
     @classmethod
-    def load_generators(cls, pkg):
+    def load_generators(cls, pkg_root, exclude_pkgs):
 
         cls.registry = dict()
 
+        all_modules = []
+
+        all_modules += find_modules(pkg_root, False, True)
+
         # Check for SERVER mode
-        if config.SERVER_MODE:
-            for module_name in find_modules(pkg, False, True):
-                try:
-                    if "tests." in str(module_name):
-                        import_module(module_name)
-                except ImportError:
-                    traceback.print_exc(file=sys.stderr)
-        else:
-            for module_name in find_modules(pkg, False, True):
-                try:
-                    # Exclude the test cases in browser node if SERVER_MODE
-                    # is False
-                    if "pgadmin.browser.tests" not in module_name:
-                        import_module(module_name)
-                except ImportError:
-                    traceback.print_exc(file=sys.stderr)
+        for module_name in all_modules:
+            try:
+                if "tests." in str(module_name) and not any(
+                        str(module_name).startswith('pgadmin.' + str(exclude_pkg)) for exclude_pkg in exclude_pkgs
+                ):
+                    import_module(module_name)
+            except ImportError:
+                traceback.print_exc(file=sys.stderr)
 
 
 import six

web/regression/README

@@ -115,10 +115,26 @@ Execution:
      run 'python runtests.py --pkg all' or just:
          'python runtests.py'
 
+- Execute test framework for entire package
+
+     Example 1) Run test framework for 'browser' package
+     run 'python runtests.py --pkg browser'
+
+     Example 2) Run test framework for 'database' package
+     run 'python runtests.py --pkg browser.server_groups.servers.databases'
+
 - Execute test framework for single node at a time
 
      Example 1) Run test framework for 'browser' node
-     run 'python runtests.py --pkg browser'
+     run 'python runtests.py --pkg browser.tests'
 
      Example 2) Run test framework for 'database' node
-     run 'python runtests.py --pkg browser.server_groups.servers.databases'
+     run 'python runtests.py --pkg browser.server_groups.servers.databases.tests'
+
+- Exclude a package and its subpackages when running tests:
+
+    Example: exclude acceptance tests but run all others:
+    run 'python runtests.py --exclude acceptance'
+
+    Example: exclude multiple packages:
+    run 'python runtests.py --exclude browser.server_groups.servers.databases,browser.server_groups.servers.tablespaces'

web/regression/runtests.py

@@ -138,12 +138,20 @@ def get_test_modules(arguments):
 
     from pgadmin.utils.route import TestsGeneratorRegistry
 
+    exclude_pkgs = []
+
+    if not config.SERVER_MODE:
+        exclude_pkgs.append("browser.tests")
+    if arguments['exclude'] is not None:
+        exclude_pkgs += arguments['exclude'].split(',')
+
     # Load the test modules which are in given package(i.e. in arguments.pkg)
     if arguments['pkg'] is None or arguments['pkg'] == "all":
-        TestsGeneratorRegistry.load_generators('pgadmin')
+        TestsGeneratorRegistry.load_generators('pgadmin', exclude_pkgs)
     else:
-        TestsGeneratorRegistry.load_generators('pgadmin.%s.tests' %
-                                               arguments['pkg'])
+        TestsGeneratorRegistry.load_generators('pgadmin.%s' %
+                                               arguments['pkg'],
+                                               exclude_pkgs)
 
     # Sort module list so that test suite executes the test cases sequentially
     module_list = TestsGeneratorRegistry.registry.items()
@@ -163,7 +171,9 @@ def add_arguments():
 
     parser = argparse.ArgumentParser(description='Test suite for pgAdmin4')
     parser.add_argument('--pkg', help='Executes the test cases of particular'
-                                      ' package')
+                                      ' package and subpackages')
+    parser.add_argument('--exclude', help='Skips execution of the test '
+                                          'cases of particular package and sub-packages')
     arg = parser.parse_args()
 
     return arg
@@ -268,7 +278,7 @@ if __name__ == '__main__':
             print("\n=============Running the test cases for '%s'============="
                   % server['name'], file=sys.stderr)
             # Create test server
-            test_utils.create_parent_server_node(server, node_name)
+            test_utils.create_parent_server_node(server)
 
             suite = get_suite(test_module_list, server, test_client)
             tests = unittest.TextTestRunner(stream=sys.stderr,
@@ -284,7 +294,7 @@ if __name__ == '__main__':
                 failure = True
 
             # Delete test server
-            # test_utils.delete_test_server(test_client)
+            test_utils.delete_test_server(test_client)
     except SystemExit:
         drop_objects()
 

web/regression/test_utils.py

@@ -230,53 +230,30 @@ def add_schema_to_parent_node_dict(srv_id, db_id, schema_id, schema_name):
                                                   "schema_name": schema_name})
 
 
-def create_parent_server_node(server_info, node_name):
+def create_parent_server_node(server_info):
     """
     This function create the test server which will act as parent server,
     the other node will add under this server
     :param server_info: server details
     :type server_info: dict
-    :param node_name: node name
-    :type node_name: str
     :return: None
     """
     srv_id = create_server(server_info)
-    if node_name == "databases":
-        # Create test database
-        test_db_name = "test_db_%s" % str(uuid.uuid4())[1:6]
-        db_id = create_database(server_info, test_db_name)
-        add_db_to_parent_node_dict(srv_id, db_id, test_db_name)
-    elif node_name == "schemas":
-        test_db_name = "test_db_%s" % str(uuid.uuid4())[1:6]
-        db_id = create_database(server_info, test_db_name)
-        add_db_to_parent_node_dict(srv_id, db_id, test_db_name)
-        # Create schema
-        schema_name = "test_schema_%s" % str(uuid.uuid4())[1:6]
-        connection = get_db_connection(test_db_name,
-                                       server_info['username'],
-                                       server_info['db_password'],
-                                       server_info['host'],
-                                       server_info['port'])
+    # Create database
+    test_db_name = "test_db_%s" % str(uuid.uuid4())[1:6]
+    db_id = create_database(server_info, test_db_name)
+    add_db_to_parent_node_dict(srv_id, db_id, test_db_name)
+    # Create schema
+    schema_name = "test_schema_%s" % str(uuid.uuid4())[1:6]
+    connection = get_db_connection(test_db_name,
+                                   server_info['username'],
+                                   server_info['db_password'],
+                                   server_info['host'],
+                                   server_info['port'])
 
-        schema = regression.schema_utils.create_schema(connection, schema_name)
-        add_schema_to_parent_node_dict(srv_id, db_id, schema[0],
-                                       schema[1])
-    elif node_name not in ["servers", "roles", "tablespaces", "browser"]:
-        # Create test database
-        test_db_name = "test_db_%s" % str(uuid.uuid4())[1:6]
-        db_id = create_database(server_info, test_db_name)
-        add_db_to_parent_node_dict(srv_id, db_id, test_db_name)
-        # Create schema
-        schema_name = "test_schema_%s" % str(uuid.uuid4())[1:6]
-        connection = get_db_connection(test_db_name,
-                                       server_info['username'],
-                                       server_info['db_password'],
-                                       server_info['host'],
-                                       server_info['port'])
-
-        schema = regression.schema_utils.create_schema(connection, schema_name)
-        add_schema_to_parent_node_dict(srv_id, db_id, schema[0],
-                                       schema[1])
+    schema = regression.schema_utils.create_schema(connection, schema_name)
+    add_schema_to_parent_node_dict(srv_id, db_id, schema[0],
+                                   schema[1])
 
 
 def delete_test_server(tester):