Make feature test app teardown more reliable, and tests faster

- don't spin up app and chromedriver between each test
    - catching signals also tears down the app
    - do layout reset between tests, but assume that tests will not leave a modal opened.

 Use selenium built-in waiting function and fix flakiness around clicking the alertify OK button

    - we think the OK button does not have its event bound when it is created.

If you see more flakiness around clicking the alertify OK button, let us know. The element is clickable but we have to arbitrarily wait for the event to be bound and that timing may vary system to system.

The feature tests are about 7 seconds faster now.

Tira & Joao

web/pgadmin/feature_tests/connect_to_server_feature_test.py

@@ -40,7 +40,6 @@ class ConnectsToServerFeatureTest(BaseFeatureTest):
 
     def tearDown(self):
         self.page.remove_server(self.server)
-        self.app_starter.stop_app()
 
         connection = test_utils.get_db_connection(self.server['db'],
                                                   self.server['username'],

web/pgadmin/feature_tests/template_selection_feature_test.py

@@ -1,4 +1,3 @@
-from selenium import webdriver
 from selenium.webdriver import ActionChains
 
 from regression import test_utils
@@ -44,7 +43,6 @@ class TemplateSelectionFeatureTest(BaseFeatureTest):
     def tearDown(self):
         self.page.find_by_xpath("//button[contains(.,'Cancel')]").click()
         self.page.remove_server(self.server)
-        self.app_starter.stop_app()
         connection = test_utils.get_db_connection(self.server['db'],
                                                   self.server['username'],
                                                   self.server['db_password'],

web/pgadmin/utils/route.py

@@ -96,3 +96,7 @@ class BaseTestGenerator(unittest.TestCase):
     @classmethod
     def setTestClient(cls, test_client):
         cls.tester = test_client
+
+    @classmethod
+    def setDriver(cls, driver):
+        cls.driver = driver

web/regression/feature_utils/base_feature_test.py

@@ -1,8 +1,5 @@
-from selenium import webdriver
-
 import config as app_config
 from pgadmin.utils.route import BaseTestGenerator
-from regression.feature_utils.app_starter import AppStarter
 from regression.feature_utils.pgadmin_page import PgadminPage
 
 
@@ -12,11 +9,11 @@ class BaseFeatureTest(BaseTestGenerator):
             self.skipTest("Currently, config is set to start pgadmin in server mode. "
                           "This test doesn't know username and password so doesn't work in server mode")
 
-        driver = webdriver.Chrome()
+        self.page = PgadminPage(self.driver, app_config)
-        self.app_starter = AppStarter(driver, app_config)
-        self.page = PgadminPage(driver, app_config)
-        self.app_starter.start_app()
         self.page.wait_for_app()
+        self.page.wait_for_spinner_to_disappear()
+        self.page.reset_layout()
+        self.page.wait_for_spinner_to_disappear()
 
     def failureException(self, *args, **kwargs):
         self.page.driver.save_screenshot('/tmp/feature_test_failure.png')

web/regression/feature_utils/pgadmin_page.py

@@ -1,21 +1,33 @@
 import time
 
-from selenium.common.exceptions import NoSuchElementException
+from selenium.common.exceptions import NoSuchElementException, WebDriverException
 from selenium.webdriver import ActionChains
 from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
 
 
 class PgadminPage:
     """
     Helper class for interacting with the page, given a selenium driver
     """
+
     def __init__(self, driver, app_config):
         self.driver = driver
         self.app_config = app_config
+        self.timeout = 10
+
+    def reset_layout(self):
+        self.click_element(self.find_by_partial_link_text("File"))
+        self.find_by_partial_link_text("Reset Layout").click()
+        self.click_modal_ok()
+
+    def click_modal_ok(self):
+        time.sleep(0.1)
+        self.click_element(self.find_by_xpath("//button[contains(.,'OK')]"))
 
     def add_server(self, server_config):
-        self.wait_for_spinner_to_disappear()
-
         self.find_by_xpath("//*[@class='aciTreeText' and contains(.,'Servers')]").click()
         self.driver.find_element_by_link_text("Object").click()
         ActionChains(self.driver) \
@@ -37,24 +49,36 @@ class PgadminPage:
         self.find_by_xpath("//*[@id='tree']//*[.='" + server_config['name'] + "' and @class='aciTreeItem']").click()
         self.find_by_partial_link_text("Object").click()
         self.find_by_partial_link_text("Delete/Drop").click()
-        time.sleep(0.5)
+        self.click_modal_ok()
-        self.find_by_xpath("//button[contains(.,'OK')]").click()
 
     def toggle_open_tree_item(self, tree_item_text):
         self.find_by_xpath("//*[@id='tree']//*[.='" + tree_item_text + "']/../*[@class='aciTreeButton']").click()
 
     def find_by_xpath(self, xpath):
-        return self.wait_for_element(lambda: self.driver.find_element_by_xpath(xpath))
+        return self.wait_for_element(lambda (driver): driver.find_element_by_xpath(xpath))
 
     def find_by_id(self, element_id):
-        return self.wait_for_element(lambda: self.driver.find_element_by_id(element_id))
+        return self.wait_for_element(lambda (driver): driver.find_element_by_id(element_id))
 
     def find_by_partial_link_text(self, link_text):
-        return self.wait_for_element(lambda: self.driver.find_element_by_partial_link_text(link_text))
+        return self._wait_for(
+            'link with text "#{0}"'.format(link_text),
+            EC.element_to_be_clickable((By.PARTIAL_LINK_TEXT, link_text))
+        )
+
+    def click_element(self, element):
+        def click_succeeded(driver):
+            try:
+                element.click()
+                return True
+            except WebDriverException:
+                return False
+
+        return self._wait_for("clicking the element not to throw an exception", click_succeeded)
 
     def fill_input_by_field_name(self, field_name, field_content):
         field = self.find_by_xpath("//input[@name='" + field_name + "']")
-        backspaces = [Keys.BACKSPACE]*len(field.get_attribute('value'))
+        backspaces = [Keys.BACKSPACE] * len(field.get_attribute('value'))
 
         field.click()
         field.send_keys(backspaces)
@@ -70,8 +94,8 @@ class PgadminPage:
         self.find_by_xpath("//*[contains(@class,'wcPanelTab') and contains(.,'" + tab_name + "')]").click()
 
     def wait_for_input_field_content(self, field_name, content):
-        def input_field_has_content():
+        def input_field_has_content(driver):
-            element = self.driver.find_element_by_xpath(
+            element = driver.find_element_by_xpath(
                 "//input[@name='" + field_name + "']")
 
             return str(content) == element.get_attribute('value')
@@ -79,10 +103,10 @@ class PgadminPage:
         return self._wait_for("field to contain '" + str(content) + "'", input_field_has_content)
 
     def wait_for_element(self, find_method_with_args):
-        def element_if_it_exists():
+        def element_if_it_exists(driver):
             try:
-                element = find_method_with_args()
+                element = find_method_with_args(driver)
-                if element.is_displayed() & element.is_enabled():
+                if element.is_displayed() and element.is_enabled():
                     return element
             except NoSuchElementException:
                 return False
@@ -90,9 +114,9 @@ class PgadminPage:
         return self._wait_for("element to exist", element_if_it_exists)
 
     def wait_for_spinner_to_disappear(self):
-        def spinner_has_disappeared():
+        def spinner_has_disappeared(driver):
             try:
-                self.driver.find_element_by_id("pg-spinner")
+                driver.find_element_by_id("pg-spinner")
                 return False
             except NoSuchElementException:
                 return True
@@ -100,25 +124,15 @@ class PgadminPage:
         self._wait_for("spinner to disappear", spinner_has_disappeared)
 
     def wait_for_app(self):
-        def page_shows_app():
+        def page_shows_app(driver):
-            if self.driver.title == self.app_config.APP_NAME:
+            if driver.title == self.app_config.APP_NAME:
                 return True
             else:
-                self.driver.refresh()
+                driver.refresh()
                 return False
 
         self._wait_for("app to start", page_shows_app)
 
     def _wait_for(self, waiting_for_message, condition_met_function):
-        timeout = 10
+        return WebDriverWait(self.driver, self.timeout, 0.01).until(condition_met_function,
-        time_waited = 0
+                                                                    "Timed out waiting for " + waiting_for_message)
-        sleep_time = 0.01
-
-        while time_waited < timeout:
-            result = condition_met_function()
-            if result:
-                return result
-            time_waited += sleep_time
-            time.sleep(sleep_time)
-
-        raise AssertionError("timed out waiting for " + waiting_for_message)

web/regression/runtests.py

@@ -10,14 +10,17 @@
 """ This file collect all modules/files present in tests directory and add
 them to TestSuite. """
 from __future__ import print_function
+
 import argparse
-import os
-import sys
-import signal
 import atexit
 import logging
+import os
+import signal
+import sys
 import traceback
 
+from selenium import webdriver
+
 if sys.version_info < (2, 7):
     import unittest2 as unittest
 else:
@@ -40,6 +43,7 @@ if sys.path[0] != root:
 from pgadmin import create_app
 import config
 from regression import test_setup
+from regression.feature_utils.app_starter import AppStarter
 
 # Delete SQLite db file if exists
 if os.path.isfile(config.TEST_SQLITE_PATH):
@@ -88,7 +92,10 @@ config.CONSOLE_LOG_LEVEL = WARNING
 app = create_app()
 app.config['WTF_CSRF_ENABLED'] = False
 test_client = app.test_client()
-drop_objects = test_utils.get_cleanup_handler(test_client)
+driver = webdriver.Chrome()
+app_starter = AppStarter(driver, config)
+app_starter.start_app()
+handle_cleanup = test_utils.get_cleanup_handler(test_client, app_starter)
 
 
 def get_suite(module_list, test_server, test_app_client):
@@ -118,6 +125,7 @@ def get_suite(module_list, test_server, test_app_client):
         obj.setApp(app)
         obj.setTestClient(test_app_client)
         obj.setTestServer(test_server)
+        obj.setDriver(driver)
         scenario = generate_scenarios(obj)
         pgadmin_suite.addTests(scenario)
 
@@ -180,7 +188,7 @@ def add_arguments():
 
 
 def sig_handler(signo, frame):
-    drop_objects()
+    handle_cleanup()
 
 
 def get_tests_result(test_suite):
@@ -242,7 +250,7 @@ if __name__ == '__main__':
 
     test_result = dict()
     # Register cleanup function to cleanup on exit
-    atexit.register(drop_objects)
+    atexit.register(handle_cleanup)
     # Set signal handler for cleanup
     signal_list = dir(signal)
     required_signal_list = ['SIGTERM', 'SIGABRT', 'SIGQUIT', 'SIGINT']

web/regression/test_utils.py

@@ -365,7 +365,7 @@ def remove_db_file():
         os.remove(config.TEST_SQLITE_PATH)
 
 
-def _drop_objects(tester):
+def _cleanup(tester, app_starter):
     """This function use to cleanup the created the objects(servers, databases,
      schemas etc) during the test suite run"""
     try:
@@ -404,11 +404,12 @@ def _drop_objects(tester):
         logout_tester_account(tester)
         # Remove SQLite db file
         remove_db_file()
+        app_starter.stop_app()
 
 
-def get_cleanup_handler(tester):
+def get_cleanup_handler(tester, app_starter):
     """This function use to bind variable to drop_objects function"""
-    return partial(_drop_objects, tester)
+    return partial(_cleanup, tester, app_starter)
 
 
 class Database: