Harness: Allow generic ignoring of failures.

frameworks\utest/source/default_handlers.cpp

@@ -21,6 +21,28 @@
 
 using namespace utest::v1;
 
+const handlers_t utest::v1::verbose_continue_handlers = {
+    verbose_test_setup_handler,
+    verbose_test_teardown_handler,
+    verbose_case_setup_handler,
+    verbose_case_teardown_handler,
+    verbose_case_failure_handler
+};
+const handlers_t utest::v1::greentea_abort_handlers = {
+    greentea_test_setup_handler,
+    greentea_test_teardown_handler,
+    verbose_case_setup_handler,
+    verbose_case_teardown_handler,
+    greentea_case_failure_handler
+};
+const handlers_t utest::v1::greentea_continue_handlers = {
+    greentea_test_setup_handler,
+    greentea_test_teardown_handler,
+    verbose_case_setup_handler,
+    verbose_case_teardown_handler,
+    verbose_case_failure_handler
+};
+
 status_t utest::v1::verbose_test_setup_handler(const size_t number_of_cases)
 {
     printf(">>> Running %u test cases...\n", number_of_cases);
@@ -57,11 +79,11 @@ status_t utest::v1::verbose_case_teardown_handler(const Case *const source, cons
 
 status_t utest::v1::verbose_case_failure_handler(const Case *const /*source*/, const failure_t reason)
 {
-    if (reason != FAILURE_ASSERTION) {
+    if (!(reason & FAILURE_ASSERTION)) {
         printf(">>> failure with reason '%s'\n", stringify(reason));
     }
-    if (reason == FAILURE_TEARDOWN) return STATUS_ABORT;
+    if (reason & FAILURE_TEARDOWN) return STATUS_ABORT;
-    if (reason & FAILURE_IGNORE)    return STATUS_IGNORE;
+    if (reason & FAILURE_IGNORE)   return STATUS_IGNORE;
     return STATUS_CONTINUE;
 }
 
@@ -78,7 +100,7 @@ status_t utest::v1::greentea_test_setup_handler(const size_t /*number_of_cases*/
 void utest::v1::greentea_test_teardown_handler(const size_t passed, const size_t failed, const failure_t failure)
 {
     verbose_test_teardown_handler(passed, failed, failure);
-    if (failed || failure != FAILURE_NONE) {
+    if (failed || (failure && !(failure & FAILURE_IGNORE))) {
         printf("{{failure}}\n");
     } else {
         printf("{{success}}\n");
@@ -89,5 +111,5 @@ void utest::v1::greentea_test_teardown_handler(const size_t passed, const size_t
 status_t utest::v1::greentea_case_failure_handler(const Case *const source, const failure_t reason)
 {
     status_t status = verbose_case_failure_handler(source, reason);
-    return (status == STATUS_IGNORE) ? STATUS_IGNORE : STATUS_ABORT;
+    return (status & STATUS_IGNORE) ? STATUS_IGNORE : STATUS_ABORT;
 }

frameworks\utest/source/harness.cpp

@@ -52,12 +52,12 @@ static void die() {
     while(1) ;
 }
 
-bool Harness::run(const Specification specification)
+bool Harness::run(const Specification& specification)
 {
     return run(specification, 0);
 }
 
-bool Harness::run(const Specification specification, std::size_t start_case)
+bool Harness::run(const Specification& specification, std::size_t start_case)
 {
     // ignore any invalid start index
     if (start_case >= specification.length)
@@ -99,17 +99,17 @@ void Harness::raise_failure(failure_t reason)
         mbed::util::CriticalSectionLock lock;
 
         if (handlers.case_failure) fail_status = handlers.case_failure(case_current, reason);
-        if (fail_status != STATUS_IGNORE) case_failed++;
+        if (!(fail_status & STATUS_IGNORE)) case_failed++;
 
-        if (fail_status == STATUS_ABORT && case_timeout_handle)
+        if ((fail_status & STATUS_ABORT) && case_timeout_handle)
         {
             minar::Scheduler::cancelCallback(case_timeout_handle);
             case_timeout_handle = NULL;
         }
     }
 
-    if (fail_status == STATUS_ABORT || reason == FAILURE_SETUP) {
+    if (fail_status & STATUS_ABORT || reason & FAILURE_SETUP) {
-        if (handlers.case_teardown && reason != FAILURE_TEARDOWN) {
+        if (handlers.case_teardown && !(reason & FAILURE_TEARDOWN)) {
             status_t teardown_status = handlers.case_teardown(case_current, case_passed, case_failed, reason);
             if (teardown_status != STATUS_CONTINUE) {
                 raise_failure(FAILURE_TEARDOWN);
@@ -117,7 +117,7 @@ void Harness::raise_failure(failure_t reason)
             else handlers.case_teardown = NULL;
         }
     }
-    if (fail_status == STATUS_ABORT) {
+    if (fail_status & STATUS_ABORT) {
         test_failed++;
         if (handlers.test_teardown) handlers.test_teardown(test_passed, test_failed, reason);
         die();
@@ -126,8 +126,9 @@ void Harness::raise_failure(failure_t reason)
 
 void Harness::schedule_next_case()
 {
-    if (!(case_control.repeat & REPEAT_ON_TIMEOUT) &&
+    if (!(case_control.repeat & REPEAT_ON_TIMEOUT) && case_failed_before == case_failed) {
-        case_failed_before == case_failed) case_passed++;
+        case_passed++;
+    }
 
     if (case_control.repeat & REPEAT_ALL || case_control.repeat == REPEAT_NO_REPEAT) {
         if (handlers.case_teardown &&

frameworks\utest/utest/default_handlers.h

@@ -138,31 +138,13 @@ namespace v1 {
     status_t greentea_case_failure_handler (const Case *const source, const failure_t reason);
 
     /// The verbose default handlers that always continue on failure
-    const handlers_t verbose_continue_handlers = {
+    extern const handlers_t verbose_continue_handlers;
-        verbose_test_setup_handler,
-        verbose_test_teardown_handler,
-        verbose_case_setup_handler,
-        verbose_case_teardown_handler,
-        verbose_case_failure_handler
-    };
 
     /// The greentea default handlers that always abort on the first encountered failure
-    const handlers_t greentea_abort_handlers = {
+    extern const handlers_t greentea_abort_handlers;
-        greentea_test_setup_handler,
-        greentea_test_teardown_handler,
-        verbose_case_setup_handler,
-        verbose_case_teardown_handler,
-        greentea_case_failure_handler
-    };
 
     /// The greentea default handlers that always continue on failure
-    const handlers_t greentea_continue_handlers = {
+    extern const handlers_t greentea_continue_handlers;
-        greentea_test_setup_handler,
-        greentea_test_teardown_handler,
-        verbose_case_setup_handler,
-        verbose_case_teardown_handler,
-        verbose_case_failure_handler
-    };
 
     /// The greentea aborting handlers are the default
     const handlers_t default_handlers = greentea_abort_handlers;

frameworks\utest/utest/harness.h

@@ -46,13 +46,13 @@ namespace v1 {
         /// Runs a test specification
         /// @retval `true`  if the specification can be run
         /// @retval `false` if another specification is currently running
-        static bool run(const Specification specification);
+        static bool run(const Specification& specification);
 
         /// Runs a test specification starting at the specified case index
         /// @warning if the start index is out of bounds, the call has no effect!
         /// @retval `true`  if the specification can be run
         /// @retval `false` if another specification is currently running, or the start index was out of bounds
-        static bool run(const Specification specification, size_t start_case);
+        static bool run(const Specification& specification, size_t start_case);
 
         /// @returns `true` if a test specification is being executed, `false` otherwise
         static bool is_busy();

frameworks\utest/utest/types.h

@@ -31,25 +31,25 @@ namespace v1 {
         REPEAT_NO_REPEAT  = 0,  ///< continue with the next test case
         REPEAT_CASE_ONLY  = 1,  ///< repeat the current test case without the setup and teardown handlers
         REPEAT_ALL        = 2,  ///< repeat the current test case with the setup and teardown handlers
-        REPEAT_ON_TIMEOUT = 4   ///< repeat the current on timeout only89
+        REPEAT_ON_TIMEOUT = 4   ///< repeat the current on timeout only
     };
 
     enum status_t {
-        STATUS_CONTINUE = 0,        ///< continues testing
+        STATUS_CONTINUE = 0,    ///< continues testing
-        STATUS_IGNORE,              ///< ignores failure and continues testing
+        STATUS_IGNORE = 1,      ///< ignores failure and continues testing
-        STATUS_ABORT,               ///< stops testing
+        STATUS_ABORT = 2        ///< stops testing
     };
 
     enum failure_t {
-        FAILURE_NONE = 0,   ///< No failure occurred
+        FAILURE_NONE = 0,       ///< No failure occurred
-        FAILURE,            ///< An unknown failure occurred
+        FAILURE = 1,            ///< An unknown failure occurred
-        FAILURE_CASES,      ///< A failure occurred in at least one test case
+        FAILURE_CASES = 2,      ///< A failure occurred in at least one test case
-        FAILURE_EMPTY_CASE, ///< The test case contains only empty handlers
+        FAILURE_EMPTY_CASE = 4, ///< The test case contains only empty handlers
-        FAILURE_SETUP,      ///< A failure occurred on setup
+        FAILURE_SETUP = 8,      ///< A failure occurred on setup
-        FAILURE_TEARDOWN,   ///< A failure occurred on teardown
+        FAILURE_TEARDOWN = 16,  ///< A failure occurred on teardown
-        FAILURE_TIMEOUT,    ///< An expected asynchronous call timed out
+        FAILURE_TIMEOUT = 32,   ///< An expected asynchronous call timed out
-        FAILURE_ASSERTION,  ///< An assertion failed
+        FAILURE_ASSERTION = 64, ///< An assertion failed
-        FAILURE_IGNORE = 0x8000,    ///< A failure occurred, but may be ignored
+        FAILURE_IGNORE = 0x8000 ///< A failure occurred, but may be ignored
     };
 
     /// Stringifies a failure for understandable error messages.