Add location to failure information.

Rename `failure_t` to `failure_reason_t`.
Add `location_t` with stringify function.
Add new `failure_t` struct with location information.
Adapt harness logic to keep track of location.
Add a test failure handler that reports assertion failures
on test setup and test teardown.

frameworks\utest/source/default_handlers.cpp

@@ -22,12 +22,13 @@
 using namespace utest::v1;
 
 static status_t greentea_unknown_test_setup_handler(const size_t);
-static void selftest_failure_handler(const failure_t reason);
+static void selftest_failure_handler(const failure_t failure);
+static void test_failure_handler(const failure_t failure);
 
 const handlers_t utest::v1::verbose_continue_handlers = {
     verbose_test_setup_handler,
     verbose_test_teardown_handler,
-    ignore_handler,
+    test_failure_handler,
     verbose_case_setup_handler,
     verbose_case_teardown_handler,
     verbose_case_failure_handler
@@ -35,7 +36,7 @@ const handlers_t utest::v1::verbose_continue_handlers = {
 const handlers_t utest::v1::greentea_abort_handlers = {
     greentea_unknown_test_setup_handler,
     greentea_test_teardown_handler,
-    ignore_handler,
+    test_failure_handler,
     greentea_case_setup_handler,
     greentea_case_teardown_handler,
     greentea_case_failure_abort_handler
@@ -43,7 +44,7 @@ const handlers_t utest::v1::greentea_abort_handlers = {
 const handlers_t utest::v1::greentea_continue_handlers = {
     greentea_unknown_test_setup_handler,
     greentea_test_teardown_handler,
-    ignore_handler,
+    test_failure_handler,
     greentea_case_setup_handler,
     greentea_case_teardown_handler,
     greentea_case_failure_continue_handler
@@ -65,9 +66,19 @@ static status_t greentea_unknown_test_setup_handler(const size_t) {
 
     return STATUS_ABORT;
 }
-static void selftest_failure_handler(const failure_t reason) {
+static void selftest_failure_handler(const failure_t failure) {
-    if (reason == FAILURE_ASSERTION) {
+    if (failure.location == LOCATION_TEST_SETUP || failure.location == LOCATION_TEST_TEARDOWN || failure.reason == REASON_ASSERTION) {
-        printf(">>> failure with reason '%s (in selftest)'\n{{failure}}\n{{end}}\n", stringify(reason));
+        verbose_test_failure_handler(failure);
+    }
+    if (failure.reason == REASON_ASSERTION) {
+        printf("{{failure}}\n{{end}}\n");
+        while(1) ;
+    }
+}
+static void test_failure_handler(const failure_t failure) {
+    if (failure.location == LOCATION_TEST_SETUP || failure.location == LOCATION_TEST_TEARDOWN) {
+        verbose_test_failure_handler(failure);
+        printf("{{failure}}\n{{end}}\n");
         while(1) ;
     }
 }
@@ -82,17 +93,17 @@ status_t utest::v1::verbose_test_setup_handler(const size_t number_of_cases)
 void utest::v1::verbose_test_teardown_handler(const size_t passed, const size_t failed, const failure_t failure)
 {
     printf("\n>>> Test cases: %u passed, %u failed", passed, failed);
-    if (failure == FAILURE_NONE) {
+    if (failure.reason == REASON_NONE) {
         printf("\n");
     } else  {
-        printf(" with reason '%s'\n", stringify(failure));
+        printf(" with reason '%s'\n", stringify(failure.reason));
     }
     if (failed) printf(">>> TESTS FAILED!\n");
 }
 
-void utest::v1::verbose_test_failure_handler(const failure_t reason)
+void utest::v1::verbose_test_failure_handler(const failure_t failure)
 {
-    printf(">>> failure with reason '%s'\n", stringify(reason));
+    printf(">>> failure with reason '%s' during '%s'\n", stringify(failure.reason), stringify(failure.location));
 }
 
 // --- VERBOSE CASE HANDLERS ---
@@ -105,21 +116,21 @@ status_t utest::v1::verbose_case_setup_handler(const Case *const source, const s
 status_t utest::v1::verbose_case_teardown_handler(const Case *const source, const size_t passed, const size_t failed, const failure_t failure)
 {
     printf(">>> '%s': %u passed, %u failed", source->get_description(), passed, failed);
-    if (failure == FAILURE_NONE) {
+    if (failure.reason == REASON_NONE) {
         printf("\n");
     } else  {
-        printf(" with reason '%s'\n", stringify(failure));
+        printf(" with reason '%s'\n", stringify(failure.reason));
     }
     return STATUS_CONTINUE;
 }
 
-status_t utest::v1::verbose_case_failure_handler(const Case *const /*source*/, const failure_t reason)
+status_t utest::v1::verbose_case_failure_handler(const Case *const /*source*/, const failure_t failure)
 {
-    if (!(reason & FAILURE_ASSERTION)) {
+    if (!(failure.reason & REASON_ASSERTION)) {
-        verbose_test_failure_handler(reason);
+        verbose_test_failure_handler(failure);
     }
-    if (reason & FAILURE_TEARDOWN) return STATUS_ABORT;
+    if (failure.reason & (REASON_TEST_TEARDOWN | REASON_CASE_TEARDOWN)) return STATUS_ABORT;
-    if (reason & FAILURE_IGNORE)   return STATUS_IGNORE;
+    if (failure.reason & REASON_IGNORE) return STATUS_IGNORE;
     return STATUS_CONTINUE;
 }
 
@@ -133,7 +144,7 @@ status_t utest::v1::greentea_test_setup_handler(const size_t number_of_cases)
 void utest::v1::greentea_test_teardown_handler(const size_t passed, const size_t failed, const failure_t failure)
 {
     verbose_test_teardown_handler(passed, failed, failure);
-    if (failed || (failure && !(failure & FAILURE_IGNORE))) {
+    if (failed || (failure.reason && !(failure.reason & REASON_IGNORE))) {
         printf("{{failure}}\n");
     } else {
         printf("{{success}}\n");
@@ -157,13 +168,13 @@ status_t utest::v1::greentea_case_teardown_handler(const Case *const source, con
     return verbose_case_teardown_handler(source, passed, failed, failure);
 }
 
-status_t utest::v1::greentea_case_failure_abort_handler(const Case *const source, const failure_t reason)
+status_t utest::v1::greentea_case_failure_abort_handler(const Case *const source, const failure_t failure)
 {
-    status_t status = verbose_case_failure_handler(source, reason);
+    status_t status = verbose_case_failure_handler(source, failure);
     return (status & STATUS_IGNORE) ? STATUS_IGNORE : STATUS_ABORT;
 }
 
-status_t utest::v1::greentea_case_failure_continue_handler(const Case *const source, const failure_t reason)
+status_t utest::v1::greentea_case_failure_continue_handler(const Case *const source, const failure_t failure)
 {
-    return verbose_case_failure_handler(source, reason);
+    return verbose_case_failure_handler(source, failure);
 }

frameworks\utest/source/harness.cpp

@@ -46,6 +46,8 @@ namespace
 
     handlers_t defaults = default_handlers;
     handlers_t handlers = defaults;
+
+    location_t location = LOCATION_UNKNOWN;
 }
 
 static void die() {
@@ -82,10 +84,11 @@ bool Harness::run(const Specification& specification, std::size_t start_case)
     case_failed = 0;
     case_failed_before = 0;
     case_current = &test_cases[start_case];
+    location = LOCATION_TEST_SETUP;
 
     if (handlers.test_setup && (handlers.test_setup(test_length) != STATUS_CONTINUE)) {
-        if (handlers.test_failure) handlers.test_failure(FAILURE_SETUP);
+        if (handlers.test_failure) handlers.test_failure(failure_t(REASON_TEST_SETUP, location));
-        if (handlers.test_teardown) handlers.test_teardown(0, 0, FAILURE_SETUP);
+        if (handlers.test_teardown) handlers.test_teardown(0, 0, failure_t(REASON_TEST_SETUP, location));
         test_cases = NULL;
         return true;
     }
@@ -94,14 +97,14 @@ bool Harness::run(const Specification& specification, std::size_t start_case)
     return true;
 }
 
-void Harness::raise_failure(const failure_t reason)
+void Harness::raise_failure(const failure_reason_t reason)
 {
     status_t fail_status = STATUS_ABORT;
     {
         mbed::util::CriticalSectionLock lock;
 
-        if (handlers.test_failure) handlers.test_failure(reason);
+        if (handlers.test_failure) handlers.test_failure(failure_t(reason, location));
-        if (handlers.case_failure) fail_status = handlers.case_failure(case_current, reason);
+        if (handlers.case_failure) fail_status = handlers.case_failure(case_current, failure_t(reason, location));
         if (!(fail_status & STATUS_IGNORE)) case_failed++;
 
         if ((fail_status & STATUS_ABORT) && case_timeout_handle)
@@ -111,18 +114,22 @@ void Harness::raise_failure(const failure_t reason)
         }
     }
 
-    if (fail_status & STATUS_ABORT || reason & FAILURE_SETUP) {
+    if (fail_status & STATUS_ABORT || reason & REASON_CASE_SETUP) {
-        if (handlers.case_teardown && !(reason & FAILURE_TEARDOWN)) {
+        if (handlers.case_teardown && location != LOCATION_CASE_TEARDOWN) {
-            status_t teardown_status = handlers.case_teardown(case_current, case_passed, case_failed, reason);
+            location_t fail_loc(location);
+            location = LOCATION_CASE_TEARDOWN;
+            status_t teardown_status = handlers.case_teardown(case_current, case_passed, case_failed, failure_t(reason, fail_loc));
             if (teardown_status != STATUS_CONTINUE) {
-                raise_failure(FAILURE_TEARDOWN);
+                raise_failure(REASON_CASE_TEARDOWN);
             }
             else handlers.case_teardown = NULL;
         }
     }
     if (fail_status & STATUS_ABORT) {
         test_failed++;
-        if (handlers.test_teardown) handlers.test_teardown(test_passed, test_failed, reason);
+        failure_t fail(reason, location);
+        location = LOCATION_TEST_TEARDOWN;
+        if (handlers.test_teardown) handlers.test_teardown(test_passed, test_failed, fail);
         die();
     }
 }
@@ -134,10 +141,11 @@ void Harness::schedule_next_case()
     }
 
     if (case_control.repeat & REPEAT_SETUP_TEARDOWN || !(case_control.repeat & (REPEAT_ON_TIMEOUT | REPEAT_ON_VALIDATE))) {
+        location = LOCATION_CASE_TEARDOWN;
         if (handlers.case_teardown &&
             (handlers.case_teardown(case_current, case_passed, case_failed,
-                                     case_failed ? FAILURE_CASES : FAILURE_NONE) != STATUS_CONTINUE)) {
+                    case_failed ? failure_t(REASON_CASES, LOCATION_UNKNOWN) : failure_t(REASON_NONE)) != STATUS_CONTINUE)) {
-            raise_failure(FAILURE_TEARDOWN);
+            raise_failure(REASON_CASE_TEARDOWN);
         }
     }
 
@@ -167,7 +175,7 @@ void Harness::handle_timeout()
         }
     }
     if (case_timeout_occurred) {
-        raise_failure(failure_t(FAILURE_TIMEOUT | ((case_control.repeat & REPEAT_ON_TIMEOUT) ? FAILURE_IGNORE : 0)));
+        raise_failure(failure_reason_t(REASON_TIMEOUT | ((case_control.repeat & REPEAT_ON_TIMEOUT) ? REASON_IGNORE : 0)));
         minar::Scheduler::postCallback(schedule_next_case);
     }
 }
@@ -206,7 +214,8 @@ void Harness::run_next_case()
         handlers.case_failure  = defaults.get_handler(case_current->failure_handler);
 
         if (case_current->is_empty()) {
-            raise_failure(FAILURE_EMPTY_CASE);
+            location = LOCATION_UNKNOWN;
+            raise_failure(REASON_EMPTY_CASE);
             schedule_next_case();
             return;
         }
@@ -221,14 +230,16 @@ void Harness::run_next_case()
         }
 
         if (setup_repeat & REPEAT_SETUP_TEARDOWN) {
+            location = LOCATION_CASE_SETUP;
             if (handlers.case_setup && (handlers.case_setup(case_current, test_index_of_case) != STATUS_CONTINUE)) {
-                raise_failure(FAILURE_SETUP);
+                raise_failure(REASON_CASE_SETUP);
                 schedule_next_case();
                 return;
             }
         }
 
         case_failed_before = case_failed;
+        location = LOCATION_CASE_HANDLER;
 
         if (case_current->handler) {
             case_current->handler();
@@ -258,7 +269,8 @@ void Harness::run_next_case()
         }
     }
     else if (handlers.test_teardown) {
-        handlers.test_teardown(test_passed, test_failed, test_failed ? FAILURE_CASES : FAILURE_NONE);
+        location = LOCATION_TEST_TEARDOWN;
+        handlers.test_teardown(test_passed, test_failed, test_failed ? failure_t(REASON_CASES, LOCATION_UNKNOWN) : failure_t(REASON_NONE));
         test_cases = NULL;
     }
 }

frameworks\utest/source/types.cpp

@@ -18,40 +18,80 @@
 
  #include "utest/types.h"
 
-const char* utest::v1::stringify(utest::v1::failure_t failure)
+const char* utest::v1::stringify(utest::v1::failure_reason_t reason)
 {
     const char *string;
-    switch(failure & ~FAILURE_IGNORE)
+    switch(reason & ~REASON_IGNORE)
     {
-        case FAILURE_NONE:
+        case REASON_NONE:
             string = "Ignored: No Failure";
             break;
-        case FAILURE:
+        case REASON_CASES:
-            string = "Ignored: Unspecified Failure";
-            break;
-        case FAILURE_CASES:
             string = "Ignored: Test Cases Failed";
             break;
-        case FAILURE_EMPTY_CASE:
+        case REASON_EMPTY_CASE:
             string = "Ignored: Test Case is Empty";
             break;
-        case FAILURE_SETUP:
+        case REASON_TIMEOUT:
-            string = "Ignored: Setup Failed";
-            break;
-        case FAILURE_TEARDOWN:
-            string = "Ignored: Teardown Failed";
-            break;
-        case FAILURE_TIMEOUT:
             string = "Ignored: Timed Out";
             break;
-        case FAILURE_ASSERTION:
+        case REASON_ASSERTION:
             string = "Ignored: Assertion Failed";
             break;
+        case REASON_TEST_SETUP:
+            string = "Ignored: Test Setup Failed";
+            break;
+        case REASON_TEST_TEARDOWN:
+            string = "Ignored: Test Teardown Failed";
+            break;
+        case REASON_CASE_SETUP:
+            string = "Ignored: Case Setup Failed";
+            break;
+        case REASON_CASE_HANDLER:
+            string = "Ignored: Case Handler Failed";
+            break;
+        case REASON_CASE_TEARDOWN:
+            string = "Ignored: Case Teardown Failed";
+            break;
         default:
+        case REASON_UNKNOWN:
             string = "Ignored: Unknown Failure";
             break;
     }
-    if (!(failure & FAILURE_IGNORE)) string += 9;
+    if (!(reason & REASON_IGNORE)) string += 9;
+    return string;
+}
+
+const char* utest::v1::stringify(utest::v1::failure_t failure)
+{
+    return stringify(failure.reason);
+}
+
+const char* utest::v1::stringify(utest::v1::location_t location)
+{
+    const char *string;
+    switch(location)
+    {
+        case LOCATION_TEST_SETUP:
+            string = "Test Setup Handler";
+            break;
+        case LOCATION_TEST_TEARDOWN:
+            string = "Test Teardown Handler";
+            break;
+        case LOCATION_CASE_SETUP:
+            string = "Case Setup Handler";
+            break;
+        case LOCATION_CASE_HANDLER:
+            string = "Case Handler";
+            break;
+        case LOCATION_CASE_TEARDOWN:
+            string = "Case Teardown Handler";
+            break;
+        default:
+        case LOCATION_UNKNOWN:
+            string = "Unknown Location";
+            break;
+    }
     return string;
 }
 

frameworks\utest/source/unity_handler.cpp

@@ -21,11 +21,11 @@
 extern "C"
 void utest_unity_assert_failure()
 {
-    utest::v1::Harness::raise_failure(utest::v1::FAILURE_ASSERTION);
+    utest::v1::Harness::raise_failure(utest::v1::REASON_ASSERTION);
 }
 
 extern "C"
 void utest_unity_ignore_failure()
 {
-    utest::v1::Harness::raise_failure(utest::v1::failure_t(utest::v1::FAILURE_ASSERTION | utest::v1::FAILURE_IGNORE));
+    utest::v1::Harness::raise_failure(utest::v1::failure_reason_t(utest::v1::REASON_ASSERTION | utest::v1::REASON_IGNORE));
 }

frameworks\utest/utest/harness.h

@@ -76,7 +76,7 @@ namespace v1 {
 
         /// Raising a failure causes the failure to be counted and the failure handler to be called.
         /// Further action then depends on its return state.
-        static void raise_failure(const failure_t reason);
+        static void raise_failure(const failure_reason_t reason);
 
     protected:
         static void run_next_case();

frameworks\utest/utest/types.h

@@ -50,16 +50,45 @@ namespace v1 {
         STATUS_ABORT = 2        ///< stops testing
     };
 
-    enum failure_t {
+    enum failure_reason_t {
-        FAILURE_NONE = 0,       ///< No failure occurred
+        REASON_NONE          = 0,           ///< No failure occurred
-        FAILURE = 1,            ///< An unknown failure occurred
+
-        FAILURE_CASES = 2,      ///< A failure occurred in at least one test case
+        REASON_UNKNOWN       = (1 << 0),    ///< An unknown failure occurred
-        FAILURE_EMPTY_CASE = 4, ///< The test case contains only empty handlers
+        REASON_CASES         = (1 << 1),    ///< A failure occurred in at least one test case
-        FAILURE_SETUP = 8,      ///< A failure occurred on setup
+        REASON_EMPTY_CASE    = (1 << 2),    ///< The test case contains only empty handlers
-        FAILURE_TEARDOWN = 16,  ///< A failure occurred on teardown
+        REASON_TIMEOUT       = (1 << 3),    ///< An expected asynchronous call timed out
-        FAILURE_TIMEOUT = 32,   ///< An expected asynchronous call timed out
+        REASON_ASSERTION     = (1 << 4),    ///< An assertion failed
-        FAILURE_ASSERTION = 64, ///< An assertion failed
+
-        FAILURE_IGNORE = 0x8000 ///< A failure occurred, but may be ignored
+        REASON_TEST_SETUP    = (1 << 5),    ///< Test setup failed
+        REASON_TEST_TEARDOWN = (1 << 6),    ///< Test teardown failed
+        REASON_CASE_SETUP    = (1 << 7),    ///< Case setup failed
+        REASON_CASE_HANDLER  = (1 << 8),    ///< Case handler failed
+        REASON_CASE_TEARDOWN = (1 << 9),    ///< Case teardown failed
+
+        REASON_IGNORE        = 0x8000       ///< The failure may be ignored
+    };
+
+    enum location_t {
+        LOCATION_NONE = 0,      ///< No location information
+        LOCATION_TEST_SETUP,    ///< A failure occurred in the test setup
+        LOCATION_TEST_TEARDOWN, ///< A failure occurred in the test teardown
+        LOCATION_CASE_SETUP,    ///< A failure occurred in the case setup
+        LOCATION_CASE_HANDLER,  ///< A failure occurred in the case handler
+        LOCATION_CASE_TEARDOWN, ///< A failure occurred in the case teardown
+        LOCATION_UNKNOWN        ///< A failure occurred in an unknown location
+    };
+
+    struct failure_t {
+        failure_t(failure_reason_t reason) : reason(reason) {}
+        failure_t(location_t location) : location(location) {}
+        failure_t(failure_reason_t reason, location_t location) : reason(reason), location(location) {}
+
+        failure_t ignored() const {
+            return failure_t(failure_reason_t(reason | REASON_IGNORE), location);
+        }
+
+        failure_reason_t reason = REASON_NONE;
+        location_t location = LOCATION_NONE;
     };
 
     enum {
@@ -68,9 +97,13 @@ namespace v1 {
         TIMEOUT_FOREVER = uint32_t(-3)  ///< Never time out
     };
 
+    /// Stringifies a failure reason for understandable error messages.
+    const char* stringify(failure_reason_t reason);
     /// Stringifies a failure for understandable error messages.
     const char* stringify(failure_t failure);
-    /// Stringifies a status for understandable status messages.
+    /// Stringifies a location.
+    const char* stringify(location_t location);
+    /// Stringifies a status.
     const char* stringify(status_t status);
 
     /** Control class for specifying test case attributes
@@ -194,7 +227,7 @@ namespace v1 {
      *
      * @returns
      *    You can return `STATUS_ABORT` if you initialization failed and the test teardown handler will
-     *    then be called with the `FAILURE_SETUP`.
+     *    then be called with the `REASON_SETUP`.
      */
     typedef status_t (*test_setup_handler_t)(const size_t number_of_cases);
 
@@ -203,7 +236,7 @@ namespace v1 {
      * This handler is called after execution of all test case or if test execution is aborted.
      * You can use this handler to de-initialize your test environment and output test statistics.
      * The failure argument contains the immediate reason why this handler is called.
-     * If the test completed normally without failures, this will contain `FAILURE_NONE`.
+     * If the test completed normally without failures, this will contain `REASON_NONE`.
      *
      * After execution of this handler, the test harness will stop execution.
      *
@@ -232,7 +265,7 @@ namespace v1 {
      *
      * @returns
      *    You can return `STATUS_ABORT` to indicate that your setup failed, which will call the case
-     *    failure handler with `FAILURE_SETUP` and then the case teardown handler with `FAILURE_SETUP`.
+     *    failure handler with `REASON_SETUP` and then the case teardown handler with `REASON_SETUP`.
      *    This gives the teardown handler a chance to clean up a failed setup.
      */
     typedef status_t (*case_setup_handler_t)(const Case *const source, const size_t index_of_case);
@@ -279,7 +312,7 @@ namespace v1 {
      *
      * @returns
      *    You can return `STATUS_ABORT` to indicate that your teardown failed, which will call the case
-     *    failure handler with `FAILURE_TEARDOWN`.
+     *    failure handler with `REASON_TEARDOWN`.
      */
     typedef status_t (*case_teardown_handler_t)(const Case *const source, const size_t passed, const size_t failed, const failure_t reason);