From e68d00ded54d18b0ee4dd5a18d6811122b4154a7 Mon Sep 17 00:00:00 2001 From: Niklas Hauser Date: Thu, 7 Jan 2016 09:44:58 +0000 Subject: [PATCH] Add location to failure information. Rename `failure_t` to `failure_reason_t`. Add `location_t` with stringify function. Add new `failure_t` struct with location information. Adapt harness logic to keep track of location. Add a test failure handler that reports assertion failures on test setup and test teardown. --- .../source/default_handlers.cpp" | 57 ++++++++------ "frameworks\\utest/source/harness.cpp" | 44 +++++++---- "frameworks\\utest/source/types.cpp" | 74 ++++++++++++++----- "frameworks\\utest/source/unity_handler.cpp" | 4 +- "frameworks\\utest/utest/harness.h" | 2 +- "frameworks\\utest/utest/types.h" | 63 ++++++++++++---- 6 files changed, 170 insertions(+), 74 deletions(-) diff --git "a/frameworks\\utest/source/default_handlers.cpp" "b/frameworks\\utest/source/default_handlers.cpp" index 21d50149aa..72fbcd80f9 100644 --- "a/frameworks\\utest/source/default_handlers.cpp" +++ "b/frameworks\\utest/source/default_handlers.cpp" @@ -22,12 +22,13 @@ using namespace utest::v1; static status_t greentea_unknown_test_setup_handler(const size_t); -static void selftest_failure_handler(const failure_t reason); +static void selftest_failure_handler(const failure_t failure); +static void test_failure_handler(const failure_t failure); const handlers_t utest::v1::verbose_continue_handlers = { verbose_test_setup_handler, verbose_test_teardown_handler, - ignore_handler, + test_failure_handler, verbose_case_setup_handler, verbose_case_teardown_handler, verbose_case_failure_handler @@ -35,7 +36,7 @@ const handlers_t utest::v1::verbose_continue_handlers = { const handlers_t utest::v1::greentea_abort_handlers = { greentea_unknown_test_setup_handler, greentea_test_teardown_handler, - ignore_handler, + test_failure_handler, greentea_case_setup_handler, greentea_case_teardown_handler, greentea_case_failure_abort_handler @@ -43,7 +44,7 @@ const handlers_t utest::v1::greentea_abort_handlers = { const handlers_t utest::v1::greentea_continue_handlers = { greentea_unknown_test_setup_handler, greentea_test_teardown_handler, - ignore_handler, + test_failure_handler, greentea_case_setup_handler, greentea_case_teardown_handler, greentea_case_failure_continue_handler @@ -65,9 +66,19 @@ static status_t greentea_unknown_test_setup_handler(const size_t) { return STATUS_ABORT; } -static void selftest_failure_handler(const failure_t reason) { - if (reason == FAILURE_ASSERTION) { - printf(">>> failure with reason '%s (in selftest)'\n{{failure}}\n{{end}}\n", stringify(reason)); +static void selftest_failure_handler(const failure_t failure) { + if (failure.location == LOCATION_TEST_SETUP || failure.location == LOCATION_TEST_TEARDOWN || failure.reason == REASON_ASSERTION) { + verbose_test_failure_handler(failure); + } + if (failure.reason == REASON_ASSERTION) { + printf("{{failure}}\n{{end}}\n"); + while(1) ; + } +} +static void test_failure_handler(const failure_t failure) { + if (failure.location == LOCATION_TEST_SETUP || failure.location == LOCATION_TEST_TEARDOWN) { + verbose_test_failure_handler(failure); + printf("{{failure}}\n{{end}}\n"); while(1) ; } } @@ -82,17 +93,17 @@ status_t utest::v1::verbose_test_setup_handler(const size_t number_of_cases) void utest::v1::verbose_test_teardown_handler(const size_t passed, const size_t failed, const failure_t failure) { printf("\n>>> Test cases: %u passed, %u failed", passed, failed); - if (failure == FAILURE_NONE) { + if (failure.reason == REASON_NONE) { printf("\n"); } else { - printf(" with reason '%s'\n", stringify(failure)); + printf(" with reason '%s'\n", stringify(failure.reason)); } if (failed) printf(">>> TESTS FAILED!\n"); } -void utest::v1::verbose_test_failure_handler(const failure_t reason) +void utest::v1::verbose_test_failure_handler(const failure_t failure) { - printf(">>> failure with reason '%s'\n", stringify(reason)); + printf(">>> failure with reason '%s' during '%s'\n", stringify(failure.reason), stringify(failure.location)); } // --- VERBOSE CASE HANDLERS --- @@ -105,21 +116,21 @@ status_t utest::v1::verbose_case_setup_handler(const Case *const source, const s status_t utest::v1::verbose_case_teardown_handler(const Case *const source, const size_t passed, const size_t failed, const failure_t failure) { printf(">>> '%s': %u passed, %u failed", source->get_description(), passed, failed); - if (failure == FAILURE_NONE) { + if (failure.reason == REASON_NONE) { printf("\n"); } else { - printf(" with reason '%s'\n", stringify(failure)); + printf(" with reason '%s'\n", stringify(failure.reason)); } return STATUS_CONTINUE; } -status_t utest::v1::verbose_case_failure_handler(const Case *const /*source*/, const failure_t reason) +status_t utest::v1::verbose_case_failure_handler(const Case *const /*source*/, const failure_t failure) { - if (!(reason & FAILURE_ASSERTION)) { - verbose_test_failure_handler(reason); + if (!(failure.reason & REASON_ASSERTION)) { + verbose_test_failure_handler(failure); } - if (reason & FAILURE_TEARDOWN) return STATUS_ABORT; - if (reason & FAILURE_IGNORE) return STATUS_IGNORE; + if (failure.reason & (REASON_TEST_TEARDOWN | REASON_CASE_TEARDOWN)) return STATUS_ABORT; + if (failure.reason & REASON_IGNORE) return STATUS_IGNORE; return STATUS_CONTINUE; } @@ -133,7 +144,7 @@ status_t utest::v1::greentea_test_setup_handler(const size_t number_of_cases) void utest::v1::greentea_test_teardown_handler(const size_t passed, const size_t failed, const failure_t failure) { verbose_test_teardown_handler(passed, failed, failure); - if (failed || (failure && !(failure & FAILURE_IGNORE))) { + if (failed || (failure.reason && !(failure.reason & REASON_IGNORE))) { printf("{{failure}}\n"); } else { printf("{{success}}\n"); @@ -157,13 +168,13 @@ status_t utest::v1::greentea_case_teardown_handler(const Case *const source, con return verbose_case_teardown_handler(source, passed, failed, failure); } -status_t utest::v1::greentea_case_failure_abort_handler(const Case *const source, const failure_t reason) +status_t utest::v1::greentea_case_failure_abort_handler(const Case *const source, const failure_t failure) { - status_t status = verbose_case_failure_handler(source, reason); + status_t status = verbose_case_failure_handler(source, failure); return (status & STATUS_IGNORE) ? STATUS_IGNORE : STATUS_ABORT; } -status_t utest::v1::greentea_case_failure_continue_handler(const Case *const source, const failure_t reason) +status_t utest::v1::greentea_case_failure_continue_handler(const Case *const source, const failure_t failure) { - return verbose_case_failure_handler(source, reason); + return verbose_case_failure_handler(source, failure); } diff --git "a/frameworks\\utest/source/harness.cpp" "b/frameworks\\utest/source/harness.cpp" index 521e809e16..a41a980af0 100644 --- "a/frameworks\\utest/source/harness.cpp" +++ "b/frameworks\\utest/source/harness.cpp" @@ -46,6 +46,8 @@ namespace handlers_t defaults = default_handlers; handlers_t handlers = defaults; + + location_t location = LOCATION_UNKNOWN; } static void die() { @@ -82,10 +84,11 @@ bool Harness::run(const Specification& specification, std::size_t start_case) case_failed = 0; case_failed_before = 0; case_current = &test_cases[start_case]; + location = LOCATION_TEST_SETUP; if (handlers.test_setup && (handlers.test_setup(test_length) != STATUS_CONTINUE)) { - if (handlers.test_failure) handlers.test_failure(FAILURE_SETUP); - if (handlers.test_teardown) handlers.test_teardown(0, 0, FAILURE_SETUP); + if (handlers.test_failure) handlers.test_failure(failure_t(REASON_TEST_SETUP, location)); + if (handlers.test_teardown) handlers.test_teardown(0, 0, failure_t(REASON_TEST_SETUP, location)); test_cases = NULL; return true; } @@ -94,14 +97,14 @@ bool Harness::run(const Specification& specification, std::size_t start_case) return true; } -void Harness::raise_failure(const failure_t reason) +void Harness::raise_failure(const failure_reason_t reason) { status_t fail_status = STATUS_ABORT; { mbed::util::CriticalSectionLock lock; - if (handlers.test_failure) handlers.test_failure(reason); - if (handlers.case_failure) fail_status = handlers.case_failure(case_current, reason); + if (handlers.test_failure) handlers.test_failure(failure_t(reason, location)); + if (handlers.case_failure) fail_status = handlers.case_failure(case_current, failure_t(reason, location)); if (!(fail_status & STATUS_IGNORE)) case_failed++; if ((fail_status & STATUS_ABORT) && case_timeout_handle) @@ -111,18 +114,22 @@ void Harness::raise_failure(const failure_t reason) } } - if (fail_status & STATUS_ABORT || reason & FAILURE_SETUP) { - if (handlers.case_teardown && !(reason & FAILURE_TEARDOWN)) { - status_t teardown_status = handlers.case_teardown(case_current, case_passed, case_failed, reason); + if (fail_status & STATUS_ABORT || reason & REASON_CASE_SETUP) { + if (handlers.case_teardown && location != LOCATION_CASE_TEARDOWN) { + location_t fail_loc(location); + location = LOCATION_CASE_TEARDOWN; + status_t teardown_status = handlers.case_teardown(case_current, case_passed, case_failed, failure_t(reason, fail_loc)); if (teardown_status != STATUS_CONTINUE) { - raise_failure(FAILURE_TEARDOWN); + raise_failure(REASON_CASE_TEARDOWN); } else handlers.case_teardown = NULL; } } if (fail_status & STATUS_ABORT) { test_failed++; - if (handlers.test_teardown) handlers.test_teardown(test_passed, test_failed, reason); + failure_t fail(reason, location); + location = LOCATION_TEST_TEARDOWN; + if (handlers.test_teardown) handlers.test_teardown(test_passed, test_failed, fail); die(); } } @@ -134,10 +141,11 @@ void Harness::schedule_next_case() } if (case_control.repeat & REPEAT_SETUP_TEARDOWN || !(case_control.repeat & (REPEAT_ON_TIMEOUT | REPEAT_ON_VALIDATE))) { + location = LOCATION_CASE_TEARDOWN; if (handlers.case_teardown && (handlers.case_teardown(case_current, case_passed, case_failed, - case_failed ? FAILURE_CASES : FAILURE_NONE) != STATUS_CONTINUE)) { - raise_failure(FAILURE_TEARDOWN); + case_failed ? failure_t(REASON_CASES, LOCATION_UNKNOWN) : failure_t(REASON_NONE)) != STATUS_CONTINUE)) { + raise_failure(REASON_CASE_TEARDOWN); } } @@ -167,7 +175,7 @@ void Harness::handle_timeout() } } if (case_timeout_occurred) { - raise_failure(failure_t(FAILURE_TIMEOUT | ((case_control.repeat & REPEAT_ON_TIMEOUT) ? FAILURE_IGNORE : 0))); + raise_failure(failure_reason_t(REASON_TIMEOUT | ((case_control.repeat & REPEAT_ON_TIMEOUT) ? REASON_IGNORE : 0))); minar::Scheduler::postCallback(schedule_next_case); } } @@ -206,7 +214,8 @@ void Harness::run_next_case() handlers.case_failure = defaults.get_handler(case_current->failure_handler); if (case_current->is_empty()) { - raise_failure(FAILURE_EMPTY_CASE); + location = LOCATION_UNKNOWN; + raise_failure(REASON_EMPTY_CASE); schedule_next_case(); return; } @@ -221,14 +230,16 @@ void Harness::run_next_case() } if (setup_repeat & REPEAT_SETUP_TEARDOWN) { + location = LOCATION_CASE_SETUP; if (handlers.case_setup && (handlers.case_setup(case_current, test_index_of_case) != STATUS_CONTINUE)) { - raise_failure(FAILURE_SETUP); + raise_failure(REASON_CASE_SETUP); schedule_next_case(); return; } } case_failed_before = case_failed; + location = LOCATION_CASE_HANDLER; if (case_current->handler) { case_current->handler(); @@ -258,7 +269,8 @@ void Harness::run_next_case() } } else if (handlers.test_teardown) { - handlers.test_teardown(test_passed, test_failed, test_failed ? FAILURE_CASES : FAILURE_NONE); + location = LOCATION_TEST_TEARDOWN; + handlers.test_teardown(test_passed, test_failed, test_failed ? failure_t(REASON_CASES, LOCATION_UNKNOWN) : failure_t(REASON_NONE)); test_cases = NULL; } } diff --git "a/frameworks\\utest/source/types.cpp" "b/frameworks\\utest/source/types.cpp" index 8ef482b5e6..3f8f692c58 100644 --- "a/frameworks\\utest/source/types.cpp" +++ "b/frameworks\\utest/source/types.cpp" @@ -18,40 +18,80 @@ #include "utest/types.h" -const char* utest::v1::stringify(utest::v1::failure_t failure) +const char* utest::v1::stringify(utest::v1::failure_reason_t reason) { const char *string; - switch(failure & ~FAILURE_IGNORE) + switch(reason & ~REASON_IGNORE) { - case FAILURE_NONE: + case REASON_NONE: string = "Ignored: No Failure"; break; - case FAILURE: - string = "Ignored: Unspecified Failure"; - break; - case FAILURE_CASES: + case REASON_CASES: string = "Ignored: Test Cases Failed"; break; - case FAILURE_EMPTY_CASE: + case REASON_EMPTY_CASE: string = "Ignored: Test Case is Empty"; break; - case FAILURE_SETUP: - string = "Ignored: Setup Failed"; - break; - case FAILURE_TEARDOWN: - string = "Ignored: Teardown Failed"; - break; - case FAILURE_TIMEOUT: + case REASON_TIMEOUT: string = "Ignored: Timed Out"; break; - case FAILURE_ASSERTION: + case REASON_ASSERTION: string = "Ignored: Assertion Failed"; break; + case REASON_TEST_SETUP: + string = "Ignored: Test Setup Failed"; + break; + case REASON_TEST_TEARDOWN: + string = "Ignored: Test Teardown Failed"; + break; + case REASON_CASE_SETUP: + string = "Ignored: Case Setup Failed"; + break; + case REASON_CASE_HANDLER: + string = "Ignored: Case Handler Failed"; + break; + case REASON_CASE_TEARDOWN: + string = "Ignored: Case Teardown Failed"; + break; default: + case REASON_UNKNOWN: string = "Ignored: Unknown Failure"; break; } - if (!(failure & FAILURE_IGNORE)) string += 9; + if (!(reason & REASON_IGNORE)) string += 9; + return string; +} + +const char* utest::v1::stringify(utest::v1::failure_t failure) +{ + return stringify(failure.reason); +} + +const char* utest::v1::stringify(utest::v1::location_t location) +{ + const char *string; + switch(location) + { + case LOCATION_TEST_SETUP: + string = "Test Setup Handler"; + break; + case LOCATION_TEST_TEARDOWN: + string = "Test Teardown Handler"; + break; + case LOCATION_CASE_SETUP: + string = "Case Setup Handler"; + break; + case LOCATION_CASE_HANDLER: + string = "Case Handler"; + break; + case LOCATION_CASE_TEARDOWN: + string = "Case Teardown Handler"; + break; + default: + case LOCATION_UNKNOWN: + string = "Unknown Location"; + break; + } return string; } diff --git "a/frameworks\\utest/source/unity_handler.cpp" "b/frameworks\\utest/source/unity_handler.cpp" index b40978c75d..bc595233aa 100644 --- "a/frameworks\\utest/source/unity_handler.cpp" +++ "b/frameworks\\utest/source/unity_handler.cpp" @@ -21,11 +21,11 @@ extern "C" void utest_unity_assert_failure() { - utest::v1::Harness::raise_failure(utest::v1::FAILURE_ASSERTION); + utest::v1::Harness::raise_failure(utest::v1::REASON_ASSERTION); } extern "C" void utest_unity_ignore_failure() { - utest::v1::Harness::raise_failure(utest::v1::failure_t(utest::v1::FAILURE_ASSERTION | utest::v1::FAILURE_IGNORE)); + utest::v1::Harness::raise_failure(utest::v1::failure_reason_t(utest::v1::REASON_ASSERTION | utest::v1::REASON_IGNORE)); } diff --git "a/frameworks\\utest/utest/harness.h" "b/frameworks\\utest/utest/harness.h" index fa2ad9e102..726f146497 100644 --- "a/frameworks\\utest/utest/harness.h" +++ "b/frameworks\\utest/utest/harness.h" @@ -76,7 +76,7 @@ namespace v1 { /// Raising a failure causes the failure to be counted and the failure handler to be called. /// Further action then depends on its return state. - static void raise_failure(const failure_t reason); + static void raise_failure(const failure_reason_t reason); protected: static void run_next_case(); diff --git "a/frameworks\\utest/utest/types.h" "b/frameworks\\utest/utest/types.h" index d2be7688a0..c936666bec 100644 --- "a/frameworks\\utest/utest/types.h" +++ "b/frameworks\\utest/utest/types.h" @@ -50,16 +50,45 @@ namespace v1 { STATUS_ABORT = 2 ///< stops testing }; - enum failure_t { - FAILURE_NONE = 0, ///< No failure occurred - FAILURE = 1, ///< An unknown failure occurred - FAILURE_CASES = 2, ///< A failure occurred in at least one test case - FAILURE_EMPTY_CASE = 4, ///< The test case contains only empty handlers - FAILURE_SETUP = 8, ///< A failure occurred on setup - FAILURE_TEARDOWN = 16, ///< A failure occurred on teardown - FAILURE_TIMEOUT = 32, ///< An expected asynchronous call timed out - FAILURE_ASSERTION = 64, ///< An assertion failed - FAILURE_IGNORE = 0x8000 ///< A failure occurred, but may be ignored + enum failure_reason_t { + REASON_NONE = 0, ///< No failure occurred + + REASON_UNKNOWN = (1 << 0), ///< An unknown failure occurred + REASON_CASES = (1 << 1), ///< A failure occurred in at least one test case + REASON_EMPTY_CASE = (1 << 2), ///< The test case contains only empty handlers + REASON_TIMEOUT = (1 << 3), ///< An expected asynchronous call timed out + REASON_ASSERTION = (1 << 4), ///< An assertion failed + + REASON_TEST_SETUP = (1 << 5), ///< Test setup failed + REASON_TEST_TEARDOWN = (1 << 6), ///< Test teardown failed + REASON_CASE_SETUP = (1 << 7), ///< Case setup failed + REASON_CASE_HANDLER = (1 << 8), ///< Case handler failed + REASON_CASE_TEARDOWN = (1 << 9), ///< Case teardown failed + + REASON_IGNORE = 0x8000 ///< The failure may be ignored + }; + + enum location_t { + LOCATION_NONE = 0, ///< No location information + LOCATION_TEST_SETUP, ///< A failure occurred in the test setup + LOCATION_TEST_TEARDOWN, ///< A failure occurred in the test teardown + LOCATION_CASE_SETUP, ///< A failure occurred in the case setup + LOCATION_CASE_HANDLER, ///< A failure occurred in the case handler + LOCATION_CASE_TEARDOWN, ///< A failure occurred in the case teardown + LOCATION_UNKNOWN ///< A failure occurred in an unknown location + }; + + struct failure_t { + failure_t(failure_reason_t reason) : reason(reason) {} + failure_t(location_t location) : location(location) {} + failure_t(failure_reason_t reason, location_t location) : reason(reason), location(location) {} + + failure_t ignored() const { + return failure_t(failure_reason_t(reason | REASON_IGNORE), location); + } + + failure_reason_t reason = REASON_NONE; + location_t location = LOCATION_NONE; }; enum { @@ -68,9 +97,13 @@ namespace v1 { TIMEOUT_FOREVER = uint32_t(-3) ///< Never time out }; + /// Stringifies a failure reason for understandable error messages. + const char* stringify(failure_reason_t reason); /// Stringifies a failure for understandable error messages. const char* stringify(failure_t failure); - /// Stringifies a status for understandable status messages. + /// Stringifies a location. + const char* stringify(location_t location); + /// Stringifies a status. const char* stringify(status_t status); /** Control class for specifying test case attributes @@ -194,7 +227,7 @@ namespace v1 { * * @returns * You can return `STATUS_ABORT` if you initialization failed and the test teardown handler will - * then be called with the `FAILURE_SETUP`. + * then be called with the `REASON_SETUP`. */ typedef status_t (*test_setup_handler_t)(const size_t number_of_cases); @@ -203,7 +236,7 @@ namespace v1 { * This handler is called after execution of all test case or if test execution is aborted. * You can use this handler to de-initialize your test environment and output test statistics. * The failure argument contains the immediate reason why this handler is called. - * If the test completed normally without failures, this will contain `FAILURE_NONE`. + * If the test completed normally without failures, this will contain `REASON_NONE`. * * After execution of this handler, the test harness will stop execution. * @@ -232,7 +265,7 @@ namespace v1 { * * @returns * You can return `STATUS_ABORT` to indicate that your setup failed, which will call the case - * failure handler with `FAILURE_SETUP` and then the case teardown handler with `FAILURE_SETUP`. + * failure handler with `REASON_SETUP` and then the case teardown handler with `REASON_SETUP`. * This gives the teardown handler a chance to clean up a failed setup. */ typedef status_t (*case_setup_handler_t)(const Case *const source, const size_t index_of_case); @@ -279,7 +312,7 @@ namespace v1 { * * @returns * You can return `STATUS_ABORT` to indicate that your teardown failed, which will call the case - * failure handler with `FAILURE_TEARDOWN`. + * failure handler with `REASON_TEARDOWN`. */ typedef status_t (*case_teardown_handler_t)(const Case *const source, const size_t passed, const size_t failed, const failure_t reason);