ARMmbed
/
mbed-os
mirror of https://github.com/ARMmbed/mbed-os.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add location to failure information. 

Rename `failure_t` to `failure_reason_t`.
Add `location_t` with stringify function.
Add new `failure_t` struct with location information.
Adapt harness logic to keep track of location.
Add a test failure handler that reports assertion failures
on test setup and test teardown.
Niklas Hauser 2016-01-07 09:44:58 +00:00 committed by Martin Kojtal
parent 5a052a523a
commit e68d00ded5
6 changed files with 170 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
frameworks\utest/source/default_handlers.cpp
View File

@ -22,12 +22,13 @@
using namespace utest::v1;
static status_t greentea_unknown_test_setup_handler(const size_t);
static void selftest_failure_handler(const failure_t reason);
static void selftest_failure_handler(const failure_t failure);
static void test_failure_handler(const failure_t failure);
const handlers_t utest::v1::verbose_continue_handlers = {
verbose_test_setup_handler,
verbose_test_teardown_handler,
ignore_handler,
test_failure_handler,
verbose_case_setup_handler,
verbose_case_teardown_handler,
verbose_case_failure_handler
@ -35,7 +36,7 @@ const handlers_t utest::v1::verbose_continue_handlers = {
const handlers_t utest::v1::greentea_abort_handlers = {
greentea_unknown_test_setup_handler,
greentea_test_teardown_handler,
ignore_handler,
test_failure_handler,
greentea_case_setup_handler,
greentea_case_teardown_handler,
greentea_case_failure_abort_handler
@ -43,7 +44,7 @@ const handlers_t utest::v1::greentea_abort_handlers = {
const handlers_t utest::v1::greentea_continue_handlers = {
greentea_unknown_test_setup_handler,
greentea_test_teardown_handler,
ignore_handler,
test_failure_handler,
greentea_case_setup_handler,
greentea_case_teardown_handler,
greentea_case_failure_continue_handler
@ -65,9 +66,19 @@ static status_t greentea_unknown_test_setup_handler(const size_t) {
return STATUS_ABORT;
}
static void selftest_failure_handler(const failure_t reason) {
if (reason == FAILURE_ASSERTION) {
printf(">>> failure with reason '%s (in selftest)'\n{{failure}}\n{{end}}\n", stringify(reason));
static void selftest_failure_handler(const failure_t failure) {
if (failure.location == LOCATION_TEST_SETUP || failure.location == LOCATION_TEST_TEARDOWN || failure.reason == REASON_ASSERTION) {
verbose_test_failure_handler(failure);
}
if (failure.reason == REASON_ASSERTION) {
printf("{{failure}}\n{{end}}\n");
while(1) ;
}
}
static void test_failure_handler(const failure_t failure) {
if (failure.location == LOCATION_TEST_SETUP || failure.location == LOCATION_TEST_TEARDOWN) {
verbose_test_failure_handler(failure);
printf("{{failure}}\n{{end}}\n");
while(1) ;
}
}
@ -82,17 +93,17 @@ status_t utest::v1::verbose_test_setup_handler(const size_t number_of_cases)
void utest::v1::verbose_test_teardown_handler(const size_t passed, const size_t failed, const failure_t failure)
{
printf("\n>>> Test cases: %u passed, %u failed", passed, failed);
if (failure == FAILURE_NONE) {
if (failure.reason == REASON_NONE) {
printf("\n");
} else {
printf(" with reason '%s'\n", stringify(failure));
printf(" with reason '%s'\n", stringify(failure.reason));
}
if (failed) printf(">>> TESTS FAILED!\n");
}
void utest::v1::verbose_test_failure_handler(const failure_t reason)
void utest::v1::verbose_test_failure_handler(const failure_t failure)
{
printf(">>> failure with reason '%s'\n", stringify(reason));
printf(">>> failure with reason '%s' during '%s'\n", stringify(failure.reason), stringify(failure.location));
}
// --- VERBOSE CASE HANDLERS ---
@ -105,21 +116,21 @@ status_t utest::v1::verbose_case_setup_handler(const Case *const source, const s
status_t utest::v1::verbose_case_teardown_handler(const Case *const source, const size_t passed, const size_t failed, const failure_t failure)
{
printf(">>> '%s': %u passed, %u failed", source->get_description(), passed, failed);
if (failure == FAILURE_NONE) {
if (failure.reason == REASON_NONE) {
printf("\n");
} else {
printf(" with reason '%s'\n", stringify(failure));
printf(" with reason '%s'\n", stringify(failure.reason));
}
return STATUS_CONTINUE;
}
status_t utest::v1::verbose_case_failure_handler(const Case *const /*source*/, const failure_t reason)
status_t utest::v1::verbose_case_failure_handler(const Case *const /*source*/, const failure_t failure)
{
if (!(reason & FAILURE_ASSERTION)) {
verbose_test_failure_handler(reason);
if (!(failure.reason & REASON_ASSERTION)) {
verbose_test_failure_handler(failure);
}
if (reason & FAILURE_TEARDOWN) return STATUS_ABORT;
if (reason & FAILURE_IGNORE) return STATUS_IGNORE;
if (failure.reason & (REASON_TEST_TEARDOWN | REASON_CASE_TEARDOWN)) return STATUS_ABORT;
if (failure.reason & REASON_IGNORE) return STATUS_IGNORE;
return STATUS_CONTINUE;
}
@ -133,7 +144,7 @@ status_t utest::v1::greentea_test_setup_handler(const size_t number_of_cases)
void utest::v1::greentea_test_teardown_handler(const size_t passed, const size_t failed, const failure_t failure)
{
verbose_test_teardown_handler(passed, failed, failure);
if (failed || (failure && !(failure & FAILURE_IGNORE))) {
if (failed || (failure.reason && !(failure.reason & REASON_IGNORE))) {
printf("{{failure}}\n");
} else {
printf("{{success}}\n");
@ -157,13 +168,13 @@ status_t utest::v1::greentea_case_teardown_handler(const Case *const source, con
return verbose_case_teardown_handler(source, passed, failed, failure);
}
status_t utest::v1::greentea_case_failure_abort_handler(const Case *const source, const failure_t reason)
status_t utest::v1::greentea_case_failure_abort_handler(const Case *const source, const failure_t failure)
{
status_t status = verbose_case_failure_handler(source, reason);
status_t status = verbose_case_failure_handler(source, failure);
return (status & STATUS_IGNORE) ? STATUS_IGNORE : STATUS_ABORT;
}
status_t utest::v1::greentea_case_failure_continue_handler(const Case *const source, const failure_t reason)
status_t utest::v1::greentea_case_failure_continue_handler(const Case *const source, const failure_t failure)
{
return verbose_case_failure_handler(source, reason);
return verbose_case_failure_handler(source, failure);
}

44
frameworks\utest/source/harness.cpp
View File

@ -46,6 +46,8 @@ namespace
handlers_t defaults = default_handlers;
handlers_t handlers = defaults;
location_t location = LOCATION_UNKNOWN;
}
static void die() {
@ -82,10 +84,11 @@ bool Harness::run(const Specification& specification, std::size_t start_case)
case_failed = 0;
case_failed_before = 0;
case_current = &test_cases[start_case];
location = LOCATION_TEST_SETUP;
if (handlers.test_setup && (handlers.test_setup(test_length) != STATUS_CONTINUE)) {
if (handlers.test_failure) handlers.test_failure(FAILURE_SETUP);
if (handlers.test_teardown) handlers.test_teardown(0, 0, FAILURE_SETUP);
if (handlers.test_failure) handlers.test_failure(failure_t(REASON_TEST_SETUP, location));
if (handlers.test_teardown) handlers.test_teardown(0, 0, failure_t(REASON_TEST_SETUP, location));
test_cases = NULL;
return true;
}
@ -94,14 +97,14 @@ bool Harness::run(const Specification& specification, std::size_t start_case)
return true;
}
void Harness::raise_failure(const failure_t reason)
void Harness::raise_failure(const failure_reason_t reason)
{
status_t fail_status = STATUS_ABORT;
{
mbed::util::CriticalSectionLock lock;
if (handlers.test_failure) handlers.test_failure(reason);
if (handlers.case_failure) fail_status = handlers.case_failure(case_current, reason);
if (handlers.test_failure) handlers.test_failure(failure_t(reason, location));
if (handlers.case_failure) fail_status = handlers.case_failure(case_current, failure_t(reason, location));
if (!(fail_status & STATUS_IGNORE)) case_failed++;
if ((fail_status & STATUS_ABORT) && case_timeout_handle)
@ -111,18 +114,22 @@ void Harness::raise_failure(const failure_t reason)
}
}
if (fail_status & STATUS_ABORT || reason & FAILURE_SETUP) {
if (handlers.case_teardown && !(reason & FAILURE_TEARDOWN)) {
status_t teardown_status = handlers.case_teardown(case_current, case_passed, case_failed, reason);
if (fail_status & STATUS_ABORT || reason & REASON_CASE_SETUP) {
if (handlers.case_teardown && location != LOCATION_CASE_TEARDOWN) {
location_t fail_loc(location);
location = LOCATION_CASE_TEARDOWN;
status_t teardown_status = handlers.case_teardown(case_current, case_passed, case_failed, failure_t(reason, fail_loc));
if (teardown_status != STATUS_CONTINUE) {
raise_failure(FAILURE_TEARDOWN);
raise_failure(REASON_CASE_TEARDOWN);
}
else handlers.case_teardown = NULL;
}
}
if (fail_status & STATUS_ABORT) {
test_failed++;
if (handlers.test_teardown) handlers.test_teardown(test_passed, test_failed, reason);
failure_t fail(reason, location);
location = LOCATION_TEST_TEARDOWN;
if (handlers.test_teardown) handlers.test_teardown(test_passed, test_failed, fail);
die();
}
}
@ -134,10 +141,11 @@ void Harness::schedule_next_case()
}
if (case_control.repeat & REPEAT_SETUP_TEARDOWN || !(case_control.repeat & (REPEAT_ON_TIMEOUT | REPEAT_ON_VALIDATE))) {
location = LOCATION_CASE_TEARDOWN;
if (handlers.case_teardown &&
(handlers.case_teardown(case_current, case_passed, case_failed,
case_failed ? FAILURE_CASES : FAILURE_NONE) != STATUS_CONTINUE)) {
raise_failure(FAILURE_TEARDOWN);
case_failed ? failure_t(REASON_CASES, LOCATION_UNKNOWN) : failure_t(REASON_NONE)) != STATUS_CONTINUE)) {
raise_failure(REASON_CASE_TEARDOWN);
}
}
@ -167,7 +175,7 @@ void Harness::handle_timeout()
}
}
if (case_timeout_occurred) {
raise_failure(failure_t(FAILURE_TIMEOUT | ((case_control.repeat & REPEAT_ON_TIMEOUT) ? FAILURE_IGNORE : 0)));
raise_failure(failure_reason_t(REASON_TIMEOUT | ((case_control.repeat & REPEAT_ON_TIMEOUT) ? REASON_IGNORE : 0)));
minar::Scheduler::postCallback(schedule_next_case);
}
}
@ -206,7 +214,8 @@ void Harness::run_next_case()
handlers.case_failure = defaults.get_handler(case_current->failure_handler);
if (case_current->is_empty()) {
raise_failure(FAILURE_EMPTY_CASE);
location = LOCATION_UNKNOWN;
raise_failure(REASON_EMPTY_CASE);
schedule_next_case();
return;
}
@ -221,14 +230,16 @@ void Harness::run_next_case()
}
if (setup_repeat & REPEAT_SETUP_TEARDOWN) {
location = LOCATION_CASE_SETUP;
if (handlers.case_setup && (handlers.case_setup(case_current, test_index_of_case) != STATUS_CONTINUE)) {
raise_failure(FAILURE_SETUP);
raise_failure(REASON_CASE_SETUP);
schedule_next_case();
return;
}
}
case_failed_before = case_failed;
location = LOCATION_CASE_HANDLER;
if (case_current->handler) {
case_current->handler();
@ -258,7 +269,8 @@ void Harness::run_next_case()
}
}
else if (handlers.test_teardown) {
handlers.test_teardown(test_passed, test_failed, test_failed ? FAILURE_CASES : FAILURE_NONE);
location = LOCATION_TEST_TEARDOWN;
handlers.test_teardown(test_passed, test_failed, test_failed ? failure_t(REASON_CASES, LOCATION_UNKNOWN) : failure_t(REASON_NONE));
test_cases = NULL;
}
}

74
frameworks\utest/source/types.cpp
View File

@ -18,40 +18,80 @@
#include "utest/types.h"
const char* utest::v1::stringify(utest::v1::failure_t failure)
const char* utest::v1::stringify(utest::v1::failure_reason_t reason)
{
const char *string;
switch(failure & ~FAILURE_IGNORE)
switch(reason & ~REASON_IGNORE)
{
case FAILURE_NONE:
case REASON_NONE:
string = "Ignored: No Failure";
break;
case FAILURE:
string = "Ignored: Unspecified Failure";
break;
case FAILURE_CASES:
case REASON_CASES:
string = "Ignored: Test Cases Failed";
break;
case FAILURE_EMPTY_CASE:
case REASON_EMPTY_CASE:
string = "Ignored: Test Case is Empty";
break;
case FAILURE_SETUP:
string = "Ignored: Setup Failed";
break;
case FAILURE_TEARDOWN:
string = "Ignored: Teardown Failed";
break;
case FAILURE_TIMEOUT:
case REASON_TIMEOUT:
string = "Ignored: Timed Out";
break;
case FAILURE_ASSERTION:
case REASON_ASSERTION:
string = "Ignored: Assertion Failed";
break;
case REASON_TEST_SETUP:
string = "Ignored: Test Setup Failed";
break;
case REASON_TEST_TEARDOWN:
string = "Ignored: Test Teardown Failed";
break;
case REASON_CASE_SETUP:
string = "Ignored: Case Setup Failed";
break;
case REASON_CASE_HANDLER:
string = "Ignored: Case Handler Failed";
break;
case REASON_CASE_TEARDOWN:
string = "Ignored: Case Teardown Failed";
break;
default:
case REASON_UNKNOWN:
string = "Ignored: Unknown Failure";
break;
}
if (!(failure & FAILURE_IGNORE)) string += 9;
if (!(reason & REASON_IGNORE)) string += 9;
return string;
}
const char* utest::v1::stringify(utest::v1::failure_t failure)
{
return stringify(failure.reason);
}
const char* utest::v1::stringify(utest::v1::location_t location)
{
const char *string;
switch(location)
{
case LOCATION_TEST_SETUP:
string = "Test Setup Handler";
break;
case LOCATION_TEST_TEARDOWN:
string = "Test Teardown Handler";
break;
case LOCATION_CASE_SETUP:
string = "Case Setup Handler";
break;
case LOCATION_CASE_HANDLER:
string = "Case Handler";
break;
case LOCATION_CASE_TEARDOWN:
string = "Case Teardown Handler";
break;
default:
case LOCATION_UNKNOWN:
string = "Unknown Location";
break;
}
return string;
}

4
frameworks\utest/source/unity_handler.cpp
View File

@ -21,11 +21,11 @@
extern "C"
void utest_unity_assert_failure()
{
utest::v1::Harness::raise_failure(utest::v1::FAILURE_ASSERTION);
utest::v1::Harness::raise_failure(utest::v1::REASON_ASSERTION);
}
extern "C"
void utest_unity_ignore_failure()
{
utest::v1::Harness::raise_failure(utest::v1::failure_t(utest::v1::FAILURE_ASSERTION | utest::v1::FAILURE_IGNORE));
utest::v1::Harness::raise_failure(utest::v1::failure_reason_t(utest::v1::REASON_ASSERTION | utest::v1::REASON_IGNORE));
}

2
frameworks\utest/utest/harness.h
View File

@ -76,7 +76,7 @@ namespace v1 {
/// Raising a failure causes the failure to be counted and the failure handler to be called.
/// Further action then depends on its return state.
static void raise_failure(const failure_t reason);
static void raise_failure(const failure_reason_t reason);
protected:
static void run_next_case();

63
frameworks\utest/utest/types.h
View File

@ -50,16 +50,45 @@ namespace v1 {
STATUS_ABORT = 2 ///< stops testing
};
enum failure_t {
FAILURE_NONE = 0, ///< No failure occurred
FAILURE = 1, ///< An unknown failure occurred
FAILURE_CASES = 2, ///< A failure occurred in at least one test case
FAILURE_EMPTY_CASE = 4, ///< The test case contains only empty handlers
FAILURE_SETUP = 8, ///< A failure occurred on setup
FAILURE_TEARDOWN = 16, ///< A failure occurred on teardown
FAILURE_TIMEOUT = 32, ///< An expected asynchronous call timed out
FAILURE_ASSERTION = 64, ///< An assertion failed
FAILURE_IGNORE = 0x8000 ///< A failure occurred, but may be ignored
enum failure_reason_t {
REASON_NONE = 0, ///< No failure occurred
REASON_UNKNOWN = (1 << 0), ///< An unknown failure occurred
REASON_CASES = (1 << 1), ///< A failure occurred in at least one test case
REASON_EMPTY_CASE = (1 << 2), ///< The test case contains only empty handlers
REASON_TIMEOUT = (1 << 3), ///< An expected asynchronous call timed out
REASON_ASSERTION = (1 << 4), ///< An assertion failed
REASON_TEST_SETUP = (1 << 5), ///< Test setup failed
REASON_TEST_TEARDOWN = (1 << 6), ///< Test teardown failed
REASON_CASE_SETUP = (1 << 7), ///< Case setup failed
REASON_CASE_HANDLER = (1 << 8), ///< Case handler failed
REASON_CASE_TEARDOWN = (1 << 9), ///< Case teardown failed
REASON_IGNORE = 0x8000 ///< The failure may be ignored
};
enum location_t {
LOCATION_NONE = 0, ///< No location information
LOCATION_TEST_SETUP, ///< A failure occurred in the test setup
LOCATION_TEST_TEARDOWN, ///< A failure occurred in the test teardown
LOCATION_CASE_SETUP, ///< A failure occurred in the case setup
LOCATION_CASE_HANDLER, ///< A failure occurred in the case handler
LOCATION_CASE_TEARDOWN, ///< A failure occurred in the case teardown
LOCATION_UNKNOWN ///< A failure occurred in an unknown location
};
struct failure_t {
failure_t(failure_reason_t reason) : reason(reason) {}
failure_t(location_t location) : location(location) {}
failure_t(failure_reason_t reason, location_t location) : reason(reason), location(location) {}
failure_t ignored() const {
return failure_t(failure_reason_t(reason | REASON_IGNORE), location);
}
failure_reason_t reason = REASON_NONE;
location_t location = LOCATION_NONE;
};
enum {
@ -68,9 +97,13 @@ namespace v1 {
TIMEOUT_FOREVER = uint32_t(-3) ///< Never time out
};
/// Stringifies a failure reason for understandable error messages.
const char* stringify(failure_reason_t reason);
/// Stringifies a failure for understandable error messages.
const char* stringify(failure_t failure);
/// Stringifies a status for understandable status messages.
/// Stringifies a location.
const char* stringify(location_t location);
/// Stringifies a status.
const char* stringify(status_t status);
/** Control class for specifying test case attributes
@ -194,7 +227,7 @@ namespace v1 {
*
* @returns
* You can return `STATUS_ABORT` if you initialization failed and the test teardown handler will
* then be called with the `FAILURE_SETUP`.
* then be called with the `REASON_SETUP`.
*/
typedef status_t (*test_setup_handler_t)(const size_t number_of_cases);
@ -203,7 +236,7 @@ namespace v1 {
* This handler is called after execution of all test case or if test execution is aborted.
* You can use this handler to de-initialize your test environment and output test statistics.
* The failure argument contains the immediate reason why this handler is called.
* If the test completed normally without failures, this will contain `FAILURE_NONE`.
* If the test completed normally without failures, this will contain `REASON_NONE`.
*
* After execution of this handler, the test harness will stop execution.
*
@ -232,7 +265,7 @@ namespace v1 {
*
* @returns
* You can return `STATUS_ABORT` to indicate that your setup failed, which will call the case
* failure handler with `FAILURE_SETUP` and then the case teardown handler with `FAILURE_SETUP`.
* failure handler with `REASON_SETUP` and then the case teardown handler with `REASON_SETUP`.
* This gives the teardown handler a chance to clean up a failed setup.
*/
typedef status_t (*case_setup_handler_t)(const Case *const source, const size_t index_of_case);
@ -279,7 +312,7 @@ namespace v1 {
*
* @returns
* You can return `STATUS_ABORT` to indicate that your teardown failed, which will call the case
* failure handler with `FAILURE_TEARDOWN`.
* failure handler with `REASON_TEARDOWN`.
*/
typedef status_t (*case_teardown_handler_t)(const Case *const source, const size_t passed, const size_t failed, const failure_t reason);