Merge pull request #4599 from studavekar/timing_tests_update

Update timing tests to be robust
pull/4726/head
Jimmy Brisson 2017-07-07 16:22:09 -05:00 committed by GitHub
commit 50fdca88e8
6 changed files with 317 additions and 282 deletions

View File

@ -16,91 +16,120 @@ limitations under the License.
"""
from mbed_host_tests import BaseHostTest
import time
class TimingDriftTest(BaseHostTest):
""" This test is reading single characters from stdio
and measures time between their occurrences.
class TimingDriftSync(BaseHostTest):
"""
This works as master-slave fashion
1) Device says its booted up and ready to run the test, wait for host to respond
2) Host sends the message to get the device current time i.e base time
#
# *
# * |
#<---* DUT<- base_time | - round_trip_base_time ------
# * | |
# * - |
# - |
# | |
# | |
# | - measurement_stretch | - nominal_time
# | |
# | |
# - |
# * - |
# * | |
#<---* DUT <-final_time | - round_trip_final_time------
# * |
# * -
#
#
# As we increase the measurement_stretch, the error because of transport delay diminishes.
# The values of measurement_stretch is propotional to round_trip_base_time(transport delays)
# by factor time_measurement_multiplier.This multiplier is used is 80 to tolerate 2 sec of
# transport delay and test time ~ 180 secs
#
# Failure in timing can occur if we are ticking too fast or we are ticking too slow, hence we have
# min_range and max_range. if we cross on either side tests would be marked fail. The range is a function of
# tolerance/acceptable drift currently its 5%.
#
"""
__result = None
# This is calculated later: average_drift_max * number of tick events
total_drift_max = None
average_drift_max = 0.05
ticks = []
start_time = None
finish_time = None
dut_seconds_passed = None
total_time = None
total_drift = None
average_drift = None
def _callback_result(self, key, value, timestamp):
# We should not see result data in this test
self.__result = False
mega = 1000000.0
max_measurement_time = 180
def _callback_end(self, key, value, timestamp):
""" {{end;%s}}} """
self.log("Received end event, timestamp: %f" % timestamp)
self.notify_complete(result=self.result(print_stats=True))
# this value is obtained for measurements when there is 0 transport delay and we want accurancy of 5%
time_measurement_multiplier = 80
def _callback_timing_drift_check_start(self, key, value, timestamp):
self.round_trip_base_start = timestamp
self.send_kv("base_time", 0)
def _callback_tick(self, key, value, timestamp):
""" {{tick;%d}}} """
self.log("tick! %f" % timestamp)
self.ticks.append((key, value, timestamp))
def _callback_base_time(self, key, value, timestamp):
self.round_trip_base_end = timestamp
self.device_time_base = float(value)
self.round_trip_base_time = self.round_trip_base_end - self.round_trip_base_start
self.log("Device base time {}".format(value))
measurement_stretch = (self.round_trip_base_time * self.time_measurement_multiplier) + 5
if measurement_stretch > self.max_measurement_time:
self.log("Time required {} to determine device timer is too high due to transport delay, skipping".format(measurement_stretch))
else:
self.log("sleeping for {} to measure drift accurately".format(measurement_stretch))
time.sleep(measurement_stretch)
self.round_trip_final_start = time.time()
self.send_kv("final_time", 0)
def _callback_final_time(self, key, value, timestamp):
self.round_trip_final_end = timestamp
self.device_time_final = float(value)
self.round_trip_final_time = self.round_trip_final_end - self.round_trip_final_start
self.log("Device final time {} ".format(value))
# compute the test results and send to device
results = "pass" if self.compute_parameter() else "fail"
self.send_kv(results, "0")
def setup(self):
self.register_callback("end", self._callback_end)
self.register_callback('tick', self._callback_tick)
self.register_callback('timing_drift_check_start', self._callback_timing_drift_check_start)
self.register_callback('base_time', self._callback_base_time)
self.register_callback('final_time', self._callback_final_time)
def compute_parameter(self, failure_criteria=0.05):
t_max = self.round_trip_final_end - self.round_trip_base_start
t_min = self.round_trip_final_start - self.round_trip_base_end
t_max_hi = t_max * (1 + failure_criteria)
t_max_lo = t_max * (1 - failure_criteria)
t_min_hi = t_min * (1 + failure_criteria)
t_min_lo = t_min * (1 - failure_criteria)
device_time = (self.device_time_final - self.device_time_base) / self.mega
def result(self, print_stats=True):
self.dut_seconds_passed = len(self.ticks) - 1
if self.dut_seconds_passed < 1:
if print_stats:
self.log("FAIL: failed to receive at least two tick events")
self.__result = False
return self.__result
self.log("Compute host events")
self.log("Transport delay 0: {}".format(self.round_trip_base_time))
self.log("Transport delay 1: {}".format(self.round_trip_final_time))
self.log("DUT base time : {}".format(self.device_time_base))
self.log("DUT end time : {}".format(self.device_time_final))
self.total_drift_max = self.dut_seconds_passed * self.average_drift_max
self.log("min_pass : {} , max_pass : {} for {}%%".format(t_max_lo, t_min_hi, failure_criteria * 100))
self.log("min_inconclusive : {} , max_inconclusive : {}".format(t_min_lo, t_max_hi))
self.log("Time reported by device: {}".format(device_time))
self.start_time = self.ticks[0][2]
self.finish_time = self.ticks[-1][2]
self.total_time = self.finish_time - self.start_time
self.total_drift = self.total_time - self.dut_seconds_passed
self.average_drift = self.total_drift / self.dut_seconds_passed
if print_stats:
self.log("Start: %f" % self.start_time)
self.log("Finish: %f" % self.finish_time)
self.log("Total time taken: %f" % self.total_time)
total_drift_ratio_string = "Total drift/Max total drift: %f/%f"
self.log(total_drift_ratio_string % (self.total_drift,
self.total_drift_max))
average_drift_ratio_string = "Average drift/Max average drift: %f/%f"
self.log(average_drift_ratio_string % (self.average_drift,
self.average_drift_max))
if abs(self.total_drift) > self.total_drift_max:
if print_stats:
self.log("FAIL: Total drift exceeded max total drift")
self.__result = False
elif self.average_drift > self.average_drift_max:
if print_stats:
self.log("FAIL: Average drift exceeded max average drift")
if t_max_lo <= device_time <= t_min_hi:
self.log("Test passed !!!")
self.__result = True
elif t_min_lo <= device_time <= t_max_hi:
self.log("Test inconclusive due to transport delay, retrying")
self.__result = False
else:
self.__result = True
self.log("Time outside of passing range. Timing drift seems to be present !!!")
self.__result = False
return self.__result
def result(self):
return self.__result
def teardown(self):
pass
pass

View File

@ -14,13 +14,31 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* Tests is to measure the accuracy of Ticker over a period of time
*
*
* 1) DUT would start to update callback_trigger_count every milli sec, in 2x callback we use 2 tickers
* to update the count alternatively.
* 2) Host would query what is current count base_time, Device responds by the callback_trigger_count
* 3) Host after waiting for measurement stretch. It will query for device time again final_time.
* 4) Host computes the drift considering base_time, final_time, transport delay and measurement stretch
* 5) Finally host send the results back to device pass/fail based on tolerance.
* 6) More details on tests can be found in timing_drift_auto.py
*/
#include "mbed.h"
#include "greentea-client/test_env.h"
#include "utest/utest.h"
#include "unity/unity.h"
using namespace utest::v1;
static const int ONE_SECOND_MS = 1000;
#define ONE_MILLI_SEC 1000
volatile uint32_t callback_trigger_count = 0;
static const int test_timeout = 240;
static const int total_ticks = 10;
DigitalOut led1(LED1);
@ -32,53 +50,37 @@ Ticker *ticker2;
volatile int ticker_count = 0;
volatile bool print_tick = false;
void send_kv_tick() {
if (ticker_count <= total_ticks) {
print_tick = true;
}
}
void ticker_callback_1_switch_to_2(void);
void ticker_callback_2_switch_to_1(void);
void ticker_callback_0(void) {
static int fast_ticker_count = 0;
if (fast_ticker_count >= ONE_SECOND_MS) {
send_kv_tick();
fast_ticker_count = 0;
led1 = !led1;
}
fast_ticker_count++;
++callback_trigger_count;
}
void ticker_callback_1(void) {
void ticker_callback_1_led(void) {
led1 = !led1;
send_kv_tick();
}
void ticker_callback_2_led(void) {
led2 = !led2;
}
void ticker_callback_2(void) {
ticker_callback_2_led();
send_kv_tick();
}
void ticker_callback_1_switch_to_2(void);
void ticker_callback_2_switch_to_1(void);
void ticker_callback_1_switch_to_2(void) {
++callback_trigger_count;
ticker1->detach();
ticker1->attach(ticker_callback_2_switch_to_1, 1.0);
ticker_callback_1();
ticker1->attach_us(ticker_callback_2_switch_to_1, ONE_MILLI_SEC);
ticker_callback_1_led();
}
void ticker_callback_2_switch_to_1(void) {
++callback_trigger_count;
ticker2->detach();
ticker2->attach(ticker_callback_1_switch_to_2, 1.0);
ticker_callback_2();
ticker2->attach_us(ticker_callback_1_switch_to_2, ONE_MILLI_SEC);
ticker_callback_2_led();
}
void wait_and_print() {
while(ticker_count <= total_ticks) {
while (ticker_count <= total_ticks) {
if (print_tick) {
print_tick = false;
greentea_send_kv("tick", ticker_count++);
@ -87,61 +89,94 @@ void wait_and_print() {
}
void test_case_1x_ticker() {
led1 = 0;
led2 = 0;
ticker_count = 0;
ticker1->attach_us(ticker_callback_0, ONE_SECOND_MS);
wait_and_print();
}
void test_case_2x_ticker() {
led1 = 0;
led2 = 0;
ticker_count = 0;
ticker1->attach(&ticker_callback_1, 1.0);
ticker2->attach(&ticker_callback_2_led, 2.0);
wait_and_print();
char _key[11] = { };
char _value[128] = { };
uint8_t results_size = 0;
int expected_key = 1;
greentea_send_kv("timing_drift_check_start", 0);
ticker1->attach_us(&ticker_callback_0, ONE_MILLI_SEC);
// wait for 1st signal from host
do {
greentea_parse_kv(_key, _value, sizeof(_key), sizeof(_value));
expected_key = strcmp(_key, "base_time");
} while (expected_key);
greentea_send_kv(_key, callback_trigger_count * ONE_MILLI_SEC);
// wait for 2nd signal from host
greentea_parse_kv(_key, _value, sizeof(_key), sizeof(_value));
greentea_send_kv(_key, callback_trigger_count * ONE_MILLI_SEC);
//get the results from host
greentea_parse_kv(_key, _value, sizeof(_key), sizeof(_value));
TEST_ASSERT_EQUAL_STRING_MESSAGE("pass", _key,"Host side script reported a fail...");
}
void test_case_2x_callbacks() {
char _key[11] = { };
char _value[128] = { };
uint8_t results_size = 0;
int expected_key = 1;
led1 = 0;
led2 = 0;
ticker_count = 0;
ticker1->attach(ticker_callback_1_switch_to_2, 1.0);
wait_and_print();
callback_trigger_count = 0;
greentea_send_kv("timing_drift_check_start", 0);
ticker1->attach_us(ticker_callback_1_switch_to_2, ONE_MILLI_SEC);
// wait for 1st signal from host
do {
greentea_parse_kv(_key, _value, sizeof(_key), sizeof(_value));
expected_key = strcmp(_key, "base_time");
} while (expected_key);
greentea_send_kv(_key, callback_trigger_count * ONE_MILLI_SEC);
// wait for 2nd signal from host
greentea_parse_kv(_key, _value, sizeof(_key), sizeof(_value));
greentea_send_kv(_key, callback_trigger_count * ONE_MILLI_SEC);
//get the results from host
greentea_parse_kv(_key, _value, sizeof(_key), sizeof(_value));
TEST_ASSERT_EQUAL_STRING_MESSAGE("pass", _key,"Host side script reported a fail...");
}
utest::v1::status_t one_ticker_case_setup_handler_t(const Case *const source, const size_t index_of_case) {
ticker1 = new Ticker();
return greentea_case_setup_handler(source, index_of_case);
ticker1 = new Ticker();
return greentea_case_setup_handler(source, index_of_case);
}
utest::v1::status_t two_ticker_case_setup_handler_t(const Case *const source, const size_t index_of_case) {
ticker1 = new Ticker();
ticker2 = new Ticker();
return greentea_case_setup_handler(source, index_of_case);
ticker1 = new Ticker();
ticker2 = new Ticker();
return greentea_case_setup_handler(source, index_of_case);
}
utest::v1::status_t one_ticker_case_teardown_handler_t(const Case *const source, const size_t passed, const size_t failed, const failure_t reason) {
delete ticker1;
return greentea_case_teardown_handler(source, passed, failed, reason);
delete ticker1;
return greentea_case_teardown_handler(source, passed, failed, reason);
}
utest::v1::status_t two_ticker_case_teardown_handler_t(const Case *const source, const size_t passed, const size_t failed, const failure_t reason) {
delete ticker1;
delete ticker2;
return greentea_case_teardown_handler(source, passed, failed, reason);
delete ticker1;
delete ticker2;
return greentea_case_teardown_handler(source, passed, failed, reason);
}
// Test cases
Case cases[] = {
Case("Timers: 1x ticker", one_ticker_case_setup_handler_t, test_case_1x_ticker, one_ticker_case_teardown_handler_t),
Case("Timers: 2x tickers", two_ticker_case_setup_handler_t, test_case_2x_ticker, two_ticker_case_teardown_handler_t),
Case("Timers: 2x callbacks", two_ticker_case_setup_handler_t, test_case_2x_callbacks,two_ticker_case_teardown_handler_t),
Case("Timers: 1x ticker", one_ticker_case_setup_handler_t,test_case_1x_ticker, one_ticker_case_teardown_handler_t),
Case("Timers: 2x callbacks", two_ticker_case_setup_handler_t,test_case_2x_callbacks, two_ticker_case_teardown_handler_t),
};
utest::v1::status_t greentea_test_setup(const size_t number_of_cases) {
GREENTEA_SETUP((total_ticks + 5) * 3, "timing_drift_auto");
GREENTEA_SETUP(test_timeout, "timing_drift_auto");
return greentea_test_setup_handler(number_of_cases);
}

View File

@ -14,48 +14,73 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* Tests is to measure the accuracy of Timeout over a period of time
*
*
* 1) DUT would start to update callback_trigger_count every milli sec
* 2) Host would query what is current count base_time, Device responds by the callback_trigger_count
* 3) Host after waiting for measurement stretch. It will query for device time again final_time.
* 4) Host computes the drift considering base_time, final_time, transport delay and measurement stretch
* 5) Finally host send the results back to device pass/fail based on tolerance.
* 6) More details on tests can be found in timing_drift_auto.py
*
*/
#include "mbed.h"
#include "greentea-client/test_env.h"
#include "utest/utest.h"
#include "unity/unity.h"
using namespace utest::v1;
Timeout timeout;
DigitalOut led(LED1);
#define ONE_MILLI_SEC 1000
volatile int ticker_count = 0;
volatile bool print_tick = false;
static const int total_ticks = 10;
const int ONE_SECOND_US = 1000000;
volatile uint32_t callback_trigger_count = 0;
static const int test_timeout = 240;
Timeout timeout;
void send_kv_tick() {
if (ticker_count <= total_ticks) {
timeout.attach_us(send_kv_tick, ONE_SECOND_US);
print_tick = true;
}
void set_incremeant_count() {
timeout.attach_us(set_incremeant_count, ONE_MILLI_SEC);
++callback_trigger_count;
}
void wait_and_print() {
while(ticker_count <= total_ticks) {
if (print_tick) {
print_tick = false;
greentea_send_kv("tick", ticker_count++);
led = !led;
}
}
void test_case_timeout() {
char _key[11] = { };
char _value[128] = { };
int expected_key = 1;
uint8_t results_size = 0;
greentea_send_kv("timing_drift_check_start", 0);
timeout.attach_us(set_incremeant_count, ONE_MILLI_SEC);
// wait for 1st signal from host
do {
greentea_parse_kv(_key, _value, sizeof(_key), sizeof(_value));
expected_key = strcmp(_key, "base_time");
} while (expected_key);
greentea_send_kv(_key, callback_trigger_count * ONE_MILLI_SEC);
// wait for 2nd signal from host
greentea_parse_kv(_key, _value, sizeof(_key), sizeof(_value));
greentea_send_kv(_key, callback_trigger_count * ONE_MILLI_SEC);
//get the results from host
greentea_parse_kv(_key, _value, sizeof(_key), sizeof(_value));
TEST_ASSERT_EQUAL_STRING_MESSAGE("pass", _key,"Host side script reported a fail...");
}
void test_case_ticker() {
timeout.attach_us(send_kv_tick, ONE_SECOND_US);
wait_and_print();
}
// Test cases
Case cases[] = {
Case("Timers: toggle on/off", test_case_ticker),
};
// Test casess
Case cases[] = { Case("Timers: toggle on/off", test_case_timeout), };
utest::v1::status_t greentea_test_setup(const size_t number_of_cases) {
GREENTEA_SETUP(total_ticks + 5, "timing_drift_auto");
GREENTEA_SETUP(test_timeout, "timing_drift_auto");
return greentea_test_setup_handler(number_of_cases);
}
@ -64,4 +89,3 @@ Specification specification(greentea_test_setup, cases, greentea_test_teardown_h
int main() {
Harness::run(specification);
}

View File

@ -1,56 +0,0 @@
/*
* Copyright (c) 2013-2016, ARM Limited, All Rights Reserved
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "mbed.h"
#include "greentea-client/test_env.h"
#include "utest/utest.h"
/**
NOTE: This test will have a bit of inherent drift due to it being
single-threaded, so having a drift that is non-zero should be ok. However,
it should still be well under the limit.
**/
using namespace utest::v1;
DigitalOut led(LED1);
volatile bool print_tick = false;
const int ONE_SECOND_US = 1000000;
const int total_ticks = 10;
void test_case_ticker() {
for (int i = 0; i <= total_ticks; i++) {
wait_us(ONE_SECOND_US);
greentea_send_kv("tick", i);
}
}
// Test cases
Case cases[] = {
Case("Timers: wait_us", test_case_ticker),
};
utest::v1::status_t greentea_test_setup(const size_t number_of_cases) {
GREENTEA_SETUP(total_ticks + 5, "timing_drift_auto");
return greentea_test_setup_handler(number_of_cases);
}
Specification specification(greentea_test_setup, cases, greentea_test_teardown_handler);
int main() {
Harness::run(specification);
}

View File

@ -1,38 +1,92 @@
/*
* Copyright (c) 2013-2017, ARM Limited, All Rights Reserved
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* Tests is to measure the accuracy of Thread::wait() over a period of time
*
*
* 1) DUT would start to update callback_trigger_count every milli sec
* 2) Host would query what is current count base_time, Device responds by the callback_trigger_count
* 3) Host after waiting for measurement stretch. It will query for device time again final_time.
* 4) Host computes the drift considering base_time, final_time, transport delay and measurement stretch
* 5) Finally host send the results back to device pass/fail based on tolerance.
* 6) More details on tests can be found in timing_drift_auto.py
*
*/
#include "mbed.h"
#include "greentea-client/test_env.h"
#include "rtos.h"
#include "unity/unity.h"
#if defined(MBED_RTOS_SINGLE_THREAD)
#error [NOT_SUPPORTED] test not supported
#error [NOT_SUPPORTED] test not supported
#endif
#define TEST_STACK_SIZE 768
#define TEST_STACK_SIZE 1024
#define ONE_MILLI_SEC 1000
#define SIGNAL_PRINT_TICK 0x01
volatile uint32_t callback_trigger_count = 0;
DigitalOut led1(LED1);
static const int test_timeout = 240;
bool test_result = false;
const int total_ticks = 10;
void update_tick_thread() {
while (true) {
Thread::wait(1);
++callback_trigger_count;
}
}
void print_tick_thread() {
for (int i = 0; i <= total_ticks; i++) {
Thread::signal_wait(SIGNAL_PRINT_TICK);
greentea_send_kv("tick", i);
led1 = !led1;
void gt_comm_wait_thread() {
char _key[11] = { };
char _value[128] = { };
int expected_key = 1;
greentea_send_kv("timing_drift_check_start", 0);
// wait for 1st signal from host
do {
greentea_parse_kv(_key, _value, sizeof(_key), sizeof(_value));
expected_key = strcmp(_key, "base_time");
} while (expected_key);
greentea_send_kv(_key, callback_trigger_count * ONE_MILLI_SEC);
// wait for 2nd signal from host
greentea_parse_kv(_key, _value, sizeof(_key), sizeof(_value));
greentea_send_kv(_key, callback_trigger_count * ONE_MILLI_SEC);
//get the results from host
greentea_parse_kv(_key, _value, sizeof(_key), sizeof(_value));
if (strcmp("pass", _key) == 0) {
test_result = true;
}
}
int main() {
GREENTEA_SETUP(total_ticks + 5, "timing_drift_auto");
GREENTEA_SETUP(test_timeout, "timing_drift_auto");
Thread tick_thread(osPriorityHigh, TEST_STACK_SIZE);
Thread gt_conn_thread(osPriorityNormal, TEST_STACK_SIZE);
Thread tick_thread(osPriorityNormal, TEST_STACK_SIZE);
tick_thread.start(print_tick_thread);
tick_thread.start(update_tick_thread);
gt_conn_thread.start(gt_comm_wait_thread);
gt_conn_thread.join();
for (int i = 0; i <= total_ticks; i++) {
Thread::wait(1000);
tick_thread.signal_set(SIGNAL_PRINT_TICK);
}
tick_thread.join();
GREENTEA_TESTSUITE_RESULT(1);
GREENTEA_TESTSUITE_RESULT(test_result);
}

View File

@ -1,51 +0,0 @@
#include "mbed.h"
#include "greentea-client/test_env.h"
#include "rtos.h"
#if defined(MBED_RTOS_SINGLE_THREAD)
#error [NOT_SUPPORTED] test not supported
#endif
int total_ticks = 10;
volatile int current_tick = 0;
DigitalOut LEDs[4] = {
DigitalOut(LED1), DigitalOut(LED2), DigitalOut(LED3), DigitalOut(LED4)
};
void blink(void const *n) {
static int blink_counter = 0;
const int led_id = int(n);
LEDs[led_id] = !LEDs[led_id];
if (++blink_counter == 75 && current_tick <= total_ticks) {
greentea_send_kv("tick", current_tick++);
blink_counter = 0;
}
}
int main(void) {
GREENTEA_SETUP(total_ticks + 5, "timing_drift_auto");
RtosTimer led_1_timer(blink, osTimerPeriodic, (void *)0);
RtosTimer led_2_timer(blink, osTimerPeriodic, (void *)1);
RtosTimer led_3_timer(blink, osTimerPeriodic, (void *)2);
RtosTimer led_4_timer(blink, osTimerPeriodic, (void *)3);
led_1_timer.start(200);
led_2_timer.start(100);
led_3_timer.start(50);
led_4_timer.start(25);
while(current_tick <= total_ticks) {
Thread::wait(10);
}
led_4_timer.stop();
led_3_timer.stop();
led_2_timer.stop();
led_1_timer.stop();
GREENTEA_TESTSUITE_RESULT(1);
Thread::wait(osWaitForever);
}