Merge pull request #2588 from bridadan/timing-tests-drift-refactor

Timing tests drift refactor

TESTS/host_tests/timing_drift_auto.py

@@ -0,0 +1,106 @@
+"""
+mbed SDK
+Copyright (c) 2011-2013 ARM Limited
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from mbed_host_tests import BaseHostTest
+
+
+class TimingDriftTest(BaseHostTest):
+    """ This test is reading single characters from stdio
+        and measures time between their occurrences.
+    """
+    __result = None
+    
+    # This is calculated later: average_drift_max * number of tick events
+    total_drift_max = None
+    
+    average_drift_max = 0.05
+    ticks = []
+    start_time = None
+    finish_time = None
+    dut_seconds_passed = None
+    total_time = None
+    total_drift = None
+    average_drift = None
+    
+    def _callback_result(self, key, value, timestamp):
+        # We should not see result data in this test
+        self.__result = False
+
+    def _callback_end(self, key, value, timestamp):
+        """ {{end;%s}}} """
+        self.log("Received end event, timestamp: %f" % timestamp)
+        self.notify_complete(result=self.result(print_stats=True))
+
+
+    def _callback_tick(self, key, value, timestamp):
+        """ {{tick;%d}}} """
+        self.log("tick! %f" % timestamp)
+        self.ticks.append((key, value, timestamp))
+
+
+    def setup(self):
+        self.register_callback("end", self._callback_end)
+        self.register_callback('tick', self._callback_tick)
+
+
+    def result(self, print_stats=True):        
+        self.dut_seconds_passed = len(self.ticks) - 1
+        
+        if self.dut_seconds_passed < 1:
+            if print_stats:
+                self.log("FAIL: failed to receive at least two tick events")
+            self.__result = False
+            return self.__result
+
+        self.total_drift_max = self.dut_seconds_passed * self.average_drift_max
+
+        self.start_time = self.ticks[0][2]
+        self.finish_time = self.ticks[-1][2]
+        self.total_time = self.finish_time - self.start_time
+        self.total_drift = self.total_time - self.dut_seconds_passed
+        self.average_drift = self.total_drift / self.dut_seconds_passed
+        
+        if print_stats:
+            self.log("Start: %f" % self.start_time)
+            self.log("Finish: %f" % self.finish_time)
+            self.log("Total time taken: %f" % self.total_time)
+        
+            total_drift_ratio_string = "Total drift/Max total drift: %f/%f"
+            self.log(total_drift_ratio_string % (self.total_drift,
+                                                 self.total_drift_max))
+                                                 
+            average_drift_ratio_string = "Average drift/Max average drift: %f/%f"
+            self.log(average_drift_ratio_string % (self.average_drift,
+                                                   self.average_drift_max))
+        
+
+        if abs(self.total_drift) > self.total_drift_max:
+            if print_stats:
+                self.log("FAIL: Total drift exceeded max total drift")
+            self.__result = False
+        elif self.average_drift > self.average_drift_max:
+            if print_stats:
+                self.log("FAIL: Average drift exceeded max average drift")
+            self.__result = False
+        else:
+            self.__result = True
+        
+        return self.__result
+
+
+    def teardown(self):
+        pass

TESTS/integration/threaded_blinky/main.cpp

@@ -1,29 +0,0 @@
-#include "test_env.h"
-#include "mbed.h"
-#include "rtos.h"
-
-#if defined(MBED_RTOS_SINGLE_THREAD)
-  #error [NOT_SUPPORTED] test not supported
-#endif
-
-DigitalOut led1(LED1);
-
-  
-void led1_thread(void const *args) {
-    int count = 0;
-    while (true) {
-        Thread::wait(1000);
-        greentea_send_kv("tick", count);
-        count++;
-        led1 = !led1;
-    }
-}
- 
-int main() {
-    GREENTEA_SETUP(20, "wait_us_auto");
-    
-    Thread thread(led1_thread);
-    
-    while (true) {
-    }
-}

TESTS/mbed_drivers/ticker/main.cpp

@@ -21,6 +21,7 @@
 using namespace utest::v1;
 
 static const int ONE_SECOND_MS = 1000;
+static const int total_ticks = 10;
 
 DigitalOut led1(LED1);
 DigitalOut led2(LED2);
@@ -28,25 +29,23 @@ DigitalOut led2(LED2);
 Ticker *ticker1;
 Ticker *ticker2;
 
+volatile int ticker_count = 0;
+volatile bool print_tick = false;
+
 void send_kv_tick() {
-    static int count = 0;
-    if (count < 10) {
-        greentea_send_kv("tick", count);
-    } else if (count == 10) {
-        count = 0;
-        Harness::validate_callback();
+    if (ticker_count <= total_ticks) {
+        print_tick = true;
     }
-    count++;
 }
 
 void ticker_callback_0(void) {
-    static int ticker_count = 0;
-    if (ticker_count >= ONE_SECOND_MS) {
+    static int fast_ticker_count = 0;
+    if (fast_ticker_count >= ONE_SECOND_MS) {
         send_kv_tick();
-        ticker_count = 0;
+        fast_ticker_count = 0;
         led1 = !led1;
     }
-    ticker_count++;
+    fast_ticker_count++;
 }
 
 void ticker_callback_1(void) {
@@ -78,26 +77,38 @@ void ticker_callback_2_switch_to_1(void) {
     ticker_callback_2();
 }
 
-utest::v1::control_t test_case_1x_ticker() {
-    led1 = 0;
-    led2 = 0;
-    ticker1->attach_us(ticker_callback_0, ONE_SECOND_MS);
-    return CaseTimeout(15 * ONE_SECOND_MS);
+void wait_and_print() {
+    while(ticker_count <= total_ticks) {
+        if (print_tick) {
+            print_tick = false;
+            greentea_send_kv("tick", ticker_count++);
+        }
+    }
 }
 
-control_t test_case_2x_ticker() {
+void test_case_1x_ticker() {
     led1 = 0;
     led2 = 0;
+    ticker_count = 0;
+    ticker1->attach_us(ticker_callback_0, ONE_SECOND_MS);
+    wait_and_print();
+}
+
+void test_case_2x_ticker() {
+    led1 = 0;
+    led2 = 0;
+    ticker_count = 0;
     ticker1->attach(&ticker_callback_1, 1.0);
     ticker2->attach(&ticker_callback_2_led, 2.0);
-    return CaseTimeout(15 * ONE_SECOND_MS);
+    wait_and_print();
 }
 
-utest::v1::control_t test_case_2x_callbacks() {
+void test_case_2x_callbacks() {
     led1 = 0;
     led2 = 0;
+    ticker_count = 0;
     ticker1->attach(ticker_callback_1_switch_to_2, 1.0);
-    return CaseTimeout(15 * ONE_SECOND_MS);
+    wait_and_print();
 }
 
 utest::v1::status_t one_ticker_case_setup_handler_t(const Case *const source, const size_t index_of_case) {
@@ -130,7 +141,7 @@ Case cases[] = {
 };
 
 utest::v1::status_t greentea_test_setup(const size_t number_of_cases) {
-    GREENTEA_SETUP(60, "wait_us_auto");
+    GREENTEA_SETUP((total_ticks + 5) * 3, "timing_drift_auto");
     return greentea_test_setup_handler(number_of_cases);
 }
 

TESTS/mbed_drivers/timeout/main.cpp

@@ -20,43 +20,33 @@
 
 using namespace utest::v1;
 
-Timeout timer;
+Timeout timeout;
 DigitalOut led(LED1);
-
-namespace {
-    const int MS_INTERVALS = 1000;
-}
+volatile int ticker_count = 0;
+volatile bool print_tick = false;
+static const int total_ticks = 10;
+const int ONE_SECOND_US = 1000000;
 
 void send_kv_tick() {
-    static int count = 0;
-    if (count < 10) {
-        greentea_send_kv("tick", count);
-    } else if (count == 10) {
-        Harness::validate_callback();
+    if (ticker_count <= total_ticks) {
+        timeout.attach_us(send_kv_tick, ONE_SECOND_US);
+        print_tick = true;
     }
-    count++;
 }
 
-void toggleOff(void);
-
-void toggleOn(void) {
-    static int toggle_counter = 0;
-    if (toggle_counter == MS_INTERVALS) {
-        led = !led;
-        send_kv_tick();
-        toggle_counter = 0;
+void wait_and_print() {
+    while(ticker_count <= total_ticks) {
+        if (print_tick) {
+            print_tick = false;
+            greentea_send_kv("tick", ticker_count++);
+            led = !led;
+        }
     }
-    toggle_counter++;
-    timer.attach_us(toggleOff, 500);
 }
 
-void toggleOff(void) {
-    timer.attach_us(toggleOn, 500);
-}
-
-control_t test_case_ticker() {
-    toggleOn();
-    return CaseTimeout(15 * 1000);
+void test_case_ticker() {
+    timeout.attach_us(send_kv_tick, ONE_SECOND_US);
+    wait_and_print();
 }
 
 // Test cases
@@ -65,7 +55,7 @@ Case cases[] = {
 };
 
 utest::v1::status_t greentea_test_setup(const size_t number_of_cases) {
-    GREENTEA_SETUP(20, "wait_us_auto");
+    GREENTEA_SETUP(total_ticks + 5, "timing_drift_auto");
     return greentea_test_setup_handler(number_of_cases);
 }
 

TESTS/mbed_drivers/wait_us/main.cpp

@@ -18,18 +18,23 @@
 #include "greentea-client/test_env.h"
 #include "utest/utest.h"
 
+/**
+  NOTE: This test will have a bit of inherent drift due to it being
+  single-threaded, so having a drift that is non-zero should be ok. However,
+  it should still be well under the limit.
+**/
+
+
 using namespace utest::v1;
 
 DigitalOut led(LED1);
+volatile bool print_tick = false;
+const int ONE_SECOND_US = 1000000;
+const int total_ticks = 10;
 
-void test_case_ticker() {
-    for (int i=0; i < 10; ++i) {
-        // 10 secs...
-        for (int j = 0; j < 1000; ++j) {
-            // 1000 * 1000us = 1 sec
-            wait_us(1000);
-        }
-        led = !led; // Blink
+void test_case_ticker() {  
+    for (int i = 0; i <= total_ticks; i++) {
+        wait_us(ONE_SECOND_US);
         greentea_send_kv("tick", i);
     }
 }
@@ -40,7 +45,7 @@ Case cases[] = {
 };
 
 utest::v1::status_t greentea_test_setup(const size_t number_of_cases) {
-    GREENTEA_SETUP(20, "wait_us_auto");
+    GREENTEA_SETUP(total_ticks + 5, "timing_drift_auto");
     return greentea_test_setup_handler(number_of_cases);
 }
 

TESTS/mbedmicro-rtos-mbed/basic/main.cpp

@@ -29,25 +29,31 @@
 #define STACK_SIZE DEFAULT_STACK_SIZE
 #endif
 
-DigitalOut led1(LED1);
-DigitalOut led2(LED2);
+#define SIGNAL_PRINT_TICK 0x01
 
-void led2_thread(void const *argument) {
-    static int count = 0;
-    while (true) {
-        led2 = !led2;
-        Thread::wait(1000);
-        greentea_send_kv("tick", count++);
+DigitalOut led1(LED1);
+
+const int total_ticks = 10;
+
+void print_tick_thread() {
+    for (int i = 0; i <= total_ticks; i++) {
+      Thread::signal_wait(SIGNAL_PRINT_TICK);
+      greentea_send_kv("tick", i);
+      led1 = !led1;
     }
 }
 
 int main() {
-    GREENTEA_SETUP(15, "wait_us_auto");
-
-    Thread thread(led2_thread, NULL, osPriorityNormal, STACK_SIZE);
-
-    while (true) {
-        led1 = !led1;
-        Thread::wait(500);
+    GREENTEA_SETUP(total_ticks + 5, "timing_drift_auto");
+    
+    Thread tick_thread(osPriorityNormal, STACK_SIZE);
+    tick_thread.start(print_tick_thread);
+    
+    for (int i = 0; i <= total_ticks; i++) {
+        Thread::wait(1000);
+        tick_thread.signal_set(SIGNAL_PRINT_TICK);
     }
+    
+    tick_thread.join();
+    GREENTEA_TESTSUITE_RESULT(1);
 }

TESTS/mbedmicro-rtos-mbed/timer/main.cpp

@@ -6,23 +6,25 @@
   #error [NOT_SUPPORTED] test not supported
 #endif
 
+int total_ticks = 10;
+volatile int current_tick = 0;
+
 DigitalOut LEDs[4] = {
     DigitalOut(LED1), DigitalOut(LED2), DigitalOut(LED3), DigitalOut(LED4)
 };
 
 void blink(void const *n) {
     static int blink_counter = 0;
-    static int count = 0;
     const int led_id = int(n);
     LEDs[led_id] = !LEDs[led_id];
-    if (++blink_counter == 75) {
-        greentea_send_kv("tick", count++);
+    if (++blink_counter == 75 && current_tick <= total_ticks) {
+        greentea_send_kv("tick", current_tick++);
         blink_counter = 0;
     }
 }
 
 int main(void) {
-    GREENTEA_SETUP(15, "wait_us_auto");
+    GREENTEA_SETUP(total_ticks + 5, "timing_drift_auto");
 
     RtosTimer led_1_timer(blink, osTimerPeriodic, (void *)0);
     RtosTimer led_2_timer(blink, osTimerPeriodic, (void *)1);
@@ -33,6 +35,17 @@ int main(void) {
     led_2_timer.start(100);
     led_3_timer.start(50);
     led_4_timer.start(25);
+    
+    while(current_tick <= total_ticks) {
+        Thread::wait(10);
+    }
+    
+    led_4_timer.stop();
+    led_3_timer.stop();
+    led_2_timer.stop();
+    led_1_timer.stop();
+    
+    GREENTEA_TESTSUITE_RESULT(1);
 
     Thread::wait(osWaitForever);
 }