From 5947838b2b0d6664150bcf02d1c908c504ee98c7 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Fri, 8 Aug 2014 13:57:19 +0100 Subject: [PATCH 01/18] BUgfix: jobs (-j) value was not properly set in SingletestRunner class parameters --- workspace_tools/singletest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workspace_tools/singletest.py b/workspace_tools/singletest.py index 0620bbad71..90d123a060 100644 --- a/workspace_tools/singletest.py +++ b/workspace_tools/singletest.py @@ -184,7 +184,8 @@ if __name__ == '__main__': _opts_suppress_summary=opts.suppress_summary, _opts_test_x_toolchain_summary=opts.test_x_toolchain_summary, _opts_copy_method=opts.copy_method, - _opts_mut_reset_type=opts.mut_reset_type + _opts_mut_reset_type=opts.mut_reset_type, + _opts_jobs=opts.jobs ) # Runs test suite in CLI mode From 64640c88803b91c4a8ac6fd4089ca20d51adea57 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Fri, 8 Aug 2014 16:46:20 +0100 Subject: [PATCH 02/18] Feature: MPS2 reset functionality implementation (not finished and will not break current implementation) --- workspace_tools/host_tests/host_test.py | 9 ++++----- workspace_tools/test_api.py | 7 +++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/workspace_tools/host_tests/host_test.py b/workspace_tools/host_tests/host_test.py index 9f6f31de85..4cda45f1cd 100644 --- a/workspace_tools/host_tests/host_test.py +++ b/workspace_tools/host_tests/host_test.py @@ -133,17 +133,16 @@ class Mbed: return result def touch_file(self, path, name): - with os.open(path, 'a'): + with open(path, 'a'): os.utime(path, None) def reset(self): """ reboot.txt - startup from standby state, reboots when in run mode. shutdown.txt - shutdown from run mode reset.txt - reset fpga during run mode """ - if self.options.forced_reset_type: - path = os.path.join([self.disk, self.options.forced_reset_type.lower()]) - if self.options.forced_reset_type.endswith('.txt'): - self.touch_file(path) + if self.options.forced_reset_type and self.options.forced_reset_type.endswith('.txt'): + reset_file_path = os.path.join(self.disk, self.options.forced_reset_type.lower()) + self.touch_file(reset_file_path) else: self.safe_sendBreak(self.serial) # Instead of serial.sendBreak() # Give time to wait for the image loading diff --git a/workspace_tools/test_api.py b/workspace_tools/test_api.py index c18eea0cc2..900280f7e8 100644 --- a/workspace_tools/test_api.py +++ b/workspace_tools/test_api.py @@ -510,6 +510,7 @@ class SingleTestRunner(object): disk = mut['disk'] port = mut['port'] + reset_type = mut.get('reset_type') target_by_mcu = TARGET_MAP[mut['mcu']] # Program @@ -552,8 +553,10 @@ class SingleTestRunner(object): start_host_exec_time = time() host_test_verbose = self.opts_verbose_test_result_only or self.opts_verbose - host_test_reset = self.opts_mut_reset_type - single_test_result = self.run_host_test(test.host_test, disk, port, duration, verbose=host_test_verbose, reset=host_test_reset) + host_test_reset = self.opts_mut_reset_type if reset_type is None else reset_type + single_test_result = self.run_host_test(test.host_test, disk, port, duration, + verbose=host_test_verbose, + reset=host_test_reset) # Store test result test_all_result.append(single_test_result) From c1ebcb498b6d261b9ab45ade0ee62039b2c9de5b Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Mon, 11 Aug 2014 10:38:06 +0100 Subject: [PATCH 03/18] Simple indent for function call parameters --- workspace_tools/singletest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/workspace_tools/singletest.py b/workspace_tools/singletest.py index 90d123a060..cbba6b610a 100644 --- a/workspace_tools/singletest.py +++ b/workspace_tools/singletest.py @@ -185,8 +185,7 @@ if __name__ == '__main__': _opts_test_x_toolchain_summary=opts.test_x_toolchain_summary, _opts_copy_method=opts.copy_method, _opts_mut_reset_type=opts.mut_reset_type, - _opts_jobs=opts.jobs - ) + _opts_jobs=opts.jobs) # Runs test suite in CLI mode singletest_in_cli_mode(single_test) From 0f8c5dc494bcc68b4c3bfa77f0d29b7e550ff03e Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Mon, 11 Aug 2014 10:41:58 +0100 Subject: [PATCH 04/18] MPS2: Added function which will modify image cfg. file for MPS2 board. Added skeleton for log functionality --- workspace_tools/test_api.py | 86 ++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 2 deletions(-) diff --git a/workspace_tools/test_api.py b/workspace_tools/test_api.py index 900280f7e8..66504e71d7 100644 --- a/workspace_tools/test_api.py +++ b/workspace_tools/test_api.py @@ -20,6 +20,7 @@ Author: Przemyslaw Wirkus import os import re import json +import time import pprint import random import optparse @@ -35,10 +36,11 @@ from threading import Thread from subprocess import Popen, PIPE, call # Imports related to mbed build api +from workspace_tools.tests import TESTS +from workspace_tools.tests import TEST_MAP from workspace_tools.paths import BUILD_DIR from workspace_tools.paths import HOST_TESTS -from workspace_tools.tests import TEST_MAP -from workspace_tools.tests import TESTS +from workspace_tools.utils import construct_enum from workspace_tools.targets import TARGET_MAP from workspace_tools.build_api import build_project, build_mbed_libs, build_lib from workspace_tools.build_api import get_target_supported_toolchains @@ -962,6 +964,86 @@ def singletest_in_cli_mode(single_test): print "Completed in %d sec"% (elapsed_time) +def mps2_check_board_image_file(disk, board_image_path, image0file_path, image_name='images.txt'): + """ This function will modify 'Versatile Express Images Configuration File' file + for Versatile Express V2M-MPS2 board. + Main goal of this function is to change number of images to 1, comment all + existing image entries and append at the end of file new entry with test. + """ + MBED_SDK_TEST_STAMP = 'test suite entry' + image_path = os.path.join(disk, board_image_path, image_name) + new_file_lines = [] # New configuration file lines (entries) + + # Check each line of the image configuration file + try: + with open(image_path, 'r') as file: + for line in file: + if re.search('^TOTALIMAGES', line): + # Check number of total images, should be 1 + new_file_lines.append(re.sub('^TOTALIMAGES:[\t ]*[\d]+', 'TOTALIMAGES: 1', line)) + pass + + elif re.search('; - %s[\n\r]*$'% MBED_SDK_TEST_STAMP, line): + # Look for test suite entries and remove them + pass # Omit all test suite entries + + elif re.search('^IMAGE[\d]+FILE', line): + # Check all image entries and mark the ';' + new_file_lines.append(';' + line) # Comment non test suite lines + else: + # Append line to new file + new_file_lines.append(line) + except IOError as e: + return False + + # Add new image entry with proper commented stamp + new_file_lines.append('IMAGE0FILE: %s ; - %s\r\n'% (image0file_path, MBED_SDK_TEST_STAMP)) + + # Write all lines to file + try: + with open(image_path, 'w') as file: + for line in new_file_lines: + file.write(line), + except IOError as e: + return False + + return True + + +class TestLogger(): + """ Super-class for logging and printing ongoing events for test suite pass """ + def __init__(self): + self.Log = [] + + self.LogType = construct_enum(INFO='Info', + NOTIF='Notification', + WARN='Warning', + ERROR='Error') + + self.LogToFileAttr = construct_enum(CREATE=1, # Create or overwrite existing log file + APPEND=2) # Append to existing log file + + def log_line(self, LogType, log_line, log_time=None): + log_timestamp = time.time() if log_time is None else log_time + log_entry = {'log_type' : LogType, + 'log_timestamp' : log_timestamp, + 'log_line' : log_line, + '_future' : None} + self.Log.append(log_entry) + + def log_to_file(self, LogToFileAttr, file_name): + """ Class will log to file current log entries. + Note: you should be able to see log file like this: + + tail -f log_file.txt + """ + pass + + +class CLITestLogger(TestLogger): + pass + + def get_default_test_options_parser(): """ Get common test script options used by CLI, webservices etc. """ parser = optparse.OptionParser() From 17c0132d56c1a2fc2a19f55c161fbcd043249b62 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Tue, 12 Aug 2014 10:20:41 +0100 Subject: [PATCH 05/18] Added parts of functionality related to extra reset types, global timeout increments and image configuration file modifications --- workspace_tools/host_tests/host_test.py | 23 +++++++-- workspace_tools/singletest.py | 3 +- workspace_tools/test_api.py | 69 +++++++++++++++++++------ workspace_tools/test_webapi.py | 2 +- 4 files changed, 74 insertions(+), 23 deletions(-) diff --git a/workspace_tools/host_tests/host_test.py b/workspace_tools/host_tests/host_test.py index 4cda45f1cd..09a477357e 100644 --- a/workspace_tools/host_tests/host_test.py +++ b/workspace_tools/host_tests/host_test.py @@ -63,8 +63,17 @@ class Mbed: dest="forced_reset_type", help="Forces different type of reset") + parser.add_option("-R", "--reset-timeout", + dest="forced_reset_timeout", + metavar="NUMBER", + type="int", + help="When forcing a reset using option -r you can set up after reset timeout in seconds") + (self.options, _) = parser.parse_args() + self.DEFAULT_RESET_TOUT = 2 + self.DEFAULT_TOUT = 10 + if self.options.port is None: raise Exception("The serial port of the target mbed have to be provided as command line arguments") @@ -73,7 +82,7 @@ class Mbed: self.extra_port = self.options.extra self.extra_serial = None self.serial = None - self.timeout = 10 if self.options.timeout is None else self.options.timeout + self.timeout = self.DEFAULT_TOUT if self.options.timeout is None else self.options.timeout print 'Mbed: "%s" "%s"' % (self.port, self.disk) def init_serial(self, baud=9600, extra_baud=9600): @@ -132,21 +141,27 @@ class Mbed: result = False return result - def touch_file(self, path, name): + def touch_file(self, path): with open(path, 'a'): os.utime(path, None) + def reset_timeout(self, timeout): + for n in range(0, timeout): + sleep(1) + def reset(self): """ reboot.txt - startup from standby state, reboots when in run mode. shutdown.txt - shutdown from run mode - reset.txt - reset fpga during run mode """ + reset.txt - reset FPGA during run mode + """ if self.options.forced_reset_type and self.options.forced_reset_type.endswith('.txt'): reset_file_path = os.path.join(self.disk, self.options.forced_reset_type.lower()) self.touch_file(reset_file_path) else: self.safe_sendBreak(self.serial) # Instead of serial.sendBreak() # Give time to wait for the image loading - sleep(2) + reset_tout_s = self.options.forced_reset_timeout if self.options.forced_reset_timeout is not None else self.DEFAULT_RESET_TOUT + self.reset_timeout(reset_tout_s) def flush(self): self.serial.flushInput() diff --git a/workspace_tools/singletest.py b/workspace_tools/singletest.py index cbba6b610a..3d867aeb7e 100644 --- a/workspace_tools/singletest.py +++ b/workspace_tools/singletest.py @@ -185,7 +185,8 @@ if __name__ == '__main__': _opts_test_x_toolchain_summary=opts.test_x_toolchain_summary, _opts_copy_method=opts.copy_method, _opts_mut_reset_type=opts.mut_reset_type, - _opts_jobs=opts.jobs) + _opts_jobs=opts.jobs, + _opts_extend_test_timeout=opts.extend_test_timeout) # Runs test suite in CLI mode singletest_in_cli_mode(single_test) diff --git a/workspace_tools/test_api.py b/workspace_tools/test_api.py index 66504e71d7..69dfec4f8c 100644 --- a/workspace_tools/test_api.py +++ b/workspace_tools/test_api.py @@ -19,6 +19,7 @@ Author: Przemyslaw Wirkus import os import re +import sys import json import time import pprint @@ -88,7 +89,7 @@ class SingleTestExecutor(threading.Thread): # prints well-formed summary with results (SQL table like) # table shows text x toolchain test result matrix print self.single_test.generate_test_summary_by_target(test_summary, shuffle_seed) - print "Completed in %d sec"% (elapsed_time) + print "Completed in %.2f sec"% (elapsed_time) class SingleTestRunner(object): @@ -146,7 +147,8 @@ class SingleTestRunner(object): _opts_test_x_toolchain_summary=False, _opts_copy_method=None, _opts_mut_reset_type=None, - _opts_jobs=None + _opts_jobs=None, + _opts_extend_test_timeout=None ): """ Let's try hard to init this object """ PATTERN = "\\{(" + "|".join(self.TEST_RESULT_MAPPING.keys()) + ")\\}" @@ -186,7 +188,8 @@ class SingleTestRunner(object): self.opts_test_x_toolchain_summary = _opts_test_x_toolchain_summary self.opts_copy_method = _opts_copy_method self.opts_mut_reset_type = _opts_mut_reset_type - self.opts_jobs = _opts_jobs + self.opts_jobs = _opts_jobs if _opts_jobs is not None else 1 + self.opts_extend_test_timeout = _opts_extend_test_timeout def shuffle_random_func(self): return self.shuffle_random_seed @@ -312,9 +315,14 @@ class SingleTestRunner(object): # With this option we are skipping testing phase continue + # Test duration can be increased by global value + test_duration = test.duration + if self.opts_extend_test_timeout is not None: + test_duration += self.opts_extend_test_timeout + # For an automated test the duration act as a timeout after # which the test gets interrupted - test_spec = self.shape_test_request(target, path, test_id, test.duration) + test_spec = self.shape_test_request(target, path, test_id, test_duration) test_loops = self.get_test_loop_count(test_id) single_test_result = self.handle(test_spec, target, toolchain, test_loops=test_loops) if single_test_result is not None: @@ -435,7 +443,6 @@ class SingleTestRunner(object): profile.set_preference('browser.download.manager.showWhenStarting', False) profile.set_preference('browser.download.dir', dest_disk) profile.set_preference('browser.helperApps.neverAsk.saveToDisk', 'application/octet-stream') - # Launch browser with profile and get file browser = webdriver.Firefox(profile) browser.get(file_path) @@ -449,7 +456,6 @@ class SingleTestRunner(object): if copy_method == 'cp' or copy_method == 'copy' or copy_method == 'xcopy': source_path = image_path.encode('ascii', 'ignore') destination_path = os.path.join(disk.encode('ascii', 'ignore'), basename(image_path).encode('ascii', 'ignore')) - cmd = [copy_method, source_path, destination_path] try: ret = call(cmd, shell=True) @@ -512,8 +518,12 @@ class SingleTestRunner(object): disk = mut['disk'] port = mut['port'] - reset_type = mut.get('reset_type') target_by_mcu = TARGET_MAP[mut['mcu']] + # Some extra stuff can be declared in MUTs structure + reset_type = mut.get('reset_type') + reset_tout = mut.get('reset_tout') + images_config = mut.get('images_config') + mobo_config = mut.get('mobo_config') # Program # When the build and test system were separate, this was relative to a @@ -558,7 +568,8 @@ class SingleTestRunner(object): host_test_reset = self.opts_mut_reset_type if reset_type is None else reset_type single_test_result = self.run_host_test(test.host_test, disk, port, duration, verbose=host_test_verbose, - reset=host_test_reset) + reset=host_test_reset, + reset_tout=reset_tout) # Store test result test_all_result.append(single_test_result) @@ -597,7 +608,7 @@ class SingleTestRunner(object): result = test_all_result[0] return result - def run_host_test(self, name, disk, port, duration, reset=None, verbose=False, extra_serial=None): + def run_host_test(self, name, disk, port, duration, reset=None, reset_tout=None, verbose=False, extra_serial=None): """ Function creates new process with host test configured with particular test case. Function also is pooling for serial port activity from process to catch all data printed by test runner and host test during test execution.""" @@ -609,6 +620,11 @@ class SingleTestRunner(object): cmd += ["-e", extra_serial] if reset is not None: cmd += ["-r", reset] + if reset_tout is not None: + cmd += ["-R", str(reset_tout)] + + if verbose: + print "Host test cmd: " + " ".join(cmd) proc = Popen(cmd, stdout=PIPE, cwd=HOST_TESTS) obs = ProcessObserver(proc) @@ -617,7 +633,7 @@ class SingleTestRunner(object): output = [] while (time() - start_time) < duration: try: - c = obs.queue.get(block=True, timeout=1) + c = obs.queue.get(block=True, timeout=0.5) except Empty, _: c = None @@ -625,7 +641,8 @@ class SingleTestRunner(object): output.append(c) # Give the mbed under test a way to communicate the end of the test if c in ['\n', '\r']: - if '{end}' in line: break + if '{end}' in line: + break line = '' else: line += c @@ -961,17 +978,17 @@ def singletest_in_cli_mode(single_test): # prints well-formed summary with results (SQL table like) # table shows text x toolchain test result matrix print single_test.generate_test_summary_by_target(test_summary, shuffle_seed) - print "Completed in %d sec"% (elapsed_time) + print "Completed in %.2f sec"% (elapsed_time) -def mps2_check_board_image_file(disk, board_image_path, image0file_path, image_name='images.txt'): - """ This function will modify 'Versatile Express Images Configuration File' file - for Versatile Express V2M-MPS2 board. +def mps2_set_board_image_file(disk, images_cfg_path, image0file_path, image_name='images.txt'): + """ This function will alter image cfg file. Main goal of this function is to change number of images to 1, comment all - existing image entries and append at the end of file new entry with test. + existing image entries and append at the end of file new entry with test path. + @return True when all steps succeed. """ MBED_SDK_TEST_STAMP = 'test suite entry' - image_path = os.path.join(disk, board_image_path, image_name) + image_path = os.path.join(disk, images_cfg_path, image_name) new_file_lines = [] # New configuration file lines (entries) # Check each line of the image configuration file @@ -1010,6 +1027,18 @@ def mps2_check_board_image_file(disk, board_image_path, image0file_path, image_n return True +def mps2_select_core(disk, mobo_config_name=""): + """ Function selects actual core """ + # TODO: implement core selection + pass + + +def mps2_switch_usb_auto_mounting_after_restart(disk, usb_config_name=""): + """ Function alters configuration to allow USB MSD to be mounted after restarts """ + # TODO: implement USB MSD restart detection + pass + + class TestLogger(): """ Super-class for logging and printing ongoing events for test suite pass """ def __init__(self): @@ -1170,6 +1199,12 @@ def get_default_test_options_parser(): default=None, help='For some commands you can use filter to filter out results') + parser.add_option('', '--inc-timeout', + dest='extend_test_timeout', + metavar="NUMBER", + type="int", + help='You can increase global timeout for each test by specifying additional test timeout in seconds') + parser.add_option('', '--verbose-skipped', dest='verbose_skipped_tests', default=False, diff --git a/workspace_tools/test_webapi.py b/workspace_tools/test_webapi.py index 59fb97c016..0f2c22c0b4 100644 --- a/workspace_tools/test_webapi.py +++ b/workspace_tools/test_webapi.py @@ -54,7 +54,7 @@ class SingleTestRunnerWebService(SingleTestRunner): def get_rest_result_template(self, result, command, success_code): """ Returns common part of every web service request """ - result = {"result": result, + result = {"result" : result, "command" : command, "success_code": success_code} # 0 - OK, >0 - Error number return result From 28b48ae4d012cb4128bfbd4a13c2a3c61b616521 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Tue, 12 Aug 2014 11:40:17 +0100 Subject: [PATCH 06/18] reset functionality extensions: adding 'image_copy_method_selector' - function which will apply if necessary image configuration file modifications when image is copied onto MUT --- workspace_tools/test_api.py | 41 ++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/workspace_tools/test_api.py b/workspace_tools/test_api.py index 69dfec4f8c..f789061d5f 100644 --- a/workspace_tools/test_api.py +++ b/workspace_tools/test_api.py @@ -448,14 +448,33 @@ class SingleTestRunner(object): browser.get(file_path) browser.close() - def file_copy_method_selector(self, image_path, disk, copy_method): + def image_copy_method_selector(self, target, image_path, disk, copy_method, + images_config=None, image_dest=None): + """ Function copied image file and fiddles with image configuration files in needed. + This function will select proper image configuration (modify image config file + if needed) after image is copied. + """ + image_dest = image_dest if image_dest is not None else '' + _copy_res, _err_msg, _copy_method = self.file_copy_method_selector(image_path, disk, self.opts_copy_method, image_dest=image_dest) + + if images_config is not None: + if target == 'ARM_MPS2': + images_cfg_path = images_config + image0file_path = os.path.join(disk, image_dest, basename(image_path)) + mps2_set_board_image_file(disk, images_cfg_path, image0file_path): + + return _copy_res, _err_msg, _copy_method + + def file_copy_method_selector(self, image_path, disk, copy_method, + image_dest='', image_cfg_func=None, image_cfg_func_params=None): """ Copy file depending on method you want to use. Handles exception - and return code from shell copy commands. """ + and return code from shell copy commands. + """ result = True resutl_msg = "" if copy_method == 'cp' or copy_method == 'copy' or copy_method == 'xcopy': source_path = image_path.encode('ascii', 'ignore') - destination_path = os.path.join(disk.encode('ascii', 'ignore'), basename(image_path).encode('ascii', 'ignore')) + destination_path = os.path.join(disk.encode('ascii', 'ignore'), image_dest, basename(image_path).encode('ascii', 'ignore')) cmd = [copy_method, source_path, destination_path] try: ret = call(cmd, shell=True) @@ -465,10 +484,10 @@ class SingleTestRunner(object): except Exception, e: resutl_msg = e result = False - if copy_method == 'firefox': + elif copy_method == 'firefox': try: source_path = image_path.encode('ascii', 'ignore') - destination_path = disk.encode('ascii', 'ignore') + destination_path = os.path.join(disk.encode('ascii', 'ignore'), image_dest) self.file_store_firefox(source_path, destination_path) except Exception, e: resutl_msg = e @@ -481,6 +500,7 @@ class SingleTestRunner(object): except Exception, e: resutl_msg = e result = False + return result, resutl_msg, copy_method def delete_file(self, file_path): @@ -520,10 +540,11 @@ class SingleTestRunner(object): port = mut['port'] target_by_mcu = TARGET_MAP[mut['mcu']] # Some extra stuff can be declared in MUTs structure - reset_type = mut.get('reset_type') - reset_tout = mut.get('reset_tout') - images_config = mut.get('images_config') - mobo_config = mut.get('mobo_config') + reset_type = mut.get('reset_type') # reboot.txt, reset.txt, shutdown.txt + reset_tout = mut.get('reset_tout') # COPY_IMAGE -> RESET_PROC -> SLEEP(RESET_TOUT) + image_dest = mut.get('image_dest') # Image file destination DISK + IMAGE_DEST + BINARY_NAME + images_config = mut.get('images_config') # Available images selection via config file + mobo_config = mut.get('mobo_config') # Available board configuration selection e.g. core selection etc. # Program # When the build and test system were separate, this was relative to a @@ -545,7 +566,7 @@ class SingleTestRunner(object): test_all_result = [] for test_index in range(test_loops): # Choose one method of copy files to mbed virtual drive - _copy_res, _err_msg, _copy_method = self.file_copy_method_selector(image_path, disk, self.opts_copy_method) + _copy_res, _err_msg, _copy_method = self.file_copy_method_selector(image_path, disk, self.opts_copy_method, image_dest=image_dest) # Host test execution start_host_exec_time = time() From fd23d125db56fc8f3ffa5bd20c06f107d0f8261e Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Tue, 12 Aug 2014 13:19:00 +0100 Subject: [PATCH 07/18] reset functionality extensions: added function which will handle image configuration file changes after binary (image) copy to target. Now we can copy binary (image) into specified directory on MUTs disk --- workspace_tools/test_api.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/workspace_tools/test_api.py b/workspace_tools/test_api.py index f789061d5f..11efc6031a 100644 --- a/workspace_tools/test_api.py +++ b/workspace_tools/test_api.py @@ -331,7 +331,8 @@ class SingleTestRunner(object): def generate_test_summary_by_target(self, test_summary, shuffle_seed=None): """ Prints well-formed summary with results (SQL table like) - table shows text x toolchain test result matrix """ + table shows text x toolchain test result matrix + """ RESULT_INDEX = 0 TARGET_INDEX = 1 TOOLCHAIN_INDEX = 2 @@ -448,7 +449,7 @@ class SingleTestRunner(object): browser.get(file_path) browser.close() - def image_copy_method_selector(self, target, image_path, disk, copy_method, + def image_copy_method_selector(self, target_name, image_path, disk, copy_method, images_config=None, image_dest=None): """ Function copied image file and fiddles with image configuration files in needed. This function will select proper image configuration (modify image config file @@ -458,15 +459,16 @@ class SingleTestRunner(object): _copy_res, _err_msg, _copy_method = self.file_copy_method_selector(image_path, disk, self.opts_copy_method, image_dest=image_dest) if images_config is not None: + # For different targets additional configuration file has to be changed + # Here we select target and proper function to handle configuration change if target == 'ARM_MPS2': images_cfg_path = images_config image0file_path = os.path.join(disk, image_dest, basename(image_path)) - mps2_set_board_image_file(disk, images_cfg_path, image0file_path): + mps2_set_board_image_file(disk, images_cfg_path, image0file_path) return _copy_res, _err_msg, _copy_method - def file_copy_method_selector(self, image_path, disk, copy_method, - image_dest='', image_cfg_func=None, image_cfg_func_params=None): + def file_copy_method_selector(self, image_path, disk, copy_method, image_dest=''): """ Copy file depending on method you want to use. Handles exception and return code from shell copy commands. """ @@ -566,7 +568,10 @@ class SingleTestRunner(object): test_all_result = [] for test_index in range(test_loops): # Choose one method of copy files to mbed virtual drive - _copy_res, _err_msg, _copy_method = self.file_copy_method_selector(image_path, disk, self.opts_copy_method, image_dest=image_dest) + #_copy_res, _err_msg, _copy_method = self.file_copy_method_selector(image_path, disk, self.opts_copy_method, image_dest=image_dest) + + _copy_res, _err_msg, _copy_method = self.image_copy_method_selector(target_name, image_path, disk, self.opts_copy_method, + images_config, image_dest) # Host test execution start_host_exec_time = time() From 698fe930e622ebb7ec74598a9c11955937c1488b Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Tue, 12 Aug 2014 13:24:04 +0100 Subject: [PATCH 08/18] Changed functions' docstring format a little to mach singletest, test api and test webapi files --- workspace_tools/singletest.py | 3 +- workspace_tools/test_api.py | 91 ++++++++++++++++++++++------------ workspace_tools/test_webapi.py | 18 ++++--- 3 files changed, 73 insertions(+), 39 deletions(-) diff --git a/workspace_tools/singletest.py b/workspace_tools/singletest.py index 3d867aeb7e..608f23ce0f 100644 --- a/workspace_tools/singletest.py +++ b/workspace_tools/singletest.py @@ -91,7 +91,8 @@ from workspace_tools.test_api import get_default_test_options_parser def get_version(): - """ Returns test script version """ + """ Returns test script version + """ single_test_version_major = 1 single_test_version_minor = 1 return (single_test_version_major, single_test_version_minor) diff --git a/workspace_tools/test_api.py b/workspace_tools/test_api.py index 11efc6031a..98360fe6aa 100644 --- a/workspace_tools/test_api.py +++ b/workspace_tools/test_api.py @@ -71,7 +71,8 @@ class ProcessObserver(Thread): class SingleTestExecutor(threading.Thread): - """ Example: Single test class in separate thread usage """ + """ Example: Single test class in separate thread usage + """ def __init__(self, single_test): self.single_test = single_test threading.Thread.__init__(self) @@ -93,8 +94,8 @@ class SingleTestExecutor(threading.Thread): class SingleTestRunner(object): - """ Object wrapper for single test run which may involve multiple MUTs.""" - + """ Object wrapper for single test run which may involve multiple MUTs + """ RE_DETECT_TESTCASE_RESULT = None # Return codes for test script @@ -150,7 +151,8 @@ class SingleTestRunner(object): _opts_jobs=None, _opts_extend_test_timeout=None ): - """ Let's try hard to init this object """ + """ Let's try hard to init this object + """ PATTERN = "\\{(" + "|".join(self.TEST_RESULT_MAPPING.keys()) + ")\\}" self.RE_DETECT_TESTCASE_RESULT = re.compile(PATTERN) # Settings related to test loops counters @@ -195,7 +197,8 @@ class SingleTestRunner(object): return self.shuffle_random_seed def is_shuffle_seed_float(self): - """ return true if function parameter can be converted to float """ + """ return true if function parameter can be converted to float + """ result = True try: float(self.shuffle_random_seed) @@ -372,7 +375,8 @@ class SingleTestRunner(object): def generate_test_summary(self, test_summary, shuffle_seed=None): """ Prints well-formed summary with results (SQL table like) - table shows target x test results matrix across """ + table shows target x test results matrix across + """ result = "Test summary:\n" # Pretty table package is used to print results pt = PrettyTable(["Result", "Target", "Toolchain", "Test ID", "Test Description", @@ -410,7 +414,8 @@ class SingleTestRunner(object): return result def test_loop_list_to_dict(self, test_loops_str): - """ Transforms test_id=X,test_id=X,test_id=X into dictionary {test_id : test_id_loops_count} """ + """ Transforms test_id=X,test_id=X,test_id=X into dictionary {test_id : test_id_loops_count} + """ result = {} if test_loops_str: test_loops = test_loops_str.split(',') @@ -427,7 +432,8 @@ class SingleTestRunner(object): def get_test_loop_count(self, test_id): """ This function returns no. of loops per test (deducted by test_id_. - If test is not in list of redefined loop counts it will use default value. """ + If test is not in list of redefined loop counts it will use default value. + """ result = self.GLOBAL_LOOPS_COUNT if test_id in self.TEST_LOOPS_DICT: result = self.TEST_LOOPS_DICT[test_id] @@ -506,7 +512,8 @@ class SingleTestRunner(object): return result, resutl_msg, copy_method def delete_file(self, file_path): - """ Remove file from the system """ + """ Remove file from the system + """ result = True resutl_msg = "" try: @@ -518,7 +525,8 @@ class SingleTestRunner(object): def handle(self, test_spec, target_name, toolchain_name, test_loops=1): """ Function determines MUT's mbed disk/port and copies binary to - target. Test is being invoked afterwards. """ + target. Test is being invoked afterwards. + """ data = json.loads(test_spec) # Get test information, image and test timeout test_id = data['test_id'] @@ -609,7 +617,8 @@ class SingleTestRunner(object): def print_test_result(self, test_result, target_name, toolchain_name, test_id, test_description, elapsed_time, duration): - """ Use specific convention to print test result and related data.""" + """ Use specific convention to print test result and related data + """ tokens = [] tokens.append("TargetTest") tokens.append(target_name) @@ -622,13 +631,15 @@ class SingleTestRunner(object): return result def shape_test_loop_ok_result_count(self, test_all_result): - """ Reformats list of results to simple string """ + """ Reformats list of results to simple string + """ test_loop_count = len(test_all_result) test_loop_ok_result = test_all_result.count(self.TEST_RESULT_OK) return "%d/%d"% (test_loop_ok_result, test_loop_count) def shape_global_test_loop_result(self, test_all_result): - """ Reformats list of results to simple string """ + """ Reformats list of results to simple string + """ result = self.TEST_RESULT_FAIL if all(test_all_result[0] == res for res in test_all_result): result = test_all_result[0] @@ -637,7 +648,8 @@ class SingleTestRunner(object): def run_host_test(self, name, disk, port, duration, reset=None, reset_tout=None, verbose=False, extra_serial=None): """ Function creates new process with host test configured with particular test case. Function also is pooling for serial port activity from process to catch all data - printed by test runner and host test during test execution.""" + printed by test runner and host test during test execution + """ # print "{%s} port:%s disk:%s" % (name, port, disk), cmd = ["python", "%s.py" % name, '-p', port, '-d', disk, '-t', str(duration)] @@ -692,7 +704,8 @@ class SingleTestRunner(object): return result def is_peripherals_available(self, target_mcu_name, peripherals=None): - """ Checks if specified target should run specific peripheral test case.""" + """ Checks if specified target should run specific peripheral test case + """ if peripherals is not None: peripherals = set(peripherals) for id, mut in self.muts.iteritems(): @@ -709,7 +722,8 @@ class SingleTestRunner(object): return False def shape_test_request(self, mcu, image_path, test_id, duration=10): - """ Function prepares JOSN structure describing test specification.""" + """ Function prepares JOSN structure describing test specification + """ test_spec = { "mcu": mcu, "image": image_path, @@ -720,7 +734,8 @@ class SingleTestRunner(object): def get_unique_value_from_summary(test_summary, index): - """ Gets list of unique target names """ + """ Gets list of unique target names + """ result = [] for test in test_summary: target_name = test[index] @@ -730,7 +745,8 @@ def get_unique_value_from_summary(test_summary, index): def get_unique_value_from_summary_ext(test_summary, index_key, index_val): - """ Gets list of unique target names and return dictionary """ + """ Gets list of unique target names and return dictionary + """ result = {} for test in test_summary: key = test[index_key] @@ -741,7 +757,8 @@ def get_unique_value_from_summary_ext(test_summary, index_key, index_val): def show_json_file_format_error(json_spec_filename, line, column): - """ Prints JSON broken content """ + """ Prints JSON broken content + """ with open(json_spec_filename) as data_file: line_no = 1 for json_line in data_file: @@ -755,7 +772,8 @@ def show_json_file_format_error(json_spec_filename, line, column): def json_format_error_defect_pos(json_error_msg): """ Gets first error line and column in JSON file format. - Parsed from exception thrown by json.loads() string """ + Parsed from exception thrown by json.loads() string + """ result = None line, column = 0, 0 # Line value search @@ -775,7 +793,8 @@ def json_format_error_defect_pos(json_error_msg): def get_json_data_from_file(json_spec_filename, verbose=False): - """ Loads from file JSON formatted string to data structure """ + """ Loads from file JSON formatted string to data structure + """ result = None try: with open(json_spec_filename) as data_file: @@ -801,7 +820,8 @@ def get_json_data_from_file(json_spec_filename, verbose=False): def print_muts_configuration_from_json(json_data, join_delim=", "): - """ Prints MUTs configuration passed to test script for verboseness. """ + """ Prints MUTs configuration passed to test script for verboseness + """ muts_info_cols = [] # We need to check all unique properties for each defined MUT for k in json_data: @@ -830,7 +850,8 @@ def print_muts_configuration_from_json(json_data, join_delim=", "): def print_test_configuration_from_json(json_data, join_delim=", "): - """ Prints test specification configuration passed to test script for verboseness. """ + """ Prints test specification configuration passed to test script for verboseness + """ toolchains_info_cols = [] # We need to check all toolchains for each device for k in json_data: @@ -893,7 +914,8 @@ def print_test_configuration_from_json(json_data, join_delim=", "): def get_avail_tests_summary_table(cols=None, result_summary=True, join_delim=','): """ Generates table summary with all test cases and additional test cases information using pretty print functionality. Allows test suite user to - see test cases. """ + see test cases + """ # get all unique test ID prefixes unique_test_id = [] for test in TESTS: @@ -980,7 +1002,8 @@ def get_avail_tests_summary_table(cols=None, result_summary=True, join_delim=',' def progress_bar(percent_progress, saturation=0): - """ This function creates progress bar with optional simple saturation mark""" + """ This function creates progress bar with optional simple saturation mark + """ step = int(percent_progress / 2) # Scale by to (scale: 1 - 50) str_progress = '#' * step + '.' * int(50 - step) c = '!' if str_progress[38] == '.' else '|' @@ -991,7 +1014,8 @@ def progress_bar(percent_progress, saturation=0): def singletest_in_cli_mode(single_test): - """ Runs SingleTestRunner object in CLI (Command line interface) mode """ + """ Runs SingleTestRunner object in CLI (Command line interface) mode + """ start = time() # Execute tests depending on options and filter applied test_summary, shuffle_seed = single_test.execute() @@ -1054,19 +1078,22 @@ def mps2_set_board_image_file(disk, images_cfg_path, image0file_path, image_name def mps2_select_core(disk, mobo_config_name=""): - """ Function selects actual core """ + """ Function selects actual core + """ # TODO: implement core selection pass def mps2_switch_usb_auto_mounting_after_restart(disk, usb_config_name=""): - """ Function alters configuration to allow USB MSD to be mounted after restarts """ + """ Function alters configuration to allow USB MSD to be mounted after restarts + """ # TODO: implement USB MSD restart detection pass class TestLogger(): - """ Super-class for logging and printing ongoing events for test suite pass """ + """ Super-class for logging and printing ongoing events for test suite pass + """ def __init__(self): self.Log = [] @@ -1089,8 +1116,7 @@ class TestLogger(): def log_to_file(self, LogToFileAttr, file_name): """ Class will log to file current log entries. Note: you should be able to see log file like this: - - tail -f log_file.txt + tail -f log_file.txt """ pass @@ -1100,7 +1126,8 @@ class CLITestLogger(TestLogger): def get_default_test_options_parser(): - """ Get common test script options used by CLI, webservices etc. """ + """ Get common test script options used by CLI, webservices etc. + """ parser = optparse.OptionParser() parser.add_option('-i', '--tests', dest='test_spec_filename', diff --git a/workspace_tools/test_webapi.py b/workspace_tools/test_webapi.py index 0f2c22c0b4..0a70b0c4b2 100644 --- a/workspace_tools/test_webapi.py +++ b/workspace_tools/test_webapi.py @@ -53,7 +53,8 @@ class SingleTestRunnerWebService(SingleTestRunner): REST_TEST_RESULTS='test_results') def get_rest_result_template(self, result, command, success_code): - """ Returns common part of every web service request """ + """ Returns common part of every web service request + """ result = {"result" : result, "command" : command, "success_code": success_code} # 0 - OK, >0 - Error number @@ -61,22 +62,26 @@ class SingleTestRunnerWebService(SingleTestRunner): # REST API handlers for Flask framework def rest_api_status(self): - """ Returns current test execution status. E.g. running / finished etc. """ + """ Returns current test execution status. E.g. running / finished etc. + """ with self.resource_lock: pass def rest_api_config(self): - """ Returns configuration passed to SingleTest executor """ + """ Returns configuration passed to SingleTest executor + """ with self.resource_lock: pass def rest_api_log(self): - """ Returns current test log """ + """ Returns current test log + """ with self.resource_lock: pass def rest_api_request_handler(self, request_type): - """ Returns various data structures. Both static and mutable during test """ + """ Returns various data structures. Both static and mutable during test + """ result = {} success_code = 0 with self.resource_lock: @@ -97,7 +102,8 @@ def singletest_in_webservice_mode(): def get_default_test_webservice_options_parser(): - """ Get test script web service options used by CLI, webservices etc. """ + """ Get test script web service options used by CLI, webservices etc. + """ parser = get_default_test_options_parser() # Things related to web services offered by test suite scripts From 0e6803b7873d505f31a196426d9125cac4859cb0 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Tue, 12 Aug 2014 13:47:08 +0100 Subject: [PATCH 09/18] Bugfix: fixed function printing table with test results per target (switch -t) --- workspace_tools/test_api.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/workspace_tools/test_api.py b/workspace_tools/test_api.py index 98360fe6aa..7454dcb0f7 100644 --- a/workspace_tools/test_api.py +++ b/workspace_tools/test_api.py @@ -348,14 +348,18 @@ class SingleTestRunner(object): unique_toolchains = get_unique_value_from_summary(test_summary, TOOLCHAIN_INDEX) result = "Test summary:\n" - result_dict = {} # test : { toolchain : result } for target in unique_targets: + result_dict = {} # test : { toolchain : result } + unique_target_toolchains = [] for test in test_summary: - if test[TEST_INDEX] not in result_dict: - result_dict[test[TEST_INDEX]] = {} - result_dict[test[TEST_INDEX]][test[TOOLCHAIN_INDEX]] = test[RESULT_INDEX] + if test[TARGET_INDEX] == target: + if test[TOOLCHAIN_INDEX] not in unique_target_toolchains: + unique_target_toolchains.append(test[TOOLCHAIN_INDEX]) + if test[TEST_INDEX] not in result_dict: + result_dict[test[TEST_INDEX]] = {} + result_dict[test[TEST_INDEX]][test[TOOLCHAIN_INDEX]] = test[RESULT_INDEX] - pt_cols = ["Target", "Test ID", "Test Description"] + unique_toolchains + pt_cols = ["Target", "Test ID", "Test Description"] + unique_target_toolchains pt = PrettyTable(pt_cols) for col in pt_cols: pt.align[col] = "l" @@ -365,7 +369,8 @@ class SingleTestRunner(object): test_results = result_dict[test] row = [target, test, unique_test_desc[test]] for toolchain in unique_toolchains: - row.append(test_results[toolchain]) + if toolchain in test_results: + row.append(test_results[toolchain]) pt.add_row(row) result += pt.get_string() shuffle_seed_text = "Shuffle Seed: %.*f"% (self.SHUFFLE_SEED_ROUND, From 23bcf850b3486d58fb3ad0696fb9767859e186db Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Tue, 12 Aug 2014 14:04:21 +0100 Subject: [PATCH 10/18] Small update to new functionality for logger --- workspace_tools/test_api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/workspace_tools/test_api.py b/workspace_tools/test_api.py index 7454dcb0f7..f818cc6558 100644 --- a/workspace_tools/test_api.py +++ b/workspace_tools/test_api.py @@ -1103,9 +1103,10 @@ class TestLogger(): self.Log = [] self.LogType = construct_enum(INFO='Info', - NOTIF='Notification', WARN='Warning', - ERROR='Error') + NOTIF='Notification', + ERROR='Error', + EXCEPT='Exception') self.LogToFileAttr = construct_enum(CREATE=1, # Create or overwrite existing log file APPEND=2) # Append to existing log file From f80d0302a7e92e9e1a9e636df59d9fc132e6eb69 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Tue, 12 Aug 2014 15:12:57 +0100 Subject: [PATCH 11/18] Added simple logger CLI logger (with logging to file) plus simple logger super class --- workspace_tools/test_api.py | 59 +++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/workspace_tools/test_api.py b/workspace_tools/test_api.py index f818cc6558..866d343790 100644 --- a/workspace_tools/test_api.py +++ b/workspace_tools/test_api.py @@ -149,8 +149,7 @@ class SingleTestRunner(object): _opts_copy_method=None, _opts_mut_reset_type=None, _opts_jobs=None, - _opts_extend_test_timeout=None - ): + _opts_extend_test_timeout=None): """ Let's try hard to init this object """ PATTERN = "\\{(" + "|".join(self.TEST_RESULT_MAPPING.keys()) + ")\\}" @@ -193,6 +192,8 @@ class SingleTestRunner(object): self.opts_jobs = _opts_jobs if _opts_jobs is not None else 1 self.opts_extend_test_timeout = _opts_extend_test_timeout + self.logger = CLITestLogger() # Default test logger + def shuffle_random_func(self): return self.shuffle_random_seed @@ -1099,8 +1100,14 @@ def mps2_switch_usb_auto_mounting_after_restart(disk, usb_config_name=""): class TestLogger(): """ Super-class for logging and printing ongoing events for test suite pass """ - def __init__(self): - self.Log = [] + def __init__(self, store_log=True): + """ We can control if logger actually stores log in memory + or just handled all log entries immediately + """ + self.log = [] + self.log_to_file = False + self.log_file_name = None + self.store_log = store_log self.LogType = construct_enum(INFO='Info', WARN='Warning', @@ -1111,24 +1118,46 @@ class TestLogger(): self.LogToFileAttr = construct_enum(CREATE=1, # Create or overwrite existing log file APPEND=2) # Append to existing log file - def log_line(self, LogType, log_line, log_time=None): - log_timestamp = time.time() if log_time is None else log_time + def log_line(self, LogType, log_line): + """ Log one line of text + """ + log_timestamp = time.time() log_entry = {'log_type' : LogType, 'log_timestamp' : log_timestamp, 'log_line' : log_line, '_future' : None} - self.Log.append(log_entry) - - def log_to_file(self, LogToFileAttr, file_name): - """ Class will log to file current log entries. - Note: you should be able to see log file like this: - tail -f log_file.txt - """ - pass + # Store log in memory + if self.store_log: + self.log.append(log_entry) + return log_entry class CLITestLogger(TestLogger): - pass + """ Logger used with CLI (Command line interface) test suite. Logs on screen and to file if needed + """ + def __init__(self, store_log=True, file_name=None): + TestLogger.__init__(self) + self.log_file_name = file_name + self.TIMESTAMP_FORMAT = '%y-%m-%d %H:%M:%S' + + def log_print(self, log_entry, timestamp=True): + """ Prints on screen formatted log entry + """ + ts = log_entry['log_timestamp'] + timestamp_str = datetime.datetime.fromtimestamp(ts).strftime("[%s] "% self.TIMESTAMP_FORMAT) if timestamp else '' + log_line_str = "%(log_type)s: %(log_line)s"% (log_entry) + return timestamp_str + log_line_str + + def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'): + log_entry = TestLogger.log_line(self, LogType, log_line) + log_line_str = self.log_print(log_entry, timestamp) + print log_line_str + if self.log_file_name is not None: + try: + with open(self.log_file_name, 'a') as file: + file.write(log_line_str + line_delim) + except IOError: + pass def get_default_test_options_parser(): From 8cc7678c4bd0cf52ef368dd7c1e273a9d06d5858 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Tue, 12 Aug 2014 16:11:22 +0100 Subject: [PATCH 12/18] Small comment indent in test web api file --- workspace_tools/test_webapi.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/workspace_tools/test_webapi.py b/workspace_tools/test_webapi.py index 0a70b0c4b2..59273e80d2 100644 --- a/workspace_tools/test_webapi.py +++ b/workspace_tools/test_webapi.py @@ -74,8 +74,7 @@ class SingleTestRunnerWebService(SingleTestRunner): pass def rest_api_log(self): - """ Returns current test log - """ + """ Returns current test log """ with self.resource_lock: pass From 61f028f0508be21d510229a1dfdd2402b66f0274 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Tue, 12 Aug 2014 16:15:33 +0100 Subject: [PATCH 13/18] Fixed commit issues regarding logging mechanism - missed during previous commits --- workspace_tools/singletest.py | 1 + workspace_tools/test_api.py | 36 +++++++++++++++++++++-------------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/workspace_tools/singletest.py b/workspace_tools/singletest.py index 608f23ce0f..3301f56901 100644 --- a/workspace_tools/singletest.py +++ b/workspace_tools/singletest.py @@ -169,6 +169,7 @@ if __name__ == '__main__': single_test = SingleTestRunner(_global_loops_count=opts.test_global_loops_value, _test_loops_list=opts.test_loops_list, _muts=MUTs, + _opts_log_file_name=opts.log_file_name, _test_spec=test_spec, _opts_goanna_for_mbed_sdk=opts.goanna_for_mbed_sdk, _opts_goanna_for_tests=opts.goanna_for_tests, diff --git a/workspace_tools/test_api.py b/workspace_tools/test_api.py index 866d343790..1e0d115318 100644 --- a/workspace_tools/test_api.py +++ b/workspace_tools/test_api.py @@ -25,6 +25,7 @@ import time import pprint import random import optparse +import datetime import threading from types import ListType from prettytable import PrettyTable @@ -131,6 +132,7 @@ class SingleTestRunner(object): _global_loops_count=1, _test_loops_list=None, _muts={}, + _opts_log_file_name=None, _test_spec={}, _opts_goanna_for_mbed_sdk=None, _opts_goanna_for_tests=None, @@ -173,6 +175,7 @@ class SingleTestRunner(object): self.test_spec = _test_spec # Settings passed e.g. from command line + self.opts_log_file_name = _opts_log_file_name self.opts_goanna_for_mbed_sdk = _opts_goanna_for_mbed_sdk self.opts_goanna_for_tests = _opts_goanna_for_tests self.opts_shuffle_test_order = _opts_shuffle_test_order @@ -192,7 +195,7 @@ class SingleTestRunner(object): self.opts_jobs = _opts_jobs if _opts_jobs is not None else 1 self.opts_extend_test_timeout = _opts_extend_test_timeout - self.logger = CLITestLogger() # Default test logger + self.logger = CLITestLogger(file_name=self.opts_log_file_name) # Default test logger def shuffle_random_func(self): return self.shuffle_random_seed @@ -223,7 +226,7 @@ class SingleTestRunner(object): # print '=== %s::%s ===' % (target, toolchain) # Let's build our test if target not in TARGET_MAP: - print 'Skipped tests for %s target. Target platform not found' % (target) + print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Target platform not found' % (target)) continue T = TARGET_MAP[target] @@ -236,7 +239,7 @@ class SingleTestRunner(object): clean=clean_mbed_libs_options, jobs=self.opts_jobs) if not build_mbed_libs_result: - print 'Skipped tests for %s target. Toolchain %s is not yet supported for this target' % (T.name, toolchain) + print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Toolchain %s is not yet supported for this target' % (T.name, toolchain)) continue build_dir = join(BUILD_DIR, "test", target, toolchain) @@ -256,19 +259,19 @@ class SingleTestRunner(object): if self.opts_test_only_peripheral and not test.peripherals: if self.opts_verbose_skipped_tests: - print "TargetTest::%s::NotPeripheralTestSkipped()" % (target) + print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target)) continue if self.opts_test_only_common and test.peripherals: if self.opts_verbose_skipped_tests: - print "TargetTest::%s::PeripheralTestSkipped()" % (target) + print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral test skipped for target %s'% (target)) continue if test.automated and test.is_supported(target, toolchain): if not self.is_peripherals_available(target, test.peripherals): if self.opts_verbose_skipped_tests: test_peripherals = test.peripherals if test.peripherals else [] - print "TargetTest::%s::TestSkipped(%s)" % (target, ",".join(test_peripherals)) + print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral %s test skipped for target %s'% (",".join(test_peripherals), target)) continue build_project_options = ["analyze"] if self.opts_goanna_for_tests else None @@ -567,7 +570,7 @@ class SingleTestRunner(object): # base network folder base path: join(NETWORK_BASE_PATH, ) image_path = image if not exists(image_path): - print "Error: Image file does not exist: %s" % image_path + print self.logger.log_line(self.logger.LogType.ERROR, 'Image file does not exist: %s' % image_path) elapsed_time = 0 test_result = self.TEST_RESULT_NO_IMAGE return (test_result, target_name, toolchain_name, @@ -593,7 +596,7 @@ class SingleTestRunner(object): single_test_result = self.TEST_RESULT_UNDEF # singe test run result if not _copy_res: # Serial port copy error single_test_result = self.TEST_RESULT_IOERR_COPY - print "Error: Copy method '%s'. %s"% (_copy_method, _err_msg) + print self.logger.log_line(self.logger.LogType.ERROR, "Copy method '%s' failed. Reason: %s"% (_copy_method, _err_msg)) else: # Copy Extra Files if not target_by_mcu.is_disk_virtual and test.extra_files: @@ -808,7 +811,7 @@ def get_json_data_from_file(json_spec_filename, verbose=False): result = json.load(data_file) except ValueError as json_error_msg: result = None - print "Error in '%s' file. %s" % (json_spec_filename, json_error_msg) + print self.logger.log_line(self.logger.LogType.ERROR, 'JSON file %s parsing failed. Reason: %s' % (json_spec_filename, json_error_msg)) # We can print where error occurred inside JSON file if we can parse exception msg json_format_defect_pos = json_format_error_defect_pos(str(json_error_msg)) if json_format_defect_pos is not None: @@ -818,7 +821,8 @@ def get_json_data_from_file(json_spec_filename, verbose=False): show_json_file_format_error(json_spec_filename, line, column) except IOError as fileopen_error_msg: - print "Error: %s" % (fileopen_error_msg) + print self.logger.log_line(self.logger.LogType.ERROR, 'JSON file %s not opened. Reason: %s'% (json_spec_filename, fileopen_error_msg)) + print if verbose and result: pp = pprint.PrettyPrinter(indent=4) pp.pprint(result) @@ -1121,7 +1125,7 @@ class TestLogger(): def log_line(self, LogType, log_line): """ Log one line of text """ - log_timestamp = time.time() + log_timestamp = time() log_entry = {'log_type' : LogType, 'log_timestamp' : log_timestamp, 'log_line' : log_line, @@ -1138,7 +1142,8 @@ class CLITestLogger(TestLogger): def __init__(self, store_log=True, file_name=None): TestLogger.__init__(self) self.log_file_name = file_name - self.TIMESTAMP_FORMAT = '%y-%m-%d %H:%M:%S' + #self.TIMESTAMP_FORMAT = '%y-%m-%d %H:%M:%S' # Full date and time + self.TIMESTAMP_FORMAT = '%H:%M:%S' # Time only def log_print(self, log_entry, timestamp=True): """ Prints on screen formatted log entry @@ -1151,14 +1156,13 @@ class CLITestLogger(TestLogger): def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'): log_entry = TestLogger.log_line(self, LogType, log_line) log_line_str = self.log_print(log_entry, timestamp) - print log_line_str if self.log_file_name is not None: try: with open(self.log_file_name, 'a') as file: file.write(log_line_str + line_delim) except IOError: pass - + return log_line_str def get_default_test_options_parser(): """ Get common test script options used by CLI, webservices etc. @@ -1293,6 +1297,10 @@ def get_default_test_options_parser(): type="int", help='You can increase global timeout for each test by specifying additional test timeout in seconds') + parser.add_option('-l', '--log', + dest='log_file_name', + help='Log events to external file (note not all console entries may be visible in log file)') + parser.add_option('', '--verbose-skipped', dest='verbose_skipped_tests', default=False, From d076c510609d047f287bc16645c1dd644fc66649 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Wed, 13 Aug 2014 09:43:49 +0100 Subject: [PATCH 14/18] Changed short description for --firmware-name switch --- workspace_tools/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workspace_tools/test_api.py b/workspace_tools/test_api.py index 1e0d115318..16e8b74801 100644 --- a/workspace_tools/test_api.py +++ b/workspace_tools/test_api.py @@ -1268,7 +1268,7 @@ def get_default_test_options_parser(): parser.add_option('', '--firmware-name', dest='firmware_global_name', - help='Set global name for all produced projects. E.g. you can call all test binaries firmware.bin') + help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts.') parser.add_option('-u', '--shuffle', dest='shuffle_test_order', From 005c3a7b87a0208f2c8ec12815f7d606e61487d1 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Wed, 13 Aug 2014 11:11:51 +0100 Subject: [PATCH 15/18] Host test: wrapped timeout for MUTs serial port access to handle IO_SERIAL error while accessing serial port --- workspace_tools/host_tests/host_test.py | 9 +++++++++ workspace_tools/host_tests/rtc_auto.py | 4 +++- workspace_tools/host_tests/wait_us_auto.py | 4 +++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/workspace_tools/host_tests/host_test.py b/workspace_tools/host_tests/host_test.py index 09a477357e..6503d174f9 100644 --- a/workspace_tools/host_tests/host_test.py +++ b/workspace_tools/host_tests/host_test.py @@ -86,6 +86,7 @@ class Mbed: print 'Mbed: "%s" "%s"' % (self.port, self.disk) def init_serial(self, baud=9600, extra_baud=9600): + """ Initialize serial port. Function will return error is port can't be opened or initialized """ result = True try: self.serial = Serial(self.port, timeout=1) @@ -100,6 +101,14 @@ class Mbed: self.flush() return result + def serial_timeout(self, timeout): + """ Wraps self.mbed.serial object timeout property """ + result = None + if self.serial: + self.serial.timeout = timeout + result = True + return result + def serial_read(self, count=1): """ Wraps self.mbed.serial object read method """ result = None diff --git a/workspace_tools/host_tests/rtc_auto.py b/workspace_tools/host_tests/rtc_auto.py index 3e4d5f7a09..5a6e4c6c58 100644 --- a/workspace_tools/host_tests/rtc_auto.py +++ b/workspace_tools/host_tests/rtc_auto.py @@ -26,7 +26,9 @@ class RTCTest(DefaultTest): def run(self): test_result = True - c = self.mbed.serial.timeout = None + if self.mbed.serial_timeout(None) is None: + self.print_result("ioerr_serial") + return for i in range(0, 5): c = self.mbed.serial_read(38) # 38 len("[1256729742] [2009-10-28 11:35:42 AM]\n" if c is None: diff --git a/workspace_tools/host_tests/wait_us_auto.py b/workspace_tools/host_tests/wait_us_auto.py index 982b5719a8..9171bdb2e8 100644 --- a/workspace_tools/host_tests/wait_us_auto.py +++ b/workspace_tools/host_tests/wait_us_auto.py @@ -23,7 +23,9 @@ class WaitusTest(DefaultTest): def run(self): test_result = True # First character to start test (to know after reset when test starts) - self.mbed.serial.timeout = None + if self.mbed.serial_timeout(None) is None: + self.print_result("ioerr_serial") + return c = self.mbed.serial_read(1) if c is None: self.print_result("ioerr_serial") From a5bdd4a8bba57d2d519b81d25131910829bc2ff0 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Wed, 13 Aug 2014 11:29:06 +0100 Subject: [PATCH 16/18] More comments added to methods in host_test.py --- workspace_tools/host_tests/host_test.py | 61 ++++++++++++++++++------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/workspace_tools/host_tests/host_test.py b/workspace_tools/host_tests/host_test.py index 6503d174f9..3045928fc8 100644 --- a/workspace_tools/host_tests/host_test.py +++ b/workspace_tools/host_tests/host_test.py @@ -28,8 +28,7 @@ from time import sleep from sys import stdout class Mbed: - """ - Base class for a host driven test + """ Base class for a host driven test """ def __init__(self): parser = OptionParser() @@ -83,10 +82,11 @@ class Mbed: self.extra_serial = None self.serial = None self.timeout = self.DEFAULT_TOUT if self.options.timeout is None else self.options.timeout - print 'Mbed: "%s" "%s"' % (self.port, self.disk) + print 'Host test instrumentation on port: "%s" with serial: "%s"' % (self.port, self.disk) def init_serial(self, baud=9600, extra_baud=9600): - """ Initialize serial port. Function will return error is port can't be opened or initialized """ + """ Initialize serial port. Function will return error is port can't be opened or initialized + """ result = True try: self.serial = Serial(self.port, timeout=1) @@ -102,7 +102,8 @@ class Mbed: return result def serial_timeout(self, timeout): - """ Wraps self.mbed.serial object timeout property """ + """ Wraps self.mbed.serial object timeout property + """ result = None if self.serial: self.serial.timeout = timeout @@ -110,7 +111,8 @@ class Mbed: return result def serial_read(self, count=1): - """ Wraps self.mbed.serial object read method """ + """ Wraps self.mbed.serial object read method + """ result = None if self.serial: try: @@ -120,7 +122,8 @@ class Mbed: return result def serial_write(self, write_buffer): - """ Wraps self.mbed.serial object write method """ + """ Wraps self.mbed.serial object write method + """ result = -1 if self.serial: try: @@ -131,12 +134,12 @@ class Mbed: def safe_sendBreak(self, serial): """ Wraps serial.sendBreak() to avoid serial::serialposix.py exception on Linux - Traceback (most recent call last): - File "make.py", line 189, in - serial.sendBreak() - File "/usr/lib/python2.7/dist-packages/serial/serialposix.py", line 511, in sendBreak - termios.tcsendbreak(self.fd, int(duration/0.25)) - error: (32, 'Broken pipe') + Traceback (most recent call last): + File "make.py", line 189, in + serial.sendBreak() + File "/usr/lib/python2.7/dist-packages/serial/serialposix.py", line 511, in sendBreak + termios.tcsendbreak(self.fd, int(duration/0.25)) + error: (32, 'Broken pipe') """ result = True try: @@ -151,15 +154,22 @@ class Mbed: return result def touch_file(self, path): + """ Touch file and set timestamp to items + """ with open(path, 'a'): os.utime(path, None) def reset_timeout(self, timeout): + """ Timeout executed just after reset command is issued + """ for n in range(0, timeout): sleep(1) def reset(self): - """ reboot.txt - startup from standby state, reboots when in run mode. + """ Reset function. Supports 'standard' send break command via Mbed's CDC, + also handles other reset modes. + E.g. reset by touching file with specific file name: + reboot.txt - startup from standby state, reboots when in run mode. shutdown.txt - shutdown from run mode reset.txt - reset FPGA during run mode """ @@ -173,6 +183,8 @@ class Mbed: self.reset_timeout(reset_tout_s) def flush(self): + """ Flush serial ports + """ self.serial.flushInput() self.serial.flushOutput() if self.extra_serial: @@ -181,10 +193,15 @@ class Mbed: class Test: + """ Baseclass for host test's test runner + """ def __init__(self): self.mbed = Mbed() def run(self): + """ Test runner for host test. This function will start executing + test and forward test result via serial port to test suite + """ try: result = self.test() self.print_result("success" if result else "failure") @@ -193,7 +210,8 @@ class Test: self.print_result("error") def setup(self): - """ Setup and check if configuration for test is correct. E.g. if serial port can be opened """ + """ Setup and check if configuration for test is correct. E.g. if serial port can be opened + """ result = True if not self.mbed.serial: result = False @@ -201,16 +219,20 @@ class Test: return result def notify(self, message): - """ On screen notification function """ + """ On screen notification function + """ print message stdout.flush() def print_result(self, result): - """ Test result unified printing function """ + """ Test result unified printing function + """ self.notify("\n{%s}\n{end}" % result) class DefaultTest(Test): + """ Test class with serial port initialization + """ def __init__(self): Test.__init__(self) serial_init_res = self.mbed.init_serial() @@ -218,6 +240,10 @@ class DefaultTest(Test): class Simple(DefaultTest): + """ Simple, basic host test's test runner waiting for serial port + output from MUT, no supervision over test running in MUT is executed. + Just waiting for result + """ def run(self): try: while True: @@ -230,5 +256,6 @@ class Simple(DefaultTest): except KeyboardInterrupt, _: print "\n[CTRL+c] exit" + if __name__ == '__main__': Simple().run() From 278acbec5d5c375ca50e1c38d8c9550820d0e0ae Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Thu, 14 Aug 2014 11:29:56 +0100 Subject: [PATCH 17/18] Host test: wrapped mbed.serial.timeout into separate function to avoid traceback and send proper IO_SERIAL error from host test(s) --- workspace_tools/host_tests/echo.py | 2 +- workspace_tools/host_tests/host_test.py | 4 ++-- workspace_tools/host_tests/stdio_auto.py | 2 +- workspace_tools/host_tests/tcpecho_client_auto.py | 2 +- workspace_tools/host_tests/udpecho_client_auto.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/workspace_tools/host_tests/echo.py b/workspace_tools/host_tests/echo.py index f5f8300ba6..ad0f7303a8 100644 --- a/workspace_tools/host_tests/echo.py +++ b/workspace_tools/host_tests/echo.py @@ -29,7 +29,7 @@ class EchoTest(Test): TEST="longer serial test" check = True for i in range(1, 100): - self.mbed.serial.write(TEST + "\n") + self.mbed.serial_write(TEST + "\n") l = self.mbed.serial.readline().strip() if not l: continue diff --git a/workspace_tools/host_tests/host_test.py b/workspace_tools/host_tests/host_test.py index 3045928fc8..d6aa1bd8a5 100644 --- a/workspace_tools/host_tests/host_test.py +++ b/workspace_tools/host_tests/host_test.py @@ -124,12 +124,12 @@ class Mbed: def serial_write(self, write_buffer): """ Wraps self.mbed.serial object write method """ - result = -1 + result = None if self.serial: try: result = self.serial.write(write_buffer) except: - result = -1 + result = None return result def safe_sendBreak(self, serial): diff --git a/workspace_tools/host_tests/stdio_auto.py b/workspace_tools/host_tests/stdio_auto.py index 65fd272b9f..b262e065ba 100644 --- a/workspace_tools/host_tests/stdio_auto.py +++ b/workspace_tools/host_tests/stdio_auto.py @@ -31,7 +31,7 @@ class StdioTest(DefaultTest): for i in range(1, 5): random_integer = random.randint(-10000, 10000) print "Generated number: " + str(random_integer) - self.mbed.serial.write(str(random_integer) + "\n") + self.mbed.serial_write(str(random_integer) + "\n") serial_stdio_msg = "" ip_msg_timeout = self.mbed.options.timeout diff --git a/workspace_tools/host_tests/tcpecho_client_auto.py b/workspace_tools/host_tests/tcpecho_client_auto.py index 9f2f26bb8c..1d8b6b0f82 100644 --- a/workspace_tools/host_tests/tcpecho_client_auto.py +++ b/workspace_tools/host_tests/tcpecho_client_auto.py @@ -33,7 +33,7 @@ class TCPEchoClientTest(Test): self.mbed.reset() print "Sending server IP Address to target..." connection_str = ip_address + ":" + str(port_no) + "\n" - self.mbed.serial.write(connection_str) + self.mbed.serial_write(connection_str) class TCPEchoClient_Handler(BaseRequestHandler): diff --git a/workspace_tools/host_tests/udpecho_client_auto.py b/workspace_tools/host_tests/udpecho_client_auto.py index e03d2c59ff..9d65bac3a8 100644 --- a/workspace_tools/host_tests/udpecho_client_auto.py +++ b/workspace_tools/host_tests/udpecho_client_auto.py @@ -32,7 +32,7 @@ class UDPEchoClientTest(Test): self.mbed.reset() print "Sending server IP Address to target..." connection_str = ip_address + ":" + str(port_no) + "\n" - self.mbed.serial.write(connection_str) + self.mbed.serial_write(connection_str) class UDPEchoClient_Handler(BaseRequestHandler): From a09f3dd605f18c1b8ca85be3781304e98cf09110 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Thu, 14 Aug 2014 11:30:50 +0100 Subject: [PATCH 18/18] Bugfix: when displaying extra summary using -t option some indexes are not found and dictionary must be checked against key --- workspace_tools/test_api.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/workspace_tools/test_api.py b/workspace_tools/test_api.py index 16e8b74801..54754958d7 100644 --- a/workspace_tools/test_api.py +++ b/workspace_tools/test_api.py @@ -370,12 +370,14 @@ class SingleTestRunner(object): pt.padding_width = 1 # One space between column edges and contents (default) for test in unique_tests: - test_results = result_dict[test] - row = [target, test, unique_test_desc[test]] - for toolchain in unique_toolchains: - if toolchain in test_results: - row.append(test_results[toolchain]) - pt.add_row(row) + if test in result_dict: + test_results = result_dict[test] + if test in unique_test_desc: + row = [target, test, unique_test_desc[test]] + for toolchain in unique_toolchains: + if toolchain in test_results: + row.append(test_results[toolchain]) + pt.add_row(row) result += pt.get_string() shuffle_seed_text = "Shuffle Seed: %.*f"% (self.SHUFFLE_SEED_ROUND, shuffle_seed if shuffle_seed else self.shuffle_random_seed)