From cc6be1a16e4d10538757313c4b1d3509613e0fc1 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Sun, 3 May 2015 02:32:04 +0100 Subject: [PATCH 1/4] Added support for multiple targets of the same type (hardware platfrom) --- workspace_tools/singletest.py | 4 +- workspace_tools/test_api.py | 77 +++++++++++++++++++------------ workspace_tools/test_exporters.py | 69 ++++++++++++++------------- 3 files changed, 87 insertions(+), 63 deletions(-) diff --git a/workspace_tools/singletest.py b/workspace_tools/singletest.py index e7caef72ca..a2579dd93c 100644 --- a/workspace_tools/singletest.py +++ b/workspace_tools/singletest.py @@ -144,9 +144,9 @@ if __name__ == '__main__': if get_module_avail('mbed_lstools'): mbeds = mbed_lstools.create() - muts_list = mbeds.list_mbeds() + muts_list = mbeds.list_mbeds_ext() if hasattr(mbeds, 'list_mbeds_ext') else mbeds.list_mbeds() for mut in muts_list: - print "MBEDLS: Detected %s, port: %s, mounted: %s"% (mut['platform_name'], + print "MBEDLS: Detected %s, port: %s, mounted: %s"% (mut['platform_name_unique'] if 'platform_name_unique' in mut else mut['platform_name'], mut['serial_port'], mut['mount_point']) diff --git a/workspace_tools/test_api.py b/workspace_tools/test_api.py index c88134eb28..11138a865b 100644 --- a/workspace_tools/test_api.py +++ b/workspace_tools/test_api.py @@ -518,24 +518,30 @@ class SingleTestRunner(object): test_suite_properties['test.path.%s.%s.%s'% (target, toolchain, test_id)] = path # read MUTs, test specification and perform tests - handle_result = self.handle(test_spec, target, toolchain, test_loops=test_loops) - if handle_result: - single_test_result, detailed_test_results = handle_result - else: + handle_results = self.handle(test_spec, target, toolchain, test_loops=test_loops) + + if handle_results is None: continue - # Append test results to global test summary - if single_test_result is not None: - self.test_summary.append(single_test_result) + for handle_result in handle_results: + if handle_result: + single_test_result, detailed_test_results = handle_result + else: + continue - # Add detailed test result to test summary structure - if target not in self.test_summary_ext[toolchain][target]: - self.test_summary_ext[toolchain][target][test_id] = detailed_test_results # test_summary_ext : toolchain : target : test_it + # Append test results to global test summary + if single_test_result is not None: + self.test_summary.append(single_test_result) + + # Add detailed test result to test summary structure + if target not in self.test_summary_ext[toolchain][target]: + if test_id not in self.test_summary_ext[toolchain][target]: + self.test_summary_ext[toolchain][target][test_id] = [] + self.test_summary_ext[toolchain][target][test_id].append(detailed_test_results) test_suite_properties['skipped'] = ', '.join(test_suite_properties['skipped']) self.test_suite_properties_ext[target][toolchain] = test_suite_properties - # return self.test_summary, self.shuffle_random_seed, test_summary_ext, self.test_suite_properties_ext q.put(target + '_'.join(toolchains)) return @@ -821,12 +827,9 @@ class SingleTestRunner(object): result = False return result, resutl_msg - def handle(self, test_spec, target_name, toolchain_name, test_loops=1): - """ Function determines MUT's mbed disk/port and copies binary to - target. - Test is being invoked afterwards. + def handle_mut(self, mut, data, target_name, toolchain_name, test_loops=1): + """ Test is being invoked for given MUT. """ - data = json.loads(test_spec) # Get test information, image and test timeout test_id = data['test_id'] test = TEST_MAP[test_id] @@ -834,13 +837,6 @@ class SingleTestRunner(object): image = data["image"] duration = data.get("duration", 10) - # Find a suitable MUT: - mut = None - for id, m in self.muts.iteritems(): - if m['mcu'] == data['mcu']: - mut = m - break - if mut is None: print "Error: No Mbed available: MUT[%s]" % data['mcu'] return None @@ -852,6 +848,7 @@ class SingleTestRunner(object): return None target_by_mcu = TARGET_MAP[mut['mcu']] + target_name_unique = mut['mcu_unique'] if 'mcu_unique' in mut else mut['mcu'] # Some extra stuff can be declared in MUTs structure reset_type = mut.get('reset_type') # reboot.txt, reset.txt, shutdown.txt reset_tout = mut.get('reset_tout') # COPY_IMAGE -> RESET_PROC -> SLEEP(RESET_TOUT) @@ -911,6 +908,7 @@ class SingleTestRunner(object): 'single_test_result' : single_test_result, 'single_test_output' : single_test_output, 'target_name' : target_name, + 'target_name_unique' : target_name_unique, 'toolchain_name' : toolchain_name, 'test_id' : test_id, 'test_description' : test_description, @@ -919,7 +917,7 @@ class SingleTestRunner(object): 'copy_method' : _copy_method, } - print self.print_test_result(single_test_result, target_name, toolchain_name, + print self.print_test_result(single_test_result, target_name_unique, toolchain_name, test_id, test_description, elapsed_time, single_timeout) # Update database entries for ongoing test @@ -944,7 +942,7 @@ class SingleTestRunner(object): self.db_logger.disconnect() return (self.shape_global_test_loop_result(test_all_result), - target_name, + target_name_unique, toolchain_name, test_id, test_description, @@ -952,6 +950,23 @@ class SingleTestRunner(object): single_timeout, self.shape_test_loop_ok_result_count(test_all_result)), detailed_test_results + def handle(self, test_spec, target_name, toolchain_name, test_loops=1): + """ Function determines MUT's mbed disk/port and copies binary to + target. + """ + handle_results = [] + data = json.loads(test_spec) + + # Find a suitable MUT: + mut = None + for id, m in self.muts.iteritems(): + if m['mcu'] == data['mcu']: + mut = m + handle_result = self.handle_mut(mut, data, target_name, toolchain_name, test_loops=test_loops) + handle_results.append(handle_result) + + return handle_results + def print_test_result(self, test_result, target_name, toolchain_name, test_id, test_description, elapsed_time, duration): """ Use specific convention to print test result and related data @@ -1497,7 +1512,6 @@ def singletest_in_cli_mode(single_test): # Export build results as html report to sparate file write_build_report(build_report, 'tests_build/report.html', single_test.opts_report_build_file_name) - class TestLogger(): """ Super-class for logging and printing ongoing events for test suite pass """ @@ -1627,10 +1641,13 @@ def get_autodetected_MUTS(mbeds_list, platform_name_filter=None): # mbeds_list = [{'platform_name': 'NUCLEO_F302R8', 'mount_point': 'E:', 'target_id': '07050200623B61125D5EF72A', 'serial_port': u'COM34'}] index = 1 for mut in mbeds_list: - m = {'mcu' : mut['platform_name'], - 'port' : mut['serial_port'], - 'disk' : mut['mount_point'], - 'peripherals' : [] # No peripheral detection + # For mcu_unique - we are assigning 'platform_name_unique' value from mbedls output (if its existing) + # if not we are creating our own unique value (last few chars from platform's target_id). + m = {'mcu': mut['platform_name'], + 'mcu_unique' : mut['platform_name_unique'] if 'platform_name_unique' in mut else "%s[%s]" % (mut['platform_name'], mut['target_id'][-4:]), + 'port': mut['serial_port'], + 'disk': mut['mount_point'], + 'peripherals': [] # No peripheral detection } if index not in result: result[index] = {} diff --git a/workspace_tools/test_exporters.py b/workspace_tools/test_exporters.py index c4847b0e75..4d61e0981b 100644 --- a/workspace_tools/test_exporters.py +++ b/workspace_tools/test_exporters.py @@ -21,7 +21,8 @@ from workspace_tools.utils import construct_enum ResultExporterType = construct_enum(HTML='Html_Exporter', - JUNIT='JUnit_Exporter') + JUNIT='JUnit_Exporter', + BUILD='Build_Exporter') class ReportExporter(): @@ -91,10 +92,10 @@ class ReportExporter(): """ Generate simple unique tool-tip name which can be used. For example as HTML
section id attribute. """ - return "target_test_%s_%s_%s_%d"% (toolchain.lower(), target.lower(), test_id.lower(), loop_no) + return "target_test_%s_%s_%s_%s"% (toolchain.lower(), target.lower(), test_id.lower(), loop_no) def get_result_div_sections(self, test, test_no): - """ Generates separate sections which contains test results output. + """ Generates separate
sections which contains test results output. """ RESULT_COLORS = {'OK' : 'LimeGreen', @@ -102,12 +103,14 @@ class ReportExporter(): 'ERROR' : 'LightCoral',} tooltip_name = self.get_tooltip_name(test['toolchain_name'], test['target_name'], test['test_id'], test_no) - background_color = RESULT_COLORS[test['single_test_result'] if test['single_test_result'] in RESULT_COLORS else 'ERROR'] + background_color = RESULT_COLORS[test['single_test_result'] if test['single_test_result'] in RESULT_COLORS else 'LightGray'] result_div_style = "background-color: %s"% background_color result = """
%s
+ %s
+
%s in %.2f sec
@@ -120,6 +123,7 @@ class ReportExporter(): tooltip_name, test['single_test_result'], tooltip_name, + test['target_name_unique'], test['test_description'], test['elapsed_time'], test['single_test_output'].replace('\n', '
')) @@ -130,14 +134,16 @@ class ReportExporter(): we will show it in a column to see all results. This function produces HTML table with corresponding results. """ - result = '' - test_ids = sorted(test_results.keys()) - for test_no in test_ids: - test = test_results[test_no] - result += """ - - """% self.get_result_div_sections(test, test_no) - result += '
%s
' + result = '' + for i, test_result in enumerate(test_results): + result += '' + test_ids = sorted(test_result.keys()) + for test_no in test_ids: + test = test_result[test_no] + result += """ + + """% self.get_result_div_sections(test, "%d_%d" % (test_no, i)) + result += '
%s
' return result def get_all_unique_test_ids(self, test_result_ext): @@ -158,7 +164,7 @@ class ReportExporter(): # def exporter_html(self, test_result_ext, test_suite_properties=None): - """ Export test results in proprietary html format. + """ Export test results in proprietary HTML format. """ result = """ @@ -211,25 +217,26 @@ class ReportExporter(): tests = sorted(test_result_ext[toolchain][target].keys()) for test in tests: test_results = test_result_ext[toolchain][target][test] - test_ids = sorted(test_results.keys()) - for test_no in test_ids: - test_result = test_results[test_no] - name = test_result['test_description'] - classname = 'test.%s.%s.%s'% (target, toolchain, test_result['test_id']) - elapsed_sec = test_result['elapsed_time'] - _stdout = test_result['single_test_output'] - _stderr = '' - # Test case - tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr) - # Test case extra failure / error info - if test_result['single_test_result'] == 'FAIL': - message = test_result['single_test_result'] - tc.add_failure_info(message, _stdout) - elif test_result['single_test_result'] != 'OK': - message = test_result['single_test_result'] - tc.add_error_info(message, _stdout) + for test_res in test_results: + test_ids = sorted(test_res.keys()) + for test_no in test_ids: + test_result = test_res[test_no] + name = test_result['test_description'] + classname = 'test.%s.%s.%s'% (target, toolchain, test_result['test_id']) + elapsed_sec = test_result['elapsed_time'] + _stdout = test_result['single_test_output'] + _stderr = test_result['target_name_unique'] + # Test case + tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr) + # Test case extra failure / error info + if test_result['single_test_result'] == 'FAIL': + message = test_result['single_test_result'] + tc.add_failure_info(message, _stdout) + elif test_result['single_test_result'] != 'OK': + message = test_result['single_test_result'] + tc.add_error_info(message, _stdout) - test_cases.append(tc) + test_cases.append(tc) ts = TestSuite("test.suite.%s.%s"% (target, toolchain), test_cases, properties=test_suite_properties[target][toolchain]) test_suites.append(ts) return TestSuite.to_xml_string(test_suites) From f410b86b054f85bfa09d812ac3e719e8654b025b Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Sun, 3 May 2015 02:34:31 +0100 Subject: [PATCH 2/4] Updated app version to 1.5 - Biggest change: added support for multiple devices of the same type. --- workspace_tools/singletest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workspace_tools/singletest.py b/workspace_tools/singletest.py index a2579dd93c..0f7e3a23ee 100644 --- a/workspace_tools/singletest.py +++ b/workspace_tools/singletest.py @@ -85,7 +85,7 @@ def get_version(): """ Returns test script version """ single_test_version_major = 1 - single_test_version_minor = 4 + single_test_version_minor = 5 return (single_test_version_major, single_test_version_minor) From abee10eee8725764f81a1a062f5b07fa10a1ec90 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Wed, 6 May 2015 13:42:38 +0100 Subject: [PATCH 3/4] Fixed KeyError in access to background-color map --- workspace_tools/test_exporters.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/workspace_tools/test_exporters.py b/workspace_tools/test_exporters.py index 4d61e0981b..a1a74dd32a 100644 --- a/workspace_tools/test_exporters.py +++ b/workspace_tools/test_exporters.py @@ -98,12 +98,14 @@ class ReportExporter(): """ Generates separate
sections which contains test results output. """ - RESULT_COLORS = {'OK' : 'LimeGreen', - 'FAIL' : 'Orange', - 'ERROR' : 'LightCoral',} + RESULT_COLORS = {'OK': 'LimeGreen', + 'FAIL': 'Orange', + 'ERROR': 'LightCoral', + 'OTHER': 'LightGray', + } tooltip_name = self.get_tooltip_name(test['toolchain_name'], test['target_name'], test['test_id'], test_no) - background_color = RESULT_COLORS[test['single_test_result'] if test['single_test_result'] in RESULT_COLORS else 'LightGray'] + background_color = RESULT_COLORS[test['single_test_result'] if test['single_test_result'] in RESULT_COLORS else 'OTHER'] result_div_style = "background-color: %s"% background_color result = """
From 65bae00f6162bd4da33f75c8ef16a4c3711cff74 Mon Sep 17 00:00:00 2001 From: Przemek Wirkus Date: Wed, 6 May 2015 16:31:55 +0100 Subject: [PATCH 4/4] Bugfix: copy plugin assumed always that all commands like cp, copy, xcopy are shell command. In fact copy and xcopy are shell commands under Windows and 'cp' is always a separate app. not shell command e.g. in Windows. --- .../host_tests/host_tests_plugins/host_test_plugins.py | 1 + .../host_tests/host_tests_plugins/module_copy_shell.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/workspace_tools/host_tests/host_tests_plugins/host_test_plugins.py b/workspace_tools/host_tests/host_tests_plugins/host_test_plugins.py index 8bc1da35d3..ee60950dc6 100644 --- a/workspace_tools/host_tests/host_tests_plugins/host_test_plugins.py +++ b/workspace_tools/host_tests/host_tests_plugins/host_test_plugins.py @@ -106,6 +106,7 @@ class HostTestPluginBase: """ Runs command from command line. """ result = True + ret = 0 try: ret = call(cmd, shell=shell) if ret: diff --git a/workspace_tools/host_tests/host_tests_plugins/module_copy_shell.py b/workspace_tools/host_tests/host_tests_plugins/module_copy_shell.py index f7fb23b0a7..8e5e4153e0 100644 --- a/workspace_tools/host_tests/host_tests_plugins/module_copy_shell.py +++ b/workspace_tools/host_tests/host_tests_plugins/module_copy_shell.py @@ -51,10 +51,11 @@ class HostTestPluginCopyMethod_Shell(HostTestPluginBase): if capabilitity == 'shell': if os.name == 'nt': capabilitity = 'copy' elif os.name == 'posix': capabilitity = 'cp' - if capabilitity == 'cp' or capabilitity == 'copy' or capabilitity == 'copy': + if capabilitity == 'cp' or capabilitity == 'copy' or capabilitity == 'xcopy': copy_method = capabilitity cmd = [copy_method, image_path, destination_path] - result = self.run_command(cmd) + shell = not capabilitity == 'cp' + result = self.run_command(cmd, shell=shell) return result