mirror of https://github.com/ARMmbed/mbed-os.git
351 lines
15 KiB
Python
351 lines
15 KiB
Python
"""
|
|
mbed SDK
|
|
Copyright (c) 2011-2014 ARM Limited
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
|
|
Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
|
|
"""
|
|
|
|
from tools.utils import construct_enum, mkdir
|
|
import os
|
|
|
|
ResultExporterType = construct_enum(HTML='Html_Exporter',
|
|
JUNIT='JUnit_Exporter',
|
|
JUNIT_OPER='JUnit_Exporter_Interoperability',
|
|
BUILD='Build_Exporter',
|
|
PRINT='Print_Exporter')
|
|
|
|
|
|
class ReportExporter():
|
|
""" Class exports extended test result Python data structure to
|
|
different formats like HTML, JUnit XML.
|
|
|
|
Parameter 'test_result_ext' format:
|
|
|
|
u'uARM': { u'LPC1768': { 'MBED_2': { 0: { 'copy_method': 'shutils.copy()',
|
|
'duration': 20,
|
|
'elapsed_time': 1.7929999828338623,
|
|
'output': 'Host test instrumentation on ...\r\n',
|
|
'result': 'OK',
|
|
'target_name': u'LPC1768',
|
|
'description': 'stdio',
|
|
'id': u'MBED_2',
|
|
'toolchain_name': u'uARM'}},
|
|
"""
|
|
CSS_STYLE = """<style>
|
|
.name{
|
|
border: 1px solid;
|
|
border-radius: 25px;
|
|
width: 100px;
|
|
}
|
|
.tooltip{
|
|
position:absolute;
|
|
background-color: #F5DA81;
|
|
display:none;
|
|
}
|
|
</style>
|
|
"""
|
|
|
|
JAVASCRIPT = """
|
|
<script type="text/javascript">
|
|
function show (elem) {
|
|
elem.style.display = "block";
|
|
}
|
|
function hide (elem) {
|
|
elem.style.display = "";
|
|
}
|
|
</script>
|
|
"""
|
|
|
|
def __init__(self, result_exporter_type, package="test"):
|
|
self.result_exporter_type = result_exporter_type
|
|
self.package = package
|
|
|
|
def report(self, test_summary_ext, test_suite_properties=None,
|
|
print_log_for_failures=True):
|
|
""" Invokes report depending on exporter_type set in constructor
|
|
"""
|
|
if self.result_exporter_type == ResultExporterType.HTML:
|
|
# HTML exporter
|
|
return self.exporter_html(test_summary_ext, test_suite_properties)
|
|
elif self.result_exporter_type == ResultExporterType.JUNIT:
|
|
# JUNIT exporter for results from test suite
|
|
return self.exporter_junit(test_summary_ext, test_suite_properties)
|
|
elif self.result_exporter_type == ResultExporterType.JUNIT_OPER:
|
|
# JUNIT exporter for interoperability test
|
|
return self.exporter_junit_ioper(test_summary_ext, test_suite_properties)
|
|
elif self.result_exporter_type == ResultExporterType.PRINT:
|
|
# JUNIT exporter for interoperability test
|
|
return self.exporter_print(test_summary_ext, print_log_for_failures=print_log_for_failures)
|
|
return None
|
|
|
|
def report_to_file(self, test_summary_ext, file_name, test_suite_properties=None):
|
|
""" Stores report to specified file
|
|
"""
|
|
report = self.report(test_summary_ext, test_suite_properties=test_suite_properties)
|
|
self.write_to_file(report, file_name)
|
|
|
|
def write_to_file(self, report, file_name):
|
|
if report is not None:
|
|
dirname = os.path.dirname(file_name)
|
|
if dirname:
|
|
mkdir(dirname)
|
|
with open(file_name, 'w') as f:
|
|
f.write(report)
|
|
|
|
def get_tooltip_name(self, toolchain, target, test_id, loop_no):
|
|
""" Generate simple unique tool-tip name which can be used.
|
|
For example as HTML <div> section id attribute.
|
|
"""
|
|
return "target_test_%s_%s_%s_%s"% (toolchain.lower(), target.lower(), test_id.lower(), loop_no)
|
|
|
|
def get_result_div_sections(self, test, test_no):
|
|
""" Generates separate <DIV> sections which contains test results output.
|
|
"""
|
|
|
|
RESULT_COLORS = {'OK': 'LimeGreen',
|
|
'FAIL': 'Orange',
|
|
'ERROR': 'LightCoral',
|
|
'OTHER': 'LightGray',
|
|
}
|
|
|
|
tooltip_name = self.get_tooltip_name(test['toolchain_name'], test['target_name'], test['id'], test_no)
|
|
background_color = RESULT_COLORS[test['result'] if test['result'] in RESULT_COLORS else 'OTHER']
|
|
result_div_style = "background-color: %s"% background_color
|
|
|
|
result = """<div class="name" style="%s" onmouseover="show(%s)" onmouseout="hide(%s)">
|
|
<center>%s</center>
|
|
<div class = "tooltip" id= "%s">
|
|
<b>%s</b><br />
|
|
<hr />
|
|
<b>%s</b> in <b>%.2f sec</b><br />
|
|
<hr />
|
|
<small>
|
|
%s
|
|
</small>
|
|
</div>
|
|
</div>
|
|
"""% (result_div_style,
|
|
tooltip_name,
|
|
tooltip_name,
|
|
test['result'],
|
|
tooltip_name,
|
|
test['target_name_unique'],
|
|
test['description'],
|
|
test['elapsed_time'],
|
|
test['output'].replace('\n', '<br />'))
|
|
return result
|
|
|
|
def get_result_tree(self, test_results):
|
|
""" If test was run in a loop (we got few results from the same test)
|
|
we will show it in a column to see all results.
|
|
This function produces HTML table with corresponding results.
|
|
"""
|
|
result = ''
|
|
for i, test_result in enumerate(test_results):
|
|
result += '<table>'
|
|
test_ids = sorted(test_result.keys())
|
|
for test_no in test_ids:
|
|
test = test_result[test_no]
|
|
result += """<tr>
|
|
<td valign="top">%s</td>
|
|
</tr>"""% self.get_result_div_sections(test, "%d_%d" % (test_no, i))
|
|
result += '</table>'
|
|
return result
|
|
|
|
def get_all_unique_test_ids(self, test_result_ext):
|
|
""" Gets all unique test ids from all ran tests.
|
|
We need this to create complete list of all test ran.
|
|
"""
|
|
result = []
|
|
targets = test_result_ext.keys()
|
|
for target in targets:
|
|
toolchains = test_result_ext[target].keys()
|
|
for toolchain in toolchains:
|
|
tests = test_result_ext[target][toolchain].keys()
|
|
result.extend(tests)
|
|
return sorted(list(set(result)))
|
|
|
|
#
|
|
# Exporters functions
|
|
#
|
|
|
|
def exporter_html(self, test_result_ext, test_suite_properties=None):
|
|
""" Export test results in proprietary HTML format.
|
|
"""
|
|
result = """<html>
|
|
<head>
|
|
<title>mbed SDK test suite test result report</title>
|
|
%s
|
|
%s
|
|
</head>
|
|
<body>
|
|
"""% (self.CSS_STYLE, self.JAVASCRIPT)
|
|
|
|
unique_test_ids = self.get_all_unique_test_ids(test_result_ext)
|
|
targets = sorted(test_result_ext.keys())
|
|
result += '<table><tr>'
|
|
for target in targets:
|
|
toolchains = sorted(test_result_ext[target].keys())
|
|
for toolchain in toolchains:
|
|
result += '<td></td>'
|
|
result += '<td></td>'
|
|
|
|
tests = sorted(test_result_ext[target][toolchain].keys())
|
|
for test in unique_test_ids:
|
|
result += """<td align="center">%s</td>"""% test
|
|
result += """</tr>
|
|
<tr>
|
|
<td valign="center">%s</td>
|
|
<td valign="center"><b>%s</b></td>
|
|
"""% (toolchain, target)
|
|
|
|
for test in unique_test_ids:
|
|
test_result = self.get_result_tree(test_result_ext[target][toolchain][test]) if test in tests else ''
|
|
result += '<td>%s</td>'% (test_result)
|
|
|
|
result += '</tr>'
|
|
result += '</table>'
|
|
result += '</body></html>'
|
|
return result
|
|
|
|
def exporter_junit_ioper(self, test_result_ext, test_suite_properties=None):
|
|
from junit_xml import TestSuite, TestCase
|
|
test_suites = []
|
|
test_cases = []
|
|
|
|
for platform in sorted(test_result_ext.keys()):
|
|
# {platform : ['Platform', 'Result', 'Scope', 'Description'])
|
|
test_cases = []
|
|
for tr_result in test_result_ext[platform]:
|
|
result, name, scope, description = tr_result
|
|
|
|
classname = 'test.ioper.%s.%s.%s' % (platform, name, scope)
|
|
elapsed_sec = 0
|
|
_stdout = description
|
|
_stderr = ''
|
|
# Test case
|
|
tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr)
|
|
# Test case extra failure / error info
|
|
if result == 'FAIL':
|
|
tc.add_failure_info(description, _stdout)
|
|
elif result == 'ERROR':
|
|
tc.add_error_info(description, _stdout)
|
|
elif result == 'SKIP' or result == 'NOT_SUPPORTED':
|
|
tc.add_skipped_info(description, _stdout)
|
|
|
|
test_cases.append(tc)
|
|
ts = TestSuite("test.suite.ioper.%s" % (platform), test_cases)
|
|
test_suites.append(ts)
|
|
return TestSuite.to_xml_string(test_suites)
|
|
|
|
def exporter_junit(self, test_result_ext, test_suite_properties=None):
|
|
""" Export test results in JUnit XML compliant format
|
|
"""
|
|
from junit_xml import TestSuite, TestCase
|
|
test_suites = []
|
|
test_cases = []
|
|
|
|
targets = sorted(test_result_ext.keys())
|
|
for target in targets:
|
|
toolchains = sorted(test_result_ext[target].keys())
|
|
for toolchain in toolchains:
|
|
test_cases = []
|
|
tests = sorted(test_result_ext[target][toolchain].keys())
|
|
for test in tests:
|
|
test_results = test_result_ext[target][toolchain][test]
|
|
for test_res in test_results:
|
|
test_ids = sorted(test_res.keys())
|
|
for test_no in test_ids:
|
|
test_result = test_res[test_no]
|
|
name = test_result['description']
|
|
classname = '%s.%s.%s.%s'% (self.package, target, toolchain, test_result['id'])
|
|
elapsed_sec = test_result['elapsed_time']
|
|
_stdout = test_result['output']
|
|
|
|
if 'target_name_unique' in test_result:
|
|
_stderr = test_result['target_name_unique']
|
|
else:
|
|
_stderr = test_result['target_name']
|
|
|
|
# Test case
|
|
tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr)
|
|
|
|
# Test case extra failure / error info
|
|
message = test_result['result']
|
|
if test_result['result'] == 'FAIL':
|
|
tc.add_failure_info(message, _stdout)
|
|
elif test_result['result'] == 'SKIP' or test_result["result"] == 'NOT_SUPPORTED':
|
|
tc.add_skipped_info(message, _stdout)
|
|
elif test_result['result'] != 'OK':
|
|
tc.add_error_info(message, _stdout)
|
|
|
|
test_cases.append(tc)
|
|
|
|
ts = TestSuite("test.suite.%s.%s"% (target, toolchain), test_cases, properties=test_suite_properties[target][toolchain])
|
|
test_suites.append(ts)
|
|
return TestSuite.to_xml_string(test_suites)
|
|
|
|
def exporter_print_helper(self, array, print_log=False):
|
|
for item in array:
|
|
print " * %s::%s::%s" % (item["target_name"], item["toolchain_name"], item["id"])
|
|
if print_log:
|
|
log_lines = item["output"].split("\n")
|
|
for log_line in log_lines:
|
|
print " %s" % log_line
|
|
|
|
def exporter_print(self, test_result_ext, print_log_for_failures=False):
|
|
""" Export test results in print format.
|
|
"""
|
|
failures = []
|
|
skips = []
|
|
successes = []
|
|
|
|
unique_test_ids = self.get_all_unique_test_ids(test_result_ext)
|
|
targets = sorted(test_result_ext.keys())
|
|
|
|
for target in targets:
|
|
toolchains = sorted(test_result_ext[target].keys())
|
|
for toolchain in toolchains:
|
|
tests = sorted(test_result_ext[target][toolchain].keys())
|
|
for test in tests:
|
|
test_runs = test_result_ext[target][toolchain][test]
|
|
for test_runner in test_runs:
|
|
#test_run = test_result_ext[target][toolchain][test][test_run_number][0]
|
|
test_run = test_runner[0]
|
|
|
|
if test_run["result"] == "FAIL":
|
|
failures.append(test_run)
|
|
elif test_run["result"] == "SKIP" or test_run["result"] == "NOT_SUPPORTED":
|
|
skips.append(test_run)
|
|
elif test_run["result"] == "OK":
|
|
successes.append(test_run)
|
|
else:
|
|
raise Exception("Unhandled result type: %s" % (test_run["result"]))
|
|
|
|
if successes:
|
|
print "\n\nBuild successes:"
|
|
self.exporter_print_helper(successes)
|
|
|
|
if skips:
|
|
print "\n\nBuild skips:"
|
|
self.exporter_print_helper(skips)
|
|
|
|
if failures:
|
|
print "\n\nBuild failures:"
|
|
self.exporter_print_helper(failures, print_log=print_log_for_failures)
|
|
return False
|
|
else:
|
|
return True
|