ARMmbed
/
mbed-os
mirror of https://github.com/ARMmbed/mbed-os.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Added JUNIT exporters to compliance testing functionality

pull/1139/head
Przemek Wirkus 2015-05-27 09:56:35 +01:00
parent a82ffbe904
commit 7b0b9e8f32
5 changed files with 68 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
workspace_tools/compliance/ioper_runner.py
View File

@ -56,7 +56,7 @@ class IOperTestRunner():
'basic' - just simple, passive tests (no device flashing) 'basic' - just simple, passive tests (no device flashing)
""" """
self.requested_scope = scope # Test scope given by user self.requested_scope = scope # Test scope given by user
self.raw_test_results = [] # Raw test results, can be used by exporters self.raw_test_results = {} # Raw test results, can be used by exporters: { Platform: [test results]}
# Test scope definitions # Test scope definitions
self.SCOPE_BASIC = 'basic' # Basic tests, sanity checks self.SCOPE_BASIC = 'basic' # Basic tests, sanity checks
@ -84,6 +84,8 @@ class IOperTestRunner():
for i, mut in enumerate(muts_list): for i, mut in enumerate(muts_list):
result = [] result = []
self.raw_test_results[mut['platform_name']] = []
print "MBEDLS: Detected %s, port: %s, mounted: %s"% (mut['platform_name'], print "MBEDLS: Detected %s, port: %s, mounted: %s"% (mut['platform_name'],
mut['serial_port'], mut['serial_port'],
mut['mount_point']) mut['mount_point'])
@ -92,24 +94,25 @@ class IOperTestRunner():
if self.scopes[self.requested_scope] >= self.scopes[test_case.scope]: if self.scopes[self.requested_scope] >= self.scopes[test_case.scope]:
res = test_case.test(param=mut) res = test_case.test(param=mut)
result.extend(res) result.extend(res)
self.raw_test_results.append(res) self.raw_test_results[mut['platform_name']].extend(res)
columns = ['Platform', 'Result', 'Scope', 'Description'] columns = ['Platform', 'Test Case', 'Result', 'Scope', 'Description']
pt = PrettyTable(columns) pt = PrettyTable(columns)
for col in columns: for col in columns:
pt.align[col] = 'l' pt.align[col] = 'l'
for tr in result: for tr in result:
severity, tr_scope, text = tr severity, tr_name, tr_scope, text = tr
tr = (test_base.COLOR(severity, mut['platform_name']), tr = (test_base.COLOR(severity, mut['platform_name']),
test_base.COLOR(severity, tr_name),
test_base.COLOR(severity, severity), test_base.COLOR(severity, severity),
test_base.COLOR(severity, tr_scope), test_base.COLOR(severity, tr_scope),
test_base.COLOR(severity, text)) test_base.COLOR(severity, text))
pt.add_row(list(tr)) pt.add_row(list(tr))
print pt.get_string(border=True, sortby='Result') print pt.get_string(border=True, sortby='Result')
if i+1 < len(muts_list): if i + 1 < len(muts_list):
print print
return self.raw_test_results
def get_available_oper_test_scopes(): def get_available_oper_test_scopes():
""" Get list of available test scopes """ Get list of available test scopes

5
workspace_tools/compliance/ioper_test_fs.py
View File

@ -30,10 +30,11 @@ class IOperTest_FileStructure(IOperTestCaseBase):
def if_file_exist(self, fname, fail_severity=None): def if_file_exist(self, fname, fail_severity=None):
file_path = os.path.join(self.param['mount_point'], fname) file_path = os.path.join(self.param['mount_point'], fname)
exist = os.path.isfile(file_path) exist = os.path.isfile(file_path)
tr_name = "FILE_EXIST(%s)" % fname.upper()
if exist: if exist:
self.result.append((self.PASS, self.scope, "File '%s' exists" % file_path)) self.result.append((self.PASS, tr_name, self.scope, "File '%s' exists" % file_path))
else: else:
self.result.append((fail_severity if fail_severity else self.ERROR, self.scope, "File '%s' not found" % file_path)) self.result.append((fail_severity if fail_severity else self.ERROR, tr_name, self.scope, "File '%s' not found" % file_path))
def test(self, param=None): def test(self, param=None):
self.result = [] self.result = []

26
workspace_tools/compliance/ioper_test_target_id.py
View File

@ -34,24 +34,24 @@ class IOperTest_TargetID(IOperTestCaseBase):
result = [] result = []
target_id_len = len(target_id) if target_id else 0 target_id_len = len(target_id) if target_id else 0
if target_id_len == self.TARGET_ID_LEN: if target_id_len == self.TARGET_ID_LEN:
result.append((self.PASS, self.scope, "%s '%s' is %d chars long " % (target_id_name, target_id, target_id_len))) result.append((self.PASS, "TARGET_ID_LEN", self.scope, "%s '%s' is %d chars long " % (target_id_name, target_id, target_id_len)))
result.append((self.INFO, self.scope, "%s Version String is %s.%s.%s " % (target_id_name, result.append((self.INFO, "FW_VER_STR", self.scope, "%s Version String is %s.%s.%s " % (target_id_name,
target_id[0:4], target_id[0:4],
target_id[4:8], target_id[4:8],
target_id[8:24], target_id[8:24],
))) )))
else: else:
result.append((self.ERROR, self.scope, "%s '%s' is %d chars long. Expected %d chars" % (target_id_name, target_id, target_id_len, self.TARGET_ID_LEN))) result.append((self.ERROR, "TARGET_ID_LEN", self.scope, "%s '%s' is %d chars long. Expected %d chars" % (target_id_name, target_id, target_id_len, self.TARGET_ID_LEN)))
return result return result
def test_decode_target_id(self, target_id, target_id_name): def test_decode_target_id(self, target_id, target_id_name):
result = [] result = []
target_id_len = len(target_id) if target_id else 0 target_id_len = len(target_id) if target_id else 0
if target_id_len >= 4: if target_id_len >= 4:
result.append((self.INFO, self.scope, "%s Vendor Code is '%s'" % (target_id_name, target_id[0:2]))) result.append((self.INFO, "FW_VEN_CODE", self.scope, "%s Vendor Code is '%s'" % (target_id_name, target_id[0:2])))
result.append((self.INFO, self.scope, "%s Platform Code is '%s'" % (target_id_name, target_id[2:4]))) result.append((self.INFO, "FW_PLAT_CODE", self.scope, "%s Platform Code is '%s'" % (target_id_name, target_id[2:4])))
result.append((self.INFO, self.scope, "%s Firmware Version is '%s'" % (target_id_name, target_id[4:8]))) result.append((self.INFO, "FW_VER", self.scope, "%s Firmware Version is '%s'" % (target_id_name, target_id[4:8])))
result.append((self.INFO, self.scope, "%s Hash of secret is '%s'" % (target_id_name, target_id[8:24]))) result.append((self.INFO, "FW_HASH_SEC", self.scope, "%s Hash of secret is '%s'" % (target_id_name, target_id[8:24])))
return result return result
def test(self, param=None): def test(self, param=None):
@ -72,22 +72,22 @@ class IOperTest_TargetID_Basic(IOperTest_TargetID):
result = [] result = []
if param: if param:
result.append((self.PASS, self.scope, "TargetID '%s' found" % param['target_id'])) result.append((self.PASS, "TARGET_ID", self.scope, "TargetID '%s' found" % param['target_id']))
# Check if target name can be decoded with mbed-ls # Check if target name can be decoded with mbed-ls
if param['platform_name']: if param['platform_name']:
result.append((self.PASS, self.scope, "TargetID '%s' decoded as '%s'" % (param['target_id'][0:4], param['platform_name']))) result.append((self.PASS, "TARGET_ID_DECODE", self.scope, "TargetID '%s' decoded as '%s'" % (param['target_id'][0:4], param['platform_name'])))
else: else:
result.append((self.ERROR, self.scope, "TargetID '%s'... not decoded" % (param['target_id'] if param['target_id'] else ''))) result.append((self.ERROR, "TARGET_ID_DECODE", self.scope, "TargetID '%s'... not decoded" % (param['target_id'] if param['target_id'] else '')))
# Test for USBID and mbed.htm consistency # Test for USBID and mbed.htm consistency
if param['target_id_mbed_htm'] == param['target_id_usb_id']: if param['target_id_mbed_htm'] == param['target_id_usb_id']:
result.append((self.PASS, self.scope, "TargetID (USBID) and TargetID (mbed.htm) match")) result.append((self.PASS, "TARGET_ID_MATCH", self.scope, "TargetID (USBID) and TargetID (mbed.htm) match"))
else: else:
text = "TargetID (USBID) and TargetID (mbed.htm) don't match: '%s' != '%s'" % (param['target_id_usb_id'], param['target_id_mbed_htm']) text = "TargetID (USBID) and TargetID (mbed.htm) don't match: '%s' != '%s'" % (param['target_id_usb_id'], param['target_id_mbed_htm'])
result.append((self.WARN, self.scope, text)) result.append((self.WARN, "TARGET_ID_MATCH", self.scope, text))
else: else:
result.append((self.ERROR, self.scope, "TargetID not found")) result.append((self.ERROR, "TARGET_ID", self.scope, "TargetID not found"))
return result return result
class IOperTest_TargetID_MbedEnabled(IOperTest_TargetID): class IOperTest_TargetID_MbedEnabled(IOperTest_TargetID):

6
workspace_tools/singletest.py
View File

@ -77,6 +77,7 @@ from workspace_tools.test_api import get_module_avail
from workspace_tools.compliance.ioper_runner import IOperTestRunner from workspace_tools.compliance.ioper_runner import IOperTestRunner
from workspace_tools.compliance.ioper_runner import get_available_oper_test_scopes from workspace_tools.compliance.ioper_runner import get_available_oper_test_scopes
from workspace_tools.test_exporters import ReportExporter, ResultExporterType
# Importing extra modules which can be not installed but if available they can extend test suite functionality # Importing extra modules which can be not installed but if available they can extend test suite functionality
@ -201,6 +202,11 @@ if __name__ == '__main__':
if opts.operability_checks in test_scope: if opts.operability_checks in test_scope:
tests = IOperTestRunner(scope=opts.operability_checks) tests = IOperTestRunner(scope=opts.operability_checks)
test_results = tests.run() test_results = tests.run()
# Export results in form of JUnit XML report to separate file
if opts.report_junit_file_name:
report_exporter = ReportExporter(ResultExporterType.JUNIT_OPER)
report_exporter.report_to_file(test_results, opts.report_junit_file_name)
else: else:
print "Unknown interoperability test scope name: '%s'" % (opts.operability_checks) print "Unknown interoperability test scope name: '%s'" % (opts.operability_checks)
print "Available test scopes: %s" % (','.join(["'%s'" % n for n in test_scope])) print "Available test scopes: %s" % (','.join(["'%s'" % n for n in test_scope]))

39
workspace_tools/test_exporters.py
View File

@ -22,7 +22,8 @@ from workspace_tools.utils import construct_enum
ResultExporterType = construct_enum(HTML='Html_Exporter', ResultExporterType = construct_enum(HTML='Html_Exporter',
JUNIT='JUnit_Exporter', JUNIT='JUnit_Exporter',
BUILD='Build_Exporter') BUILD='Build_Exporter',
JUNIT_OPER='JUnit_Exporter_Interoperability')
class ReportExporter(): class ReportExporter():
@ -76,14 +77,20 @@ class ReportExporter():
# HTML exporter # HTML exporter
return self.exporter_html(test_summary_ext, test_suite_properties) return self.exporter_html(test_summary_ext, test_suite_properties)
elif self.result_exporter_type == ResultExporterType.JUNIT: elif self.result_exporter_type == ResultExporterType.JUNIT:
# JUNIT exporter # JUNIT exporter for results from test suite
return self.exporter_junit(test_summary_ext, test_suite_properties) return self.exporter_junit(test_summary_ext, test_suite_properties)
elif self.result_exporter_type == ResultExporterType.JUNIT_OPER:
# JUNIT exporter for interoperability test
return self.exporter_junit_ioper(test_summary_ext, test_suite_properties)
return None return None
def report_to_file(self, test_summary_ext, file_name, test_suite_properties=None): def report_to_file(self, test_summary_ext, file_name, test_suite_properties=None):
""" Stores report to specified file """ Stores report to specified file
""" """
report = self.report(test_summary_ext, test_suite_properties=test_suite_properties) report = self.report(test_summary_ext, test_suite_properties=test_suite_properties)
self.write_to_file(report, file_name)
def write_to_file(self, report, file_name):
if report is not None: if report is not None:
with open(file_name, 'w') as f: with open(file_name, 'w') as f:
f.write(report) f.write(report)
@ -204,6 +211,34 @@ class ReportExporter():
result += '</body></html>' result += '</body></html>'
return result return result
def exporter_junit_ioper(self, test_result_ext, test_suite_properties=None):
from junit_xml import TestSuite, TestCase
test_suites = []
test_cases = []
for platform in sorted(test_result_ext.keys()):
# {platform : ['Platform', 'Result', 'Scope', 'Description'])
for tr_result in test_result_ext[platform]:
result, name, scope, description = tr_result
classname = 'test.ioper.%s.%s.%s' % (platform, name, scope)
elapsed_sec = 0
_stdout = description
_stderr = ''
# Test case
tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr)
# Test case extra failure / error info
if result == 'FAIL':
tc.add_failure_info(description, _stdout)
elif result == 'ERROR':
tc.add_error_info(description, _stdout)
test_cases.append(tc)
ts = TestSuite("test.suite.ioper.%s" % (platform), test_cases)
test_suites.append(ts)
return TestSuite.to_xml_string(test_suites)
def exporter_junit(self, test_result_ext, test_suite_properties=None): def exporter_junit(self, test_result_ext, test_suite_properties=None):
""" Export test results in JUnit XML compliant format """ Export test results in JUnit XML compliant format
""" """