ARMmbed
/
mbed-os
mirror of https://github.com/ARMmbed/mbed-os.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Refactored interoperability tests, added separate classes of tests and CLI

pull/1139/head^2
Przemek Wirkus 2015-05-26 15:59:07 +01:00
parent 51be9a2346
commit 7b589e3d71
7 changed files with 372 additions and 208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

200
workspace_tools/compliance/ifchip_test.py
View File

@ -1,200 +0,0 @@
#!/usr/bin/env python2
"""
mbed SDK
Copyright (c) 2011-2015 ARM Limited
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Author: Przemyslaw Wirkus <Przemyslaw.Wirkus@arm.com>
"""
import sys
import os.path
import mbed_lstools
from prettytable import PrettyTable
try:
from colorama import init
from colorama import Fore, Back, Style
except:
pass
COLORAMA = 'colorama' in sys.modules
class IOperTestCaseBase():
""" Interoperability test case base class
@return list of tuple (severity, Description)
Example: (result.append((IOperTestSeverity.INFO, ""))
"""
def __init__(self):
self.PASS = 'PASS'
self.INFO = 'INFO'
self.ERROR = 'ERROR'
self.WARN = 'WARN'
def test(self, param=None):
result = []
return result
def RED(self, text):
return self.color_text(text, color=Fore.RED, delim=Fore.RESET)
def GREEN(self, text):
return self.color_text(text, color=Fore.GREEN, delim=Fore.RESET)
def YELLOW(self, text):
return self.color_text(text, color=Fore.YELLOW, delim=Fore.RESET)
def color_text(self, text, color='', delim=''):
return color + text + color + delim
def COLOR(self, severity, text):
colors = {
self.PASS : self.GREEN,
self.ERROR : self.RED,
self.WARN : self.YELLOW
}
if severity in colors:
return colors[severity](text)
return text
class IOperTest_FileStructure(IOperTestCaseBase):
def if_file_exist(self, fname, fail_severity=None):
file_path = os.path.join(self.param['mount_point'], fname)
exist = os.path.isfile(file_path)
if exist:
self.result.append((self.PASS, "File '%s' exists" % file_path))
else:
self.result.append((fail_severity if fail_severity else self.ERROR, "File '%s' not found" % file_path))
def test(self, param=None):
self.param = param
self.result = []
if param:
self.if_file_exist('mbed.htm', self.ERROR)
self.if_file_exist('DETAILS.TXT', self.INFO)
self.if_file_exist('FAIL.TXT', self.INFO)
return self.result
class IOperTest_TargetID(IOperTestCaseBase):
""" tests related to target_id value
"""
TARGET_ID_LEN = 24
def test_target_id_format(self, target_id, target_id_name):
# Expected length == 24, eg. "02400203D94B0E7724B7F3CF"
result = []
target_id_len = len(target_id) if target_id else 0
if target_id_len == self.TARGET_ID_LEN:
result.append((self.PASS, "%s '%s' is %d chars long " % (target_id_name, target_id, target_id_len)))
result.append((self.INFO, "%s Version String is %s.%s.%s " % (target_id_name,
target_id[0:4],
target_id[4:8],
target_id[8:24],
)))
else:
result.append((self.ERROR, "%s '%s' is %d chars long. Expected %d chars" % (target_id_name, target_id, target_id_len, self.TARGET_ID_LEN)))
return result
def test_decode_target_id(self, target_id, target_id_name):
result = []
target_id_len = len(target_id) if target_id else 0
if target_id_len >= 4:
result.append((self.INFO, "%s Vendor Code is '%s'" % (target_id_name, target_id[0:2])))
result.append((self.INFO, "%s Platform Code is '%s'" % (target_id_name, target_id[2:4])))
result.append((self.INFO, "%s Firmware Version is '%s'" % (target_id_name, target_id[4:8])))
result.append((self.INFO, "%s Hash of secret is '%s'" % (target_id_name, target_id[8:24])))
return result
def test(self, param=None):
result = []
if param:
result.append((self.PASS, "TargetID '%s' found" % param['target_id']))
# Check if target name can be decoded with mbed-ls
if param['platform_name']:
result.append((self.PASS, "TargetID '%s' decoded as '%s'" % (param['target_id'][0:4], param['platform_name'])))
else:
result.append((self.ERROR, "TargetID '%s'... not decoded" % (param['target_id'] if param['target_id'] else '')))
# Test for USBID and mbed.htm consistency
if param['target_id_mbed_htm'] == param['target_id_usb_id']:
result.append((self.PASS, "TargetID (USBID) and TargetID (mbed.htm) match"))
else:
text = "TargetID (USBID) and TargetID (mbed.htm) don't match: '%s' != '%s'" % (param['target_id_usb_id'], param['target_id_mbed_htm'])
result.append((self.WARN, text))
# Target ID tests:
result += self.test_target_id_format(param['target_id_usb_id'], "TargetId (USBID)")
result += self.test_target_id_format(param['target_id_mbed_htm'], "TargetId (mbed.htm)")
# Some extra info about TargetID itself
result += self.test_decode_target_id(param['target_id_usb_id'], "TargetId (USBID)")
result += self.test_decode_target_id(param['target_id_mbed_htm'], "TargetId (mbed.htm)")
else:
result.append((self.ERROR, "TargetID not found"))
return result
##########################################################################
class IOperTestRunenr():
""" Calls all i/face interoperability tests
"""
def run(self):
""" Run tests, calculate overall score and print test results
"""
mbeds = mbed_lstools.create()
muts_list = mbeds.list_mbeds()
test_base = IOperTestCaseBase()
for mut in muts_list:
result = []
print "MBEDLS: Detected %s, port: %s, mounted: %s"% (mut['platform_name'],
mut['serial_port'],
mut['mount_point'])
for test_case in TEST_LIST:
res = test_case.test(param=mut)
result.extend(res)
columns = ['Severity', 'Description']
pt = PrettyTable(columns)
for col in columns:
pt.align[col] = 'l'
for tr in result:
severity, text = tr
tr = (test_base.COLOR(severity, severity), test_base.COLOR(severity, text))
pt.add_row(list(tr))
print pt.get_string(border=False)
##########################################################################
TEST_LIST = [IOperTest_TargetID(),
IOperTest_FileStructure()
]
if __name__ == '__main__':
init() # colorama
tests = IOperTestRunenr()
test_results = tests.run()

69
workspace_tools/compliance/ioper_base.py Normal file
View File

@ -0,0 +1,69 @@
"""
mbed SDK
Copyright (c) 2011-2015 ARM Limited
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Author: Przemyslaw Wirkus <Przemyslaw.Wirkus@arm.com>
"""
import sys
try:
from colorama import Fore
except:
pass
COLORAMA = 'colorama' in sys.modules
class IOperTestCaseBase():
""" Interoperability test case base class
@return list of tuple (severity, Description)
Example: (result.append((IOperTestSeverity.INFO, ""))
"""
def __init__(self, scope=None):
self.PASS = 'PASS'
self.INFO = 'INFO'
self.ERROR = 'ERROR'
self.WARN = 'WARN'
self.scope = scope # Default test scope (basic, pedantic, mbed-enabled etc...)
def test(self, param=None):
result = []
return result
def RED(self, text):
return self.color_text(text, color=Fore.RED, delim=Fore.RESET) if COLORAMA else text
def GREEN(self, text):
return self.color_text(text, color=Fore.GREEN, delim=Fore.RESET) if COLORAMA else text
def YELLOW(self, text):
return self.color_text(text, color=Fore.YELLOW, delim=Fore.RESET) if COLORAMA else text
def color_text(self, text, color='', delim=''):
return color + text + color + delim
def COLOR(self, severity, text):
colors = {
self.PASS : self.GREEN,
self.ERROR : self.RED,
self.WARN : self.YELLOW
}
if severity in colors:
return colors[severity](text)
return text

117
workspace_tools/compliance/ioper_runner.py Normal file
View File

@ -0,0 +1,117 @@
#!/usr/bin/env python2
"""
mbed SDK
Copyright (c) 2011-2015 ARM Limited
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Author: Przemyslaw Wirkus <Przemyslaw.Wirkus@arm.com>
"""
import sys
import mbed_lstools
from prettytable import PrettyTable
try:
from colorama import init
except:
pass
COLORAMA = 'colorama' in sys.modules
from ioper_base import IOperTestCaseBase
from ioper_test_fs import IOperTest_FileStructure
from ioper_test_target_id import IOperTest_TargetID_Basic
from ioper_test_target_id import IOperTest_TargetID_MbedEnabled
TEST_LIST = [IOperTest_TargetID_Basic('basic'),
IOperTest_TargetID_MbedEnabled('mbed-enabled'),
IOperTest_FileStructure('basic'),
IOperTestCaseBase('all'), # Dummy used to add 'all' option
]
class IOperTestRunner():
""" Calls all i/face interoperability tests
"""
def __init__(self, scope=None):
""" Test scope:
'pedantic' - all
'mbed-enabled' - let's try to check if this device is mbed-enabled
'basic' - just simple, passive tests (no device flashing)
"""
self.requested_scope = scope
# Test scope definitions
self.SCOPE_BASIC = 'basic' # Basic tests, sanity checks
self.SCOPE_MBED_ENABLED = 'mbed-enabled' # Let's try to check if this device is mbed-enabled
self.SCOPE_PEDANTIC = 'pedantic' # Extensive tests
self.SCOPE_ALL = 'all' # All tests, equal to highest scope level
# This structure will help us sort test scopes so we can include them
# e.g. pedantic also includes basic and mbed-enabled tests
self.scopes = {self.SCOPE_BASIC : 0,
self.SCOPE_MBED_ENABLED : 1,
self.SCOPE_PEDANTIC : 2,
self.SCOPE_ALL : 99,
}
if COLORAMA:
init() # colorama.init()
def run(self):
""" Run tests, calculate overall score and print test results
"""
mbeds = mbed_lstools.create()
muts_list = mbeds.list_mbeds()
test_base = IOperTestCaseBase()
for i, mut in enumerate(muts_list):
result = []
print "MBEDLS: Detected %s, port: %s, mounted: %s"% (mut['platform_name'],
mut['serial_port'],
mut['mount_point'])
print "Running interoperability test suite, scope '%s'" % (self.requested_scope)
for test_case in TEST_LIST:
if self.scopes[self.requested_scope] >= self.scopes[test_case.scope]:
res = test_case.test(param=mut)
result.extend(res)
columns = ['Platform', 'Result', 'Scope', 'Description']
pt = PrettyTable(columns)
for col in columns:
pt.align[col] = 'l'
for tr in result:
severity, tr_scope, text = tr
tr = (test_base.COLOR(severity, mut['platform_name']),
test_base.COLOR(severity, severity),
test_base.COLOR(severity, tr_scope),
test_base.COLOR(severity, text))
pt.add_row(list(tr))
print pt.get_string(border=True, sortby='Result')
if i+1 < len(muts_list):
print
def get_available_oper_test_scopes():
""" Get list of available test scopes
"""
scopes = set()
for oper_test in TEST_LIST:
if oper_test.scope is not None:
scopes.add(oper_test.scope)
return list(scopes)

45
workspace_tools/compliance/ioper_test_fs.py Normal file
View File

@ -0,0 +1,45 @@
"""
mbed SDK
Copyright (c) 2011-2015 ARM Limited
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Author: Przemyslaw Wirkus <Przemyslaw.Wirkus@arm.com>
"""
import os.path
from ioper_base import IOperTestCaseBase
class IOperTest_FileStructure(IOperTestCaseBase):
def __init__(self, scope=None):
IOperTestCaseBase.__init__(self, scope)
def if_file_exist(self, fname, fail_severity=None):
file_path = os.path.join(self.param['mount_point'], fname)
exist = os.path.isfile(file_path)
if exist:
self.result.append((self.PASS, self.scope, "File '%s' exists" % file_path))
else:
self.result.append((fail_severity if fail_severity else self.ERROR, self.scope, "File '%s' not found" % file_path))
def test(self, param=None):
self.param = param
self.result = []
if param:
self.if_file_exist('mbed.htm', self.ERROR)
self.if_file_exist('DETAILS.TXT', self.INFO)
self.if_file_exist('FAIL.TXT', self.INFO)
return self.result

111
workspace_tools/compliance/ioper_test_target_id.py Normal file
View File

@ -0,0 +1,111 @@
"""
mbed SDK
Copyright (c) 2011-2015 ARM Limited
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Author: Przemyslaw Wirkus <Przemyslaw.Wirkus@arm.com>
"""
from ioper_base import IOperTestCaseBase
class IOperTest_TargetID(IOperTestCaseBase):
""" tests related to target_id value
"""
def __init__(self, scope=None):
IOperTestCaseBase.__init__(self, scope)
self.TARGET_ID_LEN = 24
def test_target_id_format(self, target_id, target_id_name):
# Expected length == 24, eg. "02400203D94B0E7724B7F3CF"
result = []
target_id_len = len(target_id) if target_id else 0
if target_id_len == self.TARGET_ID_LEN:
result.append((self.PASS, self.scope, "%s '%s' is %d chars long " % (target_id_name, target_id, target_id_len)))
result.append((self.INFO, self.scope, "%s Version String is %s.%s.%s " % (target_id_name,
target_id[0:4],
target_id[4:8],
target_id[8:24],
)))
else:
result.append((self.ERROR, self.scope, "%s '%s' is %d chars long. Expected %d chars" % (target_id_name, target_id, target_id_len, self.TARGET_ID_LEN)))
return result
def test_decode_target_id(self, target_id, target_id_name):
result = []
target_id_len = len(target_id) if target_id else 0
if target_id_len >= 4:
result.append((self.INFO, self.scope, "%s Vendor Code is '%s'" % (target_id_name, target_id[0:2])))
result.append((self.INFO, self.scope, "%s Platform Code is '%s'" % (target_id_name, target_id[2:4])))
result.append((self.INFO, self.scope, "%s Firmware Version is '%s'" % (target_id_name, target_id[4:8])))
result.append((self.INFO, self.scope, "%s Hash of secret is '%s'" % (target_id_name, target_id[8:24])))
return result
def test(self, param=None):
result = []
if param:
pass
return result
class IOperTest_TargetID_Basic(IOperTest_TargetID):
""" Basic interoperability tests checking TargetID compliance
"""
def __init__(self, scope=None):
IOperTest_TargetID.__init__(self, scope)
def test(self, param=None):
result = []
if param:
result.append((self.PASS, self.scope, "TargetID '%s' found" % param['target_id']))
# Check if target name can be decoded with mbed-ls
if param['platform_name']:
result.append((self.PASS, self.scope, "TargetID '%s' decoded as '%s'" % (param['target_id'][0:4], param['platform_name'])))
else:
result.append((self.ERROR, self.scope, "TargetID '%s'... not decoded" % (param['target_id'] if param['target_id'] else '')))
# Test for USBID and mbed.htm consistency
if param['target_id_mbed_htm'] == param['target_id_usb_id']:
result.append((self.PASS, self.scope, "TargetID (USBID) and TargetID (mbed.htm) match"))
else:
text = "TargetID (USBID) and TargetID (mbed.htm) don't match: '%s' != '%s'" % (param['target_id_usb_id'], param['target_id_mbed_htm'])
result.append((self.WARN, self.scope, text))
else:
result.append((self.ERROR, self.scope, "TargetID not found"))
return result
class IOperTest_TargetID_MbedEnabled(IOperTest_TargetID):
""" Basic interoperability tests checking TargetID compliance
"""
def __init__(self, scope=None):
IOperTest_TargetID.__init__(self, scope)
def test(self, param=None):
result = []
if param:
# Target ID tests:
result += self.test_target_id_format(param['target_id_usb_id'], "TargetId (USBID)")
result += self.test_target_id_format(param['target_id_mbed_htm'], "TargetId (mbed.htm)")
# Some extra info about TargetID itself
result += self.test_decode_target_id(param['target_id_usb_id'], "TargetId (USBID)")
result += self.test_decode_target_id(param['target_id_mbed_htm'], "TargetId (mbed.htm)")
return result

20
workspace_tools/singletest.py
View File

@ -75,6 +75,10 @@ from workspace_tools.test_api import get_autodetected_MUTS
from workspace_tools.test_api import get_autodetected_TEST_SPEC from workspace_tools.test_api import get_autodetected_TEST_SPEC
from workspace_tools.test_api import get_module_avail from workspace_tools.test_api import get_module_avail
from workspace_tools.compliance.ioper_runner import IOperTestRunner
from workspace_tools.compliance.ioper_runner import get_available_oper_test_scopes
# Importing extra modules which can be not installed but if available they can extend test suite functionality # Importing extra modules which can be not installed but if available they can extend test suite functionality
try: try:
import mbed_lstools import mbed_lstools
@ -182,15 +186,27 @@ if __name__ == '__main__':
exit(-1) exit(-1)
if opts.verbose_test_configuration_only: if opts.verbose_test_configuration_only:
print "MUTs configuration in %s:"% ('auto-detected' if opts.auto_detect else opts.muts_spec_filename) print "MUTs configuration in %s:" % ('auto-detected' if opts.auto_detect else opts.muts_spec_filename)
if MUTs: if MUTs:
print print_muts_configuration_from_json(MUTs, platform_filter=opts.general_filter_regex) print print_muts_configuration_from_json(MUTs, platform_filter=opts.general_filter_regex)
print print
print "Test specification in %s:"% ('auto-detected' if opts.auto_detect else opts.test_spec_filename) print "Test specification in %s:" % ('auto-detected' if opts.auto_detect else opts.test_spec_filename)
if test_spec: if test_spec:
print print_test_configuration_from_json(test_spec) print print_test_configuration_from_json(test_spec)
exit(0) exit(0)
if opts.operability_checks:
# Check if test scope is valid and run tests
test_scope = get_available_oper_test_scopes()
if opts.operability_checks in test_scope:
tests = IOperTestRunner(scope=opts.operability_checks)
test_results = tests.run()
else:
print "Unknown interoperability test scope name: '%s'" % (opts.operability_checks)
print "Available test scopes: %s" % (','.join(["'%s'" % n for n in test_scope]))
exit(0)
# Verbose test specification and MUTs configuration # Verbose test specification and MUTs configuration
if MUTs and opts.verbose: if MUTs and opts.verbose:
print print_muts_configuration_from_json(MUTs) print print_muts_configuration_from_json(MUTs)

18
workspace_tools/test_api.py
View File

@ -53,6 +53,7 @@ from workspace_tools.build_api import print_build_results
from workspace_tools.libraries import LIBRARIES, LIBRARY_MAP from workspace_tools.libraries import LIBRARIES, LIBRARY_MAP
from workspace_tools.toolchains import TOOLCHAIN_BIN_PATH from workspace_tools.toolchains import TOOLCHAIN_BIN_PATH
from workspace_tools.test_exporters import ReportExporter, ResultExporterType from workspace_tools.test_exporters import ReportExporter, ResultExporterType
from workspace_tools.compliance.ioper_runner import get_available_oper_test_scopes
import workspace_tools.host_tests.host_tests_plugins as host_tests_plugins import workspace_tools.host_tests.host_tests_plugins as host_tests_plugins
@ -1707,7 +1708,12 @@ def get_default_test_options_parser():
parser.add_option('', '--tc', parser.add_option('', '--tc',
dest='toolchains_filter', dest='toolchains_filter',
help="Toolchain filter for --auto option. Use toolcahins names separated by comma, 'default' or 'all' to select toolchains") help="Toolchain filter for --auto option. Use toolchains names separated by comma, 'default' or 'all' to select toolchains")
test_scopes = ','.join(["'%s'" % n for n in get_available_oper_test_scopes()])
parser.add_option('', '--oper',
dest='operability_checks',
help='Perform interoperability tests between host and connected mbed devices. Available test scopes are: %s' % test_scopes)
parser.add_option('', '--clean', parser.add_option('', '--clean',
dest='clean', dest='clean',
@ -1725,15 +1731,15 @@ def get_default_test_options_parser():
dest='test_only_common', dest='test_only_common',
default=False, default=False,
action="store_true", action="store_true",
help='Test only board internals. Skip perpherials tests and perform common tests.') help='Test only board internals. Skip perpherials tests and perform common tests')
parser.add_option('-n', '--test-by-names', parser.add_option('-n', '--test-by-names',
dest='test_by_names', dest='test_by_names',
help='Runs only test enumerated it this switch. Use comma to separate test case names.') help='Runs only test enumerated it this switch. Use comma to separate test case names')
parser.add_option('-p', '--peripheral-by-names', parser.add_option('-p', '--peripheral-by-names',
dest='peripheral_by_names', dest='peripheral_by_names',
help='Forces discovery of particular peripherals. Use comma to separate peripheral names.') help='Forces discovery of particular peripherals. Use comma to separate peripheral names')
copy_methods = host_tests_plugins.get_plugin_caps('CopyMethod') copy_methods = host_tests_plugins.get_plugin_caps('CopyMethod')
copy_methods_str = "Plugin support: " + ', '.join(copy_methods) copy_methods_str = "Plugin support: " + ', '.join(copy_methods)
@ -1822,11 +1828,11 @@ def get_default_test_options_parser():
dest='waterfall_test', dest='waterfall_test',
default=False, default=False,
action="store_true", action="store_true",
help='Used with --loops or --global-loops options. Tests until OK result occurs and assumes test passed.') help='Used with --loops or --global-loops options. Tests until OK result occurs and assumes test passed')
parser.add_option('-N', '--firmware-name', parser.add_option('-N', '--firmware-name',
dest='firmware_global_name', dest='firmware_global_name',
help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts.') help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts')
parser.add_option('-u', '--shuffle', parser.add_option('-u', '--shuffle',
dest='shuffle_test_order', dest='shuffle_test_order',