diff --git a/tools/build_api.py b/tools/build_api.py index e32c797861..1a9335d83f 100644 --- a/tools/build_api.py +++ b/tools/build_api.py @@ -17,18 +17,17 @@ limitations under the License. import re import tempfile -import colorama -from copy import copy from types import ListType from shutil import rmtree from os.path import join, exists, basename, abspath, normpath -from os import getcwd, walk, linesep +from os import linesep from time import time -import fnmatch -from tools.utils import mkdir, run_cmd, run_cmd_ext, NotSupportedException, ToolException, InvalidReleaseTargetException -from tools.paths import MBED_TARGETS_PATH, MBED_LIBRARIES, MBED_API, MBED_HAL, MBED_COMMON, MBED_CONFIG_FILE +from tools.utils import mkdir, run_cmd, run_cmd_ext, NotSupportedException,\ + ToolException, InvalidReleaseTargetException +from tools.paths import MBED_TARGETS_PATH, MBED_LIBRARIES, MBED_API, MBED_HAL,\ + MBED_COMMON, MBED_CONFIG_FILE from tools.targets import TARGET_NAMES, TARGET_MAP from tools.libraries import Library from tools.toolchains import TOOLCHAIN_CLASSES @@ -39,7 +38,7 @@ from tools.config import Config RELEASE_VERSIONS = ['2', '5'] def prep_report(report, target_name, toolchain_name, id_name): - # Setup report keys + """Setup report keys""" if not target_name in report: report[target_name] = {} @@ -50,7 +49,7 @@ def prep_report(report, target_name, toolchain_name, id_name): report[target_name][toolchain_name][id_name] = [] def prep_properties(properties, target_name, toolchain_name, vendor_label): - # Setup test properties + """Setup test properties""" if not target_name in properties: properties[target_name] = {} @@ -62,6 +61,7 @@ def prep_properties(properties, target_name, toolchain_name, vendor_label): properties[target_name][toolchain_name]["toolchain"] = toolchain_name def create_result(target_name, toolchain_name, id_name, description): + """Create the result dictionary""" cur_result = {} cur_result["target_name"] = target_name cur_result["toolchain_name"] = toolchain_name @@ -73,13 +73,15 @@ def create_result(target_name, toolchain_name, id_name, description): return cur_result def add_result_to_report(report, result): + """Add a single result to a report dictionary""" target = result["target_name"] toolchain = result["toolchain_name"] id_name = result['id'] - result_wrap = { 0: result } + result_wrap = {0: result} report[target][toolchain][id_name].append(result_wrap) def get_config(src_paths, target, toolchain_name): + """Get the configuration object for a target-toolchain combination""" # Convert src_paths to a list if needed if type(src_paths) != ListType: src_paths = [src_paths] @@ -120,26 +122,31 @@ def is_official_target(target_name, version): official release for the given version. target_name: Name if the target (ex. 'K64F') - version: The release version string. Should be a string contained within RELEASE_VERSIONS + version: The release version string. Should be a string contained within + RELEASE_VERSIONS """ - + result = True reason = None target = TARGET_MAP[target_name] - - if hasattr(target, 'release_versions') and version in target.release_versions: + + if hasattr(target, 'release_versions') \ + and version in target.release_versions: if version == '2': # For version 2, either ARM or uARM toolchain support is required required_toolchains = set(['ARM', 'uARM']) - - if not len(required_toolchains.intersection(set(target.supported_toolchains))) > 0: - result = False + + if not len(required_toolchains.intersection( + set(target.supported_toolchains))) > 0: + result = False reason = ("Target '%s' must support " % target.name) + \ - ("one of the folowing toolchains to be included in the mbed 2.0 ") + \ - (("official release: %s" + linesep) % ", ".join(required_toolchains)) + \ + ("one of the folowing toolchains to be included in the") + \ + ((" mbed 2.0 official release: %s" + linesep) % + ", ".join(required_toolchains)) + \ ("Currently it is only configured to support the ") + \ - ("following toolchains: %s" % ", ".join(target.supported_toolchains)) - + ("following toolchains: %s" % + ", ".join(target.supported_toolchains)) + elif version == '5': # For version 5, ARM, GCC_ARM, and IAR toolchain support is required required_toolchains = set(['ARM', 'GCC_ARM', 'IAR']) @@ -148,34 +155,41 @@ def is_official_target(target_name, version): supported_toolchains = set(target.supported_toolchains) supported_toolchains_sorted = list(supported_toolchains) supported_toolchains_sorted.sort() - + if not required_toolchains.issubset(supported_toolchains): result = False reason = ("Target '%s' must support " % target.name) + \ - ("ALL of the folowing toolchains to be included in the mbed OS 5.0 ") + \ - (("official release: %s" + linesep) % ", ".join(required_toolchains_sorted)) + \ + ("ALL of the folowing toolchains to be included in the") + \ + ((" mbed OS 5.0 official release: %s" + linesep) % + ", ".join(required_toolchains_sorted)) + \ ("Currently it is only configured to support the ") + \ - ("following toolchains: %s" % ", ".join(supported_toolchains_sorted)) + ("following toolchains: %s" % + ", ".join(supported_toolchains_sorted)) elif not target.default_build == 'standard': result = False - reason = ("Target '%s' must set the 'default_build' " % target.name) + \ - ("to 'standard' to be included in the mbed OS 5.0 ") + \ - ("official release." + linesep) + \ + reason = ("Target '%s' must set the " % target.name) + \ + ("'default_build' to 'standard' to be included in the ") + \ + ("mbed OS 5.0 official release." + linesep) + \ ("Currently it is set to '%s'" % target.default_build) else: result = False - reason = ("Target '%s' has set an invalid release version of '%s'" % version) + \ - ("Please choose from the following release versions: %s" + ', '.join(RELEASE_VERSIONS)) + reason = ("Target '%s' has set an invalid release version of '%s'" % + version) + \ + ("Please choose from the following release versions: %s" % + ', '.join(RELEASE_VERSIONS)) else: result = False if not hasattr(target, 'release_versions'): - reason = "Target '%s' does not have the 'release_versions' key set" % target.name + reason = "Target '%s' " % target.name + reason += "does not have the 'release_versions' key set" elif not version in target.release_versions: - reason = "Target '%s' does not contain the version '%s' in its 'release_versions' key" % (target.name, version) - + reason = "Target '%s' does not contain the version '%s' " % \ + (target.name, version) + reason += "in its 'release_versions' key" + return result, reason def transform_release_toolchains(toolchains, version): @@ -183,10 +197,9 @@ def transform_release_toolchains(toolchains, version): only the supported toolchains for that release toolchains: The list of toolchains - version: The release version string. Should be a string contained within RELEASE_VERSIONS + version: The release version string. Should be a string contained within + RELEASE_VERSIONS """ - toolchains_set = set(toolchains) - if version == '5': return ['ARM', 'GCC_ARM', 'IAR'] else: @@ -196,34 +209,40 @@ def transform_release_toolchains(toolchains, version): def get_mbed_official_release(version): """ Given a release version string, return a tuple that contains a target and the supported toolchains for that release. - Ex. Given '2', return (('LPC1768', ('ARM', 'GCC_ARM')), ('K64F', ('ARM', 'GCC_ARM')), ...) + Ex. Given '2', return (('LPC1768', ('ARM', 'GCC_ARM')), + ('K64F', ('ARM', 'GCC_ARM')), ...) - version: The version string. Should be a string contained within RELEASE_VERSIONS + version: The version string. Should be a string contained within + RELEASE_VERSIONS """ - MBED_OFFICIAL_RELEASE = ( + mbed_official_release = ( tuple( tuple( [ TARGET_MAP[target].name, - tuple(transform_release_toolchains(TARGET_MAP[target].supported_toolchains, version)) + tuple(transform_release_toolchains( + TARGET_MAP[target].supported_toolchains, version)) ] - ) for target in TARGET_NAMES if (hasattr(TARGET_MAP[target], 'release_versions') and version in TARGET_MAP[target].release_versions) + ) for target in TARGET_NAMES \ + if (hasattr(TARGET_MAP[target], 'release_versions') + and version in TARGET_MAP[target].release_versions) ) ) - - for target in MBED_OFFICIAL_RELEASE: + + for target in mbed_official_release: is_official, reason = is_official_target(target[0], version) - + if not is_official: raise InvalidReleaseTargetException(reason) - - return MBED_OFFICIAL_RELEASE + + return mbed_official_release def prepare_toolchain(src_paths, target, toolchain_name, - macros=None, options=None, clean=False, jobs=1, - notify=None, silent=False, verbose=False, extra_verbose=False, config=None): + macros=None, options=None, clean=False, jobs=1, + notify=None, silent=False, verbose=False, + extra_verbose=False, config=None): """ Prepares resource related objects - toolchain, target, config src_paths: the paths to source directories target: ['LPC1768', 'LPC11U24', 'LPC2368'] @@ -249,8 +268,10 @@ def prepare_toolchain(src_paths, target, toolchain_name, # Toolchain instance try: - toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, options, notify, macros, silent, extra_verbose=extra_verbose) - except KeyError as e: + toolchain = TOOLCHAIN_CLASSES[toolchain_name]( + target, options, notify, macros, silent, + extra_verbose=extra_verbose) + except KeyError: raise KeyError("Toolchain %s not supported" % toolchain_name) toolchain.config = config @@ -260,12 +281,14 @@ def prepare_toolchain(src_paths, target, toolchain_name, return toolchain -def scan_resources(src_paths, toolchain, dependencies_paths=None, inc_dirs=None): +def scan_resources(src_paths, toolchain, dependencies_paths=None, + inc_dirs=None): """ Scan resources using initialized toolcain src_paths: the paths to source directories toolchain: valid toolchain object dependencies_paths: dependency paths that we should scan for include dirs - inc_dirs: additional include directories which should be added to thescanner resources + inc_dirs: additional include directories which should be added to + the scanner resources """ # Scan src_path @@ -286,7 +309,8 @@ def scan_resources(src_paths, toolchain, dependencies_paths=None, inc_dirs=None) else: resources.inc_dirs.append(inc_dirs) - # Load resources into the config system which might expand/modify resources based on config data + # Load resources into the config system which might expand/modify resources + # based on config data resources = toolchain.config.load_resources(resources) # Set the toolchain's configuration data @@ -295,10 +319,11 @@ def scan_resources(src_paths, toolchain, dependencies_paths=None, inc_dirs=None) return resources def build_project(src_paths, build_path, target, toolchain_name, - libraries_paths=None, options=None, linker_script=None, - clean=False, notify=None, verbose=False, name=None, macros=None, inc_dirs=None, - jobs=1, silent=False, report=None, properties=None, project_id=None, project_description=None, - extra_verbose=False, config=None): + libraries_paths=None, options=None, linker_script=None, + clean=False, notify=None, verbose=False, name=None, + macros=None, inc_dirs=None, jobs=1, silent=False, + report=None, properties=None, project_id=None, + project_description=None, extra_verbose=False, config=None): """ This function builds project. Project can be for example one test / UT """ @@ -310,20 +335,21 @@ def build_project(src_paths, build_path, target, toolchain_name, src_paths.extend(libraries_paths) # Build Directory - if clean: - if exists(build_path): - rmtree(build_path) + if clean and exists(build_path): + rmtree(build_path) mkdir(build_path) # Pass all params to the unified prepare_toolchain() - toolchain = prepare_toolchain(src_paths, target, toolchain_name, - macros=macros, options=options, clean=clean, jobs=jobs, - notify=notify, silent=silent, verbose=verbose, extra_verbose=extra_verbose, config=config) + toolchain = prepare_toolchain( + src_paths, target, toolchain_name, macros=macros, options=options, + clean=clean, jobs=jobs, notify=notify, silent=silent, verbose=verbose, + extra_verbose=extra_verbose, config=config) # The first path will give the name to the library if name is None: name = basename(normpath(abspath(src_paths[0]))) - toolchain.info("Building project %s (%s, %s)" % (name, toolchain.target.name, toolchain_name)) + toolchain.info("Building project %s (%s, %s)" % + (name, toolchain.target.name, toolchain_name)) # Initialize reporting if report != None: @@ -333,9 +359,11 @@ def build_project(src_paths, build_path, target, toolchain_name, description = project_description if project_description else name vendor_label = toolchain.target.extra_labels[0] prep_report(report, toolchain.target.name, toolchain_name, id_name) - cur_result = create_result(toolchain.target.name, toolchain_name, id_name, description) + cur_result = create_result(toolchain.target.name, toolchain_name, + id_name, description) if properties != None: - prep_properties(properties, toolchain.target.name, toolchain_name, vendor_label) + prep_properties(properties, toolchain.target.name, toolchain_name, + vendor_label) try: # Call unified scan_resources @@ -346,7 +374,8 @@ def build_project(src_paths, build_path, target, toolchain_name, resources.linker_script = linker_script # Compile Sources - objects = toolchain.compile_sources(resources, build_path, resources.inc_dirs) + objects = toolchain.compile_sources(resources, build_path, + resources.inc_dirs) resources.objects.extend(objects) # Link Program @@ -363,11 +392,11 @@ def build_project(src_paths, build_path, target, toolchain_name, return res - except Exception, e: + except Exception as exc: if report != None: end = time() - if isinstance(e, NotSupportedException): + if isinstance(exc, NotSupportedException): cur_result["result"] = "NOT_SUPPORTED" else: cur_result["result"] = "FAIL" @@ -384,10 +413,10 @@ def build_project(src_paths, build_path, target, toolchain_name, raise def build_library(src_paths, build_path, target, toolchain_name, - dependencies_paths=None, options=None, name=None, clean=False, archive=True, - notify=None, verbose=False, macros=None, inc_dirs=None, - jobs=1, silent=False, report=None, properties=None, extra_verbose=False, - project_id=None): + dependencies_paths=None, options=None, name=None, clean=False, + archive=True, notify=None, verbose=False, macros=None, + inc_dirs=None, jobs=1, silent=False, report=None, + properties=None, extra_verbose=False, project_id=None): """ Prepares resource related objects - toolchain, target, config src_paths: the paths to source directories build_path: the path of the build directory @@ -412,20 +441,21 @@ def build_library(src_paths, build_path, target, toolchain_name, tmp_path = build_path # Clean the build directory - if clean: - if exists(tmp_path): - rmtree(tmp_path) + if clean and exists(tmp_path): + rmtree(tmp_path) mkdir(tmp_path) # Pass all params to the unified prepare_toolchain() - toolchain = prepare_toolchain(src_paths, target, toolchain_name, - macros=macros, options=options, clean=clean, jobs=jobs, - notify=notify, silent=silent, verbose=verbose, extra_verbose=extra_verbose) + toolchain = prepare_toolchain( + src_paths, target, toolchain_name, macros=macros, options=options, + clean=clean, jobs=jobs, notify=notify, silent=silent, verbose=verbose, + extra_verbose=extra_verbose) # The first path will give the name to the library if name is None: name = basename(normpath(abspath(src_paths[0]))) - toolchain.info("Building library %s (%s, %s)" % (name, toolchain.target.name, toolchain_name)) + toolchain.info("Building library %s (%s, %s)" % + (name, toolchain.target.name, toolchain_name)) # Initialize reporting if report != None: @@ -435,9 +465,11 @@ def build_library(src_paths, build_path, target, toolchain_name, description = name vendor_label = toolchain.target.extra_labels[0] prep_report(report, toolchain.target.name, toolchain_name, id_name) - cur_result = create_result(toolchain.target.name, toolchain_name, id_name, description) + cur_result = create_result(toolchain.target.name, toolchain_name, + id_name, description) if properties != None: - prep_properties(properties, toolchain.target.name, toolchain_name, vendor_label) + prep_properties(properties, toolchain.target.name, toolchain_name, + vendor_label) for src_path in src_paths: if not exists(src_path): @@ -450,21 +482,28 @@ def build_library(src_paths, build_path, target, toolchain_name, try: # Call unified scan_resources - resources = scan_resources(src_paths, toolchain, dependencies_paths=dependencies_paths, inc_dirs=inc_dirs) + resources = scan_resources(src_paths, toolchain, + dependencies_paths=dependencies_paths, + inc_dirs=inc_dirs) - # Copy headers, objects and static libraries - all files needed for static lib + # Copy headers, objects and static libraries - all files needed for + # static lib toolchain.copy_files(resources.headers, build_path, resources=resources) toolchain.copy_files(resources.objects, build_path, resources=resources) - toolchain.copy_files(resources.libraries, build_path, resources=resources) + toolchain.copy_files(resources.libraries, build_path, + resources=resources) if resources.linker_script: - toolchain.copy_files(resources.linker_script, build_path, resources=resources) + toolchain.copy_files(resources.linker_script, build_path, + resources=resources) if resources.hex_files: - toolchain.copy_files(resources.hex_files, build_path, resources=resources) + toolchain.copy_files(resources.hex_files, build_path, + resources=resources) # Compile Sources - objects = toolchain.compile_sources(resources, abspath(tmp_path), resources.inc_dirs) + objects = toolchain.compile_sources(resources, abspath(tmp_path), + resources.inc_dirs) resources.objects.extend(objects) if archive: @@ -476,16 +515,17 @@ def build_library(src_paths, build_path, target, toolchain_name, cur_result["output"] = toolchain.get_output() cur_result["result"] = "OK" + add_result_to_report(report, cur_result) return True - except Exception, e: + except Exception as exc: if report != None: end = time() - if isinstance(e, ToolException): + if isinstance(exc, ToolException): cur_result["result"] = "FAIL" - elif isinstance(e, NotSupportedException): + elif isinstance(exc, NotSupportedException): cur_result["result"] = "NOT_SUPPORTED" cur_result["elapsed_time"] = end - start @@ -497,27 +537,33 @@ def build_library(src_paths, build_path, target, toolchain_name, add_result_to_report(report, cur_result) # Let Exception propagate - raise e + raise ###################### ### Legacy methods ### ###################### -def build_lib(lib_id, target, toolchain_name, options=None, verbose=False, clean=False, macros=None, notify=None, jobs=1, silent=False, report=None, properties=None, extra_verbose=False): +def build_lib(lib_id, target, toolchain_name, options=None, verbose=False, + clean=False, macros=None, notify=None, jobs=1, silent=False, + report=None, properties=None, extra_verbose=False): """ Legacy method for building mbed libraries - Function builds library in proper directory using all dependencies and macros defined by user. + + Function builds library in proper directory using all dependencies and + macros defined by user. """ lib = Library(lib_id) if not lib.is_supported(target, toolchain_name): - print 'Library "%s" is not yet supported on target %s with toolchain %s' % (lib_id, target.name, toolchain) + print('Library "%s" is not yet supported on target %s with toolchain %s' + % (lib_id, target.name, toolchain_name)) return False - # We need to combine macros from parameter list with macros from library definition - MACROS = lib.macros if lib.macros else [] + # We need to combine macros from parameter list with macros from library + # definition + lib_macros = lib.macros if lib.macros else [] if macros: - macros.extend(MACROS) + macros.extend(lib_macros) else: - macros = MACROS + macros = lib_macros src_paths = lib.source_dir build_path = lib.build_dir @@ -525,17 +571,6 @@ def build_lib(lib_id, target, toolchain_name, options=None, verbose=False, clean inc_dirs = lib.inc_dirs inc_dirs_ext = lib.inc_dirs_ext - """ src_path: the path of the source directory - build_path: the path of the build directory - target: ['LPC1768', 'LPC11U24', 'LPC2368'] - toolchain: ['ARM', 'uARM', 'GCC_ARM', 'GCC_CR'] - library_paths: List of paths to additional libraries - clean: Rebuild everything if True - notify: Notify function for logs - verbose: Write the actual tools command lines if True - inc_dirs: additional include directories which should be included in build - inc_dirs_ext: additional include directories which should be copied to library directory - """ if type(src_paths) != ListType: src_paths = [src_paths] @@ -549,10 +584,12 @@ def build_lib(lib_id, target, toolchain_name, options=None, verbose=False, clean vendor_label = target.extra_labels[0] cur_result = None prep_report(report, target.name, toolchain_name, id_name) - cur_result = create_result(target.name, toolchain_name, id_name, description) + cur_result = create_result(target.name, toolchain_name, id_name, + description) if properties != None: - prep_properties(properties, target.name, toolchain_name, vendor_label) + prep_properties(properties, target.name, toolchain_name, + vendor_label) for src_path in src_paths: if not exists(src_path): @@ -567,12 +604,15 @@ def build_lib(lib_id, target, toolchain_name, options=None, verbose=False, clean try: # Toolchain instance - toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, options, macros=macros, notify=notify, silent=silent, extra_verbose=extra_verbose) + toolchain = TOOLCHAIN_CLASSES[toolchain_name]( + target, options, macros=macros, notify=notify, silent=silent, + extra_verbose=extra_verbose) toolchain.VERBOSE = verbose toolchain.jobs = jobs toolchain.build_all = clean - toolchain.info("Building library %s (%s, %s)" % (name.upper(), target.name, toolchain_name)) + toolchain.info("Building library %s (%s, %s)" % + (name.upper(), target.name, toolchain_name)) # Take into account the library configuration (MBED_CONFIG_FILE) config = Config(target) @@ -602,8 +642,8 @@ def build_lib(lib_id, target, toolchain_name, options=None, verbose=False, clean dependencies_include_dir.extend(inc_dirs) # Add other discovered configuration data to the configuration object - for r in resources: - config.load_resources(r) + for res in resources: + config.load_resources(res) toolchain.set_config_data(toolchain.config.get_config_data()) # Create the desired build directory structure @@ -614,14 +654,17 @@ def build_lib(lib_id, target, toolchain_name, options=None, verbose=False, clean # Copy Headers for resource in resources: - toolchain.copy_files(resource.headers, build_path, resources=resource) + toolchain.copy_files(resource.headers, build_path, + resources=resource) - dependencies_include_dir.extend(toolchain.scan_resources(build_path).inc_dirs) + dependencies_include_dir.extend( + toolchain.scan_resources(build_path).inc_dirs) # Compile Sources objects = [] for resource in resources: - objects.extend(toolchain.compile_sources(resource, tmp_path, dependencies_include_dir)) + objects.extend(toolchain.compile_sources(resource, tmp_path, + dependencies_include_dir)) needed_update = toolchain.build_library(objects, bin_path, name) @@ -634,7 +677,7 @@ def build_lib(lib_id, target, toolchain_name, options=None, verbose=False, clean add_result_to_report(report, cur_result) return True - except Exception, e: + except Exception: if report != None: end = time() cur_result["result"] = "FAIL" @@ -647,11 +690,16 @@ def build_lib(lib_id, target, toolchain_name, options=None, verbose=False, clean add_result_to_report(report, cur_result) # Let Exception propagate - raise e + raise -# We do have unique legacy conventions about how we build and package the mbed library -def build_mbed_libs(target, toolchain_name, options=None, verbose=False, clean=False, macros=None, notify=None, jobs=1, silent=False, report=None, properties=None, extra_verbose=False): - """ Function returns True is library was built and false if building was skipped """ +# We do have unique legacy conventions about how we build and package the mbed +# library +def build_mbed_libs(target, toolchain_name, options=None, verbose=False, + clean=False, macros=None, notify=None, jobs=1, silent=False, + report=None, properties=None, extra_verbose=False): + """ Function returns True is library was built and false if building was + skipped + """ if report != None: start = time() @@ -660,16 +708,21 @@ def build_mbed_libs(target, toolchain_name, options=None, verbose=False, clean=F vendor_label = target.extra_labels[0] cur_result = None prep_report(report, target.name, toolchain_name, id_name) - cur_result = create_result(target.name, toolchain_name, id_name, description) + cur_result = create_result(target.name, toolchain_name, id_name, + description) if properties != None: - prep_properties(properties, target.name, toolchain_name, vendor_label) + prep_properties(properties, target.name, toolchain_name, + vendor_label) # Check toolchain support if toolchain_name not in target.supported_toolchains: supported_toolchains_text = ", ".join(target.supported_toolchains) - print '%s target is not yet supported by toolchain %s' % (target.name, toolchain_name) - print '%s target supports %s toolchain%s' % (target.name, supported_toolchains_text, 's' if len(target.supported_toolchains) > 1 else '') + print('%s target is not yet supported by toolchain %s' % + (target.name, toolchain_name)) + print('%s target supports %s toolchain%s' % + (target.name, supported_toolchains_text, 's' + if len(target.supported_toolchains) > 1 else '')) if report != None: cur_result["result"] = "SKIP" @@ -679,7 +732,9 @@ def build_mbed_libs(target, toolchain_name, options=None, verbose=False, clean=F try: # Toolchain - toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, options, macros=macros, notify=notify, silent=silent, extra_verbose=extra_verbose) + toolchain = TOOLCHAIN_CLASSES[toolchain_name]( + target, options, macros=macros, notify=notify, silent=silent, + extra_verbose=extra_verbose) toolchain.VERBOSE = verbose toolchain.jobs = jobs toolchain.build_all = clean @@ -691,63 +746,75 @@ def build_mbed_libs(target, toolchain_name, options=None, verbose=False, clean=F toolchain.set_config_data(toolchain.config.get_config_data()) # Source and Build Paths - BUILD_TARGET = join(MBED_LIBRARIES, "TARGET_" + target.name) - BUILD_TOOLCHAIN = join(BUILD_TARGET, "TOOLCHAIN_" + toolchain.name) - mkdir(BUILD_TOOLCHAIN) + build_target = join(MBED_LIBRARIES, "TARGET_" + target.name) + build_toolchain = join(build_target, "TOOLCHAIN_" + toolchain.name) + mkdir(build_toolchain) - TMP_PATH = join(MBED_LIBRARIES, '.temp', toolchain.obj_path) - mkdir(TMP_PATH) + tmp_path = join(MBED_LIBRARIES, '.temp', toolchain.obj_path) + mkdir(tmp_path) # CMSIS - toolchain.info("Building library %s (%s, %s)"% ('CMSIS', target.name, toolchain_name)) + toolchain.info("Building library %s (%s, %s)" % + ('CMSIS', target.name, toolchain_name)) cmsis_src = join(MBED_TARGETS_PATH, "cmsis") resources = toolchain.scan_resources(cmsis_src) - toolchain.copy_files(resources.headers, BUILD_TARGET) - toolchain.copy_files(resources.linker_script, BUILD_TOOLCHAIN) - toolchain.copy_files(resources.bin_files, BUILD_TOOLCHAIN) + toolchain.copy_files(resources.headers, build_target) + toolchain.copy_files(resources.linker_script, build_toolchain) + toolchain.copy_files(resources.bin_files, build_toolchain) - objects = toolchain.compile_sources(resources, TMP_PATH) - toolchain.copy_files(objects, BUILD_TOOLCHAIN) + objects = toolchain.compile_sources(resources, tmp_path) + toolchain.copy_files(objects, build_toolchain) # mbed - toolchain.info("Building library %s (%s, %s)" % ('MBED', target.name, toolchain_name)) + toolchain.info("Building library %s (%s, %s)" % + ('MBED', target.name, toolchain_name)) # Common Headers - toolchain.copy_files(toolchain.scan_resources(MBED_API).headers, MBED_LIBRARIES) - toolchain.copy_files(toolchain.scan_resources(MBED_HAL).headers, MBED_LIBRARIES) + toolchain.copy_files(toolchain.scan_resources(MBED_API).headers, + MBED_LIBRARIES) + toolchain.copy_files(toolchain.scan_resources(MBED_HAL).headers, + MBED_LIBRARIES) # Target specific sources - HAL_SRC = join(MBED_TARGETS_PATH, "hal") - hal_implementation = toolchain.scan_resources(HAL_SRC) - toolchain.copy_files(hal_implementation.headers + hal_implementation.hex_files + hal_implementation.libraries, BUILD_TARGET, resources=hal_implementation) - incdirs = toolchain.scan_resources(BUILD_TARGET).inc_dirs - objects = toolchain.compile_sources(hal_implementation, TMP_PATH, [MBED_LIBRARIES] + incdirs) + hal_src = join(MBED_TARGETS_PATH, "hal") + hal_implementation = toolchain.scan_resources(hal_src) + toolchain.copy_files(hal_implementation.headers + + hal_implementation.hex_files + + hal_implementation.libraries, + build_target, resources=hal_implementation) + incdirs = toolchain.scan_resources(build_target).inc_dirs + objects = toolchain.compile_sources(hal_implementation, tmp_path, + [MBED_LIBRARIES] + incdirs) # Common Sources mbed_resources = toolchain.scan_resources(MBED_COMMON) - objects += toolchain.compile_sources(mbed_resources, TMP_PATH, [MBED_LIBRARIES] + incdirs) + objects += toolchain.compile_sources(mbed_resources, tmp_path, + [MBED_LIBRARIES] + incdirs) # A number of compiled files need to be copied as objects as opposed to - # being part of the mbed library, for reasons that have to do with the way - # the linker search for symbols in archives. These are: - # - retarget.o: to make sure that the C standard lib symbols get overridden + # being part of the mbed library, for reasons that have to do with the + # way the linker search for symbols in archives. These are: + # - retarget.o: to make sure that the C standard lib symbols get + # overridden # - board.o: mbed_die is weak - # - mbed_overrides.o: this contains platform overrides of various weak SDK functions - separate_names, separate_objects = ['retarget.o', 'board.o', 'mbed_overrides.o'], [] + # - mbed_overrides.o: this contains platform overrides of various + # weak SDK functions + separate_names, separate_objects = ['retarget.o', 'board.o', + 'mbed_overrides.o'], [] - for o in objects: + for obj in objects: for name in separate_names: - if o.endswith(name): - separate_objects.append(o) + if obj.endswith(name): + separate_objects.append(obj) - for o in separate_objects: - objects.remove(o) + for obj in separate_objects: + objects.remove(obj) - toolchain.build_library(objects, BUILD_TOOLCHAIN, "mbed") + toolchain.build_library(objects, build_toolchain, "mbed") - for o in separate_objects: - toolchain.copy_files(o, BUILD_TOOLCHAIN) + for obj in separate_objects: + toolchain.copy_files(obj, build_toolchain) if report != None: end = time() @@ -759,7 +826,7 @@ def build_mbed_libs(target, toolchain_name, options=None, verbose=False, clean=F return True - except Exception, e: + except Exception as exc: if report != None: end = time() cur_result["result"] = "FAIL" @@ -769,12 +836,12 @@ def build_mbed_libs(target, toolchain_name, options=None, verbose=False, clean=F if toolchain_output: cur_result["output"] += toolchain_output - cur_result["output"] += str(e) + cur_result["output"] += str(exc) add_result_to_report(report, cur_result) # Let Exception propagate - raise e + raise def get_unique_supported_toolchains(release_targets=None): @@ -798,9 +865,11 @@ def get_unique_supported_toolchains(release_targets=None): return unique_supported_toolchains -def mcu_toolchain_matrix(verbose_html=False, platform_filter=None, release_version='5'): +def mcu_toolchain_matrix(verbose_html=False, platform_filter=None, + release_version='5'): """ Shows target map using prettytable """ - from prettytable import PrettyTable # Only use it in this function so building works without extra modules + # Only use it in this function so building works without extra modules + from prettytable import PrettyTable if isinstance(release_version, basestring): # Force release_version to lowercase if it is a string @@ -815,23 +884,26 @@ def mcu_toolchain_matrix(verbose_html=False, platform_filter=None, release_versi for version in RELEASE_VERSIONS: version_release_targets[version] = get_mbed_official_release(version) - version_release_target_names[version] = [x[0] for x in version_release_targets[version]] + version_release_target_names[version] = [x[0] for x in + version_release_targets[ + version]] if release_version in RELEASE_VERSIONS: release_targets = version_release_targets[release_version] else: release_targets = None - unique_supported_toolchains = get_unique_supported_toolchains(release_targets) + unique_supported_toolchains = get_unique_supported_toolchains( + release_targets) prepend_columns = ["Target"] + ["mbed OS %s" % x for x in RELEASE_VERSIONS] # All tests status table print columns = prepend_columns + unique_supported_toolchains - pt = PrettyTable(columns) + table_printer = PrettyTable(columns) # Align table for col in columns: - pt.align[col] = "c" - pt.align["Target"] = "l" + table_printer.align[col] = "c" + table_printer.align["Target"] = "l" perm_counter = 0 target_counter = 0 @@ -867,9 +939,10 @@ def mcu_toolchain_matrix(verbose_html=False, platform_filter=None, release_versi text = "-" row.append(text) - pt.add_row(row) + table_printer.add_row(row) - result = pt.get_html_string() if verbose_html else pt.get_string() + result = table_printer.get_html_string() if verbose_html \ + else table_printer.get_string() result += "\n" result += "Supported targets: %d\n"% (target_counter) if target_counter == 1: @@ -879,56 +952,66 @@ def mcu_toolchain_matrix(verbose_html=False, platform_filter=None, release_versi def get_target_supported_toolchains(target): """ Returns target supported toolchains list """ - return TARGET_MAP[target].supported_toolchains if target in TARGET_MAP else None + return TARGET_MAP[target].supported_toolchains if target in TARGET_MAP \ + else None -def static_analysis_scan(target, toolchain_name, CPPCHECK_CMD, CPPCHECK_MSG_FORMAT, options=None, verbose=False, clean=False, macros=None, notify=None, jobs=1, extra_verbose=False): +def static_analysis_scan(target, toolchain_name, cppcheck_cmd, + cppcheck_msg_format, options=None, verbose=False, + clean=False, macros=None, notify=None, jobs=1, + extra_verbose=False): + """Perform static analysis on a target and toolchain combination""" # Toolchain - toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, options, macros=macros, notify=notify, extra_verbose=extra_verbose) + toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, options, + macros=macros, notify=notify, + extra_verbose=extra_verbose) toolchain.VERBOSE = verbose toolchain.jobs = jobs toolchain.build_all = clean # Source and Build Paths - BUILD_TARGET = join(MBED_LIBRARIES, "TARGET_" + target.name) - BUILD_TOOLCHAIN = join(BUILD_TARGET, "TOOLCHAIN_" + toolchain.name) - mkdir(BUILD_TOOLCHAIN) + build_target = join(MBED_LIBRARIES, "TARGET_" + target.name) + build_toolchain = join(build_target, "TOOLCHAIN_" + toolchain.name) + mkdir(build_toolchain) - TMP_PATH = join(MBED_LIBRARIES, '.temp', toolchain.obj_path) - mkdir(TMP_PATH) + tmp_path = join(MBED_LIBRARIES, '.temp', toolchain.obj_path) + mkdir(tmp_path) # CMSIS - toolchain.info("Static analysis for %s (%s, %s)" % ('CMSIS', target.name, toolchain_name)) + toolchain.info("Static analysis for %s (%s, %s)" % + ('CMSIS', target.name, toolchain_name)) cmsis_src = join(MBED_TARGETS_PATH, "cmsis") resources = toolchain.scan_resources(cmsis_src) # Copy files before analysis - toolchain.copy_files(resources.headers, BUILD_TARGET) - toolchain.copy_files(resources.linker_script, BUILD_TOOLCHAIN) + toolchain.copy_files(resources.headers, build_target) + toolchain.copy_files(resources.linker_script, build_toolchain) - # Gather include paths, c, cpp sources and macros to transfer to cppcheck command line + # Gather include paths, c, cpp sources and macros to transfer to cppcheck + # command line includes = ["-I%s"% i for i in resources.inc_dirs] - includes.append("-I%s"% str(BUILD_TARGET)) + includes.append("-I%s"% str(build_target)) c_sources = " ".join(resources.c_sources) cpp_sources = " ".join(resources.cpp_sources) macros = ["-D%s"% s for s in toolchain.get_symbols() + toolchain.macros] - includes = map(str.strip, includes) - macros = map(str.strip, macros) + includes = [inc.strip() for inc in includes] + macros = [mac.strip() for mac in macros] - check_cmd = CPPCHECK_CMD - check_cmd += CPPCHECK_MSG_FORMAT + check_cmd = cppcheck_cmd + check_cmd += cppcheck_msg_format check_cmd += includes check_cmd += macros - # We need to pass some params via file to avoid "command line too long in some OSs" + # We need to pass some params via file to avoid "command line too long in + # some OSs" tmp_file = tempfile.NamedTemporaryFile(delete=False) tmp_file.writelines(line + '\n' for line in c_sources.split()) tmp_file.writelines(line + '\n' for line in cpp_sources.split()) tmp_file.close() check_cmd += ["--file-list=%s"% tmp_file.name] - _stdout, _stderr, _rc = run_cmd(check_cmd) + _stdout, _stderr, _ = run_cmd(check_cmd) if verbose: print _stdout print _stderr @@ -936,50 +1019,58 @@ def static_analysis_scan(target, toolchain_name, CPPCHECK_CMD, CPPCHECK_MSG_FORM # ========================================================================= # MBED - toolchain.info("Static analysis for %s (%s, %s)" % ('MBED', target.name, toolchain_name)) + toolchain.info("Static analysis for %s (%s, %s)" % + ('MBED', target.name, toolchain_name)) # Common Headers - toolchain.copy_files(toolchain.scan_resources(MBED_API).headers, MBED_LIBRARIES) - toolchain.copy_files(toolchain.scan_resources(MBED_HAL).headers, MBED_LIBRARIES) + toolchain.copy_files(toolchain.scan_resources(MBED_API).headers, + MBED_LIBRARIES) + toolchain.copy_files(toolchain.scan_resources(MBED_HAL).headers, + MBED_LIBRARIES) # Target specific sources - HAL_SRC = join(MBED_TARGETS_PATH, "hal") - hal_implementation = toolchain.scan_resources(HAL_SRC) + hal_src = join(MBED_TARGETS_PATH, "hal") + hal_implementation = toolchain.scan_resources(hal_src) # Copy files before analysis - toolchain.copy_files(hal_implementation.headers + hal_implementation.hex_files, BUILD_TARGET, resources=hal_implementation) - incdirs = toolchain.scan_resources(BUILD_TARGET) + toolchain.copy_files(hal_implementation.headers + + hal_implementation.hex_files, build_target, + resources=hal_implementation) + incdirs = toolchain.scan_resources(build_target) target_includes = ["-I%s" % i for i in incdirs.inc_dirs] - target_includes.append("-I%s"% str(BUILD_TARGET)) - target_includes.append("-I%s"% str(HAL_SRC)) + target_includes.append("-I%s"% str(build_target)) + target_includes.append("-I%s"% str(hal_src)) target_c_sources = " ".join(incdirs.c_sources) target_cpp_sources = " ".join(incdirs.cpp_sources) - target_macros = ["-D%s"% s for s in toolchain.get_symbols() + toolchain.macros] + target_macros = ["-D%s"% s for s in + toolchain.get_symbols() + toolchain.macros] # Common Sources mbed_resources = toolchain.scan_resources(MBED_COMMON) - # Gather include paths, c, cpp sources and macros to transfer to cppcheck command line + # Gather include paths, c, cpp sources and macros to transfer to cppcheck + # command line mbed_includes = ["-I%s" % i for i in mbed_resources.inc_dirs] - mbed_includes.append("-I%s"% str(BUILD_TARGET)) + mbed_includes.append("-I%s"% str(build_target)) mbed_includes.append("-I%s"% str(MBED_COMMON)) mbed_includes.append("-I%s"% str(MBED_API)) mbed_includes.append("-I%s"% str(MBED_HAL)) mbed_c_sources = " ".join(mbed_resources.c_sources) mbed_cpp_sources = " ".join(mbed_resources.cpp_sources) - target_includes = map(str.strip, target_includes) - mbed_includes = map(str.strip, mbed_includes) - target_macros = map(str.strip, target_macros) + target_includes = [inc.strip() for inc in target_includes] + mbed_includes = [inc.strip() for inc in mbed_includes] + target_macros = [mac.strip() for mac in target_macros] - check_cmd = CPPCHECK_CMD - check_cmd += CPPCHECK_MSG_FORMAT + check_cmd = cppcheck_cmd + check_cmd += cppcheck_msg_format check_cmd += target_includes check_cmd += mbed_includes check_cmd += target_macros - # We need to pass some parames via file to avoid "command line too long in some OSs" + # We need to pass some parames via file to avoid "command line too long in + # some OSs" tmp_file = tempfile.NamedTemporaryFile(delete=False) tmp_file.writelines(line + '\n' for line in target_c_sources.split()) tmp_file.writelines(line + '\n' for line in target_cpp_sources.split()) @@ -988,42 +1079,59 @@ def static_analysis_scan(target, toolchain_name, CPPCHECK_CMD, CPPCHECK_MSG_FORM tmp_file.close() check_cmd += ["--file-list=%s"% tmp_file.name] - _stdout, _stderr, _rc = run_cmd_ext(check_cmd) + _stdout, _stderr, _ = run_cmd_ext(check_cmd) if verbose: print _stdout print _stderr -def static_analysis_scan_lib(lib_id, target, toolchain, cppcheck_cmd, cppcheck_msg_format, - options=None, verbose=False, clean=False, macros=None, notify=None, jobs=1, extra_verbose=False): +def static_analysis_scan_lib(lib_id, target, toolchain, cppcheck_cmd, + cppcheck_msg_format, options=None, verbose=False, + clean=False, macros=None, notify=None, jobs=1, + extra_verbose=False): + """Perform static analysis on a library as if it were to be compiled for a + particular target and toolchain combination + """ lib = Library(lib_id) if lib.is_supported(target, toolchain): - static_analysis_scan_library(lib.source_dir, lib.build_dir, target, toolchain, cppcheck_cmd, cppcheck_msg_format, - lib.dependencies, options, - verbose=verbose, clean=clean, macros=macros, notify=notify, jobs=jobs, extra_verbose=extra_verbose) + static_analysis_scan_library( + lib.source_dir, lib.build_dir, target, toolchain, cppcheck_cmd, + cppcheck_msg_format, lib.dependencies, options, verbose=verbose, + clean=clean, macros=macros, notify=notify, jobs=jobs, + extra_verbose=extra_verbose) else: - print 'Library "%s" is not yet supported on target %s with toolchain %s'% (lib_id, target.name, toolchain) + print('Library "%s" is not yet supported on target %s with toolchain %s' + % (lib_id, target.name, toolchain)) -def static_analysis_scan_library(src_paths, build_path, target, toolchain_name, cppcheck_cmd, cppcheck_msg_format, - dependencies_paths=None, options=None, name=None, clean=False, - notify=None, verbose=False, macros=None, jobs=1, extra_verbose=False): - """ Function scans library (or just some set of sources/headers) for staticly detectable defects """ +def static_analysis_scan_library(src_paths, build_path, target, toolchain_name, + cppcheck_cmd, cppcheck_msg_format, + dependencies_paths=None, options=None, + name=None, clean=False, notify=None, + verbose=False, macros=None, jobs=1, + extra_verbose=False): + """ Function scans library (or just some set of sources/headers) for + statically detectable defects + """ if type(src_paths) != ListType: src_paths = [src_paths] for src_path in src_paths: if not exists(src_path): - raise Exception("The library source folder does not exist: %s", src_path) + raise Exception("The library source folder does not exist: %s", + src_path) # Toolchain instance - toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, options, macros=macros, notify=notify, extra_verbose=extra_verbose) + toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, options, + macros=macros, notify=notify, + extra_verbose=extra_verbose) toolchain.VERBOSE = verbose toolchain.jobs = jobs # The first path will give the name to the library name = basename(src_paths[0]) - toolchain.info("Static analysis for library %s (%s, %s)" % (name.upper(), target.name, toolchain_name)) + toolchain.info("Static analysis for library %s (%s, %s)" % + (name.upper(), target.name, toolchain_name)) # Scan Resources resources = [] @@ -1043,7 +1151,8 @@ def static_analysis_scan_library(src_paths, build_path, target, toolchain_name, tmp_path = join(build_path, '.temp', toolchain.obj_path) mkdir(tmp_path) - # Gather include paths, c, cpp sources and macros to transfer to cppcheck command line + # Gather include paths, c, cpp sources and macros to transfer to cppcheck + # command line includes = ["-I%s" % i for i in dependencies_include_dir + src_paths] c_sources = " " cpp_sources = " " @@ -1056,27 +1165,30 @@ def static_analysis_scan_library(src_paths, build_path, target, toolchain_name, c_sources += " ".join(resource.c_sources) + " " cpp_sources += " ".join(resource.cpp_sources) + " " - dependencies_include_dir.extend(toolchain.scan_resources(build_path).inc_dirs) + dependencies_include_dir.extend( + toolchain.scan_resources(build_path).inc_dirs) - includes = map(str.strip, includes) - macros = map(str.strip, macros) + includes = [inc.strip() for inc in includes] + macros = [mac.strip() for mac in macros] check_cmd = cppcheck_cmd check_cmd += cppcheck_msg_format check_cmd += includes check_cmd += macros - # We need to pass some parameters via file to avoid "command line too long in some OSs" - # Temporary file is created to store e.g. cppcheck list of files for command line + # We need to pass some parameters via file to avoid "command line too long + # in some OSs". A temporary file is created to store e.g. cppcheck list of + # files for command line tmp_file = tempfile.NamedTemporaryFile(delete=False) tmp_file.writelines(line + '\n' for line in c_sources.split()) tmp_file.writelines(line + '\n' for line in cpp_sources.split()) tmp_file.close() check_cmd += ["--file-list=%s"% tmp_file.name] - # This will allow us to grab result from both stdio and stderr outputs (so we can show them) - # We assume static code analysis tool is outputting defects on STDERR - _stdout, _stderr, _rc = run_cmd_ext(check_cmd) + # This will allow us to grab result from both stdio and stderr outputs (so + # we can show them) We assume static code analysis tool is outputting + # defects on STDERR + _stdout, _stderr, _ = run_cmd_ext(check_cmd) if verbose: print _stdout print _stderr @@ -1091,10 +1203,10 @@ def print_build_results(result_list, build_name): result += "\n" return result -def print_build_memory_usage_results(report): +def print_build_memory_usage(report): """ Generate result table with memory usage values for build results - Agregates (puts together) reports obtained from self.get_memory_summary() - @param report Report generated during build procedure. See + Aggregates (puts together) reports obtained from self.get_memory_summary() + @param report Report generated during build procedure. See """ from prettytable import PrettyTable columns_text = ['name', 'target', 'toolchain'] @@ -1112,21 +1224,25 @@ def print_build_memory_usage_results(report): for name in report[target][toolchain]: for dlist in report[target][toolchain][name]: for dlistelem in dlist: - # Get 'memory_usage' record and build table with statistics + # Get 'memory_usage' record and build table with + # statistics record = dlist[dlistelem] if 'memory_usage' in record and record['memory_usage']: # Note that summary should be in the last record of - # 'memory_usage' section. This is why we are grabbing - # last "[-1]" record. + # 'memory_usage' section. This is why we are + # grabbing last "[-1]" record. row = [ record['description'], record['target_name'], record['toolchain_name'], - record['memory_usage'][-1]['summary']['static_ram'], + record['memory_usage'][-1]['summary'][ + 'static_ram'], record['memory_usage'][-1]['summary']['stack'], record['memory_usage'][-1]['summary']['heap'], - record['memory_usage'][-1]['summary']['total_ram'], - record['memory_usage'][-1]['summary']['total_flash'], + record['memory_usage'][-1]['summary'][ + 'total_ram'], + record['memory_usage'][-1]['summary'][ + 'total_flash'], ] table.add_row(row) @@ -1135,6 +1251,7 @@ def print_build_memory_usage_results(report): return result def write_build_report(build_report, template_filename, filename): + """write a build report to disk using a template file""" build_report_failing = [] build_report_passing = [] @@ -1148,5 +1265,7 @@ def write_build_report(build_report, template_filename, filename): env.loader = FileSystemLoader('ci_templates') template = env.get_template(template_filename) - with open(filename, 'w+') as f: - f.write(template.render(failing_builds=build_report_failing, passing_builds=build_report_passing)) + with open(filename, 'w+') as placeholder: + placeholder.write(template.render( + failing_builds=build_report_failing, + passing_builds=build_report_passing)) diff --git a/tools/test.py b/tools/test.py index d794bb9bca..6cf205ca11 100644 --- a/tools/test.py +++ b/tools/test.py @@ -29,7 +29,7 @@ sys.path.insert(0, ROOT) from tools.test_api import test_path_to_name, find_tests, print_tests, build_tests, test_spec_from_test_builds from tools.options import get_default_options_parser from tools.build_api import build_project, build_library -from tools.build_api import print_build_memory_usage_results +from tools.build_api import print_build_memory_usage from tools.targets import TARGET_MAP from tools.utils import mkdir, ToolException, NotSupportedException, args_error from tools.test_exporters import ReportExporter, ResultExporterType @@ -230,7 +230,7 @@ if __name__ == '__main__': # Print memory map summary on screen if build_report: print - print print_build_memory_usage_results(build_report) + print print_build_memory_usage(build_report) print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build") status = print_report_exporter.report(build_report)