From b969fa5bb12077cee3103324d54e2da4e165883f Mon Sep 17 00:00:00 2001 From: Sarah Marsh Date: Fri, 15 Jul 2016 11:14:21 -0500 Subject: [PATCH] IDE build tests with progen --- requirements.txt | 2 +- tools/export/__init__.py | 31 ++++-- tools/export/exporters.py | 9 +- tools/export/iar.py | 7 +- tools/export/uvision4.py | 7 +- tools/export/uvision5.py | 7 +- tools/project.py | 66 ++----------- tools/project_api.py | 111 +++++++++++++++++++++ tools/test/export/build_test.py | 169 ++++++++++++++++++++++++++++++++ 9 files changed, 336 insertions(+), 73 deletions(-) create mode 100644 tools/project_api.py create mode 100644 tools/test/export/build_test.py diff --git a/requirements.txt b/requirements.txt index 53efae0aef..fcec27e282 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ PySerial>=2.7 PrettyTable>=0.7.2 Jinja2>=2.7.3 IntelHex>=1.3 -project-generator>=0.9.3,<0.10.0 +project-generator>=0.9.7,<0.10.0 project_generator_definitions>=0.2.26,<0.3.0 junit-xml pyYAML diff --git a/tools/export/__init__.py b/tools/export/__init__.py index e766bd2708..8ae7c69070 100644 --- a/tools/export/__init__.py +++ b/tools/export/__init__.py @@ -21,7 +21,7 @@ import yaml from tools.utils import mkdir from tools.export import uvision4, uvision5, codered, gccarm, ds5_5, iar, emblocks, coide, kds, zip, simplicityv3, atmelstudio, sw4stm32, e2studio -from tools.export.exporters import zip_working_directory_and_clean_up, OldLibrariesException +from tools.export.exporters import zip_working_directory_and_clean_up, OldLibrariesException, FailedBuildException from tools.targets import TARGET_NAMES, EXPORT_MAP, TARGET_MAP from project_generator_definitions.definitions import ProGenDef @@ -58,7 +58,8 @@ def online_build_url_resolver(url): def export(project_path, project_name, ide, target, destination='/tmp/', - tempdir=None, clean=True, extra_symbols=None, make_zip=True, sources_relative=False, build_url_resolver=online_build_url_resolver): + tempdir=None, pgen_build = False, clean=True, extra_symbols=None, make_zip=True, sources_relative=False, + build_url_resolver=online_build_url_resolver, progen_build=False): # Convention: we are using capitals for toolchain and target names if target is not None: target = target.upper() @@ -68,8 +69,8 @@ def export(project_path, project_name, ide, target, destination='/tmp/', use_progen = False supported = True - report = {'success': False, 'errormsg':''} - + report = {'success': False, 'errormsg':'', 'skip': False} + if ide is None or ide == "zip": # Simple ZIP exporter try: @@ -83,6 +84,7 @@ def export(project_path, project_name, ide, target, destination='/tmp/', else: if ide not in EXPORTERS: report['errormsg'] = ERROR_MESSAGE_UNSUPPORTED_TOOLCHAIN % (target, ide) + report['skip'] = True else: Exporter = EXPORTERS[ide] target = EXPORT_MAP.get(target, target) @@ -91,24 +93,35 @@ def export(project_path, project_name, ide, target, destination='/tmp/', use_progen = True except AttributeError: pass + + if target not in Exporter.TARGETS or Exporter.TOOLCHAIN not in TARGET_MAP[target].supported_toolchains: + supported = False + if use_progen: if not ProGenDef(ide).is_supported(TARGET_MAP[target].progen['target']): supported = False - else: - if target not in Exporter.TARGETS: - supported = False if supported: # target checked, export try: exporter = Exporter(target, tempdir, project_name, build_url_resolver, extra_symbols=extra_symbols, sources_relative=sources_relative) exporter.scan_and_copy_resources(project_path, tempdir, sources_relative) - exporter.generate() - report['success'] = True + if progen_build: + #try to build with pgen ide builders + try: + exporter.generate(progen_build=True) + report['success'] = True + except FailedBuildException, f: + report['errormsg'] = "Build Failed" + else: + exporter.generate() + report['success'] = True except OldLibrariesException, e: report['errormsg'] = ERROR_MESSAGE_NOT_EXPORT_LIBS + else: report['errormsg'] = ERROR_MESSAGE_UNSUPPORTED_TOOLCHAIN % (target, ide) + report['skip'] = True zip_path = None if report['success']: diff --git a/tools/export/exporters.py b/tools/export/exporters.py index deb8926ddd..2273bf0482 100644 --- a/tools/export/exporters.py +++ b/tools/export/exporters.py @@ -21,6 +21,8 @@ from tools.config import Config class OldLibrariesException(Exception): pass +class FailedBuildException(Exception) : pass + class Exporter(object): TEMPLATE_DIR = dirname(__file__) DOT_IN_RELATIVE_PATH = False @@ -107,7 +109,7 @@ class Exporter(object): } return project_data - def progen_gen_file(self, tool_name, project_data): + def progen_gen_file(self, tool_name, project_data, progen_build=False): """ Generate project using ProGen Project API """ settings = ProjectSettings() project = Project(self.program_name, [project_data], settings) @@ -115,6 +117,11 @@ class Exporter(object): # thinks it is not dict but a file, and adds them to workspace. project.project['common']['include_paths'] = self.resources.inc_dirs project.generate(tool_name, copied=not self.sources_relative) + if progen_build: + print("Project exported, building...") + result = project.build(tool_name) + if result == -1: + raise FailedBuildException("Build Failed") def __scan_all(self, path): resources = [] diff --git a/tools/export/iar.py b/tools/export/iar.py index 16a626a9ba..f9f18d4199 100644 --- a/tools/export/iar.py +++ b/tools/export/iar.py @@ -46,7 +46,7 @@ class IAREmbeddedWorkbench(Exporter): # target is not supported yet continue - def generate(self): + def generate(self, progen_build=False): """ Generates the project files """ project_data = self.progen_get_project_data() tool_specific = {} @@ -75,7 +75,10 @@ class IAREmbeddedWorkbench(Exporter): # VLA is enabled via template IccAllowVLA project_data['tool_specific']['iar']['misc']['c_flags'].remove("--vla") project_data['common']['build_dir'] = os.path.join(project_data['common']['build_dir'], 'iar_arm') - self.progen_gen_file('iar_arm', project_data) + if progen_build: + self.progen_gen_file('iar_arm', project_data, True) + else: + self.progen_gen_file('iar_arm', project_data) # Currently not used, we should reuse folder_name to create virtual folders class IarFolder(): diff --git a/tools/export/uvision4.py b/tools/export/uvision4.py index 17c62f2903..94dc3a8ff4 100644 --- a/tools/export/uvision4.py +++ b/tools/export/uvision4.py @@ -50,7 +50,7 @@ class Uvision4(Exporter): def get_toolchain(self): return TARGET_MAP[self.target].default_toolchain - def generate(self): + def generate(self, progen_build=False): """ Generates the project files """ project_data = self.progen_get_project_data() tool_specific = {} @@ -96,4 +96,7 @@ class Uvision4(Exporter): i += 1 project_data['common']['macros'].append('__ASSERT_MSG') project_data['common']['build_dir'] = join(project_data['common']['build_dir'], 'uvision4') - self.progen_gen_file('uvision', project_data) + if progen_build: + self.progen_gen_file('uvision', project_data, True) + else: + self.progen_gen_file('uvision', project_data) diff --git a/tools/export/uvision5.py b/tools/export/uvision5.py index 50ebf91906..4e789712d0 100644 --- a/tools/export/uvision5.py +++ b/tools/export/uvision5.py @@ -50,7 +50,7 @@ class Uvision5(Exporter): def get_toolchain(self): return TARGET_MAP[self.target].default_toolchain - def generate(self): + def generate(self, progen_build=False): """ Generates the project files """ project_data = self.progen_get_project_data() tool_specific = {} @@ -95,4 +95,7 @@ class Uvision5(Exporter): project_data['common']['macros'].pop(i) i += 1 project_data['common']['macros'].append('__ASSERT_MSG') - self.progen_gen_file('uvision5', project_data) + if progen_build: + self.progen_gen_file('uvision5', project_data, True) + else: + self.progen_gen_file('uvision5', project_data) diff --git a/tools/project.py b/tools/project.py index cabdfdf7b9..96f86d2f64 100644 --- a/tools/project.py +++ b/tools/project.py @@ -7,16 +7,14 @@ from shutil import move, rmtree from argparse import ArgumentParser from os import path -from tools.paths import EXPORT_DIR, EXPORT_WORKSPACE, EXPORT_TMP -from tools.paths import MBED_BASE, MBED_LIBRARIES -from tools.export import export, setup_user_prj, EXPORTERS, mcu_ide_matrix -from tools.utils import args_error, mkdir -from tools.tests import TESTS, Test, TEST_MAP +from tools.paths import EXPORT_DIR +from tools.export import export, EXPORTERS, mcu_ide_matrix +from tools.tests import TESTS, TEST_MAP from tools.tests import test_known, test_name_known from tools.targets import TARGET_NAMES -from tools.libraries import LIBRARIES -from utils import argparse_lowercase_type, argparse_uppercase_type, argparse_filestring_type, argparse_many +from utils import argparse_filestring_type, argparse_many from utils import argparse_force_lowercase_type, argparse_force_uppercase_type +from project_api import setup_project, perform_export, print_results, get_lib_symbols @@ -129,8 +127,6 @@ if __name__ == '__main__': # Export results successes = [] failures = [] - zip = True - clean = True # source_dir = use relative paths, otherwise sources are copied sources_relative = True if options.source_dir else False @@ -138,47 +134,13 @@ if __name__ == '__main__': for mcu in options.mcu: # Program Number or name p, src, ide = options.program, options.source_dir, options.ide + project_dir, project_name, project_temp = setup_project(mcu, ide, p, src, options.build) - if src: - # --source is used to generate IDE files to toolchain directly in the source tree and doesn't generate zip file - project_dir = options.source_dir - project_name = TESTS[p] if p else "Unnamed_project" - project_temp = path.join(options.source_dir[0], 'projectfiles', '%s_%s' % (ide, mcu)) - mkdir(project_temp) - lib_symbols = [] - if options.macros: - lib_symbols += options.macros - zip = False # don't create zip - clean = False # don't cleanup because we use the actual source tree to generate IDE files - else: - test = Test(p) - - # Some libraries have extra macros (called by exporter symbols) to we need to pass - # them to maintain compilation macros integrity between compiled library and - # header files we might use with it - lib_symbols = [] - if options.macros: - lib_symbols += options.macros - for lib in LIBRARIES: - if lib['build_dir'] in test.dependencies: - lib_macros = lib.get('macros', None) - if lib_macros is not None: - lib_symbols.extend(lib_macros) - - if not options.build: - # Substitute the library builds with the sources - # TODO: Substitute also the other library build paths - if MBED_LIBRARIES in test.dependencies: - test.dependencies.remove(MBED_LIBRARIES) - test.dependencies.append(MBED_BASE) - - # Build the project with the same directory structure of the mbed online IDE - project_name = test.id - project_dir = [join(EXPORT_WORKSPACE, project_name)] - project_temp = EXPORT_TMP - setup_user_prj(project_dir[0], test.source_dir, test.dependencies) + zip = src is [] # create zip when no src_dir provided + clean = src is [] # don't clean when source is provided, use acrual source tree for IDE files # Export to selected toolchain + lib_symbols = get_lib_symbols(options.macros, src, p) tmp_path, report = export(project_dir, project_name, ide, mcu, project_dir[0], project_temp, clean=clean, make_zip=zip, extra_symbols=lib_symbols, sources_relative=sources_relative) if report['success']: if not zip: @@ -191,12 +153,4 @@ if __name__ == '__main__': failures.append("%s::%s\t%s"% (mcu, ide, report['errormsg'])) # Prints export results - print - if len(successes) > 0: - print "Successful exports:" - for success in successes: - print " * %s"% success - if len(failures) > 0: - print "Failed exports:" - for failure in failures: - print " * %s"% failure + print_results(successes, failures) diff --git a/tools/project_api.py b/tools/project_api.py new file mode 100644 index 0000000000..f0bfb04795 --- /dev/null +++ b/tools/project_api.py @@ -0,0 +1,111 @@ +import sys +from os.path import join, abspath, dirname, exists, basename +ROOT = abspath(join(dirname(__file__), "..")) +sys.path.insert(0, ROOT) + +from tools.paths import EXPORT_WORKSPACE, EXPORT_TMP +from tools.paths import MBED_BASE, MBED_LIBRARIES +from tools.export import export, setup_user_prj +from tools.utils import mkdir +from tools.tests import Test, TEST_MAP, TESTS +from tools.libraries import LIBRARIES + +try: + import tools.private_settings as ps +except: + ps = object() + + +def get_program(n): + p = TEST_MAP[n].n + return p + + +def get_test(p): + return Test(p) + + +def get_test_from_name(n): + if not n in TEST_MAP.keys(): + # Check if there is an alias for this in private_settings.py + if getattr(ps, "test_alias", None) is not None: + alias = ps.test_alias.get(n, "") + if not alias in TEST_MAP.keys(): + return None + else: + n = alias + else: + return None + return get_program(n) + + +def get_lib_symbols(macros, src, program): + # Some libraries have extra macros (called by exporter symbols) to we need to pass + # them to maintain compilation macros integrity between compiled library and + # header files we might use with it + lib_symbols = [] + if macros: + lib_symbols += macros + if src: + return lib_symbols + test = get_test(program) + for lib in LIBRARIES: + if lib['build_dir'] in test.dependencies: + lib_macros = lib.get('macros', None) + if lib_macros is not None: + lib_symbols.extend(lib_macros) + + +def setup_project(mcu, ide, program=None, source_dir=None, build=None): + + # Some libraries have extra macros (called by exporter symbols) to we need to pass + # them to maintain compilation macros integrity between compiled library and + # header files we might use with it + if source_dir: + # --source is used to generate IDE files to toolchain directly in the source tree and doesn't generate zip file + project_dir = source_dir + project_name = TESTS[program] if program else "Unnamed_Project" + project_temp = join(source_dir[0], 'projectfiles', '%s_%s' % (ide, mcu)) + mkdir(project_temp) + else: + test = get_test(program) + if not build: + # Substitute the library builds with the sources + # TODO: Substitute also the other library build paths + if MBED_LIBRARIES in test.dependencies: + test.dependencies.remove(MBED_LIBRARIES) + test.dependencies.append(MBED_BASE) + + # Build the project with the same directory structure of the mbed online IDE + project_name = test.id + project_dir = [join(EXPORT_WORKSPACE, project_name)] + project_temp = EXPORT_TMP + setup_user_prj(project_dir[0], test.source_dir, test.dependencies) + + return project_dir, project_name, project_temp + + +def perform_export(dir, name, ide, mcu, temp, clean=False, zip=False, lib_symbols='', + sources_relative=False, progen_build=False): + + tmp_path, report = export(dir, name, ide, mcu, dir[0], temp, clean=clean, + make_zip=zip, extra_symbols=lib_symbols, sources_relative=sources_relative, + progen_build=progen_build) + return tmp_path, report + + +def print_results(successes, failures, skips = []): + print + if len(successes) > 0: + print "Successful: " + for success in successes: + print " * %s" % success + if len(failures) > 0: + print "Failed: " + for failure in failures: + print " * %s" % failure + if len(skips) > 0: + print "Skipped: " + for skip in skips: + print " * %s" % skip + diff --git a/tools/test/export/build_test.py b/tools/test/export/build_test.py new file mode 100644 index 0000000000..ec80fcfd00 --- /dev/null +++ b/tools/test/export/build_test.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python +""" +mbed SDK +Copyright (c) 2011-2013 ARM Limited + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + + +import sys +import argparse +import os +import shutil +from os.path import join, abspath, dirname, exists, basename +r=dirname(__file__) +ROOT = abspath(join(r, "..","..","..")) +sys.path.insert(0, ROOT) + +from tools.export import EXPORTERS +from tools.targets import TARGET_NAMES, TARGET_MAP +from tools.project_api import setup_project, perform_export, print_results, get_test_from_name, get_lib_symbols +from project_generator_definitions.definitions import ProGenDef +from tools.utils import args_error + + +class ProgenBuildTest(): + def __init__(self, desired_ides, targets): + #map of targets and the ides that can build programs for them + self.target_ides = {} + for target in targets: + self.target_ides[target] =[] + for ide in desired_ides: + if target in EXPORTERS[ide].TARGETS: + #target is supported by ide + self.target_ides[target].append(ide) + if len(self.target_ides[target]) == 0: + del self.target_ides[target] + + + @staticmethod + def get_pgen_targets(ides): + #targets supported by pgen and desired ides for tests + targs = [] + for ide in ides: + for target in TARGET_NAMES: + if target not in targs and hasattr(TARGET_MAP[target],'progen') \ + and ProGenDef(ide).is_supported(TARGET_MAP[target].progen['target']): + targs.append(target) + return targs + + @staticmethod + def handle_project_files(project_dir, mcu, test, tool, clean=False): + log = '' + if tool == 'uvision' or tool == 'uvision5': + log = os.path.join(project_dir,"build","build_log.txt") + elif tool == 'iar': + log = os.path.join(project_dir, 'build_log.txt') + try: + with open(log, 'r') as f: + print f.read() + except: + return + + prefix = "_".join([test, mcu, tool]) + log_name = os.path.join(os.path.dirname(project_dir), prefix+"_log.txt") + + #check if a log already exists for this platform+test+ide + if os.path.exists(log_name): + #delete it if so + os.remove(log_name) + os.rename(log, log_name) + + if clean: + shutil.rmtree(project_dir, ignore_errors=True) + return + + def generate_and_build(self, tests, clean=False): + + #build results + successes = [] + failures = [] + skips = [] + for mcu, ides in self.target_ides.items(): + for test in tests: + #resolve name alias + test = get_test_from_name(test) + for ide in ides: + lib_symbols = get_lib_symbols(None, None, test) + project_dir, project_name, project_temp = setup_project(mcu, ide, test) + + dest_dir = os.path.dirname(project_temp) + destination = os.path.join(dest_dir,"_".join([project_name, mcu, ide])) + + tmp_path, report = perform_export(project_dir, project_name, ide, mcu, destination, + lib_symbols=lib_symbols, progen_build = True) + + if report['success']: + successes.append("build for %s::%s\t%s" % (mcu, ide, project_name)) + elif report['skip']: + skips.append("%s::%s\t%s" % (mcu, ide, project_name)) + else: + failures.append("%s::%s\t%s for %s" % (mcu, ide, report['errormsg'], project_name)) + + ProgenBuildTest.handle_project_files(destination, mcu, project_name, ide, clean) + return successes, failures, skips + + +if __name__ == '__main__': + accepted_ides = ["iar", "uvision", "uvision5"] + accepted_targets = sorted(ProgenBuildTest.get_pgen_targets(accepted_ides)) + default_tests = ["MBED_BLINKY"] + + parser = argparse.ArgumentParser(description = "Test progen builders. Leave any flag off to run with all possible options.") + parser.add_argument("-i", "--IDEs", + nargs = '+', + dest="ides", + help="tools you wish to perfrom build tests. (%s)" % ', '.join(accepted_ides), + default = accepted_ides) + + parser.add_argument("-n", + nargs='+', + dest="tests", + help="names of desired test programs", + default = default_tests) + + parser.add_argument("-m", "--mcus", + nargs='+', + dest ="targets", + help="generate project for the given MCUs (%s)" % '\n '.join(accepted_targets), + default = accepted_targets) + + parser.add_argument("-c", "--clean", + dest="clean", + action = "store_true", + help="clean up the exported project files", + default=False) + + options = parser.parse_args() + + tests = options.tests + ides = [ide.lower() for ide in options.ides] + targets = [target.upper() for target in options.targets] + + if any(get_test_from_name(test) is None for test in tests): + args_error(parser, "[ERROR] test name not recognized") + + if any(target not in accepted_targets for target in targets): + args_error(parser, "[ERROR] mcu must be one of the following:\n %s" % '\n '.join(accepted_targets)) + + if any(ide not in accepted_ides for ide in ides): + args_error(parser, "[ERROR] ide must be in %s" % ', '.join(accepted_ides)) + + build_test = ProgenBuildTest(ides, targets) + successes, failures, skips = build_test.generate_and_build(tests, options.clean) + print_results(successes, failures, skips) + sys.exit(len(failures)) + + +