ARMmbed
/
mbed-os
mirror of https://github.com/ARMmbed/mbed-os.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

IDE build tests with progen

pull/2179/head
Sarah Marsh 2016-07-15 11:14:21 -05:00
parent 38ae4f9289
commit b969fa5bb1
9 changed files with 336 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
requirements.txt
View File

@ -3,7 +3,7 @@ PySerial>=2.7
PrettyTable>=0.7.2
Jinja2>=2.7.3
IntelHex>=1.3
project-generator>=0.9.3,<0.10.0
project-generator>=0.9.7,<0.10.0
project_generator_definitions>=0.2.26,<0.3.0
junit-xml
pyYAML

31
tools/export/__init__.py
View File

@ -21,7 +21,7 @@ import yaml
from tools.utils import mkdir
from tools.export import uvision4, uvision5, codered, gccarm, ds5_5, iar, emblocks, coide, kds, zip, simplicityv3, atmelstudio, sw4stm32, e2studio
from tools.export.exporters import zip_working_directory_and_clean_up, OldLibrariesException
from tools.export.exporters import zip_working_directory_and_clean_up, OldLibrariesException, FailedBuildException
from tools.targets import TARGET_NAMES, EXPORT_MAP, TARGET_MAP
from project_generator_definitions.definitions import ProGenDef
@ -58,7 +58,8 @@ def online_build_url_resolver(url):
def export(project_path, project_name, ide, target, destination='/tmp/',
tempdir=None, clean=True, extra_symbols=None, make_zip=True, sources_relative=False, build_url_resolver=online_build_url_resolver):
tempdir=None, pgen_build = False, clean=True, extra_symbols=None, make_zip=True, sources_relative=False,
build_url_resolver=online_build_url_resolver, progen_build=False):
# Convention: we are using capitals for toolchain and target names
if target is not None:
target = target.upper()
@ -68,8 +69,8 @@ def export(project_path, project_name, ide, target, destination='/tmp/',
use_progen = False
supported = True
report = {'success': False, 'errormsg':''}
report = {'success': False, 'errormsg':'', 'skip': False}
if ide is None or ide == "zip":
# Simple ZIP exporter
try:
@ -83,6 +84,7 @@ def export(project_path, project_name, ide, target, destination='/tmp/',
else:
if ide not in EXPORTERS:
report['errormsg'] = ERROR_MESSAGE_UNSUPPORTED_TOOLCHAIN % (target, ide)
report['skip'] = True
else:
Exporter = EXPORTERS[ide]
target = EXPORT_MAP.get(target, target)
@ -91,24 +93,35 @@ def export(project_path, project_name, ide, target, destination='/tmp/',
use_progen = True
except AttributeError:
pass
if target not in Exporter.TARGETS or Exporter.TOOLCHAIN not in TARGET_MAP[target].supported_toolchains:
supported = False
if use_progen:
if not ProGenDef(ide).is_supported(TARGET_MAP[target].progen['target']):
supported = False
else:
if target not in Exporter.TARGETS:
supported = False
if supported:
# target checked, export
try:
exporter = Exporter(target, tempdir, project_name, build_url_resolver, extra_symbols=extra_symbols, sources_relative=sources_relative)
exporter.scan_and_copy_resources(project_path, tempdir, sources_relative)
exporter.generate()
report['success'] = True
if progen_build:
#try to build with pgen ide builders
try:
exporter.generate(progen_build=True)
report['success'] = True
except FailedBuildException, f:
report['errormsg'] = "Build Failed"
else:
exporter.generate()
report['success'] = True
except OldLibrariesException, e:
report['errormsg'] = ERROR_MESSAGE_NOT_EXPORT_LIBS
else:
report['errormsg'] = ERROR_MESSAGE_UNSUPPORTED_TOOLCHAIN % (target, ide)
report['skip'] = True
zip_path = None
if report['success']:

9
tools/export/exporters.py
View File

@ -21,6 +21,8 @@ from tools.config import Config
class OldLibrariesException(Exception): pass
class FailedBuildException(Exception) : pass
class Exporter(object):
TEMPLATE_DIR = dirname(__file__)
DOT_IN_RELATIVE_PATH = False
@ -107,7 +109,7 @@ class Exporter(object):
}
return project_data
def progen_gen_file(self, tool_name, project_data):
def progen_gen_file(self, tool_name, project_data, progen_build=False):
""" Generate project using ProGen Project API """
settings = ProjectSettings()
project = Project(self.program_name, [project_data], settings)
@ -115,6 +117,11 @@ class Exporter(object):
# thinks it is not dict but a file, and adds them to workspace.
project.project['common']['include_paths'] = self.resources.inc_dirs
project.generate(tool_name, copied=not self.sources_relative)
if progen_build:
print("Project exported, building...")
result = project.build(tool_name)
if result == -1:
raise FailedBuildException("Build Failed")
def __scan_all(self, path):
resources = []

7
tools/export/iar.py
View File

@ -46,7 +46,7 @@ class IAREmbeddedWorkbench(Exporter):
# target is not supported yet
continue
def generate(self):
def generate(self, progen_build=False):
""" Generates the project files """
project_data = self.progen_get_project_data()
tool_specific = {}
@ -75,7 +75,10 @@ class IAREmbeddedWorkbench(Exporter):
# VLA is enabled via template IccAllowVLA
project_data['tool_specific']['iar']['misc']['c_flags'].remove("--vla")
project_data['common']['build_dir'] = os.path.join(project_data['common']['build_dir'], 'iar_arm')
self.progen_gen_file('iar_arm', project_data)
if progen_build:
self.progen_gen_file('iar_arm', project_data, True)
else:
self.progen_gen_file('iar_arm', project_data)
# Currently not used, we should reuse folder_name to create virtual folders
class IarFolder():

7
tools/export/uvision4.py
View File

@ -50,7 +50,7 @@ class Uvision4(Exporter):
def get_toolchain(self):
return TARGET_MAP[self.target].default_toolchain
def generate(self):
def generate(self, progen_build=False):
""" Generates the project files """
project_data = self.progen_get_project_data()
tool_specific = {}
@ -96,4 +96,7 @@ class Uvision4(Exporter):
i += 1
project_data['common']['macros'].append('__ASSERT_MSG')
project_data['common']['build_dir'] = join(project_data['common']['build_dir'], 'uvision4')
self.progen_gen_file('uvision', project_data)
if progen_build:
self.progen_gen_file('uvision', project_data, True)
else:
self.progen_gen_file('uvision', project_data)

7
tools/export/uvision5.py
View File

@ -50,7 +50,7 @@ class Uvision5(Exporter):
def get_toolchain(self):
return TARGET_MAP[self.target].default_toolchain
def generate(self):
def generate(self, progen_build=False):
""" Generates the project files """
project_data = self.progen_get_project_data()
tool_specific = {}
@ -95,4 +95,7 @@ class Uvision5(Exporter):
project_data['common']['macros'].pop(i)
i += 1
project_data['common']['macros'].append('__ASSERT_MSG')
self.progen_gen_file('uvision5', project_data)
if progen_build:
self.progen_gen_file('uvision5', project_data, True)
else:
self.progen_gen_file('uvision5', project_data)

66
tools/project.py
View File

@ -7,16 +7,14 @@ from shutil import move, rmtree
from argparse import ArgumentParser
from os import path
from tools.paths import EXPORT_DIR, EXPORT_WORKSPACE, EXPORT_TMP
from tools.paths import MBED_BASE, MBED_LIBRARIES
from tools.export import export, setup_user_prj, EXPORTERS, mcu_ide_matrix
from tools.utils import args_error, mkdir
from tools.tests import TESTS, Test, TEST_MAP
from tools.paths import EXPORT_DIR
from tools.export import export, EXPORTERS, mcu_ide_matrix
from tools.tests import TESTS, TEST_MAP
from tools.tests import test_known, test_name_known
from tools.targets import TARGET_NAMES
from tools.libraries import LIBRARIES
from utils import argparse_lowercase_type, argparse_uppercase_type, argparse_filestring_type, argparse_many
from utils import argparse_filestring_type, argparse_many
from utils import argparse_force_lowercase_type, argparse_force_uppercase_type
from project_api import setup_project, perform_export, print_results, get_lib_symbols
@ -129,8 +127,6 @@ if __name__ == '__main__':
# Export results
successes = []
failures = []
zip = True
clean = True
# source_dir = use relative paths, otherwise sources are copied
sources_relative = True if options.source_dir else False
@ -138,47 +134,13 @@ if __name__ == '__main__':
for mcu in options.mcu:
# Program Number or name
p, src, ide = options.program, options.source_dir, options.ide
project_dir, project_name, project_temp = setup_project(mcu, ide, p, src, options.build)
if src:
# --source is used to generate IDE files to toolchain directly in the source tree and doesn't generate zip file
project_dir = options.source_dir
project_name = TESTS[p] if p else "Unnamed_project"
project_temp = path.join(options.source_dir[0], 'projectfiles', '%s_%s' % (ide, mcu))
mkdir(project_temp)
lib_symbols = []
if options.macros:
lib_symbols += options.macros
zip = False # don't create zip
clean = False # don't cleanup because we use the actual source tree to generate IDE files
else:
test = Test(p)
# Some libraries have extra macros (called by exporter symbols) to we need to pass
# them to maintain compilation macros integrity between compiled library and
# header files we might use with it
lib_symbols = []
if options.macros:
lib_symbols += options.macros
for lib in LIBRARIES:
if lib['build_dir'] in test.dependencies:
lib_macros = lib.get('macros', None)
if lib_macros is not None:
lib_symbols.extend(lib_macros)
if not options.build:
# Substitute the library builds with the sources
# TODO: Substitute also the other library build paths
if MBED_LIBRARIES in test.dependencies:
test.dependencies.remove(MBED_LIBRARIES)
test.dependencies.append(MBED_BASE)
# Build the project with the same directory structure of the mbed online IDE
project_name = test.id
project_dir = [join(EXPORT_WORKSPACE, project_name)]
project_temp = EXPORT_TMP
setup_user_prj(project_dir[0], test.source_dir, test.dependencies)
zip = src is [] # create zip when no src_dir provided
clean = src is [] # don't clean when source is provided, use acrual source tree for IDE files
# Export to selected toolchain
lib_symbols = get_lib_symbols(options.macros, src, p)
tmp_path, report = export(project_dir, project_name, ide, mcu, project_dir[0], project_temp, clean=clean, make_zip=zip, extra_symbols=lib_symbols, sources_relative=sources_relative)
if report['success']:
if not zip:
@ -191,12 +153,4 @@ if __name__ == '__main__':
failures.append("%s::%s\t%s"% (mcu, ide, report['errormsg']))
# Prints export results
print
if len(successes) > 0:
print "Successful exports:"
for success in successes:
print " * %s"% success
if len(failures) > 0:
print "Failed exports:"
for failure in failures:
print " * %s"% failure
print_results(successes, failures)

111
tools/project_api.py Normal file
View File

@ -0,0 +1,111 @@
import sys
from os.path import join, abspath, dirname, exists, basename
ROOT = abspath(join(dirname(__file__), ".."))
sys.path.insert(0, ROOT)
from tools.paths import EXPORT_WORKSPACE, EXPORT_TMP
from tools.paths import MBED_BASE, MBED_LIBRARIES
from tools.export import export, setup_user_prj
from tools.utils import mkdir
from tools.tests import Test, TEST_MAP, TESTS
from tools.libraries import LIBRARIES
try:
import tools.private_settings as ps
except:
ps = object()
def get_program(n):
p = TEST_MAP[n].n
return p
def get_test(p):
return Test(p)
def get_test_from_name(n):
if not n in TEST_MAP.keys():
# Check if there is an alias for this in private_settings.py
if getattr(ps, "test_alias", None) is not None:
alias = ps.test_alias.get(n, "")
if not alias in TEST_MAP.keys():
return None
else:
n = alias
else:
return None
return get_program(n)
def get_lib_symbols(macros, src, program):
# Some libraries have extra macros (called by exporter symbols) to we need to pass
# them to maintain compilation macros integrity between compiled library and
# header files we might use with it
lib_symbols = []
if macros:
lib_symbols += macros
if src:
return lib_symbols
test = get_test(program)
for lib in LIBRARIES:
if lib['build_dir'] in test.dependencies:
lib_macros = lib.get('macros', None)
if lib_macros is not None:
lib_symbols.extend(lib_macros)
def setup_project(mcu, ide, program=None, source_dir=None, build=None):
# Some libraries have extra macros (called by exporter symbols) to we need to pass
# them to maintain compilation macros integrity between compiled library and
# header files we might use with it
if source_dir:
# --source is used to generate IDE files to toolchain directly in the source tree and doesn't generate zip file
project_dir = source_dir
project_name = TESTS[program] if program else "Unnamed_Project"
project_temp = join(source_dir[0], 'projectfiles', '%s_%s' % (ide, mcu))
mkdir(project_temp)
else:
test = get_test(program)
if not build:
# Substitute the library builds with the sources
# TODO: Substitute also the other library build paths
if MBED_LIBRARIES in test.dependencies:
test.dependencies.remove(MBED_LIBRARIES)
test.dependencies.append(MBED_BASE)
# Build the project with the same directory structure of the mbed online IDE
project_name = test.id
project_dir = [join(EXPORT_WORKSPACE, project_name)]
project_temp = EXPORT_TMP
setup_user_prj(project_dir[0], test.source_dir, test.dependencies)
return project_dir, project_name, project_temp
def perform_export(dir, name, ide, mcu, temp, clean=False, zip=False, lib_symbols='',
sources_relative=False, progen_build=False):
tmp_path, report = export(dir, name, ide, mcu, dir[0], temp, clean=clean,
make_zip=zip, extra_symbols=lib_symbols, sources_relative=sources_relative,
progen_build=progen_build)
return tmp_path, report
def print_results(successes, failures, skips = []):
print
if len(successes) > 0:
print "Successful: "
for success in successes:
print " * %s" % success
if len(failures) > 0:
print "Failed: "
for failure in failures:
print " * %s" % failure
if len(skips) > 0:
print "Skipped: "
for skip in skips:
print " * %s" % skip

169
tools/test/export/build_test.py Normal file
View File

@ -0,0 +1,169 @@
#!/usr/bin/env python
"""
mbed SDK
Copyright (c) 2011-2013 ARM Limited
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import sys
import argparse
import os
import shutil
from os.path import join, abspath, dirname, exists, basename
r=dirname(__file__)
ROOT = abspath(join(r, "..","..",".."))
sys.path.insert(0, ROOT)
from tools.export import EXPORTERS
from tools.targets import TARGET_NAMES, TARGET_MAP
from tools.project_api import setup_project, perform_export, print_results, get_test_from_name, get_lib_symbols
from project_generator_definitions.definitions import ProGenDef
from tools.utils import args_error
class ProgenBuildTest():
def __init__(self, desired_ides, targets):
#map of targets and the ides that can build programs for them
self.target_ides = {}
for target in targets:
self.target_ides[target] =[]
for ide in desired_ides:
if target in EXPORTERS[ide].TARGETS:
#target is supported by ide
self.target_ides[target].append(ide)
if len(self.target_ides[target]) == 0:
del self.target_ides[target]
@staticmethod
def get_pgen_targets(ides):
#targets supported by pgen and desired ides for tests
targs = []
for ide in ides:
for target in TARGET_NAMES:
if target not in targs and hasattr(TARGET_MAP[target],'progen') \
and ProGenDef(ide).is_supported(TARGET_MAP[target].progen['target']):
targs.append(target)
return targs
@staticmethod
def handle_project_files(project_dir, mcu, test, tool, clean=False):
log = ''
if tool == 'uvision' or tool == 'uvision5':
log = os.path.join(project_dir,"build","build_log.txt")
elif tool == 'iar':
log = os.path.join(project_dir, 'build_log.txt')
try:
with open(log, 'r') as f:
print f.read()
except:
return
prefix = "_".join([test, mcu, tool])
log_name = os.path.join(os.path.dirname(project_dir), prefix+"_log.txt")
#check if a log already exists for this platform+test+ide
if os.path.exists(log_name):
#delete it if so
os.remove(log_name)
os.rename(log, log_name)
if clean:
shutil.rmtree(project_dir, ignore_errors=True)
return
def generate_and_build(self, tests, clean=False):
#build results
successes = []
failures = []
skips = []
for mcu, ides in self.target_ides.items():
for test in tests:
#resolve name alias
test = get_test_from_name(test)
for ide in ides:
lib_symbols = get_lib_symbols(None, None, test)
project_dir, project_name, project_temp = setup_project(mcu, ide, test)
dest_dir = os.path.dirname(project_temp)
destination = os.path.join(dest_dir,"_".join([project_name, mcu, ide]))
tmp_path, report = perform_export(project_dir, project_name, ide, mcu, destination,
lib_symbols=lib_symbols, progen_build = True)
if report['success']:
successes.append("build for %s::%s\t%s" % (mcu, ide, project_name))
elif report['skip']:
skips.append("%s::%s\t%s" % (mcu, ide, project_name))
else:
failures.append("%s::%s\t%s for %s" % (mcu, ide, report['errormsg'], project_name))
ProgenBuildTest.handle_project_files(destination, mcu, project_name, ide, clean)
return successes, failures, skips
if __name__ == '__main__':
accepted_ides = ["iar", "uvision", "uvision5"]
accepted_targets = sorted(ProgenBuildTest.get_pgen_targets(accepted_ides))
default_tests = ["MBED_BLINKY"]
parser = argparse.ArgumentParser(description = "Test progen builders. Leave any flag off to run with all possible options.")
parser.add_argument("-i", "--IDEs",
nargs = '+',
dest="ides",
help="tools you wish to perfrom build tests. (%s)" % ', '.join(accepted_ides),
default = accepted_ides)
parser.add_argument("-n",
nargs='+',
dest="tests",
help="names of desired test programs",
default = default_tests)
parser.add_argument("-m", "--mcus",
nargs='+',
dest ="targets",
help="generate project for the given MCUs (%s)" % '\n '.join(accepted_targets),
default = accepted_targets)
parser.add_argument("-c", "--clean",
dest="clean",
action = "store_true",
help="clean up the exported project files",
default=False)
options = parser.parse_args()
tests = options.tests
ides = [ide.lower() for ide in options.ides]
targets = [target.upper() for target in options.targets]
if any(get_test_from_name(test) is None for test in tests):
args_error(parser, "[ERROR] test name not recognized")
if any(target not in accepted_targets for target in targets):
args_error(parser, "[ERROR] mcu must be one of the following:\n %s" % '\n '.join(accepted_targets))
if any(ide not in accepted_ides for ide in ides):
args_error(parser, "[ERROR] ide must be in %s" % ', '.join(accepted_ides))
build_test = ProgenBuildTest(ides, targets)
successes, failures, skips = build_test.generate_and_build(tests, options.clean)
print_results(successes, failures, skips)
sys.exit(len(failures))