From da9c10be487c9d27794323ad5329ce31623ee7ce Mon Sep 17 00:00:00 2001 From: Jimmy Brisson Date: Tue, 4 Apr 2017 11:35:00 -0500 Subject: [PATCH 1/7] Expand build reports in the tools --- tools/build_api.py | 5 +++++ tools/config/__init__.py | 23 ++++++++++++++--------- tools/toolchains/__init__.py | 13 +++++++++++++ 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/tools/build_api.py b/tools/build_api.py index fe0b4c0e8b..dad25b99a2 100644 --- a/tools/build_api.py +++ b/tools/build_api.py @@ -17,6 +17,7 @@ limitations under the License. import re import tempfile +import datetime from types import ListType from shutil import rmtree from os.path import join, exists, dirname, basename, abspath, normpath, splitext @@ -102,6 +103,7 @@ def add_result_to_report(report, result): report - the report to append to result - the result to append """ + result["date"] = str(datetime.datetime.now()) target = result["target_name"] toolchain = result["toolchain_name"] id_name = result['id'] @@ -550,6 +552,9 @@ def build_project(src_paths, build_path, target, toolchain_name, cur_result["output"] = toolchain.get_output() + memap_table cur_result["result"] = "OK" cur_result["memory_usage"] = toolchain.map_outputs + cur_result["bin"] = res + cur_result["elf"] = splitext(res)[0] + ".elf" + cur_result.update(toolchain.report) add_result_to_report(report, cur_result) diff --git a/tools/config/__init__.py b/tools/config/__init__.py index fd4a2d3d86..57d9db0f0e 100644 --- a/tools/config/__init__.py +++ b/tools/config/__init__.py @@ -20,7 +20,7 @@ import os from os.path import dirname, abspath import sys from collections import namedtuple -from os.path import splitext +from os.path import splitext, relpath from intelhex import IntelHex from jinja2 import FileSystemLoader, StrictUndefined from jinja2.environment import Environment @@ -389,25 +389,25 @@ class Config(object): search for a configuration file). """ config_errors = [] - app_config_location = app_config - if app_config_location is None: + self.app_config_location = app_config + if self.app_config_location is None: for directory in top_level_dirs or []: full_path = os.path.join(directory, self.__mbed_app_config_name) if os.path.isfile(full_path): - if app_config_location is not None: + if self.app_config_location is not None: raise ConfigException("Duplicate '%s' file in '%s' and '%s'" % (self.__mbed_app_config_name, - app_config_location, full_path)) + self.app_config_location, full_path)) else: - app_config_location = full_path + self.app_config_location = full_path try: - self.app_config_data = json_file_to_dict(app_config_location) \ - if app_config_location else {} + self.app_config_data = json_file_to_dict(self.app_config_location) \ + if self.app_config_location else {} except ValueError as exc: self.app_config_data = {} config_errors.append( ConfigException("Could not parse mbed app configuration from %s" - % app_config_location)) + % self.app_config_location)) # Check the keys in the application configuration data unknown_keys = set(self.app_config_data.keys()) - \ @@ -527,6 +527,11 @@ class Config(object): raise ConfigException("Not enough memory on device to fit all " "application regions") + @property + def report(self): + return {'app_config': self.app_config_location, + 'library_configs': map(relpath, self.processed_configs.keys())} + def _process_config_and_overrides(self, data, params, unit_name, unit_kind): """Process "config_parameters" and "target_config_overrides" into a given dictionary diff --git a/tools/toolchains/__init__.py b/tools/toolchains/__init__.py index 7b69b8230c..d63dc5f2ec 100644 --- a/tools/toolchains/__init__.py +++ b/tools/toolchains/__init__.py @@ -1393,6 +1393,19 @@ class mbedToolchain: def get_config_macros(self): return Config.config_to_macros(self.config_data) if self.config_data else [] + @property + def report(self): + to_ret = {} + to_ret['c_compiler'] = {'flags': copy(self.flags['c']), + 'symbols': self.get_symbols()} + to_ret['cxx_compiler'] = {'flags': copy(self.flags['cxx']), + 'symbols': self.get_symbols()} + to_ret['assembler'] = {'flags': copy(self.flags['asm']), + 'symbols': self.get_symbols(True)} + to_ret['linker'] = {'flags': copy(self.flags['ld'])} + to_ret.update(self.config.report) + return to_ret + from tools.settings import ARM_PATH from tools.settings import GCC_ARM_PATH from tools.settings import IAR_PATH From 035ddfb057e62ffdef2e21f4069e77677f666d96 Mon Sep 17 00:00:00 2001 From: Jimmy Brisson Date: Tue, 4 Apr 2017 11:35:37 -0500 Subject: [PATCH 2/7] Add an option to make.py to dump build metadata --- tools/make.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tools/make.py b/tools/make.py index 8822e34878..35de3b258f 100644 --- a/tools/make.py +++ b/tools/make.py @@ -23,6 +23,7 @@ import json from time import sleep from shutil import copy from os.path import join, abspath, dirname +from json import load, dump # Be sure that the tools directory is in the search path ROOT = abspath(join(dirname(__file__), "..")) @@ -54,6 +55,18 @@ from utils import argparse_dir_not_parent from tools.toolchains import mbedToolchain, TOOLCHAIN_CLASSES, TOOLCHAIN_PATHS from tools.settings import CLI_COLOR_MAP +def merge_metadata(filename, toolchain_report): + try: + metadata = load(open(filename)) + except (IOError, ValueError): + metadata = {'builds': []} + for tgt in toolchain_report.values(): + for tc in tgt.values(): + for project in tc.values(): + for build in project: + metadata['builds'].append(build[0]) + dump(metadata, open(filename, "wb"), indent=4, separators=(',', ': ')) + if __name__ == '__main__': # Parse Options parser = get_default_options_parser(add_app_config=True) @@ -177,6 +190,11 @@ if __name__ == '__main__': default=False, help="Link with mbed test library") + parser.add_argument("--metadata", + dest="metadata", + default=None, + help="Dump metadata to this file") + # Specify a different linker script parser.add_argument("-l", "--linker", dest="linker_script", type=argparse_filestring_type, @@ -249,6 +267,7 @@ if __name__ == '__main__': %(toolchain,search_path)) # Test + metadata_blob = {} if options.metadata else None for test_no in p: test = Test(test_no) if options.automated is not None: test.automated = options.automated @@ -287,6 +306,7 @@ if __name__ == '__main__': clean=options.clean, verbose=options.verbose, notify=notify, + report=metadata_blob, silent=options.silent, macros=options.macros, jobs=options.jobs, @@ -342,3 +362,5 @@ if __name__ == '__main__': print "[ERROR] %s" % str(e) sys.exit(1) + if options.metadata: + merge_metadata(options.metadata, metadata_blob) From aeb6109717bd5df91f723cf08cb86702f563c643 Mon Sep 17 00:00:00 2001 From: Jimmy Brisson Date: Wed, 5 Apr 2017 16:40:01 -0500 Subject: [PATCH 3/7] Rename switch to --build-data --- tools/make.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tools/make.py b/tools/make.py index 35de3b258f..c053875ce6 100644 --- a/tools/make.py +++ b/tools/make.py @@ -55,17 +55,17 @@ from utils import argparse_dir_not_parent from tools.toolchains import mbedToolchain, TOOLCHAIN_CLASSES, TOOLCHAIN_PATHS from tools.settings import CLI_COLOR_MAP -def merge_metadata(filename, toolchain_report): +def merge_build_data(filename, toolchain_report): try: - metadata = load(open(filename)) + build_data = load(open(filename)) except (IOError, ValueError): - metadata = {'builds': []} + build_data = {'builds': []} for tgt in toolchain_report.values(): for tc in tgt.values(): for project in tc.values(): for build in project: - metadata['builds'].append(build[0]) - dump(metadata, open(filename, "wb"), indent=4, separators=(',', ': ')) + build_data['builds'].append(build[0]) + dump(build_data, open(filename, "wb"), indent=4, separators=(',', ': ')) if __name__ == '__main__': # Parse Options @@ -190,10 +190,10 @@ if __name__ == '__main__': default=False, help="Link with mbed test library") - parser.add_argument("--metadata", - dest="metadata", + parser.add_argument("--build-data", + dest="build_data", default=None, - help="Dump metadata to this file") + help="Dump build_data to this file") # Specify a different linker script parser.add_argument("-l", "--linker", dest="linker_script", @@ -267,7 +267,7 @@ if __name__ == '__main__': %(toolchain,search_path)) # Test - metadata_blob = {} if options.metadata else None + build_data_blob = {} if options.build_data else None for test_no in p: test = Test(test_no) if options.automated is not None: test.automated = options.automated @@ -306,7 +306,7 @@ if __name__ == '__main__': clean=options.clean, verbose=options.verbose, notify=notify, - report=metadata_blob, + report=build_data_blob, silent=options.silent, macros=options.macros, jobs=options.jobs, @@ -362,5 +362,5 @@ if __name__ == '__main__': print "[ERROR] %s" % str(e) sys.exit(1) - if options.metadata: - merge_metadata(options.metadata, metadata_blob) + if options.build_data: + merge_build_data(options.build_data, build_data_blob) From 808279911cf21c97a0170e7aa96c8dbcbcb807d7 Mon Sep 17 00:00:00 2001 From: Jimmy Brisson Date: Thu, 6 Apr 2017 11:17:54 -0500 Subject: [PATCH 4/7] Add --build-data flag to mbed test --- tools/build_api.py | 21 +++++++++++++++++++++ tools/make.py | 13 +------------ tools/test.py | 25 ++++++++++++++----------- 3 files changed, 36 insertions(+), 23 deletions(-) diff --git a/tools/build_api.py b/tools/build_api.py index dad25b99a2..dfdcd00082 100644 --- a/tools/build_api.py +++ b/tools/build_api.py @@ -21,9 +21,11 @@ import datetime from types import ListType from shutil import rmtree from os.path import join, exists, dirname, basename, abspath, normpath, splitext +from os.path import relpath from os import linesep, remove, makedirs from time import time from intelhex import IntelHex +from json import load, dump from tools.utils import mkdir, run_cmd, run_cmd_ext, NotSupportedException,\ ToolException, InvalidReleaseTargetException, intelhex_offset @@ -1365,3 +1367,22 @@ def write_build_report(build_report, template_filename, filename): placeholder.write(template.render( failing_builds=build_report_failing, passing_builds=build_report_passing)) + + +def merge_build_data(filename, toolchain_report): + path_to_file = dirname(abspath(filename)) + try: + build_data = load(open(filename)) + except (IOError, ValueError): + build_data = {'builds': []} + for tgt in toolchain_report.values(): + for tc in tgt.values(): + for project in tc.values(): + for build in project: + try: + build[0]['elf'] = relpath(build[0]['elf'], path_to_file) + build[0]['bin'] = relpath(build[0]['bin'], path_to_file) + except KeyError: + pass + build_data['builds'].append(build[0]) + dump(build_data, open(filename, "wb"), indent=4, separators=(',', ': ')) diff --git a/tools/make.py b/tools/make.py index c053875ce6..6512e8b3ed 100644 --- a/tools/make.py +++ b/tools/make.py @@ -49,24 +49,13 @@ from tools.build_api import build_project from tools.build_api import mcu_toolchain_matrix from tools.build_api import mcu_toolchain_list from tools.build_api import mcu_target_list +from tools.build_api import merge_build_data from utils import argparse_filestring_type from utils import argparse_many from utils import argparse_dir_not_parent from tools.toolchains import mbedToolchain, TOOLCHAIN_CLASSES, TOOLCHAIN_PATHS from tools.settings import CLI_COLOR_MAP -def merge_build_data(filename, toolchain_report): - try: - build_data = load(open(filename)) - except (IOError, ValueError): - build_data = {'builds': []} - for tgt in toolchain_report.values(): - for tc in tgt.values(): - for project in tc.values(): - for build in project: - build_data['builds'].append(build[0]) - dump(build_data, open(filename, "wb"), indent=4, separators=(',', ': ')) - if __name__ == '__main__': # Parse Options parser = get_default_options_parser(add_app_config=True) diff --git a/tools/test.py b/tools/test.py index ec153d499a..722b0cb197 100644 --- a/tools/test.py +++ b/tools/test.py @@ -31,6 +31,7 @@ from tools.test_api import test_path_to_name, find_tests, print_tests, build_tes from tools.options import get_default_options_parser, extract_profile from tools.build_api import build_project, build_library from tools.build_api import print_build_memory_usage +from tools.build_api import merge_build_data from tools.targets import TARGET_MAP from tools.utils import mkdir, ToolException, NotSupportedException, args_error from tools.test_exporters import ReportExporter, ResultExporterType @@ -88,6 +89,10 @@ if __name__ == '__main__': parser.add_argument("--build-report-junit", dest="build_report_junit", default=None, help="Destination path for a build report in the JUnit xml format") + parser.add_argument("--build-data", + dest="build_data", + default=None, + help="Dump build_data to this file") parser.add_argument("-v", "--verbose", action="store_true", @@ -175,17 +180,13 @@ if __name__ == '__main__': profile = extract_profile(parser, options, toolchain) try: # Build sources - build_library(base_source_paths, options.build_dir, mcu, toolchain, - jobs=options.jobs, - clean=options.clean, - report=build_report, - properties=build_properties, - name="mbed-build", - macros=options.macros, - verbose=options.verbose, - notify=notify, - archive=False, - app_config=options.app_config, + build_library(base_source_paths, options.build_dir, mcu, + toolchain, jobs=options.jobs, + clean=options.clean, report=build_report, + properties=build_properties, name="mbed-build", + macros=options.macros, verbose=options.verbose, + notify=notify, archive=False, + app_config=options.app_config, build_profile=profile) library_build_success = True @@ -245,6 +246,8 @@ if __name__ == '__main__': print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build") status = print_report_exporter.report(build_report) + if options.build_data: + merge_build_data(options.build_data, build_report) if status: sys.exit(0) From 6ef384b7fee24d67f557f20a1bf5c03a71c5cb4d Mon Sep 17 00:00:00 2001 From: Jimmy Brisson Date: Fri, 7 Apr 2017 12:38:49 -0500 Subject: [PATCH 5/7] Add type to build_data.json --- tools/build_api.py | 5 ++++- tools/make.py | 2 +- tools/test.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/build_api.py b/tools/build_api.py index dfdcd00082..f7c6fb469d 100644 --- a/tools/build_api.py +++ b/tools/build_api.py @@ -658,6 +658,7 @@ def build_library(src_paths, build_path, target, toolchain_name, prep_report(report, toolchain.target.name, toolchain_name, id_name) cur_result = create_result(toolchain.target.name, toolchain_name, id_name, description) + cur_result['type'] = 'library' if properties != None: prep_properties(properties, toolchain.target.name, toolchain_name, vendor_label) @@ -1369,7 +1370,7 @@ def write_build_report(build_report, template_filename, filename): passing_builds=build_report_passing)) -def merge_build_data(filename, toolchain_report): +def merge_build_data(filename, toolchain_report, app_type): path_to_file = dirname(abspath(filename)) try: build_data = load(open(filename)) @@ -1384,5 +1385,7 @@ def merge_build_data(filename, toolchain_report): build[0]['bin'] = relpath(build[0]['bin'], path_to_file) except KeyError: pass + if 'type' not in build[0]: + build[0]['type'] = app_type build_data['builds'].append(build[0]) dump(build_data, open(filename, "wb"), indent=4, separators=(',', ': ')) diff --git a/tools/make.py b/tools/make.py index 6512e8b3ed..93c9fc8d61 100644 --- a/tools/make.py +++ b/tools/make.py @@ -352,4 +352,4 @@ if __name__ == '__main__': sys.exit(1) if options.build_data: - merge_build_data(options.build_data, build_data_blob) + merge_build_data(options.build_data, build_data_blob, "application") diff --git a/tools/test.py b/tools/test.py index 722b0cb197..0058c97d37 100644 --- a/tools/test.py +++ b/tools/test.py @@ -247,7 +247,7 @@ if __name__ == '__main__': print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build") status = print_report_exporter.report(build_report) if options.build_data: - merge_build_data(options.build_data, build_report) + merge_build_data(options.build_data, build_report, "test") if status: sys.exit(0) From 05dce1206ecdcda047fd94023a23b1fe9128b489 Mon Sep 17 00:00:00 2001 From: Jimmy Brisson Date: Fri, 7 Apr 2017 10:54:41 -0500 Subject: [PATCH 6/7] Change date key to UTC ISO 8601 format --- tools/build_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build_api.py b/tools/build_api.py index f7c6fb469d..7c82e10eb5 100644 --- a/tools/build_api.py +++ b/tools/build_api.py @@ -105,7 +105,7 @@ def add_result_to_report(report, result): report - the report to append to result - the result to append """ - result["date"] = str(datetime.datetime.now()) + result["date"] = datetime.datetime.utcnow().isoformat() target = result["target_name"] toolchain = result["toolchain_name"] id_name = result['id'] From c4c6e139b658a17c5067cb053d2c70c10f364a59 Mon Sep 17 00:00:00 2001 From: Jimmy Brisson Date: Fri, 7 Apr 2017 11:03:02 -0500 Subject: [PATCH 7/7] Add UUID to all builds --- tools/build_api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/build_api.py b/tools/build_api.py index 7c82e10eb5..58234ca887 100644 --- a/tools/build_api.py +++ b/tools/build_api.py @@ -18,6 +18,7 @@ limitations under the License. import re import tempfile import datetime +import uuid from types import ListType from shutil import rmtree from os.path import join, exists, dirname, basename, abspath, normpath, splitext @@ -106,6 +107,7 @@ def add_result_to_report(report, result): result - the result to append """ result["date"] = datetime.datetime.utcnow().isoformat() + result["uuid"] = str(uuid.uuid1()) target = result["target_name"] toolchain = result["toolchain_name"] id_name = result['id']