Merge pull request #4110 from theotherjimmy/build-metadata

Add --build-data switch to mbed compile and test

tools/build_api.py

@@ -17,12 +17,16 @@ limitations under the License.
 
 import re
 import tempfile
+import datetime
+import uuid
 from types import ListType
 from shutil import rmtree
 from os.path import join, exists, dirname, basename, abspath, normpath, splitext
+from os.path import relpath
 from os import linesep, remove, makedirs
 from time import time
 from intelhex import IntelHex
+from json import load, dump
 
 from tools.utils import mkdir, run_cmd, run_cmd_ext, NotSupportedException,\
     ToolException, InvalidReleaseTargetException, intelhex_offset
@@ -102,6 +106,8 @@ def add_result_to_report(report, result):
     report - the report to append to
     result - the result to append
     """
+    result["date"] = datetime.datetime.utcnow().isoformat()
+    result["uuid"] = str(uuid.uuid1())
     target = result["target_name"]
     toolchain = result["toolchain_name"]
     id_name = result['id']
@@ -552,6 +558,9 @@ def build_project(src_paths, build_path, target, toolchain_name,
             cur_result["output"] = toolchain.get_output() + memap_table
             cur_result["result"] = "OK"
             cur_result["memory_usage"] = toolchain.map_outputs
+            cur_result["bin"] = res
+            cur_result["elf"] = splitext(res)[0] + ".elf"
+            cur_result.update(toolchain.report)
 
             add_result_to_report(report, cur_result)
 
@@ -653,6 +662,7 @@ def build_library(src_paths, build_path, target, toolchain_name,
         prep_report(report, toolchain.target.name, toolchain_name, id_name)
         cur_result = create_result(toolchain.target.name, toolchain_name,
                                    id_name, description)
+        cur_result['type'] = 'library'
         if properties != None:
             prep_properties(properties, toolchain.target.name, toolchain_name,
                             vendor_label)
@@ -1362,3 +1372,24 @@ def write_build_report(build_report, template_filename, filename):
         placeholder.write(template.render(
             failing_builds=build_report_failing,
             passing_builds=build_report_passing))
+
+
+def merge_build_data(filename, toolchain_report, app_type):
+    path_to_file = dirname(abspath(filename))
+    try:
+        build_data = load(open(filename))
+    except (IOError, ValueError):
+        build_data = {'builds': []}
+    for tgt in toolchain_report.values():
+        for tc in tgt.values():
+            for project in tc.values():
+                for build in project:
+                    try:
+                        build[0]['elf'] = relpath(build[0]['elf'], path_to_file)
+                        build[0]['bin'] = relpath(build[0]['bin'], path_to_file)
+                    except KeyError:
+                        pass
+                    if 'type' not in build[0]:
+                        build[0]['type'] = app_type
+                    build_data['builds'].append(build[0])
+    dump(build_data, open(filename, "wb"), indent=4, separators=(',', ': '))

tools/config/__init__.py

@@ -20,7 +20,7 @@ import os
 from os.path import dirname, abspath, exists
 import sys
 from collections import namedtuple
-from os.path import splitext
+from os.path import splitext, relpath
 from intelhex import IntelHex
 from jinja2 import FileSystemLoader, StrictUndefined
 from jinja2.environment import Environment
@@ -389,25 +389,25 @@ class Config(object):
         search for a configuration file).
         """
         config_errors = []
-        app_config_location = app_config
-        if app_config_location is None:
+        self.app_config_location = app_config
+        if self.app_config_location is None:
             for directory in top_level_dirs or []:
                 full_path = os.path.join(directory, self.__mbed_app_config_name)
                 if os.path.isfile(full_path):
-                    if app_config_location is not None:
+                    if self.app_config_location is not None:
                         raise ConfigException("Duplicate '%s' file in '%s' and '%s'"
                                               % (self.__mbed_app_config_name,
-                                                 app_config_location, full_path))
+                                                 self.app_config_location, full_path))
                     else:
-                        app_config_location = full_path
+                        self.app_config_location = full_path
         try:
-            self.app_config_data = json_file_to_dict(app_config_location) \
-                                   if app_config_location else {}
+            self.app_config_data = json_file_to_dict(self.app_config_location) \
+                                   if self.app_config_location else {}
         except ValueError as exc:
             self.app_config_data = {}
             config_errors.append(
                 ConfigException("Could not parse mbed app configuration from %s"
-                                % app_config_location))
+                                % self.app_config_location))
 
         # Check the keys in the application configuration data
         unknown_keys = set(self.app_config_data.keys()) - \
@@ -529,6 +529,11 @@ class Config(object):
             raise ConfigException("Not enough memory on device to fit all "
                                   "application regions")
 
+    @property
+    def report(self):
+        return {'app_config': self.app_config_location,
+                'library_configs': map(relpath, self.processed_configs.keys())}
+
     def _process_config_and_overrides(self, data, params, unit_name, unit_kind):
         """Process "config_parameters" and "target_config_overrides" into a
         given dictionary

tools/make.py

@@ -23,6 +23,7 @@ import json
 from time import sleep
 from shutil import copy
 from os.path import join, abspath, dirname
+from json import load, dump
 
 # Be sure that the tools directory is in the search path
 ROOT = abspath(join(dirname(__file__), ".."))
@@ -48,6 +49,7 @@ from tools.build_api import build_project
 from tools.build_api import mcu_toolchain_matrix
 from tools.build_api import mcu_toolchain_list
 from tools.build_api import mcu_target_list
+from tools.build_api import merge_build_data
 from utils import argparse_filestring_type
 from utils import argparse_many
 from utils import argparse_dir_not_parent
@@ -177,6 +179,11 @@ if __name__ == '__main__':
                       default=False,
                       help="Link with mbed test library")
 
+    parser.add_argument("--build-data",
+                        dest="build_data",
+                        default=None,
+                        help="Dump build_data to this file")
+
     # Specify a different linker script
     parser.add_argument("-l", "--linker", dest="linker_script",
                       type=argparse_filestring_type,
@@ -249,6 +256,7 @@ if __name__ == '__main__':
                            %(toolchain,search_path))
 
     # Test
+    build_data_blob = {} if options.build_data else None
     for test_no in p:
         test = Test(test_no)
         if options.automated is not None:    test.automated = options.automated
@@ -287,6 +295,7 @@ if __name__ == '__main__':
                                      clean=options.clean,
                                      verbose=options.verbose,
                                      notify=notify,
+                                     report=build_data_blob,
                                      silent=options.silent,
                                      macros=options.macros,
                                      jobs=options.jobs,
@@ -342,3 +351,5 @@ if __name__ == '__main__':
                 print "[ERROR] %s" % str(e)
             
             sys.exit(1)
+    if options.build_data:
+        merge_build_data(options.build_data, build_data_blob, "application")

tools/test.py

@@ -31,6 +31,7 @@ from tools.test_api import test_path_to_name, find_tests, print_tests, build_tes
 from tools.options import get_default_options_parser, extract_profile
 from tools.build_api import build_project, build_library
 from tools.build_api import print_build_memory_usage
+from tools.build_api import merge_build_data
 from tools.targets import TARGET_MAP
 from tools.utils import mkdir, ToolException, NotSupportedException, args_error
 from tools.test_exporters import ReportExporter, ResultExporterType
@@ -88,6 +89,10 @@ if __name__ == '__main__':
         
         parser.add_argument("--build-report-junit", dest="build_report_junit",
                           default=None, help="Destination path for a build report in the JUnit xml format")
+        parser.add_argument("--build-data",
+                            dest="build_data",
+                            default=None,
+                            help="Dump build_data to this file")
         
         parser.add_argument("-v", "--verbose",
                           action="store_true",
@@ -175,17 +180,13 @@ if __name__ == '__main__':
             profile = extract_profile(parser, options, toolchain)
             try:
                 # Build sources
-                build_library(base_source_paths, options.build_dir, mcu, toolchain,
-                                                jobs=options.jobs,
-                                                clean=options.clean,
-                                                report=build_report,
-                                                properties=build_properties,
-                                                name="mbed-build",
-                                                macros=options.macros,
-                                                verbose=options.verbose,
-                                                notify=notify,
-                                                archive=False,
-                                                app_config=options.app_config,
+                build_library(base_source_paths, options.build_dir, mcu,
+                              toolchain, jobs=options.jobs,
+                              clean=options.clean, report=build_report,
+                              properties=build_properties, name="mbed-build",
+                              macros=options.macros, verbose=options.verbose,
+                              notify=notify, archive=False,
+                              app_config=options.app_config,
                               build_profile=profile)
 
                 library_build_success = True
@@ -245,6 +246,8 @@ if __name__ == '__main__':
 
             print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build")
             status = print_report_exporter.report(build_report)
+            if options.build_data:
+                merge_build_data(options.build_data, build_report, "test")
 
             if status:
                 sys.exit(0)

tools/toolchains/__init__.py

@@ -1393,6 +1393,19 @@ class mbedToolchain:
     def get_config_macros(self):
         return Config.config_to_macros(self.config_data) if self.config_data else []
 
+    @property
+    def report(self):
+        to_ret = {}
+        to_ret['c_compiler'] = {'flags': copy(self.flags['c']),
+                                'symbols': self.get_symbols()}
+        to_ret['cxx_compiler'] = {'flags': copy(self.flags['cxx']),
+                                  'symbols': self.get_symbols()}
+        to_ret['assembler'] = {'flags': copy(self.flags['asm']),
+                               'symbols': self.get_symbols(True)}
+        to_ret['linker'] = {'flags': copy(self.flags['ld'])}
+        to_ret.update(self.config.report)
+        return to_ret
+
 from tools.settings import ARM_PATH
 from tools.settings import GCC_ARM_PATH
 from tools.settings import IAR_PATH