Merge pull request #10021 from bridadan/uvision_postbuild_regions

Enable post build bootloader merging in uvision

tools/export/__init__.py

@@ -136,7 +136,7 @@ def get_exporter_toolchain(ide):
 
 
 def generate_project_files(resources, export_path, target, name, toolchain, ide,
-                           macros=None):
+                           zip, macros=None):
     """Generate the project files for a project
 
     Positional arguments:
@@ -147,13 +147,14 @@ def generate_project_files(resources, export_path, target, name, toolchain, ide,
     toolchain - a toolchain class that corresponds to the toolchain used by the
       IDE or makefile
     ide - IDE name to export to
+    zip - True if the exported project will be zipped
 
     Optional arguments:
     macros - additional macros that should be defined within the exported
       project
     """
     exporter_cls, _ = get_exporter_toolchain(ide)
-    exporter = exporter_cls(target, export_path, name, toolchain,
+    exporter = exporter_cls(target, export_path, name, toolchain, zip,
                             extra_symbols=macros, resources=resources)
     exporter.generate()
     files = exporter.generated_files
@@ -278,9 +279,9 @@ def export_project(src_paths, export_path, target, ide, libraries_paths=None,
     if toolchain.config.name:
         name = toolchain.config.name
 
-    files, exporter = generate_project_files(resources, export_path,
+    files, exporter = generate_project_files(
-                                             target, name, toolchain, ide,
+        resources, export_path, target, name, toolchain, ide, zip_proj, macros=macros
-                                             macros=macros)
+    )
     if zip_proj:
         resources.add_features(ALLOWED_FEATURES)
         if isinstance(zip_proj, basestring):

tools/export/exporters.py

@@ -73,7 +73,7 @@ class Exporter(object):
     CLEAN_FILES = ("GettingStarted.html",)
 
 
-    def __init__(self, target, export_dir, project_name, toolchain,
+    def __init__(self, target, export_dir, project_name, toolchain, zip,
                  extra_symbols=None, resources=None):
         """Initialize an instance of class exporter
         Positional arguments:
@@ -81,6 +81,7 @@ class Exporter(object):
         export_dir    - the directory of the exported project files
         project_name  - the name of the project
         toolchain     - an instance of class toolchain
+        zip           - True if the exported project will be zipped
 
         Keyword arguments:
         extra_symbols - a list of extra macros for the toolchain
@@ -94,6 +95,7 @@ class Exporter(object):
         self.jinja_environment = Environment(loader=jinja_loader)
         resources.win_to_unix()
         self.resources = resources
+        self.zip = zip
         self.generated_files = []
         getting_started_name = "GettingStarted.html"
         dot_mbed_name = ".mbed"

tools/export/uvision/__init__.py

@@ -2,13 +2,14 @@ from __future__ import print_function, absolute_import
 from builtins import str
 
 import os
-from os.path import normpath, exists, dirname
+from os.path import normpath, exists, dirname, join, abspath, relpath
 import ntpath
 import copy
 from collections import namedtuple
 import shutil
 from subprocess import Popen, PIPE
 import re
+import json
 
 from tools.resources import FileType
 from tools.targets import TARGET_MAP, CORE_ARCH
@@ -243,7 +244,31 @@ class Uvision(Exporter):
             'include_paths': ';'.join(self.filter_dot(d) for d in
                                       self.resources.inc_dirs),
             'device': DeviceUvision(self.target),
+            'postbuild_step_active': 0,
         }
+
+        if self.toolchain.config.has_regions and not self.zip:
+            # Serialize region information
+            export_info = {}
+            restrict_size = getattr(self.toolchain.config.target, "restrict_size")
+            if restrict_size:
+                export_info["target"] = {
+                    "restrict_size": restrict_size
+                }
+
+            binary_path = "BUILD/{}.hex".format(self.project_name)
+            region_list = list(self.toolchain.config.regions)
+            export_info["region_list"] = [
+                r._replace(filename=binary_path) if r.active else r for r in region_list
+            ]
+            # Enable the post build step
+            postbuild_script_path = join(relpath(dirname(__file__)), "postbuild.py")
+            ctx['postbuild_step'] = (
+                'python {} "$K\\" "#L"'.format(postbuild_script_path)
+            )
+            ctx['postbuild_step_active'] = 1
+            ctx['export_info'] = json.dumps(export_info, indent=4)
+
         sct_file_ref = self.resources.get_file_refs(FileType.LD_SCRIPT)[0]
         sct_file_ref = self.toolchain.correct_scatter_shebang(
             sct_file_ref, dirname(sct_file_ref.name)
@@ -271,10 +296,29 @@ class Uvision(Exporter):
             'uvision/uvision_debug.tmpl', ctx, self.project_name + ".uvoptx"
         )
 
+        if ctx['postbuild_step_active']:
+            self.gen_file(
+                'uvision/debug_init.ini', ctx, 'debug_init.ini'
+            )
+            self.gen_file(
+                'uvision/flash_init.ini', ctx, 'flash_init.ini'
+            )
+            self.gen_file(
+                'uvision/export_info.tmpl', ctx, 'export_info.json'
+            )
+
     @staticmethod
     def clean(project_name):
         os.remove(project_name + ".uvprojx")
         os.remove(project_name + ".uvoptx")
+
+        if exists("export_info.json"):
+            os.remove("export_info.json")
+        if exists("debug_init.ini"):
+            os.remove("debug_init.ini")
+        if exists("flash_init.ini"):
+            os.remove("flash_init.ini")
+
         # legacy .build directory cleaned if exists
         if exists('.build'):
             shutil.rmtree('.build')

tools/export/uvision/debug_init.ini

@@ -0,0 +1,2 @@
+// Add the debug symbols
+Load "$L@L.axf" NOCODE

tools/export/uvision/export_info.tmpl

@@ -0,0 +1 @@
+{{export_info}}

tools/export/uvision/flash_init.ini

@@ -0,0 +1,2 @@
+// Flash device
+Load "$L@L_combined.hex" INCREMENTAL

tools/export/uvision/postbuild.py

@@ -0,0 +1,109 @@
+#! /usr/bin/env python2
+"""
+mbed SDK
+Copyright (c) 2019 ARM Limited
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+LIBRARIES BUILD
+"""
+from __future__ import print_function, division, absolute_import
+
+import sys
+from os.path import join, abspath, dirname, normpath
+import json
+from shutil import copyfile
+from argparse import ArgumentParser
+from copy import copy
+
+# Be sure that the tools directory is in the search path
+ROOT = abspath(join(dirname(__file__), "../../../"))
+sys.path.insert(0, ROOT)
+
+from tools.regions import merge_region_list, UPDATE_WHITELIST
+from tools.notifier.term import TerminalNotifier
+from tools.config import Region
+from tools.utils import split_path, run_cmd_ext, generate_update_filename
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+
+    parser.add_argument(
+        "toolchain_path",
+        help="Path to the Keil folder"
+    )
+
+    parser.add_argument(
+        "linker_output",
+        help="Path to the built axf file"
+    )
+
+    options = parser.parse_args()
+    axf_file = normpath(options.linker_output)
+    output_directory, output_name, output_ext = split_path(axf_file)
+    hex_file = join(output_directory, output_name + ".hex")
+    combined_hex_file = join(output_directory, output_name + "_combined.hex")
+
+    command = [
+        join(normpath(options.toolchain_path), "ARM/ARMCC/bin/fromelf.exe"),
+        "--i32", "--output", hex_file, axf_file
+    ]
+    stdout, stderr, retcode = run_cmd_ext(command)
+
+    if retcode:
+        err_msg = (
+            "Failed to convert axf to hex.\r\n"
+            "Command: {}\r\n"
+            "retcode: {}\r\n"
+            "stdout: {}\r\n"
+            "stderr: {}"
+        ).format(command, retcode, stdout, stderr)
+        raise Exception(err_msg)
+
+    with open(join("export_info.json"), "r") as export_info_file:
+        export_info_data = json.load(export_info_file)
+
+    region_list = [Region(*r) for r in export_info_data.get("region_list", [])]
+
+    for index, region in enumerate(copy(region_list)):
+        if region.name == "application":
+            region_data = region._asdict()
+            region_data["filename"] = hex_file
+            region_list[index] = Region(**region_data)
+            break
+    else:
+        raise Exception("No application region found")
+
+    notify = TerminalNotifier()
+    restrict_size = export_info_data.get("target", {}).get("restrict_size")
+    merge_region_list(
+        region_list, combined_hex_file, notify, restrict_size=restrict_size
+    )
+
+    update_regions = [
+        r for r in region_list if r.name in UPDATE_WHITELIST
+    ]
+
+    if update_regions:
+        update_res = normpath(
+            join(
+                output_directory,
+                generate_update_filename(output_name, None)
+            )
+        )
+        merge_region_list(
+            update_regions, update_res, notify, restrict_size=restrict_size
+        )
+
+    sys.exit(0)

tools/export/uvision/uvision.tmpl

@@ -82,9 +82,9 @@
             <nStopB2X>0</nStopB2X>
           </BeforeMake>
           <AfterMake>
-            <RunUserProg1>0</RunUserProg1>
+            <RunUserProg1>{{postbuild_step_active}}</RunUserProg1>
             <RunUserProg2>0</RunUserProg2>
-            <UserProg1Name></UserProg1Name>
+            <UserProg1Name>{{postbuild_step if postbuild_step_active else ''}}</UserProg1Name>
             <UserProg2Name></UserProg2Name>
             <UserProg1Dos16Mode>0</UserProg1Dos16Mode>
             <UserProg2Dos16Mode>0</UserProg2Dos16Mode>
@@ -182,7 +182,7 @@
           <bUseTDR>1</bUseTDR>
           <Flash2>BIN\UL2CM3.DLL</Flash2>
           <Flash3></Flash3>
-          <Flash4></Flash4>
+          <Flash4>{{'./flash_init.ini' if postbuild_step_active else ''}}</Flash4>
           <pFcarmOut></pFcarmOut>
           <pFcarmGrp></pFcarmGrp>
           <pFcArmRoot></pFcArmRoot>

tools/export/uvision/uvision_debug.tmpl

@@ -6,11 +6,20 @@
 		<ToolsetNumber>0x4</ToolsetNumber>
 		<ToolsetName>ARM-ADS</ToolsetName>
 		<TargetOption>
+			{% if postbuild_step_active %}
+			<OPTTT>
+				<RunAbUc>1</RunAbUc>
+			</OPTTT>
+			{% endif %}
 			<DebugOpt>
 				<uSim>0</uSim>
 				<uTrg>1</uTrg>
 				<nTsel>11</nTsel>
 				<pMon>{{device.debug_interface.bin_loc}}</pMon>
+				{% if postbuild_step_active %}
+				<tIfile>./debug_init.ini</tIfile>
+				<tLdApp>0</tLdApp>
+				{% endif %}
 			</DebugOpt>
 			<TargetDriverDllRegistry>
 				<SetRegEntry>

tools/regions.py

@@ -106,8 +106,8 @@ def merge_region_list(
         region_list,
         destination,
         notify,
-        config,
+        padding=b'\xFF',
-        padding=b'\xFF'
+        restrict_size=None
 ):
     """Merge the region_list into a single image
 
@@ -115,6 +115,7 @@ def merge_region_list(
     region_list - list of regions, which should contain filenames
     destination - file name to write all regions to
     padding - bytes to fill gaps with
+    restrict_size - check to ensure a region fits within the given size
     """
     merged = IntelHex()
     _, format = splitext(destination)
@@ -145,7 +146,7 @@ def merge_region_list(
             # Normally, we assume that part.maxddr() can be beyond
             # end of rom. If the size is restricted with config, don't
             # allow this.
-            if config.target.restrict_size is not None:
+            if restrict_size is not None:
                 part_size = (part.maxaddr() - part.minaddr()) + 1
                 if part_size > region.size:
                     raise ToolException(

tools/test/build_api/build_api_test.py

@@ -247,11 +247,9 @@ class BuildApiTests(unittest.TestCase):
                          "prepare_toolchain was called with an incorrect app_config")
 
     @patch('tools.regions.intelhex_offset')
-    @patch('tools.config')
+    def test_merge_region_no_fit(self, mock_intelhex_offset):
-    def test_merge_region_no_fit(self, mock_config, mock_intelhex_offset):
         """
         Test that merge_region_list call fails when part size overflows region size.
-        :param mock_config: config object that is mocked.
         :param mock_intelhex_offset: mocked intel_hex_offset call.
         :return:
         """
@@ -267,15 +265,12 @@ class BuildApiTests(unittest.TestCase):
         region_list = [region_application, region_post_application]
         # path to store the result in, should not get used as we expect exception.
         res = "./"
-        mock_config.target.restrict_size = 90000
         toolexception = False
 
         try:
-            merge_region_list(region_list, res, notify, mock_config)
+            merge_region_list(region_list, res, notify, restrict_size=90000)
         except ToolException:
             toolexception = True
-        except Exception as e:
-            print("%s %s" % (e.message, e.args))
 
         self.assertTrue(toolexception, "Expected ToolException not raised")
 

tools/toolchains/mbed_toolchain.py

@@ -663,7 +663,10 @@ class mbedToolchain:
         region_list = [r._replace(filename=binary) if r.active else r
                        for r in region_list]
         res = "{}.{}".format(join(self.build_dir, name), ext)
-        merge_region_list(region_list, res, self.notify, self.config)
+        merge_region_list(
+            region_list, res, self.notify,
+            restrict_size=self.config.target.restrict_size
+        )
         update_regions = [
             r for r in region_list if r.name in UPDATE_WHITELIST
         ]
@@ -676,7 +679,7 @@ class mbedToolchain:
                 update_regions,
                 update_res,
                 self.notify,
-                self.config
+                restrict_size=self.config.target.restrict_size
             )
             return res, update_res
         else: