Support for response files for ARMCC, GCC and IAR archiving (static library)

Support for compiling static libraries via build.py
Support for build.py --no-archive flag which compiles static library as multiple objects
Change default number of jobs when compiling to 0 (auto)
Fix for relative path issue when compiling, which flattened the output whenever absolute path is passed to --build (make.py and build.py)
Fix for temporary files when pre-processing of assembler files with ARMCC
Fix issue with response files where one of the elements is empty string

tools/build.py

@@ -30,7 +30,7 @@ sys.path.insert(0, ROOT)
 from tools.toolchains import TOOLCHAINS
 from tools.targets import TARGET_NAMES, TARGET_MAP
 from tools.options import get_default_options_parser
-from tools.build_api import build_mbed_libs, build_lib
+from tools.build_api import build_library, build_mbed_libs, build_lib
 from tools.build_api import mcu_toolchain_matrix
 from tools.build_api import static_analysis_scan, static_analysis_scan_lib, static_analysis_scan_library
 from tools.build_api import print_build_results
@@ -42,6 +42,15 @@ if __name__ == '__main__':
     # Parse Options
     parser = get_default_options_parser()
 
+    parser.add_option("--source", dest="source_dir",
+                      default=None, help="The source (input) directory", action="append")
+
+    parser.add_option("--build", dest="build_dir",
+                      default=None, help="The build (output) directory")
+
+    parser.add_option("--no-archive", dest="no_archive", action="store_true",
+                      default=False, help="Do not produce archive (.ar) file, but rather .o")
+
     # Extra libraries
     parser.add_option("-r", "--rtos",
                       action="store_true",
@@ -119,7 +128,7 @@ if __name__ == '__main__':
                       help='For some commands you can use filter to filter out results')
 
     parser.add_option("-j", "--jobs", type="int", dest="jobs",
-                      default=1, help="Number of concurrent jobs (default 1). Use 0 for auto based on host machine's number of CPUs")
+                      default=0, help="Number of concurrent jobs. Default: 0/auto (based on host machine's number of CPUs)")
 
     parser.add_option("-v", "--verbose",
                       action="store_true",
@@ -224,13 +233,14 @@ if __name__ == '__main__':
                 tt_id = "%s::%s" % (toolchain, target)
                 try:
                     mcu = TARGET_MAP[target]
-                    lib_build_res = build_mbed_libs(mcu, toolchain,
+                    lib_build_res = build_library(options.source_dir, options.build_dir, mcu, toolchain,
                                                     options=options.options,
                                                     extra_verbose=options.extra_verbose_notify,
                                                     verbose=options.verbose,
                                                     silent=options.silent,
                                                     jobs=options.jobs,
                                                     clean=options.clean,
+                                                    archive=(not options.no_archive),
                                                     macros=options.macros)
                     for lib_id in libraries:
                         build_lib(lib_id, mcu, toolchain,

tools/build_api.py

@@ -22,7 +22,7 @@ import colorama
 
 from types import ListType
 from shutil import rmtree
-from os.path import join, exists, basename
+from os.path import join, exists, basename, abspath
 from os import getcwd
 from time import time
 
@@ -188,7 +188,7 @@ def build_project(src_path, build_path, target, toolchain_name,
 
 
 def build_library(src_paths, build_path, target, toolchain_name,
-         dependencies_paths=None, options=None, name=None, clean=False,
+         dependencies_paths=None, options=None, name=None, clean=False, archive=True,
          notify=None, verbose=False, macros=None, inc_dirs=None, inc_dirs_ext=None,
          jobs=1, silent=False, report=None, properties=None, extra_verbose=False):
     """ src_path: the path of the source directory
@@ -206,7 +206,10 @@ def build_library(src_paths, build_path, target, toolchain_name,
         src_paths = [src_paths]
 
     # The first path will give the name to the library
-    name = basename(src_paths[0])
+    project_name = basename(src_paths[0] if src_paths[0] != "." and src_paths[0] != "./" else getcwd())
+    if name is None:
+        # We will use default project name based on project folder name
+        name = project_name
 
     if report != None:
         start = time()
@@ -238,7 +241,7 @@ def build_library(src_paths, build_path, target, toolchain_name,
         toolchain.jobs = jobs
         toolchain.build_all = clean
 
-        toolchain.info("Building library %s (%s, %s)" % (name.upper(), target.name, toolchain_name))
+        toolchain.info("Building library %s (%s, %s)" % (name, target.name, toolchain_name))
 
         # Scan Resources
         resources = []
@@ -262,23 +265,28 @@ def build_library(src_paths, build_path, target, toolchain_name,
         if inc_dirs:
             dependencies_include_dir.extend(inc_dirs)
 
-        # Create the desired build directory structure
+        if archive:
-        bin_path = join(build_path, toolchain.obj_path)
+            # Use temp path when building archive
-        mkdir(bin_path)
+            tmp_path = join(build_path, '.temp')
-        tmp_path = join(build_path, '.temp', toolchain.obj_path)
+            mkdir(tmp_path)
-        mkdir(tmp_path)
+        else:
+            tmp_path = build_path
 
         # Copy Headers
         for resource in resources:
             toolchain.copy_files(resource.headers, build_path, rel_path=resource.base_path)
-        dependencies_include_dir.extend(toolchain.scan_resources(build_path).inc_dirs)
+            if resource.linker_script:
+                toolchain.copy_files(resource.linker_script, build_path, rel_path=resource.base_path)
 
         # Compile Sources
         objects = []
         for resource in resources:
-            objects.extend(toolchain.compile_sources(resource, tmp_path, dependencies_include_dir))
+            objects.extend(toolchain.compile_sources(resource, abspath(tmp_path), dependencies_include_dir))
 
-        needed_update = toolchain.build_library(objects, bin_path, name)
+        if archive:
+            needed_update = toolchain.build_library(objects, build_path, name)
+        else:
+            needed_update = True
 
         if report != None and needed_update:
             end = time()

tools/make.py

@@ -63,7 +63,7 @@ if __name__ == '__main__':
                       type="int",
                       dest="jobs",
                       default=1,
-                      help="Number of concurrent jobs (default 1). Use 0 for auto based on host machine's number of CPUs")
+                      help="Number of concurrent jobs. Default: 0/auto (based on host machine's number of CPUs)")
 
     parser.add_option("-v", "--verbose",
                       action="store_true",

tools/toolchains/__init__.py

@@ -472,12 +472,7 @@ class mbedToolchain:
     def relative_object_path(self, build_path, base_dir, source):
         source_dir, name, _ = split_path(source)
         
-        if build_path.startswith(base_dir):
+        obj_dir = join(build_path, relpath(source_dir, base_dir))
-            # absolute path
-            obj_dir = join(build_path, relpath(source_dir, base_dir))
-        else:
-            # relative path
-            obj_dir = join(base_dir, build_path)
         mkdir(obj_dir)
         return join(obj_dir, name + '.o')
 

tools/toolchains/arm.py

@@ -15,12 +15,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 import re
-from os.path import join, dirname
+from os.path import join, dirname, basename
 
 from tools.toolchains import mbedToolchain
-from tools.settings import ARM_BIN, ARM_INC, ARM_LIB, MY_ARM_CLIB, ARM_CPPLIB
+from tools.settings import ARM_BIN, ARM_INC, ARM_LIB, MY_ARM_CLIB, ARM_CPPLIB, GOANNA_PATH
 from tools.hooks import hook_tool
-from tools.settings import GOANNA_PATH
+from tools.utils import mkdir
 
 class ARM(mbedToolchain):
     LINKER_EXT = '.sct'
@@ -115,13 +115,12 @@ class ARM(mbedToolchain):
     def get_dep_opt(self, dep_path):
         return ["--depend", dep_path]
 
-    def archive(self, objects, lib_path):
-        self.default_cmd([self.ar, '-r', lib_path] + objects)
-
     @hook_tool
     def assemble(self, source, object, includes):
         # Preprocess first, then assemble
-        tempfile = object + '.E.s'
+        dir = join(dirname(object), '.temp')
+        mkdir(dir)
+        tempfile = join(dir, basename(object) + '.E.s')
         
         # Build preprocess assemble command
         cmd_pre = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros] + ["-I%s" % i for i in includes] + ["-E", "-o", tempfile, source]
@@ -159,13 +158,27 @@ class ARM(mbedToolchain):
             cmd_linker = cmd[0]
             cmd_list = []
             for c in cmd[1:]:
-                cmd_list.append(('"%s"' % c) if not c.startswith('-') else c)                    
+                if c:
+                    cmd_list.append(('"%s"' % c) if not c.startswith('-') else c)                    
             string = " ".join(cmd_list).replace("\\", "/")
             f.write(string)
 
         # Exec command
         self.default_cmd([cmd_linker, '--via', link_files])
 
+    @hook_tool
+    def archive(self, objects, lib_path):
+        archive_files = join(dirname(lib_path), ".archive_files.txt")
+        with open(archive_files, "wb") as f:
+            o_list = []
+            for o in objects:
+                o_list.append('"%s"' % o)                    
+            string = " ".join(o_list).replace("\\", "/")
+            f.write(string)
+
+        # Exec command
+        self.default_cmd([self.ar, '-r', lib_path, '--via', archive_files])
+
     @hook_tool
     def binary(self, resources, elf, bin):
         # Build binary command

tools/toolchains/gcc.py

@@ -161,13 +161,6 @@ class GCC(mbedToolchain):
                     message + match.group('message')
                 )
 
-    def archive(self, objects, lib_path):
-        # Build archive command
-        cmd = [self.ar, "rcs", lib_path] + objects
-        
-        # Exec cmd
-        self.default_cmd(cmd)
-
     @hook_tool
     def assemble(self, source, object, includes):
         # Build assemble command
@@ -195,7 +188,11 @@ class GCC(mbedToolchain):
             libs.extend(libs)
         
         # Build linker command
-        cmd = self.ld + ["-T", mem_map, "-o", output] + objects
+        cmd = self.ld + ["-o", output] + objects
+        
+        if mem_map:
+            cmd.extend(['-T', mem_map])
+            
         for L in lib_dirs:
             cmd.extend(['-L', L])
         cmd.extend(libs)
@@ -209,13 +206,27 @@ class GCC(mbedToolchain):
             cmd_linker = cmd[0]
             cmd_list = []
             for c in cmd[1:]:
-                cmd_list.append(('"%s"' % c) if not c.startswith('-') else c)   
+                if c:
+                    cmd_list.append(('"%s"' % c) if not c.startswith('-') else c)   
             string = " ".join(cmd_list).replace("\\", "/")
             f.write(string)
 
         # Exec command
         self.default_cmd([cmd_linker, "@%s" % link_files])
 
+    @hook_tool
+    def archive(self, objects, lib_path):
+        archive_files = join(dirname(lib_path), ".archive_files.txt")
+        with open(archive_files, "wb") as f:
+            o_list = []
+            for o in objects:
+                o_list.append('"%s"' % o)                    
+            string = " ".join(o_list).replace("\\", "/")
+            f.write(string)
+
+        # Exec command
+        self.default_cmd([self.ar, 'rcs', lib_path, "@%s" % archive_files])
+
     @hook_tool
     def binary(self, resources, elf, bin):
         # Build binary command

tools/toolchains/iar.py

@@ -114,16 +114,13 @@ class IAR(mbedToolchain):
         # Return command array, don't execute
         return [cmd]
 
-    @hook_tool
-    def archive(self, objects, lib_path):
-        if exists(lib_path):
-            remove(lib_path)
-        self.default_cmd([self.ar, lib_path] + objects)
-
     @hook_tool
     def link(self, output, objects, libraries, lib_dirs, mem_map):
         # Build linker command
-        cmd = [self.ld, "-o", output, "--config", mem_map, "--skip_dynamic_initialization"] + objects + libraries
+        cmd = [self.ld, "-o", output, "--skip_dynamic_initialization"] + objects + libraries
+
+        if mem_map:
+            args.extend(["--config", mem_map])
 
         # Call cmdline hook
         cmd = self.hook.get_cmdline_linker(cmd)
@@ -134,13 +131,29 @@ class IAR(mbedToolchain):
             cmd_linker = cmd[0]
             cmd_list = []
             for c in cmd[1:]:
-                cmd_list.append(('"%s"' % c) if not c.startswith('-') else c)                    
+                if c:
+                    cmd_list.append(('"%s"' % c) if not c.startswith('-') else c)                    
             string = " ".join(cmd_list).replace("\\", "/")
             f.write(string)
 
         # Exec command
         self.default_cmd([cmd_linker, '-f', link_files])
 
+    @hook_tool
+    def archive(self, objects, lib_path):
+        archive_files = join(dirname(lib_path), ".archive_files.txt")
+        with open(archive_files, "wb") as f:
+            o_list = []
+            for o in objects:
+                o_list.append('"%s"' % o)                    
+            string = " ".join(o_list).replace("\\", "/")
+            f.write(string)
+
+        if exists(lib_path):
+            remove(lib_path)
+
+        self.default_cmd([self.ar, lib_path, '-f', archive_files])
+
     @hook_tool
     def binary(self, resources, elf, bin):
         # Build binary command