Merge pull request #84 from screamerbg/compile-response-files

Response files for includes, de-duplicate includes and more

tools/toolchains/__init__.py

@@ -29,6 +29,7 @@ from multiprocessing import Pool, cpu_count
 from tools.utils import run_cmd, mkdir, rel_path, ToolException, NotSupportedException, split_path
 from tools.settings import BUILD_OPTIONS, MBED_ORG_USER
 import tools.hooks as hooks
+from hashlib import md5
 
 
 #Disables multiprocessing if set to higher number than the host machine CPUs
@@ -210,6 +211,7 @@ class mbedToolchain:
         self.has_config = False
 
         self.build_all = False
+        self.build_dir = None
         self.timestamp = time()
         self.jobs = 1
 
@@ -476,19 +478,35 @@ class mbedToolchain:
         mkdir(obj_dir)
         return join(obj_dir, name + '.o')
 
+    def get_inc_file(self, includes):
+        include_file = join(self.build_dir, ".includes_%s.txt" % self.inc_md5)
+        if not exists(include_file):
+            with open(include_file, "wb") as f:
+                cmd_list = []
+                for c in includes:
+                    if c:
+                        cmd_list.append(('-I%s' % c).replace("\\", "/"))                    
+                string = " ".join(cmd_list)
+                f.write(string)
+        return include_file
+
     def compile_sources(self, resources, build_path, inc_dirs=None):
         # Web IDE progress bar for project build
         files_to_compile = resources.s_sources + resources.c_sources + resources.cpp_sources
         self.to_be_compiled = len(files_to_compile)
         self.compiled = 0
 
-        #for i in self.build_params:
-        #    self.debug(i)
-        #    self.debug("%s" % self.build_params[i])
-
         inc_paths = resources.inc_dirs
         if inc_dirs is not None:
             inc_paths.extend(inc_dirs)
+        # De-duplicate include paths
+        inc_paths = set(inc_paths)
+        # Sort include paths for consistency
+        inc_paths = sorted(set(inc_paths))
+        # Unique id of all include paths
+        self.inc_md5 = md5(' '.join(inc_paths)).hexdigest()
+        # Where to store response files
+        self.build_dir = build_path
 
         objects = []
         queue = []
@@ -496,6 +514,7 @@ class mbedToolchain:
 
         # The dependency checking for C/C++ is delegated to the compiler
         base_path = resources.base_path
+        # Sort compile queue for consistency
         files_to_compile.sort()
         work_dir = getcwd()
 
@@ -641,28 +660,6 @@ class mbedToolchain:
             else:
                 raise ToolException(_stderr)
 
-    def compile(self, cc, source, object, includes):
-        _, ext = splitext(source)
-        ext = ext.lower()
-
-        command = cc + ['-D%s' % s for s in self.get_symbols()] + ["-I%s" % i for i in includes] + ["-o", object, source]
-
-        if hasattr(self, "get_dep_opt"):
-            base, _ = splitext(object)
-            dep_path = base + '.d'
-            command.extend(self.get_dep_opt(dep_path))
-
-        if hasattr(self, "cc_extra"):
-            command.extend(self.cc_extra(base))
-
-        return [command]
-
-    def compile_c(self, source, object, includes):
-        return self.compile(self.cc, source, object, includes)
-
-    def compile_cpp(self, source, object, includes):
-        return self.compile(self.cppc, source, object, includes)
-
     def build_library(self, objects, dir, name):
         needed_update = False
         lib = self.STD_LIB_NAME % name
@@ -712,12 +709,12 @@ class mbedToolchain:
         return bin, needed_update
 
     def default_cmd(self, command):
+        self.debug("Command: %s"% ' '.join(command))
         _stdout, _stderr, _rc = run_cmd(command)
         # Print all warning / erros from stderr to console output
         for error_line in _stderr.splitlines():
             print error_line
 
-        self.debug("Command: %s"% ' '.join(command))
         self.debug("Return: %s"% _rc)
 
         for output_line in _stdout.splitlines():

tools/toolchains/arm.py

@@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 import re
-from os.path import join, dirname, basename
+from os.path import join, dirname, splitext, basename, exists
 
 from tools.toolchains import mbedToolchain
 from tools.settings import ARM_BIN, ARM_INC, ARM_LIB, MY_ARM_CLIB, ARM_CPPLIB, GOANNA_PATH
@@ -78,11 +78,6 @@ class ARM(mbedToolchain):
         self.ar = join(ARM_BIN, "armar")
         self.elf2bin = join(ARM_BIN, "fromelf")
 
-    def remove_option(self, option):
-        for tool in [self.asm, self.cc, self.cppc]:
-            if option in tool:
-                tool.remove(option)
-
     def parse_dependencies(self, dep_path):
         dependencies = []
         for line in open(dep_path).readlines():
@@ -112,9 +107,14 @@ class ARM(mbedToolchain):
                     match.group('message')
                 )
 
-    def get_dep_opt(self, dep_path):
+    def get_dep_option(self, object):
+        base, _ = splitext(object)
+        dep_path = base + '.d'
         return ["--depend", dep_path]
 
+    def get_compile_options(self, defines, includes):        
+        return ['-D%s' % d for d in defines] + ['--via', self.get_inc_file(includes)]
+
     @hook_tool
     def assemble(self, source, object, includes):
         # Preprocess first, then assemble
@@ -123,7 +123,7 @@ class ARM(mbedToolchain):
         tempfile = join(dir, basename(object) + '.E.s')
         
         # Build preprocess assemble command
-        cmd_pre = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros] + ["-I%s" % i for i in includes] + ["-E", "-o", tempfile, source]
+        cmd_pre = self.asm + self.get_compile_options(self.get_symbols(), includes) + ["-E", "-o", tempfile, source]
 
         # Build main assemble command
         cmd = self.asm + ["-o", object, tempfile]
@@ -135,6 +135,25 @@ class ARM(mbedToolchain):
         # Return command array, don't execute
         return [cmd_pre, cmd]
 
+    @hook_tool
+    def compile(self, cc, source, object, includes):
+        # Build compile command
+        cmd = cc + self.get_compile_options(self.get_symbols(), includes)
+        
+        cmd.extend(self.get_dep_option(object))
+            
+        cmd.extend(["-o", object, source])
+
+        # Call cmdline hook
+        cmd = self.hook.get_cmdline_compiler(cmd)
+
+        return [cmd]
+
+    def compile_c(self, source, object, includes):
+        return self.compile(self.cc, source, object, includes)
+
+    def compile_cpp(self, source, object, includes):
+        return self.compile(self.cppc, source, object, includes)
 
     @hook_tool
     def link(self, output, objects, libraries, lib_dirs, mem_map):

tools/toolchains/gcc.py

@@ -15,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 import re
-from os.path import join, basename, splitext, dirname
+from os.path import join, basename, splitext, dirname, exists
 
 from tools.toolchains import mbedToolchain
 from tools.settings import GCC_ARM_PATH, GCC_CR_PATH
@@ -68,7 +68,7 @@ class GCC(mbedToolchain):
             "-Wno-unused-parameter", "-Wno-missing-field-initializers",
             "-fmessage-length=0", "-fno-exceptions", "-fno-builtin",
             "-ffunction-sections", "-fdata-sections",
-            "-MMD", "-fno-delete-null-pointer-checks", "-fomit-frame-pointer"
+            "-fno-delete-null-pointer-checks", "-fomit-frame-pointer"
             ] + self.cpu
 
         if "save-asm" in self.options:
@@ -161,10 +161,18 @@ class GCC(mbedToolchain):
                     message + match.group('message')
                 )
 
+    def get_dep_option(self, object):
+        base, _ = splitext(object)
+        dep_path = base + '.d'
+        return ["-MD", "-MF", dep_path]
+
+    def get_compile_options(self, defines, includes):
+        return ['-D%s' % d for d in defines] + ['@%s' % self.get_inc_file(includes)]
+
     @hook_tool
     def assemble(self, source, object, includes):
         # Build assemble command
-        cmd = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros] + ["-I%s" % i for i in includes] + ["-o", object, source]
+        cmd = self.asm + self.get_compile_options(self.get_symbols(), includes) + ["-o", object, source]
 
         # Call cmdline hook
         cmd = self.hook.get_cmdline_assembler(cmd)
@@ -172,6 +180,26 @@ class GCC(mbedToolchain):
         # Return command array, don't execute
         return [cmd]
 
+    @hook_tool
+    def compile(self, cc, source, object, includes):
+        # Build compile command
+        cmd = cc + self.get_compile_options(self.get_symbols(), includes)
+
+        cmd.extend(self.get_dep_option(object))
+
+        cmd.extend(["-o", object, source])
+        
+        # Call cmdline hook
+        cmd = self.hook.get_cmdline_compiler(cmd)
+
+        return [cmd]
+
+    def compile_c(self, source, object, includes):
+        return self.compile(self.cc, source, object, includes)
+
+    def compile_cpp(self, source, object, includes):
+        return self.compile(self.cppc, source, object, includes)
+
     @hook_tool
     def link(self, output, objects, libraries, lib_dirs, mem_map):
         libs = []

tools/toolchains/iar.py

@@ -16,7 +16,7 @@ limitations under the License.
 """
 import re
 from os import remove
-from os.path import join, exists, dirname
+from os.path import join, exists, dirname, splitext, exists
 
 from tools.toolchains import mbedToolchain
 from tools.settings import IAR_PATH
@@ -72,6 +72,10 @@ class IAR(mbedToolchain):
         self.ar = join(IAR_BIN, "iarchive")
         self.elf2bin = join(IAR_BIN, "ielftool")
 
+    def parse_dependencies(self, dep_path):
+        return [path.strip() for path in open(dep_path).readlines()
+                if (path and not path.isspace())]
+
     def parse_output(self, output):
         for line in output.splitlines():
             match = IAR.DIAGNOSTIC_PATTERN.match(line)
@@ -93,20 +97,22 @@ class IAR(mbedToolchain):
                     match.group('message')
                 )
 
-    def get_dep_opt(self, dep_path):
+    def get_dep_option(self, object):
+        base, _ = splitext(object)
+        dep_path = base + '.d'
         return ["--dependencies", dep_path]
 
-    def cc_extra(self, base):
+    def cc_extra(self, object):
+        base, _ = splitext(object)
         return ["-l", base + '.s']
 
-    def parse_dependencies(self, dep_path):
+    def get_compile_options(self, defines, includes):
-        return [path.strip() for path in open(dep_path).readlines()
+        return ['-D%s' % d for d in defines] + ['-f', self.get_inc_file(includes)]
-                if (path and not path.isspace())]
 
     @hook_tool
     def assemble(self, source, object, includes):
         # Build assemble command
-        cmd = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros] + ["-I%s" % i for i in includes] + ["-o", object, source]
+        cmd = self.asm + self.get_compile_options(self.get_symbols(), includes) + ["-o", object, source]
 
         # Call cmdline hook
         cmd = self.hook.get_cmdline_assembler(cmd)
@@ -114,6 +120,28 @@ class IAR(mbedToolchain):
         # Return command array, don't execute
         return [cmd]
 
+    @hook_tool
+    def compile(self, cc, source, object, includes):
+        # Build compile command
+        cmd = cc +  self.get_compile_options(self.get_symbols(), includes)
+
+        cmd.extend(self.get_dep_option(object))
+
+        cmd.extend(self.cc_extra(object))
+        
+        cmd.extend(["-o", object, source])
+
+        # Call cmdline hook
+        cmd = self.hook.get_cmdline_compiler(cmd)
+
+        return [cmd]
+
+    def compile_c(self, source, object, includes):
+        return self.compile(self.cc, source, object, includes)
+
+    def compile_cpp(self, source, object, includes):
+        return self.compile(self.cppc, source, object, includes)
+
     @hook_tool
     def link(self, output, objects, libraries, lib_dirs, mem_map):
         # Build linker command

tools/utils.py

@@ -34,8 +34,12 @@ def cmd(l, check=True, verbose=False, shell=False, cwd=None):
 
 def run_cmd(command, wd=None, redirect=False):
     assert is_cmd_valid(command[0])
-    p = Popen(command, stdout=PIPE, stderr=STDOUT if redirect else PIPE, cwd=wd)
+    try:
-    _stdout, _stderr = p.communicate()
+        p = Popen(command, stdout=PIPE, stderr=STDOUT if redirect else PIPE, cwd=wd)
+        _stdout, _stderr = p.communicate()
+    except:
+        print "[OS ERROR] Command: "+(' '.join(command))
+        raise
     return _stdout, _stderr, p.returncode