Response files for includes during assemble() and compile()

Moved unified compile to toolchains to enable specific toolchain support for response files

tools/toolchains/__init__.py

@@ -24,6 +24,7 @@ from types import ListType
 from shutil import copyfile
 from os.path import join, splitext, exists, relpath, dirname, basename, split
 from inspect import getmro
+from tempfile import mkdtemp
 
 from multiprocessing import Pool, cpu_count
 from tools.utils import run_cmd, mkdir, rel_path, ToolException, NotSupportedException, split_path
@@ -211,6 +212,7 @@ class mbedToolchain:
 
         self.build_all = False
         self.timestamp = time()
+        self.temp_dir = None
         self.jobs = 1
 
         self.CHROOT = None
@@ -481,6 +483,7 @@ class mbedToolchain:
         files_to_compile = resources.s_sources + resources.c_sources + resources.cpp_sources
         self.to_be_compiled = len(files_to_compile)
         self.compiled = 0
+        self.temp_dir = build_path
 
         #for i in self.build_params:
         #    self.debug(i)
@@ -641,28 +644,6 @@ class mbedToolchain:
             else:
                 raise ToolException(_stderr)
 
-    def compile(self, cc, source, object, includes):
-        _, ext = splitext(source)
-        ext = ext.lower()
-
-        command = cc + ['-D%s' % s for s in self.get_symbols()] + ["-I%s" % i for i in includes] + ["-o", object, source]
-
-        if hasattr(self, "get_dep_opt"):
-            base, _ = splitext(object)
-            dep_path = base + '.d'
-            command.extend(self.get_dep_opt(dep_path))
-
-        if hasattr(self, "cc_extra"):
-            command.extend(self.cc_extra(base))
-
-        return [command]
-
-    def compile_c(self, source, object, includes):
-        return self.compile(self.cc, source, object, includes)
-
-    def compile_cpp(self, source, object, includes):
-        return self.compile(self.cppc, source, object, includes)
-
     def build_library(self, objects, dir, name):
         needed_update = False
         lib = self.STD_LIB_NAME % name

tools/toolchains/arm.py

@@ -15,7 +15,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 import re
-from os.path import join, dirname, basename
+from os.path import join, dirname, splitext, basename, exists
+from hashlib import md5
 
 from tools.toolchains import mbedToolchain
 from tools.settings import ARM_BIN, ARM_INC, ARM_LIB, MY_ARM_CLIB, ARM_CPPLIB, GOANNA_PATH
@@ -123,7 +124,26 @@ class ARM(mbedToolchain):
         tempfile = join(dir, basename(object) + '.E.s')
         
         # Build preprocess assemble command
-        cmd_pre = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros] + ["-I%s" % i for i in includes] + ["-E", "-o", tempfile, source]
+        cmd_pre = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros]
+
+        # Response file
+        inc_str = ' '.join(includes)
+        if len(inc_str) > 16000:
+            sum = md5(inc_str).hexdigest()
+            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
+            if not exists(include_files):
+                with open(include_files, "wb") as f:
+                    cmd_list = []
+                    for c in includes:
+                        if c:
+                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)
+                    string = " ".join(cmd_list).replace("\\", "/")
+                    f.write(string)
+            cmd_pre.extend(['--via', include_files])
+        else:
+            cmd_pre.extend(['-I"%s"' % i for i in includes])
+
+        cmd_pre.extend(["-E", "-o", tempfile, source])
 
         # Build main assemble command
         cmd = self.asm + ["-o", object, tempfile]
@@ -135,6 +155,40 @@ class ARM(mbedToolchain):
         # Return command array, don't execute
         return [cmd_pre, cmd]
 
+    @hook_tool
+    def compile(self, cc, source, object, includes):
+        cmd = cc + ['-D %s' % s for s in self.get_symbols()]
+
+        inc_str = ' '.join(includes)
+        if len(inc_str) > 16000:
+            sum = md5(inc_str).hexdigest()
+            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
+            if not exists(include_files):
+                with open(include_files, "wb") as f:
+                    cmd_list = []
+                    for c in includes:
+                        if c:
+                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)
+                    string = " ".join(cmd_list).replace("\\", "/")
+                    f.write(string)
+            cmd.extend(['--via', include_files])
+        else:
+            cmd.extend(['-I"%s"' % i for i in includes])
+
+        
+        base, _ = splitext(object)
+        dep_path = base + '.d'
+        cmd.extend(self.get_dep_opt(dep_path))
+            
+        cmd.extend(["-o", object, source])
+
+        return [cmd]
+
+    def compile_c(self, source, object, includes):
+        return self.compile(self.cc, source, object, includes)
+
+    def compile_cpp(self, source, object, includes):
+        return self.compile(self.cppc, source, object, includes)
 
     @hook_tool
     def link(self, output, objects, libraries, lib_dirs, mem_map):

tools/toolchains/gcc.py

@@ -15,7 +15,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 import re
-from os.path import join, basename, splitext, dirname
+from os.path import join, basename, splitext, dirname, exists
+from hashlib import md5
 
 from tools.toolchains import mbedToolchain
 from tools.settings import GCC_ARM_PATH, GCC_CR_PATH
@@ -164,7 +165,25 @@ class GCC(mbedToolchain):
     @hook_tool
     def assemble(self, source, object, includes):
         # Build assemble command
-        cmd = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros] + ["-I%s" % i for i in includes] + ["-o", object, source]
+        cmd = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros]
+        
+        inc_str = ' '.join(includes)
+        if len(inc_str) > 16000:
+            sum = md5(inc_str).hexdigest()
+            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
+            if not exists(include_files):
+                with open(include_files, "wb") as f:
+                    cmd_list = []
+                    for c in includes:
+                        if c:
+                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)                    
+                    string = " ".join(cmd_list).replace("\\", "/")
+                    f.write(string)
+            cmd.extend(['@%s' % include_files])
+        else:
+            cmd.extend(['-I"%s"' % i for i in includes])
+        
+        cmd.extend(["-o", object, source])
 
         # Call cmdline hook
         cmd = self.hook.get_cmdline_assembler(cmd)
@@ -172,6 +191,36 @@ class GCC(mbedToolchain):
         # Return command array, don't execute
         return [cmd]
 
+    @hook_tool
+    def compile(self, cc, source, object, includes):
+        cmd = cc + ['-D%s' % s for s in self.get_symbols()]
+
+        inc_str = ' '.join(includes)
+        if len(inc_str) > 16000:
+            sum = md5(inc_str).hexdigest()
+            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
+            if not exists(include_files):
+                with open(include_files, "wb") as f:
+                    cmd_list = []
+                    for c in includes:
+                        if c:
+                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)                    
+                    string = " ".join(cmd_list).replace("\\", "/")
+                    f.write(string)
+            cmd.extend(['@%s' % include_files])
+        else:
+            cmd.extend(['-I"%s"' % i for i in includes])
+
+        cmd.extend(["-o", object, source])
+
+        return [cmd]
+
+    def compile_c(self, source, object, includes):
+        return self.compile(self.cc, source, object, includes)
+
+    def compile_cpp(self, source, object, includes):
+        return self.compile(self.cppc, source, object, includes)
+
     @hook_tool
     def link(self, output, objects, libraries, lib_dirs, mem_map):
         libs = []

tools/toolchains/iar.py

@@ -16,7 +16,8 @@ limitations under the License.
 """
 import re
 from os import remove
-from os.path import join, exists, dirname
+from os.path import join, exists, dirname, splitext, exists
+from hashlib import md5
 
 from tools.toolchains import mbedToolchain
 from tools.settings import IAR_PATH
@@ -114,6 +115,42 @@ class IAR(mbedToolchain):
         # Return command array, don't execute
         return [cmd]
 
+    @hook_tool
+    def compile(self, cc, source, object, includes):
+        cmd = cc + ['-D%s' % s for s in self.get_symbols()]
+
+        inc_str = ' '.join(includes)
+        if len(inc_str) > 1000:
+            sum = md5(inc_str).hexdigest()
+            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
+            if not exists(include_files):
+                with open(include_files, "wb") as f:
+                    cmd_list = []
+                    for c in includes:
+                        if c:
+                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)                    
+                    string = " ".join(cmd_list).replace("\\", "/")
+                    f.write(string)
+            cmd.extend(['-f', include_files])
+        else:
+            cmd.extend(['-I"%s"' % i for i in includes])
+
+        base, _ = splitext(object)
+        dep_path = base + '.d'
+        cmd.extend(self.get_dep_opt(dep_path))
+
+        cmd.extend(self.cc_extra(base))
+        
+        cmd.extend(["-o", object, source])
+
+        return [cmd]
+
+    def compile_c(self, source, object, includes):
+        return self.compile(self.cc, source, object, includes)
+
+    def compile_cpp(self, source, object, includes):
+        return self.compile(self.cppc, source, object, includes)
+
     @hook_tool
     def link(self, output, objects, libraries, lib_dirs, mem_map):
         # Build linker command