Unified command-line handling of defines and includes with response files for all toolchains (ARM, GCC, IAR)

Capture system/OS error during command execution and report the command
Report command before executing it in default_cmd()

tools/toolchains/__init__.py

@@ -688,12 +688,12 @@ class mbedToolchain:
         return bin, needed_update
 
     def default_cmd(self, command):
+        self.debug("Command: %s"% ' '.join(command))
         _stdout, _stderr, _rc = run_cmd(command)
         # Print all warning / erros from stderr to console output
         for error_line in _stderr.splitlines():
             print error_line
 
-        self.debug("Command: %s"% ' '.join(command))
         self.debug("Return: %s"% _rc)
 
         for output_line in _stdout.splitlines():

tools/toolchains/arm.py

@@ -116,6 +116,27 @@ class ARM(mbedToolchain):
     def get_dep_opt(self, dep_path):
         return ["--depend", dep_path]
 
+    def get_compile_options(self, defines, includes):
+        cmd = []
+        
+        str = (' '.join(defines))+"|"+(' '.join(includes))
+        if len(str) > 160:
+            sum = md5(str).hexdigest()
+            options_file = join(self.temp_dir, "options_%s.txt" % sum)
+            if not exists(options_file):
+                with open(options_file, "wb") as f:
+                    cmd_list = ['-D%s' % d for d in defines]
+                    for c in includes:
+                        if c:
+                            cmd_list.append(('-I%s' % c) if not c.startswith('-') else c)                    
+                    string = " ".join(cmd_list).replace("\\", "/")
+                    f.write(string)
+            cmd.extend(['--via', options_file])
+        else:
+            cmd.extend(['-D%s' % d for d in defines] + ['-I%s' % i for i in includes])
+        
+        return cmd
+
     @hook_tool
     def assemble(self, source, object, includes):
         # Preprocess first, then assemble
@@ -124,26 +145,7 @@ class ARM(mbedToolchain):
         tempfile = join(dir, basename(object) + '.E.s')
         
         # Build preprocess assemble command
-        cmd_pre = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros]
+        cmd_pre = self.asm + self.get_compile_options(self.get_symbols(), includes) + ["-E", "-o", tempfile, source]
-
-        # Response file
-        inc_str = ' '.join(includes)
-        if len(inc_str) > 16000:
-            sum = md5(inc_str).hexdigest()
-            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
-            if not exists(include_files):
-                with open(include_files, "wb") as f:
-                    cmd_list = []
-                    for c in includes:
-                        if c:
-                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)
-                    string = " ".join(cmd_list).replace("\\", "/")
-                    f.write(string)
-            cmd_pre.extend(['--via', include_files])
-        else:
-            cmd_pre.extend(['-I"%s"' % i for i in includes])
-
-        cmd_pre.extend(["-E", "-o", tempfile, source])
 
         # Build main assemble command
         cmd = self.asm + ["-o", object, tempfile]
@@ -157,24 +159,8 @@ class ARM(mbedToolchain):
 
     @hook_tool
     def compile(self, cc, source, object, includes):
-        cmd = cc + ['-D %s' % s for s in self.get_symbols()]
+        # Build compile command
-
+        cmd = cc + self.get_compile_options(self.get_symbols(), includes)
-        inc_str = ' '.join(includes)
-        if len(inc_str) > 16000:
-            sum = md5(inc_str).hexdigest()
-            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
-            if not exists(include_files):
-                with open(include_files, "wb") as f:
-                    cmd_list = []
-                    for c in includes:
-                        if c:
-                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)
-                    string = " ".join(cmd_list).replace("\\", "/")
-                    f.write(string)
-            cmd.extend(['--via', include_files])
-        else:
-            cmd.extend(['-I"%s"' % i for i in includes])
-
         
         base, _ = splitext(object)
         dep_path = base + '.d'
@@ -182,6 +168,9 @@ class ARM(mbedToolchain):
             
         cmd.extend(["-o", object, source])
 
+        # Call cmdline hook
+        cmd = self.hook.get_cmdline_compiler(cmd)
+
         return [cmd]
 
     def compile_c(self, source, object, includes):

tools/toolchains/gcc.py

@@ -162,28 +162,31 @@ class GCC(mbedToolchain):
                     message + match.group('message')
                 )
 
+    def get_compile_options(self, defines, includes):
+        cmd = []
+        
+        str = (' '.join(defines))+"|"+(' '.join(includes))
+        if len(str) > 160:
+            sum = md5(str).hexdigest()
+            options_file = join(self.temp_dir, "options_%s.txt" % sum)
+            if not exists(options_file):
+                with open(options_file, "wb") as f:
+                    cmd_list = ['-D%s' % d for d in defines]
+                    for c in includes:
+                        if c:
+                            cmd_list.append(('-I%s' % c) if not c.startswith('-') else c)                    
+                    string = " ".join(cmd_list).replace("\\", "/")
+                    f.write(string)
+            cmd.extend(['@%s' % options_file])
+        else:
+            cmd.extend(['-D%s' % d for d in defines] + ['-I%s' % i for i in includes])
+        
+        return cmd
+
     @hook_tool
     def assemble(self, source, object, includes):
         # Build assemble command
-        cmd = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros]
+        cmd = self.asm + self.get_compile_options(self.get_symbols(), includes) + ["-o", object, source]
-        
-        inc_str = ' '.join(includes)
-        if len(inc_str) > 16000:
-            sum = md5(inc_str).hexdigest()
-            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
-            if not exists(include_files):
-                with open(include_files, "wb") as f:
-                    cmd_list = []
-                    for c in includes:
-                        if c:
-                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)                    
-                    string = " ".join(cmd_list).replace("\\", "/")
-                    f.write(string)
-            cmd.extend(['@%s' % include_files])
-        else:
-            cmd.extend(['-I"%s"' % i for i in includes])
-        
-        cmd.extend(["-o", object, source])
 
         # Call cmdline hook
         cmd = self.hook.get_cmdline_assembler(cmd)
@@ -193,25 +196,11 @@ class GCC(mbedToolchain):
 
     @hook_tool
     def compile(self, cc, source, object, includes):
-        cmd = cc + ['-D%s' % s for s in self.get_symbols()]
+        # Build compile command
+        cmd = cc + self.get_compile_options(self.get_symbols(), includes) + ["-o", object, source]
 
-        inc_str = ' '.join(includes)
+        # Call cmdline hook
-        if len(inc_str) > 16000:
+        cmd = self.hook.get_cmdline_compiler(cmd)
-            sum = md5(inc_str).hexdigest()
-            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
-            if not exists(include_files):
-                with open(include_files, "wb") as f:
-                    cmd_list = []
-                    for c in includes:
-                        if c:
-                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)                    
-                    string = " ".join(cmd_list).replace("\\", "/")
-                    f.write(string)
-            cmd.extend(['@%s' % include_files])
-        else:
-            cmd.extend(['-I"%s"' % i for i in includes])
-
-        cmd.extend(["-o", object, source])
 
         return [cmd]
 

tools/toolchains/iar.py

@@ -104,10 +104,31 @@ class IAR(mbedToolchain):
         return [path.strip() for path in open(dep_path).readlines()
                 if (path and not path.isspace())]
 
+    def get_compile_options(self, defines, includes):
+        cmd = []
+        
+        str = (' '.join(defines))+"|"+(' '.join(includes))
+        if len(str) > 160:
+            sum = md5(str).hexdigest()
+            options_file = join(self.temp_dir, "options_%s.txt" % sum)
+            if not exists(options_file):
+                with open(options_file, "wb") as f:
+                    cmd_list = ['-D%s' % d for d in defines]
+                    for c in includes:
+                        if c:
+                            cmd_list.append(('-I%s' % c) if not c.startswith('-') else c)                    
+                    string = " ".join(cmd_list).replace("\\", "/")
+                    f.write(string)
+            cmd.extend(['-f', options_file])
+        else:
+            cmd.extend(['-D%s' % d for d in defines] + ['-I%s' % i for i in includes])
+        
+        return cmd
+
     @hook_tool
     def assemble(self, source, object, includes):
         # Build assemble command
-        cmd = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros] + ["-I%s" % i for i in includes] + ["-o", object, source]
+        cmd = self.asm + self.get_compile_options(self.get_symbols(), includes) + ["-o", object, source]
 
         # Call cmdline hook
         cmd = self.hook.get_cmdline_assembler(cmd)
@@ -117,23 +138,8 @@ class IAR(mbedToolchain):
 
     @hook_tool
     def compile(self, cc, source, object, includes):
-        cmd = cc + ['-D%s' % s for s in self.get_symbols()]
+        # Build compile command
-
+        cmd = cc +  self.get_compile_options(self.get_symbols(), includes)
-        inc_str = ' '.join(includes)
-        if len(inc_str) > 1000:
-            sum = md5(inc_str).hexdigest()
-            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
-            if not exists(include_files):
-                with open(include_files, "wb") as f:
-                    cmd_list = []
-                    for c in includes:
-                        if c:
-                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)                    
-                    string = " ".join(cmd_list).replace("\\", "/")
-                    f.write(string)
-            cmd.extend(['-f', include_files])
-        else:
-            cmd.extend(['-I"%s"' % i for i in includes])
 
         base, _ = splitext(object)
         dep_path = base + '.d'
@@ -143,6 +149,9 @@ class IAR(mbedToolchain):
         
         cmd.extend(["-o", object, source])
 
+        # Call cmdline hook
+        cmd = self.hook.get_cmdline_compiler(cmd)
+
         return [cmd]
 
     def compile_c(self, source, object, includes):

tools/utils.py

@@ -34,8 +34,12 @@ def cmd(l, check=True, verbose=False, shell=False, cwd=None):
 
 def run_cmd(command, wd=None, redirect=False):
     assert is_cmd_valid(command[0])
-    p = Popen(command, stdout=PIPE, stderr=STDOUT if redirect else PIPE, cwd=wd)
+    try:
-    _stdout, _stderr = p.communicate()
+        p = Popen(command, stdout=PIPE, stderr=STDOUT if redirect else PIPE, cwd=wd)
+        _stdout, _stderr = p.communicate()
+    except:
+        print "[OS ERROR] Command: "+(' '.join(command))
+        raise
     return _stdout, _stderr, p.returncode