From 2be0385f73056c41c4f4d4ffc2e3af441a4c720e Mon Sep 17 00:00:00 2001 From: Mihail Stoyanov Date: Wed, 4 May 2016 18:35:29 +0100 Subject: [PATCH 1/7] Response files for includes during assemble() and compile() Moved unified compile to toolchains to enable specific toolchain support for response files --- tools/toolchains/__init__.py | 25 ++------------- tools/toolchains/arm.py | 60 ++++++++++++++++++++++++++++++++++-- tools/toolchains/gcc.py | 53 +++++++++++++++++++++++++++++-- tools/toolchains/iar.py | 39 ++++++++++++++++++++++- 4 files changed, 149 insertions(+), 28 deletions(-) diff --git a/tools/toolchains/__init__.py b/tools/toolchains/__init__.py index 4683b691a4..8534772fed 100644 --- a/tools/toolchains/__init__.py +++ b/tools/toolchains/__init__.py @@ -24,6 +24,7 @@ from types import ListType from shutil import copyfile from os.path import join, splitext, exists, relpath, dirname, basename, split from inspect import getmro +from tempfile import mkdtemp from multiprocessing import Pool, cpu_count from tools.utils import run_cmd, mkdir, rel_path, ToolException, NotSupportedException, split_path @@ -211,6 +212,7 @@ class mbedToolchain: self.build_all = False self.timestamp = time() + self.temp_dir = None self.jobs = 1 self.CHROOT = None @@ -481,6 +483,7 @@ class mbedToolchain: files_to_compile = resources.s_sources + resources.c_sources + resources.cpp_sources self.to_be_compiled = len(files_to_compile) self.compiled = 0 + self.temp_dir = build_path #for i in self.build_params: # self.debug(i) @@ -641,28 +644,6 @@ class mbedToolchain: else: raise ToolException(_stderr) - def compile(self, cc, source, object, includes): - _, ext = splitext(source) - ext = ext.lower() - - command = cc + ['-D%s' % s for s in self.get_symbols()] + ["-I%s" % i for i in includes] + ["-o", object, source] - - if hasattr(self, "get_dep_opt"): - base, _ = splitext(object) - dep_path = base + '.d' - command.extend(self.get_dep_opt(dep_path)) - - if hasattr(self, "cc_extra"): - command.extend(self.cc_extra(base)) - - return [command] - - def compile_c(self, source, object, includes): - return self.compile(self.cc, source, object, includes) - - def compile_cpp(self, source, object, includes): - return self.compile(self.cppc, source, object, includes) - def build_library(self, objects, dir, name): needed_update = False lib = self.STD_LIB_NAME % name diff --git a/tools/toolchains/arm.py b/tools/toolchains/arm.py index 1e524a7b03..323418a39e 100644 --- a/tools/toolchains/arm.py +++ b/tools/toolchains/arm.py @@ -15,7 +15,8 @@ See the License for the specific language governing permissions and limitations under the License. """ import re -from os.path import join, dirname, basename +from os.path import join, dirname, splitext, basename, exists +from hashlib import md5 from tools.toolchains import mbedToolchain from tools.settings import ARM_BIN, ARM_INC, ARM_LIB, MY_ARM_CLIB, ARM_CPPLIB, GOANNA_PATH @@ -123,8 +124,27 @@ class ARM(mbedToolchain): tempfile = join(dir, basename(object) + '.E.s') # Build preprocess assemble command - cmd_pre = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros] + ["-I%s" % i for i in includes] + ["-E", "-o", tempfile, source] - + cmd_pre = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros] + + # Response file + inc_str = ' '.join(includes) + if len(inc_str) > 16000: + sum = md5(inc_str).hexdigest() + include_files = join(self.temp_dir, "includes_%s.txt" % sum) + if not exists(include_files): + with open(include_files, "wb") as f: + cmd_list = [] + for c in includes: + if c: + cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c) + string = " ".join(cmd_list).replace("\\", "/") + f.write(string) + cmd_pre.extend(['--via', include_files]) + else: + cmd_pre.extend(['-I"%s"' % i for i in includes]) + + cmd_pre.extend(["-E", "-o", tempfile, source]) + # Build main assemble command cmd = self.asm + ["-o", object, tempfile] @@ -135,6 +155,40 @@ class ARM(mbedToolchain): # Return command array, don't execute return [cmd_pre, cmd] + @hook_tool + def compile(self, cc, source, object, includes): + cmd = cc + ['-D %s' % s for s in self.get_symbols()] + + inc_str = ' '.join(includes) + if len(inc_str) > 16000: + sum = md5(inc_str).hexdigest() + include_files = join(self.temp_dir, "includes_%s.txt" % sum) + if not exists(include_files): + with open(include_files, "wb") as f: + cmd_list = [] + for c in includes: + if c: + cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c) + string = " ".join(cmd_list).replace("\\", "/") + f.write(string) + cmd.extend(['--via', include_files]) + else: + cmd.extend(['-I"%s"' % i for i in includes]) + + + base, _ = splitext(object) + dep_path = base + '.d' + cmd.extend(self.get_dep_opt(dep_path)) + + cmd.extend(["-o", object, source]) + + return [cmd] + + def compile_c(self, source, object, includes): + return self.compile(self.cc, source, object, includes) + + def compile_cpp(self, source, object, includes): + return self.compile(self.cppc, source, object, includes) @hook_tool def link(self, output, objects, libraries, lib_dirs, mem_map): diff --git a/tools/toolchains/gcc.py b/tools/toolchains/gcc.py index 4bc81a7b8e..4a54b36c06 100644 --- a/tools/toolchains/gcc.py +++ b/tools/toolchains/gcc.py @@ -15,7 +15,8 @@ See the License for the specific language governing permissions and limitations under the License. """ import re -from os.path import join, basename, splitext, dirname +from os.path import join, basename, splitext, dirname, exists +from hashlib import md5 from tools.toolchains import mbedToolchain from tools.settings import GCC_ARM_PATH, GCC_CR_PATH @@ -164,7 +165,25 @@ class GCC(mbedToolchain): @hook_tool def assemble(self, source, object, includes): # Build assemble command - cmd = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros] + ["-I%s" % i for i in includes] + ["-o", object, source] + cmd = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros] + + inc_str = ' '.join(includes) + if len(inc_str) > 16000: + sum = md5(inc_str).hexdigest() + include_files = join(self.temp_dir, "includes_%s.txt" % sum) + if not exists(include_files): + with open(include_files, "wb") as f: + cmd_list = [] + for c in includes: + if c: + cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c) + string = " ".join(cmd_list).replace("\\", "/") + f.write(string) + cmd.extend(['@%s' % include_files]) + else: + cmd.extend(['-I"%s"' % i for i in includes]) + + cmd.extend(["-o", object, source]) # Call cmdline hook cmd = self.hook.get_cmdline_assembler(cmd) @@ -172,6 +191,36 @@ class GCC(mbedToolchain): # Return command array, don't execute return [cmd] + @hook_tool + def compile(self, cc, source, object, includes): + cmd = cc + ['-D%s' % s for s in self.get_symbols()] + + inc_str = ' '.join(includes) + if len(inc_str) > 16000: + sum = md5(inc_str).hexdigest() + include_files = join(self.temp_dir, "includes_%s.txt" % sum) + if not exists(include_files): + with open(include_files, "wb") as f: + cmd_list = [] + for c in includes: + if c: + cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c) + string = " ".join(cmd_list).replace("\\", "/") + f.write(string) + cmd.extend(['@%s' % include_files]) + else: + cmd.extend(['-I"%s"' % i for i in includes]) + + cmd.extend(["-o", object, source]) + + return [cmd] + + def compile_c(self, source, object, includes): + return self.compile(self.cc, source, object, includes) + + def compile_cpp(self, source, object, includes): + return self.compile(self.cppc, source, object, includes) + @hook_tool def link(self, output, objects, libraries, lib_dirs, mem_map): libs = [] diff --git a/tools/toolchains/iar.py b/tools/toolchains/iar.py index 3d297a80f3..a9eb4deadf 100644 --- a/tools/toolchains/iar.py +++ b/tools/toolchains/iar.py @@ -16,7 +16,8 @@ limitations under the License. """ import re from os import remove -from os.path import join, exists, dirname +from os.path import join, exists, dirname, splitext, exists +from hashlib import md5 from tools.toolchains import mbedToolchain from tools.settings import IAR_PATH @@ -114,6 +115,42 @@ class IAR(mbedToolchain): # Return command array, don't execute return [cmd] + @hook_tool + def compile(self, cc, source, object, includes): + cmd = cc + ['-D%s' % s for s in self.get_symbols()] + + inc_str = ' '.join(includes) + if len(inc_str) > 1000: + sum = md5(inc_str).hexdigest() + include_files = join(self.temp_dir, "includes_%s.txt" % sum) + if not exists(include_files): + with open(include_files, "wb") as f: + cmd_list = [] + for c in includes: + if c: + cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c) + string = " ".join(cmd_list).replace("\\", "/") + f.write(string) + cmd.extend(['-f', include_files]) + else: + cmd.extend(['-I"%s"' % i for i in includes]) + + base, _ = splitext(object) + dep_path = base + '.d' + cmd.extend(self.get_dep_opt(dep_path)) + + cmd.extend(self.cc_extra(base)) + + cmd.extend(["-o", object, source]) + + return [cmd] + + def compile_c(self, source, object, includes): + return self.compile(self.cc, source, object, includes) + + def compile_cpp(self, source, object, includes): + return self.compile(self.cppc, source, object, includes) + @hook_tool def link(self, output, objects, libraries, lib_dirs, mem_map): # Build linker command From ecb9651873910cf3cc942097976ed8ba5f8950b9 Mon Sep 17 00:00:00 2001 From: Mihail Stoyanov Date: Thu, 5 May 2016 13:13:38 +0100 Subject: [PATCH 2/7] Unified command-line handling of defines and includes with response files for all toolchains (ARM, GCC, IAR) Capture system/OS error during command execution and report the command Report command before executing it in default_cmd() --- tools/toolchains/__init__.py | 2 +- tools/toolchains/arm.py | 65 +++++++++++++++--------------------- tools/toolchains/gcc.py | 63 +++++++++++++++------------------- tools/toolchains/iar.py | 45 +++++++++++++++---------- tools/utils.py | 8 +++-- 5 files changed, 87 insertions(+), 96 deletions(-) diff --git a/tools/toolchains/__init__.py b/tools/toolchains/__init__.py index 8534772fed..3fe0e67796 100644 --- a/tools/toolchains/__init__.py +++ b/tools/toolchains/__init__.py @@ -688,12 +688,12 @@ class mbedToolchain: return bin, needed_update def default_cmd(self, command): + self.debug("Command: %s"% ' '.join(command)) _stdout, _stderr, _rc = run_cmd(command) # Print all warning / erros from stderr to console output for error_line in _stderr.splitlines(): print error_line - self.debug("Command: %s"% ' '.join(command)) self.debug("Return: %s"% _rc) for output_line in _stdout.splitlines(): diff --git a/tools/toolchains/arm.py b/tools/toolchains/arm.py index 323418a39e..8f89d986ae 100644 --- a/tools/toolchains/arm.py +++ b/tools/toolchains/arm.py @@ -116,6 +116,27 @@ class ARM(mbedToolchain): def get_dep_opt(self, dep_path): return ["--depend", dep_path] + def get_compile_options(self, defines, includes): + cmd = [] + + str = (' '.join(defines))+"|"+(' '.join(includes)) + if len(str) > 160: + sum = md5(str).hexdigest() + options_file = join(self.temp_dir, "options_%s.txt" % sum) + if not exists(options_file): + with open(options_file, "wb") as f: + cmd_list = ['-D%s' % d for d in defines] + for c in includes: + if c: + cmd_list.append(('-I%s' % c) if not c.startswith('-') else c) + string = " ".join(cmd_list).replace("\\", "/") + f.write(string) + cmd.extend(['--via', options_file]) + else: + cmd.extend(['-D%s' % d for d in defines] + ['-I%s' % i for i in includes]) + + return cmd + @hook_tool def assemble(self, source, object, includes): # Preprocess first, then assemble @@ -124,26 +145,7 @@ class ARM(mbedToolchain): tempfile = join(dir, basename(object) + '.E.s') # Build preprocess assemble command - cmd_pre = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros] - - # Response file - inc_str = ' '.join(includes) - if len(inc_str) > 16000: - sum = md5(inc_str).hexdigest() - include_files = join(self.temp_dir, "includes_%s.txt" % sum) - if not exists(include_files): - with open(include_files, "wb") as f: - cmd_list = [] - for c in includes: - if c: - cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c) - string = " ".join(cmd_list).replace("\\", "/") - f.write(string) - cmd_pre.extend(['--via', include_files]) - else: - cmd_pre.extend(['-I"%s"' % i for i in includes]) - - cmd_pre.extend(["-E", "-o", tempfile, source]) + cmd_pre = self.asm + self.get_compile_options(self.get_symbols(), includes) + ["-E", "-o", tempfile, source] # Build main assemble command cmd = self.asm + ["-o", object, tempfile] @@ -157,24 +159,8 @@ class ARM(mbedToolchain): @hook_tool def compile(self, cc, source, object, includes): - cmd = cc + ['-D %s' % s for s in self.get_symbols()] - - inc_str = ' '.join(includes) - if len(inc_str) > 16000: - sum = md5(inc_str).hexdigest() - include_files = join(self.temp_dir, "includes_%s.txt" % sum) - if not exists(include_files): - with open(include_files, "wb") as f: - cmd_list = [] - for c in includes: - if c: - cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c) - string = " ".join(cmd_list).replace("\\", "/") - f.write(string) - cmd.extend(['--via', include_files]) - else: - cmd.extend(['-I"%s"' % i for i in includes]) - + # Build compile command + cmd = cc + self.get_compile_options(self.get_symbols(), includes) base, _ = splitext(object) dep_path = base + '.d' @@ -182,6 +168,9 @@ class ARM(mbedToolchain): cmd.extend(["-o", object, source]) + # Call cmdline hook + cmd = self.hook.get_cmdline_compiler(cmd) + return [cmd] def compile_c(self, source, object, includes): diff --git a/tools/toolchains/gcc.py b/tools/toolchains/gcc.py index 4a54b36c06..8120f669d5 100644 --- a/tools/toolchains/gcc.py +++ b/tools/toolchains/gcc.py @@ -162,28 +162,31 @@ class GCC(mbedToolchain): message + match.group('message') ) + def get_compile_options(self, defines, includes): + cmd = [] + + str = (' '.join(defines))+"|"+(' '.join(includes)) + if len(str) > 160: + sum = md5(str).hexdigest() + options_file = join(self.temp_dir, "options_%s.txt" % sum) + if not exists(options_file): + with open(options_file, "wb") as f: + cmd_list = ['-D%s' % d for d in defines] + for c in includes: + if c: + cmd_list.append(('-I%s' % c) if not c.startswith('-') else c) + string = " ".join(cmd_list).replace("\\", "/") + f.write(string) + cmd.extend(['@%s' % options_file]) + else: + cmd.extend(['-D%s' % d for d in defines] + ['-I%s' % i for i in includes]) + + return cmd + @hook_tool def assemble(self, source, object, includes): # Build assemble command - cmd = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros] - - inc_str = ' '.join(includes) - if len(inc_str) > 16000: - sum = md5(inc_str).hexdigest() - include_files = join(self.temp_dir, "includes_%s.txt" % sum) - if not exists(include_files): - with open(include_files, "wb") as f: - cmd_list = [] - for c in includes: - if c: - cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c) - string = " ".join(cmd_list).replace("\\", "/") - f.write(string) - cmd.extend(['@%s' % include_files]) - else: - cmd.extend(['-I"%s"' % i for i in includes]) - - cmd.extend(["-o", object, source]) + cmd = self.asm + self.get_compile_options(self.get_symbols(), includes) + ["-o", object, source] # Call cmdline hook cmd = self.hook.get_cmdline_assembler(cmd) @@ -193,25 +196,11 @@ class GCC(mbedToolchain): @hook_tool def compile(self, cc, source, object, includes): - cmd = cc + ['-D%s' % s for s in self.get_symbols()] + # Build compile command + cmd = cc + self.get_compile_options(self.get_symbols(), includes) + ["-o", object, source] - inc_str = ' '.join(includes) - if len(inc_str) > 16000: - sum = md5(inc_str).hexdigest() - include_files = join(self.temp_dir, "includes_%s.txt" % sum) - if not exists(include_files): - with open(include_files, "wb") as f: - cmd_list = [] - for c in includes: - if c: - cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c) - string = " ".join(cmd_list).replace("\\", "/") - f.write(string) - cmd.extend(['@%s' % include_files]) - else: - cmd.extend(['-I"%s"' % i for i in includes]) - - cmd.extend(["-o", object, source]) + # Call cmdline hook + cmd = self.hook.get_cmdline_compiler(cmd) return [cmd] diff --git a/tools/toolchains/iar.py b/tools/toolchains/iar.py index a9eb4deadf..258830eaa0 100644 --- a/tools/toolchains/iar.py +++ b/tools/toolchains/iar.py @@ -104,10 +104,31 @@ class IAR(mbedToolchain): return [path.strip() for path in open(dep_path).readlines() if (path and not path.isspace())] + def get_compile_options(self, defines, includes): + cmd = [] + + str = (' '.join(defines))+"|"+(' '.join(includes)) + if len(str) > 160: + sum = md5(str).hexdigest() + options_file = join(self.temp_dir, "options_%s.txt" % sum) + if not exists(options_file): + with open(options_file, "wb") as f: + cmd_list = ['-D%s' % d for d in defines] + for c in includes: + if c: + cmd_list.append(('-I%s' % c) if not c.startswith('-') else c) + string = " ".join(cmd_list).replace("\\", "/") + f.write(string) + cmd.extend(['-f', options_file]) + else: + cmd.extend(['-D%s' % d for d in defines] + ['-I%s' % i for i in includes]) + + return cmd + @hook_tool def assemble(self, source, object, includes): # Build assemble command - cmd = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros] + ["-I%s" % i for i in includes] + ["-o", object, source] + cmd = self.asm + self.get_compile_options(self.get_symbols(), includes) + ["-o", object, source] # Call cmdline hook cmd = self.hook.get_cmdline_assembler(cmd) @@ -117,23 +138,8 @@ class IAR(mbedToolchain): @hook_tool def compile(self, cc, source, object, includes): - cmd = cc + ['-D%s' % s for s in self.get_symbols()] - - inc_str = ' '.join(includes) - if len(inc_str) > 1000: - sum = md5(inc_str).hexdigest() - include_files = join(self.temp_dir, "includes_%s.txt" % sum) - if not exists(include_files): - with open(include_files, "wb") as f: - cmd_list = [] - for c in includes: - if c: - cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c) - string = " ".join(cmd_list).replace("\\", "/") - f.write(string) - cmd.extend(['-f', include_files]) - else: - cmd.extend(['-I"%s"' % i for i in includes]) + # Build compile command + cmd = cc + self.get_compile_options(self.get_symbols(), includes) base, _ = splitext(object) dep_path = base + '.d' @@ -143,6 +149,9 @@ class IAR(mbedToolchain): cmd.extend(["-o", object, source]) + # Call cmdline hook + cmd = self.hook.get_cmdline_compiler(cmd) + return [cmd] def compile_c(self, source, object, includes): diff --git a/tools/utils.py b/tools/utils.py index 21f0e1496b..a03558a019 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -34,8 +34,12 @@ def cmd(l, check=True, verbose=False, shell=False, cwd=None): def run_cmd(command, wd=None, redirect=False): assert is_cmd_valid(command[0]) - p = Popen(command, stdout=PIPE, stderr=STDOUT if redirect else PIPE, cwd=wd) - _stdout, _stderr = p.communicate() + try: + p = Popen(command, stdout=PIPE, stderr=STDOUT if redirect else PIPE, cwd=wd) + _stdout, _stderr = p.communicate() + except: + print "[OS ERROR] Command: "+(' '.join(command)) + raise return _stdout, _stderr, p.returncode From 868942a14878f09c6b831446875ade22ba3514d3 Mon Sep 17 00:00:00 2001 From: Mihail Stoyanov Date: Thu, 5 May 2016 19:26:19 +0100 Subject: [PATCH 3/7] Build system response files for all macros and includes. This also allows developers to check the response file after/during compile for the list of all macros and includes being used. --- tools/toolchains/arm.py | 26 +++++++++++--------------- tools/toolchains/gcc.py | 26 +++++++++++--------------- tools/toolchains/iar.py | 26 +++++++++++--------------- 3 files changed, 33 insertions(+), 45 deletions(-) diff --git a/tools/toolchains/arm.py b/tools/toolchains/arm.py index 8f89d986ae..462369f591 100644 --- a/tools/toolchains/arm.py +++ b/tools/toolchains/arm.py @@ -119,21 +119,17 @@ class ARM(mbedToolchain): def get_compile_options(self, defines, includes): cmd = [] - str = (' '.join(defines))+"|"+(' '.join(includes)) - if len(str) > 160: - sum = md5(str).hexdigest() - options_file = join(self.temp_dir, "options_%s.txt" % sum) - if not exists(options_file): - with open(options_file, "wb") as f: - cmd_list = ['-D%s' % d for d in defines] - for c in includes: - if c: - cmd_list.append(('-I%s' % c) if not c.startswith('-') else c) - string = " ".join(cmd_list).replace("\\", "/") - f.write(string) - cmd.extend(['--via', options_file]) - else: - cmd.extend(['-D%s' % d for d in defines] + ['-I%s' % i for i in includes]) + sum = md5(' '.join(includes)).hexdigest() + options_file = join(self.temp_dir, "options_%s.txt" % sum) + if not exists(options_file): + with open(options_file, "wb") as f: + cmd_list = ['-D%s' % d for d in defines] + for c in includes: + if c: + cmd_list.append(('-I%s' % c) if not c.startswith('-') else c) + string = " ".join(cmd_list).replace("\\", "/") + f.write(string) + cmd.extend(['--via', options_file]) return cmd diff --git a/tools/toolchains/gcc.py b/tools/toolchains/gcc.py index 8120f669d5..8fae2fb667 100644 --- a/tools/toolchains/gcc.py +++ b/tools/toolchains/gcc.py @@ -165,21 +165,17 @@ class GCC(mbedToolchain): def get_compile_options(self, defines, includes): cmd = [] - str = (' '.join(defines))+"|"+(' '.join(includes)) - if len(str) > 160: - sum = md5(str).hexdigest() - options_file = join(self.temp_dir, "options_%s.txt" % sum) - if not exists(options_file): - with open(options_file, "wb") as f: - cmd_list = ['-D%s' % d for d in defines] - for c in includes: - if c: - cmd_list.append(('-I%s' % c) if not c.startswith('-') else c) - string = " ".join(cmd_list).replace("\\", "/") - f.write(string) - cmd.extend(['@%s' % options_file]) - else: - cmd.extend(['-D%s' % d for d in defines] + ['-I%s' % i for i in includes]) + sum = md5(' '.join(includes)).hexdigest() + options_file = join(self.temp_dir, "options_%s.txt" % sum) + if not exists(options_file): + with open(options_file, "wb") as f: + cmd_list = ['-D%s' % d for d in defines] + for c in includes: + if c: + cmd_list.append(('-I%s' % c) if not c.startswith('-') else c) + string = " ".join(cmd_list).replace("\\", "/") + f.write(string) + cmd.extend(['@%s' % options_file]) return cmd diff --git a/tools/toolchains/iar.py b/tools/toolchains/iar.py index 258830eaa0..c449d560a6 100644 --- a/tools/toolchains/iar.py +++ b/tools/toolchains/iar.py @@ -107,21 +107,17 @@ class IAR(mbedToolchain): def get_compile_options(self, defines, includes): cmd = [] - str = (' '.join(defines))+"|"+(' '.join(includes)) - if len(str) > 160: - sum = md5(str).hexdigest() - options_file = join(self.temp_dir, "options_%s.txt" % sum) - if not exists(options_file): - with open(options_file, "wb") as f: - cmd_list = ['-D%s' % d for d in defines] - for c in includes: - if c: - cmd_list.append(('-I%s' % c) if not c.startswith('-') else c) - string = " ".join(cmd_list).replace("\\", "/") - f.write(string) - cmd.extend(['-f', options_file]) - else: - cmd.extend(['-D%s' % d for d in defines] + ['-I%s' % i for i in includes]) + sum = md5(' '.join(includes)).hexdigest() + options_file = join(self.temp_dir, "options_%s.txt" % sum) + if not exists(options_file): + with open(options_file, "wb") as f: + cmd_list = ['-D%s' % d for d in defines] + for c in includes: + if c: + cmd_list.append(('-I%s' % c) if not c.startswith('-') else c) + string = " ".join(cmd_list).replace("\\", "/") + f.write(string) + cmd.extend(['-f', options_file]) return cmd From f01e1363d15715ba2a94f4c2b80eb78f5b174d4e Mon Sep 17 00:00:00 2001 From: Mihail Stoyanov Date: Fri, 6 May 2016 12:18:07 +0100 Subject: [PATCH 4/7] De-duplicate the include paths. Without this all compiles are passed include paths twice --- tools/toolchains/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/toolchains/__init__.py b/tools/toolchains/__init__.py index 3fe0e67796..441630dba6 100644 --- a/tools/toolchains/__init__.py +++ b/tools/toolchains/__init__.py @@ -485,13 +485,11 @@ class mbedToolchain: self.compiled = 0 self.temp_dir = build_path - #for i in self.build_params: - # self.debug(i) - # self.debug("%s" % self.build_params[i]) - inc_paths = resources.inc_dirs if inc_dirs is not None: inc_paths.extend(inc_dirs) + # De-duplicate include paths and sort for consistency + inc_paths = sorted(set(inc_paths)) objects = [] queue = [] @@ -499,6 +497,7 @@ class mbedToolchain: # The dependency checking for C/C++ is delegated to the compiler base_path = resources.base_path + # Sort compile queue for consistency files_to_compile.sort() work_dir = getcwd() From 7e11174a0065a9f348a583e0e4ba2813277ccb25 Mon Sep 17 00:00:00 2001 From: Mihail Stoyanov Date: Fri, 6 May 2016 13:06:48 +0100 Subject: [PATCH 5/7] Calculate md5 of all include paths in compile_sources() and remove calculation from get_compile_options(), thus significantly reduce repetitive md5 calculations Unify handling of the include response file in mbedToolchain::get_inc_file() Sanitize obsolete no-longer needed methods --- tools/toolchains/__init__.py | 22 +++++++++++++++++++-- tools/toolchains/arm.py | 32 ++++++------------------------ tools/toolchains/gcc.py | 30 ++++++++++++---------------- tools/toolchains/iar.py | 38 ++++++++++++------------------------ 4 files changed, 50 insertions(+), 72 deletions(-) diff --git a/tools/toolchains/__init__.py b/tools/toolchains/__init__.py index 441630dba6..1a60ad0758 100644 --- a/tools/toolchains/__init__.py +++ b/tools/toolchains/__init__.py @@ -30,6 +30,7 @@ from multiprocessing import Pool, cpu_count from tools.utils import run_cmd, mkdir, rel_path, ToolException, NotSupportedException, split_path from tools.settings import BUILD_OPTIONS, MBED_ORG_USER import tools.hooks as hooks +from hashlib import md5 #Disables multiprocessing if set to higher number than the host machine CPUs @@ -478,18 +479,35 @@ class mbedToolchain: mkdir(obj_dir) return join(obj_dir, name + '.o') + def get_inc_file(self, includes): + include_file = join(self.temp_dir, "includes_%s.txt" % self.inc_md5) + if not exists(include_file): + with open(include_file, "wb") as f: + cmd_list = [] + for c in includes: + if c: + cmd_list.append(('-I%s' % c).replace("\\", "/")) + string = " ".join(cmd_list) + f.write(string) + return include_file + def compile_sources(self, resources, build_path, inc_dirs=None): # Web IDE progress bar for project build files_to_compile = resources.s_sources + resources.c_sources + resources.cpp_sources self.to_be_compiled = len(files_to_compile) self.compiled = 0 - self.temp_dir = build_path inc_paths = resources.inc_dirs if inc_dirs is not None: inc_paths.extend(inc_dirs) - # De-duplicate include paths and sort for consistency + # De-duplicate include paths + inc_paths = set(inc_paths) + # Sort include paths for consistency inc_paths = sorted(set(inc_paths)) + # Unique id of all include paths + self.inc_md5 = md5(' '.join(inc_paths)).hexdigest() + # Where to store response files + self.temp_dir = build_path objects = [] queue = [] diff --git a/tools/toolchains/arm.py b/tools/toolchains/arm.py index 462369f591..c4633debf7 100644 --- a/tools/toolchains/arm.py +++ b/tools/toolchains/arm.py @@ -16,7 +16,6 @@ limitations under the License. """ import re from os.path import join, dirname, splitext, basename, exists -from hashlib import md5 from tools.toolchains import mbedToolchain from tools.settings import ARM_BIN, ARM_INC, ARM_LIB, MY_ARM_CLIB, ARM_CPPLIB, GOANNA_PATH @@ -79,11 +78,6 @@ class ARM(mbedToolchain): self.ar = join(ARM_BIN, "armar") self.elf2bin = join(ARM_BIN, "fromelf") - def remove_option(self, option): - for tool in [self.asm, self.cc, self.cppc]: - if option in tool: - tool.remove(option) - def parse_dependencies(self, dep_path): dependencies = [] for line in open(dep_path).readlines(): @@ -113,25 +107,13 @@ class ARM(mbedToolchain): match.group('message') ) - def get_dep_opt(self, dep_path): + def get_dep_option(self, object): + base, _ = splitext(object) + dep_path = base + '.d' return ["--depend", dep_path] - def get_compile_options(self, defines, includes): - cmd = [] - - sum = md5(' '.join(includes)).hexdigest() - options_file = join(self.temp_dir, "options_%s.txt" % sum) - if not exists(options_file): - with open(options_file, "wb") as f: - cmd_list = ['-D%s' % d for d in defines] - for c in includes: - if c: - cmd_list.append(('-I%s' % c) if not c.startswith('-') else c) - string = " ".join(cmd_list).replace("\\", "/") - f.write(string) - cmd.extend(['--via', options_file]) - - return cmd + def get_compile_options(self, defines, includes): + return ['-D%s' % d for d in defines] + ['--via', self.get_inc_file(includes)] @hook_tool def assemble(self, source, object, includes): @@ -158,9 +140,7 @@ class ARM(mbedToolchain): # Build compile command cmd = cc + self.get_compile_options(self.get_symbols(), includes) - base, _ = splitext(object) - dep_path = base + '.d' - cmd.extend(self.get_dep_opt(dep_path)) + cmd.extend(self.get_dep_option(object)) cmd.extend(["-o", object, source]) diff --git a/tools/toolchains/gcc.py b/tools/toolchains/gcc.py index 8fae2fb667..df4be262ac 100644 --- a/tools/toolchains/gcc.py +++ b/tools/toolchains/gcc.py @@ -16,7 +16,6 @@ limitations under the License. """ import re from os.path import join, basename, splitext, dirname, exists -from hashlib import md5 from tools.toolchains import mbedToolchain from tools.settings import GCC_ARM_PATH, GCC_CR_PATH @@ -69,7 +68,7 @@ class GCC(mbedToolchain): "-Wno-unused-parameter", "-Wno-missing-field-initializers", "-fmessage-length=0", "-fno-exceptions", "-fno-builtin", "-ffunction-sections", "-fdata-sections", - "-MMD", "-fno-delete-null-pointer-checks", "-fomit-frame-pointer" + "-fno-delete-null-pointer-checks", "-fomit-frame-pointer" ] + self.cpu if "save-asm" in self.options: @@ -162,22 +161,13 @@ class GCC(mbedToolchain): message + match.group('message') ) + def get_dep_option(self, object): + base, _ = splitext(object) + dep_path = base + '.d' + return ["-MD", "-MF", dep_path] + def get_compile_options(self, defines, includes): - cmd = [] - - sum = md5(' '.join(includes)).hexdigest() - options_file = join(self.temp_dir, "options_%s.txt" % sum) - if not exists(options_file): - with open(options_file, "wb") as f: - cmd_list = ['-D%s' % d for d in defines] - for c in includes: - if c: - cmd_list.append(('-I%s' % c) if not c.startswith('-') else c) - string = " ".join(cmd_list).replace("\\", "/") - f.write(string) - cmd.extend(['@%s' % options_file]) - - return cmd + return ['-D%s' % d for d in defines] + ['@%s' % self.get_inc_file(includes)] @hook_tool def assemble(self, source, object, includes): @@ -193,8 +183,12 @@ class GCC(mbedToolchain): @hook_tool def compile(self, cc, source, object, includes): # Build compile command - cmd = cc + self.get_compile_options(self.get_symbols(), includes) + ["-o", object, source] + cmd = cc + self.get_compile_options(self.get_symbols(), includes) + cmd.extend(self.get_dep_option(object)) + + cmd.extend(["-o", object, source]) + # Call cmdline hook cmd = self.hook.get_cmdline_compiler(cmd) diff --git a/tools/toolchains/iar.py b/tools/toolchains/iar.py index c449d560a6..7d2fb60992 100644 --- a/tools/toolchains/iar.py +++ b/tools/toolchains/iar.py @@ -17,7 +17,6 @@ limitations under the License. import re from os import remove from os.path import join, exists, dirname, splitext, exists -from hashlib import md5 from tools.toolchains import mbedToolchain from tools.settings import IAR_PATH @@ -73,6 +72,10 @@ class IAR(mbedToolchain): self.ar = join(IAR_BIN, "iarchive") self.elf2bin = join(IAR_BIN, "ielftool") + def parse_dependencies(self, dep_path): + return [path.strip() for path in open(dep_path).readlines() + if (path and not path.isspace())] + def parse_output(self, output): for line in output.splitlines(): match = IAR.DIAGNOSTIC_PATTERN.match(line) @@ -94,32 +97,17 @@ class IAR(mbedToolchain): match.group('message') ) - def get_dep_opt(self, dep_path): + def get_dep_option(self, object): + base, _ = splitext(object) + dep_path = base + '.d' return ["--dependencies", dep_path] - def cc_extra(self, base): + def cc_extra(self, object): + base, _ = splitext(object) return ["-l", base + '.s'] - def parse_dependencies(self, dep_path): - return [path.strip() for path in open(dep_path).readlines() - if (path and not path.isspace())] - def get_compile_options(self, defines, includes): - cmd = [] - - sum = md5(' '.join(includes)).hexdigest() - options_file = join(self.temp_dir, "options_%s.txt" % sum) - if not exists(options_file): - with open(options_file, "wb") as f: - cmd_list = ['-D%s' % d for d in defines] - for c in includes: - if c: - cmd_list.append(('-I%s' % c) if not c.startswith('-') else c) - string = " ".join(cmd_list).replace("\\", "/") - f.write(string) - cmd.extend(['-f', options_file]) - - return cmd + return ['-D%s' % d for d in defines] + ['-f', self.get_inc_file(includes)] @hook_tool def assemble(self, source, object, includes): @@ -137,11 +125,9 @@ class IAR(mbedToolchain): # Build compile command cmd = cc + self.get_compile_options(self.get_symbols(), includes) - base, _ = splitext(object) - dep_path = base + '.d' - cmd.extend(self.get_dep_opt(dep_path)) + cmd.extend(self.get_dep_option(object)) - cmd.extend(self.cc_extra(base)) + cmd.extend(self.cc_extra(object)) cmd.extend(["-o", object, source]) From d33f890fdd27c7849a7b7811a8d0a0db377024a5 Mon Sep 17 00:00:00 2001 From: Mihail Stoyanov Date: Fri, 6 May 2016 13:10:25 +0100 Subject: [PATCH 6/7] Minor code cleanup --- tools/toolchains/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/toolchains/__init__.py b/tools/toolchains/__init__.py index 1a60ad0758..9880b2a616 100644 --- a/tools/toolchains/__init__.py +++ b/tools/toolchains/__init__.py @@ -24,7 +24,6 @@ from types import ListType from shutil import copyfile from os.path import join, splitext, exists, relpath, dirname, basename, split from inspect import getmro -from tempfile import mkdtemp from multiprocessing import Pool, cpu_count from tools.utils import run_cmd, mkdir, rel_path, ToolException, NotSupportedException, split_path @@ -212,8 +211,8 @@ class mbedToolchain: self.has_config = False self.build_all = False + self.build_dir = None self.timestamp = time() - self.temp_dir = None self.jobs = 1 self.CHROOT = None @@ -480,7 +479,7 @@ class mbedToolchain: return join(obj_dir, name + '.o') def get_inc_file(self, includes): - include_file = join(self.temp_dir, "includes_%s.txt" % self.inc_md5) + include_file = join(self.build_dir, "includes_%s.txt" % self.inc_md5) if not exists(include_file): with open(include_file, "wb") as f: cmd_list = [] @@ -507,7 +506,7 @@ class mbedToolchain: # Unique id of all include paths self.inc_md5 = md5(' '.join(inc_paths)).hexdigest() # Where to store response files - self.temp_dir = build_path + self.build_dir = build_path objects = [] queue = [] From 7e3b69ec13fba7d451e8563219f10af8d95bb040 Mon Sep 17 00:00:00 2001 From: Mihail Stoyanov Date: Fri, 6 May 2016 13:35:48 +0100 Subject: [PATCH 7/7] Add dot to include response file name, which follows the naming convention with .link_totals.txt and .link_files.txt --- tools/toolchains/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/toolchains/__init__.py b/tools/toolchains/__init__.py index 9880b2a616..bce80cdb4b 100644 --- a/tools/toolchains/__init__.py +++ b/tools/toolchains/__init__.py @@ -479,7 +479,7 @@ class mbedToolchain: return join(obj_dir, name + '.o') def get_inc_file(self, includes): - include_file = join(self.build_dir, "includes_%s.txt" % self.inc_md5) + include_file = join(self.build_dir, ".includes_%s.txt" % self.inc_md5) if not exists(include_file): with open(include_file, "wb") as f: cmd_list = []