From 7e11174a0065a9f348a583e0e4ba2813277ccb25 Mon Sep 17 00:00:00 2001 From: Mihail Stoyanov Date: Fri, 6 May 2016 13:06:48 +0100 Subject: [PATCH] Calculate md5 of all include paths in compile_sources() and remove calculation from get_compile_options(), thus significantly reduce repetitive md5 calculations Unify handling of the include response file in mbedToolchain::get_inc_file() Sanitize obsolete no-longer needed methods --- tools/toolchains/__init__.py | 22 +++++++++++++++++++-- tools/toolchains/arm.py | 32 ++++++------------------------ tools/toolchains/gcc.py | 30 ++++++++++++---------------- tools/toolchains/iar.py | 38 ++++++++++++------------------------ 4 files changed, 50 insertions(+), 72 deletions(-) diff --git a/tools/toolchains/__init__.py b/tools/toolchains/__init__.py index 441630dba6..1a60ad0758 100644 --- a/tools/toolchains/__init__.py +++ b/tools/toolchains/__init__.py @@ -30,6 +30,7 @@ from multiprocessing import Pool, cpu_count from tools.utils import run_cmd, mkdir, rel_path, ToolException, NotSupportedException, split_path from tools.settings import BUILD_OPTIONS, MBED_ORG_USER import tools.hooks as hooks +from hashlib import md5 #Disables multiprocessing if set to higher number than the host machine CPUs @@ -478,18 +479,35 @@ class mbedToolchain: mkdir(obj_dir) return join(obj_dir, name + '.o') + def get_inc_file(self, includes): + include_file = join(self.temp_dir, "includes_%s.txt" % self.inc_md5) + if not exists(include_file): + with open(include_file, "wb") as f: + cmd_list = [] + for c in includes: + if c: + cmd_list.append(('-I%s' % c).replace("\\", "/")) + string = " ".join(cmd_list) + f.write(string) + return include_file + def compile_sources(self, resources, build_path, inc_dirs=None): # Web IDE progress bar for project build files_to_compile = resources.s_sources + resources.c_sources + resources.cpp_sources self.to_be_compiled = len(files_to_compile) self.compiled = 0 - self.temp_dir = build_path inc_paths = resources.inc_dirs if inc_dirs is not None: inc_paths.extend(inc_dirs) - # De-duplicate include paths and sort for consistency + # De-duplicate include paths + inc_paths = set(inc_paths) + # Sort include paths for consistency inc_paths = sorted(set(inc_paths)) + # Unique id of all include paths + self.inc_md5 = md5(' '.join(inc_paths)).hexdigest() + # Where to store response files + self.temp_dir = build_path objects = [] queue = [] diff --git a/tools/toolchains/arm.py b/tools/toolchains/arm.py index 462369f591..c4633debf7 100644 --- a/tools/toolchains/arm.py +++ b/tools/toolchains/arm.py @@ -16,7 +16,6 @@ limitations under the License. """ import re from os.path import join, dirname, splitext, basename, exists -from hashlib import md5 from tools.toolchains import mbedToolchain from tools.settings import ARM_BIN, ARM_INC, ARM_LIB, MY_ARM_CLIB, ARM_CPPLIB, GOANNA_PATH @@ -79,11 +78,6 @@ class ARM(mbedToolchain): self.ar = join(ARM_BIN, "armar") self.elf2bin = join(ARM_BIN, "fromelf") - def remove_option(self, option): - for tool in [self.asm, self.cc, self.cppc]: - if option in tool: - tool.remove(option) - def parse_dependencies(self, dep_path): dependencies = [] for line in open(dep_path).readlines(): @@ -113,25 +107,13 @@ class ARM(mbedToolchain): match.group('message') ) - def get_dep_opt(self, dep_path): + def get_dep_option(self, object): + base, _ = splitext(object) + dep_path = base + '.d' return ["--depend", dep_path] - def get_compile_options(self, defines, includes): - cmd = [] - - sum = md5(' '.join(includes)).hexdigest() - options_file = join(self.temp_dir, "options_%s.txt" % sum) - if not exists(options_file): - with open(options_file, "wb") as f: - cmd_list = ['-D%s' % d for d in defines] - for c in includes: - if c: - cmd_list.append(('-I%s' % c) if not c.startswith('-') else c) - string = " ".join(cmd_list).replace("\\", "/") - f.write(string) - cmd.extend(['--via', options_file]) - - return cmd + def get_compile_options(self, defines, includes): + return ['-D%s' % d for d in defines] + ['--via', self.get_inc_file(includes)] @hook_tool def assemble(self, source, object, includes): @@ -158,9 +140,7 @@ class ARM(mbedToolchain): # Build compile command cmd = cc + self.get_compile_options(self.get_symbols(), includes) - base, _ = splitext(object) - dep_path = base + '.d' - cmd.extend(self.get_dep_opt(dep_path)) + cmd.extend(self.get_dep_option(object)) cmd.extend(["-o", object, source]) diff --git a/tools/toolchains/gcc.py b/tools/toolchains/gcc.py index 8fae2fb667..df4be262ac 100644 --- a/tools/toolchains/gcc.py +++ b/tools/toolchains/gcc.py @@ -16,7 +16,6 @@ limitations under the License. """ import re from os.path import join, basename, splitext, dirname, exists -from hashlib import md5 from tools.toolchains import mbedToolchain from tools.settings import GCC_ARM_PATH, GCC_CR_PATH @@ -69,7 +68,7 @@ class GCC(mbedToolchain): "-Wno-unused-parameter", "-Wno-missing-field-initializers", "-fmessage-length=0", "-fno-exceptions", "-fno-builtin", "-ffunction-sections", "-fdata-sections", - "-MMD", "-fno-delete-null-pointer-checks", "-fomit-frame-pointer" + "-fno-delete-null-pointer-checks", "-fomit-frame-pointer" ] + self.cpu if "save-asm" in self.options: @@ -162,22 +161,13 @@ class GCC(mbedToolchain): message + match.group('message') ) + def get_dep_option(self, object): + base, _ = splitext(object) + dep_path = base + '.d' + return ["-MD", "-MF", dep_path] + def get_compile_options(self, defines, includes): - cmd = [] - - sum = md5(' '.join(includes)).hexdigest() - options_file = join(self.temp_dir, "options_%s.txt" % sum) - if not exists(options_file): - with open(options_file, "wb") as f: - cmd_list = ['-D%s' % d for d in defines] - for c in includes: - if c: - cmd_list.append(('-I%s' % c) if not c.startswith('-') else c) - string = " ".join(cmd_list).replace("\\", "/") - f.write(string) - cmd.extend(['@%s' % options_file]) - - return cmd + return ['-D%s' % d for d in defines] + ['@%s' % self.get_inc_file(includes)] @hook_tool def assemble(self, source, object, includes): @@ -193,8 +183,12 @@ class GCC(mbedToolchain): @hook_tool def compile(self, cc, source, object, includes): # Build compile command - cmd = cc + self.get_compile_options(self.get_symbols(), includes) + ["-o", object, source] + cmd = cc + self.get_compile_options(self.get_symbols(), includes) + cmd.extend(self.get_dep_option(object)) + + cmd.extend(["-o", object, source]) + # Call cmdline hook cmd = self.hook.get_cmdline_compiler(cmd) diff --git a/tools/toolchains/iar.py b/tools/toolchains/iar.py index c449d560a6..7d2fb60992 100644 --- a/tools/toolchains/iar.py +++ b/tools/toolchains/iar.py @@ -17,7 +17,6 @@ limitations under the License. import re from os import remove from os.path import join, exists, dirname, splitext, exists -from hashlib import md5 from tools.toolchains import mbedToolchain from tools.settings import IAR_PATH @@ -73,6 +72,10 @@ class IAR(mbedToolchain): self.ar = join(IAR_BIN, "iarchive") self.elf2bin = join(IAR_BIN, "ielftool") + def parse_dependencies(self, dep_path): + return [path.strip() for path in open(dep_path).readlines() + if (path and not path.isspace())] + def parse_output(self, output): for line in output.splitlines(): match = IAR.DIAGNOSTIC_PATTERN.match(line) @@ -94,32 +97,17 @@ class IAR(mbedToolchain): match.group('message') ) - def get_dep_opt(self, dep_path): + def get_dep_option(self, object): + base, _ = splitext(object) + dep_path = base + '.d' return ["--dependencies", dep_path] - def cc_extra(self, base): + def cc_extra(self, object): + base, _ = splitext(object) return ["-l", base + '.s'] - def parse_dependencies(self, dep_path): - return [path.strip() for path in open(dep_path).readlines() - if (path and not path.isspace())] - def get_compile_options(self, defines, includes): - cmd = [] - - sum = md5(' '.join(includes)).hexdigest() - options_file = join(self.temp_dir, "options_%s.txt" % sum) - if not exists(options_file): - with open(options_file, "wb") as f: - cmd_list = ['-D%s' % d for d in defines] - for c in includes: - if c: - cmd_list.append(('-I%s' % c) if not c.startswith('-') else c) - string = " ".join(cmd_list).replace("\\", "/") - f.write(string) - cmd.extend(['-f', options_file]) - - return cmd + return ['-D%s' % d for d in defines] + ['-f', self.get_inc_file(includes)] @hook_tool def assemble(self, source, object, includes): @@ -137,11 +125,9 @@ class IAR(mbedToolchain): # Build compile command cmd = cc + self.get_compile_options(self.get_symbols(), includes) - base, _ = splitext(object) - dep_path = base + '.d' - cmd.extend(self.get_dep_opt(dep_path)) + cmd.extend(self.get_dep_option(object)) - cmd.extend(self.cc_extra(base)) + cmd.extend(self.cc_extra(object)) cmd.extend(["-o", object, source])