From bdb23b3cfbce9caf88b8a60ebda7f04ec6f0b1bf Mon Sep 17 00:00:00 2001 From: Brian Daniels Date: Thu, 5 Nov 2015 17:21:21 -0600 Subject: [PATCH] Simplifying and reducing code --- workspace_tools/build_api.py | 77 ++++++++------ workspace_tools/toolchains/__init__.py | 140 ++++++++++++------------- workspace_tools/toolchains/arm.py | 9 +- workspace_tools/toolchains/gcc.py | 7 +- workspace_tools/toolchains/iar.py | 9 +- 5 files changed, 116 insertions(+), 126 deletions(-) diff --git a/workspace_tools/build_api.py b/workspace_tools/build_api.py index fee96d3423..ce820f50c8 100644 --- a/workspace_tools/build_api.py +++ b/workspace_tools/build_api.py @@ -95,10 +95,10 @@ def build_project(src_path, build_path, target, toolchain_name, if name is None: # We will use default project name based on project folder name name = PROJECT_BASENAME - cur_result["output"] += toolchain.info("Building project %s (%s, %s)" % (PROJECT_BASENAME.upper(), target.name, toolchain_name)) + toolchain.info("Building project %s (%s, %s)" % (PROJECT_BASENAME.upper(), target.name, toolchain_name)) else: # User used custom global project name to have the same name for the - cur_result["output"] += toolchain.info("Building project %s to %s (%s, %s)" % (PROJECT_BASENAME.upper(), name, target.name, toolchain_name)) + toolchain.info("Building project %s to %s (%s, %s)" % (PROJECT_BASENAME.upper(), name, target.name, toolchain_name)) start = time() id_name = project_id.upper() @@ -141,17 +141,16 @@ def build_project(src_path, build_path, target, toolchain_name, # Compile Sources for path in src_paths: src = toolchain.scan_resources(path) - objects, build_output = toolchain.compile_sources(src, build_path, resources.inc_dirs) + objects = toolchain.compile_sources(src, build_path, resources.inc_dirs) resources.objects.extend(objects) - cur_result["output"] += build_output # Link Program - res, needed_update, build_output = toolchain.link_program(resources, build_path, name) - cur_result["output"] += build_output + res, needed_update = toolchain.link_program(resources, build_path, name) if report != None and needed_update: end = time() cur_result["elapsed_time"] = end - start + cur_result["output"] = toolchain.get_output() cur_result["result"] = "OK" add_result_to_report(report, cur_result) @@ -161,9 +160,14 @@ def build_project(src_path, build_path, target, toolchain_name, except Exception, e: end = time() cur_result["result"] = "FAIL" - cur_result["output"] = str(e) cur_result["elapsed_time"] = end - start + toolchain_output = toolchain.get_output() + if toolchain_output: + cur_result["output"] += toolchain_output + + cur_result["output"] += str(e) + add_result_to_report(report, cur_result) # Let Exception propagate @@ -221,7 +225,7 @@ def build_library(src_paths, build_path, target, toolchain_name, toolchain.jobs = jobs toolchain.build_all = clean - cur_result["output"] += toolchain.info("Building library %s (%s, %s)" % (name.upper(), target.name, toolchain_name)) + toolchain.info("Building library %s (%s, %s)" % (name.upper(), target.name, toolchain_name)) # Scan Resources resources = [] @@ -253,23 +257,21 @@ def build_library(src_paths, build_path, target, toolchain_name, # Copy Headers for resource in resources: - cur_result["output"] += toolchain.copy_files(resource.headers, build_path, rel_path=resource.base_path) + toolchain.copy_files(resource.headers, build_path, rel_path=resource.base_path) dependencies_include_dir.extend(toolchain.scan_resources(build_path).inc_dirs) # Compile Sources objects = [] for resource in resources: - objects, tmp_output = toolchain.compile_sources(resource, tmp_path, dependencies_include_dir) + objects = toolchain.compile_sources(resource, tmp_path, dependencies_include_dir) objects.extend(objects) - cur_result["output"] += tmp_output - needed_update, build_output = toolchain.build_library(objects, bin_path, name) - - cur_result["output"] += build_output + needed_update = toolchain.build_library(objects, bin_path, name) if report != None and needed_update: end = time() cur_result["elapsed_time"] = end - start + cur_result["output"] = toolchain.get_output() cur_result["result"] = "OK" add_result_to_report(report, cur_result) @@ -278,9 +280,14 @@ def build_library(src_paths, build_path, target, toolchain_name, if report != None: end = time() cur_result["result"] = "FAIL" - cur_result["output"] += str(e) cur_result["elapsed_time"] = end - start + toolchain_output = toolchain.get_output() + if toolchain_output: + cur_result["output"] += toolchain_output + + cur_result["output"] += str(e) + add_result_to_report(report, cur_result) # Let Exception propagate @@ -356,37 +363,34 @@ def build_mbed_libs(target, toolchain_name, options=None, verbose=False, clean=F mkdir(TMP_PATH) # CMSIS - cur_result["output"] += toolchain.info("Building library %s (%s, %s)"% ('CMSIS', target.name, toolchain_name)) + toolchain.info("Building library %s (%s, %s)"% ('CMSIS', target.name, toolchain_name)) cmsis_src = join(MBED_TARGETS_PATH, "cmsis") resources = toolchain.scan_resources(cmsis_src) - cur_result["output"] += toolchain.copy_files(resources.headers, BUILD_TARGET) - cur_result["output"] += toolchain.copy_files(resources.linker_script, BUILD_TOOLCHAIN) - cur_result["output"] += toolchain.copy_files(resources.bin_files, BUILD_TOOLCHAIN) + toolchain.copy_files(resources.headers, BUILD_TARGET) + toolchain.copy_files(resources.linker_script, BUILD_TOOLCHAIN) + toolchain.copy_files(resources.bin_files, BUILD_TOOLCHAIN) - objects, build_output = toolchain.compile_sources(resources, TMP_PATH) - cur_result["output"] += build_output - cur_result["output"] += toolchain.copy_files(objects, BUILD_TOOLCHAIN) + objects = toolchain.compile_sources(resources, TMP_PATH) + toolchain.copy_files(objects, BUILD_TOOLCHAIN) # mbed - cur_result["output"] += toolchain.info("Building library %s (%s, %s)" % ('MBED', target.name, toolchain_name)) + toolchain.info("Building library %s (%s, %s)" % ('MBED', target.name, toolchain_name)) # Common Headers - cur_result["output"] += toolchain.copy_files(toolchain.scan_resources(MBED_API).headers, MBED_LIBRARIES) - cur_result["output"] += toolchain.copy_files(toolchain.scan_resources(MBED_HAL).headers, MBED_LIBRARIES) + toolchain.copy_files(toolchain.scan_resources(MBED_API).headers, MBED_LIBRARIES) + toolchain.copy_files(toolchain.scan_resources(MBED_HAL).headers, MBED_LIBRARIES) # Target specific sources HAL_SRC = join(MBED_TARGETS_PATH, "hal") hal_implementation = toolchain.scan_resources(HAL_SRC) - cur_result["output"] += toolchain.copy_files(hal_implementation.headers + hal_implementation.hex_files + hal_implementation.libraries, BUILD_TARGET, HAL_SRC) + toolchain.copy_files(hal_implementation.headers + hal_implementation.hex_files + hal_implementation.libraries, BUILD_TARGET, HAL_SRC) incdirs = toolchain.scan_resources(BUILD_TARGET).inc_dirs - objects, build_output = toolchain.compile_sources(hal_implementation, TMP_PATH, [MBED_LIBRARIES] + incdirs) - cur_result["output"] += build_output + objects = toolchain.compile_sources(hal_implementation, TMP_PATH, [MBED_LIBRARIES] + incdirs) # Common Sources mbed_resources = toolchain.scan_resources(MBED_COMMON) - objects, build_output = toolchain.compile_sources(mbed_resources, TMP_PATH, [MBED_LIBRARIES] + incdirs) - cur_result["output"] += build_output + objects += toolchain.compile_sources(mbed_resources, TMP_PATH, [MBED_LIBRARIES] + incdirs) # A number of compiled files need to be copied as objects as opposed to # being part of the mbed library, for reasons that have to do with the way @@ -404,15 +408,15 @@ def build_mbed_libs(target, toolchain_name, options=None, verbose=False, clean=F for o in separate_objects: objects.remove(o) - needed_update, build_output = toolchain.build_library(objects, BUILD_TOOLCHAIN, "mbed") - cur_result["output"] += build_output + needed_update = toolchain.build_library(objects, BUILD_TOOLCHAIN, "mbed") for o in separate_objects: - cur_result["output"] += toolchain.copy_files(o, BUILD_TOOLCHAIN) + toolchain.copy_files(o, BUILD_TOOLCHAIN) if report != None and needed_update: end = time() cur_result["elapsed_time"] = end - start + cur_result["output"] = toolchain.get_output() cur_result["result"] = "OK" add_result_to_report(report, cur_result) @@ -423,9 +427,14 @@ def build_mbed_libs(target, toolchain_name, options=None, verbose=False, clean=F if report != None: end = time() cur_result["result"] = "FAIL" - cur_result["output"] += str(e) cur_result["elapsed_time"] = end - start + toolchain_output = toolchain.get_output() + if toolchain_output: + cur_result["output"] += toolchain_output + + cur_result["output"] += str(e) + add_result_to_report(report, cur_result) # Let Exception propagate diff --git a/workspace_tools/toolchains/__init__.py b/workspace_tools/toolchains/__init__.py index 2606720037..ccff696206 100644 --- a/workspace_tools/toolchains/__init__.py +++ b/workspace_tools/toolchains/__init__.py @@ -34,47 +34,6 @@ import workspace_tools.hooks as hooks #Disables multiprocessing if set to higher number than the host machine CPUs CPU_COUNT_MIN = 1 -def print_notify(event, silent=False): - """ Default command line notification - """ - msg = None - - if event['type'] in ['info', 'debug']: - msg = event['message'] - - elif event['type'] == 'cc': - event['severity'] = event['severity'].title() - event['file'] = basename(event['file']) - msg = '[%(severity)s] %(file)s@%(line)s: %(message)s' % event - - elif event['type'] == 'progress': - if not silent: - msg = '%s: %s' % (event['action'].title(), basename(event['file'])) - - if msg: - print msg - return msg + "\n" - -def print_notify_verbose(event, silent=False): - """ Default command line notification with more verbose mode - """ - if event['type'] in ['info', 'debug']: - return print_notify(event) # standard handle - - elif event['type'] == 'cc': - event['severity'] = event['severity'].title() - event['file'] = basename(event['file']) - event['mcu_name'] = "None" - event['toolchain'] = "None" - event['target_name'] = event['target_name'].upper() if event['target_name'] else "Unknown" - event['toolchain_name'] = event['toolchain_name'].upper() if event['toolchain_name'] else "Unknown" - msg = '[%(severity)s] %(target_name)s::%(toolchain_name)s::%(file)s@%(line)s: %(message)s' % event - print msg - return msg + "\n" - - elif event['type'] == 'progress': - return print_notify(event) # standard handle - def compile_worker(job): results = [] for command in job['commands']: @@ -226,10 +185,11 @@ class mbedToolchain: self.name = self.__class__.__name__ self.hook = hooks.Hook(target, self) self.silent = silent + self.output = "" self.legacy_ignore_dirs = LEGACY_IGNORE_DIRS - set([target.name, LEGACY_TOOLCHAIN_NAMES[self.name]]) - self.notify_fun = notify if notify is not None else print_notify + self.notify_fun = notify if notify is not None else self.print_notify self.options = options if options is not None else [] self.macros = macros or [] @@ -251,6 +211,50 @@ class mbedToolchain: self.mp_pool = None + def get_output(self): + return self.output + + def print_notify(self, event, silent=False): + """ Default command line notification + """ + msg = None + + if event['type'] in ['info', 'debug']: + msg = event['message'] + + elif event['type'] == 'cc': + event['severity'] = event['severity'].title() + event['file'] = basename(event['file']) + msg = '[%(severity)s] %(file)s@%(line)s: %(message)s' % event + + elif event['type'] == 'progress': + if not silent: + msg = '%s: %s' % (event['action'].title(), basename(event['file'])) + + if msg: + print msg + self.output += msg + "\n" + + def print_notify_verbose(self, event, silent=False): + """ Default command line notification with more verbose mode + """ + if event['type'] in ['info', 'debug']: + self.print_notify(event) # standard handle + + elif event['type'] == 'cc': + event['severity'] = event['severity'].title() + event['file'] = basename(event['file']) + event['mcu_name'] = "None" + event['toolchain'] = "None" + event['target_name'] = event['target_name'].upper() if event['target_name'] else "Unknown" + event['toolchain_name'] = event['toolchain_name'].upper() if event['toolchain_name'] else "Unknown" + msg = '[%(severity)s] %(target_name)s::%(toolchain_name)s::%(file)s@%(line)s: %(message)s' % event + print msg + self.output += msg + "\n" + + elif event['type'] == 'progress': + self.print_notify(event) # standard handle + def notify(self, event): """ Little closure for notify functions """ @@ -423,7 +427,6 @@ class mbedToolchain: return resources def copy_files(self, files_paths, trg_path, rel_path=None): - output = "" # Handle a single file if type(files_paths) != ListType: files_paths = [files_paths] @@ -441,12 +444,10 @@ class mbedToolchain: target = join(trg_path, relative_path) if (target != source) and (self.need_update(target, [source])): - output += self.progress("copy", relative_path) + self.progress("copy", relative_path) mkdir(dirname(target)) copyfile(source, target) - return output - def relative_object_path(self, build_path, base_dir, source): source_dir, name, _ = split_path(source) obj_dir = join(build_path, relpath(source_dir, base_dir)) @@ -505,25 +506,22 @@ class mbedToolchain: return self.compile_seq(queue, objects) def compile_seq(self, queue, objects): - output = "" - for item in queue: result = compile_worker(item) self.compiled += 1 - output += self.progress("compile", item['source'], build_update=True) + self.progress("compile", item['source'], build_update=True) for res in result['results']: - output += self.debug("Command: %s" % ' '.join(res['command'])) - output += self.compile_output([ + self.debug("Command: %s" % ' '.join(res['command'])) + self.compile_output([ res['code'], res['output'], res['command'] ]) objects.append(result['object']) - return objects, output + return objects def compile_queue(self, queue, objects): - output = "" jobs_count = int(self.jobs if self.jobs else cpu_count()) p = Pool(processes=jobs_count) @@ -547,10 +545,10 @@ class mbedToolchain: results.remove(r) self.compiled += 1 - output += self.progress("compile", result['source'], build_update=True) + self.progress("compile", result['source'], build_update=True) for res in result['results']: - output += self.debug("Command: %s" % ' '.join(res['command'])) - output += self.compile_output([ + self.debug("Command: %s" % ' '.join(res['command'])) + self.compile_output([ res['code'], res['output'], res['command'] @@ -575,7 +573,7 @@ class mbedToolchain: p.terminate() p.join() - return objects, output + return objects def compile_command(self, source, object, includes): # Check dependencies @@ -601,27 +599,24 @@ class mbedToolchain: return None def compile_output(self, output=[]): - tmp_output = "" _rc = output[0] _stderr = output[1] command = output[2] # Parse output for Warnings and Errors - tmp_output += self.parse_output(_stderr) - tmp_output += self.debug("Return: %s"% _rc) + self.parse_output(_stderr) + self.debug("Return: %s"% _rc) for error_line in _stderr.splitlines(): - tmp_output += self.debug("Output: %s"% error_line) + self.debug("Output: %s"% error_line) # Check return code if _rc != 0: for line in _stderr.splitlines(): - tmp_output += self.tool_error(line) + self.tool_error(line) raise ToolException(_stderr) - return tmp_output - def compile(self, cc, source, object, includes): _, ext = splitext(source) ext = ext.lower() @@ -646,18 +641,16 @@ class mbedToolchain: def build_library(self, objects, dir, name): needed_update = False - output = "" lib = self.STD_LIB_NAME % name fout = join(dir, lib) if self.need_update(fout, objects): - output = self.info("Library: %s" % lib) + self.info("Library: %s" % lib) self.archive(objects, fout) needed_update = True - return needed_update, output + return needed_update def link_program(self, r, tmp_path, name): - output = "" needed_update = False ext = 'bin' if hasattr(self.target, 'OUTPUT_EXT'): @@ -675,19 +668,19 @@ class mbedToolchain: if self.need_update(elf, r.objects + r.libraries + [r.linker_script]): needed_update = True - output += self.progress("link", name) + self.progress("link", name) self.link(elf, r.objects, r.libraries, r.lib_dirs, r.linker_script) if self.need_update(bin, [elf]): needed_update = True - output += self.progress("elf2bin", name) + self.progress("elf2bin", name) self.binary(r, elf, bin) self.var("compile_succeded", True) self.var("binary", filename) - return bin, needed_update, output + return bin, needed_update def default_cmd(self, command): _stdout, _stderr, _rc = run_cmd(command) @@ -713,14 +706,11 @@ class mbedToolchain: return self.notify({'type': 'info', 'message': message}) def debug(self, message): - output = "" if self.VERBOSE: if type(message) is ListType: message = ' '.join(message) message = "[DEBUG] " + message - output = self.notify({'type': 'debug', 'message': message}) - - return output + self.notify({'type': 'debug', 'message': message}) def cc_info(self, severity, file, line, message, target_name=None, toolchain_name=None): return self.notify({'type': 'cc', diff --git a/workspace_tools/toolchains/arm.py b/workspace_tools/toolchains/arm.py index f98bfb9fa4..36cddd042d 100644 --- a/workspace_tools/toolchains/arm.py +++ b/workspace_tools/toolchains/arm.py @@ -99,11 +99,10 @@ class ARM(mbedToolchain): return dependencies def parse_output(self, output): - tmp_output = "" for line in output.splitlines(): match = ARM.DIAGNOSTIC_PATTERN.match(line) if match is not None: - tmp_output += self.cc_info( + self.cc_info( match.group('severity').lower(), match.group('file'), match.group('line'), @@ -113,18 +112,16 @@ class ARM(mbedToolchain): ) match = self.goanna_parse_line(line) if match is not None: - tmp_output += self.cc_info( + self.cc_info( match.group('severity').lower(), match.group('file'), match.group('line'), match.group('message') ) - return tmp_output - def get_dep_opt(self, dep_path): return ["--depend", dep_path] - + def archive(self, objects, lib_path): self.default_cmd([self.ar, '-r', lib_path] + objects) diff --git a/workspace_tools/toolchains/gcc.py b/workspace_tools/toolchains/gcc.py index 9b21602810..d6746ccc4b 100644 --- a/workspace_tools/toolchains/gcc.py +++ b/workspace_tools/toolchains/gcc.py @@ -115,13 +115,12 @@ class GCC(mbedToolchain): def parse_output(self, output): # The warning/error notification is multiline - tmp_output = "" WHERE, WHAT = 0, 1 state, file, message = WHERE, None, None for line in output.splitlines(): match = self.goanna_parse_line(line) if match is not None: - tmp_output += self.cc_info( + self.cc_info( match.group('severity').lower(), match.group('file'), match.group('line'), @@ -148,14 +147,12 @@ class GCC(mbedToolchain): state = WHERE continue - tmp_output += self.cc_info( + self.cc_info( match.group('severity'), file, match.group('line'), message + match.group('message') ) - return tmp_output - def archive(self, objects, lib_path): self.default_cmd([self.ar, "rcs", lib_path] + objects) diff --git a/workspace_tools/toolchains/iar.py b/workspace_tools/toolchains/iar.py index f0d720ca98..6aa8539656 100644 --- a/workspace_tools/toolchains/iar.py +++ b/workspace_tools/toolchains/iar.py @@ -65,11 +65,10 @@ class IAR(mbedToolchain): self.elf2bin = join(IAR_BIN, "ielftool") def parse_output(self, output): - tmp_output = "" for line in output.splitlines(): match = IAR.DIAGNOSTIC_PATTERN.match(line) if match is not None: - tmp_output += self.cc_info( + self.cc_info( match.group('severity').lower(), match.group('file'), match.group('line'), @@ -79,14 +78,12 @@ class IAR(mbedToolchain): ) match = self.goanna_parse_line(line) if match is not None: - tmp_output += self.cc_info( + self.cc_info( match.group('severity').lower(), match.group('file'), match.group('line'), match.group('message') ) - - return tmp_output def get_dep_opt(self, dep_path): return ["--dependencies", dep_path] @@ -97,7 +94,7 @@ class IAR(mbedToolchain): def parse_dependencies(self, dep_path): return [path.strip() for path in open(dep_path).readlines() if (path and not path.isspace())] - + def assemble(self, source, object, includes): return [self.hook.get_cmdline_assembler(self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros] + ["-I%s" % i for i in includes] + ["-o", object, source])]