From d101c74e25f47a518f77c3734ca18c3d404d15e1 Mon Sep 17 00:00:00 2001 From: Brian Daniels Date: Thu, 9 Jun 2016 12:52:28 +0100 Subject: [PATCH] Adding .mbedignore logic to find_tests function Addresses issues #247 and #197 --- tools/build_api.py | 65 ++++++++++++++++++++++++++++++++++++++++++++-- tools/test_api.py | 49 ++++++++++++++++++++++------------ 2 files changed, 95 insertions(+), 19 deletions(-) diff --git a/tools/build_api.py b/tools/build_api.py index 354d4c05e5..fcbbad3c62 100644 --- a/tools/build_api.py +++ b/tools/build_api.py @@ -19,12 +19,13 @@ import re import tempfile import colorama - +from copy import copy from types import ListType from shutil import rmtree from os.path import join, exists, basename, abspath, normpath -from os import getcwd +from os import getcwd, walk from time import time +import fnmatch from tools.utils import mkdir, run_cmd, run_cmd_ext, NotSupportedException, ToolException from tools.paths import MBED_TARGETS_PATH, MBED_LIBRARIES, MBED_API, MBED_HAL, MBED_COMMON @@ -827,3 +828,63 @@ def write_build_report(build_report, template_filename, filename): with open(filename, 'w+') as f: f.write(template.render(failing_builds=build_report_failing, passing_builds=build_report_passing)) + + +def scan_for_source_paths(path, exclude_paths=None): + ignorepatterns = [] + paths = [] + + def is_ignored(file_path): + for pattern in ignorepatterns: + if fnmatch.fnmatch(file_path, pattern): + return True + return False + + + """ os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]]) + When topdown is True, the caller can modify the dirnames list in-place + (perhaps using del or slice assignment), and walk() will only recurse into + the subdirectories whose names remain in dirnames; this can be used to prune + the search, impose a specific order of visiting, or even to inform walk() + about directories the caller creates or renames before it resumes walk() + again. Modifying dirnames when topdown is False is ineffective, because in + bottom-up mode the directories in dirnames are generated before dirpath + itself is generated. + """ + for root, dirs, files in walk(path, followlinks=True): + # Remove ignored directories + # Check if folder contains .mbedignore + if ".mbedignore" in files : + with open (join(root,".mbedignore"), "r") as f: + lines=f.readlines() + lines = [l.strip() for l in lines] # Strip whitespaces + lines = [l for l in lines if l != ""] # Strip empty lines + lines = [l for l in lines if not re.match("^#",l)] # Strip comment lines + # Append root path to glob patterns + # and append patterns to ignorepatterns + ignorepatterns.extend([join(root,line.strip()) for line in lines]) + + for d in copy(dirs): + dir_path = join(root, d) + + # Always ignore hidden directories + if d.startswith('.'): + dirs.remove(d) + + # Remove dirs that already match the ignorepatterns + # to avoid travelling into them and to prevent them + # on appearing in include path. + if is_ignored(join(dir_path,"")): + dirs.remove(d) + + if exclude_paths: + for exclude_path in exclude_paths: + rel_path = relpath(dir_path, exclude_path) + if not (rel_path.startswith('..')): + dirs.remove(d) + break + + # Add root to include paths + paths.append(root) + + return paths diff --git a/tools/test_api.py b/tools/test_api.py index d238cf8172..b7d88c8492 100644 --- a/tools/test_api.py +++ b/tools/test_api.py @@ -55,6 +55,7 @@ from tools.build_api import prep_report from tools.build_api import prep_properties from tools.build_api import create_result from tools.build_api import add_result_to_report +from tools.build_api import scan_for_source_paths from tools.libraries import LIBRARIES, LIBRARY_MAP from tools.toolchains import TOOLCHAIN_BIN_PATH from tools.test_exporters import ReportExporter, ResultExporterType @@ -1965,6 +1966,12 @@ def test_path_to_name(path): def find_tests(base_dir): """Given any directory, walk through the subdirectories and find all tests""" + def is_subdir(path, directory): + path = os.path.realpath(path) + directory = os.path.realpath(directory) + relative = os.path.relpath(path, directory) + return not (relative.startswith(os.pardir + os.sep) and relative.startswith(os.pardir)) + def find_tests_in_tests_directory(directory): """Given a 'TESTS' directory, return a dictionary of test names and test paths. The formate of the dictionary is {"test-name": "./path/to/test"}""" @@ -1975,10 +1982,11 @@ def find_tests(base_dir): if d != "host_tests": # Loop on test case directories for td in os.listdir(os.path.join(directory, d)): - # Add test case to the results if it is a directory - test_case_path = os.path.join(directory, d, td) - if os.path.isdir(test_case_path): - tests[test_path_to_name(test_case_path)] = test_case_path + # Add test case to the results if it is a directory and not "host_tests" + if td != "host_tests": + test_case_path = os.path.join(directory, d, td) + if os.path.isdir(test_case_path): + tests[test_path_to_name(test_case_path)] = test_case_path return tests @@ -1994,20 +2002,27 @@ def find_tests(base_dir): # Not pointing at a "TESTS" directory, so go find one! tests = {} - for root, dirs, files in os.walk(base_dir): - # Don't search build directories - if '.build' in dirs: - dirs.remove('.build') + dirs = scan_for_source_paths(base_dir) + + test_and_sub_dirs = [x for x in dirs if tests_path in x] + test_dirs = [] + for potential_test_dir in test_and_sub_dirs: + good_to_add = True + if test_dirs: + for test_dir in test_dirs: + if is_subdir(potential_test_dir, test_dir): + good_to_add = False + break - # If a "TESTS" directory is found, find the tests inside of it - if tests_path in dirs: - # Remove it from the directory walk - dirs.remove(tests_path) - - # Get the tests inside of the "TESTS" directory - new_tests = find_tests_in_tests_directory(os.path.join(root, tests_path)) - if new_tests: - tests.update(new_tests) + if good_to_add: + test_dirs.append(potential_test_dir) + + # Only look at valid paths + for path in test_dirs: + # Get the tests inside of the "TESTS" directory + new_tests = find_tests_in_tests_directory(path) + if new_tests: + tests.update(new_tests) return tests