diff --git a/tools/resources/__init__.py b/tools/resources/__init__.py index 79ec719ad9..d19cabde5d 100644 --- a/tools/resources/__init__.py +++ b/tools/resources/__init__.py @@ -41,6 +41,8 @@ from os import walk, sep from os.path import (join, splitext, dirname, relpath, basename, split, normcase, abspath, exists) +from .ignore import MbedIgnoreSet, IGNORE_FILENAME + # Support legacy build conventions: the original mbed build system did not have # standard labels for the "TARGET_" and "TOOLCHAIN_" specific directories, but # had the knowledge of a list of these directories to be ignored. @@ -135,16 +137,15 @@ class Resources(object): # Incremental scan related self._label_paths = [] - self._labels = {"TARGET": [], "TOOLCHAIN": [], "FEATURE": []} + self._labels = { + "TARGET": [], "TOOLCHAIN": [], "FEATURE": [], "COMPONENT": [] + } self._prefixed_labels = set() # Path seperator style (defaults to OS-specific seperator) self._sep = sep - # Ignore patterns from .mbedignore files and add_ignore_patters - self._ignore_patterns = [] - self._ignore_regex = re.compile("$^") - + self._ignoreset = MbedIgnoreSet() def ignore_dir(self, directory): if self._collect_ignores: @@ -227,6 +228,8 @@ class Resources(object): def add_target_labels(self, target): self._add_labels("TARGET", target.labels) + self._add_labels("COMPONENT", target.components) + self.add_features(target.features) def add_features(self, features): self._add_labels("FEATURE", features) @@ -237,27 +240,9 @@ class Resources(object): self._legacy_ignore_dirs -= set( [toolchain.target.name, LEGACY_TOOLCHAIN_NAMES[toolchain.name]]) - def is_ignored(self, file_path): - """Check if file path is ignored by any .mbedignore thus far""" - return self._ignore_regex.match(normcase(file_path)) - def add_ignore_patterns(self, root, base_path, patterns): - """Add a series of patterns to the ignored paths - - Positional arguments: - root - the directory containing the ignore file - base_path - the location that the scan started from - patterns - the list of patterns we will ignore in the future - """ real_base = relpath(root, base_path) - if real_base == ".": - self._ignore_patterns.extend(normcase(p) for p in patterns) - else: - self._ignore_patterns.extend( - normcase(join(real_base, pat)) for pat in patterns) - if self._ignore_patterns: - self._ignore_regex = re.compile("|".join( - fnmatch.translate(p) for p in self._ignore_patterns)) + self._ignoreset.add_ignore_patterns(real_base, patterns) def _not_current_label(self, dirname, label_type): return (dirname.startswith(label_type + "_") and @@ -401,15 +386,13 @@ class Resources(object): for root, dirs, files in walk(path, followlinks=True): # Check if folder contains .mbedignore - if ".mbedignore" in files: - with open (join(root,".mbedignore"), "r") as f: - lines=f.readlines() - lines = [l.strip() for l in lines - if l.strip() != "" and not l.startswith("#")] - self.add_ignore_patterns(root, base_path, lines) + if IGNORE_FILENAME in files: + real_base = relpath(root, base_path) + self._ignoreset.add_mbedignore( + real_base, join(root, IGNORE_FILENAME)) root_path =join(relpath(root, base_path)) - if self.is_ignored(join(root_path,"")): + if self._ignoreset.is_ignored(join(root_path,"")): self.ignore_dir(root_path) dirs[:] = [] continue @@ -421,12 +404,12 @@ class Resources(object): self.add_file_ref(FileType.REPO_DIR, fake_path, dir_path) if (any(self._not_current_label(d, t) for t - in ['TARGET', 'TOOLCHAIN', 'FEATURE'])): + in self._labels.keys())): self._label_paths.append((dir_path, base_path, into_path)) self.ignore_dir(dir_path) dirs.remove(d) elif (d.startswith('.') or d in self._legacy_ignore_dirs or - self.is_ignored(join(root_path, d, ""))): + self._ignoreset.is_ignored(join(root_path, d, ""))): self.ignore_dir(dir_path) dirs.remove(d) @@ -468,7 +451,7 @@ class Resources(object): scanning starting as base_path """ - if (self.is_ignored(relpath(file_path, base_path)) or + if (self._ignoreset.is_ignored(relpath(file_path, base_path)) or basename(file_path).startswith(".")): self.ignore_dir(relpath(file_path, base_path)) return @@ -546,3 +529,4 @@ class Resources(object): self.add_directory(path) config.load_resources(self) return self + diff --git a/tools/resources/ignore.py b/tools/resources/ignore.py new file mode 100644 index 0000000000..b22b281af9 --- /dev/null +++ b/tools/resources/ignore.py @@ -0,0 +1,69 @@ +# mbed SDK +# Copyright (c) 2011-2013 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fnmatch +import re +from os.path import normcase, join + + +IGNORE_FILENAME = ".mbedignore" + + +class MbedIgnoreSet(object): + """ + # The mbedignore rules as an object + + A project in Mbed OS contains metadata files that exclude files from a build. + These rules are stored as `fnmatch` patterns in text in a files named `.mbedignore`. + """ + + def __init__(self): + self._ignore_patterns = [] + self._ignore_regex = re.compile("$^") + + def is_ignored(self, file_path): + """Check if file path is ignored by any .mbedignore thus far""" + return self._ignore_regex.match(normcase(file_path)) + + def add_ignore_patterns(self, in_name, patterns): + """Ignore all files and directories matching the paterns in + directories named by in_name. + + Positional arguments: + in_name - the filename prefix that this ignore will apply to + patterns - the list of patterns we will ignore in the future + """ + if in_name == ".": + self._ignore_patterns.extend(normcase(p) for p in patterns) + else: + self._ignore_patterns.extend( + normcase(join(in_name, pat)) for pat in patterns) + if self._ignore_patterns: + self._ignore_regex = re.compile("|".join( + fnmatch.translate(p) for p in self._ignore_patterns)) + + def add_mbedignore(self, in_name, filepath): + """Add a series of patterns to the ignored paths + + Positional arguments: + in_name - the filename prefix that this ignore will apply to + patterns - the list of patterns we will ignore in the future + """ + with open (filepath) as f: + patterns = [l.strip() for l in f + if l.strip() != "" and not l.startswith("#")] + self.add_ignore_patterns(in_name, patterns) + + diff --git a/tools/targets/__init__.py b/tools/targets/__init__.py index 141f16af18..66e741f1ec 100644 --- a/tools/targets/__init__.py +++ b/tools/targets/__init__.py @@ -91,7 +91,7 @@ def cached(func): # Cumulative attributes can have values appended to them, so they # need to be computed differently than regular attributes -CUMULATIVE_ATTRIBUTES = ['extra_labels', 'macros', 'device_has', 'features'] +CUMULATIVE_ATTRIBUTES = ['extra_labels', 'macros', 'device_has', 'features', 'components'] def get_resolution_order(json_data, target_name, order, level=0): @@ -222,8 +222,7 @@ class Target(namedtuple("Target", "name json_data resolution_order resolution_or def_idx = idx break else: - raise AttributeError("Attribute '%s' not found in target '%s'" - % (attrname, self.name)) + return [] # Get the starting value of the attribute starting_value = (tdata[self.resolution_order[def_idx][0]][attrname] or [])[:] diff --git a/tools/test_api.py b/tools/test_api.py index ccde1158d7..121b18016c 100644 --- a/tools/test_api.py +++ b/tools/test_api.py @@ -40,7 +40,7 @@ try: from Queue import Queue, Empty except ImportError: from queue import Queue, Empty -from os.path import join, exists, basename, relpath, isdir +from os.path import join, exists, basename, relpath, isdir, isfile from threading import Thread, Lock from multiprocessing import Pool, cpu_count from subprocess import Popen, PIPE @@ -55,6 +55,7 @@ from tools.utils import NotSupportedException from tools.utils import construct_enum from tools.memap import MemapParser from tools.targets import TARGET_MAP, Target +from tools.config import Config import tools.test_configs as TestConfig from tools.test_db import BaseDBAccess from tools.build_api import build_project, build_mbed_libs, build_lib @@ -66,7 +67,7 @@ from tools.build_api import create_result from tools.build_api import add_result_to_report from tools.build_api import prepare_toolchain from tools.build_api import get_config -from tools.resources import Resources +from tools.resources import Resources, MbedIgnoreSet, IGNORE_FILENAME from tools.libraries import LIBRARIES, LIBRARY_MAP from tools.options import extract_profile from tools.toolchains import TOOLCHAIN_PATHS @@ -2082,19 +2083,29 @@ def find_tests(base_dir, target_name, toolchain_name, app_config=None): # List of common folders: (predicate function, path) tuple commons = [] + config = Config(target_name, base_dir, app_config) + # Scan the directory for paths to probe for 'TESTS' folders base_resources = Resources(MockNotifier(), collect_ignores=True) - base_resources.add_directory(base_dir) + base_resources.scan_with_config(base_dir, config) dirs = [d for d in base_resources.ignored_dirs if basename(d) == 'TESTS'] + ignoreset = MbedIgnoreSet() + for directory in dirs: + ignorefile = join(directory, IGNORE_FILENAME) + if isfile(ignorefile): + ignoreset.add_mbedignore(directory, ignorefile) for test_group_directory in os.listdir(directory): grp_dir = join(directory, test_group_directory) - if not isdir(grp_dir): + if not isdir(grp_dir) or ignoreset.is_ignored(grp_dir): continue + grpignorefile = join(grp_dir, IGNORE_FILENAME) + if isfile(grpignorefile): + ignoreset.add_mbedignore(grp_dir, grpignorefile) for test_case_directory in os.listdir(grp_dir): d = join(directory, test_group_directory, test_case_directory) - if not isdir(d): + if not isdir(d) or ignoreset.is_ignored(d): continue special_dirs = ['host_tests', 'COMMON'] if test_group_directory not in special_dirs and test_case_directory not in special_dirs: diff --git a/tools/toolchains/__init__.py b/tools/toolchains/__init__.py index f645615c89..edf901bcab 100644 --- a/tools/toolchains/__init__.py +++ b/tools/toolchains/__init__.py @@ -202,6 +202,8 @@ class mbedToolchain: self.cxx_symbols += ["DEVICE_" + data + "=1" for data in self.target.device_has] # Add target's features self.cxx_symbols += ["FEATURE_" + data + "=1" for data in self.target.features] + # Add target's components + self.cxx_symbols += ["COMPONENT_" + data + "=1" for data in self.target.components] # Add extra symbols passed via 'macros' parameter self.cxx_symbols += self.macros @@ -221,6 +223,7 @@ class mbedToolchain: self.labels = { 'TARGET': self.target.labels, 'FEATURE': self.target.features, + 'COMPONENT': self.target.components, 'TOOLCHAIN': toolchain_labels }