Merge pull request #7644 from theotherjimmy/components

Tools: Scan for "components"

tools/resources/__init__.py

@@ -41,6 +41,8 @@ from os import walk, sep
 from os.path import (join, splitext, dirname, relpath, basename, split, normcase,
                      abspath, exists)
 
+from .ignore import MbedIgnoreSet, IGNORE_FILENAME
+
 # Support legacy build conventions: the original mbed build system did not have
 # standard labels for the "TARGET_" and "TOOLCHAIN_" specific directories, but
 # had the knowledge of a list of these directories to be ignored.
@@ -135,16 +137,15 @@ class Resources(object):
 
         # Incremental scan related
         self._label_paths = []
-        self._labels = {"TARGET": [], "TOOLCHAIN": [], "FEATURE": []}
+        self._labels = {
+            "TARGET": [], "TOOLCHAIN": [], "FEATURE": [], "COMPONENT": []
+        }
         self._prefixed_labels = set()
 
         # Path seperator style (defaults to OS-specific seperator)
         self._sep = sep
 
-        # Ignore patterns from .mbedignore files and add_ignore_patters
-        self._ignore_patterns = []
-        self._ignore_regex = re.compile("$^")
-
+        self._ignoreset = MbedIgnoreSet()
 
     def ignore_dir(self, directory):
         if self._collect_ignores:
@@ -227,6 +228,8 @@ class Resources(object):
 
     def add_target_labels(self, target):
         self._add_labels("TARGET", target.labels)
+        self._add_labels("COMPONENT", target.components)
+        self.add_features(target.features)
 
     def add_features(self, features):
         self._add_labels("FEATURE", features)
@@ -237,27 +240,9 @@ class Resources(object):
         self._legacy_ignore_dirs -= set(
             [toolchain.target.name, LEGACY_TOOLCHAIN_NAMES[toolchain.name]])
 
-    def is_ignored(self, file_path):
-        """Check if file path is ignored by any .mbedignore thus far"""
-        return self._ignore_regex.match(normcase(file_path))
-
     def add_ignore_patterns(self, root, base_path, patterns):
-        """Add a series of patterns to the ignored paths
-
-        Positional arguments:
-        root - the directory containing the ignore file
-        base_path - the location that the scan started from
-        patterns - the list of patterns we will ignore in the future
-        """
         real_base = relpath(root, base_path)
-        if real_base == ".":
-            self._ignore_patterns.extend(normcase(p) for p in patterns)
-        else:
-            self._ignore_patterns.extend(
-                normcase(join(real_base, pat)) for pat in patterns)
-        if self._ignore_patterns:
-            self._ignore_regex = re.compile("|".join(
-                fnmatch.translate(p) for p in self._ignore_patterns))
+        self._ignoreset.add_ignore_patterns(real_base, patterns)
 
     def _not_current_label(self, dirname, label_type):
         return (dirname.startswith(label_type + "_") and
@@ -401,15 +386,13 @@ class Resources(object):
 
         for root, dirs, files in walk(path, followlinks=True):
             # Check if folder contains .mbedignore
-            if ".mbedignore" in files:
-                with open (join(root,".mbedignore"), "r") as f:
-                    lines=f.readlines()
-                    lines = [l.strip() for l in lines
-                             if l.strip() != "" and not l.startswith("#")]
-                    self.add_ignore_patterns(root, base_path, lines)
+            if IGNORE_FILENAME in files:
+                real_base = relpath(root, base_path)
+                self._ignoreset.add_mbedignore(
+                    real_base, join(root, IGNORE_FILENAME))
 
             root_path =join(relpath(root, base_path))
-            if self.is_ignored(join(root_path,"")):
+            if self._ignoreset.is_ignored(join(root_path,"")):
                 self.ignore_dir(root_path)
                 dirs[:] = []
                 continue
@@ -421,12 +404,12 @@ class Resources(object):
                     self.add_file_ref(FileType.REPO_DIR, fake_path, dir_path)
 
                 if (any(self._not_current_label(d, t) for t
-                        in ['TARGET', 'TOOLCHAIN', 'FEATURE'])):
+                        in self._labels.keys())):
                     self._label_paths.append((dir_path, base_path, into_path))
                     self.ignore_dir(dir_path)
                     dirs.remove(d)
                 elif (d.startswith('.') or d in self._legacy_ignore_dirs or
-                      self.is_ignored(join(root_path, d, ""))):
+                      self._ignoreset.is_ignored(join(root_path, d, ""))):
                     self.ignore_dir(dir_path)
                     dirs.remove(d)
 
@@ -468,7 +451,7 @@ class Resources(object):
         scanning starting as base_path
         """
 
-        if  (self.is_ignored(relpath(file_path, base_path)) or
+        if  (self._ignoreset.is_ignored(relpath(file_path, base_path)) or
              basename(file_path).startswith(".")):
             self.ignore_dir(relpath(file_path, base_path))
             return
@@ -546,3 +529,4 @@ class Resources(object):
                 self.add_directory(path)
         config.load_resources(self)
         return self
+

tools/resources/ignore.py

@@ -0,0 +1,69 @@
+# mbed SDK
+# Copyright (c) 2011-2013 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import fnmatch
+import re
+from os.path import normcase, join
+
+
+IGNORE_FILENAME = ".mbedignore"
+
+
+class MbedIgnoreSet(object):
+    """
+    # The mbedignore rules as an object
+
+    A project in Mbed OS contains metadata files that exclude files from a build.
+    These rules are stored as `fnmatch` patterns in text in a files named `.mbedignore`.
+    """
+
+    def __init__(self):
+        self._ignore_patterns = []
+        self._ignore_regex = re.compile("$^")
+
+    def is_ignored(self, file_path):
+        """Check if file path is ignored by any .mbedignore thus far"""
+        return self._ignore_regex.match(normcase(file_path))
+
+    def add_ignore_patterns(self, in_name, patterns):
+        """Ignore all files and directories matching the paterns in 
+        directories named by in_name.
+
+        Positional arguments:
+        in_name - the filename prefix that this ignore will apply to
+        patterns - the list of patterns we will ignore in the future
+        """
+        if in_name == ".":
+            self._ignore_patterns.extend(normcase(p) for p in patterns)
+        else:
+            self._ignore_patterns.extend(
+                normcase(join(in_name, pat)) for pat in patterns)
+        if self._ignore_patterns:
+            self._ignore_regex = re.compile("|".join(
+                fnmatch.translate(p) for p in self._ignore_patterns))
+
+    def add_mbedignore(self, in_name, filepath):
+        """Add a series of patterns to the ignored paths
+
+        Positional arguments:
+        in_name - the filename prefix that this ignore will apply to
+        patterns - the list of patterns we will ignore in the future
+        """
+        with open (filepath) as f:
+            patterns = [l.strip() for l in f
+                        if l.strip() != "" and not l.startswith("#")]
+            self.add_ignore_patterns(in_name, patterns)
+
+

tools/targets/__init__.py

@@ -91,7 +91,7 @@ def cached(func):
 
 # Cumulative attributes can have values appended to them, so they
 # need to be computed differently than regular attributes
-CUMULATIVE_ATTRIBUTES = ['extra_labels', 'macros', 'device_has', 'features']
+CUMULATIVE_ATTRIBUTES = ['extra_labels', 'macros', 'device_has', 'features', 'components']
 
 
 def get_resolution_order(json_data, target_name, order, level=0):
@@ -222,8 +222,7 @@ class Target(namedtuple("Target", "name json_data resolution_order resolution_or
                 def_idx = idx
                 break
         else:
-            raise AttributeError("Attribute '%s' not found in target '%s'"
-                                 % (attrname, self.name))
+            return []
         # Get the starting value of the attribute
         starting_value = (tdata[self.resolution_order[def_idx][0]][attrname]
                           or [])[:]

tools/test_api.py

@@ -40,7 +40,7 @@ try:
     from Queue import Queue, Empty
 except ImportError:
     from queue import Queue, Empty
-from os.path import join, exists, basename, relpath, isdir
+from os.path import join, exists, basename, relpath, isdir, isfile
 from threading import Thread, Lock
 from multiprocessing import Pool, cpu_count
 from subprocess import Popen, PIPE
@@ -55,6 +55,7 @@ from tools.utils import NotSupportedException
 from tools.utils import construct_enum
 from tools.memap import MemapParser
 from tools.targets import TARGET_MAP, Target
+from tools.config import Config
 import tools.test_configs as TestConfig
 from tools.test_db import BaseDBAccess
 from tools.build_api import build_project, build_mbed_libs, build_lib
@@ -66,7 +67,7 @@ from tools.build_api import create_result
 from tools.build_api import add_result_to_report
 from tools.build_api import prepare_toolchain
 from tools.build_api import get_config
-from tools.resources import Resources
+from tools.resources import Resources, MbedIgnoreSet, IGNORE_FILENAME
 from tools.libraries import LIBRARIES, LIBRARY_MAP
 from tools.options import extract_profile
 from tools.toolchains import TOOLCHAIN_PATHS
@@ -2082,19 +2083,29 @@ def find_tests(base_dir, target_name, toolchain_name, app_config=None):
     # List of common folders: (predicate function, path) tuple
     commons = []
 
+    config = Config(target_name, base_dir, app_config)
+
     # Scan the directory for paths to probe for 'TESTS' folders
     base_resources = Resources(MockNotifier(), collect_ignores=True)
-    base_resources.add_directory(base_dir)
+    base_resources.scan_with_config(base_dir, config)
 
     dirs = [d for d in base_resources.ignored_dirs if basename(d) == 'TESTS']
+    ignoreset = MbedIgnoreSet()
+
     for directory in dirs:
+        ignorefile = join(directory, IGNORE_FILENAME)
+        if isfile(ignorefile):
+            ignoreset.add_mbedignore(directory, ignorefile)
         for test_group_directory in os.listdir(directory):
             grp_dir = join(directory, test_group_directory)
-            if not isdir(grp_dir):
+            if not isdir(grp_dir) or ignoreset.is_ignored(grp_dir):
                 continue
+            grpignorefile = join(grp_dir, IGNORE_FILENAME)
+            if isfile(grpignorefile):
+                ignoreset.add_mbedignore(grp_dir, grpignorefile)
             for test_case_directory in os.listdir(grp_dir):
                 d = join(directory, test_group_directory, test_case_directory)
-                if not isdir(d):
+                if not isdir(d) or ignoreset.is_ignored(d):
                     continue
                 special_dirs = ['host_tests', 'COMMON']
                 if test_group_directory not in special_dirs and test_case_directory not in special_dirs:

tools/toolchains/__init__.py

@@ -202,6 +202,8 @@ class mbedToolchain:
                 self.cxx_symbols += ["DEVICE_" + data + "=1" for data in self.target.device_has]
                 # Add target's features
                 self.cxx_symbols += ["FEATURE_" + data + "=1" for data in self.target.features]
+                # Add target's components
+                self.cxx_symbols += ["COMPONENT_" + data + "=1" for data in self.target.components]
                 # Add extra symbols passed via 'macros' parameter
                 self.cxx_symbols += self.macros
 
@@ -221,6 +223,7 @@ class mbedToolchain:
             self.labels = {
                 'TARGET': self.target.labels,
                 'FEATURE': self.target.features,
+                'COMPONENT': self.target.components,
                 'TOOLCHAIN': toolchain_labels
             }