mbed-os/tools/python/mbed_tools/build/_internal/find_files.py

#
# Copyright (c) 2020-2021 Arm Limited and Contributors. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
"""Find files in MbedOS program directory."""
from pathlib import Path
import fnmatch
from typing import Callable, Iterable, Optional, List, Tuple

from mbed_tools.lib.json_helpers import decode_json_file


def find_files(filename: str, directory: Path) -> List[Path]:
    """Proxy to `_find_files`, which applies legacy filtering rules."""
    # Temporary workaround, which replicates hardcoded ignore rules from old tools.
    # Legacy list of ignored directories is longer, however "TESTS" and
    # "TEST_APPS" were the only ones that actually exist in the MbedOS source.
    # Ideally, this should be solved by putting an `.mbedignore` file in the root of MbedOS repo,
    # similarly to what the code below pretends is happening.
    legacy_ignore = MbedignoreFilter(("*/TESTS", "*/TEST_APPS"))
    return _find_files(filename, directory, [legacy_ignore])


def _find_files(filename: str, directory: Path, filters: Optional[List[Callable]] = None) -> List[Path]:
    """Recursively find files by name under a given directory.

    This function automatically applies rules from .mbedignore files found during traversal.

    It is important to realise that applied filters are "greedy". The moment a directory is filtered out,
    its children won't be traversed.

    Args:
        filename: Name of the file to look for.
        directory: Location where search starts.
        filters: Optional list of exclude filters to apply.
    """
    if filters is None:
        filters = []

    result: List[Path] = []

    # Directories and files to process
    children = list(directory.iterdir())

    # If .mbedignore is one of the children, we need to add it to filter list,
    # as it might contain rules for currently processed directory, as well as its descendants.
    mbedignore = Path(directory, ".mbedignore")
    if mbedignore in children:
        filters = filters + [MbedignoreFilter.from_file(mbedignore)]

    # Remove files and directories that don't match current set of filters
    filtered_children = filter_files(children, filters)

    for child in filtered_children:
        if child.is_symlink():
            child = child.absolute().resolve()

        if child.is_dir():
            # If processed child is a directory, recurse with current set of filters
            result += _find_files(filename, child, filters)

        if child.is_file() and child.name == filename:
            # We've got a match
            result.append(child)

    return result


def filter_files(files: Iterable[Path], filters: Iterable[Callable]) -> Iterable[Path]:
    """Filter given paths to files using filter callables."""
    return [file for file in files if all(f(file) for f in filters)]


class RequiresFilter:
    """Filter out mbed libraries not needed by application.

    The 'requires' config option in mbed_app.json can specify list of mbed
    libraries (mbed_lib.json) that application requires. Apply 'requires'
    filter to remove mbed_lib.json files not required by application.
    """

    def __init__(self, requires: Iterable[str]):
        """Initialise the filter attributes.

        Args:
            requires: List of required mbed libraries.
        """
        self._requires = requires

    def __call__(self, path: Path) -> bool:
        """Return True if no requires are specified or our lib name is in the list of required libs."""
        return decode_json_file(path).get("name", "") in self._requires or not self._requires


class LabelFilter:
    """Filter out given paths using path labelling rules.

    If a path is labelled with given type, but contains label value which is
    not allowed, it will be filtered out.

    An example of labelled path is "/mbed-os/rtos/source/TARGET_CORTEX/mbed_lib.json",
    where label type is "TARGET" and label value is "CORTEX".

    For example, given a label type "FEATURE" and allowed values ["FOO"]:
    - "/path/FEATURE_FOO/somefile.txt" will not be filtered out
    - "/path/FEATURE_BAZ/somefile.txt" will be filtered out
    - "/path/FEATURE_FOO/FEATURE_BAR/somefile.txt" will be filtered out
    """

    def __init__(self, label_type: str, allowed_label_values: Iterable[str]):
        """Initialise the filter attributes.

        Args:
            label_type: Type of the label to filter with. In filtered paths, it prefixes the value.
            allowed_label_values: Values which are allowed for the given label type.
        """
        self._label_type = label_type
        self._allowed_labels = set(f"{label_type}_{label_value}" for label_value in allowed_label_values)

    def __call__(self, path: Path) -> bool:
        """Return True if given path contains only allowed labels - should not be filtered out."""
        labels = set(part for part in path.parts if self._label_type in part)
        return labels.issubset(self._allowed_labels)


class MbedignoreFilter:
    """Filter out given paths based on rules found in .mbedignore files.

    Patterns in .mbedignore use unix shell-style wildcards (fnmatch). It means
    that functionality, although similar is different to that found in
    .gitignore and friends.
    """

    def __init__(self, patterns: Tuple[str, ...]):
        """Initialise the filter attributes.

        Args:
            patterns: List of patterns from .mbedignore to filter against.
        """
        self._patterns = patterns

    def __call__(self, path: Path) -> bool:
        """Return True if given path doesn't match .mbedignore patterns - should not be filtered out."""
        stringified = str(path)
        return not any(fnmatch.fnmatch(stringified, pattern) for pattern in self._patterns)

    @classmethod
    def from_file(cls, mbedignore_path: Path) -> "MbedignoreFilter":
        """Return new instance with patterns read from .mbedignore file.

        Constructed patterns are rooted in the directory of .mbedignore file.
        """
        lines = mbedignore_path.read_text().splitlines()
        pattern_lines = (line for line in lines if line.strip() and not line.startswith("#"))
        ignore_root = mbedignore_path.parent
        patterns = tuple(str(ignore_root.joinpath(pattern)) for pattern in pattern_lines)
        return cls(patterns)