Merge pull request #9561 from theotherjimmy/test-resources

Tools changes for bare metal
pull/9628/head
Cruz Monrreal 2019-02-13 12:31:54 -06:00 committed by GitHub
commit b820ec8922
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
77 changed files with 543 additions and 10 deletions

View File

@ -0,0 +1,3 @@
{
"name": "greentea-client"
}

View File

@ -0,0 +1,3 @@
{
"name": "mbed-client-cli"
}

View File

@ -0,0 +1,3 @@
{
"name": "mbed-client-randlib"
}

View File

@ -0,0 +1,3 @@
{
"name": "mbed-coap"
}

View File

@ -0,0 +1,3 @@
{
"name": "nanostack-libservice"
}

View File

@ -0,0 +1,3 @@
{
"name": "unity"
}

View File

@ -0,0 +1,3 @@
{
"name": "mbedtls"
}

View File

@ -0,0 +1,3 @@
{
"name": "coap-service"
}

View File

@ -1,5 +1,6 @@
{
"name": "mbed-mesh-api",
"requires": ["nanostack"],
"config": {
"heap-size": {
"help": "Nanostack's heap size [bytes: 0-65534]",

View File

@ -0,0 +1,4 @@
{
"name": "nanostack-interface",
"requires": ["nanostack"]
}

View File

@ -1,5 +1,6 @@
{
"name": "nanostack-eventloop",
"requires": ["nanostack-hal"],
"config": {
"use_platform_tick_timer": {
"help": "Use platform provided low resolution tick timer for eventloop",

View File

@ -1,5 +1,6 @@
{
"name": "nanostack",
"requires": ["nanostack-eventloop", "coap-service"],
"config": {
"configuration": {
"help": "Build time configuration. Refer to Handbook for valid values. Default: full stack",

View File

@ -0,0 +1,3 @@
{
"name": "nfc"
}

View File

@ -0,0 +1,3 @@
{
"name": "system-storage"
}

View File

@ -156,7 +156,7 @@ def get_config(src_paths, target, toolchain_name=None, app_config=None):
cfg, macros = config.get_config_data()
features = config.get_features()
return cfg, macros, features
return cfg, macros, features, res
def is_official_target(target_name, version):
""" Returns True, None if a target is part of the official release for the

View File

@ -545,7 +545,6 @@ class Config(object):
# Check that we didn't already process this file
if full_path in self.processed_configs:
continue
self.processed_configs[full_path] = True
# Read the library configuration and add a "__full_config_path"
# attribute to it
try:
@ -570,6 +569,12 @@ class Config(object):
raise ConfigException("; ".join(
self.format_validation_error(x, config_file)
for x in errors))
if "requires" in self.app_config_data:
if cfg["name"] not in self.app_config_data["requires"]:
continue
self.app_config_data["requires"].extend(cfg.get("requires", []))
self.processed_configs[full_path] = True
cfg["__config_path"] = full_path
@ -1279,6 +1284,7 @@ class Config(object):
"""
# Update configuration files until added features creates no changes
prev_features = set()
prev_requires = set()
while True:
# Add/update the configuration with any .json files found while
# scanning
@ -1288,14 +1294,37 @@ class Config(object):
# Add features while we find new ones
features = set(self.get_features())
if features == prev_features:
requires = set(self.app_config_data.get("requires", []))
if features == prev_features and requires == prev_requires:
break
resources.add_features(features)
prev_features = features
prev_requires = requires
self.validate_config()
missing_requirements = {}
for name, lib in self.lib_config_data.items():
for req in lib.get("requires", []):
if req not in self.lib_config_data:
missing_requirements.setdefault(name, [])
missing_requirements[name].append(req)
if missing_requirements:
message = "; ".join(
"library '{}' requires {} which is not present".format(
name, ", ".join("'{}'".format(i) for i in missing)
)
for name, missing in missing_requirements.items()
)
raise ConfigException(message)
all_json_paths = [
cfg["__config_path"] for cfg in self.lib_config_data.values()
]
included_json_files = [
ref for ref in resources.get_file_refs(FileType.JSON)
if abspath(ref.path) in all_json_paths
]
resources.filter_by_libraries(included_json_files)
if (hasattr(self.target, "release_versions") and
"5" not in self.target.release_versions and
"rtos" in self.lib_config_data):

View File

@ -6,6 +6,13 @@
"type": "string"
}
},
"requires_definition": {
"description": "Required libraries",
"type": "array",
"items": {
"$ref": "#/name_definition"
}
},
"macro_definition": {
"description": "A list of extra macros that will be defined when compiling a project that includes this library.",
"type": "array",

View File

@ -16,6 +16,9 @@
"macros": {
"$ref": "definitions.json#/macro_definition"
},
"requires": {
"$ref": "definitions.json#/requires_definition"
},
"artifact_name": {
"type": "string"
}

View File

@ -13,6 +13,9 @@
"target_overrides": {
"$ref": "definitions.json#/target_overrides_definition"
},
"requires": {
"$ref": "definitions.json#/requires_definition"
},
"macros": {
"$ref": "definitions.json#/macro_definition"
}

View File

@ -56,7 +56,7 @@ if __name__ == '__main__':
options.prefix = options.prefix or [""]
try:
params, macros, features = get_config(
params, macros, features, _ = get_config(
options.source_dir,
target,
options.tool[0] if options.tool else None,

View File

@ -80,9 +80,17 @@ LEGACY_TOOLCHAIN_NAMES = {
'ARMC6': 'ARMC6',
}
MBED_LIB_FILENAME = 'mbed_lib.json'
MBED_APP_FILENAME = 'mbed_app.json'
CONFIG_FILES = set([
MBED_LIB_FILENAME,
MBED_APP_FILENAME
])
FileRef = namedtuple("FileRef", "name path")
class FileType(object):
C_SRC = "c"
CPP_SRC = "c++"
@ -126,6 +134,9 @@ class Resources(object):
# publicly accessible things
self.ignored_dirs = []
# library requirements
self._libs_filtered = None
# Pre-mbed 2.0 ignore dirs
self._legacy_ignore_dirs = (LEGACY_IGNORE_DIRS)
@ -260,9 +271,47 @@ class Resources(object):
file_name = file_name.replace(sep, self._sep)
self._file_refs[file_type].add(FileRef(file_name, file_path))
def _include_file(self, ref):
"""Determine if a given file ref should be included in the build
Files may be part of a library if a parent directory contains an
mbed_lib.json. If a file is part of a library, include or exclude
it based on the library it's part of.
If a file is not part of a library, it's included.
"""
_, path = ref
cur_dir = dirname(path)
included_lib_paths = [dirname(e.path) for e in self._libs_filtered]
excluded_lib_paths = [dirname(e.path) for e in self._excluded_libs]
while dirname(cur_dir) != cur_dir:
if cur_dir in included_lib_paths:
return True
elif cur_dir in excluded_lib_paths:
return False
cur_dir = dirname(cur_dir)
return True
def get_file_refs(self, file_type):
"""Return a list of FileRef for every file of the given type"""
return list(self._file_refs[file_type])
if self._libs_filtered is None:
return list(self._file_refs[file_type])
else:
return [
ref for ref in self._file_refs[file_type]
if self._include_file(ref)
]
def filter_by_libraries(self, libraries_included):
"""
Call after completely done scanning to filter resources based on
libraries
"""
self._libs_filtered = set(libraries_included)
all_library_refs = set(
ref for ref in self._file_refs[FileType.JSON]
if ref.name.endswith(MBED_LIB_FILENAME)
)
self._excluded_libs = all_library_refs - self._libs_filtered
def _get_from_refs(self, file_type, key):
return sorted([key(f) for f in self.get_file_refs(file_type)])

View File

@ -26,6 +26,14 @@ from os.path import join, isfile, dirname, abspath
from tools.build_api import get_config
from tools.targets import set_targets_json_location, Target, TARGET_NAMES
from tools.config import ConfigException, Config, ConfigParameter, ConfigMacro
from tools.resources import Resources
NOT_CONFIG = [
"expected_macros",
"expected_features",
"included_source",
"excluded_source",
]
def compare_config(cfg, expected):
"""Compare the output of config against a dictionary of known good results
@ -40,7 +48,7 @@ def compare_config(cfg, expected):
except KeyError:
return "Unexpected key '%s' in configuration data" % k
for k in expected:
if k not in ["expected_macros", "expected_features"] + list(cfg.keys()):
if k not in NOT_CONFIG + list(cfg.keys()):
return "Expected key '%s' was not found in configuration data" % k
return ""
@ -73,7 +81,7 @@ def test_config(name):
set_targets_json_location(targets_json if isfile(targets_json) else None)
for target, expected in test_data.items():
try:
cfg, macros, features = get_config(test_dir, target, "GCC_ARM")
cfg, macros, features, resources = get_config(test_dir, target, "GCC_ARM")
res = compare_config(cfg, expected)
assert not(res), res
expected_macros = expected.get("expected_macros", None)
@ -84,6 +92,24 @@ def test_config(name):
assert sorted(expected_macros) == sorted(macros)
if expected_features is not None:
assert sorted(expected_features) == sorted(features)
included_source = [
join(test_dir, src) for src in
expected.get("included_source", [])
]
excluded_source = [
join(test_dir, src) for src in
expected.get("excluded_source", [])
]
for typ in Resources.ALL_FILE_TYPES:
for _, path in resources.get_file_refs(typ):
if included_source and path in included_source:
included_source.remove(path)
if excluded_source:
assert(path not in excluded_source)
assert(not included_source)
if included_source:
assert(False)
except ConfigException as e:
err_msg = str(e)
if "exception_msg" not in expected:

View File

@ -1,5 +1,6 @@
{
"test_target": {
"expected_features": ["BOOTLOADER", "STORAGE"]
"expected_features": ["BOOTLOADER", "STORAGE"],
"included_source": ["FEATURE_BOOTLOADER/lib1/lib1.c", "FEATURE_STORAGE/lib2/lib2.c"]
}
}

View File

@ -0,0 +1,7 @@
{
"name": "lib1",
"requires": ["lib2"],
"config": {
"test": "BAD"
}
}

View File

@ -0,0 +1,9 @@
{
"name": "lib2",
"requires": ["lib3"],
"config": {
"test": {
"value": "BAD"
}
}
}

View File

@ -0,0 +1,9 @@
{
"requires" : ["lib1"],
"target_overrides": {
"test_target": {
"lib2.test": "GOOD",
"lib1.test": "GOOD"
}
}
}

View File

@ -0,0 +1,9 @@
{
"test_target": {
"supported_toolchains": ["GCC_ARM"],
"core": "Cortex-M0",
"extra_labels": [],
"features": [],
"default_lib": "std"
}
}

View File

@ -0,0 +1,5 @@
{
"test_target": {
"exception_msg": "'lib2' requires 'lib3' which is not present"
}
}

View File

@ -0,0 +1,8 @@
{
"name": "lib2",
"config": {
"test": {
"value": "BAD"
}
}
}

View File

@ -0,0 +1,6 @@
{
"name": "lib1",
"config": {
"test": "BAD"
}
}

View File

@ -0,0 +1,11 @@
{
"requires" : ["lib1"],
"target_overrides": {
"should_fail": {
"lib2.test": "GOOD"
},
"should_pass": {
"lib1.test": "GOOD"
}
}
}

View File

@ -0,0 +1,16 @@
{
"should_fail": {
"supported_toolchains": ["GCC_ARM"],
"core": "Cortex-M0",
"extra_labels": [],
"features": [],
"default_lib": "std"
},
"should_pass": {
"supported_toolchains": ["GCC_ARM"],
"core": "Cortex-M0",
"extra_labels": [],
"features": [],
"default_lib": "std"
}
}

View File

@ -0,0 +1,10 @@
{
"should_fail": {
"exception_msg": "Attempt to override undefined parameter 'lib2.test' in 'application[should_fail]'"
},
"should_pass": {
"lib1.test": "GOOD",
"excluded_source": ["lib1/lib2/lib2.c"],
"included_source": ["lib1/lib1.cpp"]
}
}

View File

@ -0,0 +1,7 @@
{
"name": "lib1",
"requires": ["lib2"],
"config": {
"test": "BAD"
}
}

View File

@ -0,0 +1,9 @@
{
"name": "lib2",
"requires": ["lib3"],
"config": {
"test": {
"value": "BAD"
}
}
}

View File

@ -0,0 +1,6 @@
{
"name": "lib3",
"config": {
"test": "BAD"
}
}

View File

@ -0,0 +1,10 @@
{
"requires" : ["lib1"],
"target_overrides": {
"test_target": {
"lib3.test": "GOOD",
"lib2.test": "GOOD",
"lib1.test": "GOOD"
}
}
}

View File

@ -0,0 +1,9 @@
{
"test_target": {
"supported_toolchains": ["GCC_ARM"],
"core": "Cortex-M0",
"extra_labels": [],
"features": [],
"default_lib": "std"
}
}

View File

@ -0,0 +1,8 @@
{
"test_target": {
"lib3.test": "GOOD",
"lib2.test": "GOOD",
"lib1.test": "GOOD",
"included_source": ["lib3/lib3.cpp"]
}
}

View File

@ -0,0 +1,6 @@
{
"name": "lib1",
"config": {
"test": "BAD"
}
}

View File

@ -0,0 +1,8 @@
{
"name": "lib2",
"config": {
"test": {
"value": "BAD"
}
}
}

View File

@ -0,0 +1,11 @@
{
"requires" : ["lib1"],
"target_overrides": {
"should_fail": {
"lib2.test": "GOOD"
},
"should_pass": {
"lib1.test": "GOOD"
}
}
}

View File

@ -0,0 +1,16 @@
{
"should_fail": {
"supported_toolchains": ["GCC_ARM"],
"core": "Cortex-M0",
"extra_labels": [],
"features": [],
"default_lib": "std"
},
"should_pass": {
"supported_toolchains": ["GCC_ARM"],
"core": "Cortex-M0",
"extra_labels": [],
"features": [],
"default_lib": "std"
}
}

View File

@ -0,0 +1,10 @@
{
"should_fail": {
"exception_msg": "Attempt to override undefined parameter 'lib2.test' in 'application[should_fail]'"
},
"should_pass": {
"lib1.test": "GOOD",
"excluded_source": ["lib2/lib2.c"],
"included_source": ["lib2/lib1/lib1.cpp"]
}
}

View File

@ -0,0 +1,6 @@
{
"name": "lib1",
"config": {
"test": "BAD"
}
}

View File

@ -0,0 +1,8 @@
{
"name": "lib2",
"config": {
"test": {
"value": "BAD"
}
}
}

View File

@ -0,0 +1,11 @@
{
"requires" : ["lib1"],
"target_overrides": {
"should_fail": {
"lib2.test": "GOOD"
},
"should_pass": {
"lib1.test": "GOOD"
}
}
}

View File

@ -0,0 +1,16 @@
{
"should_fail": {
"supported_toolchains": ["GCC_ARM"],
"core": "Cortex-M0",
"extra_labels": [],
"features": [],
"default_lib": "std"
},
"should_pass": {
"supported_toolchains": ["GCC_ARM"],
"core": "Cortex-M0",
"extra_labels": [],
"features": [],
"default_lib": "std"
}
}

View File

@ -0,0 +1,10 @@
{
"should_fail": {
"exception_msg": "Attempt to override undefined parameter 'lib2.test' in 'application[should_fail]'"
},
"should_pass": {
"lib1.test": "GOOD",
"excluded_source": ["lib2/lib2.c"],
"included_source": ["lib1/lib1.cpp"]
}
}

View File

@ -0,0 +1,137 @@
# mbed SDK
# Copyright (c) 2019 ARM Limited
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from os.path import dirname, join
from tools.resources import Resources, FileType
from tools.notifier.mock import MockNotifier
SRC_PATHS = {
'': join(dirname(__file__), 'source'),
# The online compiler uses a similar mapping, with the `.lib` suffix.
'mbed-os': join(dirname(__file__), 'mbed-os.lib'),
}
class ResourcesTest(unittest.TestCase):
"""
Tests for Resources objects
"""
def setUp(self):
"""
Called before each test case
:return:
"""
def test_basic_scan(self):
"""
Verify that the ordering of Target info addition and directory addition
does not matter, so long as all the Target info and all directories are
added.
"""
first = Resources(MockNotifier())
first._add_labels('TARGET', ['K64F'])
first._add_labels('TARGET', ['FRDM'])
for name, loc in SRC_PATHS.items():
print(name, loc)
first.add_directory(loc, into_path=name)
assert("main.cpp" in first.get_file_names(FileType.CPP_SRC))
def test_add_target_info(self):
"""
Verify that the ordering of Target info addition and directory addition
does not matter, so long as all the Target info and all directories are
added.
"""
first = Resources(MockNotifier())
middle = Resources(MockNotifier())
last = Resources(MockNotifier())
first._add_labels('TARGET', ['K64F'])
first._add_labels('TARGET', ['FRDM'])
middle._add_labels('TARGET', ['FRDM'])
for name, loc in SRC_PATHS.items():
first.add_directory(loc, into_path=name)
middle.add_directory(loc, into_path=name)
last.add_directory(loc, into_path=name)
middle._add_labels('TARGET', ['K64F'])
last._add_labels('TARGET', ['K64F'])
last._add_labels('TARGET', ['FRDM'])
for ftype in Resources.ALL_FILE_TYPES:
assert(set(first.get_file_refs(ftype))
== set(middle.get_file_refs(ftype)))
assert(set(last.get_file_refs(ftype))
== set(middle.get_file_refs(ftype)))
def test_detect_duplicates(self):
"""
Verify that detect_duplicates finds all of the duplicate object files
in the scanned tree.
"""
notifier = MockNotifier()
first = Resources(notifier)
first._add_labels('TARGET', ['K64F'])
for name, loc in SRC_PATHS.items():
first.add_directory(loc, into_path=name)
notifier.messages = []
first.detect_duplicates()
error_messages = "\n".join(
m['message'] for m in notifier.messages if m['type'] == 'tool_error'
)
assert(" eggs.o " in error_messages)
first._add_labels('TARGET', ['FRDM'])
first.detect_duplicates()
error_messages = "\n".join(
m['message'] for m in notifier.messages if m['type'] == 'tool_error'
)
assert(" eggs.o " in error_messages)
assert(" not-main.o " in error_messages)
assert(" main.o " in error_messages)
def test_filter_by_all_libraries(self):
"""
Assert something
"""
res = Resources(MockNotifier())
res._add_labels('TARGET', ['K64F', 'FRDM'])
for name, loc in SRC_PATHS.items():
res.add_directory(loc, into_path=name)
res.filter_by_libraries(res.get_file_refs(FileType.JSON))
assert("main.cpp" in res.get_file_names(FileType.CPP_SRC))
def test_filter_by_bm_lib(self):
res = Resources(MockNotifier())
res._add_labels('TARGET', ['K64F', 'FRDM'])
for name, loc in SRC_PATHS.items():
res.add_directory(loc, into_path=name)
filter_by = [
ref for ref in res.get_file_refs(FileType.JSON)
if join("platform", "bm", "mbed_lib.json") in ref.name
]
res.filter_by_libraries(filter_by)
assert("main.cpp" not in res.get_file_names(FileType.CPP_SRC))
assert(
join("mbed-os", "platform", "bm", "bm.cpp")
in res.get_file_names(FileType.CPP_SRC)
)
assert(
join("mbed-os", "TARGET_FRDM", "not-main.cpp")
in res.get_file_names(FileType.CPP_SRC)
)
if __name__ == '__main__':
unittest.main()

View File

View File

View File

View File

View File

View File

View File

View File

View File

@ -2245,7 +2245,7 @@ def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
else:
target_name = target
target = TARGET_MAP[target_name]
cfg, _, _ = get_config(base_source_paths, target, app_config=app_config)
cfg, _, _, _ = get_config(base_source_paths, target, app_config=app_config)
baud_rate = 9600
if 'platform.stdio-baud-rate' in cfg: