Clean all lint errors in configuration system

pull/10113/head
Jimmy Brisson 2019-03-14 12:11:59 -05:00
parent b80c961dae
commit a666e976a2
1 changed files with 249 additions and 124 deletions

View File

@ -18,15 +18,11 @@ from __future__ import print_function, division, absolute_import
from copy import deepcopy
from six import moves
import json
import six
import os
import re
from os.path import dirname, abspath, exists, join, isabs
import sys
from collections import namedtuple
from os.path import splitext, relpath
from intelhex import IntelHex
from os.path import relpath
from jinja2 import FileSystemLoader, StrictUndefined
from jinja2.environment import Environment
from jsonschema import Draft4Validator, RefResolver
@ -77,19 +73,21 @@ ALLOWED_FEATURES = [
]
# List of all possible ram memories that can be available for a target
RAM_ALL_MEMORIES = ['IRAM1', 'IRAM2', 'IRAM3', 'IRAM4', 'SRAM_OC', \
'SRAM_ITC', 'SRAM_DTC', 'SRAM_UPPER', 'SRAM_LOWER', \
RAM_ALL_MEMORIES = ['IRAM1', 'IRAM2', 'IRAM3', 'IRAM4', 'SRAM_OC',
'SRAM_ITC', 'SRAM_DTC', 'SRAM_UPPER', 'SRAM_LOWER',
'SRAM']
# List of all possible rom memories that can be available for a target
ROM_ALL_MEMORIES = ['IROM1', 'PROGRAM_FLASH', 'IROM2']
# Base class for all configuration exceptions
class ConfigException(Exception):
"""Config system only exception. Makes it easier to distinguish config
errors"""
pass
class UndefinedParameter(ConfigException):
def __init__(self, param, name, kind, label):
self.param = param
@ -103,6 +101,7 @@ class UndefinedParameter(ConfigException):
ConfigParameter.get_display_name(self.name, self.kind, self.label),
)
class ConfigParameter(object):
"""This class keeps information about a single configuration parameter"""
@ -121,13 +120,13 @@ class ConfigParameter(object):
allow_prefix=False)
self.defined_by = self.get_display_name(unit_name, unit_kind)
self.set_value(data.get("value", None), unit_name, unit_kind)
self.value_min = data.get("value_min")
self.value_max = data.get("value_max")
self.value_min = data.get("value_min")
self.value_max = data.get("value_max")
self.accepted_values = data.get("accepted_values")
self.help_text = data.get("help", None)
self.required = data.get("required", False)
self.conflicts = data.get("conflicts", [])
self.macro_name = data.get("macro_name", "MBED_CONF_%s" %
self.help_text = data.get("help", None)
self.required = data.get("required", False)
self.conflicts = data.get("conflicts", [])
self.macro_name = data.get("macro_name", "MBED_CONF_%s" %
self.sanitize(self.name.upper()))
self.config_errors = []
@ -148,7 +147,8 @@ class ConfigParameter(object):
allow_prefix - True to allow the original name to have a prefix, False
otherwise
"""
if name.find('.') == -1: # the name is not prefixed
# the name is not prefixed
if name.find('.') == -1:
if unit_kind == "target":
prefix = "target."
elif unit_kind == "application":
@ -172,8 +172,10 @@ class ConfigParameter(object):
unit_name, unit_kind, label)))
prefix = temp[0]
# Check if the given parameter prefix matches the expected prefix
if (unit_kind == "library" and prefix not in [unit_name, "target"]) or \
(unit_kind == "target" and prefix != "target"):
if (
(unit_kind == "library" and prefix not in [unit_name, "target"]) or
(unit_kind == "target" and prefix != "target")
):
raise ConfigException(
"Invalid prefix '%s' for parameter name '%s' in '%s'" %
(prefix, name, ConfigParameter.get_display_name(
@ -197,8 +199,11 @@ class ConfigParameter(object):
return "target:" + unit_name
elif unit_kind == "application":
return "application%s" % ("[%s]" % label if label else "")
else: # library
return "library:%s%s" % (unit_name, "[%s]" % label if label else "")
else: # library
return "library:%s%s" % (
unit_name,
"[%s]" % label if label else ""
)
@staticmethod
def sanitize(name):
@ -258,6 +263,7 @@ class ConfigParameter(object):
desc += " Conflicts with %s" % ", ".join(self.conflicts)
return desc
class ConfigMacro(object):
""" A representation of a configuration macro. It handles both macros
without a value (MACRO) and with a value (MACRO=VALUE)
@ -271,7 +277,9 @@ class ConfigMacro(object):
unit_kind - the type of macro this is
"""
self.name = name
self.defined_by = ConfigParameter.get_display_name(unit_name, unit_kind)
self.defined_by = ConfigParameter.get_display_name(
unit_name, unit_kind
)
if name.find("=") != -1:
tmp = name.split("=")
if len(tmp) != 2:
@ -283,6 +291,7 @@ class ConfigMacro(object):
self.macro_name = name
self.macro_value = None
class ConfigCumulativeOverride(object):
"""Representation of overrides for cumulative attributes"""
def __init__(self, name, additions=None, removals=None, strict=False):
@ -327,8 +336,8 @@ class ConfigCumulativeOverride(object):
"""Extend the list of override additions.
Positional arguments:
overrides - a list of a names that, when the override is evaluated, will
be added to the list
overrides - a list of a names that, when the override is evaluated,
will be added to the list
"""
for override in overrides:
if override in self.removals or \
@ -414,6 +423,7 @@ def _process_macros(mlist, macros, unit_name, unit_kind):
Region = namedtuple("Region", "name start size active filename")
RamRegion = namedtuple("RamRegion", "name start size active")
class Config(object):
"""'Config' implements the mbed configuration mechanism"""
@ -432,9 +442,13 @@ class Config(object):
full_path = os.path.join(directory, cls.__mbed_app_config_name)
if os.path.isfile(full_path):
if app_config_location is not None:
raise ConfigException("Duplicate '%s' file in '%s' and '%s'"
% (cls.__mbed_app_config_name,
cls.app_config_location, full_path))
raise ConfigException(
"Duplicate '{}' file in '{}' and '{}'".format(
cls.__mbed_app_config_name,
cls.app_config_location,
full_path
)
)
else:
app_config_location = full_path
return app_config_location
@ -445,7 +459,7 @@ class Config(object):
else:
return "in {} element {}: {}".format(
path, ".".join(p for p in error.absolute_path),
error.message.replace('u\'','\''))
error.message.replace('u\'', '\''))
def __init__(self, tgt, top_level_dirs=None, app_config=None):
"""Construct a mbed configuration
@ -470,20 +484,26 @@ class Config(object):
if self.app_config_location is None and top_level_dirs:
self.app_config_location = self.find_app_config(top_level_dirs)
try:
self.app_config_data = json_file_to_dict(self.app_config_location) \
if self.app_config_location else {}
except ValueError as exc:
if self.app_config_location:
self.app_config_data = json_file_to_dict(
self.app_config_location
)
else:
self.app_config_data = {}
except ValueError:
self.app_config_data = {}
config_errors.append(
ConfigException("Could not parse mbed app configuration from %s"
% self.app_config_location))
ConfigException(
"Could not parse mbed app configuration from %s"
% self.app_config_location
)
)
if self.app_config_location is not None:
# Validate the format of the JSON file based on schema_app.json
schema_root = os.path.dirname(os.path.abspath(__file__))
schema_path = os.path.join(schema_root, "schema_app.json")
schema = json_file_to_dict(schema_path)
schema = json_file_to_dict(schema_path)
url = moves.urllib.request.pathname2url(schema_path)
uri = moves.urllib_parse.urljoin("file://", url)
@ -523,7 +543,7 @@ class Config(object):
dirname(self.target._from_file),
getattr(self.target, attr)
)
setattr( self.target, attr, new_path)
setattr(self.target, attr, new_path)
self.cumulative_overrides = {key: ConfigCumulativeOverride(key)
for key in CUMULATIVE_ATTRIBUTES}
@ -572,7 +592,9 @@ class Config(object):
if "requires" in self.app_config_data:
if cfg["name"] not in self.app_config_data["requires"]:
continue
self.app_config_data["requires"].extend(cfg.get("requires", []))
self.app_config_data["requires"].extend(
cfg.get("requires", [])
)
self.processed_configs[full_path] = True
@ -582,9 +604,13 @@ class Config(object):
# name, exit with error
if cfg["name"] in self.lib_config_data:
raise ConfigException(
"Library name '%s' is not unique (defined in '%s' and '%s')"
% (cfg["name"], full_path,
self.lib_config_data[cfg["name"]]["__config_path"]))
"Library name '%s' is not unique "
"(defined in '%s' and '%s')".format(
cfg["name"],
full_path,
self.lib_config_data[cfg["name"]]["__config_path"]
)
)
self.lib_config_data[cfg["name"]] = cfg
@property
@ -664,9 +690,10 @@ class Config(object):
)
def get_all_active_memories(self, memory_list):
"""Get information of all available rom/ram memories in the form of dictionary
{Memory: [start_addr, size]}. Takes in the argument, a list of all available
regions within the ram/rom memory"""
"""Get information of all available rom/ram memories in the form of
dictionary {Memory: [start_addr, size]}. Takes in the argument, a
list of all available regions within the ram/rom memory
"""
# Override rom_start/rom_size
#
# This is usually done for a target which:
@ -676,7 +703,10 @@ class Config(object):
# Counter to keep track of ROM/RAM memories supported by target
active_memory_counter = 0
# Find which memory we are dealing with, RAM/ROM
active_memory = 'ROM' if any('ROM' in mem_list for mem_list in memory_list) else 'RAM'
if any('ROM' in mem_list for mem_list in memory_list):
active_memory = 'ROM'
else:
active_memory = 'RAM'
try:
cmsis_part = self._get_cmsis_part()
@ -685,14 +715,21 @@ class Config(object):
with ram and rom start/size defined"""
if getattr(self.target, "mbed_ram_start") and \
getattr(self.target, "mbed_rom_start"):
mem_start = int(getattr(self.target, "mbed_" + active_memory.lower() + "_start"), 0)
mem_size = int(getattr(self.target, "mbed_" + active_memory.lower() + "_size"), 0)
mem_start = int(getattr(
self.target,
"mbed_" + active_memory.lower() + "_start"
), 0)
mem_size = int(getattr(
self.target,
"mbed_" + active_memory.lower() + "_size"
), 0)
available_memories[active_memory] = [mem_start, mem_size]
return available_memories
else:
raise ConfigException("Bootloader not supported on this target. "
"ram/rom start/size not found in "
"targets.json.")
raise ConfigException(
"Bootloader not supported on this target. ram/rom "
"start/size not found in targets.json."
)
present_memories = set(cmsis_part['memories'].keys())
valid_memories = set(memory_list).intersection(present_memories)
@ -704,16 +741,19 @@ class Config(object):
for memory in valid_memories:
mem_start = memories[memory]["start"]
mem_size = memories[memory]["size"]
if memory=='IROM1' or memory=='PROGRAM_FLASH':
mem_start = getattr(self.target, "mbed_rom_start", False) or mem_start
mem_size = getattr(self.target, "mbed_rom_size", False) or mem_size
if memory in ['IROM1', 'PROGRAM_FLASH']:
if getattr(self.target, "mbed_rom_start", False):
mem_start = getattr(self.target, "mbed_rom_start")
if getattr(self.target, "mbed_rom_size", False):
mem_size = getattr(self.target, "mbed_rom_size")
memory = 'ROM'
elif memory == 'IRAM1' or memory == 'SRAM_OC' or \
memory == 'SRAM_UPPER' or memory == 'SRAM':
elif memory in ['IRAM1', 'SRAM_OC', 'SRAM_UPPER', 'SRAM']:
if (self.has_ram_regions):
continue
mem_start = getattr(self.target, "mbed_ram_start", False) or mem_start
mem_size = getattr(self.target, "mbed_ram_size", False) or mem_size
if getattr(self.target, "mbed_ram_start", False):
mem_start = getattr(self.target, "mbed_ram_start")
if getattr(self.target, "mbed_ram_start", False):
mem_size = getattr(self.target, "mbed_ram_size")
memory = 'RAM'
else:
active_memory_counter += 1
@ -754,23 +794,26 @@ class Config(object):
if not getattr(self.target, "bootloader_supported", False):
raise ConfigException("Bootloader not supported on this target.")
"""Generate a list of regions from the config"""
if ((self.target.bootloader_img or self.target.restrict_size) and
(self.target.mbed_app_start or self.target.mbed_app_size)):
if (
(self.target.bootloader_img or self.target.restrict_size) and
(self.target.mbed_app_start or self.target.mbed_app_size)
):
raise ConfigException(
"target.bootloader_img and target.restrict_size are "
"incompatible with target.mbed_app_start and "
"target.mbed_app_size")
rom = self.get_all_active_memories(ROM_ALL_MEMORIES)
if self.target.bootloader_img or self.target.restrict_size:
return self._generate_bootloader_build(self.get_all_active_memories(ROM_ALL_MEMORIES))
return self._generate_bootloader_build(rom)
else:
return self._generate_linker_overrides(self.get_all_active_memories(ROM_ALL_MEMORIES))
return self._generate_linker_overrides(rom)
@staticmethod
def header_member_size(member):
_, _, subtype, _ = member
try:
return int(subtype[:-2]) // 8
except:
except Exception:
if subtype.startswith("CRCITT32"):
return 32 // 8
elif subtype == "SHA256":
@ -799,7 +842,10 @@ class Config(object):
for s, e in regions:
if newstart > s and newstart < e:
raise ConfigException(
"Can not place %r region inside previous region" % region_name)
"Can not place {} region inside previous region".format(
region_name
)
)
return newstart
@staticmethod
@ -810,7 +856,7 @@ class Config(object):
next section, whichever is smaller
"""
# Sort the list by starting address
region_list = sorted(region_list, key=lambda x:x[0])
region_list = sorted(region_list, key=lambda x: x[0])
for s, e in region_list:
if start_address < s:
return s
@ -848,17 +894,32 @@ class Config(object):
end_address = stop
else:
break
if end_address == None:
raise ConfigException("bootloader segments don't fit within rom")
part_size = Config._align_ceiling(end_address, self.sectors) - rom_start
# Generate the region in the loop (bootloader0, bootloader1, ...)
yield Region("bootloader"+str(part_count), start, part_size, False, filename)
if end_address is None:
raise ConfigException(
"bootloader segments don't fit within rom"
)
part_size = Config._align_ceiling(
end_address, self.sectors
) - rom_start
# Generate the region in the loop
# (bootloader0, bootloader1, ...)
yield Region(
"bootloader{}".format(str(part_count)),
start,
part_size,
False,
filename
)
else:
# Number of segments is 1
_, end_address = part.segments()[0]
if (end_address > rom_end):
raise ConfigException("bootloader segments don't fit within rom")
part_size = Config._align_ceiling(end_address, self.sectors) - rom_start
raise ConfigException(
"bootloader segments don't fit within rom"
)
part_size = Config._align_ceiling(
end_address, self.sectors
) - rom_start
yield Region("bootloader", rom_start, part_size, False,
filename)
@ -866,26 +927,42 @@ class Config(object):
if self.target.header_format:
if self.target.header_offset:
start = self._assign_new_offset(
rom_start, self.target.header_offset, "header", regions)
rom_start,
self.target.header_offset,
"header",
regions
)
start, region = self._make_header_region(
start, self.target.header_format)
yield region._replace(filename=self.target.header_format)
if self.target.restrict_size is not None:
new_size = int(self.target.restrict_size, 0)
new_size = Config._align_floor(start + new_size, self.sectors) - start
new_size = Config._align_floor(
start + new_size, self.sectors
) - start
if self.target.app_offset:
start = self._assign_new_offset(rom_start, self.target.app_offset, "application", regions)
start = self._assign_new_offset(
rom_start,
self.target.app_offset,
"application",
regions
)
yield Region("application", start, new_size, True, None)
start += new_size
if self.target.header_format and not self.target.bootloader_img:
if self.target.header_offset:
start = self._assign_new_offset(
rom_start, self.target.header_offset, "header", regions)
rom_start,
self.target.header_offset,
"header",
regions
)
start, region = self._make_header_region(
start, self.target.header_format)
start, self.target.header_format
)
yield region
yield Region("post_application", start, rom_end - start,
@ -894,7 +971,8 @@ class Config(object):
if self.target.app_offset:
start = self._assign_new_offset(
rom_start, self.target.app_offset, "application", regions)
# compute the end address of the application region based on existing segments
# compute the end address of the application region based on
# existing segments
end = self._get_end_address(regions, start, rom_end)
yield Region("application", start, end - start,
True, None)
@ -924,13 +1002,18 @@ class Config(object):
@staticmethod
def _align_ceiling(address, sectors):
target_start, target_size = Config._find_sector(address, sectors)
sector_num = ((address - target_start) + target_size - 1) // target_size
return target_start + (sector_num * target_size)
sector = ((address - target_start) + target_size - 1) // target_size
return target_start + (sector * target_size)
@property
def report(self):
return {'app_config': self.app_config_location,
'library_configs': list(map(relpath, self.processed_configs.keys()))}
return {
'app_config': self.app_config_location,
'library_configs': list(map(
relpath,
self.processed_configs.keys()
))
}
def _generate_linker_overrides(self, rom_memories):
rom_start, rom_size = rom_memories.get('ROM')
@ -948,7 +1031,13 @@ class Config(object):
raise ConfigException("Application ends after ROM")
yield Region("application", start, size, True, None)
def _process_config_and_overrides(self, data, params, unit_name, unit_kind):
def _process_config_and_overrides(
self,
data,
params,
unit_name,
unit_kind
):
"""Process "config_parameters" and "target_config_overrides" into a
given dictionary
@ -1018,9 +1107,11 @@ class Config(object):
if full_name in params:
params[full_name].set_value(val, unit_name, unit_kind,
label)
elif (name.startswith("target.") and
elif (
name.startswith("target.") and
(unit_kind is "application" or
name in BOOTLOADER_OVERRIDES)):
name in BOOTLOADER_OVERRIDES)
):
_, attribute = name.split(".")
setattr(self.target, attribute, val)
continue
@ -1039,9 +1130,10 @@ class Config(object):
We consider the resolution order for our target and sort it by level
reversed, so that we first look at the top level target (the parent),
then its direct children, then the children of those children and so on,
until we reach self.target
TODO: this might not work so well in some multiple inheritance scenarios
then its direct children, then the children of those children and so
on, until we reach self.target
TODO: this might not work so well in some multiple inheritance
scenarios.
At each step, look at two keys of the target data:
- config_parameters: used to define new configuration parameters
- config_overrides: used to override already defined configuration
@ -1062,12 +1154,14 @@ class Config(object):
tname, "target")
# Then process overrides
for name, val in target_data.get("overrides", {}).items():
full_name = ConfigParameter.get_full_name(name, tname, "target")
full_name = ConfigParameter.get_full_name(
name, tname, "target"
)
# If the parameter name is not defined or if there isn't a path
# from this target to the target where the parameter was defined
# in the target inheritance tree, raise an error We need to use
# 'defined_by[7:]' to remove the "target:" prefix from
# defined_by
# from this target to the target where the parameter was
# defined in the target inheritance tree, raise an error We
# need to use 'defined_by[7:]' to remove the "target:"
# prefix from defined_by
rel_names = [tgt for tgt, _ in
get_resolution_order(self.target.json_data, tname,
[])]
@ -1082,10 +1176,10 @@ class Config(object):
def get_lib_config_data(self, target_data):
""" Read and interpret configuration data defined by libraries. It is
assumed that "add_config_files" above was already called and the library
configuration data exists in self.lib_config_data
assumed that "add_config_files" above was already called and the
library configuration data exists in self.lib_config_data
Arguments: None
Arguments: target_data
"""
macros = {}
for lib_name, lib_data in self.lib_config_data.items():
@ -1161,7 +1255,8 @@ class Config(object):
}
parameter_macros = {
p.macro_name: p.value for p in params.values() if p.value is not None
p.macro_name: p.value for p in params.values()
if p.value is not None
}
all_macros.update(parameter_macros)
@ -1235,23 +1330,29 @@ class Config(object):
err_msg = ""
for name, param in sorted(params.items()):
min = param.value_min
max = param.value_max
min = param.value_min
max = param.value_max
accepted = param.accepted_values
value = param.value
value = param.value
# Config parameters that are only defined but do not have a default
# value should not be range limited
if value is not None:
if (min is not None or max is not None) and (accepted is not None):
err_msg += "\n%s has both a range and list of accepted values specified. Please only "\
"specify either value_min and/or value_max, or accepted_values"\
% param
if (
(min is not None or max is not None) and
(accepted is not None)
):
err_msg += (
"\n%s has both a range and list of accepted values "
"specified. Please only specify either value_min "
"and/or value_max, or accepted_values"
% param
)
else:
if re.match(r'^(0[xX])[A-Fa-f0-9]+$|^[0-9]+$', str(value)):
# Value is a hexadecimal or numerical string value
# Convert to a python integer and range check/compare to
# accepted list accordingly
# Convert to a python integer and range check/compare
# to accepted list accordingly
if min is not None or max is not None:
# Numerical range check
@ -1261,31 +1362,54 @@ class Config(object):
min = int(str(min), 0) if min is not None else None
max = int(str(max), 0) if max is not None else None
if (min is not None and value < min) or (max is not None and value > max):
err_msg += "\nInvalid config range for %s, is not in the required range: [%s:%s]"\
% (param,
min if min is not None else "-inf",
max if max is not None else "inf")
if (
(min is not None and value < min) or
(max is not None and value > max)
):
err_msg += (
"\nInvalid config range for {}, is not in "
"the required range: [{}:{}]".format(
param,
min if min is not None else "-inf",
max if max is not None else "inf"
)
)
# Numerical accepted value check
elif accepted is not None and value not in accepted:
err_msg += "\nInvalid value for %s, is not an accepted value: %s"\
% (param, ", ".join(map(str, accepted)))
err_msg += (
"\nInvalid value for {}, is not an accepted "
"value: {}".format(
param,
", ".join(map(str, accepted))
)
)
else:
if min is not None or max is not None:
err_msg += "\nInvalid config range settings for %s. Range specifiers are not "\
"applicable to non-decimal/hexadecimal string values" % param
err_msg += (
"\nInvalid config range settings for {}. "
"Range specifiers are not applicable to "
"non-decimal/hexadecimal string values".format(
param
)
)
if accepted is not None and value not in accepted:
err_msg += "\nInvalid config range for %s, is not an accepted value: %s"\
% (param, ", ".join(accepted))
err_msg += (
"\nInvalid config range for {}, is not an "
"accepted value: {}".format(
param, ", ".join(accepted)
)
)
if (err_msg):
raise ConfigException(err_msg)
for error in self.config_errors:
if (isinstance(error, UndefinedParameter) and
error.param in params):
if (
isinstance(error, UndefinedParameter) and
error.param in params
):
continue
else:
raise error
@ -1309,11 +1433,8 @@ class Config(object):
param.name, param.value, conf.name, conf.value
)
)
return True
@property
def name(self):
if "artifact_name" in self.app_config_data:
@ -1371,9 +1492,11 @@ class Config(object):
if abspath(ref.path) in all_json_paths
]
resources.filter_by_libraries(included_json_files)
if (hasattr(self.target, "release_versions") and
"5" not in self.target.release_versions and
"rtos" in self.lib_config_data):
if (
hasattr(self.target, "release_versions") and
"5" not in self.target.release_versions and
"rtos" in self.lib_config_data
):
raise NotSupportedException("Target does not support mbed OS 5")
@staticmethod
@ -1393,7 +1516,9 @@ class Config(object):
"""
params, macros = config[0] or {}, config[1] or {}
Config._check_required_parameters(params)
params_with_values = [p for p in params.values() if p.value is not None]
params_with_values = [
p for p in params.values() if p.value is not None
]
ctx = {
"cfg_params": sorted([
(p.macro_name, str(p.value), p.set_by)
@ -1406,9 +1531,9 @@ class Config(object):
"name_len": max([len(m.macro_name) for m in macros.values()] +
[len(m.macro_name) for m in params_with_values]
+ [0]),
"val_len" : max([len(str(m.value)) for m in params_with_values] +
[len(m.macro_value or "") for m in macros.values()]
+ [0]),
"val_len": max([len(str(m.value)) for m in params_with_values] +
[len(m.macro_value or "") for m in macros.values()]
+ [0]),
}
jinja_loader = FileSystemLoader(dirname(abspath(__file__)))
jinja_environment = Environment(loader=jinja_loader,