ARMmbed
/
mbed-os
mirror of https://github.com/ARMmbed/mbed-os.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Allow memap.py to process memory bank information (#345) 

* Begin modernizing memap, add parsing of start address and symbol names

* Memap can now process and print memory bank info!

* Rename symbols back to modules, it was correct before

* Remove trailing whitespace

* Add type hints to modernized functions

* Apply suggestions from code review

Co-authored-by: VictorWTang <33789988+VictorWTang@users.noreply.github.com>

* Remove else

Co-authored-by: VictorWTang <33789988+VictorWTang@users.noreply.github.com>

* Clarify statement

---------

Co-authored-by: Victor Tang <me@victorwtang.dev>
Co-authored-by: VictorWTang <33789988+VictorWTang@users.noreply.github.com>
pull/15530/head
Jamie Smith 2024-09-23 20:50:05 -07:00 committed by GitHub
parent 037fe9b619
commit 0a7652033c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 398 additions and 552 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
targets/TARGET_NXP/TARGET_LPC176X/device/TOOLCHAIN_GCC_ARM/startup_LPC17xx.S
View File

@ -30,20 +30,6 @@
aborting compilation, it is not the run time limit: aborting compilation, it is not the run time limit:
Heap_Size + Stack_Size = 0x80 + 0x80 = 0x100 Heap_Size + Stack_Size = 0x80 + 0x80 = 0x100
*/ */
.section .stack
.align 3
#ifdef __STACK_SIZE
.equ Stack_Size, __STACK_SIZE
#else
.equ Stack_Size, 0xc00
#endif
.globl __StackTop
.globl __StackLimit
__StackLimit:
.space Stack_Size
.size __StackLimit, . - __StackLimit
__StackTop:
.size __StackTop, . - __StackTop
.section .heap .section .heap
.align 3 .align 3

7
tools/cmake/mbed_target_functions.cmake
View File

@ -48,6 +48,9 @@ function(mbed_generate_map_file target)
set(MBED_MEMAP_CREATE_JSON FALSE CACHE BOOL "create report in json file") set(MBED_MEMAP_CREATE_JSON FALSE CACHE BOOL "create report in json file")
set(MBED_MEMAP_CREATE_HTML FALSE CACHE BOOL "create report in html file") set(MBED_MEMAP_CREATE_HTML FALSE CACHE BOOL "create report in html file")
# Config process saves the JSON file here
set(MEMORY_BANKS_JSON_PATH ${CMAKE_BINARY_DIR}/memory_banks.json)
# generate table for screen # generate table for screen
add_custom_command( add_custom_command(
TARGET TARGET
@ -55,7 +58,7 @@ function(mbed_generate_map_file target)
POST_BUILD POST_BUILD
COMMAND ${Python3_EXECUTABLE} -m memap.memap COMMAND ${Python3_EXECUTABLE} -m memap.memap
-t ${MBED_TOOLCHAIN} ${CMAKE_CURRENT_BINARY_DIR}/${target}${CMAKE_EXECUTABLE_SUFFIX}.map -t ${MBED_TOOLCHAIN} ${CMAKE_CURRENT_BINARY_DIR}/${target}${CMAKE_EXECUTABLE_SUFFIX}.map
--depth ${MBED_MEMAP_DEPTH} --depth ${MBED_MEMAP_DEPTH} --memory-banks-json ${MEMORY_BANKS_JSON_PATH}
WORKING_DIRECTORY WORKING_DIRECTORY
${mbed-os_SOURCE_DIR}/tools/python ${mbed-os_SOURCE_DIR}/tools/python
) )
@ -71,6 +74,7 @@ function(mbed_generate_map_file target)
--depth ${MBED_MEMAP_DEPTH} --depth ${MBED_MEMAP_DEPTH}
-e json -e json
-o ${CMAKE_CURRENT_BINARY_DIR}/${target}${CMAKE_EXECUTABLE_SUFFIX}.memmap.json -o ${CMAKE_CURRENT_BINARY_DIR}/${target}${CMAKE_EXECUTABLE_SUFFIX}.memmap.json
--memory-banks-json ${MEMORY_BANKS_JSON_PATH}
WORKING_DIRECTORY WORKING_DIRECTORY
${mbed-os_SOURCE_DIR}/tools/python ${mbed-os_SOURCE_DIR}/tools/python
) )
@ -87,6 +91,7 @@ function(mbed_generate_map_file target)
--depth ${MBED_MEMAP_DEPTH} --depth ${MBED_MEMAP_DEPTH}
-e html -e html
-o ${CMAKE_CURRENT_BINARY_DIR}/${target}${CMAKE_EXECUTABLE_SUFFIX}.memmap.html -o ${CMAKE_CURRENT_BINARY_DIR}/${target}${CMAKE_EXECUTABLE_SUFFIX}.memmap.html
--memory-banks-json ${MEMORY_BANKS_JSON_PATH}
WORKING_DIRECTORY WORKING_DIRECTORY
${mbed-os_SOURCE_DIR}/tools/python ${mbed-os_SOURCE_DIR}/tools/python
) )

616
tools/python/memap/memap.py
View File

@ -17,9 +17,39 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
""" """
from __future__ import print_function, division, absolute_import from __future__ import annotations
from abc import abstractmethod, ABCMeta """
memap term glossary, for code reviewers and for developers working on this script
--------------------------------------------------------------------------------------------
- Module: In this script, a module refers to the code library (i.e. the .o file) where an object came from.
- Symbol: Any entity declared in the program that has a global address. Generally this means any global
variables and all functions. Note that symbol names have to be alphanumeric, so C++ implemented
"mangling" to encode class and function names as valid symbol names. This means that C++ symbols will look
like "_ZN4mbed8AnalogIn6_mutexE" to the linker. You can use the "c++filt" tool to convert those back
into a human readable name like "mbed::AnalogIn::_mutex."
- Section: A logical region of an elf of object file. Each section has a specific name and occupies contiguous memory.
It's a vague term.
- Input section: The section in the object (.o/.obj) file that a symbol comes from. It generally has a specific name,
e.g. a function could be from the .text.my_function input section.
- Output section: The section in the linked application file (.elf) that a symbol has been put into. The output
section *might* match the input section, but not always! A linker script can happily put stuff from
any input section into any output section if so desired.
- VMA (Virtual Memory Address): The address that an output section will have when the application runs.
Note that this name is something of a misnomer as it is inherited from desktop Linux. There is no virtual
memory on microcontrollers!
- LMA (Load Memory Address): The address that an output section is loaded from in flash when the program boots.
- .bss: Output section for global variables which have zero values at boot. This region of RAM is zeroed at boot.
- .data: Output section for global variables with nonzero default values. This region is copied, as a single block
of data, from the LMA to the VMA at boot.
- .text: Output section for code and constant data (e.g. the values of constant arrays). This region
is mapped directly into flash and does not need to be copied at runtime.
"""
import dataclasses
from typing import Optional, TextIO
from abc import abstractmethod, ABC
from sys import stdout, exit, argv, path from sys import stdout, exit, argv, path
from os import sep from os import sep
from os.path import (basename, dirname, join, relpath, abspath, commonprefix, from os.path import (basename, dirname, join, relpath, abspath, commonprefix,
@ -47,7 +77,28 @@ from .utils import (
) # noqa: E402 ) # noqa: E402
class _Parser(with_metaclass(ABCMeta, object)): @dataclasses.dataclass
class MemoryBankInfo:
name: str
"""Name of the bank, from cmsis_mcu_descriptions.json"""
start_addr: int
"""Start address of memory bank"""
total_size: int
"""Total size of the memory bank in bytes"""
used_size: int = 0
"""Size used in the memory bank in bytes (sum of the sizes of all symbols)"""
def contains_addr(self, addr: int) -> bool:
"""
:return: True if the given address is contained inside this memory bank
"""
return addr >= self.start_addr and addr < self.start_addr + self.total_size
class _Parser(ABC):
"""Internal interface for parsing""" """Internal interface for parsing"""
SECTIONS = ('.text', '.data', '.bss', '.heap', '.stack') SECTIONS = ('.text', '.data', '.bss', '.heap', '.stack')
MISC_FLASH_SECTIONS = ('.interrupts', '.flash_config') MISC_FLASH_SECTIONS = ('.interrupts', '.flash_config')
@ -57,19 +108,62 @@ class _Parser(with_metaclass(ABCMeta, object)):
'.stabstr', '.ARM.exidx', '.ARM') '.stabstr', '.ARM.exidx', '.ARM')
def __init__(self): def __init__(self):
self.modules = dict() self.modules: dict[str, dict[str, int]] = {}
"""Dict of object name to {section name, size}"""
def module_add(self, object_name, size, section): self.memory_banks: dict[str, list[MemoryBankInfo]] = {"RAM": [], "ROM": []}
""" Adds a module or section to the list """Memory bank info, by type (RAM/ROM)"""
def _add_symbol_to_memory_banks(self, symbol_name: str, symbol_start_addr: int, size: int) -> None:
"""
Update the memory banks structure to add the space used by a symbol.
"""
if len(self.memory_banks["RAM"]) == 0 and len(self.memory_banks["ROM"]) == 0:
# No memory banks loaded, skip
return
end_addr = symbol_start_addr + size
for banks in self.memory_banks.values():
for bank_info in banks:
if bank_info.contains_addr(symbol_start_addr):
if bank_info.contains_addr(end_addr):
# Symbol fully inside this memory bank
bank_info.used_size += size
# Uncomment to show debug info about each symbol
# print(f"Symbol {symbol_name} uses {size} bytes in {bank_info.name}")
return
print(f"Warning: Symbol {symbol_name} is only partially contained by memory bank {bank_info.name}")
first_addr_after_bank = bank_info.start_addr + bank_info.total_size
bank_info.used_size += first_addr_after_bank - symbol_start_addr
print(f"Warning: Symbol {symbol_name} (at address 0x{symbol_start_addr:x}, size {size}) is not inside a "
f"defined memory bank for this target.")
def add_symbol(self, symbol_name: str, object_name: str, start_addr: int, size: int, section: str, vma_lma_offset: int) -> None:
""" Adds information about a symbol (e.g. a function or global variable) to the data structures.
Positional arguments: Positional arguments:
object_name - name of the entry to add symbol_name - Descriptive name of the symbol, e.g. ".text.some_function"
size - the size of the module being added object_name - name of the object file containing the symbol
section - the section the module contributes to start addr - start address of symbol
size - the size of the symbol being added
section - Name of the output section, e.g. ".text". Can also be "unknown".
vma_lma_offset - Offset from where the output section exists in memory to where it's loaded from. If nonzero,
the initializer for this section will be considered too
""" """
if not object_name or not size or not section: if not object_name or not size:
return return
# Don't count the heap output section for memory bank size tracking, because the linker scripts (almost always?)
# configure that section to expand to fill the remaining amount of space
if section not in {".heap"}:
self._add_symbol_to_memory_banks(symbol_name, start_addr, size)
if vma_lma_offset != 0:
self._add_symbol_to_memory_banks(f"<initializer for {symbol_name}>", start_addr + vma_lma_offset, size)
if object_name in self.modules: if object_name in self.modules:
self.modules[object_name].setdefault(section, 0) self.modules[object_name].setdefault(section, 0)
self.modules[object_name][section] += size self.modules[object_name][section] += size
@ -82,19 +176,25 @@ class _Parser(with_metaclass(ABCMeta, object)):
contents[section] += size contents[section] += size
return return
new_module = defaultdict(int) new_symbol = defaultdict(int)
new_module[section] = size new_symbol[section] = size
self.modules[object_name] = new_module self.modules[object_name] = new_symbol
def module_replace(self, old_object, new_object): def load_memory_banks_info(self, memory_banks_json_file: TextIO) -> None:
""" Replaces an object name with a new one
""" """
if old_object in self.modules: Load the memory bank information from a memory_banks.json file
self.modules[new_object] = self.modules[old_object] """
del self.modules[old_object] memory_banks_json = json.load(memory_banks_json_file)
for bank_type, banks in memory_banks_json["configured_memory_banks"].items():
for bank_name, bank_data in banks.items():
self.memory_banks[bank_type].append(MemoryBankInfo(
name=bank_name,
start_addr=bank_data["start"],
total_size=bank_data["size"]
))
@abstractmethod @abstractmethod
def parse_mapfile(self, mapfile): def parse_mapfile(self, file_desc: TextIO) -> dict[str, dict[str, int]]:
"""Parse a given file object pointing to a map file """Parse a given file object pointing to a map file
Positional arguments: Positional arguments:
@ -116,32 +216,84 @@ class _GccParser(_Parser):
RE_TRANS_FILE = re.compile(r'^(.+\/|.+\.ltrans.o(bj)?)$') RE_TRANS_FILE = re.compile(r'^(.+\/|.+\.ltrans.o(bj)?)$')
OBJECT_EXTENSIONS = (".o", ".obj") OBJECT_EXTENSIONS = (".o", ".obj")
# Parses a line beginning a new output section in the map file that has a load address
# Groups:
# 1 = section name, including dot
# 2 = in-memory address, hex, no 0x
# 3 = section size
# 4 = load address, i.e. where is the data for this section stored in flash
RE_OUTPUT_SECTION_WITH_LOAD_ADDRESS = re.compile(r'^(.\w+) +0x([0-9a-f]+) +0x([0-9a-f]+) +load address +0x([0-9a-f]+)')
# Parses a line beginning a new output section in the map file does not have a load address
# Groups:
# 1 = section name, including dot
# 2 = in-memory address, hex, no 0x
# 3 = section size
# 4 = load address, i.e. where is the data for this section stored in flash
RE_OUTPUT_SECTION_NO_LOAD_ADDRESS = re.compile(r'^(.\w+) +0x([0-9a-f]+) +0x([0-9a-f]+)')
# Gets the input section name from the line, if it exists.
# Input section names are always indented 1 space.
RE_INPUT_SECTION_NAME = re.compile(r'^ (\.\w+\.?\w*\.?\w*)') # Note: This allows up to 3 dots... hopefully that's enough...
ALL_SECTIONS = ( ALL_SECTIONS = (
_Parser.SECTIONS _Parser.SECTIONS
+ _Parser.OTHER_SECTIONS + _Parser.OTHER_SECTIONS
+ _Parser.MISC_FLASH_SECTIONS + _Parser.MISC_FLASH_SECTIONS
+ ('unknown', 'OUTPUT') + ('unknown', )
) )
def check_new_section(self, line): def check_new_output_section(self, line: str) -> tuple[str, int] | None:
""" Check whether a new section in a map file has been detected """ Check whether a new output section in a map file has been detected
Positional arguments: Positional arguments:
line - the line to check for a new section line - the line to check for a new section
return value - A section name, if a new section was found, None return value - Tuple of (name, vma to lma offset), if a new section was found, None
otherwise otherwise
The vma to lma offset is the offset to be added to a memory address to get the
address where it's loaded from. If this is zero, the section is not loaded from flash to RAM at startup.
""" """
line_s = line.strip()
for i in self.ALL_SECTIONS: match = re.match(self.RE_OUTPUT_SECTION_WITH_LOAD_ADDRESS, line)
if line_s.startswith(i): if match:
return i section_name = match.group(1)
if line.startswith('.'): memory_address = int(match.group(2), 16)
return 'unknown' load_address = int(match.group(4), 16)
load_addr_offset = load_address - memory_address
else: else:
match = re.match(self.RE_OUTPUT_SECTION_NO_LOAD_ADDRESS, line)
if not match:
return None
section_name = match.group(1)
load_addr_offset = 0
# Ensure that this is a known section name, remove if not
if section_name not in self.ALL_SECTIONS:
section_name = "unknown"
# Strangely, GCC still generates load address info for sections that are not loaded, such as .bss.
# For now, suppress this for all sections other than .data.
if section_name != ".data":
load_addr_offset = 0
return section_name, load_addr_offset
def check_input_section(self, line) -> Optional[str]:
""" Check whether a new input section in a map file has been detected.
Positional arguments:
line - the line to check for a new section
return value - Input section name if found, None otherwise
"""
match = re.match(self.RE_INPUT_SECTION_NAME, line)
if not match:
return None return None
def parse_object_name(self, line): return match.group(1)
def parse_object_name(self, line: str) -> str:
""" Parse a path to object file """ Parse a path to object file
Positional arguments: Positional arguments:
@ -177,8 +329,8 @@ class _GccParser(_Parser):
% line) % line)
return '[misc]' return '[misc]'
def parse_section(self, line): def parse_section(self, line: str) -> tuple[str, int, int]:
""" Parse data from a section of gcc map file """ Parse data from a section of gcc map file describing one symbol in the code.
examples: examples:
0x00004308 0x7c ./BUILD/K64F/GCC_ARM/spi_api.o 0x00004308 0x7c ./BUILD/K64F/GCC_ARM/spi_api.o
@ -186,46 +338,64 @@ class _GccParser(_Parser):
Positional arguments: Positional arguments:
line - the line to parse a section from line - the line to parse a section from
Returns tuple of (name, start addr, size)
""" """
is_fill = re.match(self.RE_FILL_SECTION, line) is_fill = re.match(self.RE_FILL_SECTION, line)
if is_fill: if is_fill:
o_name = '[fill]' o_name = '[fill]'
o_start_addr = int(is_fill.group(1), 16)
o_size = int(is_fill.group(2), 16) o_size = int(is_fill.group(2), 16)
return [o_name, o_size] return o_name, o_start_addr, o_size
is_section = re.match(self.RE_STD_SECTION, line) is_section = re.match(self.RE_STD_SECTION, line)
if is_section: if is_section:
o_start_addr = int(is_section.group(1), 16)
o_size = int(is_section.group(2), 16) o_size = int(is_section.group(2), 16)
if o_size: if o_size:
o_name = self.parse_object_name(is_section.group(3)) o_name = self.parse_object_name(is_section.group(3))
return [o_name, o_size] return o_name, o_start_addr, o_size
return ["", 0] return "", 0, 0
def parse_mapfile(self, file_desc): def parse_mapfile(self, file_desc: TextIO) -> dict[str, dict[str, int]]:
""" Main logic to decode gcc map files """ Main logic to decode gcc map files
Positional arguments: Positional arguments:
file_desc - a stream object to parse as a gcc map file file_desc - a stream object to parse as a gcc map file
""" """
current_section = 'unknown'
# GCC can put the section/symbol info on its own line or on the same line as the size and address.
# So since this is a line oriented parser, we have to remember the most recently seen input & output
# section name for later.
current_output_section = 'unknown'
current_output_section_addr_offset = 0
current_input_section = 'unknown'
with file_desc as infile: with file_desc as infile:
for line in infile: for line in infile:
if line.startswith('Linker script and memory map'): if line.startswith('Linker script and memory map'):
current_section = "unknown"
break break
for line in infile: for line in infile:
next_section = self.check_new_section(line) if line.startswith("OUTPUT("):
# Done with memory map part of the map file
if next_section == "OUTPUT":
break break
elif next_section:
current_section = next_section
object_name, object_size = self.parse_section(line) next_section = self.check_new_output_section(line)
self.module_add(object_name, object_size, current_section) if next_section is not None:
current_output_section, current_output_section_addr_offset = next_section
next_input_section = self.check_input_section(line)
if next_input_section is not None:
current_input_section = next_input_section
symbol_name, symbol_start_addr, symbol_size = self.parse_section(line)
# With GCC at least, the closest we can get to a descriptive symbol name is the input section
# name. Thanks to the -ffunction-sections and -fdata-sections options, the section names should
# be unique for each symbol.
self.add_symbol(current_input_section, symbol_name, symbol_start_addr, symbol_size, current_output_section, current_output_section_addr_offset)
common_prefix = dirname(commonprefix([ common_prefix = dirname(commonprefix([
o for o in self.modules.keys() o for o in self.modules.keys()
@ -244,280 +414,6 @@ class _GccParser(_Parser):
return new_modules return new_modules
class _ArmccParser(_Parser):
RE = re.compile(
r'^\s+0x(\w{8})\s+0x(\w{8})\s+(\w+)\s+(\w+)\s+(\d+)\s+[*]?.+\s+(.+)$')
RE_OBJECT = re.compile(r'(.+\.(l|a|ar))\((.+\.o(bj)?)\)')
OBJECT_EXTENSIONS = (".o", ".obj")
def parse_object_name(self, line):
""" Parse object file
Positional arguments:
line - the line containing the object or library
"""
if line.endswith(self.OBJECT_EXTENSIONS):
return line
else:
is_obj = re.match(self.RE_OBJECT, line)
if is_obj:
return join(
'[lib]', basename(is_obj.group(1)), is_obj.group(3)
)
else:
print(
"Malformed input found when parsing ARMCC map: %s" % line
)
return '[misc]'
def parse_section(self, line):
""" Parse data from an armcc map file
Examples of armcc map file:
Base_Addr Size Type Attr Idx E Section Name Object
0x00000000 0x00000400 Data RO 11222 self.RESET startup_MK64F12.o
0x00000410 0x00000008 Code RO 49364 * !!!main c_w.l(__main.o)
Positional arguments:
line - the line to parse the section data from
""" # noqa: E501
test_re = re.match(self.RE, line)
if (
test_re
and "ARM_LIB_HEAP" not in line
):
size = int(test_re.group(2), 16)
if test_re.group(4) == 'RO':
section = '.text'
else:
if test_re.group(3) == 'Data':
section = '.data'
elif test_re.group(3) == 'Zero':
section = '.bss'
elif test_re.group(3) == 'Code':
section = '.text'
else:
print(
"Malformed input found when parsing armcc map: %s, %r"
% (line, test_re.groups())
)
return ["", 0, ""]
# check name of object or library
object_name = self.parse_object_name(
test_re.group(6))
return [object_name, size, section]
else:
return ["", 0, ""]
def parse_mapfile(self, file_desc):
""" Main logic to decode armc5 map files
Positional arguments:
file_desc - a file like object to parse as an armc5 map file
"""
with file_desc as infile:
# Search area to parse
for line in infile:
if line.startswith(' Base Addr Size'):
break
# Start decoding the map file
for line in infile:
self.module_add(*self.parse_section(line))
common_prefix = dirname(commonprefix([
o for o in self.modules.keys()
if (
o.endswith(self.OBJECT_EXTENSIONS)
and o != "anon$$obj.o"
and o != "anon$$obj.obj"
and not o.startswith("[lib]")
)]))
new_modules = {}
for name, stats in self.modules.items():
if (
name == "anon$$obj.o"
or name == "anon$$obj.obj"
or name.startswith("[lib]")
):
new_modules[name] = stats
elif name.endswith(self.OBJECT_EXTENSIONS):
new_modules[relpath(name, common_prefix)] = stats
else:
new_modules[name] = stats
return new_modules
class _IarParser(_Parser):
RE = re.compile(
r'^\s+(.+)\s+(zero|const|ro code|inited|uninit)\s'
r'+0x([\'\w]+)\s+0x(\w+)\s+(.+)\s.+$')
RE_CMDLINE_FILE = re.compile(r'^#\s+(.+\.o(bj)?)')
RE_LIBRARY = re.compile(r'^(.+\.a)\:.+$')
RE_OBJECT_LIBRARY = re.compile(r'^\s+(.+\.o(bj)?)\s.*')
OBJECT_EXTENSIONS = (".o", ".obj")
def __init__(self):
_Parser.__init__(self)
# Modules passed to the linker on the command line
# this is a dict because modules are looked up by their basename
self.cmd_modules = {}
def parse_object_name(self, object_name):
""" Parse object file
Positional arguments:
line - the line containing the object or library
"""
if object_name.endswith(self.OBJECT_EXTENSIONS):
try:
return self.cmd_modules[object_name]
except KeyError:
return object_name
else:
return '[misc]'
def parse_section(self, line):
""" Parse data from an IAR map file
Examples of IAR map file:
Section Kind Address Size Object
.intvec ro code 0x00000000 0x198 startup_MK64F12.o [15]
.rodata const 0x00000198 0x0 zero_init3.o [133]
.iar.init_table const 0x00008384 0x2c - Linker created -
Initializer bytes const 0x00000198 0xb2 <for P3 s0>
.data inited 0x20000000 0xd4 driverAtmelRFInterface.o [70]
.bss zero 0x20000598 0x318 RTX_Conf_CM.o [4]
.iar.dynexit uninit 0x20001448 0x204 <Block tail>
HEAP uninit 0x20001650 0x10000 <Block tail>
Positional_arguments:
line - the line to parse section data from
""" # noqa: E501
test_re = re.match(self.RE, line)
if test_re:
if (
test_re.group(2) == 'const' or
test_re.group(2) == 'ro code'
):
section = '.text'
elif (test_re.group(2) == 'zero' or
test_re.group(2) == 'uninit'):
if test_re.group(1)[0:4] == 'HEAP':
section = '.heap'
elif test_re.group(1)[0:6] == 'CSTACK':
section = '.stack'
else:
section = '.bss' # default section
elif test_re.group(2) == 'inited':
section = '.data'
else:
print("Malformed input found when parsing IAR map: %s" % line)
return ["", 0, ""]
# lookup object in dictionary and return module name
object_name = self.parse_object_name(test_re.group(5))
size = int(test_re.group(4), 16)
return [object_name, size, section]
else:
return ["", 0, ""]
def check_new_library(self, line):
"""
Searches for libraries and returns name. Example:
m7M_tls.a: [43]
"""
test_address_line = re.match(self.RE_LIBRARY, line)
if test_address_line:
return test_address_line.group(1)
else:
return ""
def check_new_object_lib(self, line):
"""
Searches for objects within a library section and returns name.
Example:
rt7M_tl.a: [44]
ABImemclr4.o 6
ABImemcpy_unaligned.o 118
ABImemset48.o 50
I64DivMod.o 238
I64DivZer.o 2
"""
test_address_line = re.match(self.RE_OBJECT_LIBRARY, line)
if test_address_line:
return test_address_line.group(1)
else:
return ""
def parse_command_line(self, lines):
"""Parse the files passed on the command line to the iar linker
Positional arguments:
lines -- an iterator over the lines within a file
"""
for line in lines:
if line.startswith("*"):
break
for arg in line.split(" "):
arg = arg.rstrip(" \n")
if (
not arg.startswith("-")
and arg.endswith(self.OBJECT_EXTENSIONS)
):
self.cmd_modules[basename(arg)] = arg
common_prefix = dirname(commonprefix(list(self.cmd_modules.values())))
self.cmd_modules = {s: relpath(f, common_prefix)
for s, f in self.cmd_modules.items()}
def parse_mapfile(self, file_desc):
""" Main logic to decode IAR map files
Positional arguments:
file_desc - a file like object to parse as an IAR map file
"""
with file_desc as infile:
self.parse_command_line(infile)
for line in infile:
if line.startswith(' Section '):
break
for line in infile:
self.module_add(*self.parse_section(line))
if line.startswith('*** MODULE SUMMARY'): # finish section
break
current_library = ""
for line in infile:
library = self.check_new_library(line)
if library:
current_library = library
object_name = self.check_new_object_lib(line)
if object_name and current_library:
temp = join('[lib]', current_library, object_name)
self.module_replace(object_name, temp)
return self.modules
class MemapParser(object): class MemapParser(object):
"""An object that represents parsed results, parses the memory map files, """An object that represents parsed results, parses the memory map files,
and writes out different file types of memory results and writes out different file types of memory results
@ -757,7 +653,7 @@ class MemapParser(object):
"Total Flash memory (text + data): {}({:+}) bytes\n" "Total Flash memory (text + data): {}({:+}) bytes\n"
) )
def generate_csv(self, file_desc): def generate_csv(self, file_desc: TextIO) -> None:
"""Generate a CSV file from a memoy map """Generate a CSV file from a memoy map
Positional arguments: Positional arguments:
@ -830,12 +726,22 @@ class MemapParser(object):
self.mem_summary['total_flash_delta'] self.mem_summary['total_flash_delta']
) )
output += '\n'
for bank_type, banks in self.memory_banks.items():
for bank_info in banks:
this_bank_deltas = self.memory_bank_summary[bank_type][bank_info.name]
output += (f"{bank_type} Bank {bank_info.name}: {bank_info.used_size}({this_bank_deltas['delta_bytes_used']:+})/"
f"{bank_info.total_size} bytes used, "
f"{this_bank_deltas['percent_used']:.01f}% ({this_bank_deltas['delta_percent_used']:+.01f}%) used\n")
return output return output
toolchains = ["ARM", "ARM_STD", "ARM_MICRO", "GCC_ARM", "IAR"] toolchains = ["ARM", "ARM_STD", "ARM_MICRO", "GCC_ARM", "IAR"]
def compute_report(self): def compute_report(self):
""" Generates summary of memory usage for main areas """
Generates summary of memory usage for main areas. Result is put into the 'self.mem_report'
dict, which is processed by tests and also dumped as JSON for the JSON output format.
""" """
self.subtotal = defaultdict(int) self.subtotal = defaultdict(int)
@ -857,22 +763,49 @@ class MemapParser(object):
self.subtotal['.text-delta'] + self.subtotal['.data-delta'], self.subtotal['.text-delta'] + self.subtotal['.data-delta'],
} }
self.mem_report = [] self.mem_report = {}
modules = []
if self.short_modules: if self.short_modules:
for name, sizes in sorted(self.short_modules.items()): for name, sizes in sorted(self.short_modules.items()):
self.mem_report.append({ modules.append({
"module": name, "module": name,
"size": { "size": {
k: sizes.get(k, 0) for k in (self.print_sections + k: sizes.get(k, 0) for k in (self.print_sections +
self.delta_sections) self.delta_sections)
} }
}) })
self.mem_report["modules"] = modules
self.mem_report.append({ self.mem_report["summary"] = self.mem_summary
'summary': self.mem_summary
})
def parse(self, mapfile, toolchain): # Calculate the delta sizes for each memory bank in a couple different formats
self.memory_bank_summary: dict[str, dict[str, dict[str, float|int]]] = {}
for bank_type, banks in self.memory_banks.items():
self.memory_bank_summary[bank_type] = {}
for bank_info in banks:
this_bank_info = {}
# Find matching memory bank in old memory banks. Compare by name as it would be possible
# for the indices to change between builds if someone edited the memory bank definition
old_bank_info = None
if self.old_memory_banks is not None and bank_type in self.old_memory_banks:
for curr_old_bank_info in self.old_memory_banks[bank_type]:
if curr_old_bank_info.name == bank_info.name:
old_bank_info = curr_old_bank_info
break
this_bank_info["bytes_used"] = bank_info.used_size
this_bank_info["total_size"] = bank_info.total_size
this_bank_info["delta_bytes_used"] = 0 if old_bank_info is None else bank_info.used_size - old_bank_info.used_size
this_bank_info["percent_used"] = 100 * bank_info.used_size/bank_info.total_size
this_bank_info["delta_percent_used"] = 100 * this_bank_info["delta_bytes_used"]/bank_info.total_size
self.memory_bank_summary[bank_type][bank_info.name] = this_bank_info
self.mem_report["memory_bank_usage"] = self.memory_bank_summary
def parse(self, mapfile: str, toolchain: str, memory_banks_json_path: str | None) -> bool:
""" Parse and decode map file depending on the toolchain """ Parse and decode map file depending on the toolchain
Positional arguments: Positional arguments:
@ -880,22 +813,28 @@ class MemapParser(object):
toolchain - the toolchain used to create the file toolchain - the toolchain used to create the file
""" """
self.tc_name = toolchain.title() self.tc_name = toolchain.title()
if toolchain in ("ARM", "ARM_STD", "ARM_MICRO", "ARMC6"): if toolchain == "GCC_ARM":
parser = _ArmccParser parser_class = _GccParser
elif toolchain == "GCC_ARM":
parser = _GccParser
elif toolchain == "IAR":
parser = _IarParser
else: else:
return False return False
parser = parser_class()
old_map_parser = parser_class()
if memory_banks_json_path is not None:
with open(memory_banks_json_path, 'r') as memory_banks_json_file:
parser.load_memory_banks_info(memory_banks_json_file)
try: try:
with open(mapfile, 'r') as file_input: with open(mapfile, 'r') as file_input:
self.modules = parser().parse_mapfile(file_input) self.modules = parser.parse_mapfile(file_input)
self.memory_banks = parser.memory_banks
try: try:
with open("%s.old" % mapfile, 'r') as old_input: with open("%s.old" % mapfile, 'r') as old_input:
self.old_modules = parser().parse_mapfile(old_input) self.old_modules = old_map_parser.parse_mapfile(old_input)
self.old_memory_banks = old_map_parser.memory_banks
except IOError: except IOError:
self.old_modules = None self.old_modules = None
self.old_memory_banks = None
return True return True
except IOError as error: except IOError as error:
@ -938,6 +877,11 @@ def main():
parser.add_argument('-v', '--version', action='version', version=version) parser.add_argument('-v', '--version', action='version', version=version)
parser.add_argument(
'-m', '--memory-banks-json',
type=argparse_filestring_type,
help='Path to memory bank JSON file. If passed, memap will track the used space in each memory bank.')
# Parse/run command # Parse/run command
if len(argv) <= 1: if len(argv) <= 1:
parser.print_help() parser.print_help()
@ -950,7 +894,7 @@ def main():
# Parse and decode a map file # Parse and decode a map file
if args.file and args.toolchain: if args.file and args.toolchain:
if memap.parse(args.file, args.toolchain) is False: if memap.parse(args.file, args.toolchain, args.memory_banks_json) is False:
exit(0) exit(0)
if args.depth is None: if args.depth is None:

22
tools/python/memap/utils.py
View File

@ -33,10 +33,6 @@ import logging
from intelhex import IntelHex from intelhex import IntelHex
import io import io
try:
unicode
except NameError:
unicode = str
def remove_if_in(lst, thing): def remove_if_in(lst, thing):
if thing in lst: if thing in lst:
@ -427,7 +423,7 @@ def argparse_type(casedness, prefer_hyphen=False):
the string, or the hyphens/underscores do not match the expected the string, or the hyphens/underscores do not match the expected
style of the argument. style of the argument.
""" """
if not isinstance(string, unicode): if not isinstance(string, str):
string = string.decode() string = string.decode()
if prefer_hyphen: if prefer_hyphen:
newstring = casedness(string).replace("_", "-") newstring = casedness(string).replace("_", "-")
@ -447,10 +443,10 @@ def argparse_type(casedness, prefer_hyphen=False):
return middle return middle
# short cuts for the argparse_type versions # short cuts for the argparse_type versions
argparse_uppercase_type = argparse_type(unicode.upper, False) argparse_uppercase_type = argparse_type(str.upper, False)
argparse_lowercase_type = argparse_type(unicode.lower, False) argparse_lowercase_type = argparse_type(str.lower, False)
argparse_uppercase_hyphen_type = argparse_type(unicode.upper, True) argparse_uppercase_hyphen_type = argparse_type(str.upper, True)
argparse_lowercase_hyphen_type = argparse_type(unicode.lower, True) argparse_lowercase_hyphen_type = argparse_type(str.lower, True)
def argparse_force_type(case): def argparse_force_type(case):
""" validate that an argument passed in (as string) is a member of the list """ validate that an argument passed in (as string) is a member of the list
@ -458,11 +454,11 @@ def argparse_force_type(case):
""" """
def middle(lst, type_name): def middle(lst, type_name):
""" The parser type generator""" """ The parser type generator"""
if not isinstance(lst[0], unicode): if not isinstance(lst[0], str):
lst = [o.decode() for o in lst] lst = [o.decode() for o in lst]
def parse_type(string): def parse_type(string):
""" The parser type""" """ The parser type"""
if not isinstance(string, unicode): if not isinstance(string, str):
string = string.decode() string = string.decode()
for option in lst: for option in lst:
if case(string) == case(option): if case(string) == case(option):
@ -474,8 +470,8 @@ def argparse_force_type(case):
return middle return middle
# these two types convert the case of their arguments _before_ validation # these two types convert the case of their arguments _before_ validation
argparse_force_uppercase_type = argparse_force_type(unicode.upper) argparse_force_uppercase_type = argparse_force_type(str.upper)
argparse_force_lowercase_type = argparse_force_type(unicode.lower) argparse_force_lowercase_type = argparse_force_type(str.lower)
def argparse_many(func): def argparse_many(func):
""" An argument parser combinator that takes in an argument parser and """ An argument parser combinator that takes in an argument parser and

47
tools/python/python_tests/memap/arm.map
View File

@ -1,47 +0,0 @@
Component: ARM Compiler 5.06 update 5 (build 528) Tool: armlink [4d35e2]
==============================================================================
Memory Map of the image
Image Entry point : 0x0001b0c1
Load Region LR_IROM1 (Base: 0x0001b000, Size: 0x0000ed04, Max: 0x00025000, ABSOLUTE, COMPRESSED[0x0000e23c])
Execution Region ER_IROM1 (Base: 0x0001b000, Size: 0x0000e1c4, Max: 0x00025000, ABSOLUTE)
Base Addr Size Type Attr Idx E Section Name Object
0x0001b000 0x000000c0 Data RO 7002 RESET /common/path/startup/startup.o
0x0001b0c0 0x00000008 Code RO 8820 * !!!main /installed/libs/../lib/armlib/c_p.l(__main.o)
0x0001b26c 0x00000098 Code RO 6076 .text /common/path/irqs/irqs.o
0x000206a0 0x00000036 Code RO 27 i._Z9time_funcPN4mbed5TimerEi /common/path/main.o
0x200039b4 0x00000018 Data RW 8092 .data /common/path/data/data.o
0x20003af8 0x00000198 Zero RW 57 .bss /common/path/data/data.o
==============================================================================
Image component sizes
Code (inc. data) RO Data RW Data ZI Data Debug
344 368 0 24 408 36188 Object Totals
8 0 0 0 0 7596 Library Totals
==============================================================================
Code (inc. data) RO Data RW Data ZI Data Debug
352 376 0 24 408 17208 Grand Totals
352 376 0 24 408 17208 ELF Image Totals (compressed)
352 376 0 24 0 0 ROM Totals
==============================================================================
Total RO Size (Code + RO Data) 352 ( 0.35kB)
Total RW Size (RW Data + ZI Data) 432 ( 0.43kB)
Total ROM Size (Code + RO Data + RW Data) 376 ( 0.37kB)
==============================================================================

83
tools/python/python_tests/memap/iar.map
View File

@ -1,83 +0,0 @@
###############################################################################
#
# IAR ELF Linker V7.80.1.28/LNX for ARM 18/Sep/2017 14:26:09
# Copyright 2007-2016 IAR Systems AB.
#
# Output file =
# /common/path/project.elf
# Map file =
# /common/path/project.map
# Command line =
# -f
# /common/path/.link_files.txt
# (-o /common/path/project.elf
# --map=/common/path/project.map /common/path/main.o
# /common/path/startup/startup.o /common/path/irqs/irqs.o
# /common/path/data/data.o
#
###############################################################################
*******************************************************************************
*** RUNTIME MODEL ATTRIBUTES
***
CppFlavor = *
__CPP_Exceptions = Disabled
__CPP_Language = C++
__Heap_Handler = DLMalloc
__SystemLibrary = DLib
__dlib_dynamic_initialization = postponed
__dlib_has_iterator_debugging = 0
__dlib_jmp_buf_num_elements = 8
*******************************************************************************
*** PLACEMENT SUMMARY
***
"A0": place at 0x0001b000 { ro section .intvec };
"P1": place in [from 0x0001b0c0 to 0x0003ffff] { ro };
"P2": place in [from 0x20002ef8 to 0x20007fff] { rw, block HEAP, block CSTACK };
do not initialize { section .noinit };
initialize by copy { rw };
{ section .intvec };
Section Kind Address Size Object
------- ---- ------- ---- ------
"A0": 0xc0
.intvec ro code 0x0001b000 0xc0 startup.o [4]
- 0x0001b0c0 0xc0
"P1": 0x
.text ro code 0x0001c753 0x36 main.o [3]
.text ro code 0x0001cfff 0x98 irqs.o [5]
.text ro code 0x0001c778 0x8 __main.o [67]
"P2", part 1 of 2: 0x18
P2-1 0x20002ef8 0x18 <Init block>
.data inited 0x20002fa8 0x18 data.o [6]
"P2", part 2 of 2: 0x198
P2-2 0x20005388 0x198 <Init block>
.bss zero 0x20002fa8 0x198 data.o [6]
*******************************************************************************
*** INIT TABLE
***
*******************************************************************************
*** MODULE SUMMARY
***
d16M_tlf.a: [67]
__main.o 8
------------------------------------------------
Total: 8
Linker created
---------------------------------------------------
Grand Total:
*******************************************************************************
*** ENTRY LIST
***

40
tools/python/python_tests/memap/memap_test.py
View File

@ -22,7 +22,7 @@ import json
import pytest import pytest
from memap import memap from memap import memap
from memap.memap import MemapParser from memap.memap import MemapParser, MemoryBankInfo
from copy import deepcopy from copy import deepcopy
""" """
@ -38,7 +38,7 @@ def memap_parser():
""" """
memap_parser = MemapParser() memap_parser = MemapParser()
memap_parser.modules = { memap_parser.symbols = {
"mbed-os/targets/TARGET/TARGET_MCUS/api/pinmap.o": { "mbed-os/targets/TARGET/TARGET_MCUS/api/pinmap.o": {
".text": 1, ".text": 1,
".data": 2, ".data": 2,
@ -136,6 +136,24 @@ def memap_parser():
"OUTPUT":0, "OUTPUT":0,
}, },
} }
memap_parser.memory_banks = {
"RAM": [
MemoryBankInfo(name="IRAM1", start_addr=0x20000000, total_size=32768, used_size=2000)
],
"ROM": [
MemoryBankInfo(name="IROM1", start_addr=0x20000000, total_size=65536, used_size=10000)
]
}
memap_parser.old_memory_banks = {
"RAM": [
MemoryBankInfo(name="IRAM1", start_addr=0x20000000, total_size=32768, used_size=2014)
],
"ROM": [
MemoryBankInfo(name="IROM1", start_addr=0x20000000, total_size=65536, used_size=9000)
]
}
return memap_parser return memap_parser
@ -217,3 +235,21 @@ def test_generate_output_csv_ci(memap_parser, tmpdir, depth, sep):
file_name = str(tmpdir.join('output.csv').realpath()) file_name = str(tmpdir.join('output.csv').realpath())
generate_test_helper(memap_parser, 'csv-ci', depth, sep, file_name) generate_test_helper(memap_parser, 'csv-ci', depth, sep, file_name)
assert isfile(file_name), "Failed to create csv-ci file" assert isfile(file_name), "Failed to create csv-ci file"
def test_memory_bank_summary(memap_parser: MemapParser):
"""
Test that the memory bank summary has the expected information in it.
"""
memap_parser.generate_output('table', 1, "/")
assert memap_parser.memory_bank_summary["RAM"].keys() == {"IRAM1"}
assert memap_parser.memory_bank_summary["ROM"].keys() == {"IROM1"}
# Check details of the ROM bank more closely
assert memap_parser.memory_bank_summary["ROM"]["IROM1"]["bytes_used"] == 10000
assert memap_parser.memory_bank_summary["ROM"]["IROM1"]["total_size"] == 65536
assert memap_parser.memory_bank_summary["ROM"]["IROM1"]["delta_bytes_used"] == 1000
assert memap_parser.memory_bank_summary["ROM"]["IROM1"]["percent_used"] == pytest.approx(15.3, abs=0.1)
assert memap_parser.memory_bank_summary["ROM"]["IROM1"]["delta_percent_used"] == pytest.approx(1.5, abs=0.1)

83
tools/python/python_tests/memap/parse_test.py
View File

@ -20,84 +20,55 @@ import sys
from io import open from io import open
from os import sep from os import sep
from os.path import isfile, join, dirname from os.path import isfile, join, dirname
import json from collections import defaultdict
import pytest import pytest
from memap.memap import MemapParser, _ArmccParser from memap.memap import MemapParser, _GccParser
from copy import deepcopy from copy import deepcopy
PARSED_ARM_DATA = {
"startup/startup.o": {".text": 0xc0},
"[lib]/c_p.l/__main.o": {".text": 8},
"irqs/irqs.o": {".text": 0x98},
"data/data.o": {".data": 0x18, ".bss": 0x198},
"main.o": {".text": 0x36},
}
def test_parse_armcc():
memap = MemapParser()
memap.parse(join(dirname(__file__), "arm.map"), "ARM")
parsed_data_os_agnostic = dict()
for k in PARSED_ARM_DATA:
parsed_data_os_agnostic[k.replace('/', sep)] = PARSED_ARM_DATA[k]
assert memap.modules == parsed_data_os_agnostic
PARSED_IAR_DATA = {
"startup/startup.o": {".text": 0xc0},
"[lib]/d16M_tlf.a/__main.o": {".text": 8},
"irqs/irqs.o": {".text": 0x98},
"data/data.o": {".data": 0x18, ".bss": 0x198},
"main.o": {".text": 0x36},
}
def test_parse_iar():
memap = MemapParser()
memap.parse(join(dirname(__file__), "iar.map"), "IAR")
parsed_data_os_agnostic = dict()
for k in PARSED_IAR_DATA:
parsed_data_os_agnostic[k.replace('/', sep)] = PARSED_IAR_DATA[k]
assert memap.modules == parsed_data_os_agnostic
PARSED_GCC_DATA = { PARSED_GCC_DATA = {
"startup/startup.o": {".text": 0xc0}, "startup/startup.o": defaultdict(int, {".text": 0xc0}),
"[lib]/d16M_tlf.a/__main.o": {".text": 8}, "[lib]/d16M_tlf.a/__main.o": defaultdict(int, {".text": 8}),
"[lib]/misc/foo.o": {".text": 8}, "[lib]/misc/foo.o": defaultdict(int, {".text": 8}),
"irqs/irqs.o": {".text": 0x98}, "irqs/irqs.o": defaultdict(int, {".text": 0x98}),
"data/data.o": {".data": 0x18, ".bss": 0x198}, "data/data.o":defaultdict(int, {".data": 0x18, ".bss": 0x198}),
"main.o": {".text": 0x36}, "main.o": defaultdict(int, {".text": 0x36}),
} }
def test_parse_gcc(): def test_parse_gcc():
memap = MemapParser() memap = MemapParser()
memap.parse(join(dirname(__file__), "gcc.map"), "GCC_ARM")
this_script_dir = dirname(__file__)
memap.parse(join(this_script_dir, "gcc.map"), "GCC_ARM", join(this_script_dir, "test_memory_banks.json"))
parsed_data_os_agnostic = dict() parsed_data_os_agnostic = dict()
for k in PARSED_GCC_DATA: for k in PARSED_GCC_DATA:
parsed_data_os_agnostic[k.replace('/', sep)] = PARSED_GCC_DATA[k] parsed_data_os_agnostic[k.replace('/', sep)] = PARSED_GCC_DATA[k]
# Sum of everything in .text and .data
assert memap.memory_banks["ROM"][0].used_size == 0x1B6
# Sum of everything in .bss and .data
assert memap.memory_banks["RAM"][0].used_size == 0x1B0
assert memap.modules == parsed_data_os_agnostic assert memap.modules == parsed_data_os_agnostic
def test_add_empty_module(): def test_add_symbol_missing_info():
memap = _ArmccParser() memap = _GccParser()
old_modules = deepcopy(memap.modules) old_symbols = deepcopy(memap.modules)
memap.module_add("", 8, ".data") memap.add_symbol(".data.some_func", "", 8, 10, ".data", 1000)
assert(old_modules == memap.modules) assert(old_symbols == memap.modules)
memap.module_add("main.o", 0, ".text") memap.add_symbol(".data.some_func", "foo.o", 8, 0, ".data", 1000)
assert(old_modules == memap.modules) assert(old_symbols == memap.modules)
memap.module_add("main.o", 8, "")
assert(old_modules == memap.modules)
def test_add_full_module(): def test_add_full_module():
memap = _ArmccParser() memap = _GccParser()
old_modules = deepcopy(memap.modules) old_modules = deepcopy(memap.modules)
memap.module_add("main.o", 8, ".data") memap.add_symbol(".data.foo", "main.o", 5, 8, ".data", 1000)
assert(old_modules != memap.modules) assert(old_modules != memap.modules)
assert("main.o" in memap.modules) assert("main.o" in memap.modules)
assert(".data" in memap.modules["main.o"]) assert(".data" in memap.modules["main.o"])

38
tools/python/python_tests/memap/test_memory_banks.json Normal file
View File

@ -0,0 +1,38 @@
{
"configured_memory_banks": {
"ROM": {
"IROM1": {
"access": {
"execute": true,
"non_secure": false,
"non_secure_callable": false,
"peripheral": false,
"read": true,
"secure": false,
"write": false
},
"default": true,
"size": 524288,
"start": 0,
"startup": true
}
},
"RAM": {
"IRAM1": {
"access": {
"execute": false,
"non_secure": false,
"non_secure_callable": false,
"peripheral": false,
"read": true,
"secure": false,
"write": true
},
"default": true,
"size": 32768,
"start": 536870912,
"startup": false
}
}
}
}