Merge pull request #218 from screamerbg/use-memap

Add memory analysis to mbed-os build system
Martin Kojtal 2016-06-07 12:57:59 +01:00
commit 07c3a108ef
4 changed files with 634 additions and 40 deletions

93
docs/memap.md Normal file
View File

@ -0,0 +1,93 @@
# memap - Static Memory Map Analysis
## Introduction
*memap* is a simple utility useful to display static memory information required by [mbed-OS](https://github.com/ARMmbed/mbed-os) applications. This information is produced by analysing the memory map file previously generated by your toolchain.
Note: this tool is only showing static RAM usage and the total size of allocated heap and stack space, not the actual heap and stack usage.
## Table of Contents
1. [Using memap](#using-memap)
1. [Running pytest](#running-pytest)
1. [Current support](#current-support)
1. [Known problems](#known-problems)
## Using memap
*memap* is a simple utility which is automatically invoked after an mbed-os build finishes succesfully.
It's also possible to manually run the program with differnt command line options:
```
$> python memap.py
usage: memap-script.py [-h] -t TOOLCHAIN [-o OUTPUT] [-j] [-v] file
Memory Map File Analyser for ARM mbed OS version 0.3.7
positional arguments:
file memory map file
optional arguments:
-h, --help show this help message and exit
-t TOOLCHAIN, --toolchain TOOLCHAIN
select a toolchain that corresponds to the memory map
file (ARM, GCC_ARM, IAR)
-o OUTPUT, --output OUTPUT
output file name
-j, --json output in JSON formatted list
-v, --version show program's version number and exit
```
Example:
```
$> python memap.py GCC_ARM\myprog3.map -t GCC_ARM
+-----------------------------------+-------+-------+------+
| Module | .text | .data | .bss |
+-----------------------------------+-------+-------+------+
| Misc | 248 | 12 | 28 |
| core/mbed-rtos | 5638 | 24 | 2494 |
| core/util | 323 | 0 | 8 |
| core/uvisor-mbed-lib | 4 | 0 | 0 |
| frameworks/greentea-client | 416 | 28 | 0 |
| frameworks/utest | 3055 | 32 | 688 |
| hal/common | 2453 | 4 | 272 |
| hal/targets | 9796 | 0 | 672 |
| net/LWIPInterface | 172 | 4 | 272 |
| net/atmel-rf-driver | 332 | 8 | 212 |
| net/mbed-client | 64 | 4 | 12 |
| net/nanostack-hal-mbed-cmsis-rtos | 100 | 4 | 56 |
| net/nanostack-interface | 40 | 4 | 16 |
| Subtotals | 22641 | 124 | 4730 |
+-----------------------------------+-------+-------+------+
Static RAM memory (data + bss): 4854
Heap: 131072
Stack: 3072
Total RAM memory (data + bss + heap + stack): 138998
Total Flash memory (text + data): 22765
Elapsed time: 55mS
```
Information on memory sections:
- text: is where the code application and constants are located in Flash
- data: initialized variables, allocated in both RAM and Flash memory
- bss: uninitialized data allocated in RAM
- heap: dynamic allocated memory, usually used by malloc, etc, in RAM
- stack: used to store local data, temporary data when branching to a subroutine and context switch info; considered dynamic allocated memory region in RAM.
## Current support
*memap* has been tested on Windows 7, Linux and Mac.
Supported map files generated by the following toolchains: gcc, ARM Compiler 5 and IAR.
## Known problems & New features
This utility is considered 'Alpha' quality at the moment.
The report generated may not be accurate and may vary from one toolchain to another.
If you are seeing problems or would like new features to be added then please raise a ticket on [GitHub](https://github.com/ARMmbed/mbed-os/issues) and use ```[memap] ``` in the title.

View File

@ -40,7 +40,7 @@ setup(name='mbed-tools',
url='https://github.com/mbedmicro/mbed',
packages=find_packages(),
license=LICENSE,
install_requires=["PrettyTable>=0.7.2", "PySerial>=2.7", "IntelHex>=1.3", "colorama>=0.3.3", "Jinja2>=2.7.3", "project-generator>=0.8.11,<0.9.0", "junit-xml", "requests", "pyelftools"])
install_requires=["PrettyTable>=0.7.2", "PySerial>=2.7", "IntelHex>=1.3", "colorama>=0.3.3", "Jinja2>=2.7.3", "project-generator>=0.8.11,<0.9.0", "junit-xml", "requests"])
# Restore previous private_settings if needed
if backup:

508
tools/memap.py Normal file
View File

@ -0,0 +1,508 @@
#! /usr/bin/env python
# Memory Map File Analyser for ARM mbed OS
import argparse
import sys
import string
import os
import re
import json
import time
import string
import StringIO
from prettytable import PrettyTable
debug = False
class MemmapParser(object):
def __init__(self):
"""
General initialization
"""
# list of all modules and their sections
self.modules = dict()
self.misc_flash_sections = ('.interrupts', '.flash_config')
self.other_sections = ('.interrupts_ram', '.init', '.ARM.extab', '.ARM.exidx', '.ARM.attributes', \
'.eh_frame', '.init_array', '.fini_array', '.jcr', '.stab', '.stabstr', \
'.ARM.exidx','.ARM' )
# sections to print info (generic for all toolchains)
self.sections = ('.text', '.data', '.bss', '.heap', '.stack',)
# need to have sections merged in this order ()
self.all_sections = self.sections + self.other_sections + \
self.misc_flash_sections + ('unknown', 'OUTPUT')
self.print_sections = ('.text', '.data', '.bss')
# list of all object files and mappting to module names
self.object_to_module = dict()
def generate_output(self, file, json_mode):
"""
Generates summary of memory map data
Parameters
file: descriptor (either stdout or file)
json_mode: generates output in json formal (True/False)
"""
buf = StringIO.StringIO()
# Calculate misc flash sections
misc_flash_mem = 0
for i in self.modules:
for k in self.misc_flash_sections:
if self.modules[i][k]:
misc_flash_mem += self.modules[i][k]
# Create table
colums = ['Module']
for i in list(self.print_sections):
colums.append(i)
table = PrettyTable(colums)
table.align["Module"] = "l"
subtotal = dict()
for k in self.sections:
subtotal[k] = 0
json_obj = []
for i in sorted(self.modules):
row = []
row.append(i)
for k in self.sections:
subtotal[k] += self.modules[i][k]
for k in self.print_sections:
row.append(self.modules[i][k])
json_obj.append({ "module":i, "size":{k:self.modules[i][k] for k in self.print_sections}})
table.add_row(row)
subtotal_row = ['Subtotals']
for k in self.print_sections:
subtotal_row.append(subtotal[k])
table.add_row(subtotal_row)
if json_mode:
json_obj.append({ "summary":{'static_ram':(subtotal['.data']+subtotal['.bss']),
'heap':(subtotal['.heap']),
'stack':(subtotal['.stack']),
'total_ram':(subtotal['.data']+subtotal['.bss']+subtotal['.heap']+subtotal['.stack']),
'total_flash':(subtotal['.text']+subtotal['.data']+misc_flash_mem),}})
file.write(json.dumps(json_obj, indent=4))
file.write('\n')
else:
file.write(table.get_string())
file.write('\n')
file.write("Static RAM memory (data + bss): %s\n" % (str(subtotal['.data']+subtotal['.bss'])))
file.write("Heap: %s\n" % str(subtotal['.heap']))
file.write("Stack: %s\n" % str(subtotal['.stack']))
file.write("Total RAM memory (data + bss + heap + stack): %s\n" % (str(subtotal['.data']+subtotal['.bss']+subtotal['.heap']+subtotal['.stack'])))
file.write("Total Flash memory (text + data + misc): %s\n" % (str(subtotal['.text']+subtotal['.data']+misc_flash_mem)))
return
def module_add(self, module_name, size, section):
"""
Adds a module / section to the list
"""
if module_name in self.modules:
self.modules[module_name][section] += size
else:
temp_dic = dict()
for x in self.all_sections:
temp_dic[x] = 0
temp_dic[section] = size
self.modules[module_name] = temp_dic
def find_start_gcc(self,line):
"""
Checks location of gcc map file to start parsing map file
"""
if line.startswith('Linker script and memory map'):
return True
else:
return False
def find_start_armcc(self,line):
"""
Checks location of armcc map file to start parsing map file
"""
if line.startswith(' Base Addr Size'):
return True
else:
return False
def find_start_iar(self,line):
"""
Checks location of armcc map file to start parsing map file
"""
if line.startswith(' Section '):
return True
else:
return False
def check_new_section_gcc(self,line):
"""
Check whether a new section in a map file has been detected (only applies to gcc)
"""
for i in self.all_sections:
if line.startswith(i):
return i # should name of the section (assuming it's a known one)
if line.startswith('.'):
return 'unknown' # all others are clasified are unknown
else:
return False # everything else, means no change in section
def path_object_to_module_name(self,txt):
"""
Parses path to object file and extracts module / object data
"""
txt = txt.replace('\\','/')
rex_mbed_os_name = r'^.+mbed-os\/(.+)\/(.+\.o)$'
test_rex_mbed_os_name = re.match(rex_mbed_os_name,txt)
if test_rex_mbed_os_name:
object_name = test_rex_mbed_os_name.group(2)
data = test_rex_mbed_os_name.group(1).split('/')
ndata = len(data)
if ndata == 1:
module_name = data[0]
else:
module_name = data[0] + '/' + data[1]
return [module_name, object_name]
else:
return ['Misc', ""]
def parse_section_gcc(self,line):
"""
Parse data from a section of gcc map file
"""
# examples
# 0x00004308 0x7c ./.build/K64F/GCC_ARM/mbed-os/hal/targets/hal/TARGET_Freescale/TARGET_KPSDK_MCUS/spi_api.o
# .text 0x00000608 0x198 ./.build/K64F/GCC_ARM/mbed-os/core/mbed-rtos/rtx/TARGET_CORTEX_M/TARGET_RTOS_M4_M7/TOOLCHAIN_GCC/HAL_CM4.o
rex_address_len_name = r'^\s+.*0x(\w{8,16})\s+0x(\w+)\s(.+)$'
test_address_len_name = re.match(rex_address_len_name,line)
if test_address_len_name:
if int(test_address_len_name.group(2),16) == 0: # size == 0
return ["",0] # no valid entry
else:
m_name, m_object = self.path_object_to_module_name(test_address_len_name.group(3))
m_size = int(test_address_len_name.group(2),16)
return [m_name,m_size]
else: # special cortner case for *fill* sections
# example
# *fill* 0x0000abe4 0x4
rex_address_len = r'^\s+\*fill\*\s+0x(\w{8,16})\s+0x(\w+).*$'
test_address_len = re.match(rex_address_len,line)
if test_address_len:
if int(test_address_len.group(2),16) == 0: # size == 0
return ["",0] # no valid entry
else:
m_name = 'Misc'
m_size = int(test_address_len.group(2),16)
return [m_name,m_size]
else:
return ["",0] # no valid entry
def parse_map_file_gcc(self, file):
"""
Main logic to decode gcc map files
"""
current_section = 'unknown'
with file as infile:
# Search area to parse
for line in infile:
if self.find_start_gcc(line) == True:
current_section = "unknown"
break
# Start decoding the map file
for line in infile:
change_section = self.check_new_section_gcc(line)
if change_section == "OUTPUT": # finish parsing file: exit
break
elif change_section != False:
current_section = change_section
[module_name, module_size] = self.parse_section_gcc(line)
if module_size == 0 or module_name == "":
pass
else:
self.module_add(module_name, module_size, current_section)
if debug:
print "Line: %s" % line,
print "Module: %s\tSection: %s\tSize: %s" % (module_name,current_section,module_size)
raw_input("----------")
def parse_section_armcc(self,line):
"""
Parse data from an armcc map file
"""
# Examples of armcc map file:
# Base_Addr Size Type Attr Idx E Section Name Object
# 0x00000000 0x00000400 Data RO 11222 RESET startup_MK64F12.o
# 0x00000410 0x00000008 Code RO 49364 * !!!main c_w.l(__main.o)
rex_armcc = r'^\s+0x(\w{8})\s+0x(\w{8})\s+(\w+)\s+(\w+)\s+(\d+)\s+[*]?.+\s+(.+)$'
test_rex_armcc = re.match(rex_armcc,line)
if test_rex_armcc:
size = int(test_rex_armcc.group(2),16)
if test_rex_armcc.group(4) == 'RO':
section = '.text'
else:
if test_rex_armcc.group(3) == 'Data':
section = '.data'
elif test_rex_armcc.group(3) == 'Zero':
section = '.bss'
else:
print "BUG armcc map parser"
raw_input()
# lookup object in dictionary and return module name
object_name = test_rex_armcc.group(6)
if object_name in self.object_to_module:
module_name = self.object_to_module[object_name]
else:
module_name = 'Misc'
return [module_name,size,section]
else:
return ["",0,""] # no valid entry
def parse_section_iar(self,line):
"""
Parse data from an IAR map file
"""
# Examples of IAR map file:
# Section Kind Address Size Object
# .intvec ro code 0x00000000 0x198 startup_MK64F12.o [15]
# .rodata const 0x00000198 0x0 zero_init3.o [133]
# .iar.init_table const 0x00008384 0x2c - Linker created -
# Initializer bytes const 0x00000198 0xb2 <for P3 s0>
# .data inited 0x20000000 0xd4 driverAtmelRFInterface.o [70]
# .bss zero 0x20000598 0x318 RTX_Conf_CM.o [4]
# .iar.dynexit uninit 0x20001448 0x204 <Block tail>
# HEAP uninit 0x20001650 0x10000 <Block tail>
rex_iar = r'^\s+(.+)\s+(zero|const|ro code|inited|uninit)\s+0x(\w{8})\s+0x(\w+)\s+(.+)\s.+$'
test_rex_iar = re.match(rex_iar,line)
if test_rex_iar:
size = int(test_rex_iar.group(4),16)
if test_rex_iar.group(2) == 'const' or test_rex_iar.group(2) == 'ro code':
section = '.text'
elif test_rex_iar.group(2) == 'zero' or test_rex_iar.group(2) == 'uninit':
if test_rex_iar.group(1)[0:4] == 'HEAP':
section = '.heap'
elif test_rex_iar.group(1)[0:6] == 'CSTACK':
section = '.stack'
else:
section = '.bss' # default section
elif test_rex_iar.group(2) == 'inited':
section = '.data'
else:
print "BUG IAR map parser"
raw_input()
# lookup object in dictionary and return module name
object_name = test_rex_iar.group(5)
if object_name in self.object_to_module:
module_name = self.object_to_module[object_name]
else:
module_name = 'Misc'
return [module_name,size,section]
else:
return ["",0,""] # no valid entry
def parse_map_file_armcc(self, file):
"""
Main logic to decode armcc map files
"""
with file as infile:
# Search area to parse
for line in infile:
if self.find_start_armcc(line) == True:
break
# Start decoding the map file
for line in infile:
[name, size, section] = self.parse_section_armcc(line)
if size == 0 or name == "" or section == "":
pass
else:
self.module_add(name, size, section)
def parse_map_file_iar(self, file):
"""
Main logic to decode armcc map files
"""
with file as infile:
# Search area to parse
for line in infile:
if self.find_start_iar(line) == True:
break
# Start decoding the map file
for line in infile:
[name, size, section] = self.parse_section_iar(line)
if size == 0 or name == "" or section == "":
pass
else:
self.module_add(name, size, section)
def search_objects(self,path,toolchain):
"""
Check whether the specified map file matches with the toolchain.
Searches for object files and creates mapping: object --> module
"""
path = path.replace('\\','/')
# check location of map file
rex = r'^(.+\/)' + re.escape(toolchain) + r'\/(.+\.map)$'
test_rex = re.match(rex,path)
if test_rex:
search_path = test_rex.group(1) + toolchain + '/mbed-os/'
else:
# It looks this is not an mbed OS project
# object-to-module mapping cannot be generated
print "Warning: specified toolchain doesn't match with path to the memory map file."
return
for root, dirs, files in os.walk(search_path):
for file in files:
if file.endswith(".o"):
module_name, object_name = self.path_object_to_module_name(os.path.join(root, file))
if object_name in self.object_to_module:
print "WARNING: multiple usages of object file: %s" % object_name
print " Current: %s" % self.object_to_module[object_name]
print " New: %s" % module_name
print " "
else:
self.object_to_module.update({object_name:module_name})
def main():
version = '0.3.7'
time_start = time.clock()
# Parser handling
parser = argparse.ArgumentParser(description="Memory Map File Analyser for ARM mbed OS\nversion %s" % version)
parser.add_argument('file', help='memory map file')
parser.add_argument('-t','--toolchain', dest='toolchain', help='select a toolchain that corresponds to the memory map file (ARM, GCC_ARM, IAR)',
required=True)
parser.add_argument('-o','--output',help='output file name', required=False)
parser.add_argument('-j', '--json', dest='json', required=False, action="store_true",
help='output in JSON formatted list')
parser.add_argument('-v', '--version', action='version', version=version)
# Parse/run command
if len(sys.argv) <= 1:
parser.print_help()
sys.exit(1)
args, remainder = parser.parse_known_args()
try:
file_input = open(args.file,'rt')
except IOError as e:
print "I/O error({0}): {1}".format(e.errno, e.strerror)
sys.exit(0)
# Creates parser object
t = MemmapParser()
# Decode map file depending on the toolchain
if args.toolchain == "ARM":
t.search_objects(os.path.abspath(args.file),args.toolchain)
t.parse_map_file_armcc(file_input)
elif args.toolchain == "GCC_ARM":
t.parse_map_file_gcc(file_input)
elif args.toolchain == "IAR":
print "WARNING: IAR Compiler not fully supported (yet)"
print " "
t.search_objects(os.path.abspath(args.file),args.toolchain)
t.parse_map_file_iar(file_input)
else:
print "Invalid toolchain. Options are: ARM, GCC_ARM, IAR"
sys.exit(0)
# Write output in file
if args.output != None:
try:
file_output = open(args.output,'w')
t.generate_output(file_output,args.json)
file_output.close()
except IOError as e:
print "I/O error({0}): {1}".format(e.errno, e.strerror)
sys.exit(0)
else: # Write output in screen
t.generate_output(sys.stdout,args.json)
file_input.close()
print "Elapsed time: %smS" %int(round((time.clock()-time_start)*1000))
sys.exit(0)
if __name__ == "__main__":
main()

View File

@ -22,18 +22,18 @@ from copy import copy
from time import time, sleep
from types import ListType
from shutil import copyfile
from os.path import join, splitext, exists, relpath, dirname, basename, split
from os.path import join, splitext, exists, relpath, dirname, basename, split, abspath
from inspect import getmro
from elftools.elf.elffile import ELFFile
from multiprocessing import Pool, cpu_count
from tools.utils import run_cmd, mkdir, rel_path, ToolException, NotSupportedException, split_path
from tools.settings import BUILD_OPTIONS, MBED_ORG_USER
import tools.hooks as hooks
from tools.memap import MemmapParser
from hashlib import md5
import fnmatch
#Disables multiprocessing if set to higher number than the host machine CPUs
CPU_COUNT_MIN = 1
@ -406,6 +406,7 @@ class mbedToolchain:
# Append root path to glob patterns
# and append patterns to ignorepatterns
self.ignorepatterns.extend([join(root,line.strip()) for line in lines])
for d in copy(dirs):
dir_path = join(root, d)
if d == '.hg':
@ -747,6 +748,7 @@ class mbedToolchain:
filename = name+'.'+ext
elf = join(tmp_path, name + '.elf')
bin = join(tmp_path, filename)
map = join(tmp_path, name + '.map')
if self.need_update(elf, r.objects + r.libraries + [r.linker_script]):
needed_update = True
@ -759,10 +761,7 @@ class mbedToolchain:
self.binary(r, elf, bin)
self.info("Memory sections sizes:")
size_dict = self.static_sizes(elf)
for section, size in size_dict.iteritems():
print("{:20} {}".format(section, size))
self.mem_stats(map)
self.var("compile_succeded", True)
self.var("binary", filename)
@ -820,41 +819,35 @@ class mbedToolchain:
def var(self, key, value):
self.notify({'type': 'var', 'key': key, 'val': value})
def static_sizes(self, elf):
"""Accepts elf, returns a dict sizes per section (text, data, bss)"""
section_sizes = {}
def mem_stats(self, map):
# Creates parser object
toolchain = self.__class__.__name__
t = MemmapParser()
SHF_WRITE = 0x1
SHF_ALLOC = 0x2
SHF_EXECINSTR = 0x4
SHT_PROGBITS = "SHT_PROGBITS"
SHT_NOBITS = "SHT_NOBITS"
text = 0
data = 0
bss = 0
with open(elf, 'rb') as f:
elffile = ELFFile(f)
for section in elffile.iter_sections():
flags = section['sh_flags']
size = section['sh_size']
if (flags & SHF_ALLOC) == 0:
# Section has no relevant data so ignore it
continue
if (flags & SHF_EXECINSTR) or not (flags & SHF_WRITE):
# Executable code or read only data
text += size
elif section['sh_type'] != SHT_NOBITS:
# Non-zero read/write data
data += size
try:
with open(map, 'rt') as f:
# Decode map file depending on the toolchain
if toolchain == "ARM_STD" or toolchain == "ARM_MICRO":
t.search_objects(abspath(map), "ARM")
t.parse_map_file_armcc(f)
elif toolchain == "GCC_ARM":
t.parse_map_file_gcc(f)
elif toolchain == "IAR":
self.info("[WARNING] IAR Compiler not fully supported (yet)")
t.search_objects(abspath(map), toolchain)
t.parse_map_file_iar(f)
else:
# Zero init read/write data
bss += size
section_sizes["text"] = text
section_sizes["data"] = data
section_sizes["bss"] = bss
return section_sizes
self.info("Unknown toolchain for memory statistics %s" % toolchain)
return
t.generate_output(sys.stdout, False)
map_out = splitext(map)[0] + "_map.json"
with open(map_out, 'w') as fo:
t.generate_output(fo, True)
except OSError:
return
from tools.settings import ARM_BIN
from tools.settings import GCC_ARM_PATH, GCC_CR_PATH
from tools.settings import IAR_PATH