Merge pull request #4392 from MarceloSalazar/memap_enhancements

Enhance memap, and configure depth level

tools/build_api.py

@@ -446,7 +446,7 @@ def build_project(src_paths, build_path, target, toolchain_name,
                   macros=None, inc_dirs=None, jobs=1, silent=False,
                   report=None, properties=None, project_id=None,
                   project_description=None, extra_verbose=False, config=None,
-                  app_config=None, build_profile=None):
+                  app_config=None, build_profile=None, stats_depth=None):
     """ Build a project. A project may be a test or a user program.
 
     Positional arguments:
@@ -475,6 +475,7 @@ def build_project(src_paths, build_path, target, toolchain_name,
     config - a Config object to use instead of creating one
     app_config - location of a chosen mbed_app.json file
     build_profile - a dict of flags that will be passed to the compiler
+    stats_depth - depth level for memap to display file/dirs
     """
 
     # Convert src_path to a list if needed
@@ -553,18 +554,18 @@ def build_project(src_paths, build_path, target, toolchain_name,
         memap_table = ''
         if memap_instance:
             # Write output to stdout in text (pretty table) format
-            memap_table = memap_instance.generate_output('table')
+            memap_table = memap_instance.generate_output('table', stats_depth)
 
             if not silent:
                 print memap_table
 
             # Write output to file in JSON format
             map_out = join(build_path, name + "_map.json")
-            memap_instance.generate_output('json', map_out)
+            memap_instance.generate_output('json', stats_depth, map_out)
 
             # Write output to file in CSV format for the CI
             map_csv = join(build_path, name + "_map.csv")
-            memap_instance.generate_output('csv-ci', map_csv)
+            memap_instance.generate_output('csv-ci', stats_depth, map_csv)
 
         resources.detect_duplicates(toolchain)
 
@@ -573,7 +574,7 @@ def build_project(src_paths, build_path, target, toolchain_name,
             cur_result["elapsed_time"] = end - start
             cur_result["output"] = toolchain.get_output() + memap_table
             cur_result["result"] = "OK"
-            cur_result["memory_usage"] = toolchain.map_outputs
+            cur_result["memory_usage"] = memap_instance.mem_report
             cur_result["bin"] = res
             cur_result["elf"] = splitext(res)[0] + ".elf"
             cur_result.update(toolchain.report)
@@ -1323,7 +1324,7 @@ def print_build_memory_usage(report):
     """
     from prettytable import PrettyTable
     columns_text = ['name', 'target', 'toolchain']
-    columns_int = ['static_ram', 'stack', 'heap', 'total_ram', 'total_flash']
+    columns_int = ['static_ram', 'total_flash']
     table = PrettyTable(columns_text + columns_int)
 
     for col in columns_text:
@@ -1350,10 +1351,6 @@ def print_build_memory_usage(report):
                                 record['toolchain_name'],
                                 record['memory_usage'][-1]['summary'][
                                     'static_ram'],
-                                record['memory_usage'][-1]['summary']['stack'],
-                                record['memory_usage'][-1]['summary']['heap'],
-                                record['memory_usage'][-1]['summary'][
-                                    'total_ram'],
                                 record['memory_usage'][-1]['summary'][
                                     'total_flash'],
                             ]

tools/make.py

@@ -58,40 +58,47 @@ if __name__ == '__main__':
     # Parse Options
     parser = get_default_options_parser(add_app_config=True)
     group = parser.add_mutually_exclusive_group(required=False)
-    group.add_argument("-p",
+    group.add_argument(
+        "-p",
         type=argparse_many(test_known),
         dest="program",
         help="The index of the desired test program: [0-%d]" % (len(TESTS)-1))
 
-    group.add_argument("-n",
+    group.add_argument(
+        "-n",
         type=argparse_many(test_name_known),
         dest="program",
         help="The name of the desired test program")
 
-    parser.add_argument("-j", "--jobs",
+    parser.add_argument(
+        "-j", "--jobs",
         type=int,
         dest="jobs",
         default=0,
         help="Number of concurrent jobs. Default: 0/auto (based on host machine's number of CPUs)")
 
-    parser.add_argument("-v", "--verbose",
+    parser.add_argument(
+        "-v", "--verbose",
         action="store_true",
         dest="verbose",
         default=False,
         help="Verbose diagnostic output")
 
-    parser.add_argument("--silent",
+    parser.add_argument(
+        "--silent",
         action="store_true",
         dest="silent",
         default=False,
         help="Silent diagnostic output (no copy, compile notification)")
 
-    parser.add_argument("-D",
+    parser.add_argument(
+        "-D",
         action="append",
         dest="macros",
         help="Add a macro definition")
 
-    group.add_argument("-S", "--supported-toolchains",
+    group.add_argument(
+        "-S", "--supported-toolchains",
         dest="supported_toolchains",
         default=False,
         const="matrix",
@@ -99,11 +106,19 @@ if __name__ == '__main__':
         nargs="?",
         help="Displays supported matrix of MCUs and toolchains")
 
-    parser.add_argument('-f', '--filter',
+    parser.add_argument(
+        '-f', '--filter',
         dest='general_filter_regex',
         default=None,
         help='For some commands you can use filter to filter out results')
 
+    parser.add_argument(
+        "--stats-depth",
+        type=int,
+        dest="stats_depth",
+        default=2,
+        help="Depth level for static memory report")
+
     # Local run
     parser.add_argument("--automated", action="store_true", dest="automated",
                       default=False, help="Automated test")
@@ -277,7 +292,8 @@ if __name__ == '__main__':
                                      inc_dirs=[dirname(MBED_LIBRARIES)],
                                      build_profile=extract_profile(parser,
                                                                    options,
-                                                                   toolchain))
+                                                                   toolchain),
+                                     stats_depth=options.stats_depth)
             print 'Image: %s'% bin_file
 
             if options.disk:

tools/memap.py

@@ -8,13 +8,12 @@ import re
 import csv
 import json
 import argparse
+from copy import deepcopy
 from prettytable import PrettyTable
 
 from utils import argparse_filestring_type, \
     argparse_lowercase_hyphen_type, argparse_uppercase_type
 
-DEBUG = False
-
 RE_ARMCC = re.compile(
     r'^\s+0x(\w{8})\s+0x(\w{8})\s+(\w+)\s+(\w+)\s+(\d+)\s+[*]?.+\s+(.+)$')
 RE_IAR = re.compile(
@@ -38,22 +37,18 @@ class MemapParser(object):
     # sections to print info (generic for all toolchains)
     sections = ('.text', '.data', '.bss', '.heap', '.stack')
 
-    def __init__(self, detailed_misc=False):
+    def __init__(self):
         """ General initialization
         """
-        # 
-        self.detailed_misc = detailed_misc
 
         # list of all modules and their sections
-        self.modules = dict()
+        self.modules = dict()       # full list - doesn't change with depth
+        self.short_modules = dict() # short version with specific depth
 
         # sections must be defined in this order to take irrelevant out
         self.all_sections = self.sections + self.other_sections + \
                             self.misc_flash_sections + ('unknown', 'OUTPUT')
 
-        # list of all object files and mappting to module names
-        self.object_to_module = dict()
-
         # Memory report (sections + summary)
         self.mem_report = []
 
@@ -62,23 +57,68 @@ class MemapParser(object):
 
         self.subtotal = dict()
 
-    def module_add(self, module_name, size, section):
+        self.misc_flash_mem = 0
+
+
+    def remove_unused_modules(self):
+        """ Removes modules/objects that were compiled but are not used
+        """
+
+        # Using keys to be able to remove entry
+        for i in self.modules.keys():
+            size = 0
+            for k in self.print_sections:
+                size += self.modules[i][k]
+            if size == 0:
+                del self.modules[i]
+
+    def module_init(self, object_name):
+        """ Initialize a module. Just adds the name of the module
+
+        Positional arguments:
+        object_name - name of the entry to add
+        """
+
+        if object_name not in self.modules:
+            temp_dic = dict()
+            for section_idx in self.all_sections:
+                temp_dic[section_idx] = 0
+            self.modules[object_name] = temp_dic
+
+    def module_add(self, object_name, size, section):
         """ Adds a module / section to the list
 
         Positional arguments:
-        module_name - name of the module to add
+        object_name - name of the entry to add
         size - the size of the module being added
         section - the section the module contributes to
         """
 
-        if module_name in self.modules:
-            self.modules[module_name][section] += size
-        else:
-            temp_dic = dict()
+        # Check if object is a sub-string of key
+        for module_path in self.modules:
+
+            # this is required to differenciate: main.o vs xxxmain.o
+            module_split = os.path.basename(module_path)
+            obj_split = os.path.basename(object_name)
+
+            if module_split == obj_split:
+                self.modules[module_path][section] += size
+                return
+
+        new_module = dict()
         for section_idx in self.all_sections:
-                temp_dic[section_idx] = 0
-            temp_dic[section] = size
-            self.modules[module_name] = temp_dic
+            new_module[section_idx] = 0
+        new_module[section] = size
+        self.modules[object_name] = new_module
+
+    def module_replace(self, old_object, new_object):
+        """ Replaces an object name with a new one
+        """
+
+        # Check if object is a sub-string of key
+        if old_object in self.modules:
+            self.modules[new_object] = self.modules[old_object]
+            del self.modules[old_object]
 
     def check_new_section_gcc(self, line):
         """ Check whether a new section in a map file has been detected (only
@@ -99,43 +139,41 @@ class MemapParser(object):
             return False         # everything else, means no change in section
 
 
-    def path_object_to_module_name(self, txt):
-        """ Parse a path to object file to extract it's module and object data
+    def parse_object_name_gcc(self, line):
+        """ Parse a path to object file
 
         Positional arguments:
         txt - the path to parse the object and module name from
         """
 
-        txt = txt.replace('\\', '/')
-        rex_mbed_os_name = r'^.+mbed-os\/(.+)\/(.+\.o)$'
-        test_rex_mbed_os_name = re.match(rex_mbed_os_name, txt)
+        line = line.replace('\\', '/')
+        RE_OBJECT_FILE = r'^.+\/(.+\.o)$'
+        test_re_mbed_os_name = re.match(RE_OBJECT_FILE, line)
 
-        if test_rex_mbed_os_name:
+        if test_re_mbed_os_name:
 
-            object_name = test_rex_mbed_os_name.group(2)
-            data = test_rex_mbed_os_name.group(1).split('/')
-            ndata = len(data)
+            object_name = test_re_mbed_os_name.group(1)
+
+            # corner case: certain objects are provided by the GCC toolchain
+            if 'arm-none-eabi' in line:
+                object_name = '[lib]/misc/' + object_name
+
+            return object_name
 
-            if ndata == 1:
-                module_name = data[0]
         else:
-                module_name = data[0] + '/' + data[1]
 
-            if self.detailed_misc:
-                return [module_name + '/' + object_name, object_name]
+            RE_LIBRARY_OBJECT_FILE = r'^.+\/(lib.+\.a)\((.+\.o)\)$'
+            test_re_obj_name = re.match(RE_LIBRARY_OBJECT_FILE, line)
+
+            if test_re_obj_name:
+                object_name = test_re_obj_name.group(1) + '/' + \
+                              test_re_obj_name.group(2)
+
+                return '[lib]/' + object_name
+
             else:
-                return [module_name, object_name]
-            
-        elif self.detailed_misc:           
-            rex_obj_name = r'^.+\/(.+\.o\)*)$'
-            test_rex_obj_name = re.match(rex_obj_name, txt)
-            if test_rex_obj_name:
-                object_name = test_rex_obj_name.group(1)
-                return ['Misc/' + object_name, ""]        
-                
-            return ['Misc', ""]
-        else: 
-            return ['Misc', ""]
+                print "Malformed input found when parsing GCC map: %s" % line
+                return '[misc]'
 
     def parse_section_gcc(self, line):
         """ Parse data from a section of gcc map file
@@ -147,37 +185,42 @@ class MemapParser(object):
         Positional arguments:
         line - the line to parse a section from
         """
-        rex_address_len_name = re.compile(
+
+        RE_STD_SECTION_GCC = re.compile(
             r'^\s+.*0x(\w{8,16})\s+0x(\w+)\s(.+)$')
 
-        test_address_len_name = re.match(rex_address_len_name, line)
+        test_address_len_name = re.match(RE_STD_SECTION_GCC, line)
 
         if test_address_len_name:
 
             if int(test_address_len_name.group(2), 16) == 0: # size == 0
                 return ["", 0] # no valid entry
             else:
-                m_name, _ = self.path_object_to_module_name(
+                o_name = self.parse_object_name_gcc(\
                     test_address_len_name.group(3))
-                m_size = int(test_address_len_name.group(2), 16)
-                return [m_name, m_size]
+                o_size = int(test_address_len_name.group(2), 16)
+
+                return [o_name, o_size]
 
         else: # special corner case for *fill* sections
             #  example
             # *fill*         0x0000abe4        0x4
-            rex_address_len = r'^\s+\*fill\*\s+0x(\w{8,16})\s+0x(\w+).*$'
-            test_address_len = re.match(rex_address_len, line)
+
+            RE_FILL_SECTION_GCC = r'^\s+\*fill\*\s+0x(\w{8,16})\s+0x(\w+).*$'
+
+            test_address_len = re.match(RE_FILL_SECTION_GCC, line)
 
             if test_address_len:
                 if int(test_address_len.group(2), 16) == 0: # size == 0
                     return ["", 0] # no valid entry
                 else:
-                    m_name = 'Fill'
-                    m_size = int(test_address_len.group(2), 16)
-                    return [m_name, m_size]
+                    o_name = '[fill]'
+                    o_size = int(test_address_len.group(2), 16)
+                    return [o_name, o_size]
             else:
                 return ["", 0] # no valid entry
 
+
     def parse_map_file_gcc(self, file_desc):
         """ Main logic to decode gcc map files
 
@@ -205,18 +248,40 @@ class MemapParser(object):
                 elif change_section != False:
                     current_section = change_section
 
-                [module_name, module_size] = self.parse_section_gcc(line)
+                [object_name, object_size] = self.parse_section_gcc(line)
 
-                if module_size == 0 or module_name == "":
+                if object_size == 0 or object_name == "":
                     pass
                 else:
-                    self.module_add(module_name, module_size, current_section)
+                    self.module_add(object_name, object_size,\
+                                        current_section)
+
+    def parse_object_name_armcc(self, line):
+        """ Parse object file
+
+        Positional arguments:
+        line - the line containing the object or library
+        """
+
+        # simple object (not library)
+        if line[-2] == '.' and line[-1] == 'o':
+            return line
+
+        else:
+
+            RE_OBJECT_ARMCC = r'(.+\.l)\((.+\.o)\)'
+            test_re_obj_name = re.match(RE_OBJECT_ARMCC, line)
+
+            if test_re_obj_name:
+                object_name = test_re_obj_name.group(1) + '/' + \
+                              test_re_obj_name.group(2)
+
+                return '[lib]/' + object_name
+            else:
+                print "Malformed input found when parsing ARMCC map: %s" % line
+                return '[misc]'
+
 
-                if DEBUG:
-                    print "Line: %s" % line,
-                    print "Module: %s\tSection: %s\tSize: %s" % \
-                        (module_name, current_section, module_size)
-                    raw_input("----------")
 
     def parse_section_armcc(self, line):
         """ Parse data from an armcc map file
@@ -230,34 +295,47 @@ class MemapParser(object):
         line - the line to parse the section data from
         """
 
-        test_rex_armcc = re.match(RE_ARMCC, line)
+        test_re_armcc = re.match(RE_ARMCC, line)
 
-        if test_rex_armcc:
+        if test_re_armcc:
 
-            size = int(test_rex_armcc.group(2), 16)
+            size = int(test_re_armcc.group(2), 16)
 
-            if test_rex_armcc.group(4) == 'RO':
+            if test_re_armcc.group(4) == 'RO':
                 section = '.text'
             else:
-                if test_rex_armcc.group(3) == 'Data':
+                if test_re_armcc.group(3) == 'Data':
                     section = '.data'
-                elif test_rex_armcc.group(3) == 'Zero':
+                elif test_re_armcc.group(3) == 'Zero':
                     section = '.bss'
                 else:
-                    print "BUG armcc map parser"
-                    raw_input()
+                    print "Malformed input found when parsing armcc map: %s" %\
+                          line
 
-            # lookup object in dictionary and return module name
-            object_name = test_rex_armcc.group(6)
-            if object_name in self.object_to_module:
-                module_name = self.object_to_module[object_name]
-            else:
-                module_name = 'Misc'
+            # check name of object or library
+            object_name = self.parse_object_name_armcc(\
+                test_re_armcc.group(6))
 
-            return [module_name, size, section]
+            return [object_name, size, section]
 
         else:
-            return ["", 0, ""] # no valid entry
+            return ["", 0, ""]
+
+    def parse_object_name_iar(self, line):
+        """ Parse object file
+
+        Positional arguments:
+        line - the line containing the object or library
+        """
+
+        # simple object (not library)
+        if line[-2] == '.' and line[-1] == 'o':
+            object_name = line
+            return object_name
+
+        else:
+            return '[misc]'
+
 
     def parse_section_iar(self, line):
         """ Parse data from an IAR map file
@@ -277,38 +355,34 @@ class MemapParser(object):
         line - the line to parse section data from
         """
 
-        test_rex_iar = re.match(RE_IAR, line)
+        test_re_iar = re.match(RE_IAR, line)
 
-        if test_rex_iar:
+        if test_re_iar:
 
-            size = int(test_rex_iar.group(4), 16)
+            size = int(test_re_iar.group(4), 16)
 
-            if test_rex_iar.group(2) == 'const' or \
-               test_rex_iar.group(2) == 'ro code':
+            if test_re_iar.group(2) == 'const' or \
+               test_re_iar.group(2) == 'ro code':
                 section = '.text'
-            elif test_rex_iar.group(2) == 'zero' or \
-            test_rex_iar.group(2) == 'uninit':
-                if test_rex_iar.group(1)[0:4] == 'HEAP':
+            elif test_re_iar.group(2) == 'zero' or \
+            test_re_iar.group(2) == 'uninit':
+                if test_re_iar.group(1)[0:4] == 'HEAP':
                     section = '.heap'
-                elif test_rex_iar.group(1)[0:6] == 'CSTACK':
+                elif test_re_iar.group(1)[0:6] == 'CSTACK':
                     section = '.stack'
                 else:
                     section = '.bss' #  default section
 
-            elif test_rex_iar.group(2) == 'inited':
+            elif test_re_iar.group(2) == 'inited':
                 section = '.data'
             else:
-                print "BUG IAR map parser"
-                raw_input()
+                print "Malformed input found when parsing IAR map: %s" % line
 
             # lookup object in dictionary and return module name
-            object_name = test_rex_iar.group(5)
-            if object_name in self.object_to_module:
-                module_name = self.object_to_module[object_name]
-            else:
-                module_name = 'Misc'
+            temp = test_re_iar.group(5)
+            object_name = self.parse_object_name_iar(temp)
 
-            return [module_name, size, section]
+            return [object_name, size, section]
 
         else:
             return ["", 0, ""] # no valid entry
@@ -330,12 +404,51 @@ class MemapParser(object):
             # Start decoding the map file
             for line in infile:
 
-                [name, size, section] = self.parse_section_armcc(line)
+                [object_name, object_size, section] = \
+                                self.parse_section_armcc(line)
 
-                if size == 0 or name == "" or section == "":
+                if object_size == 0 or object_name == "" or section == "":
                     pass
                 else:
-                    self.module_add(name, size, section)
+                    self.module_add(object_name, object_size, section)
+
+
+    def check_new_library_iar(self, line):
+        """
+        Searches for libraries and returns name. Example:
+        m7M_tls.a: [43]
+
+        """
+
+        RE_LIBRARY_IAR = re.compile(r'^(.+\.a)\:.+$')
+
+        test_address_line = re.match(RE_LIBRARY_IAR, line)
+
+        if test_address_line:
+            return test_address_line.group(1)
+        else:
+            return ""
+
+    def check_new_object_lib_iar(self, line):
+        """
+        Searches for objects within a library section and returns name. Example:
+        rt7M_tl.a: [44]
+            ABImemclr4.o                 6
+            ABImemcpy_unaligned.o      118
+            ABImemset48.o               50
+            I64DivMod.o                238
+            I64DivZer.o                  2
+
+        """
+
+        RE_OBJECT_LIBRARY_IAR = re.compile(r'^\s+(.+\.o)\s.*')
+
+        test_address_line = re.match(RE_OBJECT_LIBRARY_IAR, line)
+
+        if test_address_line:
+            return test_address_line.group(1)
+        else:
+            return ""
 
     def parse_map_file_iar(self, file_desc):
         """ Main logic to decode IAR map files
@@ -344,8 +457,8 @@ class MemapParser(object):
         file_desc - a file like object to parse as an IAR map file
         """
 
+        # first round, search for objects
         with file_desc as infile:
-
             # Search area to parse
             for line in infile:
                 if line.startswith('  Section  '):
@@ -361,45 +474,142 @@ class MemapParser(object):
                 else:
                     self.module_add(name, size, section)
 
-    def search_objects(self, path):
-        """ Searches for object files and creates mapping: object --> module
+                if line.startswith('*** MODULE SUMMARY'): # finish section
+                    break
+
+            # Start decoding the map file
+            current_library = ""
+            for line in infile:
+
+                library = self.check_new_library_iar(line)
+
+                if library != "":
+                    current_library = library
+
+                object_name = self.check_new_object_lib_iar(line)
+
+                if object_name != "" and current_library != "":
+                    temp = '[lib]' + '/'+ current_library + '/'+ object_name
+                    self.module_replace(object_name, temp)
+
+
+    export_formats = ["json", "csv-ci", "table"]
+
+    def list_dir_obj(self, path):
+        """ Searches all objects in BUILD directory and creates list
 
         Positional arguments:
-        path - the path to an object file
+        path - the path to a map file
         """
 
         path = path.replace('\\', '/')
 
         # check location of map file
-        rex = r'^(.+)' + r'\/(.+\.map)$'
-        test_rex = re.match(rex, path)
+        RE_PATH_MAP_FILE = r'^(.+)\/.+\.map$'
+        test_re = re.match(RE_PATH_MAP_FILE, path)
 
-        if test_rex:
-            search_path = test_rex.group(1) + '/mbed-os/'
+        if test_re:
+            search_path = test_re.group(1)
         else:
             print "Warning: this doesn't look like an mbed project"
             return
 
+        # create empty disctionary
+        self.modules = dict()
+
+        # search for object files
         for root, _, obj_files in os.walk(search_path):
             for obj_file in obj_files:
                 if obj_file.endswith(".o"):
-                    module_name, object_name = self.path_object_to_module_name(
-                        os.path.join(root, obj_file))
 
-                    if object_name in self.object_to_module:
-                        if DEBUG:
-                            print "WARNING: multiple usages of object file: %s"\
-                                % object_name
-                            print "    Current: %s" % \
-                                self.object_to_module[object_name]
-                            print "    New:     %s" % module_name
-                            print " "
-                    else:
-                        self.object_to_module.update({object_name:module_name})
+                    txt = os.path.join(root, obj_file)
+
+                    txt = txt.replace('\\', '/')
+
+                    # add relative path + object to list
+                    self.module_init(txt[len(search_path)+1:])
+
+        # The code below is a special case for TESTS.
+        # mbed-os lives in a separate location and we need to explicitly search
+        # their object files skiping the TESTS folder (already scanned above)
+
+        # check location of mbed-os
+        RE_PATH_MAP_FILE = r'^(.+)\/mbed-os\/.*TESTS\/.+\.map$'
+        test_re = re.match(RE_PATH_MAP_FILE, path)
+
+        if test_re == None:
+            return
+
+        search_path = test_re.group(1)
+
+        # search for object files
+        for root, _, obj_files in os.walk(search_path):
+            for obj_file in obj_files:
+                if 'TESTS' not in root and obj_file.endswith(".o"):
+
+                    txt = os.path.join(root, obj_file)
+                    txt = txt.replace('\\', '/')
+
+                    # add relative path + object to list
+                    self.module_init(txt[len(search_path)+1:])
+
+
+    def reduce_depth(self, depth):
+        """
+        prints list of directories and objects. Examples:
+
+        (1) depth = 1:
+        main.o
+        mbed-os
+
+        (2) depth = 2:
+        main.o
+        mbed-os/test.o
+        mbed-os/drivers
+
+        """
+
+        # depth 0 or None shows all entries
+        if depth == 0 or depth == None:
+            self.short_modules = deepcopy(self.modules)
+            return
+
+        self.short_modules = dict()
+
+        # create reduced list
+        for line in self.modules:
+
+            data = line.split('/')
+            ndir = len(data)
+
+            temp = ''
+            count = 0
+
+            # iterate until the max depth level
+            max_level = min(depth, ndir)
+
+            # rebuild the path based on depth level
+            while count < max_level:
+                if count > 0:    # ignore '/' from first entry
+                    temp = temp + '/'
+
+                temp = temp + data[count]
+                count += 1
+
+            if temp not in self.short_modules:
+                temp_dic = dict()
+                for section_idx in self.all_sections:
+                    temp_dic[section_idx] = 0
+                self.short_modules[temp] = temp_dic
+
+            for section_idx in self.all_sections:
+                self.short_modules[temp][section_idx] += \
+                    self.modules[line][section_idx]
+
 
     export_formats = ["json", "csv-ci", "table"]
 
-    def generate_output(self, export_format, file_output=None):
+    def generate_output(self, export_format, depth, file_output=None):
         """ Generates summary of memory map data
 
         Positional arguments:
@@ -407,10 +617,14 @@ class MemapParser(object):
 
         Keyword arguments:
         file_desc - descriptor (either stdout or file)
+        depth - directory depth on report
 
         Returns: generated string for the 'table' format, otherwise None
         """
 
+        self.reduce_depth(depth)
+        self.compute_report()
+
         try:
             if file_output:
                 file_desc = open(file_output, 'wb')
@@ -452,29 +666,14 @@ class MemapParser(object):
 
         csv_module_section = []
         csv_sizes = []
-        for i in sorted(self.modules):
+        for i in sorted(self.short_modules):
             for k in self.print_sections:
                 csv_module_section += [i+k]
-                csv_sizes += [self.modules[i][k]]
+                csv_sizes += [self.short_modules[i][k]]
 
         csv_module_section += ['static_ram']
         csv_sizes += [self.mem_summary['static_ram']]
 
-        csv_module_section += ['heap']
-        if self.mem_summary['heap'] == 0:
-            csv_sizes += ['unknown']
-        else:
-            csv_sizes += [self.mem_summary['heap']]
-
-        csv_module_section += ['stack']
-        if self.mem_summary['stack'] == 0:
-            csv_sizes += ['unknown']
-        else:
-            csv_sizes += [self.mem_summary['stack']]
-
-        csv_module_section += ['total_ram']
-        csv_sizes += [self.mem_summary['total_ram']]
-
         csv_module_section += ['total_flash']
         csv_sizes += [self.mem_summary['total_flash']]
 
@@ -486,9 +685,6 @@ class MemapParser(object):
     def generate_table(self, file_desc):
         """Generate a table from a memoy map
 
-        Positional arguments:
-        file_desc - the file to write out the final report to
-
         Returns: string of the generated table
         """
         # Create table
@@ -503,11 +699,11 @@ class MemapParser(object):
         for i in list(self.print_sections):
             table.align[i] = 'r'
 
-        for i in sorted(self.modules):
+        for i in sorted(self.short_modules):
             row = [i]
 
             for k in self.print_sections:
-                row.append(self.modules[i][k])
+                row.append(self.short_modules[i][k])
 
             table.add_row(row)
 
@@ -520,23 +716,9 @@ class MemapParser(object):
         output = table.get_string()
         output += '\n'
 
-        if self.mem_summary['heap'] == 0:
-            output += "Allocated Heap: unknown\n"
-        else:
-            output += "Allocated Heap: %s bytes\n" % \
-                        str(self.mem_summary['heap'])
-
-        if self.mem_summary['stack'] == 0:
-            output += "Allocated Stack: unknown\n"
-        else:
-            output += "Allocated Stack: %s bytes\n" % \
-                        str(self.mem_summary['stack'])
-
         output += "Total Static RAM memory (data + bss): %s bytes\n" % \
                         str(self.mem_summary['static_ram'])
-        output += "Total RAM memory (data + bss + heap + stack): %s bytes\n" % \
-                        str(self.mem_summary['total_ram'])
-        output += "Total Flash memory (text + data + misc): %s bytes\n" % \
+        output += "Total Flash memory (text + data): %s bytes\n" % \
                         str(self.mem_summary['total_flash'])
 
         return output
@@ -544,36 +726,27 @@ class MemapParser(object):
     toolchains = ["ARM", "ARM_STD", "ARM_MICRO", "GCC_ARM", "GCC_CR", "IAR"]
 
     def compute_report(self):
+        """ Generates summary of memory usage for main areas
+        """
+
         for k in self.sections:
             self.subtotal[k] = 0
 
-        for i in sorted(self.modules):
+        for i in sorted(self.short_modules):
             for k in self.sections:
-                self.subtotal[k] += self.modules[i][k]
-
-        # Calculate misc flash sections
-        self.misc_flash_mem = 0
-        for i in self.modules:
-            for k in self.misc_flash_sections:
-                if self.modules[i][k]:
-                    self.misc_flash_mem += self.modules[i][k]
+                self.subtotal[k] += self.short_modules[i][k]
 
         self.mem_summary = {
             'static_ram': (self.subtotal['.data'] + self.subtotal['.bss']),
-            'heap': (self.subtotal['.heap']),
-            'stack': (self.subtotal['.stack']),
-            'total_ram': (self.subtotal['.data'] + self.subtotal['.bss'] +
-                          self.subtotal['.heap']+self.subtotal['.stack']),
-            'total_flash': (self.subtotal['.text'] + self.subtotal['.data'] +
-                            self.misc_flash_mem),
+            'total_flash': (self.subtotal['.text'] + self.subtotal['.data']),
         }
 
         self.mem_report = []
-        for i in sorted(self.modules):
+        for i in sorted(self.short_modules):
             self.mem_report.append({
                 "module":i,
                 "size":{
-                    k:self.modules[i][k] for k in self.print_sections
+                    k:self.short_modules[i][k] for k in self.print_sections
                 }
             })
 
@@ -592,19 +765,21 @@ class MemapParser(object):
         result = True
         try:
             with open(mapfile, 'r') as file_input:
+
+                # Common to all toolchains: first search for objects in BUILD
+                self.list_dir_obj(os.path.abspath(mapfile))
+
                 if toolchain == "ARM" or toolchain == "ARM_STD" or\
                   toolchain == "ARM_MICRO":
-                    self.search_objects(os.path.abspath(mapfile))
                     self.parse_map_file_armcc(file_input)
                 elif toolchain == "GCC_ARM" or toolchain == "GCC_CR":
                     self.parse_map_file_gcc(file_input)
                 elif toolchain == "IAR":
-                    self.search_objects(os.path.abspath(mapfile))
                     self.parse_map_file_iar(file_input)
                 else:
                     result = False
 
-            self.compute_report()
+                self.remove_unused_modules()
 
         except IOError as error:
             print "I/O error({0}): {1}".format(error.errno, error.strerror)
@@ -614,7 +789,7 @@ class MemapParser(object):
 def main():
     """Entry Point"""
 
-    version = '0.3.12'
+    version = '0.4.0'
 
     # Parser handling
     parser = argparse.ArgumentParser(
@@ -631,6 +806,10 @@ def main():
         required=True,
         type=argparse_uppercase_type(MemapParser.toolchains, "toolchain"))
 
+    parser.add_argument(
+        '-d', '--depth', dest='depth', type=int,
+        help='specify directory depth level to display report', required=False)
+
     parser.add_argument(
         '-o', '--output', help='output file name', required=False)
 
@@ -643,30 +822,33 @@ def main():
 
     parser.add_argument('-v', '--version', action='version', version=version)
 
-    parser.add_argument('-d', '--detailed', action='store_true', help='Displays the elements in "Misc" in a detailed fashion', required=False)
-
     # Parse/run command
     if len(sys.argv) <= 1:
         parser.print_help()
         sys.exit(1)
 
-
     args = parser.parse_args()
 
     # Create memap object
-    memap = MemapParser(detailed_misc=args.detailed)
+    memap = MemapParser()
 
     # Parse and decode a map file
     if args.file and args.toolchain:
         if memap.parse(args.file, args.toolchain) is False:
             sys.exit(0)
 
+    if args.depth is None:
+        depth = 2  # default depth level
+    else:
+        depth = args.depth
+
     returned_string = None
     # Write output in file
     if args.output != None:
-        returned_string = memap.generate_output(args.export, args.output)
+        returned_string = memap.generate_output(args.export, \
+            depth, args.output)
     else: # Write output in screen
-        returned_string = memap.generate_output(args.export)
+        returned_string = memap.generate_output(args.export, depth)
 
     if args.export == 'table' and returned_string:
         print returned_string

tools/test.py

@@ -100,6 +100,12 @@ if __name__ == '__main__':
                           default=False,
                           help="Verbose diagnostic output")
 
+        parser.add_argument("--stats-depth",
+                            type=int,
+                            dest="stats_depth",
+                            default=2,
+                            help="Depth level for static memory report")
+
         options = parser.parse_args()
 
         # Filter tests by path if specified
@@ -215,7 +221,8 @@ if __name__ == '__main__':
                         jobs=options.jobs,
                         continue_on_build_fail=options.continue_on_build_fail,
                         app_config=options.app_config,
-                                                             build_profile=profile)
+                        build_profile=profile,
+                        stats_depth=options.stats_depth)
 
                 # If a path to a test spec is provided, write it to a file
                 if options.test_spec:

tools/test/memap/memap_test.py

@@ -16,6 +16,7 @@ limitations under the License.
 """
 import sys
 import os
+import json
 
 ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
 sys.path.insert(0, ROOT)
@@ -42,55 +43,103 @@ class MemapParserTests(unittest.TestCase):
         self.memap_parser = MemapParser()
 
         self.memap_parser.modules = {
-            "Misc": {
-                "unknown": 0,
-                ".ARM": 8,
-                ".ARM.extab": 0,
-                ".init": 12,
-                "OUTPUT": 0,
-                ".stack": 0,
-                ".eh_frame": 0,
-                ".fini_array": 4,
+            "mbed-os/targets/TARGET/TARGET_MCUS/api/pinmap.o": {
+                ".text": 1,
+                ".data": 2,
+                ".bss": 3,
                 ".heap": 0,
-                ".stabstr": 0,
-                ".interrupts_ram": 0,
-                ".init_array": 0,
-                ".stab": 0,
-                ".ARM.attributes": 7347,
-                ".bss": 8517,
-                ".flash_config": 16,
-                ".interrupts": 1024,
-                ".data": 2325,
-                ".ARM.exidx": 0,
-                ".text": 59906,
-                ".jcr": 0
-            },
-            "Fill": {
-                "unknown": 12,
-                ".ARM": 0,
-                ".ARM.extab": 0,
-                ".init": 0,
-                "OUTPUT": 0,
                 ".stack": 0,
-                ".eh_frame": 0,
-                ".fini_array": 0,
-                ".heap": 65536,
-                ".stabstr": 0,
-                ".interrupts_ram": 1024,
-                ".init_array": 0,
-                ".stab": 0,
-                ".ARM.attributes": 0,
-                ".bss": 2235,
-                ".flash_config": 0,
-                ".interrupts": 0,
-                ".data": 3,
+                ".interrupts_ram":0,
+                ".init":0,
+                ".ARM.extab":0,
                 ".ARM.exidx":0,
-                ".text": 136,
-                ".jcr": 0
+                ".ARM.attributes":0,
+                ".eh_frame":0,
+                ".init_array":0,
+                ".fini_array":0,
+                ".jcr":0,
+                ".stab":0,
+                ".stabstr":0,
+                ".ARM.exidx":0,
+                ".ARM":0,
+                ".interrupts":0,
+                ".flash_config":0,
+                "unknown":0,
+                "OUTPUT":0,
+                },
+            "[lib]/libc.a/lib_a-printf.o": {
+                ".text": 4,
+                ".data": 5,
+                ".bss": 6,
+                ".heap": 0,
+                ".stack": 0,
+                ".interrupts_ram":0,
+                ".init":0,
+                ".ARM.extab":0,
+                ".ARM.exidx":0,
+                ".ARM.attributes":0,
+                ".eh_frame":0,
+                ".init_array":0,
+                ".fini_array":0,
+                ".jcr":0,
+                ".stab":0,
+                ".stabstr":0,
+                ".ARM.exidx":0,
+                ".ARM":0,
+                ".interrupts":0,
+                ".flash_config":0,
+                "unknown":0,
+                "OUTPUT":0,
+                },
+            "main.o": {
+                ".text": 7,
+                ".data": 8,
+                ".bss": 0,
+                ".heap": 0,
+                ".stack": 0,
+                ".interrupts_ram":0,
+                ".init":0,
+                ".ARM.extab":0,
+                ".ARM.exidx":0,
+                ".ARM.attributes":0,
+                ".eh_frame":0,
+                ".init_array":0,
+                ".fini_array":0,
+                ".jcr":0,
+                ".stab":0,
+                ".stabstr":0,
+                ".ARM.exidx":0,
+                ".ARM":0,
+                ".interrupts":0,
+                ".flash_config":0,
+                "unknown":0,
+                "OUTPUT":0,
+                },
+            "test.o": {
+                ".text": 0,
+                ".data": 0,
+                ".bss": 0,
+                ".heap": 0,
+                ".stack": 0,
+                ".interrupts_ram":0,
+                ".init":0,
+                ".ARM.extab":0,
+                ".ARM.exidx":0,
+                ".ARM.attributes":0,
+                ".eh_frame":0,
+                ".init_array":0,
+                ".fini_array":0,
+                ".jcr":0,
+                ".stab":0,
+                ".stabstr":0,
+                ".ARM.exidx":0,
+                ".ARM":0,
+                ".interrupts":0,
+                ".flash_config":0,
+                "unknown":0,
+                "OUTPUT":0,
+                },
         }
-        }
-        
-        self.memap_parser.compute_report()
 
     def tearDown(self):
         """
@@ -100,10 +149,10 @@ class MemapParserTests(unittest.TestCase):
         """
         pass
 
-    def generate_test_helper(self, output_type, file_output=None):
+    def generate_test_helper(self, output_type, depth, file_output=None):
         """
-        Helper that ensures that the member variables "modules", "mem_report",
-        and "mem_summary" are unchanged after calling "generate_output"
+        Helper that ensures that the member variables "modules" is
+        unchanged after calling "generate_output"
 
         :param output_type: type string that is passed to "generate_output"
         :param file_output: path to output file that is passed to "generate_output"
@@ -111,15 +160,12 @@ class MemapParserTests(unittest.TestCase):
         """
 
         old_modules = deepcopy(self.memap_parser.modules)
-        old_report = deepcopy(self.memap_parser.mem_report)
-        old_summary = deepcopy(self.memap_parser.mem_summary)
-        self.memap_parser.generate_output(output_type, file_output)
+
+        self.memap_parser.generate_output(output_type, depth, file_output)
+
         self.assertEqual(self.memap_parser.modules, old_modules,
                         "generate_output modified the 'modules' property")
-        self.assertEqual(self.memap_parser.mem_report, old_report,
-                         "generate_output modified the 'mem_report' property")
-        self.assertEqual(self.memap_parser.mem_summary, old_summary,
-                         "generate_output modified the 'mem_summary' property")
+
 
     def test_report_computed(self):
         """
@@ -127,11 +173,12 @@ class MemapParserTests(unittest.TestCase):
 
         :return:
         """
-        self.assertTrue(self.memap_parser.mem_report)
+
+        self.memap_parser.generate_output('table', 2)
+
+        # Report is created after generating output
         self.assertTrue(self.memap_parser.mem_summary)
-        self.assertEqual(self.memap_parser.mem_report[-1]['summary'],
-                         self.memap_parser.mem_summary,
-                         "mem_report did not contain a correct copy of mem_summary")
+        self.assertTrue(self.memap_parser.mem_report)
 
     def test_generate_output_table(self):
         """
@@ -139,7 +186,8 @@ class MemapParserTests(unittest.TestCase):
 
         :return:
         """
-        self.generate_test_helper('table')
+        depth = 2
+        self.generate_test_helper('table', depth)
 
     def test_generate_output_json(self):
         """
@@ -148,7 +196,8 @@ class MemapParserTests(unittest.TestCase):
         :return:
         """
         file_name = '.json_test_output.json'
-        self.generate_test_helper('json', file_output=file_name)
+        depth = 2
+        self.generate_test_helper('json', depth, file_output=file_name)
         self.assertTrue(os.path.exists(file_name), "Failed to create json file")
         os.remove(file_name)
 
@@ -159,7 +208,8 @@ class MemapParserTests(unittest.TestCase):
         :return:
         """
         file_name = '.csv_ci_test_output.csv'
-        self.generate_test_helper('csv-ci', file_output=file_name)
+        depth = 2
+        self.generate_test_helper('csv-ci', depth, file_output=file_name)
         self.assertTrue(os.path.exists(file_name), "Failed to create csv-ci file")
         os.remove(file_name)
 

tools/test_api.py

@@ -496,7 +496,8 @@ class SingleTestRunner(object):
                                      properties=build_properties,
                                      project_id=test_id,
                                      project_description=test.get_description(),
-                                     build_profile=profile)
+                                     build_profile=profile,
+                                     stats_depth=stats_depth)
 
                 except Exception, e:
                     project_name_str = project_name if project_name is not None else test_id
@@ -2122,7 +2123,7 @@ def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
                 clean=False, notify=None, verbose=False, jobs=1, macros=None,
                 silent=False, report=None, properties=None,
                 continue_on_build_fail=False, app_config=None,
-                build_profile=None):
+                build_profile=None, stats_depth=None):
     """Given the data structure from 'find_tests' and the typical build parameters,
     build all the tests
 
@@ -2172,7 +2173,8 @@ def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
             'app_config': app_config,
             'build_profile': build_profile,
             'silent': True,
-            'toolchain_paths': TOOLCHAIN_PATHS
+            'toolchain_paths': TOOLCHAIN_PATHS,
+            'stats_depth': stats_depth
         }
 
         results.append(p.apply_async(build_test_worker, args, kwargs))

tools/toolchains/__init__.py

@@ -416,7 +416,6 @@ class mbedToolchain:
 
         # Print output buffer
         self.output = str()
-        self.map_outputs = list()   # Place to store memmap scan results in JSON like data structures
 
         # uVisor spepcific rules
         if 'UVISOR' in self.target.features and 'UVISOR_SUPPORTED' in self.target.extra_labels:
@@ -1128,7 +1127,8 @@ class mbedToolchain:
             self.progress("elf2bin", name)
             self.binary(r, elf, bin)
 
-        self.map_outputs = self.mem_stats(map)
+        # Initialize memap and process map file. This doesn't generate output.
+        self.mem_stats(map)
 
         self.var("compile_succeded", True)
         self.var("binary", filename)
@@ -1193,8 +1193,7 @@ class mbedToolchain:
     def mem_stats(self, map):
         """! Creates parser object
         @param map Path to linker map file to parse and decode
-        @return Memory summary structure with memory usage statistics
-                None if map file can't be opened and processed
+        @return None
         """
         toolchain = self.__class__.__name__
 
@@ -1209,10 +1208,10 @@ class mbedToolchain:
         # Store the memap instance for later use
         self.memap_instance = memap
 
-        # Here we return memory statistics structure (constructed after
-        # call to generate_output) which contains raw data in bytes
-        # about sections + summary
-        return memap.mem_report
+        # Note: memory statistics are not returned.
+        # Need call to generate_output later (depends on depth & output format)
+
+        return None
 
     # Set the configuration data
     def set_config_data(self, config_data):