Merge pull request #683 from PrzemekWirkus/cpputest-export

Tools: Update exporters' capabilities to support CppUTest project exports

libraries/tests/utest/basic/basic.cpp

@@ -1,4 +1,4 @@
-#include "CppUTest/TestHarness.h"
+#include "TestHarness.h"
 
 TEST_GROUP(FirstTestGroup)
 {

libraries/tests/utest/general/general.cpp

@@ -1,4 +1,4 @@
-#include "CppUTest/TestHarness.h"
+#include "TestHarness.h"
 #include <utility>
 #include "mbed.h"
 

libraries/tests/utest/semihost_fs/semihost_fs.cpp

@@ -1,4 +1,4 @@
-#include "CppUTest/TestHarness.h"
+#include "TestHarness.h"
 #include "mbed.h"
 #include "semihost_api.h"
 #include <stdio.h>

libraries/tests/utest/testrunner/testrunner.cpp

@@ -1,14 +1,14 @@
-#include "CppUTest\CommandLineTestRunner.h"
+#include "CommandLineTestRunner.h"
 #include <stdio.h>
 #include "mbed.h"
 #include "testrunner.h"
 #include "test_env.h"
 
 /**
-Object 'console' is used to show prints on console.
+Object 'mbed_cpputest_console' is used to show prints on console.
 It is declared in \cpputest\src\Platforms\armcc\UtestPlatform.cpp
 */
-Serial console(STDIO_UART_TX, STDIO_UART_RX);
+Serial mbed_cpputest_console(STDIO_UART_TX, STDIO_UART_RX);
 
 int main(int ac, char** av)
 {

workspace_tools/build.py

@@ -85,6 +85,12 @@ if __name__ == '__main__':
                       default=False,
                       help="Compile the u-blox library")
 
+    parser.add_option("", "--cpputest",
+                      action="store_true",
+                      dest="cpputest_lib",
+                      default=False,
+                      help="Compiles 'cpputest' unit test library (library should be on the same directory level as mbed repository)")
+
     parser.add_option("-D", "",
                       action="append",
                       dest="macros",
@@ -169,6 +175,8 @@ if __name__ == '__main__':
         libraries.extend(["fat"])
     if options.ublox:
         libraries.extend(["rtx", "rtos", "usb_host", "ublox"])
+    if options.cpputest_lib:
+        libraries.extend(["cpputest"])
 
     notify = print_notify_verbose if options.extra_verbose_notify else None  # Special notify for CI (more verbose)
 

workspace_tools/build_api.py

@@ -92,7 +92,7 @@ def build_project(src_path, build_path, target, toolchain_name,
 
 def build_library(src_paths, build_path, target, toolchain_name,
          dependencies_paths=None, options=None, name=None, clean=False,
-         notify=None, verbose=False, macros=None, inc_dirs=None, jobs=1):
+         notify=None, verbose=False, macros=None, inc_dirs=None, inc_dirs_ext=None, jobs=1):
     """ src_path: the path of the source directory
     build_path: the path of the build directory
     target: ['LPC1768', 'LPC11U24', 'LPC2368']
@@ -102,6 +102,7 @@ def build_library(src_paths, build_path, target, toolchain_name,
     notify: Notify function for logs
     verbose: Write the actual tools command lines if True
     inc_dirs: additional include directories which should be included in build
+    inc_dirs_ext: additional include directories which should be copied to library directory
     """
     if type(src_paths) != ListType:
         src_paths = [src_paths]
@@ -125,6 +126,13 @@ def build_library(src_paths, build_path, target, toolchain_name,
     for src_path in src_paths:
         resources.append(toolchain.scan_resources(src_path))
 
+    # Add extra include directories / files which are required by library
+    # This files usually are not in the same directory as source files so
+    # previous scan will not include them
+    if inc_dirs_ext is not None:
+        for inc_ext in inc_dirs_ext:
+            resources.append(toolchain.scan_resources(inc_ext))
+
     # Dependencies Include Paths
     dependencies_include_dir = []
     if dependencies_paths is not None:
@@ -171,6 +179,7 @@ def build_lib(lib_id, target, toolchain, options=None, verbose=False, clean=Fals
                       macros=MACROS,
                       notify=notify,
                       inc_dirs=lib.inc_dirs,
+                      inc_dirs_ext=lib.inc_dirs_ext,
                       jobs=jobs)
     else:
         print 'Library "%s" is not yet supported on target %s with toolchain %s' % (lib_id, target.name, toolchain)

workspace_tools/export/__init__.py

@@ -16,7 +16,6 @@ limitations under the License.
 """
 import os, tempfile
 from os.path import join, exists, basename
-from os import makedirs
 from shutil import copytree, rmtree
 
 from workspace_tools.utils import mkdir
@@ -50,7 +49,8 @@ def online_build_url_resolver(url):
     return {'path':'', 'name':''}
 
 
-def export(project_path, project_name, ide, target, destination='/tmp/', tempdir=None, clean=True, build_url_resolver=online_build_url_resolver):
+def export(project_path, project_name, ide, target, destination='/tmp/',
+           tempdir=None, clean=True, extra_symbols=None, build_url_resolver=online_build_url_resolver):
     # Convention: we are using capitals for toolchain and target names
     if target is not None:
         target = target.upper()
@@ -75,7 +75,7 @@ def export(project_path, project_name, ide, target, destination='/tmp/', tempdir
                 report['errormsg'] = ERROR_MESSAGE_UNSUPPORTED_TOOLCHAIN % (target, ide)
             else:
                 try:
-                    exporter = Exporter(target, tempdir, project_name, build_url_resolver)
+                    exporter = Exporter(target, tempdir, project_name, build_url_resolver, extra_symbols=extra_symbols)
                     exporter.scan_and_copy_resources(project_path, tempdir)
                     exporter.generate()
                     report['success'] = True

workspace_tools/export/codered.py

@@ -48,7 +48,7 @@ class CodeRed(Exporter):
             'linker_script': self.resources.linker_script,
             'object_files': self.resources.objects,
             'libraries': libraries,
-            'symbols': self.toolchain.get_symbols()
+            'symbols': self.get_symbols()
         }
         self.gen_file('codered_%s_project.tmpl' % self.target.lower(), ctx, '.project')
         self.gen_file('codered_%s_cproject.tmpl' % self.target.lower(), ctx, '.cproject')

workspace_tools/export/codesourcery.py

@@ -55,6 +55,6 @@ class CodeSourcery(Exporter):
             'library_paths': self.resources.lib_dirs,
             'linker_script': self.resources.linker_script,
             'libraries': libraries,
-            'symbols': self.toolchain.get_symbols()
+            'symbols': self.get_symbols()
         }
         self.gen_file('codesourcery_%s.tmpl' % self.target.lower(), ctx, 'Makefile')

workspace_tools/export/coide.py

@@ -79,7 +79,7 @@ class CoIDE(Exporter):
             'library_paths': self.resources.lib_dirs,
             'object_files': self.resources.objects,
             'libraries': libraries,
-            'symbols': self.toolchain.get_symbols()
+            'symbols': self.get_symbols()
         }
         target = self.target.lower()
 

workspace_tools/export/ds5_5.py

@@ -56,7 +56,7 @@ class DS5_5(Exporter):
             'scatter_file': self.resources.linker_script,
             'object_files': self.resources.objects + self.resources.libraries,
             'source_files': source_files,
-            'symbols': self.toolchain.get_symbols()
+            'symbols': self.get_symbols()
         }
         target = self.target.lower()
 

workspace_tools/export/emblocks.py

@@ -66,7 +66,7 @@ class IntermediateFile(Exporter):
             'script_file': self.resources.linker_script,
             'library_paths': self.resources.lib_dirs,
             'libraries': libraries,
-            'symbols': self.toolchain.get_symbols(),
+            'symbols': self.get_symbols(),
             'object_files': self.resources.objects,
             'sys_libs': self.toolchain.sys_libs,
             'cc_org': self.toolchain.cc[1:],

workspace_tools/export/exporters.py

@@ -18,7 +18,7 @@ class Exporter():
     TEMPLATE_DIR = dirname(__file__)
     DOT_IN_RELATIVE_PATH = False
 
-    def __init__(self, target, inputDir, program_name, build_url_resolver):
+    def __init__(self, target, inputDir, program_name, build_url_resolver, extra_symbols=None):
         self.inputDir = inputDir
         self.target = target
         self.program_name = program_name
@@ -26,6 +26,7 @@ class Exporter():
         self.build_url_resolver = build_url_resolver
         jinja_loader = FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
         self.jinja_environment = Environment(loader=jinja_loader)
+        self.extra_symbols = extra_symbols
 
     def get_toolchain(self):
         return self.TOOLCHAIN
@@ -97,6 +98,16 @@ class Exporter():
         logging.debug("Generating: %s" % target_path)
         open(target_path, "w").write(target_text)
 
+    def get_symbols(self, add_extra_symbols=True):
+        """ This function returns symbols which must be exported.
+            Please add / overwrite symbols in each exporter separately
+        """
+        symbols = self.toolchain.get_symbols()
+        # We have extra symbols from e.g. libraries, we want to have them also added to export
+        if add_extra_symbols:
+            if self.extra_symbols is not None:
+                symbols.extend(self.extra_symbols)
+        return symbols
 
 def zip_working_directory_and_clean_up(tempdirectory=None, destination=None, program_name=None, clean=True):
     uid = str(uuid.uuid4())

workspace_tools/export/gccarm.py

@@ -90,6 +90,6 @@ class GccArm(Exporter):
             'library_paths': self.resources.lib_dirs,
             'linker_script': self.resources.linker_script,
             'libraries': libraries,
-            'symbols': self.toolchain.get_symbols()
+            'symbols': self.get_symbols()
         }
         self.gen_file('gcc_arm_%s.tmpl' % self.target.lower(), ctx, 'Makefile')

workspace_tools/export/iar.py

@@ -59,7 +59,7 @@ class IAREmbeddedWorkbench(Exporter):
             'linker_script': self.resources.linker_script,
             'object_files': self.resources.objects,
             'libraries': self.resources.libraries,
-            'symbols': self.toolchain.get_symbols(),
+            'symbols': self.get_symbols(),
             'source_files': sources,
         }
         self.gen_file('iar_%s.ewp.tmpl' % self.target.lower(), ctx, '%s.ewp' % self.program_name)

workspace_tools/export/kds.py

@@ -39,7 +39,7 @@ class KDS(Exporter):
             'linker_script': self.resources.linker_script,
             'object_files': self.resources.objects,
             'libraries': libraries,
-            'symbols': self.toolchain.get_symbols()
+            'symbols': self.get_symbols()
         }
         self.gen_file('kds_%s_project.tmpl' % self.target.lower(), ctx, '.project')
         self.gen_file('kds_%s_cproject.tmpl' % self.target.lower(), ctx, '.cproject')

workspace_tools/export/uvision4.py

@@ -123,7 +123,7 @@ class Uvision4(Exporter):
             'scatter_file': self.resources.linker_script,
             'object_files': self.resources.objects + self.resources.libraries,
             'source_files': source_files.items(),
-            'symbols': self.toolchain.get_symbols() + ['__ASSERT_MSG'],
+            'symbols': self.get_symbols() + ['__ASSERT_MSG'],
             'hex_files' : self.resources.hex_files,
             'flags' : self.get_flags(),
         }

workspace_tools/libraries.py

@@ -94,7 +94,7 @@ LIBRARIES = [
         "build_dir": CPPUTEST_LIBRARY,
         "dependencies": [MBED_LIBRARIES],
         'inc_dirs': [CPPUTEST_INC, CPPUTEST_PLATFORM_INC, CPPUTEST_TESTRUNNER_INC, TEST_MBED_LIB],
-        'inc_dirs_ext': [CPPUTEST_INC],
+        'inc_dirs_ext': [CPPUTEST_INC_EXT],
         'macros': ["CPPUTEST_USE_MEM_LEAK_DETECTION=0", "CPPUTEST_USE_STD_CPP_LIB=0", "CPPUTEST=1"],
     },
 ]

workspace_tools/paths.py

@@ -95,8 +95,9 @@ EXPORT_TMP = join(EXPORT_DIR, ".temp")
 
 # CppUtest library
 CPPUTEST_DIR = join(ROOT, "..")
-CPPUTEST_SRC = join(CPPUTEST_DIR, "cpputest", "src", "CppUTest")        #, "CppUTest"
+CPPUTEST_SRC = join(CPPUTEST_DIR, "cpputest", "src", "CppUTest")
-CPPUTEST_INC = join(CPPUTEST_DIR, "cpputest", "include")    #, "CppUTest"
+CPPUTEST_INC = join(CPPUTEST_DIR, "cpputest", "include")
+CPPUTEST_INC_EXT = join(CPPUTEST_DIR, "cpputest", "include", "CppUTest")
 # Platform dependant code is here (for armcc compiler)
 CPPUTEST_PLATFORM_SRC = join(CPPUTEST_DIR, "cpputest", "src", "Platforms", "armcc")
 CPPUTEST_PLATFORM_INC = join(CPPUTEST_DIR, "cpputest", "include", "Platforms", "armcc")

workspace_tools/project.py

@@ -4,7 +4,6 @@ ROOT = abspath(join(dirname(__file__), ".."))
 sys.path.insert(0, ROOT)
 
 from shutil import move, rmtree
-from os.path import join, exists, basename
 from optparse import OptionParser
 
 from workspace_tools.paths import EXPORT_DIR, EXPORT_WORKSPACE, EXPORT_TMP
@@ -13,6 +12,8 @@ from workspace_tools.export import export, setup_user_prj, EXPORTERS
 from workspace_tools.utils import args_error
 from workspace_tools.tests import TESTS, Test, TEST_MAP
 from workspace_tools.targets import TARGET_NAMES
+from workspace_tools.libraries import LIBRARIES
+
 try:
     import workspace_tools.private_settings as ps
 except:
@@ -28,30 +29,44 @@ if __name__ == '__main__':
     toolchainlist = EXPORTERS.keys()
     toolchainlist.sort()
 
-    parser.add_option("-m", "--mcu", metavar="MCU", default='LPC1768',
+    parser.add_option("-m", "--mcu",
-        help="generate project for the given MCU (%s)" % ', '.join(targetnames))
+                      metavar="MCU",
+                      default='LPC1768',
+                      help="generate project for the given MCU (%s)"% ', '.join(targetnames))
 
-    parser.add_option("-i", dest="ide", default='uvision',
+    parser.add_option("-i",
-        help="The target IDE: %s" % str(toolchainlist))
+                      dest="ide",
+                      default='uvision',
+                      help="The target IDE: %s"% str(toolchainlist))
 
-    parser.add_option("-c", "--clean", action="store_true", default=False,
+    parser.add_option("-c", "--clean",
-        help="clean the export directory")
+                      action="store_true",
+                      default=False,
+                      help="clean the export directory")
 
-    parser.add_option("-p", type="int", dest="program",
+    parser.add_option("-p",
-        help="The index of the desired test program: [0-%d]" % (len(TESTS)-1))
+                      type="int",
+                      dest="program",
+                      help="The index of the desired test program: [0-%d]"% (len(TESTS)-1))
 
-    parser.add_option("-n", dest="program_name",
+    parser.add_option("-n",
-        help="The name of the desired test program")
+                      dest="program_name",
+                      help="The name of the desired test program")
 
-    parser.add_option("-b", dest="build", action="store_true", default=False,
+    parser.add_option("-b",
-        help="use the mbed library build, instead of the sources")
+                      dest="build",
+                      action="store_true",
+                      default=False,
+                      help="use the mbed library build, instead of the sources")
 
-    parser.add_option("-L", "--list-tests", action="store_true", dest="list_tests", default=False, 
+    parser.add_option("-L", "--list-tests",
-        help="list available programs in order and exit")
+                      action="store_true",
+                      dest="list_tests",
+                      default=False,
+                      help="list available programs in order and exit")
 
     (options, args) = parser.parse_args()
 
-
     # Print available tests in order and exit
     if options.list_tests is True:
         print '\n'.join(map(str, sorted(TEST_MAP.values())))
@@ -65,58 +80,84 @@ if __name__ == '__main__':
     # Target
     if options.mcu is None :
         args_error(parser, "[ERROR] You should specify an MCU")
-    mcu = options.mcu
+    mcus = options.mcu
 
     # IDE
     if options.ide is None:
         args_error(parser, "[ERROR] You should specify an IDE")
     ide = options.ide
 
-    # Program Number or name
+    # Export results
-    p, n = options.program, options.program_name
+    successes = []
+    failures = []
 
-    if n is not None and p is not None:
+    for mcu in mcus.split(','):
-        args_error(parser, "[ERROR] specify either '-n' or '-p', not both")
+        # Program Number or name
-    if n:
+        p, n = options.program, options.program_name
-        if not n in TEST_MAP.keys():
+
-            # Check if there is an alias for this in private_settings.py
+        if n is not None and p is not None:
-            if getattr(ps, "test_alias", None) is not None:
+            args_error(parser, "[ERROR] specify either '-n' or '-p', not both")
-                alias = ps.test_alias.get(n, "")
+        if n:
-                if not alias in TEST_MAP.keys():
+            if not n in TEST_MAP.keys():
-                    args_error(parser, "[ERROR] Program with name '%s' not found" % n)
+                # Check if there is an alias for this in private_settings.py
+                if getattr(ps, "test_alias", None) is not None:
+                    alias = ps.test_alias.get(n, "")
+                    if not alias in TEST_MAP.keys():
+                        args_error(parser, "[ERROR] Program with name '%s' not found" % n)
+                    else:
+                        n = alias
                 else:
-                    n = alias
+                    args_error(parser, "[ERROR] Program with name '%s' not found" % n)
-            else:
+            p = TEST_MAP[n].n
-                args_error(parser, "[ERROR] Program with name '%s' not found" % n)
+        if p is None or (p < 0) or (p > (len(TESTS)-1)):
-        p = TEST_MAP[n].n
+            message = "[ERROR] You have to specify one of the following tests:\n"
-    if p is None or (p < 0) or (p > (len(TESTS)-1)):
+            message += '\n'.join(map(str, sorted(TEST_MAP.values())))
-        message = "[ERROR] You have to specify one of the following tests:\n"
+            args_error(parser, message)
-        message += '\n'.join(map(str, sorted(TEST_MAP.values())))
-        args_error(parser, message)
 
-    # Project
+        # Project
-    if p is None or (p < 0) or (p > (len(TESTS)-1)):
+        if p is None or (p < 0) or (p > (len(TESTS)-1)):
-        message = "[ERROR] You have to specify one of the following tests:\n"
+            message = "[ERROR] You have to specify one of the following tests:\n"
-        message += '\n'.join(map(str, sorted(TEST_MAP.values())))
+            message += '\n'.join(map(str, sorted(TEST_MAP.values())))
-        args_error(parser, message)
+            args_error(parser, message)
-    test = Test(p)
+        test = Test(p)
 
-    if not options.build:
+        # Some libraries have extra macros (called by exporter symbols) to we need to pass
-        # Substitute the library builds with the sources
+        # them to maintain compilation macros integrity between compiled library and
-        # TODO: Substitute also the other library build paths
+        # header files we might use with it
-        if MBED_LIBRARIES in test.dependencies:
+        lib_symbols = []
-            test.dependencies.remove(MBED_LIBRARIES)
+        for lib in LIBRARIES:
-            test.dependencies.append(MBED_BASE)
+            if lib['build_dir'] in test.dependencies:
+                lib_macros = lib.get('macros', None)
+                if lib_macros is not None:
+                    lib_symbols.extend(lib_macros)
 
-    # Build the project with the same directory structure of the mbed online IDE
+        if not options.build:
-    project_dir = join(EXPORT_WORKSPACE, test.id)
+            # Substitute the library builds with the sources
-    setup_user_prj(project_dir, test.source_dir, test.dependencies)
+            # TODO: Substitute also the other library build paths
+            if MBED_LIBRARIES in test.dependencies:
+                test.dependencies.remove(MBED_LIBRARIES)
+                test.dependencies.append(MBED_BASE)
 
-    # Export to selected toolchain
+        # Build the project with the same directory structure of the mbed online IDE
-    tmp_path, report = export(project_dir, test.id, ide, mcu, EXPORT_WORKSPACE, EXPORT_TMP)
+        project_dir = join(EXPORT_WORKSPACE, test.id)
-    if report['success']:
+        setup_user_prj(project_dir, test.source_dir, test.dependencies)
-        zip_path = join(EXPORT_DIR, "%s_%s_%s.zip" % (test.id, ide, mcu))
+
-        move(tmp_path, zip_path)
+        # Export to selected toolchain
-        print "[OK]"
+        tmp_path, report = export(project_dir, test.id, ide, mcu, EXPORT_WORKSPACE, EXPORT_TMP, extra_symbols=lib_symbols)
-    else:
+        if report['success']:
-        print '[ERRROR] %s' % report['errormsg']
+            zip_path = join(EXPORT_DIR, "%s_%s_%s.zip" % (test.id, ide, mcu))
+            move(tmp_path, zip_path)
+            successes.append("%s::%s\t%s"% (mcu, ide, zip_path))
+        else:
+            failures.append("%s::%s\t%s"% (mcu, ide, report['errormsg']))
+
+    # Prints export results
+    print
+    if len(successes) > 0:
+        print "Successful exports:"
+        for success in successes:
+            print "  * %s"% success
+    if len(failures) > 0:
+        print "Failed exports:"
+        for failure in failures:
+            print "  * %s"% failure

workspace_tools/tests.py

@@ -20,8 +20,6 @@ from workspace_tools.data.support import *
 TEST_CMSIS_LIB = join(TEST_DIR, "cmsis", "lib")
 TEST_MBED_LIB = join(TEST_DIR, "mbed", "env")
 
-TEST_MBED_TESTRUNNER_LIB = join(TEST_DIR, "utest", "testrunner")
-
 PERIPHERALS = join(TEST_DIR, "peripherals")
 BENCHMARKS_DIR = join(TEST_DIR, "benchmarks")
 
@@ -885,20 +883,20 @@ TESTS = [
     {
         "id": "UT_1", "description": "Basic",
         "source_dir": join(TEST_DIR, "utest", "basic"),
-        "dependencies": [MBED_LIBRARIES, TEST_MBED_LIB, CPPUTEST_LIBRARY, TEST_MBED_TESTRUNNER_LIB],
+        "dependencies": [MBED_LIBRARIES, TEST_MBED_LIB, CPPUTEST_LIBRARY],
         "automated": False,
     },
     {
         "id": "UT_2", "description": "Semihost file system",
         "source_dir": join(TEST_DIR, "utest", "semihost_fs"),
-        "dependencies": [MBED_LIBRARIES, TEST_MBED_LIB, CPPUTEST_LIBRARY, TEST_MBED_TESTRUNNER_LIB],
+        "dependencies": [MBED_LIBRARIES, TEST_MBED_LIB, CPPUTEST_LIBRARY],
         "automated": False,
         "mcu": ["LPC1768", "LPC2368", "LPC11U24"]
     },
     {
         "id": "UT_3", "description": "General tests",
         "source_dir": join(TEST_DIR, "utest", "general"),
-        "dependencies": [MBED_LIBRARIES, TEST_MBED_LIB, CPPUTEST_LIBRARY, TEST_MBED_TESTRUNNER_LIB],
+        "dependencies": [MBED_LIBRARIES, TEST_MBED_LIB, CPPUTEST_LIBRARY],
         "automated": False,
     },