From 773dab514ef1d7661e988e64b7fa37a6d08ac646 Mon Sep 17 00:00:00 2001 From: Mihail Stoyanov Date: Thu, 9 Jun 2016 23:11:23 +0100 Subject: [PATCH] Add --source option to both build.py and make.py. Also add test.py script for compiling and listing tests --- tools/build.py | 27 ++++++- tools/make.py | 26 +++++-- tools/test.py | 195 ++++++++++++++++++++++++++++++++++++++++++++++ tools/test_api.py | 162 +++++++++++++++++++++++++++++++++++++- 4 files changed, 398 insertions(+), 12 deletions(-) create mode 100644 tools/test.py diff --git a/tools/build.py b/tools/build.py index 8ae18c042d..4c8c0eff79 100644 --- a/tools/build.py +++ b/tools/build.py @@ -30,7 +30,7 @@ sys.path.insert(0, ROOT) from tools.toolchains import TOOLCHAINS from tools.targets import TARGET_NAMES, TARGET_MAP from tools.options import get_default_options_parser -from tools.build_api import build_mbed_libs, build_lib +from tools.build_api import build_library, build_mbed_libs, build_lib from tools.build_api import mcu_toolchain_matrix from tools.build_api import static_analysis_scan, static_analysis_scan_lib, static_analysis_scan_library from tools.build_api import print_build_results @@ -42,6 +42,15 @@ if __name__ == '__main__': # Parse Options parser = get_default_options_parser() + parser.add_option("--source", dest="source_dir", + default=None, help="The source (input) directory", action="append") + + parser.add_option("--build", dest="build_dir", + default=None, help="The build (output) directory") + + parser.add_option("--no-archive", dest="no_archive", action="store_true", + default=False, help="Do not produce archive (.ar) file, but rather .o") + # Extra libraries parser.add_option("-r", "--rtos", action="store_true", @@ -119,7 +128,7 @@ if __name__ == '__main__': help='For some commands you can use filter to filter out results') parser.add_option("-j", "--jobs", type="int", dest="jobs", - default=1, help="Number of concurrent jobs (default 1). Use 0 for auto based on host machine's number of CPUs") + default=0, help="Number of concurrent jobs. Default: 0/auto (based on host machine's number of CPUs)") parser.add_option("-v", "--verbose", action="store_true", @@ -224,7 +233,18 @@ if __name__ == '__main__': tt_id = "%s::%s" % (toolchain, target) try: mcu = TARGET_MAP[target] - lib_build_res = build_mbed_libs(mcu, toolchain, + if options.source_dir: + lib_build_res = build_library(options.source_dir, options.build_dir, mcu, toolchain, + options=options.options, + extra_verbose=options.extra_verbose_notify, + verbose=options.verbose, + silent=options.silent, + jobs=options.jobs, + clean=options.clean, + archive=(not options.no_archive), + macros=options.macros) + else: + lib_build_res = build_mbed_libs(mcu, toolchain, options=options.options, extra_verbose=options.extra_verbose_notify, verbose=options.verbose, @@ -232,6 +252,7 @@ if __name__ == '__main__': jobs=options.jobs, clean=options.clean, macros=options.macros) + for lib_id in libraries: build_lib(lib_id, mcu, toolchain, options=options.options, diff --git a/tools/make.py b/tools/make.py index db7f9ebe46..69bab0170f 100644 --- a/tools/make.py +++ b/tools/make.py @@ -21,7 +21,7 @@ TEST BUILD & RUN import sys from time import sleep from shutil import copy -from os.path import join, abspath, dirname +from os.path import join, abspath, dirname, isfile, isdir # Be sure that the tools directory is in the search path ROOT = abspath(join(dirname(__file__), "..")) @@ -42,11 +42,10 @@ from tools.targets import TARGET_MAP from tools.options import get_default_options_parser from tools.build_api import build_project try: - import mbed_settings as ps + import tools.private_settings as ps except: ps = object() - if __name__ == '__main__': # Parse Options parser = get_default_options_parser() @@ -62,8 +61,8 @@ if __name__ == '__main__': parser.add_option("-j", "--jobs", type="int", dest="jobs", - default=1, - help="Number of concurrent jobs (default 1). Use 0 for auto based on host machine's number of CPUs") + default=0, + help="Number of concurrent jobs. Default: 0/auto (based on host machine's number of CPUs)") parser.add_option("-v", "--verbose", action="store_true", @@ -94,11 +93,13 @@ if __name__ == '__main__': parser.add_option("--dep", dest="dependencies", default=None, help="Dependencies") parser.add_option("--source", dest="source_dir", - default=None, help="The source (input) directory") + default=None, help="The source (input) directory", action="append") parser.add_option("--duration", type="int", dest="duration", default=None, help="Duration of the test") parser.add_option("--build", dest="build_dir", default=None, help="The build (output) directory") + parser.add_option("-N", "--artifact-name", dest="artifact_name", + default=None, help="The built project's name") parser.add_option("-d", "--disk", dest="disk", default=None, help="The mbed disk") parser.add_option("-s", "--serial", dest="serial", @@ -165,6 +166,12 @@ if __name__ == '__main__': (options, args) = parser.parse_args() + if options.source_dir: + for path in options.source_dir : + if not isfile(path) and not isdir(path) : + args_error(parser, "[ERROR] you passed \"{}\" to --source, which does not exist". + format(path)) + # Print available tests in order and exit if options.list_tests is True: print '\n'.join(map(str, sorted(TEST_MAP.values()))) @@ -248,7 +255,8 @@ if __name__ == '__main__': verbose=options.verbose, silent=options.silent, macros=options.macros, - jobs=options.jobs) + jobs=options.jobs, + name=options.artifact_name) print 'Image: %s'% bin_file if options.disk: @@ -259,7 +267,7 @@ if __name__ == '__main__': # Import pyserial: https://pypi.python.org/pypi/pyserial from serial import Serial - sleep(target.program_cycle_s()) + sleep(TARGET_MAP[mcu].program_cycle_s()) serial = Serial(options.serial, timeout = 1) if options.baud: @@ -291,3 +299,5 @@ if __name__ == '__main__': traceback.print_exc(file=sys.stdout) else: print "[ERROR] %s" % str(e) + + sys.exit(1) diff --git a/tools/test.py b/tools/test.py new file mode 100644 index 0000000000..7fb0772e01 --- /dev/null +++ b/tools/test.py @@ -0,0 +1,195 @@ +#! /usr/bin/env python2 +""" +mbed SDK +Copyright (c) 2011-2013 ARM Limited + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +TEST BUILD & RUN +""" +import sys +import os +import json + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +sys.path.insert(0, ROOT) + +from tools.test_api import test_path_to_name, find_tests, print_tests, build_tests, test_spec_from_test_builds +from tools.options import get_default_options_parser +from tools.build_api import build_project, build_library +from tools.targets import TARGET_MAP +from tools.utils import mkdir +from tools.test_exporters import ReportExporter, ResultExporterType + +if __name__ == '__main__': + try: + # Parse Options + parser = get_default_options_parser() + + parser.add_option("-D", "", + action="append", + dest="macros", + help="Add a macro definition") + + parser.add_option("-j", "--jobs", + type="int", + dest="jobs", + default=0, + help="Number of concurrent jobs. Default: 0/auto (based on host machine's number of CPUs)") + + parser.add_option("--source", dest="source_dir", + default=None, help="The source (input) directory (for sources other than tests). Defaults to current directory.", action="append") + + parser.add_option("--build", dest="build_dir", + default=None, help="The build (output) directory") + + parser.add_option("-l", "--list", action="store_true", dest="list", + default=False, help="List (recursively) available tests in order and exit") + + parser.add_option("-p", "--paths", dest="paths", + default=None, help="Limit the tests to those within the specified comma separated list of paths") + + format_choices = ["list", "json"] + format_default_choice = "list" + format_help = "Change the format in which tests are listed. Choices include: %s. Default: %s" % (", ".join(format_choices), format_default_choice) + parser.add_option("-f", "--format", type="choice", dest="format", + choices=format_choices, default=format_default_choice, help=format_help) + + parser.add_option("-n", "--names", dest="names", + default=None, help="Limit the tests to a comma separated list of names") + + parser.add_option("--test-spec", dest="test_spec", + default=None, help="Destination path for a test spec file that can be used by the Greentea automated test tool") + + parser.add_option("--build-report-junit", dest="build_report_junit", + default=None, help="Destination path for a build report in the JUnit xml format") + + parser.add_option("-v", "--verbose", + action="store_true", + dest="verbose", + default=False, + help="Verbose diagnostic output") + + (options, args) = parser.parse_args() + + # Filter tests by path if specified + if options.paths: + all_paths = options.paths.split(",") + else: + all_paths = ["."] + + all_tests = {} + tests = {} + + # Find all tests in the relevant paths + for path in all_paths: + all_tests.update(find_tests(path)) + + # Filter tests by name if specified + if options.names: + all_names = options.names.split(",") + + all_tests_keys = all_tests.keys() + for name in all_names: + if name in all_tests_keys: + tests[name] = all_tests[name] + else: + print "[Warning] Test with name '%s' was not found in the available tests" % (name) + else: + tests = all_tests + + if options.list: + # Print available tests in order and exit + print_tests(tests, options.format) + sys.exit(0) + else: + # Build all tests + if not options.build_dir: + print "[ERROR] You must specify a build path" + sys.exit(1) + + base_source_paths = options.source_dir + + # Default base source path is the current directory + if not base_source_paths: + base_source_paths = ['.'] + + + target = TARGET_MAP[options.mcu] + + build_report = {} + build_properties = {} + + library_build_success = True + try: + # Build sources + build_library(base_source_paths, options.build_dir, target, options.tool, + options=options.options, + jobs=options.jobs, + clean=options.clean, + report=build_report, + properties=build_properties, + name="mbed-build", + macros=options.macros, + verbose=options.verbose, + archive=False) + except Exception, e: + library_build_success = False + print "Failed to build library" + + if library_build_success: + # Build all the tests + test_build_success, test_build = build_tests(tests, [options.build_dir], options.build_dir, target, options.tool, + options=options.options, + clean=options.clean, + report=build_report, + properties=build_properties, + macros=options.macros, + verbose=options.verbose, + jobs=options.jobs) + + # If a path to a test spec is provided, write it to a file + if options.test_spec: + test_spec_data = test_spec_from_test_builds(test_build) + + # Create the target dir for the test spec if necessary + # mkdir will not create the dir if it already exists + test_spec_dir = os.path.dirname(options.test_spec) + if test_spec_dir: + mkdir(test_spec_dir) + + try: + with open(options.test_spec, 'w') as f: + f.write(json.dumps(test_spec_data, indent=2)) + except IOError, e: + print "[ERROR] Error writing test spec to file" + print e + + # If a path to a JUnit build report spec is provided, write it to a file + if options.build_report_junit: + report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build") + report_exporter.report_to_file(build_report, options.build_report_junit, test_suite_properties=build_properties) + + if library_build_success and test_build_success: + sys.exit(0) + else: + sys.exit(1) + + except KeyboardInterrupt, e: + print "\n[CTRL+c] exit" + except Exception,e: + import traceback + traceback.print_exc(file=sys.stdout) + print "[ERROR] %s" % str(e) + sys.exit(1) diff --git a/tools/test_api.py b/tools/test_api.py index 87088c4c66..b7d88c8492 100644 --- a/tools/test_api.py +++ b/tools/test_api.py @@ -55,6 +55,7 @@ from tools.build_api import prep_report from tools.build_api import prep_properties from tools.build_api import create_result from tools.build_api import add_result_to_report +from tools.build_api import scan_for_source_paths from tools.libraries import LIBRARIES, LIBRARY_MAP from tools.toolchains import TOOLCHAIN_BIN_PATH from tools.test_exporters import ReportExporter, ResultExporterType @@ -1732,7 +1733,7 @@ def get_default_test_options_parser(): parser.add_option('-M', '--MUTS', dest='muts_spec_filename', metavar="FILE", - help='Points to file with MUTs specification (overwrites settings.py and mbed_settings.py)') + help='Points to file with MUTs specification (overwrites settings.py and private_settings.py)') parser.add_option("-j", "--jobs", dest='jobs', @@ -1949,3 +1950,162 @@ def get_default_test_options_parser(): action="store_true", help='Prints script version and exits') return parser + +def test_path_to_name(path): + """Change all slashes in a path into hyphens + This creates a unique cross-platform test name based on the path + This can eventually be overriden by a to-be-determined meta-data mechanism""" + name_parts = [] + head, tail = os.path.split(path) + while (tail and tail != "."): + name_parts.insert(0, tail) + head, tail = os.path.split(head) + + return "-".join(name_parts) + +def find_tests(base_dir): + """Given any directory, walk through the subdirectories and find all tests""" + + def is_subdir(path, directory): + path = os.path.realpath(path) + directory = os.path.realpath(directory) + relative = os.path.relpath(path, directory) + return not (relative.startswith(os.pardir + os.sep) and relative.startswith(os.pardir)) + + def find_tests_in_tests_directory(directory): + """Given a 'TESTS' directory, return a dictionary of test names and test paths. + The formate of the dictionary is {"test-name": "./path/to/test"}""" + tests = {} + + for d in os.listdir(directory): + # dir name host_tests is reserved for host python scripts. + if d != "host_tests": + # Loop on test case directories + for td in os.listdir(os.path.join(directory, d)): + # Add test case to the results if it is a directory and not "host_tests" + if td != "host_tests": + test_case_path = os.path.join(directory, d, td) + if os.path.isdir(test_case_path): + tests[test_path_to_name(test_case_path)] = test_case_path + + return tests + + tests_path = 'TESTS' + + # Determine if "base_dir" is already a "TESTS" directory + _, top_folder = os.path.split(base_dir) + + if top_folder == tests_path: + # Already pointing at a "TESTS" directory + return find_tests_in_tests_directory(base_dir) + else: + # Not pointing at a "TESTS" directory, so go find one! + tests = {} + + dirs = scan_for_source_paths(base_dir) + + test_and_sub_dirs = [x for x in dirs if tests_path in x] + test_dirs = [] + for potential_test_dir in test_and_sub_dirs: + good_to_add = True + if test_dirs: + for test_dir in test_dirs: + if is_subdir(potential_test_dir, test_dir): + good_to_add = False + break + + if good_to_add: + test_dirs.append(potential_test_dir) + + # Only look at valid paths + for path in test_dirs: + # Get the tests inside of the "TESTS" directory + new_tests = find_tests_in_tests_directory(path) + if new_tests: + tests.update(new_tests) + + return tests + +def print_tests(tests, format="list"): + """Given a dictionary of tests (as returned from "find_tests"), print them + in the specified format""" + if format == "list": + for test_name, test_path in tests.iteritems(): + print "Test Case:" + print " Name: %s" % test_name + print " Path: %s" % test_path + elif format == "json": + print json.dumps(tests, indent=2) + else: + print "Unknown format '%s'" % format + sys.exit(1) + +def build_tests(tests, base_source_paths, build_path, target, toolchain_name, + options=None, clean=False, notify=None, verbose=False, jobs=1, + macros=None, silent=False, report=None, properties=None): + """Given the data structure from 'find_tests' and the typical build parameters, + build all the tests + + Returns a tuple of the build result (True or False) followed by the test + build data structure""" + + test_build = { + "platform": target.name, + "toolchain": toolchain_name, + "base_path": build_path, + "baud_rate": 9600, + "binary_type": "bootable", + "tests": {} + } + + result = True + + for test_name, test_path in tests.iteritems(): + test_build_path = os.path.join(build_path, test_path) + src_path = base_source_paths + [test_path] + + try: + bin_file = build_project(src_path, test_build_path, target, toolchain_name, + options=options, + jobs=jobs, + clean=clean, + macros=macros, + name=test_name, + report=report, + properties=properties, + verbose=verbose) + + except Exception, e: + result = False + continue + + # If a clean build was carried out last time, disable it for the next build. + # Otherwise the previously built test will be deleted. + if clean: + clean = False + + # Normalize the path + bin_file = os.path.normpath(bin_file) + + test_build['tests'][test_name] = { + "binaries": [ + { + "path": bin_file + } + ] + } + + print 'Image: %s'% bin_file + + test_builds = {} + test_builds["%s-%s" % (target.name, toolchain_name)] = test_build + + + return result, test_builds + + +def test_spec_from_test_builds(test_builds): + return { + "builds": test_builds + } + \ No newline at end of file