ARMmbed
/
mbed-os
mirror of https://github.com/ARMmbed/mbed-os.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add --source option to both build.py and make.py. Also add test.py script for compiling and listing tests

pull/1893/head
Mihail Stoyanov 2016-06-09 23:11:23 +01:00
parent 87a978c4e8
commit 773dab514e
4 changed files with 398 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
tools/build.py
View File

@ -30,7 +30,7 @@ sys.path.insert(0, ROOT)
from tools.toolchains import TOOLCHAINS
from tools.targets import TARGET_NAMES, TARGET_MAP
from tools.options import get_default_options_parser
from tools.build_api import build_mbed_libs, build_lib
from tools.build_api import build_library, build_mbed_libs, build_lib
from tools.build_api import mcu_toolchain_matrix
from tools.build_api import static_analysis_scan, static_analysis_scan_lib, static_analysis_scan_library
from tools.build_api import print_build_results
@ -42,6 +42,15 @@ if __name__ == '__main__':
# Parse Options
parser = get_default_options_parser()
parser.add_option("--source", dest="source_dir",
default=None, help="The source (input) directory", action="append")
parser.add_option("--build", dest="build_dir",
default=None, help="The build (output) directory")
parser.add_option("--no-archive", dest="no_archive", action="store_true",
default=False, help="Do not produce archive (.ar) file, but rather .o")
# Extra libraries
parser.add_option("-r", "--rtos",
action="store_true",
@ -119,7 +128,7 @@ if __name__ == '__main__':
help='For some commands you can use filter to filter out results')
parser.add_option("-j", "--jobs", type="int", dest="jobs",
default=1, help="Number of concurrent jobs (default 1). Use 0 for auto based on host machine's number of CPUs")
default=0, help="Number of concurrent jobs. Default: 0/auto (based on host machine's number of CPUs)")
parser.add_option("-v", "--verbose",
action="store_true",
@ -224,7 +233,18 @@ if __name__ == '__main__':
tt_id = "%s::%s" % (toolchain, target)
try:
mcu = TARGET_MAP[target]
lib_build_res = build_mbed_libs(mcu, toolchain,
if options.source_dir:
lib_build_res = build_library(options.source_dir, options.build_dir, mcu, toolchain,
options=options.options,
extra_verbose=options.extra_verbose_notify,
verbose=options.verbose,
silent=options.silent,
jobs=options.jobs,
clean=options.clean,
archive=(not options.no_archive),
macros=options.macros)
else:
lib_build_res = build_mbed_libs(mcu, toolchain,
options=options.options,
extra_verbose=options.extra_verbose_notify,
verbose=options.verbose,
@ -232,6 +252,7 @@ if __name__ == '__main__':
jobs=options.jobs,
clean=options.clean,
macros=options.macros)
for lib_id in libraries:
build_lib(lib_id, mcu, toolchain,
options=options.options,

26
tools/make.py
View File

@ -21,7 +21,7 @@ TEST BUILD & RUN
import sys
from time import sleep
from shutil import copy
from os.path import join, abspath, dirname
from os.path import join, abspath, dirname, isfile, isdir
# Be sure that the tools directory is in the search path
ROOT = abspath(join(dirname(__file__), ".."))
@ -42,11 +42,10 @@ from tools.targets import TARGET_MAP
from tools.options import get_default_options_parser
from tools.build_api import build_project
try:
import mbed_settings as ps
import tools.private_settings as ps
except:
ps = object()
if __name__ == '__main__':
# Parse Options
parser = get_default_options_parser()
@ -62,8 +61,8 @@ if __name__ == '__main__':
parser.add_option("-j", "--jobs",
type="int",
dest="jobs",
default=1,
help="Number of concurrent jobs (default 1). Use 0 for auto based on host machine's number of CPUs")
default=0,
help="Number of concurrent jobs. Default: 0/auto (based on host machine's number of CPUs)")
parser.add_option("-v", "--verbose",
action="store_true",
@ -94,11 +93,13 @@ if __name__ == '__main__':
parser.add_option("--dep", dest="dependencies",
default=None, help="Dependencies")
parser.add_option("--source", dest="source_dir",
default=None, help="The source (input) directory")
default=None, help="The source (input) directory", action="append")
parser.add_option("--duration", type="int", dest="duration",
default=None, help="Duration of the test")
parser.add_option("--build", dest="build_dir",
default=None, help="The build (output) directory")
parser.add_option("-N", "--artifact-name", dest="artifact_name",
default=None, help="The built project's name")
parser.add_option("-d", "--disk", dest="disk",
default=None, help="The mbed disk")
parser.add_option("-s", "--serial", dest="serial",
@ -165,6 +166,12 @@ if __name__ == '__main__':
(options, args) = parser.parse_args()
if options.source_dir:
for path in options.source_dir :
if not isfile(path) and not isdir(path) :
args_error(parser, "[ERROR] you passed \"{}\" to --source, which does not exist".
format(path))
# Print available tests in order and exit
if options.list_tests is True:
print '\n'.join(map(str, sorted(TEST_MAP.values())))
@ -248,7 +255,8 @@ if __name__ == '__main__':
verbose=options.verbose,
silent=options.silent,
macros=options.macros,
jobs=options.jobs)
jobs=options.jobs,
name=options.artifact_name)
print 'Image: %s'% bin_file
if options.disk:
@ -259,7 +267,7 @@ if __name__ == '__main__':
# Import pyserial: https://pypi.python.org/pypi/pyserial
from serial import Serial
sleep(target.program_cycle_s())
sleep(TARGET_MAP[mcu].program_cycle_s())
serial = Serial(options.serial, timeout = 1)
if options.baud:
@ -291,3 +299,5 @@ if __name__ == '__main__':
traceback.print_exc(file=sys.stdout)
else:
print "[ERROR] %s" % str(e)
sys.exit(1)

195
tools/test.py Normal file
View File

@ -0,0 +1,195 @@
#! /usr/bin/env python2
"""
mbed SDK
Copyright (c) 2011-2013 ARM Limited
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
TEST BUILD & RUN
"""
import sys
import os
import json
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
sys.path.insert(0, ROOT)
from tools.test_api import test_path_to_name, find_tests, print_tests, build_tests, test_spec_from_test_builds
from tools.options import get_default_options_parser
from tools.build_api import build_project, build_library
from tools.targets import TARGET_MAP
from tools.utils import mkdir
from tools.test_exporters import ReportExporter, ResultExporterType
if __name__ == '__main__':
try:
# Parse Options
parser = get_default_options_parser()
parser.add_option("-D", "",
action="append",
dest="macros",
help="Add a macro definition")
parser.add_option("-j", "--jobs",
type="int",
dest="jobs",
default=0,
help="Number of concurrent jobs. Default: 0/auto (based on host machine's number of CPUs)")
parser.add_option("--source", dest="source_dir",
default=None, help="The source (input) directory (for sources other than tests). Defaults to current directory.", action="append")
parser.add_option("--build", dest="build_dir",
default=None, help="The build (output) directory")
parser.add_option("-l", "--list", action="store_true", dest="list",
default=False, help="List (recursively) available tests in order and exit")
parser.add_option("-p", "--paths", dest="paths",
default=None, help="Limit the tests to those within the specified comma separated list of paths")
format_choices = ["list", "json"]
format_default_choice = "list"
format_help = "Change the format in which tests are listed. Choices include: %s. Default: %s" % (", ".join(format_choices), format_default_choice)
parser.add_option("-f", "--format", type="choice", dest="format",
choices=format_choices, default=format_default_choice, help=format_help)
parser.add_option("-n", "--names", dest="names",
default=None, help="Limit the tests to a comma separated list of names")
parser.add_option("--test-spec", dest="test_spec",
default=None, help="Destination path for a test spec file that can be used by the Greentea automated test tool")
parser.add_option("--build-report-junit", dest="build_report_junit",
default=None, help="Destination path for a build report in the JUnit xml format")
parser.add_option("-v", "--verbose",
action="store_true",
dest="verbose",
default=False,
help="Verbose diagnostic output")
(options, args) = parser.parse_args()
# Filter tests by path if specified
if options.paths:
all_paths = options.paths.split(",")
else:
all_paths = ["."]
all_tests = {}
tests = {}
# Find all tests in the relevant paths
for path in all_paths:
all_tests.update(find_tests(path))
# Filter tests by name if specified
if options.names:
all_names = options.names.split(",")
all_tests_keys = all_tests.keys()
for name in all_names:
if name in all_tests_keys:
tests[name] = all_tests[name]
else:
print "[Warning] Test with name '%s' was not found in the available tests" % (name)
else:
tests = all_tests
if options.list:
# Print available tests in order and exit
print_tests(tests, options.format)
sys.exit(0)
else:
# Build all tests
if not options.build_dir:
print "[ERROR] You must specify a build path"
sys.exit(1)
base_source_paths = options.source_dir
# Default base source path is the current directory
if not base_source_paths:
base_source_paths = ['.']
target = TARGET_MAP[options.mcu]
build_report = {}
build_properties = {}
library_build_success = True
try:
# Build sources
build_library(base_source_paths, options.build_dir, target, options.tool,
options=options.options,
jobs=options.jobs,
clean=options.clean,
report=build_report,
properties=build_properties,
name="mbed-build",
macros=options.macros,
verbose=options.verbose,
archive=False)
except Exception, e:
library_build_success = False
print "Failed to build library"
if library_build_success:
# Build all the tests
test_build_success, test_build = build_tests(tests, [options.build_dir], options.build_dir, target, options.tool,
options=options.options,
clean=options.clean,
report=build_report,
properties=build_properties,
macros=options.macros,
verbose=options.verbose,
jobs=options.jobs)
# If a path to a test spec is provided, write it to a file
if options.test_spec:
test_spec_data = test_spec_from_test_builds(test_build)
# Create the target dir for the test spec if necessary
# mkdir will not create the dir if it already exists
test_spec_dir = os.path.dirname(options.test_spec)
if test_spec_dir:
mkdir(test_spec_dir)
try:
with open(options.test_spec, 'w') as f:
f.write(json.dumps(test_spec_data, indent=2))
except IOError, e:
print "[ERROR] Error writing test spec to file"
print e
# If a path to a JUnit build report spec is provided, write it to a file
if options.build_report_junit:
report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")
report_exporter.report_to_file(build_report, options.build_report_junit, test_suite_properties=build_properties)
if library_build_success and test_build_success:
sys.exit(0)
else:
sys.exit(1)
except KeyboardInterrupt, e:
print "\n[CTRL+c] exit"
except Exception,e:
import traceback
traceback.print_exc(file=sys.stdout)
print "[ERROR] %s" % str(e)
sys.exit(1)

162
tools/test_api.py
View File

@ -55,6 +55,7 @@ from tools.build_api import prep_report
from tools.build_api import prep_properties
from tools.build_api import create_result
from tools.build_api import add_result_to_report
from tools.build_api import scan_for_source_paths
from tools.libraries import LIBRARIES, LIBRARY_MAP
from tools.toolchains import TOOLCHAIN_BIN_PATH
from tools.test_exporters import ReportExporter, ResultExporterType
@ -1732,7 +1733,7 @@ def get_default_test_options_parser():
parser.add_option('-M', '--MUTS',
dest='muts_spec_filename',
metavar="FILE",
help='Points to file with MUTs specification (overwrites settings.py and mbed_settings.py)')
help='Points to file with MUTs specification (overwrites settings.py and private_settings.py)')
parser.add_option("-j", "--jobs",
dest='jobs',
@ -1949,3 +1950,162 @@ def get_default_test_options_parser():
action="store_true",
help='Prints script version and exits')
return parser
def test_path_to_name(path):
"""Change all slashes in a path into hyphens
This creates a unique cross-platform test name based on the path
This can eventually be overriden by a to-be-determined meta-data mechanism"""
name_parts = []
head, tail = os.path.split(path)
while (tail and tail != "."):
name_parts.insert(0, tail)
head, tail = os.path.split(head)
return "-".join(name_parts)
def find_tests(base_dir):
"""Given any directory, walk through the subdirectories and find all tests"""
def is_subdir(path, directory):
path = os.path.realpath(path)
directory = os.path.realpath(directory)
relative = os.path.relpath(path, directory)
return not (relative.startswith(os.pardir + os.sep) and relative.startswith(os.pardir))
def find_tests_in_tests_directory(directory):
"""Given a 'TESTS' directory, return a dictionary of test names and test paths.
The formate of the dictionary is {"test-name": "./path/to/test"}"""
tests = {}
for d in os.listdir(directory):
# dir name host_tests is reserved for host python scripts.
if d != "host_tests":
# Loop on test case directories
for td in os.listdir(os.path.join(directory, d)):
# Add test case to the results if it is a directory and not "host_tests"
if td != "host_tests":
test_case_path = os.path.join(directory, d, td)
if os.path.isdir(test_case_path):
tests[test_path_to_name(test_case_path)] = test_case_path
return tests
tests_path = 'TESTS'
# Determine if "base_dir" is already a "TESTS" directory
_, top_folder = os.path.split(base_dir)
if top_folder == tests_path:
# Already pointing at a "TESTS" directory
return find_tests_in_tests_directory(base_dir)
else:
# Not pointing at a "TESTS" directory, so go find one!
tests = {}
dirs = scan_for_source_paths(base_dir)
test_and_sub_dirs = [x for x in dirs if tests_path in x]
test_dirs = []
for potential_test_dir in test_and_sub_dirs:
good_to_add = True
if test_dirs:
for test_dir in test_dirs:
if is_subdir(potential_test_dir, test_dir):
good_to_add = False
break
if good_to_add:
test_dirs.append(potential_test_dir)
# Only look at valid paths
for path in test_dirs:
# Get the tests inside of the "TESTS" directory
new_tests = find_tests_in_tests_directory(path)
if new_tests:
tests.update(new_tests)
return tests
def print_tests(tests, format="list"):
"""Given a dictionary of tests (as returned from "find_tests"), print them
in the specified format"""
if format == "list":
for test_name, test_path in tests.iteritems():
print "Test Case:"
print " Name: %s" % test_name
print " Path: %s" % test_path
elif format == "json":
print json.dumps(tests, indent=2)
else:
print "Unknown format '%s'" % format
sys.exit(1)
def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
options=None, clean=False, notify=None, verbose=False, jobs=1,
macros=None, silent=False, report=None, properties=None):
"""Given the data structure from 'find_tests' and the typical build parameters,
build all the tests
Returns a tuple of the build result (True or False) followed by the test
build data structure"""
test_build = {
"platform": target.name,
"toolchain": toolchain_name,
"base_path": build_path,
"baud_rate": 9600,
"binary_type": "bootable",
"tests": {}
}
result = True
for test_name, test_path in tests.iteritems():
test_build_path = os.path.join(build_path, test_path)
src_path = base_source_paths + [test_path]
try:
bin_file = build_project(src_path, test_build_path, target, toolchain_name,
options=options,
jobs=jobs,
clean=clean,
macros=macros,
name=test_name,
report=report,
properties=properties,
verbose=verbose)
except Exception, e:
result = False
continue
# If a clean build was carried out last time, disable it for the next build.
# Otherwise the previously built test will be deleted.
if clean:
clean = False
# Normalize the path
bin_file = os.path.normpath(bin_file)
test_build['tests'][test_name] = {
"binaries": [
{
"path": bin_file
}
]
}
print 'Image: %s'% bin_file
test_builds = {}
test_builds["%s-%s" % (target.name, toolchain_name)] = test_build
return result, test_builds
def test_spec_from_test_builds(test_builds):
return {
"builds": test_builds
}