ARMmbed
/
mbed-os
mirror of https://github.com/ARMmbed/mbed-os.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Updated script to add an extra feature and general improvements. 

Added the ability to specify a branch to update rather than a fork
Replaced print commands with the use of a logger
Updated the run_cmd functions in line with previous improvements
pull/4093/head
Anna Bridge 2017-03-16 16:59:11 +00:00
parent 9649d36dbb
commit 6850ff06b9
1 changed files with 199 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

265
tools/test/examples/update.py
View File

@ -1,8 +1,43 @@
#!/usr/bin/env python #!/usr/bin/env python
# This script is used to update the version of mbed-os used within a specified set of example
# applications. The list of examples to be updated lives in the examples.json file and is
# shared with the examples.py script. Logging is used to provide varying levels of output
# during execution.
#
# There are two modes that can be used:
# 1) Update the ARMmbed/master branch of the specified example
#
# This is done by updating a user fork of the example and then raising a pull request
# against ARMmbed/master.
#
# 2) Update a different ARMmbed branch of the specified example
#
# A branch to update is specified. If it doesn't already exist then it is first created.
# This branch will be updated and the change automatically pushed.
#
# Command usage:
#
# update.py -c <config file> - T <github_token> -l <logging level> -U <github user> -b <branch> <tag>
#
# Where:
# -c <config file> - Optional path to an examples file.
# If not proved the default is 'examples.json'
# -T <github_token> - GitHub token for secure access (required)
# -l <logging level> - Optional Level for providing logging output. Can be one of,
# CRITICAL, ERROR, WARNING, INFO, DEBUG
# If not provided the default is 'INFO'
# -U <github_user> - GitHub user for forked repos
# -b <branch> - Branch to be updated
#
# NOTE only one of -U or -b can be specified.
#
# <tag> mbed-os tag to which all examples will be updated
#
import os import os
from os.path import dirname, abspath, basename from os.path import dirname, abspath, basename
import sys import sys, logging
import argparse import argparse
import json import json
import subprocess import subprocess
@ -17,43 +52,56 @@ sys.path.insert(0, ROOT)
import examples_lib as lib import examples_lib as lib
from examples_lib import SUPPORTED_TOOLCHAINS from examples_lib import SUPPORTED_TOOLCHAINS
def run_cmd(command, print_warning_on_fail=True): def run_cmd(command, exit_on_failure=False):
""" Takes the command specified and runs it in a sub-process, obtaining the return code. """ Passes a command to the system and returns a True/False result once the
command has been executed, indicating success/failure. Commands are passed
as a list of tokens.
E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v']
Args: Args:
command - command to run, provided as a list of individual fields which are combined into a command - system command as a list of tokens
single command before passing to the sub-process call. exit_on_failure - If True exit the program on failure (default = False)
return_code - result of the command.
Returns:
result - True/False indicating the success/failure of the command
""" """
print('[Exec] %s' % ' '.join(command)) update_log.debug('[Exec] %s', ' '.join(command))
return_code = subprocess.call(command) return_code = subprocess.call(command, shell=True)
if return_code: if return_code:
print("The command '%s' failed with return code: %s" % (' '.join(command), return_code)) update_log.warning("Command '%s' failed with return code: %s",
print("Ignoring and moving on to the next example") ' '.join(command), return_code)
if exit_on_failure:
sys.exit(1)
return return_code return return_code
def run_cmd_with_output(command, print_warning_on_fail=True): def run_cmd_with_output(command, exit_on_failure=False):
""" Takes the command specified and runs it in a sub-process, obtaining the return code """ Passes a command to the system and returns a True/False result once the
and the returned output. command has been executed, indicating success/failure. If the command was
successful then the output from the command is returned to the caller.
Commands are passed as a list of tokens.
E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v']
Args: Args:
command - command to run, provided as a list of individual fields which are combined into a command - system command as a list of tokens
single command before passing to the sub-process call. exit_on_failure - If True exit the program on failure (default = False)
return_code - result of the command.
output - the output of the command
Returns:
result - True/False indicating the success/failure of the command
output - The output of the command if it was successful, else empty string
""" """
print('[Exec] %s' % ' '.join(command)) update_log.debug('[Exec] %s', ' '.join(command))
returncode = 0 returncode = 0
output = None output = ""
try: try:
output = subprocess.check_output(command) output = subprocess.check_output(command, shell=True)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
print("The command '%s' failed with return code: %s" % (' '.join(command), e.returncode)) update_log.warning("Command '%s' failed with return code: %s",
' '.join(command), e.returncode)
returncode = e.returncode returncode = e.returncode
if exit_on_failure:
sys.exit(1)
return returncode, output return returncode, output
def rmtree_readonly(directory): def rmtree_readonly(directory):
@ -111,7 +159,7 @@ def upgrade_single_example(example, tag, directory, ref):
os.rename("mbed-os.lib", "mbed-os.lib_bak") os.rename("mbed-os.lib", "mbed-os.lib_bak")
else: else:
print("!! Error trying to backup mbed-os.lib prior to updating.") update_log.error("Failed to backup mbed-os.lib prior to updating.")
return False return False
# mbed-os.lib file contains one line with the following format # mbed-os.lib file contains one line with the following format
@ -149,52 +197,100 @@ def prepare_fork(arm_example):
""" """
print "In " + os.getcwd() update_log.debug("In: ", os.getcwd())
for cmd in [['git', 'remote', 'add', 'armmbed', arm_example], for cmd in [['git', 'remote', 'add', 'armmbed', arm_example],
['git', 'fetch', 'armmbed'], ['git', 'fetch', 'armmbed'],
['git', 'reset', '--hard', 'armmbed/master'], ['git', 'reset', '--hard', 'armmbed/master'],
['git', 'push', '-f', 'origin']]: ['git', 'push', '-f', 'origin']]:
if run_cmd(cmd): if run_cmd(cmd):
print("preparation of the fork failed!") update_log.error("Fork preparation failed")
return False return False
return True return True
def prepare_branch(branch):
def upgrade_example(github, example, tag, user, ref): update_log.debug("Preparing branch: %s", branch)
""" Clone a fork of the example specified.
Ensures the fork is up to date with the original and then and updates the associated # Check if branch already exists or not.
mbed-os.lib file on that fork to correspond to the version specified by the GitHub tag supplied. cmd = ['git', 'branch']
Also deals with multiple sub-examples in the GitHub repo, updating them in the same way. return_code, output = run_cmd_with_output(cmd)
The updates are pushed to the forked repo.
Finally a PR is raised against the original example repo for the changes. if not branch in output:
# OOB branch does not exist thus create it and then check it out
cmd = ['git', 'checkout', '-b', branch]
return_code = run_cmd(cmd)
if not return_code:
# Push new branch upstream
cmd = ['git', 'push', '-u', 'origin', branch]
return_code = run_cmd(cmd)
else:
cmd = ['git', 'checkout', branch]
return_code = run_cmd(cmd)
if return_code:
update_log.error("Failed to prepare branch: %s", branch)
return False
return True
def upgrade_example(github, example, tag, ref,
user='ARMmbed', branch='master'):
""" Clone a version of the example specified and upgrade all versions of
mbed-os.lib found within its tree. The version cloned and how it
is upgraded depends on the user and branch specified. Only two options
are valid:
1) ARMmbed + non master branch
This option will update the branch directly in the ARMmbed repo. If the
branch does not exist it will be first created.
2) alternative user + master branch
This option assumes that a fork of the repo exists in the specified user's
account. The fork will first be updated so that it is up to date with the
upstream version , then the fork will be updated and a PR raised against
the upstream ie ARMmbed repo.
Args: Args:
github - GitHub instance to allow internal git commands to be run github - GitHub instance to allow internal git commands to be run
example - json example object containing the GitHub repo to update. example - json example object containing the GitHub repo to update.
tag - GitHub tag corresponding to a version of mbed-os to upgrade to. tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
user - GitHub user name
ref - SHA corresponding to the tag ref - SHA corresponding to the tag
user - GitHub user name
branch - branch to update
""" """
ret = False ret = False
print("\nUpdating example '%s'" % example['name']) update_log.info("Updating example '%s'", example['name'])
update_log.debug("User: %s", user)
update_log.debug("Branch: %s", branch)
# First check validity of user/branch combination
if ((user == 'ARMmbed' and branch == 'master') or
(user != 'ARMmbed' and branch != 'master')):
update_log.error("Invalid user/branch combination")
return False
cwd = os.getcwd() cwd = os.getcwd()
full_repo_name = 'ARMmbed/'+ example['name'] upstream_repo = 'ARMmbed/'+ example['name']
fork = "https://github.com/" + user + '/' + example['name'] update_repo = "https://github.com/" + user + '/' + example['name']
update_log.debug("Upstream repository: %s", upstream_repo)
update_log.debug("Update repository: %s", update_repo)
# Check access to mbed-os repo # Check access to mbed-os repo
try: try:
repo = github.get_repo(full_repo_name, False) repo = github.get_repo(upstream_repo, False)
except: except:
print("\t\t!! Repo does not exist - skipping\n") update_log.error("Upstream repo: %s, does not exist - skipping", upstream_repo)
return False return False
# Clone the forked example repo # Clone the example repo
clone_cmd = ['git', 'clone', fork] clone_cmd = ['git', 'clone', update_repo]
return_code = run_cmd(clone_cmd) return_code = run_cmd(clone_cmd)
if not return_code: if not return_code:
@ -204,9 +300,14 @@ def upgrade_example(github, example, tag, user, ref):
os.chdir(example['name']) os.chdir(example['name'])
# checkout and synchronise the release-candidate branch # If the user is not the default, then a fork will be used. Thus
# synchronise the user fork with the upstream
if user != 'ARMmbed':
prepare_fork(example['github']) prepare_fork(example['github'])
if branch != 'master':
prepare_branch(branch)
for example_directory in example_directories: for example_directory in example_directories:
if not upgrade_single_example(example, tag, os.path.relpath(example_directory, example['name']), ref): if not upgrade_single_example(example, tag, os.path.relpath(example_directory, example['name']), ref):
os.chdir(cwd) os.chdir(cwd)
@ -225,6 +326,7 @@ def upgrade_example(github, example, tag, user, ref):
return_code = run_cmd(push_cmd) return_code = run_cmd(push_cmd)
if not return_code: if not return_code:
if user != 'ARMmbed':
body = "Please test/merge this PR and then tag Master with " + tag body = "Please test/merge this PR and then tag Master with " + tag
# Raise a PR from release-candidate to master # Raise a PR from release-candidate to master
user_fork = user + ':master' user_fork = user + ':master'
@ -233,15 +335,15 @@ def upgrade_example(github, example, tag, user, ref):
ret = True ret = True
except GithubException as e: except GithubException as e:
# Default to False # Default to False
print("Creation of Pull Request from release-candidate to master failed with the following error!") update_log.error("Pull request creation failed with error: %s", e)
print e
else: else:
print("!!! Git push command failed.") ret = True
else: else:
print("!!! Git commit command failed.") update_log.error("Git push command failed.")
else: else:
print("!!! Could not clone user fork %s\n" % fork) update_log.error("Git commit command failed.")
else:
update_log.error("Git clone %s failed", update_repo)
os.chdir(cwd) os.chdir(cwd)
return ret return ret
@ -255,36 +357,11 @@ def create_work_directory(path):
""" """
if os.path.exists(path): if os.path.exists(path):
print("'%s' directory already exists. Deleting..." % path) update_log.info("'%s' directory already exists. Deleting...", path)
rmtree_readonly(path) rmtree_readonly(path)
os.makedirs(path) os.makedirs(path)
def test_compile(config, tag):
""" For each example repo identified in the config json object, clone, update mbed-os to
the specified tag and then compile for all supported toolchains.
Args:
config - the json object imported from the file.
tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
results - summary of compilation results.
"""
# Create work directories
create_work_directory('test_compile')
# Loop through the examples
results = {}
os.chdir('test_compile')
lib.source_repos(config)
lib.update_mbedos_version(config, tag)
results = lib.compile_repos(config, SUPPORTED_TOOLCHAINS)
os.chdir("..")
return results
def main(arguments): def main(arguments):
""" Will update any mbed-os.lib files found in the example list specified by the config file. """ Will update any mbed-os.lib files found in the example list specified by the config file.
If no config file is specified the default 'examples.json' is used. If no config file is specified the default 'examples.json' is used.
@ -306,19 +383,30 @@ def main(arguments):
parser.add_argument('tag', help="mbed-os tag to which all examples will be updated") parser.add_argument('tag', help="mbed-os tag to which all examples will be updated")
parser.add_argument('-c', '--config_file', help="Path to the configuration file (default is 'examples.json')", default='examples.json') parser.add_argument('-c', '--config_file', help="Path to the configuration file (default is 'examples.json')", default='examples.json')
parser.add_argument('-T', '--github_token', help="GitHub token for secure access") parser.add_argument('-T', '--github_token', help="GitHub token for secure access")
parser.add_argument('-U', '--github_user', help="GitHub user for forked repos") parser.add_argument('-l', '--log-level',
help="Level for providing logging output",
default='INFO')
exclusive = parser.add_mutually_exclusive_group(required=True)
exclusive.add_argument('-U', '--github_user', help="GitHub user for forked repos, mutually exclusive to branch option")
exclusive.add_argument('-b', '--branch', help="Branch to be updated, mutually exclusive to user option")
args = parser.parse_args(arguments) args = parser.parse_args(arguments)
cfg = os.path.join(os.path.dirname(__file__), args.config_file) default = getattr(logging, 'INFO')
level = getattr(logging, args.log_level.upper(), default)
# Set logging level
logging.basicConfig(level=level)
update_log = logging.getLogger("Update")
# Load the config file # Load the config file
config = json.load(open(os.path.join(os.path.dirname(__file__), with open(os.path.join(os.path.dirname(__file__), args.config_file)) as config:
args.config_file)))
if not config: if not config:
print("Failed to load config file '%s'" % args.config_file) update_log.error("Failed to load config file '%s'", args.config_file)
sys.exit(1) sys.exit(1)
json_data = json.load(config)
# Create working directory # Create working directory
create_work_directory('examples') create_work_directory('examples')
@ -330,7 +418,7 @@ def main(arguments):
return_code, ref = run_cmd_with_output(cmd) return_code, ref = run_cmd_with_output(cmd)
if return_code: if return_code:
print("Could not obtain SHA for tag: %s\n" % args.tag) update_log.error("Could not obtain SHA for tag: %s", args.tag)
sys.exit(1) sys.exit(1)
# Loop through the examples # Loop through the examples
@ -343,7 +431,13 @@ def main(arguments):
# Determine if this example should be updated and if so update any found # Determine if this example should be updated and if so update any found
# mbed-os.lib files. # mbed-os.lib files.
if upgrade_example(github, example, args.tag, args.github_user, ref): # Only user or branch can be specified on the command line
if args.github_user:
result = upgrade_example(github, example, args.tag, ref, user=args.github_user)
else:
result = upgrade_example(github, example, args.tag, ref, branch=args.branch)
if result:
successes += [example['name']] successes += [example['name']]
else: else:
failures += [example['name']] failures += [example['name']]
@ -351,17 +445,14 @@ def main(arguments):
os.chdir('../') os.chdir('../')
# Finish the script and report the results # Finish the script and report the results
print(os.linesep + os.linesep +'Finished updating examples!' + os.linesep) update_log.info("Finished updating examples")
if successes: if successes:
print('\nThe following examples updated successfully:')
for success in successes: for success in successes:
print(' - %s' % success) update_log.info(" SUCCEEDED: %s", success)
if failures: if failures:
print('\nThe following examples were not updated:')
for fail in failures: for fail in failures:
print(' - %s' % fail) update_log.info(" FAILED: %s", fail)
if __name__ == '__main__': if __name__ == '__main__':
sys.exit(main(sys.argv[1:])) sys.exit(main(sys.argv[1:]))