mirror of https://github.com/ARMmbed/mbed-os.git
Merge pull request #4093 from adbridge/update
Update.py: New feature - update a branch instead of a fork, plus general improvements.pull/4225/head
Jimmy Brisson
GitHub
commit
0b5b6cba71
|
|
@ -1,8 +1,44 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# This script is used to update the version of mbed-os used within a specified set of example
|
||||
# applications. The list of examples to be updated lives in the examples.json file and is
|
||||
# shared with the examples.py script. Logging is used to provide varying levels of output
|
||||
# during execution.
|
||||
#
|
||||
# There are two modes that can be used:
|
||||
# 1) Update the ARMmbed/master branch of the specified example
|
||||
#
|
||||
# This is done by updating a user fork of the example and then raising a pull request
|
||||
# against ARMmbed/master.
|
||||
#
|
||||
# 2) Update a different ARMmbed branch of the specified example
|
||||
#
|
||||
# A branch to update is specified. If it doesn't already exist then it is first created.
|
||||
# This branch will be updated and the change automatically pushed.
|
||||
#
|
||||
# Command usage:
|
||||
#
|
||||
# update.py -c <config file> - T <github_token> -l <logging level> -U <github user> -b <branch> <tag>
|
||||
#
|
||||
# Where:
|
||||
# -c <config file> - Optional path to an examples file.
|
||||
# If not proved the default is 'examples.json'
|
||||
# -T <github_token> - GitHub token for secure access (required)
|
||||
# -l <logging level> - Optional Level for providing logging output. Can be one of,
|
||||
# CRITICAL, ERROR, WARNING, INFO, DEBUG
|
||||
# If not provided the default is 'INFO'
|
||||
# -U <github_user> - GitHub user for forked repos
|
||||
# -b <branch> - Branch to be updated
|
||||
#
|
||||
# NOTE only one of -U or -b can be specified.
|
||||
#
|
||||
# <tag> mbed-os tag to which all examples will be updated
|
||||
#
|
||||
|
||||
import os
|
||||
from os.path import dirname, abspath, basename
|
||||
import sys
|
||||
import logging
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
|
|
@ -17,43 +53,64 @@ sys.path.insert(0, ROOT)
|
|||
import examples_lib as lib
|
||||
from examples_lib import SUPPORTED_TOOLCHAINS
|
||||
|
||||
def run_cmd(command, print_warning_on_fail=True):
|
||||
""" Takes the command specified and runs it in a sub-process, obtaining the return code.
|
||||
|
||||
Args:
|
||||
command - command to run, provided as a list of individual fields which are combined into a
|
||||
single command before passing to the sub-process call.
|
||||
return_code - result of the command.
|
||||
def run_cmd(command, exit_on_failure=False):
|
||||
""" Run a system command and return the result status
|
||||
|
||||
Description:
|
||||
|
||||
Passes a command to the system and returns a True/False result, once the
|
||||
command has been executed, indicating success/failure. Commands are passed
|
||||
as a list of tokens.
|
||||
E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v']
|
||||
|
||||
Args:
|
||||
command - system command as a list of tokens
|
||||
exit_on_failure - If True exit the program on failure (default = False)
|
||||
|
||||
Returns:
|
||||
return_code - True/False indicating the success/failure of the command
|
||||
"""
|
||||
print('[Exec] %s' % ' '.join(command))
|
||||
return_code = subprocess.call(command)
|
||||
update_log.debug('[Exec] %s', ' '.join(command))
|
||||
return_code = subprocess.call(command, shell=True)
|
||||
|
||||
if return_code:
|
||||
print("The command '%s' failed with return code: %s" % (' '.join(command), return_code))
|
||||
print("Ignoring and moving on to the next example")
|
||||
update_log.warning("Command '%s' failed with return code: %s",
|
||||
' '.join(command), return_code)
|
||||
if exit_on_failure:
|
||||
sys.exit(1)
|
||||
|
||||
return return_code
|
||||
|
||||
def run_cmd_with_output(command, print_warning_on_fail=True):
|
||||
""" Takes the command specified and runs it in a sub-process, obtaining the return code
|
||||
and the returned output.
|
||||
|
||||
Args:
|
||||
command - command to run, provided as a list of individual fields which are combined into a
|
||||
single command before passing to the sub-process call.
|
||||
return_code - result of the command.
|
||||
output - the output of the command
|
||||
|
||||
def run_cmd_with_output(command, exit_on_failure=False):
|
||||
""" Run a system command and return the result status plus output
|
||||
|
||||
Description:
|
||||
|
||||
Passes a command to the system and returns a True/False result once the
|
||||
command has been executed, indicating success/failure. If the command was
|
||||
successful then the output from the command is returned to the caller.
|
||||
Commands are passed as a list of tokens.
|
||||
E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v']
|
||||
|
||||
Args:
|
||||
command - system command as a list of tokens
|
||||
exit_on_failure - If True exit the program on failure (default = False)
|
||||
|
||||
Returns:
|
||||
returncode - True/False indicating the success/failure of the command
|
||||
output - The output of the command if it was successful, else empty string
|
||||
"""
|
||||
print('[Exec] %s' % ' '.join(command))
|
||||
update_log.debug('[Exec] %s', ' '.join(command))
|
||||
returncode = 0
|
||||
output = None
|
||||
output = ""
|
||||
try:
|
||||
output = subprocess.check_output(command)
|
||||
output = subprocess.check_output(command, shell=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print("The command '%s' failed with return code: %s" % (' '.join(command), e.returncode))
|
||||
update_log.warning("Command '%s' failed with return code: %s",
|
||||
' '.join(command), e.returncode)
|
||||
returncode = e.returncode
|
||||
if exit_on_failure:
|
||||
sys.exit(1)
|
||||
return returncode, output
|
||||
|
||||
def rmtree_readonly(directory):
|
||||
|
|
@ -69,9 +126,13 @@ def rmtree_readonly(directory):
|
|||
shutil.rmtree(directory, onerror=remove_readonly)
|
||||
|
||||
def find_all_examples(path):
|
||||
""" Searches the path specified for sub-example folders, ie those containing an
|
||||
mbed-os.lib file. If found adds the path to the sub-example to a list which is
|
||||
then returned.
|
||||
""" Search the path for examples
|
||||
|
||||
Description:
|
||||
|
||||
Searches the path specified for sub-example folders, ie those containing an
|
||||
mbed-os.lib file. If found adds the path to the sub-example to a list which is
|
||||
then returned.
|
||||
|
||||
Args:
|
||||
path - path to search.
|
||||
|
|
@ -86,9 +147,13 @@ def find_all_examples(path):
|
|||
return examples
|
||||
|
||||
def upgrade_single_example(example, tag, directory, ref):
|
||||
""" Updates the mbed-os.lib file in the example specified to correspond to the
|
||||
version specified by the GitHub tag supplied. Also deals with
|
||||
multiple sub-examples in the GitHub repo, updating them in the same way.
|
||||
""" Update the mbed-os version for a single example
|
||||
|
||||
Description:
|
||||
|
||||
Updates the mbed-os.lib file in the example specified to correspond to the
|
||||
version specified by the GitHub tag supplied. Also deals with
|
||||
multiple sub-examples in the GitHub repo, updating them in the same way.
|
||||
|
||||
Args:
|
||||
example - json example object containing the GitHub repo to update.
|
||||
|
|
@ -111,7 +176,7 @@ def upgrade_single_example(example, tag, directory, ref):
|
|||
|
||||
os.rename("mbed-os.lib", "mbed-os.lib_bak")
|
||||
else:
|
||||
print("!! Error trying to backup mbed-os.lib prior to updating.")
|
||||
update_log.error("Failed to backup mbed-os.lib prior to updating.")
|
||||
return False
|
||||
|
||||
# mbed-os.lib file contains one line with the following format
|
||||
|
|
@ -142,6 +207,12 @@ def upgrade_single_example(example, tag, directory, ref):
|
|||
|
||||
def prepare_fork(arm_example):
|
||||
""" Synchronises a cloned fork to ensure it is up to date with the original.
|
||||
|
||||
Description:
|
||||
|
||||
This function sets a fork of an ARMmbed repo to be up to date with the
|
||||
repo it was forked from. It does this by hard resetting to the ARMmbed
|
||||
master branch.
|
||||
|
||||
Args:
|
||||
arm_example - Full GitHub repo path for original example
|
||||
|
|
@ -149,52 +220,120 @@ def prepare_fork(arm_example):
|
|||
|
||||
"""
|
||||
|
||||
print "In " + os.getcwd()
|
||||
logstr = "In: " + os.getcwd()
|
||||
update_log.debug(logstr)
|
||||
|
||||
for cmd in [['git', 'remote', 'add', 'armmbed', arm_example],
|
||||
['git', 'fetch', 'armmbed'],
|
||||
['git', 'reset', '--hard', 'armmbed/master'],
|
||||
['git', 'push', '-f', 'origin']]:
|
||||
if run_cmd(cmd):
|
||||
print("preparation of the fork failed!")
|
||||
update_log.error("Fork preparation failed")
|
||||
return False
|
||||
return True
|
||||
|
||||
def prepare_branch(branch):
|
||||
""" Set up at branch ready for use in updating examples
|
||||
|
||||
Description:
|
||||
|
||||
This function checks whether or not the supplied branch exists.
|
||||
If it does not, the branch is created and pushed to the origin.
|
||||
The branch is then switched to.
|
||||
|
||||
Args:
|
||||
arm_example - Full GitHub repo path for original example
|
||||
ret - True if the fork was synchronised successfully, False otherwise
|
||||
|
||||
"""
|
||||
|
||||
def upgrade_example(github, example, tag, user, ref):
|
||||
""" Clone a fork of the example specified.
|
||||
Ensures the fork is up to date with the original and then and updates the associated
|
||||
mbed-os.lib file on that fork to correspond to the version specified by the GitHub tag supplied.
|
||||
Also deals with multiple sub-examples in the GitHub repo, updating them in the same way.
|
||||
The updates are pushed to the forked repo.
|
||||
Finally a PR is raised against the original example repo for the changes.
|
||||
update_log.debug("Preparing branch: %s", branch)
|
||||
|
||||
# Check if branch already exists or not.
|
||||
cmd = ['git', 'branch']
|
||||
return_code, output = run_cmd_with_output(cmd)
|
||||
|
||||
if not branch in output:
|
||||
# OOB branch does not exist thus create it and then check it out
|
||||
cmd = ['git', 'checkout', '-b', branch]
|
||||
return_code = run_cmd(cmd)
|
||||
if not return_code:
|
||||
|
||||
# Push new branch upstream
|
||||
cmd = ['git', 'push', '-u', 'origin', branch]
|
||||
return_code = run_cmd(cmd)
|
||||
else:
|
||||
cmd = ['git', 'checkout', branch]
|
||||
return_code = run_cmd(cmd)
|
||||
|
||||
if return_code:
|
||||
update_log.error("Failed to prepare branch: %s", branch)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def upgrade_example(github, example, tag, ref,
|
||||
user='ARMmbed', branch='master'):
|
||||
""" Upgrade all versions of mbed-os.lib found in the specified example repo
|
||||
|
||||
Description:
|
||||
|
||||
Clone a version of the example specified and upgrade all versions of
|
||||
mbed-os.lib found within its tree. The version cloned and how it
|
||||
is upgraded depends on the user and branch specified. Only two options
|
||||
are valid:
|
||||
1) ARMmbed + non master branch
|
||||
This option will update the branch directly in the ARMmbed repo. If the
|
||||
branch does not exist it will be first created.
|
||||
|
||||
2) alternative user + master branch
|
||||
|
||||
This option assumes that a fork of the repo exists in the specified user's
|
||||
account. The fork will first be updated so that it is up to date with the
|
||||
upstream version , then the fork will be updated and a PR raised against
|
||||
the upstream ie ARMmbed repo.
|
||||
|
||||
Args:
|
||||
github - GitHub instance to allow internal git commands to be run
|
||||
example - json example object containing the GitHub repo to update.
|
||||
tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
|
||||
user - GitHub user name
|
||||
ref - SHA corresponding to the tag
|
||||
user - GitHub user name (defaults to 'ARMmbed' if not supplied)
|
||||
branch - branch to update (defaults to 'master' if not supplied)
|
||||
|
||||
returns True if the upgrade was successful, False otherwise
|
||||
"""
|
||||
|
||||
ret = False
|
||||
print("\nUpdating example '%s'" % example['name'])
|
||||
update_log.info("Updating example '%s'", example['name'])
|
||||
update_log.debug("User: %s", user)
|
||||
update_log.debug("Branch: %s", branch)
|
||||
|
||||
# First check validity of user/branch combination
|
||||
if ((user == 'ARMmbed' and branch == 'master') or
|
||||
(user != 'ARMmbed' and branch != 'master')):
|
||||
update_log.error("Invalid user/branch combination")
|
||||
return False
|
||||
|
||||
cwd = os.getcwd()
|
||||
|
||||
full_repo_name = 'ARMmbed/'+ example['name']
|
||||
fork = "https://github.com/" + user + '/' + example['name']
|
||||
upstream_repo = 'ARMmbed/'+ example['name']
|
||||
update_repo = "https://github.com/" + user + '/' + example['name']
|
||||
|
||||
update_log.debug("Upstream repository: %s", upstream_repo)
|
||||
update_log.debug("Update repository: %s", update_repo)
|
||||
|
||||
# Check access to mbed-os repo
|
||||
try:
|
||||
repo = github.get_repo(full_repo_name, False)
|
||||
repo = github.get_repo(upstream_repo, False)
|
||||
|
||||
except:
|
||||
print("\t\t!! Repo does not exist - skipping\n")
|
||||
update_log.error("Upstream repo: %s, does not exist - skipping", upstream_repo)
|
||||
return False
|
||||
|
||||
|
||||
# Clone the forked example repo
|
||||
clone_cmd = ['git', 'clone', fork]
|
||||
# Clone the example repo
|
||||
clone_cmd = ['git', 'clone', update_repo]
|
||||
return_code = run_cmd(clone_cmd)
|
||||
|
||||
if not return_code:
|
||||
|
|
@ -204,9 +343,14 @@ def upgrade_example(github, example, tag, user, ref):
|
|||
|
||||
os.chdir(example['name'])
|
||||
|
||||
# checkout and synchronise the release-candidate branch
|
||||
prepare_fork(example['github'])
|
||||
|
||||
# If the user is not the default, then a fork will be used. Thus
|
||||
# synchronise the user fork with the upstream
|
||||
if user != 'ARMmbed':
|
||||
prepare_fork(example['github'])
|
||||
|
||||
if branch != 'master':
|
||||
prepare_branch(branch)
|
||||
|
||||
for example_directory in example_directories:
|
||||
if not upgrade_single_example(example, tag, os.path.relpath(example_directory, example['name']), ref):
|
||||
os.chdir(cwd)
|
||||
|
|
@ -225,23 +369,24 @@ def upgrade_example(github, example, tag, user, ref):
|
|||
return_code = run_cmd(push_cmd)
|
||||
|
||||
if not return_code:
|
||||
body = "Please test/merge this PR and then tag Master with " + tag
|
||||
# Raise a PR from release-candidate to master
|
||||
user_fork = user + ':master'
|
||||
try:
|
||||
pr = repo.create_pull(title='Updating mbed-os to ' + tag, head=user_fork, base='master', body=body)
|
||||
ret = True
|
||||
except GithubException as e:
|
||||
# Default to False
|
||||
print("Creation of Pull Request from release-candidate to master failed with the following error!")
|
||||
print e
|
||||
if user != 'ARMmbed':
|
||||
body = "Please test/merge this PR and then tag Master with " + tag
|
||||
# Raise a PR from release-candidate to master
|
||||
user_fork = user + ':master'
|
||||
try:
|
||||
pr = repo.create_pull(title='Updating mbed-os to ' + tag, head=user_fork, base='master', body=body)
|
||||
ret = True
|
||||
except GithubException as e:
|
||||
# Default to False
|
||||
update_log.error("Pull request creation failed with error: %s", e)
|
||||
else:
|
||||
ret = True
|
||||
else:
|
||||
print("!!! Git push command failed.")
|
||||
update_log.error("Git push command failed.")
|
||||
else:
|
||||
print("!!! Git commit command failed.")
|
||||
update_log.error("Git commit command failed.")
|
||||
else:
|
||||
print("!!! Could not clone user fork %s\n" % fork)
|
||||
|
||||
update_log.error("Git clone %s failed", update_repo)
|
||||
|
||||
os.chdir(cwd)
|
||||
return ret
|
||||
|
|
@ -255,71 +400,43 @@ def create_work_directory(path):
|
|||
|
||||
"""
|
||||
if os.path.exists(path):
|
||||
print("'%s' directory already exists. Deleting..." % path)
|
||||
update_log.info("'%s' directory already exists. Deleting...", path)
|
||||
rmtree_readonly(path)
|
||||
|
||||
os.makedirs(path)
|
||||
|
||||
def test_compile(config, tag):
|
||||
""" For each example repo identified in the config json object, clone, update mbed-os to
|
||||
the specified tag and then compile for all supported toolchains.
|
||||
|
||||
Args:
|
||||
config - the json object imported from the file.
|
||||
tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
|
||||
results - summary of compilation results.
|
||||
|
||||
"""
|
||||
# Create work directories
|
||||
create_work_directory('test_compile')
|
||||
|
||||
# Loop through the examples
|
||||
results = {}
|
||||
os.chdir('test_compile')
|
||||
|
||||
lib.source_repos(config)
|
||||
lib.update_mbedos_version(config, tag)
|
||||
results = lib.compile_repos(config, SUPPORTED_TOOLCHAINS)
|
||||
os.chdir("..")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def main(arguments):
|
||||
""" Will update any mbed-os.lib files found in the example list specified by the config file.
|
||||
If no config file is specified the default 'examples.json' is used.
|
||||
The update is done by cloning a fork of each example (the fork must be present in the
|
||||
github account specified by the github user parameter). The fork is searched for any
|
||||
mbed-os.lib files and each one found is updated with the SHA corresponding to the supplied
|
||||
github tag. A pull request is then made from the fork to the original example repo.
|
||||
|
||||
Args:
|
||||
tag - tag to update the mbed-os.lib to. E.g. mbed-os-5.3.1
|
||||
github_token - Pre-authorised token to allow github access
|
||||
github_user - github username whose account contains the example forks
|
||||
config_file - optional parameter to specify a list of examples
|
||||
|
||||
"""
|
||||
if __name__ == '__main__':
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument('tag', help="mbed-os tag to which all examples will be updated")
|
||||
parser.add_argument('-c', '--config_file', help="Path to the configuration file (default is 'examples.json')", default='examples.json')
|
||||
parser.add_argument('-T', '--github_token', help="GitHub token for secure access")
|
||||
parser.add_argument('-U', '--github_user', help="GitHub user for forked repos")
|
||||
parser.add_argument('-l', '--log-level',
|
||||
help="Level for providing logging output",
|
||||
default='INFO')
|
||||
|
||||
args = parser.parse_args(arguments)
|
||||
exclusive = parser.add_mutually_exclusive_group(required=True)
|
||||
exclusive.add_argument('-U', '--github_user', help="GitHub user for forked repos, mutually exclusive to branch option")
|
||||
exclusive.add_argument('-b', '--branch', help="Branch to be updated, mutually exclusive to user option")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
default = getattr(logging, 'INFO')
|
||||
level = getattr(logging, args.log_level.upper(), default)
|
||||
|
||||
# Set logging level
|
||||
logging.basicConfig(level=level)
|
||||
|
||||
update_log = logging.getLogger("Update")
|
||||
|
||||
cfg = os.path.join(os.path.dirname(__file__), args.config_file)
|
||||
|
||||
# Load the config file
|
||||
config = json.load(open(os.path.join(os.path.dirname(__file__),
|
||||
args.config_file)))
|
||||
|
||||
if not config:
|
||||
print("Failed to load config file '%s'" % args.config_file)
|
||||
sys.exit(1)
|
||||
|
||||
with open(os.path.join(os.path.dirname(__file__), args.config_file)) as config:
|
||||
if not config:
|
||||
update_log.error("Failed to load config file '%s'", args.config_file)
|
||||
sys.exit(1)
|
||||
json_data = json.load(config)
|
||||
|
||||
# Create working directory
|
||||
create_work_directory('examples')
|
||||
|
||||
|
|
@ -330,7 +447,7 @@ def main(arguments):
|
|||
return_code, ref = run_cmd_with_output(cmd)
|
||||
|
||||
if return_code:
|
||||
print("Could not obtain SHA for tag: %s\n" % args.tag)
|
||||
update_log.error("Could not obtain SHA for tag: %s", args.tag)
|
||||
sys.exit(1)
|
||||
|
||||
# Loop through the examples
|
||||
|
|
@ -338,12 +455,18 @@ def main(arguments):
|
|||
successes = []
|
||||
results = {}
|
||||
os.chdir('examples')
|
||||
|
||||
for example in config['examples']:
|
||||
# Determine if this example should be updated and if so update any found
|
||||
# mbed-os.lib files.
|
||||
|
||||
if upgrade_example(github, example, args.tag, args.github_user, ref):
|
||||
for example in json_data['examples']:
|
||||
# Determine if this example should be updated and if so update any found
|
||||
# mbed-os.lib files.
|
||||
|
||||
# Only user or branch can be specified on the command line
|
||||
if args.github_user:
|
||||
result = upgrade_example(github, example, args.tag, ref, user=args.github_user)
|
||||
else:
|
||||
result = upgrade_example(github, example, args.tag, ref, branch=args.branch)
|
||||
|
||||
if result:
|
||||
successes += [example['name']]
|
||||
else:
|
||||
failures += [example['name']]
|
||||
|
|
@ -351,17 +474,11 @@ def main(arguments):
|
|||
os.chdir('../')
|
||||
|
||||
# Finish the script and report the results
|
||||
print(os.linesep + os.linesep +'Finished updating examples!' + os.linesep)
|
||||
|
||||
update_log.info("Finished updating examples")
|
||||
if successes:
|
||||
print('\nThe following examples updated successfully:')
|
||||
for success in successes:
|
||||
print(' - %s' % success)
|
||||
update_log.info(" SUCCEEDED: %s", success)
|
||||
|
||||
if failures:
|
||||
print('\nThe following examples were not updated:')
|
||||
for fail in failures:
|
||||
print(' - %s' % fail)
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
update_log.info(" FAILED: %s", fail)
|
||||
|
|
|
|||
Loading…
Reference in New Issue