diff --git a/tools/test/examples/update.py b/tools/test/examples/update.py index 0e398e7e61..0905480f4a 100644 --- a/tools/test/examples/update.py +++ b/tools/test/examples/update.py @@ -1,8 +1,43 @@ #!/usr/bin/env python +# This script is used to update the version of mbed-os used within a specified set of example +# applications. The list of examples to be updated lives in the examples.json file and is +# shared with the examples.py script. Logging is used to provide varying levels of output +# during execution. +# +# There are two modes that can be used: +# 1) Update the ARMmbed/master branch of the specified example +# +# This is done by updating a user fork of the example and then raising a pull request +# against ARMmbed/master. +# +# 2) Update a different ARMmbed branch of the specified example +# +# A branch to update is specified. If it doesn't already exist then it is first created. +# This branch will be updated and the change automatically pushed. +# +# Command usage: +# +# update.py -c - T -l -U -b +# +# Where: +# -c - Optional path to an examples file. +# If not proved the default is 'examples.json' +# -T - GitHub token for secure access (required) +# -l - Optional Level for providing logging output. Can be one of, +# CRITICAL, ERROR, WARNING, INFO, DEBUG +# If not provided the default is 'INFO' +# -U - GitHub user for forked repos +# -b - Branch to be updated +# +# NOTE only one of -U or -b can be specified. +# +# mbed-os tag to which all examples will be updated +# + import os from os.path import dirname, abspath, basename -import sys +import sys, logging import argparse import json import subprocess @@ -17,43 +52,56 @@ sys.path.insert(0, ROOT) import examples_lib as lib from examples_lib import SUPPORTED_TOOLCHAINS -def run_cmd(command, print_warning_on_fail=True): - """ Takes the command specified and runs it in a sub-process, obtaining the return code. - - Args: - command - command to run, provided as a list of individual fields which are combined into a - single command before passing to the sub-process call. - return_code - result of the command. +def run_cmd(command, exit_on_failure=False): + """ Passes a command to the system and returns a True/False result once the + command has been executed, indicating success/failure. Commands are passed + as a list of tokens. + E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v'] + Args: + command - system command as a list of tokens + exit_on_failure - If True exit the program on failure (default = False) + + Returns: + result - True/False indicating the success/failure of the command """ - print('[Exec] %s' % ' '.join(command)) - return_code = subprocess.call(command) + update_log.debug('[Exec] %s', ' '.join(command)) + return_code = subprocess.call(command, shell=True) if return_code: - print("The command '%s' failed with return code: %s" % (' '.join(command), return_code)) - print("Ignoring and moving on to the next example") + update_log.warning("Command '%s' failed with return code: %s", + ' '.join(command), return_code) + if exit_on_failure: + sys.exit(1) return return_code - -def run_cmd_with_output(command, print_warning_on_fail=True): - """ Takes the command specified and runs it in a sub-process, obtaining the return code - and the returned output. - - Args: - command - command to run, provided as a list of individual fields which are combined into a - single command before passing to the sub-process call. - return_code - result of the command. - output - the output of the command +def run_cmd_with_output(command, exit_on_failure=False): + """ Passes a command to the system and returns a True/False result once the + command has been executed, indicating success/failure. If the command was + successful then the output from the command is returned to the caller. + Commands are passed as a list of tokens. + E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v'] + + Args: + command - system command as a list of tokens + exit_on_failure - If True exit the program on failure (default = False) + + Returns: + result - True/False indicating the success/failure of the command + output - The output of the command if it was successful, else empty string """ - print('[Exec] %s' % ' '.join(command)) + update_log.debug('[Exec] %s', ' '.join(command)) returncode = 0 - output = None + output = "" try: - output = subprocess.check_output(command) + output = subprocess.check_output(command, shell=True) except subprocess.CalledProcessError as e: - print("The command '%s' failed with return code: %s" % (' '.join(command), e.returncode)) + update_log.warning("Command '%s' failed with return code: %s", + ' '.join(command), e.returncode) returncode = e.returncode + if exit_on_failure: + sys.exit(1) return returncode, output def rmtree_readonly(directory): @@ -111,7 +159,7 @@ def upgrade_single_example(example, tag, directory, ref): os.rename("mbed-os.lib", "mbed-os.lib_bak") else: - print("!! Error trying to backup mbed-os.lib prior to updating.") + update_log.error("Failed to backup mbed-os.lib prior to updating.") return False # mbed-os.lib file contains one line with the following format @@ -149,52 +197,100 @@ def prepare_fork(arm_example): """ - print "In " + os.getcwd() + update_log.debug("In: ", os.getcwd()) for cmd in [['git', 'remote', 'add', 'armmbed', arm_example], ['git', 'fetch', 'armmbed'], ['git', 'reset', '--hard', 'armmbed/master'], ['git', 'push', '-f', 'origin']]: if run_cmd(cmd): - print("preparation of the fork failed!") + update_log.error("Fork preparation failed") return False return True +def prepare_branch(branch): -def upgrade_example(github, example, tag, user, ref): - """ Clone a fork of the example specified. - Ensures the fork is up to date with the original and then and updates the associated - mbed-os.lib file on that fork to correspond to the version specified by the GitHub tag supplied. - Also deals with multiple sub-examples in the GitHub repo, updating them in the same way. - The updates are pushed to the forked repo. - Finally a PR is raised against the original example repo for the changes. + update_log.debug("Preparing branch: %s", branch) + + # Check if branch already exists or not. + cmd = ['git', 'branch'] + return_code, output = run_cmd_with_output(cmd) + + if not branch in output: + # OOB branch does not exist thus create it and then check it out + cmd = ['git', 'checkout', '-b', branch] + return_code = run_cmd(cmd) + if not return_code: + + # Push new branch upstream + cmd = ['git', 'push', '-u', 'origin', branch] + return_code = run_cmd(cmd) + else: + cmd = ['git', 'checkout', branch] + return_code = run_cmd(cmd) + + if return_code: + update_log.error("Failed to prepare branch: %s", branch) + return False + + return True + +def upgrade_example(github, example, tag, ref, + user='ARMmbed', branch='master'): + """ Clone a version of the example specified and upgrade all versions of + mbed-os.lib found within its tree. The version cloned and how it + is upgraded depends on the user and branch specified. Only two options + are valid: + 1) ARMmbed + non master branch + This option will update the branch directly in the ARMmbed repo. If the + branch does not exist it will be first created. + + 2) alternative user + master branch + + This option assumes that a fork of the repo exists in the specified user's + account. The fork will first be updated so that it is up to date with the + upstream version , then the fork will be updated and a PR raised against + the upstream ie ARMmbed repo. Args: github - GitHub instance to allow internal git commands to be run example - json example object containing the GitHub repo to update. tag - GitHub tag corresponding to a version of mbed-os to upgrade to. - user - GitHub user name ref - SHA corresponding to the tag - + user - GitHub user name + branch - branch to update """ + ret = False - print("\nUpdating example '%s'" % example['name']) + update_log.info("Updating example '%s'", example['name']) + update_log.debug("User: %s", user) + update_log.debug("Branch: %s", branch) + + # First check validity of user/branch combination + if ((user == 'ARMmbed' and branch == 'master') or + (user != 'ARMmbed' and branch != 'master')): + update_log.error("Invalid user/branch combination") + return False + cwd = os.getcwd() - full_repo_name = 'ARMmbed/'+ example['name'] - fork = "https://github.com/" + user + '/' + example['name'] + upstream_repo = 'ARMmbed/'+ example['name'] + update_repo = "https://github.com/" + user + '/' + example['name'] + + update_log.debug("Upstream repository: %s", upstream_repo) + update_log.debug("Update repository: %s", update_repo) # Check access to mbed-os repo try: - repo = github.get_repo(full_repo_name, False) + repo = github.get_repo(upstream_repo, False) except: - print("\t\t!! Repo does not exist - skipping\n") + update_log.error("Upstream repo: %s, does not exist - skipping", upstream_repo) return False - # Clone the forked example repo - clone_cmd = ['git', 'clone', fork] + # Clone the example repo + clone_cmd = ['git', 'clone', update_repo] return_code = run_cmd(clone_cmd) if not return_code: @@ -204,9 +300,14 @@ def upgrade_example(github, example, tag, user, ref): os.chdir(example['name']) - # checkout and synchronise the release-candidate branch - prepare_fork(example['github']) - + # If the user is not the default, then a fork will be used. Thus + # synchronise the user fork with the upstream + if user != 'ARMmbed': + prepare_fork(example['github']) + + if branch != 'master': + prepare_branch(branch) + for example_directory in example_directories: if not upgrade_single_example(example, tag, os.path.relpath(example_directory, example['name']), ref): os.chdir(cwd) @@ -225,23 +326,24 @@ def upgrade_example(github, example, tag, user, ref): return_code = run_cmd(push_cmd) if not return_code: - body = "Please test/merge this PR and then tag Master with " + tag - # Raise a PR from release-candidate to master - user_fork = user + ':master' - try: - pr = repo.create_pull(title='Updating mbed-os to ' + tag, head=user_fork, base='master', body=body) - ret = True - except GithubException as e: - # Default to False - print("Creation of Pull Request from release-candidate to master failed with the following error!") - print e + if user != 'ARMmbed': + body = "Please test/merge this PR and then tag Master with " + tag + # Raise a PR from release-candidate to master + user_fork = user + ':master' + try: + pr = repo.create_pull(title='Updating mbed-os to ' + tag, head=user_fork, base='master', body=body) + ret = True + except GithubException as e: + # Default to False + update_log.error("Pull request creation failed with error: %s", e) + else: + ret = True else: - print("!!! Git push command failed.") + update_log.error("Git push command failed.") else: - print("!!! Git commit command failed.") + update_log.error("Git commit command failed.") else: - print("!!! Could not clone user fork %s\n" % fork) - + update_log.error("Git clone %s failed", update_repo) os.chdir(cwd) return ret @@ -255,36 +357,11 @@ def create_work_directory(path): """ if os.path.exists(path): - print("'%s' directory already exists. Deleting..." % path) + update_log.info("'%s' directory already exists. Deleting...", path) rmtree_readonly(path) os.makedirs(path) -def test_compile(config, tag): - """ For each example repo identified in the config json object, clone, update mbed-os to - the specified tag and then compile for all supported toolchains. - - Args: - config - the json object imported from the file. - tag - GitHub tag corresponding to a version of mbed-os to upgrade to. - results - summary of compilation results. - - """ - # Create work directories - create_work_directory('test_compile') - - # Loop through the examples - results = {} - os.chdir('test_compile') - - lib.source_repos(config) - lib.update_mbedos_version(config, tag) - results = lib.compile_repos(config, SUPPORTED_TOOLCHAINS) - os.chdir("..") - - return results - - def main(arguments): """ Will update any mbed-os.lib files found in the example list specified by the config file. If no config file is specified the default 'examples.json' is used. @@ -306,20 +383,31 @@ def main(arguments): parser.add_argument('tag', help="mbed-os tag to which all examples will be updated") parser.add_argument('-c', '--config_file', help="Path to the configuration file (default is 'examples.json')", default='examples.json') parser.add_argument('-T', '--github_token', help="GitHub token for secure access") - parser.add_argument('-U', '--github_user', help="GitHub user for forked repos") + parser.add_argument('-l', '--log-level', + help="Level for providing logging output", + default='INFO') + + exclusive = parser.add_mutually_exclusive_group(required=True) + exclusive.add_argument('-U', '--github_user', help="GitHub user for forked repos, mutually exclusive to branch option") + exclusive.add_argument('-b', '--branch', help="Branch to be updated, mutually exclusive to user option") args = parser.parse_args(arguments) - cfg = os.path.join(os.path.dirname(__file__), args.config_file) - + default = getattr(logging, 'INFO') + level = getattr(logging, args.log_level.upper(), default) + + # Set logging level + logging.basicConfig(level=level) + + update_log = logging.getLogger("Update") + # Load the config file - config = json.load(open(os.path.join(os.path.dirname(__file__), - args.config_file))) - - if not config: - print("Failed to load config file '%s'" % args.config_file) - sys.exit(1) - + with open(os.path.join(os.path.dirname(__file__), args.config_file)) as config: + if not config: + update_log.error("Failed to load config file '%s'", args.config_file) + sys.exit(1) + json_data = json.load(config) + # Create working directory create_work_directory('examples') @@ -330,7 +418,7 @@ def main(arguments): return_code, ref = run_cmd_with_output(cmd) if return_code: - print("Could not obtain SHA for tag: %s\n" % args.tag) + update_log.error("Could not obtain SHA for tag: %s", args.tag) sys.exit(1) # Loop through the examples @@ -341,9 +429,15 @@ def main(arguments): for example in config['examples']: # Determine if this example should be updated and if so update any found - # mbed-os.lib files. - - if upgrade_example(github, example, args.tag, args.github_user, ref): + # mbed-os.lib files. + + # Only user or branch can be specified on the command line + if args.github_user: + result = upgrade_example(github, example, args.tag, ref, user=args.github_user) + else: + result = upgrade_example(github, example, args.tag, ref, branch=args.branch) + + if result: successes += [example['name']] else: failures += [example['name']] @@ -351,17 +445,14 @@ def main(arguments): os.chdir('../') # Finish the script and report the results - print(os.linesep + os.linesep +'Finished updating examples!' + os.linesep) - + update_log.info("Finished updating examples") if successes: - print('\nThe following examples updated successfully:') for success in successes: - print(' - %s' % success) + update_log.info(" SUCCEEDED: %s", success) if failures: - print('\nThe following examples were not updated:') for fail in failures: - print(' - %s' % fail) + update_log.info(" FAILED: %s", fail) if __name__ == '__main__': sys.exit(main(sys.argv[1:])) \ No newline at end of file