From 6850ff06b973399fac8d038644a10056e7e771de Mon Sep 17 00:00:00 2001 From: Anna Bridge Date: Thu, 16 Mar 2017 16:59:11 +0000 Subject: [PATCH 1/2] Updated script to add an extra feature and general improvements. Added the ability to specify a branch to update rather than a fork Replaced print commands with the use of a logger Updated the run_cmd functions in line with previous improvements --- tools/test/examples/update.py | 307 ++++++++++++++++++++++------------ 1 file changed, 199 insertions(+), 108 deletions(-) diff --git a/tools/test/examples/update.py b/tools/test/examples/update.py index 0e398e7e61..0905480f4a 100644 --- a/tools/test/examples/update.py +++ b/tools/test/examples/update.py @@ -1,8 +1,43 @@ #!/usr/bin/env python +# This script is used to update the version of mbed-os used within a specified set of example +# applications. The list of examples to be updated lives in the examples.json file and is +# shared with the examples.py script. Logging is used to provide varying levels of output +# during execution. +# +# There are two modes that can be used: +# 1) Update the ARMmbed/master branch of the specified example +# +# This is done by updating a user fork of the example and then raising a pull request +# against ARMmbed/master. +# +# 2) Update a different ARMmbed branch of the specified example +# +# A branch to update is specified. If it doesn't already exist then it is first created. +# This branch will be updated and the change automatically pushed. +# +# Command usage: +# +# update.py -c - T -l -U -b +# +# Where: +# -c - Optional path to an examples file. +# If not proved the default is 'examples.json' +# -T - GitHub token for secure access (required) +# -l - Optional Level for providing logging output. Can be one of, +# CRITICAL, ERROR, WARNING, INFO, DEBUG +# If not provided the default is 'INFO' +# -U - GitHub user for forked repos +# -b - Branch to be updated +# +# NOTE only one of -U or -b can be specified. +# +# mbed-os tag to which all examples will be updated +# + import os from os.path import dirname, abspath, basename -import sys +import sys, logging import argparse import json import subprocess @@ -17,43 +52,56 @@ sys.path.insert(0, ROOT) import examples_lib as lib from examples_lib import SUPPORTED_TOOLCHAINS -def run_cmd(command, print_warning_on_fail=True): - """ Takes the command specified and runs it in a sub-process, obtaining the return code. - - Args: - command - command to run, provided as a list of individual fields which are combined into a - single command before passing to the sub-process call. - return_code - result of the command. +def run_cmd(command, exit_on_failure=False): + """ Passes a command to the system and returns a True/False result once the + command has been executed, indicating success/failure. Commands are passed + as a list of tokens. + E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v'] + Args: + command - system command as a list of tokens + exit_on_failure - If True exit the program on failure (default = False) + + Returns: + result - True/False indicating the success/failure of the command """ - print('[Exec] %s' % ' '.join(command)) - return_code = subprocess.call(command) + update_log.debug('[Exec] %s', ' '.join(command)) + return_code = subprocess.call(command, shell=True) if return_code: - print("The command '%s' failed with return code: %s" % (' '.join(command), return_code)) - print("Ignoring and moving on to the next example") + update_log.warning("Command '%s' failed with return code: %s", + ' '.join(command), return_code) + if exit_on_failure: + sys.exit(1) return return_code - -def run_cmd_with_output(command, print_warning_on_fail=True): - """ Takes the command specified and runs it in a sub-process, obtaining the return code - and the returned output. - - Args: - command - command to run, provided as a list of individual fields which are combined into a - single command before passing to the sub-process call. - return_code - result of the command. - output - the output of the command +def run_cmd_with_output(command, exit_on_failure=False): + """ Passes a command to the system and returns a True/False result once the + command has been executed, indicating success/failure. If the command was + successful then the output from the command is returned to the caller. + Commands are passed as a list of tokens. + E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v'] + + Args: + command - system command as a list of tokens + exit_on_failure - If True exit the program on failure (default = False) + + Returns: + result - True/False indicating the success/failure of the command + output - The output of the command if it was successful, else empty string """ - print('[Exec] %s' % ' '.join(command)) + update_log.debug('[Exec] %s', ' '.join(command)) returncode = 0 - output = None + output = "" try: - output = subprocess.check_output(command) + output = subprocess.check_output(command, shell=True) except subprocess.CalledProcessError as e: - print("The command '%s' failed with return code: %s" % (' '.join(command), e.returncode)) + update_log.warning("Command '%s' failed with return code: %s", + ' '.join(command), e.returncode) returncode = e.returncode + if exit_on_failure: + sys.exit(1) return returncode, output def rmtree_readonly(directory): @@ -111,7 +159,7 @@ def upgrade_single_example(example, tag, directory, ref): os.rename("mbed-os.lib", "mbed-os.lib_bak") else: - print("!! Error trying to backup mbed-os.lib prior to updating.") + update_log.error("Failed to backup mbed-os.lib prior to updating.") return False # mbed-os.lib file contains one line with the following format @@ -149,52 +197,100 @@ def prepare_fork(arm_example): """ - print "In " + os.getcwd() + update_log.debug("In: ", os.getcwd()) for cmd in [['git', 'remote', 'add', 'armmbed', arm_example], ['git', 'fetch', 'armmbed'], ['git', 'reset', '--hard', 'armmbed/master'], ['git', 'push', '-f', 'origin']]: if run_cmd(cmd): - print("preparation of the fork failed!") + update_log.error("Fork preparation failed") return False return True +def prepare_branch(branch): -def upgrade_example(github, example, tag, user, ref): - """ Clone a fork of the example specified. - Ensures the fork is up to date with the original and then and updates the associated - mbed-os.lib file on that fork to correspond to the version specified by the GitHub tag supplied. - Also deals with multiple sub-examples in the GitHub repo, updating them in the same way. - The updates are pushed to the forked repo. - Finally a PR is raised against the original example repo for the changes. + update_log.debug("Preparing branch: %s", branch) + + # Check if branch already exists or not. + cmd = ['git', 'branch'] + return_code, output = run_cmd_with_output(cmd) + + if not branch in output: + # OOB branch does not exist thus create it and then check it out + cmd = ['git', 'checkout', '-b', branch] + return_code = run_cmd(cmd) + if not return_code: + + # Push new branch upstream + cmd = ['git', 'push', '-u', 'origin', branch] + return_code = run_cmd(cmd) + else: + cmd = ['git', 'checkout', branch] + return_code = run_cmd(cmd) + + if return_code: + update_log.error("Failed to prepare branch: %s", branch) + return False + + return True + +def upgrade_example(github, example, tag, ref, + user='ARMmbed', branch='master'): + """ Clone a version of the example specified and upgrade all versions of + mbed-os.lib found within its tree. The version cloned and how it + is upgraded depends on the user and branch specified. Only two options + are valid: + 1) ARMmbed + non master branch + This option will update the branch directly in the ARMmbed repo. If the + branch does not exist it will be first created. + + 2) alternative user + master branch + + This option assumes that a fork of the repo exists in the specified user's + account. The fork will first be updated so that it is up to date with the + upstream version , then the fork will be updated and a PR raised against + the upstream ie ARMmbed repo. Args: github - GitHub instance to allow internal git commands to be run example - json example object containing the GitHub repo to update. tag - GitHub tag corresponding to a version of mbed-os to upgrade to. - user - GitHub user name ref - SHA corresponding to the tag - + user - GitHub user name + branch - branch to update """ + ret = False - print("\nUpdating example '%s'" % example['name']) + update_log.info("Updating example '%s'", example['name']) + update_log.debug("User: %s", user) + update_log.debug("Branch: %s", branch) + + # First check validity of user/branch combination + if ((user == 'ARMmbed' and branch == 'master') or + (user != 'ARMmbed' and branch != 'master')): + update_log.error("Invalid user/branch combination") + return False + cwd = os.getcwd() - full_repo_name = 'ARMmbed/'+ example['name'] - fork = "https://github.com/" + user + '/' + example['name'] + upstream_repo = 'ARMmbed/'+ example['name'] + update_repo = "https://github.com/" + user + '/' + example['name'] + + update_log.debug("Upstream repository: %s", upstream_repo) + update_log.debug("Update repository: %s", update_repo) # Check access to mbed-os repo try: - repo = github.get_repo(full_repo_name, False) + repo = github.get_repo(upstream_repo, False) except: - print("\t\t!! Repo does not exist - skipping\n") + update_log.error("Upstream repo: %s, does not exist - skipping", upstream_repo) return False - # Clone the forked example repo - clone_cmd = ['git', 'clone', fork] + # Clone the example repo + clone_cmd = ['git', 'clone', update_repo] return_code = run_cmd(clone_cmd) if not return_code: @@ -204,9 +300,14 @@ def upgrade_example(github, example, tag, user, ref): os.chdir(example['name']) - # checkout and synchronise the release-candidate branch - prepare_fork(example['github']) - + # If the user is not the default, then a fork will be used. Thus + # synchronise the user fork with the upstream + if user != 'ARMmbed': + prepare_fork(example['github']) + + if branch != 'master': + prepare_branch(branch) + for example_directory in example_directories: if not upgrade_single_example(example, tag, os.path.relpath(example_directory, example['name']), ref): os.chdir(cwd) @@ -225,23 +326,24 @@ def upgrade_example(github, example, tag, user, ref): return_code = run_cmd(push_cmd) if not return_code: - body = "Please test/merge this PR and then tag Master with " + tag - # Raise a PR from release-candidate to master - user_fork = user + ':master' - try: - pr = repo.create_pull(title='Updating mbed-os to ' + tag, head=user_fork, base='master', body=body) - ret = True - except GithubException as e: - # Default to False - print("Creation of Pull Request from release-candidate to master failed with the following error!") - print e + if user != 'ARMmbed': + body = "Please test/merge this PR and then tag Master with " + tag + # Raise a PR from release-candidate to master + user_fork = user + ':master' + try: + pr = repo.create_pull(title='Updating mbed-os to ' + tag, head=user_fork, base='master', body=body) + ret = True + except GithubException as e: + # Default to False + update_log.error("Pull request creation failed with error: %s", e) + else: + ret = True else: - print("!!! Git push command failed.") + update_log.error("Git push command failed.") else: - print("!!! Git commit command failed.") + update_log.error("Git commit command failed.") else: - print("!!! Could not clone user fork %s\n" % fork) - + update_log.error("Git clone %s failed", update_repo) os.chdir(cwd) return ret @@ -255,36 +357,11 @@ def create_work_directory(path): """ if os.path.exists(path): - print("'%s' directory already exists. Deleting..." % path) + update_log.info("'%s' directory already exists. Deleting...", path) rmtree_readonly(path) os.makedirs(path) -def test_compile(config, tag): - """ For each example repo identified in the config json object, clone, update mbed-os to - the specified tag and then compile for all supported toolchains. - - Args: - config - the json object imported from the file. - tag - GitHub tag corresponding to a version of mbed-os to upgrade to. - results - summary of compilation results. - - """ - # Create work directories - create_work_directory('test_compile') - - # Loop through the examples - results = {} - os.chdir('test_compile') - - lib.source_repos(config) - lib.update_mbedos_version(config, tag) - results = lib.compile_repos(config, SUPPORTED_TOOLCHAINS) - os.chdir("..") - - return results - - def main(arguments): """ Will update any mbed-os.lib files found in the example list specified by the config file. If no config file is specified the default 'examples.json' is used. @@ -306,20 +383,31 @@ def main(arguments): parser.add_argument('tag', help="mbed-os tag to which all examples will be updated") parser.add_argument('-c', '--config_file', help="Path to the configuration file (default is 'examples.json')", default='examples.json') parser.add_argument('-T', '--github_token', help="GitHub token for secure access") - parser.add_argument('-U', '--github_user', help="GitHub user for forked repos") + parser.add_argument('-l', '--log-level', + help="Level for providing logging output", + default='INFO') + + exclusive = parser.add_mutually_exclusive_group(required=True) + exclusive.add_argument('-U', '--github_user', help="GitHub user for forked repos, mutually exclusive to branch option") + exclusive.add_argument('-b', '--branch', help="Branch to be updated, mutually exclusive to user option") args = parser.parse_args(arguments) - cfg = os.path.join(os.path.dirname(__file__), args.config_file) - + default = getattr(logging, 'INFO') + level = getattr(logging, args.log_level.upper(), default) + + # Set logging level + logging.basicConfig(level=level) + + update_log = logging.getLogger("Update") + # Load the config file - config = json.load(open(os.path.join(os.path.dirname(__file__), - args.config_file))) - - if not config: - print("Failed to load config file '%s'" % args.config_file) - sys.exit(1) - + with open(os.path.join(os.path.dirname(__file__), args.config_file)) as config: + if not config: + update_log.error("Failed to load config file '%s'", args.config_file) + sys.exit(1) + json_data = json.load(config) + # Create working directory create_work_directory('examples') @@ -330,7 +418,7 @@ def main(arguments): return_code, ref = run_cmd_with_output(cmd) if return_code: - print("Could not obtain SHA for tag: %s\n" % args.tag) + update_log.error("Could not obtain SHA for tag: %s", args.tag) sys.exit(1) # Loop through the examples @@ -341,9 +429,15 @@ def main(arguments): for example in config['examples']: # Determine if this example should be updated and if so update any found - # mbed-os.lib files. - - if upgrade_example(github, example, args.tag, args.github_user, ref): + # mbed-os.lib files. + + # Only user or branch can be specified on the command line + if args.github_user: + result = upgrade_example(github, example, args.tag, ref, user=args.github_user) + else: + result = upgrade_example(github, example, args.tag, ref, branch=args.branch) + + if result: successes += [example['name']] else: failures += [example['name']] @@ -351,17 +445,14 @@ def main(arguments): os.chdir('../') # Finish the script and report the results - print(os.linesep + os.linesep +'Finished updating examples!' + os.linesep) - + update_log.info("Finished updating examples") if successes: - print('\nThe following examples updated successfully:') for success in successes: - print(' - %s' % success) + update_log.info(" SUCCEEDED: %s", success) if failures: - print('\nThe following examples were not updated:') for fail in failures: - print(' - %s' % fail) + update_log.info(" FAILED: %s", fail) if __name__ == '__main__': sys.exit(main(sys.argv[1:])) \ No newline at end of file From 9ea605bb982e59e0e7a815ac9d56091969623832 Mon Sep 17 00:00:00 2001 From: adbridge Date: Fri, 21 Apr 2017 11:08:58 +0100 Subject: [PATCH 2/2] Update.py: Tidy up Fn headers, make logger work globally The function headers have been updated to follow the standard format that should be being used for tools in mbed. This is a one line summary followed by a descriptive block with more detail. Updated the handling of the main function so that the logger becomes global and thus works across all the functions. This has been tested with both the fork and branch options, and for levels INFO and DEBUG. --- tools/test/examples/update.py | 134 ++++++++++++++++++++-------------- 1 file changed, 80 insertions(+), 54 deletions(-) diff --git a/tools/test/examples/update.py b/tools/test/examples/update.py index 0905480f4a..1622443446 100644 --- a/tools/test/examples/update.py +++ b/tools/test/examples/update.py @@ -37,7 +37,8 @@ import os from os.path import dirname, abspath, basename -import sys, logging +import sys +import logging import argparse import json import subprocess @@ -53,17 +54,21 @@ import examples_lib as lib from examples_lib import SUPPORTED_TOOLCHAINS def run_cmd(command, exit_on_failure=False): - """ Passes a command to the system and returns a True/False result once the - command has been executed, indicating success/failure. Commands are passed - as a list of tokens. - E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v'] + """ Run a system command and return the result status + + Description: + + Passes a command to the system and returns a True/False result, once the + command has been executed, indicating success/failure. Commands are passed + as a list of tokens. + E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v'] Args: command - system command as a list of tokens exit_on_failure - If True exit the program on failure (default = False) Returns: - result - True/False indicating the success/failure of the command + return_code - True/False indicating the success/failure of the command """ update_log.debug('[Exec] %s', ' '.join(command)) return_code = subprocess.call(command, shell=True) @@ -77,18 +82,22 @@ def run_cmd(command, exit_on_failure=False): return return_code def run_cmd_with_output(command, exit_on_failure=False): - """ Passes a command to the system and returns a True/False result once the - command has been executed, indicating success/failure. If the command was - successful then the output from the command is returned to the caller. - Commands are passed as a list of tokens. - E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v'] + """ Run a system command and return the result status plus output + + Description: + + Passes a command to the system and returns a True/False result once the + command has been executed, indicating success/failure. If the command was + successful then the output from the command is returned to the caller. + Commands are passed as a list of tokens. + E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v'] Args: command - system command as a list of tokens exit_on_failure - If True exit the program on failure (default = False) Returns: - result - True/False indicating the success/failure of the command + returncode - True/False indicating the success/failure of the command output - The output of the command if it was successful, else empty string """ update_log.debug('[Exec] %s', ' '.join(command)) @@ -117,9 +126,13 @@ def rmtree_readonly(directory): shutil.rmtree(directory, onerror=remove_readonly) def find_all_examples(path): - """ Searches the path specified for sub-example folders, ie those containing an - mbed-os.lib file. If found adds the path to the sub-example to a list which is - then returned. + """ Search the path for examples + + Description: + + Searches the path specified for sub-example folders, ie those containing an + mbed-os.lib file. If found adds the path to the sub-example to a list which is + then returned. Args: path - path to search. @@ -134,9 +147,13 @@ def find_all_examples(path): return examples def upgrade_single_example(example, tag, directory, ref): - """ Updates the mbed-os.lib file in the example specified to correspond to the - version specified by the GitHub tag supplied. Also deals with - multiple sub-examples in the GitHub repo, updating them in the same way. + """ Update the mbed-os version for a single example + + Description: + + Updates the mbed-os.lib file in the example specified to correspond to the + version specified by the GitHub tag supplied. Also deals with + multiple sub-examples in the GitHub repo, updating them in the same way. Args: example - json example object containing the GitHub repo to update. @@ -190,6 +207,12 @@ def upgrade_single_example(example, tag, directory, ref): def prepare_fork(arm_example): """ Synchronises a cloned fork to ensure it is up to date with the original. + + Description: + + This function sets a fork of an ARMmbed repo to be up to date with the + repo it was forked from. It does this by hard resetting to the ARMmbed + master branch. Args: arm_example - Full GitHub repo path for original example @@ -197,7 +220,8 @@ def prepare_fork(arm_example): """ - update_log.debug("In: ", os.getcwd()) + logstr = "In: " + os.getcwd() + update_log.debug(logstr) for cmd in [['git', 'remote', 'add', 'armmbed', arm_example], ['git', 'fetch', 'armmbed'], @@ -209,6 +233,19 @@ def prepare_fork(arm_example): return True def prepare_branch(branch): + """ Set up at branch ready for use in updating examples + + Description: + + This function checks whether or not the supplied branch exists. + If it does not, the branch is created and pushed to the origin. + The branch is then switched to. + + Args: + arm_example - Full GitHub repo path for original example + ret - True if the fork was synchronised successfully, False otherwise + + """ update_log.debug("Preparing branch: %s", branch) @@ -237,28 +274,34 @@ def prepare_branch(branch): def upgrade_example(github, example, tag, ref, user='ARMmbed', branch='master'): - """ Clone a version of the example specified and upgrade all versions of - mbed-os.lib found within its tree. The version cloned and how it - is upgraded depends on the user and branch specified. Only two options - are valid: - 1) ARMmbed + non master branch - This option will update the branch directly in the ARMmbed repo. If the - branch does not exist it will be first created. + """ Upgrade all versions of mbed-os.lib found in the specified example repo + + Description: + + Clone a version of the example specified and upgrade all versions of + mbed-os.lib found within its tree. The version cloned and how it + is upgraded depends on the user and branch specified. Only two options + are valid: + 1) ARMmbed + non master branch + This option will update the branch directly in the ARMmbed repo. If the + branch does not exist it will be first created. - 2) alternative user + master branch + 2) alternative user + master branch - This option assumes that a fork of the repo exists in the specified user's - account. The fork will first be updated so that it is up to date with the - upstream version , then the fork will be updated and a PR raised against - the upstream ie ARMmbed repo. + This option assumes that a fork of the repo exists in the specified user's + account. The fork will first be updated so that it is up to date with the + upstream version , then the fork will be updated and a PR raised against + the upstream ie ARMmbed repo. Args: github - GitHub instance to allow internal git commands to be run example - json example object containing the GitHub repo to update. tag - GitHub tag corresponding to a version of mbed-os to upgrade to. ref - SHA corresponding to the tag - user - GitHub user name - branch - branch to update + user - GitHub user name (defaults to 'ARMmbed' if not supplied) + branch - branch to update (defaults to 'master' if not supplied) + + returns True if the upgrade was successful, False otherwise """ ret = False @@ -362,21 +405,7 @@ def create_work_directory(path): os.makedirs(path) -def main(arguments): - """ Will update any mbed-os.lib files found in the example list specified by the config file. - If no config file is specified the default 'examples.json' is used. - The update is done by cloning a fork of each example (the fork must be present in the - github account specified by the github user parameter). The fork is searched for any - mbed-os.lib files and each one found is updated with the SHA corresponding to the supplied - github tag. A pull request is then made from the fork to the original example repo. - - Args: - tag - tag to update the mbed-os.lib to. E.g. mbed-os-5.3.1 - github_token - Pre-authorised token to allow github access - github_user - github username whose account contains the example forks - config_file - optional parameter to specify a list of examples - - """ +if __name__ == '__main__': parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) @@ -391,7 +420,7 @@ def main(arguments): exclusive.add_argument('-U', '--github_user', help="GitHub user for forked repos, mutually exclusive to branch option") exclusive.add_argument('-b', '--branch', help="Branch to be updated, mutually exclusive to user option") - args = parser.parse_args(arguments) + args = parser.parse_args() default = getattr(logging, 'INFO') level = getattr(logging, args.log_level.upper(), default) @@ -426,8 +455,8 @@ def main(arguments): successes = [] results = {} os.chdir('examples') - - for example in config['examples']: + + for example in json_data['examples']: # Determine if this example should be updated and if so update any found # mbed-os.lib files. @@ -453,6 +482,3 @@ def main(arguments): if failures: for fail in failures: update_log.info(" FAILED: %s", fail) - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) \ No newline at end of file