diff --git a/tools/test/examples/examples.json b/tools/test/examples/examples.json index 8a38a7cb9e..43468dc102 100644 --- a/tools/test/examples/examples.json +++ b/tools/test/examples/examples.json @@ -1,17 +1,4 @@ { - "update-config" : { - "help" : "Update each example repo with a version of mbed-os identified by the tag", - "via-fork" : { - "help" : "-f cmd line option. Update a fork", - "github-user" : "adbridge" - }, - "via-branch" : { - "help" : "-b cmd line option. Update dst branch, created from src branch", - "src-branch" : "mbed-os-5.6.0-oob2", - "dst-branch" : "mbed-os-5.6.0-oob2" - }, - "tag" : "mbed-os-5.6.2" - }, "examples": [ { "name": "mbed-os-example-blinky", diff --git a/tools/test/examples/pr.tmpl b/tools/test/examples/pr.tmpl deleted file mode 100644 index eb7212aef3..0000000000 --- a/tools/test/examples/pr.tmpl +++ /dev/null @@ -1,4 +0,0 @@ -Please test this PR - -If successful then merge, otherwise provide a known issue. -Once you get notification of the release being made public then tag Master with {{ tag }} . diff --git a/tools/test/examples/update.py b/tools/test/examples/update.py deleted file mode 100644 index a385d6fbe1..0000000000 --- a/tools/test/examples/update.py +++ /dev/null @@ -1,562 +0,0 @@ -#!/usr/bin/env python - -# This script is used to update the version of mbed-os used within a specified set of example -# applications. The list of examples to be updated lives in the examples.json file and is -# shared with the examples.py script. Logging is used to provide varying levels of output -# during execution. -# -# There are two modes that can be used: -# 1) Update the ARMmbed/master branch of the specified example -# -# This is done by updating a user fork of the example and then raising a pull request -# against ARMmbed/master. -# -# 2) Update a different ARMmbed branch of the specified example -# -# A branch to update is specified. If it doesn't already exist then it is first created. -# This branch will be updated and the change automatically pushed. The new branch will -# be created from the specified source branch. -# -# The modes are controlled via configuration data in the json file. -# E.g. -# -# "update-config" : { -# "help" : "Update each example repo with a version of mbed-os identified by the tag", -# "via-fork" : { -# "help" : "-f cmd line option. Update a fork", -# "github-user" : "adbridge" -# }, -# "via-branch" : { -# "help" : "-b cmd line option. Update dst branch, created from src branch", -# "src-branch" : "mbed-os-5.5.0-rc1-oob", -# "dst-branch" : "mbed-os-5.5.0-rc2-oob" -# }, -# "tag" : "mbed-os-5.5.0-rc2" -# -# -# Command usage: -# -# update.py -c - T -f -b -s -# -# Where: -# -c - Optional path to an examples file. -# If not proved the default is 'examples.json' -# -T - GitHub token for secure access (required) -# -f - Update forked repos. This will use the 'github-user' parameter in -# the 'via-fork' section. -# -b - Update branched repos. This will use the "src-branch" and -# "dst-branch" parameters in the 'via-branch' section. The destination -# branch is created from the source branch (if it doesn't already exist). -# -s - Show the status of any pull requests with a tag matching that in the -# json config file -# -# The options -f, -b and -s are mutually exlusive. Only one can be specified. -# -# - -import os -from os.path import dirname, abspath, basename, join -import sys -import logging -import argparse -import json -import subprocess -import shutil -import stat -import re -from github import Github, GithubException -from jinja2 import FileSystemLoader, StrictUndefined -from jinja2.environment import Environment - -ROOT = abspath(dirname(dirname(dirname(dirname(__file__))))) -sys.path.insert(0, ROOT) - -import examples_lib as lib -from examples_lib import SUPPORTED_TOOLCHAINS - -userlog = logging.getLogger("Update") - -# Set logging level -userlog.setLevel(logging.DEBUG) - -# Everything is output to the log file -logfile = os.path.join(os.getcwd(), 'update.log') -fh = logging.FileHandler(logfile) -fh.setLevel(logging.DEBUG) - -# create console handler with a higher log level -ch = logging.StreamHandler() -ch.setLevel(logging.INFO) - -formatter = logging.Formatter('%(name)s: %(levelname)s - %(message)s') -ch.setFormatter(formatter) -fh.setFormatter(formatter) - -# add the handlers to the logger -userlog.addHandler(fh) -userlog.addHandler(ch) - -def run_cmd(command, exit_on_failure=False): - """ Run a system command returning a status result - - This is just a wrapper for the run_cmd_with_output() function, but - only returns the status of the call. - - Args: - command - system command as a list of tokens - exit_on_failure - If True exit the program on failure (default = False) - - Returns: - return_code - True/False indicating the success/failure of the command - """ - return_code, _ = run_cmd_with_output(command, exit_on_failure) - return return_code - -def run_cmd_with_output(command, exit_on_failure=False): - """ Run a system command returning a status result and any command output - - Passes a command to the system and returns a True/False result once the - command has been executed, indicating success/failure. If the command was - successful then the output from the command is returned to the caller. - Commands are passed as a string. - E.g. The command 'git remote -v' would be passed in as "git remote -v" - - Args: - command - system command as a string - exit_on_failure - If True exit the program on failure (default = False) - - Returns: - return_code - True/False indicating the success/failure of the command - output - The output of the command if it was successful, else empty string - """ - text = '[Exec] ' + command - userlog.debug(text) - returncode = 0 - output = "" - try: - output = subprocess.check_output(command, shell=True) - except subprocess.CalledProcessError as e: - text = "The command " + str(command) + "failed with return code: " + str(e.returncode) - userlog.warning(text) - returncode = e.returncode - if exit_on_failure: - sys.exit(1) - return returncode, output - - -def rmtree_readonly(directory): - """ Deletes a readonly directory tree. - - Args: - directory - tree to delete - """ - def remove_readonly(func, path, _): - os.chmod(path, stat.S_IWRITE) - func(path) - - shutil.rmtree(directory, onerror=remove_readonly) - -def find_all_examples(path): - """ Search the path for examples - - Description: - - Searches the path specified for sub-example folders, ie those containing an - mbed-os.lib file. If found adds the path to the sub-example to a list which is - then returned. - - Args: - path - path to search. - examples - (returned) list of paths to example directories. - - """ - examples = [] - for root, dirs, files in os.walk(path): - if 'mbed-os.lib' in files: - examples += [root] - - return examples - -def upgrade_single_example(example, tag, directory, ref): - """ Update the mbed-os version for a single example - - Description: - - Updates the mbed-os.lib file in the example specified to correspond to the - version specified by the GitHub tag supplied. Also deals with - multiple sub-examples in the GitHub repo, updating them in the same way. - - Args: - example - json example object containing the GitHub repo to update. - tag - GitHub tag corresponding to a version of mbed-os to upgrade to. - directory - directory path for the example. - ref - SHA corresponding to the supplied tag - returns - True if the upgrade was successful, False otherwise. - - """ - cwd = os.getcwd() - os.chdir(directory) - - return_code = False - - if os.path.isfile("mbed-os.lib"): - # Rename command will fail on some OS's if the target file already exist, - # so ensure if it does, it is deleted first. - if os.path.isfile("mbed-os.lib_bak"): - os.remove("mbed-os.lib_bak") - - os.rename("mbed-os.lib", "mbed-os.lib_bak") - else: - userlog.error("Failed to backup mbed-os.lib prior to updating.") - return False - - # mbed-os.lib file contains one line with the following format - # e.g. https://github.com/ARMmbed/mbed-os/#0789928ee7f2db08a419fa4a032fffd9bd477aa7 - lib_re = re.compile('https://github.com/ARMmbed/mbed-os/#[A-Za-z0-9]+') - updated = False - - # Scan through mbed-os.lib line by line - with open('mbed-os.lib_bak', 'r') as ip, open('mbed-os.lib', 'w') as op: - for line in ip: - - opline = line - - regexp = lib_re.match(line) - if regexp: - opline = 'https://github.com/ARMmbed/mbed-os/#' + ref - updated = True - - op.write(opline) - - if updated: - # Setup and run the git add command - cmd = "git add mbed-os.lib" - return_code = run_cmd(cmd) - - os.chdir(cwd) - return not return_code - -def prepare_fork(arm_example): - """ Synchronises a cloned fork to ensure it is up to date with the original. - - Description: - - This function sets a fork of an ARMmbed repo to be up to date with the - repo it was forked from. It does this by hard resetting to the ARMmbed - master branch. - - Args: - arm_example - Full GitHub repo path for original example - - """ - - logstr = "In: " + os.getcwd() - userlog.debug(logstr) - - for cmd in ["git remote add armmbed " + str(arm_example), - "git fetch armmbed", - "git reset --hard armmbed/master", - "git push -f origin"]: - run_cmd(cmd, exit_on_failure=True) - -def prepare_branch(src, dst): - """ Set up at branch ready for use in updating examples - - Description: - - This function checks whether or not the supplied dst branch exists. - If it does not, the branch is created from the src and pushed to the origin. - The branch is then switched to. - - Args: - src - branch to create the dst branch from - dst - branch to update - - """ - - userlog.debug("Preparing branch: %s", dst) - - # Check if branch already exists or not. - # We can use the 'git branch -r' command. This returns all the remote branches for - # the current repo. - # The output consists of a list of lines of the form: - # origin/ - # From these we need to extract just the branch names to a list and then check if - # the specified dst exists in that list - branches = [] - cmd = "git branch -r" - _, output = run_cmd_with_output(cmd, exit_on_failure=True) - - branches = [line.split('/')[1] for line in output.split('\n') if 'origin' in line and not '->' in line] - - if not dst in branches: - - # OOB branch does not exist thus create it, first ensuring we are on - # the src branch and then check it out - - for cmd in ["git checkout " + str(src), - "git checkout -b " + str(dst), - "git push -u origin " + str(dst)]: - - run_cmd(cmd, exit_on_failure=True) - - else: - cmd = "git checkout " + str(dst) - run_cmd(cmd, exit_on_failure=True) - -def upgrade_example(github, example, tag, ref, user, src, dst, template): - """ Upgrade all versions of mbed-os.lib found in the specified example repo - - Description: - - Clone a version of the example specified and upgrade all versions of - mbed-os.lib found within its tree. The version cloned and how it - is upgraded depends on the user, src and dst settings. - 1) user == None - The destination branch will be updated with the version of mbed-os - idenfied by the tag. If the destination branch does not exist then it - will be created from the source branch. - - 2) user != None - The master branch of a fork of the example will be updated with the - version of mbed-os identified by the tag. - - Args: - github - GitHub instance to allow internal git commands to be run - example - json example object containing the GitHub repo to update. - tag - GitHub tag corresponding to a version of mbed-os to upgrade to. - ref - SHA corresponding to the tag - user - GitHub user name - src - branch to create the dst branch from - dst - branch to update - - returns True if the upgrade was successful, False otherwise - """ - - # If a user has not been specified then branch update will be used and thus - # the git user will be ARMmbed. - if not user: - user = 'ARMmbed' - - ret = False - userlog.info("Updating example '%s'", example['name']) - userlog.debug("User: %s", user) - userlog.debug("Src branch: %s", (src or "None")) - userlog.debug("Dst branch: %s", (dst or "None")) - - cwd = os.getcwd() - - update_repo = "https://github.com/" + user + '/' + example['name'] - userlog.debug("Update repository: %s", update_repo) - - # Clone the example repo - clone_cmd = "git clone " + str(update_repo) - return_code = run_cmd(clone_cmd) - - if not return_code: - - # Find all examples - example_directories = find_all_examples(example['name']) - - os.chdir(example['name']) - - # If the user is ARMmbed then a branch is used. - if user == 'ARMmbed': - prepare_branch(src, dst) - else: - prepare_fork(example['github']) - - for example_directory in example_directories: - if not upgrade_single_example(example, tag, os.path.relpath(example_directory, example['name']), ref): - os.chdir(cwd) - return False - - # Setup and run the commit command - commit_cmd = "git commit -m \"Updating mbed-os to " + tag + "\"" - return_code = run_cmd(commit_cmd) - if not return_code: - - # Setup and run the push command - push_cmd = "git push origin" - return_code = run_cmd(push_cmd) - - if not return_code: - # If the user is not ARMmbed then a fork is being used - if user != 'ARMmbed': - - upstream_repo = 'ARMmbed/'+ example['name'] - userlog.debug("Upstream repository: %s", upstream_repo) - # Check access to mbed-os repo - try: - repo = github.get_repo(upstream_repo, False) - - except: - userlog.error("Upstream repo: %s, does not exist - skipping", upstream_repo) - return False - - jinja_loader = FileSystemLoader(template) - jinja_environment = Environment(loader=jinja_loader, - undefined=StrictUndefined) - pr_body = jinja_environment.get_template("pr.tmpl").render(tag=tag) - - # Raise a PR from release-candidate to master - user_fork = user + ':master' - try: - pr = repo.create_pull(title='Updating mbed-os to ' + tag, head=user_fork, base='master', body=pr_body) - ret = True - except GithubException as e: - # Default to False - userlog.error("Pull request creation failed with error: %s", e) - else: - ret = True - else: - userlog.error("Git push command failed.") - else: - userlog.error("Git commit command failed.") - else: - userlog.error("Git clone %s failed", update_repo) - - os.chdir(cwd) - return ret - -def create_work_directory(path): - """ Create a new directory specified in 'path', overwrite if the directory already - exists. - - Args: - path - directory path to be created. - - """ - if os.path.exists(path): - userlog.info("'%s' directory already exists. Deleting...", path) - rmtree_readonly(path) - - os.makedirs(path) - -def check_update_status(examples, github, tag): - """ Check the status of previously raised update pull requests - - Args: - examples - list of examples which should have had PRs raised against them. - github - github rest API instance - tag - release tag used for the update - - """ - - for example in examples: - - repo_name = ''.join(['ARMmbed/', example['name']]) - try: - repo = github.get_repo(repo_name, False) - - except Exception as exc: - text = "Cannot access: " + str(repo_name) - userlog.error(text) - userlog.exception(exc) - sys.exit(1) - - # Create the full repository filter component - org_str = ''.join(['repo:ARMmbed/', example['name']]) - filt = ' '.join([org_str, 'is:pr', tag]) - merged = False - - issues = github.search_issues(query=(filt)) - pr_list = [repo.get_pull(issue.number) for issue in issues] - - # Should only be one matching PR but just in case, go through paginated list - for pr in pr_list: - if pr.merged: - userlog.info("%s - '%s': MERGED", example['name'], pr.title) - elif pr.state == 'open': - userlog.info("%s - '%s': PENDING", example['name'], pr.title) - elif pr.state == 'closed': - userlog.info("%s - '%s': CLOSED NOT MERGED", example['name'], pr.title) - else: - userlog.error("%s: Cannot find a pull request for %s", example['name'], tag) - -if __name__ == '__main__': - - parser = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('-c', '--config_file', help="Path to the configuration file (default is 'examples.json')", default='examples.json') - parser.add_argument('-T', '--github_token', help="GitHub token for secure access") - - exclusive = parser.add_mutually_exclusive_group(required=True) - exclusive.add_argument('-f', '--fork', help="Update a fork", action='store_true') - exclusive.add_argument('-b', '--branch', help="Update a branch", action='store_true') - exclusive.add_argument('-s', '--status', help="Show examples update status", action='store_true') - - args = parser.parse_args() - - # Load the config file - with open(os.path.join(os.path.dirname(__file__), args.config_file)) as config: - if not config: - userlog.error("Failed to load config file '%s'", args.config_file) - sys.exit(1) - json_data = json.load(config) - - - github = Github(args.github_token) - config = json_data['update-config'] - tag = config['tag'] - - user = None - src = "master" - dst = None - - if args.status: - - # This option should only be called after an update has been performed - check_update_status(json_data['examples'], github, tag) - exit(0) - - # Create working directory - create_work_directory('examples') - - if args.fork: - user = config['via-fork']['github-user'] - elif args.branch: - src = config['via-branch']['src-branch'] - dst = config['via-branch']['dst-branch'] - else: - userlog.error("Must specify either -f or -b command line option") - exit(1) - - # Get the github sha corresponding to the specified mbed-os tag - cmd = "git rev-list -1 " + tag - return_code, ref = run_cmd_with_output(cmd) - - if return_code: - userlog.error("Could not obtain SHA for tag: %s", tag) - sys.exit(1) - - # Loop through the examples - failures = [] - successes = [] - results = {} - template = dirname(abspath(__file__)) - - os.chdir('examples') - - for example in json_data['examples']: - # Determine if this example should be updated and if so update any found - # mbed-os.lib files. - - result = upgrade_example(github, example, tag, ref, user, src, dst, template) - - if result: - successes += [example['name']] - else: - failures += [example['name']] - - os.chdir('../') - - # Finish the script and report the results - userlog.info("Finished updating examples") - if successes: - for success in successes: - userlog.info(" SUCCEEDED: %s", success) - - if failures: - for fail in failures: - userlog.info(" FAILED: %s", fail)