Merge pull request #4093 from adbridge/update

Update.py: New feature - update a branch instead of a fork, plus general improvements.

tools/test/examples/update.py

@@ -1,8 +1,44 @@
 #!/usr/bin/env python
 
+# This script is used to update the version of mbed-os used within a specified set of example
+# applications. The list of examples to be updated lives in the examples.json file and is 
+# shared with the examples.py script. Logging is used to provide varying levels of output
+# during execution.
+#
+# There are two modes that can be used:
+# 1) Update the ARMmbed/master branch of the specified example
+# 
+#    This is done by updating a user fork of the example and then raising a pull request
+#    against ARMmbed/master.
+#
+# 2) Update a different ARMmbed branch of the specified example
+#
+#    A branch to update is specified. If it doesn't already exist then it is first created.
+#    This branch will be updated and the change automatically pushed. 
+#
+# Command usage:
+#
+# update.py -c <config file> - T <github_token> -l <logging level> -U <github user> -b <branch> <tag>
+#
+# Where:
+# -c <config file> - Optional path to an examples file.
+#                    If not proved the default is 'examples.json'
+# -T <github_token> - GitHub token for secure access (required)
+# -l <logging level> - Optional Level for providing logging output. Can be one of,
+#                      CRITICAL, ERROR, WARNING, INFO, DEBUG
+#                      If not provided the default is 'INFO'
+# -U <github_user> - GitHub user for forked repos
+# -b <branch> - Branch to be updated
+#
+#               NOTE only one of -U or -b can be specified.
+#
+# <tag> mbed-os tag to which all examples will be updated
+#
+
 import os
 from os.path import dirname, abspath, basename
 import sys
+import logging
 import argparse
 import json
 import subprocess
@@ -17,43 +53,64 @@ sys.path.insert(0, ROOT)
 import examples_lib as lib
 from examples_lib import SUPPORTED_TOOLCHAINS
 
-def run_cmd(command, print_warning_on_fail=True):
+def run_cmd(command, exit_on_failure=False):
-    """ Takes the command specified and runs it in a sub-process, obtaining the return code.
+    """ Run a system command and return the result status
-        
+    
-    Args:
+    Description:
-    command - command to run, provided as a list of individual fields which are combined into a 
+    
-              single command before passing to the sub-process call.
+    Passes a command to the system and returns a True/False result, once the 
-    return_code - result of the command.
+    command has been executed, indicating success/failure. Commands are passed
+    as a list of tokens. 
+    E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v']
 
+    Args:
+    command - system command as a list of tokens
+    exit_on_failure - If True exit the program on failure (default = False)
+    
+    Returns:
+    return_code - True/False indicating the success/failure of the command
     """
-    print('[Exec] %s' % ' '.join(command))
+    update_log.debug('[Exec] %s', ' '.join(command))
-    return_code = subprocess.call(command)
+    return_code = subprocess.call(command, shell=True)
     
     if return_code:
-        print("The command '%s' failed with return code: %s" % (' '.join(command), return_code))
+        update_log.warning("Command '%s' failed with return code: %s", 
-        print("Ignoring and moving on to the next example")
+                           ' '.join(command), return_code)
+        if exit_on_failure:
+            sys.exit(1)
     
     return return_code
-    
-def run_cmd_with_output(command, print_warning_on_fail=True):
-    """ Takes the command specified and runs it in a sub-process, obtaining the return code 
-        and the returned output.
-        
-    Args:
-    command - command to run, provided as a list of individual fields which are combined into a 
-              single command before passing to the sub-process call.
-    return_code - result of the command.
-    output - the output of the command
 
+def run_cmd_with_output(command, exit_on_failure=False):
+    """ Run a system command and return the result status plus output  
+    
+    Description:
+        
+    Passes a command to the system and returns a True/False result once the 
+    command has been executed, indicating success/failure. If the command was 
+    successful then the output from the command is returned to the caller.
+    Commands are passed as a list of tokens. 
+    E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v']
+
+    Args:
+    command - system command as a list of tokens
+    exit_on_failure - If True exit the program on failure (default = False)
+    
+    Returns:
+    returncode - True/False indicating the success/failure of the command
+    output - The output of the command if it was successful, else empty string
     """
-    print('[Exec] %s' % ' '.join(command))
+    update_log.debug('[Exec] %s', ' '.join(command))
     returncode = 0
-    output = None
+    output = ""
     try:
-        output = subprocess.check_output(command)
+        output = subprocess.check_output(command, shell=True)
     except subprocess.CalledProcessError as e:
-        print("The command '%s' failed with return code: %s" % (' '.join(command), e.returncode))
+        update_log.warning("Command '%s' failed with return code: %s", 
+                           ' '.join(command), e.returncode)
         returncode = e.returncode
+        if exit_on_failure:
+            sys.exit(1)
     return returncode, output
 
 def rmtree_readonly(directory):
@@ -69,9 +126,13 @@ def rmtree_readonly(directory):
     shutil.rmtree(directory, onerror=remove_readonly)
 
 def find_all_examples(path):
-    """ Searches the path specified for sub-example folders, ie those containing an
+    """ Search the path for examples
-        mbed-os.lib file. If found adds the path to the sub-example to a list which is 
+    
-        then returned.
+    Description:
+    
+    Searches the path specified for sub-example folders, ie those containing an
+    mbed-os.lib file. If found adds the path to the sub-example to a list which is 
+    then returned.
         
     Args:
     path - path to search.
@@ -86,9 +147,13 @@ def find_all_examples(path):
     return examples
 
 def upgrade_single_example(example, tag, directory, ref):
-    """ Updates the mbed-os.lib file in the example specified to correspond to the 
+    """ Update the mbed-os version for a single example
-        version specified by the GitHub tag supplied. Also deals with 
+    
-        multiple sub-examples in the GitHub repo, updating them in the same way.
+    Description:
+    
+    Updates the mbed-os.lib file in the example specified to correspond to the 
+    version specified by the GitHub tag supplied. Also deals with 
+    multiple sub-examples in the GitHub repo, updating them in the same way.
         
     Args:
     example - json example object containing the GitHub repo to update.
@@ -111,7 +176,7 @@ def upgrade_single_example(example, tag, directory, ref):
         
         os.rename("mbed-os.lib", "mbed-os.lib_bak")
     else:
-        print("!! Error trying to backup mbed-os.lib prior to updating.")
+        update_log.error("Failed to backup mbed-os.lib prior to updating.")
         return False
     
     # mbed-os.lib file contains one line with the following format
@@ -142,6 +207,12 @@ def upgrade_single_example(example, tag, directory, ref):
 
 def prepare_fork(arm_example):
     """ Synchronises a cloned fork to ensure it is up to date with the original. 
+
+    Description:
+    
+    This function sets a fork of an ARMmbed repo to be up to date with the 
+    repo it was forked from. It does this by hard resetting to the ARMmbed
+    master branch.    
         
     Args:
     arm_example - Full GitHub repo path for original example 
@@ -149,52 +220,120 @@ def prepare_fork(arm_example):
     
     """
 
-    print "In " + os.getcwd()
+    logstr = "In: " + os.getcwd()
+    update_log.debug(logstr)
 
     for cmd in [['git', 'remote', 'add', 'armmbed', arm_example],
                 ['git', 'fetch', 'armmbed'],
                 ['git', 'reset', '--hard', 'armmbed/master'],
                 ['git', 'push', '-f', 'origin']]:
         if run_cmd(cmd):
-            print("preparation of the fork failed!")
+            update_log.error("Fork preparation failed")
             return False
     return True
 
+def prepare_branch(branch):
+    """ Set up at branch ready for use in updating examples 
+        
+    Description:
+    
+    This function checks whether or not the supplied branch exists.
+    If it does not, the branch is created and pushed to the origin.
+    The branch is then switched to.
+    
+    Args:
+    arm_example - Full GitHub repo path for original example 
+    ret - True if the fork was synchronised successfully, False otherwise
+    
+    """
 
-def upgrade_example(github, example, tag, user, ref):
+    update_log.debug("Preparing branch: %s", branch)
-    """ Clone a fork of the example specified.
+
-        Ensures the fork is up to date with the original and then and updates the associated 
+    # Check if branch already exists or not.
-        mbed-os.lib file on that fork to correspond to the version specified by the GitHub tag supplied. 
+    cmd = ['git', 'branch']
-        Also deals with multiple sub-examples in the GitHub repo, updating them in the same way.
+    return_code, output = run_cmd_with_output(cmd)
-        The updates are pushed to the forked repo.
+
-        Finally a PR is raised against the original example repo for the changes.
+    if not branch in output:
+        # OOB branch does not exist thus create it and then check it out
+        cmd = ['git', 'checkout', '-b', branch]
+        return_code = run_cmd(cmd)
+        if not return_code:
+
+            # Push new branch upstream
+            cmd = ['git', 'push', '-u', 'origin', branch]
+            return_code = run_cmd(cmd)
+    else: 
+        cmd = ['git', 'checkout', branch]
+        return_code = run_cmd(cmd)
+
+    if return_code:
+        update_log.error("Failed to prepare branch: %s", branch)
+        return False
+        
+    return True
+    
+def upgrade_example(github, example, tag, ref,
+                    user='ARMmbed', branch='master'):
+    """ Upgrade all versions of mbed-os.lib found in the specified example repo
+    
+    Description:
+    
+    Clone a version of the example specified and upgrade all versions of 
+    mbed-os.lib found within its tree.  The version cloned and how it 
+    is upgraded depends on the user and branch specified. Only two options 
+    are valid:
+    1) ARMmbed + non master branch
+       This option will update the branch directly in the ARMmbed repo. If the 
+       branch does not exist it will be first created.
+
+    2) alternative user + master branch
+                
+       This option assumes that a fork of the repo exists in the specified user's
+       account. The fork will first be updated so that it is up to date with the 
+       upstream  version , then the fork will be updated and a PR raised against 
+       the upstream ie ARMmbed repo.
         
     Args:
     github - GitHub instance to allow internal git commands to be run
     example - json example object containing the GitHub repo to update.
     tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
-    user - GitHub user name
     ref - SHA corresponding to the tag
+    user - GitHub user name (defaults to 'ARMmbed' if not supplied)
+    branch - branch to update (defaults to 'master' if not supplied)
     
+    returns True if the upgrade was successful, False otherwise
     """
+                
     ret = False
-    print("\nUpdating example '%s'" % example['name'])
+    update_log.info("Updating example '%s'", example['name'])
+    update_log.debug("User: %s", user)
+    update_log.debug("Branch: %s", branch)
+
+    # First check validity of user/branch combination
+    if ((user == 'ARMmbed' and branch == 'master') or 
+        (user != 'ARMmbed' and branch != 'master')):
+        update_log.error("Invalid user/branch combination")
+        return False
+
     cwd = os.getcwd()
 
-    full_repo_name = 'ARMmbed/'+ example['name']
+    upstream_repo = 'ARMmbed/'+ example['name']
-    fork = "https://github.com/" + user + '/' + example['name'] 
+    update_repo = "https://github.com/" + user + '/' + example['name'] 
+
+    update_log.debug("Upstream repository: %s", upstream_repo)
+    update_log.debug("Update repository: %s", update_repo)
 
     # Check access to mbed-os repo
     try:
-        repo = github.get_repo(full_repo_name, False)
+        repo = github.get_repo(upstream_repo, False)
 
     except:
-        print("\t\t!! Repo does not exist - skipping\n")
+        update_log.error("Upstream repo: %s, does not exist - skipping", upstream_repo)
         return False
 
 
-    # Clone the forked example repo
+    # Clone the example repo
-    clone_cmd = ['git', 'clone', fork]
+    clone_cmd = ['git', 'clone', update_repo]
     return_code = run_cmd(clone_cmd)
     
     if not return_code:
@@ -204,9 +343,14 @@ def upgrade_example(github, example, tag, user, ref):
         
         os.chdir(example['name'])
         
-        # checkout and synchronise the release-candidate branch
+        # If the user is not the default, then a fork will be used. Thus
-        prepare_fork(example['github'])
+        # synchronise the user fork with the upstream 
-        
+        if user != 'ARMmbed':
+            prepare_fork(example['github'])
+
+        if branch != 'master':
+            prepare_branch(branch)    
+            
         for example_directory in example_directories:
             if not upgrade_single_example(example, tag, os.path.relpath(example_directory, example['name']), ref):
                 os.chdir(cwd)
@@ -225,23 +369,24 @@ def upgrade_example(github, example, tag, user, ref):
             return_code = run_cmd(push_cmd)
             
             if not return_code:
-                body = "Please test/merge this PR and then tag Master with " + tag
+                if user != 'ARMmbed':                                
-                # Raise a PR from release-candidate to master
+                    body = "Please test/merge this PR and then tag Master with " + tag
-                user_fork = user + ':master' 
+                    # Raise a PR from release-candidate to master
-                try:
+                    user_fork = user + ':master' 
-                    pr = repo.create_pull(title='Updating mbed-os to ' + tag, head=user_fork, base='master', body=body)
+                    try:
-                    ret = True
+                        pr = repo.create_pull(title='Updating mbed-os to ' + tag, head=user_fork, base='master', body=body)
-                except GithubException as e:
+                        ret = True
-                    # Default to False
+                    except GithubException as e:
-                    print("Creation of Pull Request from release-candidate to master failed with the following error!")
+                        # Default to False
-                    print e
+                         update_log.error("Pull request creation failed with error: %s", e)
+                else:
+                    ret = True                    
             else:
-                print("!!! Git push command failed.")
+                update_log.error("Git push command failed.")
         else:
-            print("!!! Git commit command failed.")
+            update_log.error("Git commit command failed.")
     else:
-        print("!!! Could not clone user fork %s\n" % fork)
+        update_log.error("Git clone %s failed", update_repo)
-
 
     os.chdir(cwd)
     return ret
@@ -255,71 +400,43 @@ def create_work_directory(path):
     
     """
     if os.path.exists(path):
-        print("'%s' directory already exists. Deleting..." % path)
+        update_log.info("'%s' directory already exists. Deleting...", path)
         rmtree_readonly(path)
     
     os.makedirs(path)
 
-def test_compile(config, tag):
+if __name__ == '__main__':
-    """ For each example repo identified in the config json object, clone, update mbed-os to
-        the specified tag and then compile for all supported toolchains.
-        
-    Args:
-    config - the json object imported from the file. 
-    tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
-    results - summary of compilation results. 
-    
-    """
-    # Create work directories
-    create_work_directory('test_compile')
-    
-    # Loop through the examples
-    results = {}
-    os.chdir('test_compile')
-
-    lib.source_repos(config)    
-    lib.update_mbedos_version(config, tag)
-    results = lib.compile_repos(config, SUPPORTED_TOOLCHAINS)
-    os.chdir("..")
-
-    return results
-    
-
-def main(arguments):
-    """ Will update any mbed-os.lib files found in the example list specified by the config file.
-        If no config file is specified the default 'examples.json' is used. 
-        The update is done by cloning a fork of each example (the fork must be present in the 
-        github account specified by the github user parameter). The fork is searched for any
-        mbed-os.lib files and each one found is updated with the SHA corresponding to the supplied
-        github tag. A pull request is then made from the fork to the original example repo.
-        
-    Args:
-    tag - tag to update the mbed-os.lib to. E.g. mbed-os-5.3.1
-    github_token - Pre-authorised token to allow github access
-    github_user - github username whose account contains the example forks
-    config_file - optional parameter to specify a list of examples
-
-    """
 
     parser = argparse.ArgumentParser(description=__doc__,
                                      formatter_class=argparse.RawDescriptionHelpFormatter)
     parser.add_argument('tag', help="mbed-os tag to which all examples will be updated")
     parser.add_argument('-c', '--config_file', help="Path to the configuration file (default is 'examples.json')", default='examples.json')
     parser.add_argument('-T', '--github_token', help="GitHub token for secure access")
-    parser.add_argument('-U', '--github_user', help="GitHub user for forked repos")
+    parser.add_argument('-l', '--log-level', 
+                        help="Level for providing logging output", 
+                        default='INFO')
     
-    args = parser.parse_args(arguments)
+    exclusive = parser.add_mutually_exclusive_group(required=True)
+    exclusive.add_argument('-U', '--github_user', help="GitHub user for forked repos, mutually exclusive to branch option")
+    exclusive.add_argument('-b', '--branch', help="Branch to be updated, mutually exclusive to user option")
+    
+    args = parser.parse_args()
+
+    default = getattr(logging, 'INFO')
+    level = getattr(logging, args.log_level.upper(), default)
+        
+    # Set logging level
+    logging.basicConfig(level=level)
+
+    update_log = logging.getLogger("Update")
 
-    cfg = os.path.join(os.path.dirname(__file__), args.config_file)
-    
     # Load the config file
-    config = json.load(open(os.path.join(os.path.dirname(__file__),
+    with open(os.path.join(os.path.dirname(__file__), args.config_file)) as config:
-                             args.config_file)))
+        if not config:
-    
+            update_log.error("Failed to load config file '%s'", args.config_file)
-    if not config:
+            sys.exit(1)
-        print("Failed to load config file '%s'" % args.config_file)
+        json_data = json.load(config)
-        sys.exit(1)
+        
-    
     # Create working directory
     create_work_directory('examples')
 
@@ -330,7 +447,7 @@ def main(arguments):
     return_code, ref = run_cmd_with_output(cmd) 
 
     if return_code:
-        print("Could not obtain SHA for tag: %s\n" % args.tag)
+        update_log.error("Could not obtain SHA for tag: %s",  args.tag)
         sys.exit(1)
 
     # Loop through the examples
@@ -338,12 +455,18 @@ def main(arguments):
     successes = []
     results = {}
     os.chdir('examples')
-    
-    for example in config['examples']:
-        # Determine if this example should be updated and if so update any found 
-        # mbed-os.lib files.
 
-        if upgrade_example(github, example, args.tag, args.github_user, ref):
+    for example in json_data['examples']:
+        # Determine if this example should be updated and if so update any found 
+        # mbed-os.lib files. 
+        
+        # Only user or branch can be specified on the command line
+        if args.github_user:
+            result = upgrade_example(github, example, args.tag, ref, user=args.github_user)
+        else:
+            result = upgrade_example(github, example, args.tag, ref, branch=args.branch)
+            
+        if result:
             successes += [example['name']]
         else:
             failures += [example['name']]
@@ -351,17 +474,11 @@ def main(arguments):
     os.chdir('../')
     
     # Finish the script and report the results
-    print(os.linesep + os.linesep +'Finished updating examples!' + os.linesep)
+    update_log.info("Finished updating examples")
-    
     if successes:
-        print('\nThe following examples updated successfully:')
         for success in successes:
-            print('    - %s' % success)
+            update_log.info(" SUCCEEDED: %s", success)
     
     if failures:
-        print('\nThe following examples were not updated:')
         for fail in failures:
-            print('    - %s' % fail)
+            update_log.info(" FAILED: %s", fail)
-
-if __name__ == '__main__':
-    sys.exit(main(sys.argv[1:]))