ARMmbed
/
mbed-os
mirror of https://github.com/ARMmbed/mbed-os.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #5224 from adbridge/master 

Fix branch updates and make Linux compatible
pull/4955/head
Jimmy Brisson 2017-10-13 09:22:59 -05:00 committed by GitHub
parent 2f25744b07 31407a17ca
commit 5c8ef54e2f
1 changed files with 82 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
tools/test/examples/update.py
View File

@ -42,9 +42,6 @@
# -c <config file> - Optional path to an examples file. # -c <config file> - Optional path to an examples file.
# If not proved the default is 'examples.json' # If not proved the default is 'examples.json'
# -T <github_token> - GitHub token for secure access (required) # -T <github_token> - GitHub token for secure access (required)
# -l <logging level> - Optional Level for providing logging output. Can be one of,
# CRITICAL, ERROR, WARNING, INFO, DEBUG
# If not provided the default is 'INFO'
# -f - Update forked repos. This will use the 'github-user' parameter in # -f - Update forked repos. This will use the 'github-user' parameter in
# the 'via-fork' section. # the 'via-fork' section.
# -b - Update branched repos. This will use the "src-branch" and # -b - Update branched repos. This will use the "src-branch" and
@ -75,16 +72,34 @@ sys.path.insert(0, ROOT)
import examples_lib as lib import examples_lib as lib
from examples_lib import SUPPORTED_TOOLCHAINS from examples_lib import SUPPORTED_TOOLCHAINS
def run_cmd(command, exit_on_failure=False): userlog = logging.getLogger("Update")
""" Run a system command and return the result status
Description:
Passes a command to the system and returns a True/False result, once the
command has been executed, indicating success/failure. Commands are passed
as a list of tokens.
E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v']
# Set logging level
userlog.setLevel(logging.DEBUG)
# Everything is output to the log file
logfile = os.path.join(os.getcwd(), 'update.log')
fh = logging.FileHandler(logfile)
fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(name)s: %(levelname)s - %(message)s')
ch.setFormatter(formatter)
fh.setFormatter(formatter)
# add the handlers to the logger
userlog.addHandler(fh)
userlog.addHandler(ch)
def run_cmd(command, exit_on_failure=False):
""" Run a system command returning a status result
This is just a wrapper for the run_cmd_with_output() function, but
only returns the status of the call.
Args: Args:
command - system command as a list of tokens command - system command as a list of tokens
exit_on_failure - If True exit the program on failure (default = False) exit_on_failure - If True exit the program on failure (default = False)
@ -92,49 +107,41 @@ def run_cmd(command, exit_on_failure=False):
Returns: Returns:
return_code - True/False indicating the success/failure of the command return_code - True/False indicating the success/failure of the command
""" """
update_log.debug('[Exec] %s', ' '.join(command)) return_code, _ = run_cmd_with_output(command, exit_on_failure)
return_code = subprocess.call(command, shell=True)
if return_code:
update_log.warning("Command '%s' failed with return code: %s",
' '.join(command), return_code)
if exit_on_failure:
sys.exit(1)
return return_code return return_code
def run_cmd_with_output(command, exit_on_failure=False): def run_cmd_with_output(command, exit_on_failure=False):
""" Run a system command and return the result status plus output """ Run a system command returning a status result and any command output
Description:
Passes a command to the system and returns a True/False result once the Passes a command to the system and returns a True/False result once the
command has been executed, indicating success/failure. If the command was command has been executed, indicating success/failure. If the command was
successful then the output from the command is returned to the caller. successful then the output from the command is returned to the caller.
Commands are passed as a list of tokens. Commands are passed as a string.
E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v'] E.g. The command 'git remote -v' would be passed in as "git remote -v"
Args: Args:
command - system command as a list of tokens command - system command as a string
exit_on_failure - If True exit the program on failure (default = False) exit_on_failure - If True exit the program on failure (default = False)
Returns: Returns:
returncode - True/False indicating the success/failure of the command return_code - True/False indicating the success/failure of the command
output - The output of the command if it was successful, else empty string output - The output of the command if it was successful, else empty string
""" """
update_log.debug('[Exec] %s', ' '.join(command)) text = '[Exec] ' + command
userlog.debug(text)
returncode = 0 returncode = 0
output = "" output = ""
try: try:
output = subprocess.check_output(command, shell=True) output = subprocess.check_output(command, shell=True)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
update_log.warning("Command '%s' failed with return code: %s", text = "The command " + str(command) + "failed with return code: " + str(e.returncode)
' '.join(command), e.returncode) userlog.warning(text)
returncode = e.returncode returncode = e.returncode
if exit_on_failure: if exit_on_failure:
sys.exit(1) sys.exit(1)
return returncode, output return returncode, output
def rmtree_readonly(directory): def rmtree_readonly(directory):
""" Deletes a readonly directory tree. """ Deletes a readonly directory tree.
@ -198,7 +205,7 @@ def upgrade_single_example(example, tag, directory, ref):
os.rename("mbed-os.lib", "mbed-os.lib_bak") os.rename("mbed-os.lib", "mbed-os.lib_bak")
else: else:
update_log.error("Failed to backup mbed-os.lib prior to updating.") userlog.error("Failed to backup mbed-os.lib prior to updating.")
return False return False
# mbed-os.lib file contains one line with the following format # mbed-os.lib file contains one line with the following format
@ -221,7 +228,7 @@ def upgrade_single_example(example, tag, directory, ref):
if updated: if updated:
# Setup and run the git add command # Setup and run the git add command
cmd = ['git', 'add', 'mbed-os.lib'] cmd = "git add mbed-os.lib"
return_code = run_cmd(cmd) return_code = run_cmd(cmd)
os.chdir(cwd) os.chdir(cwd)
@ -242,12 +249,12 @@ def prepare_fork(arm_example):
""" """
logstr = "In: " + os.getcwd() logstr = "In: " + os.getcwd()
update_log.debug(logstr) userlog.debug(logstr)
for cmd in [['git', 'remote', 'add', 'armmbed', arm_example], for cmd in ["git remote add armmbed " + str(arm_example),
['git', 'fetch', 'armmbed'], "git fetch armmbed",
['git', 'reset', '--hard', 'armmbed/master'], "git reset --hard armmbed/master",
['git', 'push', '-f', 'origin']]: "git push -f origin"]:
run_cmd(cmd, exit_on_failure=True) run_cmd(cmd, exit_on_failure=True)
def prepare_branch(src, dst): def prepare_branch(src, dst):
@ -265,25 +272,34 @@ def prepare_branch(src, dst):
""" """
update_log.debug("Preparing branch: %s", dst) userlog.debug("Preparing branch: %s", dst)
# Check if branch already exists or not. # Check if branch already exists or not.
cmd = ['git', 'branch'] # We can use the 'git branch -r' command. This returns all the remote branches for
# the current repo.
# The output consists of a list of lines of the form:
# origin/<branch>
# From these we need to extract just the branch names to a list and then check if
# the specified dst exists in that list
branches = []
cmd = "git branch -r"
_, output = run_cmd_with_output(cmd, exit_on_failure=True) _, output = run_cmd_with_output(cmd, exit_on_failure=True)
if not dst in output: branches = [line.split('/')[1] for line in output.split('\n') if 'origin' in line and not '->' in line]
if not dst in branches:
# OOB branch does not exist thus create it, first ensuring we are on # OOB branch does not exist thus create it, first ensuring we are on
# the src branch and then check it out # the src branch and then check it out
for cmd in [['git', 'checkout', src], for cmd in ["git checkout " + str(src),
['git', 'checkout', '-b', dst], "git checkout -b " + str(dst),
['git', 'push', '-u', 'origin', dst]]: "git push -u origin " + str(dst)]:
run_cmd(cmd, exit_on_failure=True) run_cmd(cmd, exit_on_failure=True)
else: else:
cmd = ['git', 'checkout', dst] cmd = "git checkout " + str(dst)
run_cmd(cmd, exit_on_failure=True) run_cmd(cmd, exit_on_failure=True)
def upgrade_example(github, example, tag, ref, user, src, dst, template): def upgrade_example(github, example, tag, ref, user, src, dst, template):
@ -321,18 +337,18 @@ def upgrade_example(github, example, tag, ref, user, src, dst, template):
user = 'ARMmbed' user = 'ARMmbed'
ret = False ret = False
update_log.info("Updating example '%s'", example['name']) userlog.info("Updating example '%s'", example['name'])
update_log.debug("User: %s", user) userlog.debug("User: %s", user)
update_log.debug("Src branch: %s", (src or "None")) userlog.debug("Src branch: %s", (src or "None"))
update_log.debug("Dst branch: %s", (dst or "None")) userlog.debug("Dst branch: %s", (dst or "None"))
cwd = os.getcwd() cwd = os.getcwd()
update_repo = "https://github.com/" + user + '/' + example['name'] update_repo = "https://github.com/" + user + '/' + example['name']
update_log.debug("Update repository: %s", update_repo) userlog.debug("Update repository: %s", update_repo)
# Clone the example repo # Clone the example repo
clone_cmd = ['git', 'clone', update_repo] clone_cmd = "git clone " + str(update_repo)
return_code = run_cmd(clone_cmd) return_code = run_cmd(clone_cmd)
if not return_code: if not return_code:
@ -353,16 +369,13 @@ def upgrade_example(github, example, tag, ref, user, src, dst, template):
os.chdir(cwd) os.chdir(cwd)
return False return False
# Setup the default commit message
commit_message = 'Updating mbed-os to ' + tag
# Setup and run the commit command # Setup and run the commit command
commit_cmd = ['git', 'commit', '-m', commit_message] commit_cmd = "git commit -m \"Updating mbed-os to " + tag + "\""
return_code = run_cmd(commit_cmd) return_code = run_cmd(commit_cmd)
if not return_code: if not return_code:
# Setup and run the push command # Setup and run the push command
push_cmd = ['git', 'push', 'origin'] push_cmd = "git push origin"
return_code = run_cmd(push_cmd) return_code = run_cmd(push_cmd)
if not return_code: if not return_code:
@ -370,13 +383,13 @@ def upgrade_example(github, example, tag, ref, user, src, dst, template):
if user != 'ARMmbed': if user != 'ARMmbed':
upstream_repo = 'ARMmbed/'+ example['name'] upstream_repo = 'ARMmbed/'+ example['name']
update_log.debug("Upstream repository: %s", upstream_repo) userlog.debug("Upstream repository: %s", upstream_repo)
# Check access to mbed-os repo # Check access to mbed-os repo
try: try:
repo = github.get_repo(upstream_repo, False) repo = github.get_repo(upstream_repo, False)
except: except:
update_log.error("Upstream repo: %s, does not exist - skipping", upstream_repo) userlog.error("Upstream repo: %s, does not exist - skipping", upstream_repo)
return False return False
jinja_loader = FileSystemLoader(template) jinja_loader = FileSystemLoader(template)
@ -391,15 +404,15 @@ def upgrade_example(github, example, tag, ref, user, src, dst, template):
ret = True ret = True
except GithubException as e: except GithubException as e:
# Default to False # Default to False
update_log.error("Pull request creation failed with error: %s", e) userlog.error("Pull request creation failed with error: %s", e)
else: else:
ret = True ret = True
else: else:
update_log.error("Git push command failed.") userlog.error("Git push command failed.")
else: else:
update_log.error("Git commit command failed.") userlog.error("Git commit command failed.")
else: else:
update_log.error("Git clone %s failed", update_repo) userlog.error("Git clone %s failed", update_repo)
os.chdir(cwd) os.chdir(cwd)
return ret return ret
@ -413,7 +426,7 @@ def create_work_directory(path):
""" """
if os.path.exists(path): if os.path.exists(path):
update_log.info("'%s' directory already exists. Deleting...", path) userlog.info("'%s' directory already exists. Deleting...", path)
rmtree_readonly(path) rmtree_readonly(path)
os.makedirs(path) os.makedirs(path)
@ -424,9 +437,6 @@ if __name__ == '__main__':
formatter_class=argparse.RawDescriptionHelpFormatter) formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-c', '--config_file', help="Path to the configuration file (default is 'examples.json')", default='examples.json') parser.add_argument('-c', '--config_file', help="Path to the configuration file (default is 'examples.json')", default='examples.json')
parser.add_argument('-T', '--github_token', help="GitHub token for secure access") parser.add_argument('-T', '--github_token', help="GitHub token for secure access")
parser.add_argument('-l', '--log-level',
help="Level for providing logging output",
default='INFO')
exclusive = parser.add_mutually_exclusive_group(required=True) exclusive = parser.add_mutually_exclusive_group(required=True)
exclusive.add_argument('-f', '--fork', help="Update a fork", action='store_true') exclusive.add_argument('-f', '--fork', help="Update a fork", action='store_true')
@ -434,18 +444,10 @@ if __name__ == '__main__':
args = parser.parse_args() args = parser.parse_args()
default = getattr(logging, 'INFO')
level = getattr(logging, args.log_level.upper(), default)
# Set logging level
logging.basicConfig(level=level)
update_log = logging.getLogger("Update")
# Load the config file # Load the config file
with open(os.path.join(os.path.dirname(__file__), args.config_file)) as config: with open(os.path.join(os.path.dirname(__file__), args.config_file)) as config:
if not config: if not config:
update_log.error("Failed to load config file '%s'", args.config_file) userlog.error("Failed to load config file '%s'", args.config_file)
sys.exit(1) sys.exit(1)
json_data = json.load(config) json_data = json.load(config)
@ -470,11 +472,11 @@ if __name__ == '__main__':
exit(1) exit(1)
# Get the github sha corresponding to the specified mbed-os tag # Get the github sha corresponding to the specified mbed-os tag
cmd = ['git', 'rev-list', '-1', tag] cmd = "git rev-list -1 " + tag
return_code, ref = run_cmd_with_output(cmd) return_code, ref = run_cmd_with_output(cmd)
if return_code: if return_code:
update_log.error("Could not obtain SHA for tag: %s", tag) userlog.error("Could not obtain SHA for tag: %s", tag)
sys.exit(1) sys.exit(1)
# Loop through the examples # Loop through the examples
@ -499,11 +501,11 @@ if __name__ == '__main__':
os.chdir('../') os.chdir('../')
# Finish the script and report the results # Finish the script and report the results
update_log.info("Finished updating examples") userlog.info("Finished updating examples")
if successes: if successes:
for success in successes: for success in successes:
update_log.info(" SUCCEEDED: %s", success) userlog.info(" SUCCEEDED: %s", success)
if failures: if failures:
for fail in failures: for fail in failures:
update_log.info(" FAILED: %s", fail) userlog.info(" FAILED: %s", fail)