mirror of https://github.com/ARMmbed/mbed-os.git
Martin Kojtal
GitHub
commit
998d85f80b
|
|
@ -16,6 +16,7 @@ Note: Only files present in folder will be copied, directories inside the folder
|
|||
`commit_sha` is list of commits present in mbed-os repo. These commits will be applied after copying files and folders listed above.Each commit in the commit_sha list is cherry-picked and applied with the -x option, which records the SHA of the source commit in the commit message.
|
||||
Note: You must resolve any conflicts that arise during this cherry-pick process. Make sure that the "(cherry picked from commit ...)" statement is present in the commit message. Re-execute the python script to apply rest of the SHA commits.
|
||||
|
||||
```json
|
||||
{
|
||||
"files" : [
|
||||
{
|
||||
|
|
@ -41,9 +42,10 @@ Note: You must resolve any conflicts that arise during this cherry-pick process.
|
|||
],
|
||||
"commit_sha" : [
|
||||
"428acae1b2ac15c3ad523e8d40755a9301220822",
|
||||
"d9d622afe0ca8c7ab9d24c17f9fe59b54dcc61c9",
|
||||
{"sha": "d9d622afe0ca8c7ab9d24c17f9fe59b54dcc61c9", "msg": "Fix ..."}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Input to importer.py
|
||||
1. Repository: -r <repo_path> ( Example: CMSIS / Mbed-tls)
|
||||
|
|
@ -52,4 +54,3 @@ Note: You must resolve any conflicts that arise during this cherry-pick process.
|
|||
For example the command below can be used to update CMSIS:
|
||||
`python tools\importer\importer.py -c tools\importer\cmsis_importer.json -r <path to cmsis repo>`
|
||||
|
||||
Note: This script must be run from the mbed-os directory to work correctly.
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
#!/usr/bin/python
|
||||
"""
|
||||
Copyright (c) 2017-2019 ARM Limited. All rights reserved.
|
||||
|
||||
|
|
@ -22,184 +23,264 @@ import sys
|
|||
import subprocess
|
||||
import logging
|
||||
import argparse
|
||||
import re
|
||||
from os.path import dirname, abspath, join, isfile, normpath
|
||||
|
||||
# Be sure that the tools directory is in the search path
|
||||
ROOT = abspath(join(dirname(__file__), "../.."))
|
||||
ROOT = abspath(join(dirname(__file__), os.path.pardir, os.path.pardir))
|
||||
sys.path.insert(0, ROOT)
|
||||
|
||||
from tools.utils import run_cmd, delete_dir_files, mkdir, copy_file
|
||||
from tools.utils import delete_dir_files, mkdir, copy_file
|
||||
|
||||
rel_log = logging.getLogger("Importer")
|
||||
|
||||
|
||||
class StoreDir(argparse.Action):
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
directory = abspath(values)
|
||||
if not os.path.isdir(directory):
|
||||
raise argparse.ArgumentError(
|
||||
None, "The directory %s does not exist!" % directory)
|
||||
setattr(namespace, self.dest, directory)
|
||||
|
||||
|
||||
class SetLogLevel(argparse.Action):
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
logging.basicConfig(level=values)
|
||||
|
||||
|
||||
class DirContext(object):
|
||||
def __init__(self, dir):
|
||||
self.dir = dir
|
||||
|
||||
def __enter__(self):
|
||||
_backup_dir = os.getcwd()
|
||||
os.chdir(self.dir)
|
||||
self.dir = _backup_dir
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
os.chdir(self.dir)
|
||||
|
||||
|
||||
def del_file(name):
|
||||
""" Delete the file in RTOS/CMSIS/features directory of mbed-os
|
||||
Args:
|
||||
name - name of the file
|
||||
"""
|
||||
Delete the file in RTOS/CMSIS/features directory of mbed-os.
|
||||
|
||||
:param name: Name of the file.
|
||||
:return: None.
|
||||
"""
|
||||
|
||||
result = []
|
||||
search_path = [join(ROOT, 'rtos'), join(ROOT, 'cmsis'), join(ROOT, 'features')]
|
||||
search_path = [join(ROOT, 'rtos'), join(ROOT, 'cmsis'),
|
||||
join(ROOT, 'features')]
|
||||
for path in search_path:
|
||||
for root, dirs, files in os.walk(path):
|
||||
if name in files:
|
||||
result.append(join(root, name))
|
||||
for file in result:
|
||||
os.remove(file)
|
||||
rel_log.debug("Deleted %s", os.path.relpath(file, ROOT))
|
||||
for f in result:
|
||||
os.remove(f)
|
||||
rel_log.debug("Deleted %s", os.path.relpath(f, ROOT))
|
||||
|
||||
def copy_folder(src, dest):
|
||||
""" Copy contents of folder in mbed-os listed path
|
||||
Args:
|
||||
src - src folder path
|
||||
dest - destination folder path
|
||||
|
||||
def copy_folder(src, dst):
|
||||
"""
|
||||
Copy contents of folder in mbed-os listed path.
|
||||
|
||||
:param src: Source folder path.
|
||||
:param dst: Destination folder path.
|
||||
:return: None.
|
||||
"""
|
||||
|
||||
files = os.listdir(src)
|
||||
for file in files:
|
||||
abs_src_file = join(src, file)
|
||||
if os.path.isfile(abs_src_file):
|
||||
abs_dst_file = join(dest, file)
|
||||
for f in files:
|
||||
abs_src_file = join(src, f)
|
||||
if isfile(abs_src_file):
|
||||
abs_dst_file = join(dst, f)
|
||||
mkdir(dirname(abs_dst_file))
|
||||
copy_file(abs_src_file, abs_dst_file)
|
||||
|
||||
def run_cmd_with_output(command, exit_on_failure=False):
|
||||
""" Passes a command to the system and returns a True/False result once the
|
||||
command has been executed, indicating success/failure. If the command was
|
||||
successful then the output from the command is returned to the caller.
|
||||
Commands are passed as a list of tokens.
|
||||
E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v']
|
||||
|
||||
Args:
|
||||
command - system command as a list of tokens
|
||||
exit_on_failure - If True exit the program on failure (default = False)
|
||||
|
||||
Returns:
|
||||
result - True/False indicating the success/failure of the command
|
||||
output - The output of the command if it was successful, else empty string
|
||||
def import_files(repo_path, data_files, data_folders):
|
||||
"""
|
||||
Imports files and directories to mbed-os
|
||||
|
||||
:param repo_path: Path to the repo copying from.
|
||||
:param data_files: List of files to be copied. (with destination)
|
||||
:param data_folders: List of directories to be copied. (with destination)
|
||||
:return: None
|
||||
"""
|
||||
|
||||
# Remove all files listed in .json from mbed-os repo to avoid duplications
|
||||
for fh in data_files:
|
||||
src_file = fh['src_file']
|
||||
del_file(os.path.basename(src_file))
|
||||
dest_file = join(ROOT, fh['dest_file'])
|
||||
if isfile(dest_file):
|
||||
os.remove(join(ROOT, dest_file))
|
||||
rel_log.debug("Deleted %s", fh['dest_file'])
|
||||
for folder in data_folders:
|
||||
dest_folder = folder['dest_folder']
|
||||
delete_dir_files(dest_folder)
|
||||
rel_log.debug("Deleted: %s", folder['dest_folder'])
|
||||
rel_log.info("Removed files/folders listed in json file")
|
||||
|
||||
# Copy all the files listed in json file to mbed-os
|
||||
for fh in data_files:
|
||||
repo_file = join(repo_path, fh['src_file'])
|
||||
mbed_path = join(ROOT, fh['dest_file'])
|
||||
mkdir(dirname(mbed_path))
|
||||
copy_file(repo_file, mbed_path)
|
||||
rel_log.debug("Copied %s to %s", normpath(repo_file),
|
||||
normpath(mbed_path))
|
||||
for folder in data_folders:
|
||||
repo_folder = join(repo_path, folder['src_folder'])
|
||||
mbed_path = join(ROOT, folder['dest_folder'])
|
||||
copy_folder(repo_folder, mbed_path)
|
||||
rel_log.debug("Copied %s to %s", normpath(repo_folder),
|
||||
normpath(mbed_path))
|
||||
|
||||
|
||||
def run_cmd_with_output(command, exit_on_failure=False):
|
||||
"""
|
||||
Passes a command to the system and returns a True/False result once the
|
||||
command has been executed, indicating success/failure. If the command was
|
||||
successful then the output from the command is returned to the caller.
|
||||
Commands are passed as a list of tokens.
|
||||
E.g. The command 'git remote -v' would be passed in as:
|
||||
['git', 'remote', '-v']
|
||||
|
||||
:param command: System command as a list of tokens.
|
||||
:param exit_on_failure: Exit the program on failure (default=False)
|
||||
:return: Command return status code and output as tuple.
|
||||
"""
|
||||
|
||||
rel_log.debug('[Exec] %s', ' '.join(command))
|
||||
returncode = 0
|
||||
return_code = 0
|
||||
output = ""
|
||||
try:
|
||||
output = subprocess.check_output(command)
|
||||
except subprocess.CalledProcessError as e:
|
||||
returncode = e.returncode
|
||||
return_code = e.returncode
|
||||
|
||||
if exit_on_failure:
|
||||
rel_log.error("The command %s failed with return code: %s",
|
||||
(' '.join(command)), returncode)
|
||||
(' '.join(command)), return_code)
|
||||
sys.exit(1)
|
||||
return returncode, output
|
||||
return return_code, output
|
||||
|
||||
|
||||
def get_curr_sha(repo_path):
|
||||
""" Gets the latest SHA for the specified repo
|
||||
Args:
|
||||
repo_path - path to the repository
|
||||
|
||||
Returns:
|
||||
sha - last commit SHA
|
||||
"""
|
||||
cwd = os.getcwd()
|
||||
os.chdir(abspath(repo_path))
|
||||
Gets the latest SHA for the specified repo.
|
||||
|
||||
cmd = ['git', 'log', '--pretty=format:%h', '-n', '1']
|
||||
_, sha = run_cmd_with_output(cmd, exit_on_failure=True)
|
||||
:param repo_path: Path to a git repository.
|
||||
:return: Last commit SHA.
|
||||
"""
|
||||
|
||||
cmd = ['git', '-C', repo_path, 'log', '--pretty=format:%h', '-n', '1']
|
||||
_, _sha = run_cmd_with_output(cmd, exit_on_failure=True)
|
||||
|
||||
if not _sha:
|
||||
rel_log.error("Could not obtain latest SHA")
|
||||
sys.exit(1)
|
||||
|
||||
rel_log.info("%s SHA = %s", repo_path, _sha)
|
||||
return _sha
|
||||
|
||||
os.chdir(cwd)
|
||||
return sha
|
||||
|
||||
def branch_exists(name):
|
||||
""" Check if branch already exists in mbed-os local repository.
|
||||
It will not verify if branch is present in remote repository.
|
||||
Args:
|
||||
name - branch name
|
||||
Returns:
|
||||
True - If branch is already present
|
||||
"""
|
||||
Check if branch already exists in mbed-os local repository.
|
||||
|
||||
:param name: Branch name.
|
||||
:return: True if branch is already present, False otherwise.
|
||||
"""
|
||||
|
||||
cmd = ['git', 'branch']
|
||||
_, output = run_cmd_with_output(cmd, exit_on_failure=False)
|
||||
if name in output:
|
||||
return True
|
||||
return False
|
||||
|
||||
return name in output
|
||||
|
||||
|
||||
def branch_checkout(name):
|
||||
"""
|
||||
Checkout the required branch
|
||||
Args:
|
||||
name - branch name
|
||||
Checkout the required git branch.
|
||||
|
||||
:param name: Branch to checkout.
|
||||
:return: None.
|
||||
"""
|
||||
|
||||
cmd = ['git', 'checkout', name]
|
||||
_, _ = run_cmd_with_output(cmd, exit_on_failure=False)
|
||||
rel_log.info("Checkout to branch %s", name)
|
||||
|
||||
def get_last_cherry_pick_sha(branch):
|
||||
"""
|
||||
SHA of last cherry pick commit is returned. SHA should be added to all
|
||||
cherry-pick commits with -x option.
|
||||
|
||||
Args:
|
||||
branch - Hash to be verified.
|
||||
Returns - SHA if found, else None
|
||||
def get_last_cherry_pick_sha():
|
||||
"""
|
||||
Finds the SHA of last cherry picked commit.
|
||||
SHA should be added to cherry-pick commits with -x option.
|
||||
|
||||
:return: SHA if found, None otherwise.
|
||||
"""
|
||||
cmd = ['git', 'checkout', branch]
|
||||
run_cmd_with_output(cmd, exit_on_failure=False)
|
||||
|
||||
sha = None
|
||||
get_commit = ['git', 'log', '-n', '1']
|
||||
_, output = run_cmd_with_output(get_commit, exit_on_failure=True)
|
||||
lines = output.split('\n')
|
||||
for line in lines:
|
||||
if 'cherry picked from' in line:
|
||||
sha = line.split(' ')[-1]
|
||||
return sha[:-1]
|
||||
return sha
|
||||
|
||||
if __name__ == "__main__":
|
||||
shas = re.findall(
|
||||
pattern='^\s*\(cherry picked from commit ([0-9a-fA-F]+)\)$',
|
||||
string=output,
|
||||
flags=re.MULTILINE
|
||||
)
|
||||
|
||||
return shas[-1] if shas else None
|
||||
|
||||
|
||||
def normalize_commit_sha(sha_lst):
|
||||
"""
|
||||
The commit_sha section of the config file can hold commits in 2 ways:
|
||||
* "<SHA>" - E.g. "428acae1b2ac15c3ad523e8d40755a9301220822".
|
||||
* {"sha": "<SHA>", "msg": "<HELP>"} - E.g.
|
||||
{"sha": "d9d622afe0ca8c7ab9d24c17f9fe59b54dcc61c9", "msg": "Fix ..."}.
|
||||
|
||||
:param sha_lst: JSON data from config file.
|
||||
:return: list of commit SHA.
|
||||
"""
|
||||
return [_sha['sha'] if isinstance(_sha, dict) else _sha for _sha in sha_lst]
|
||||
|
||||
|
||||
def get_parser():
|
||||
parser = argparse.ArgumentParser(
|
||||
description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument('-l', '--log-level',
|
||||
help="Level for providing logging output",
|
||||
default='INFO')
|
||||
choices=['DEBUG', 'INFO', 'ERROR'],
|
||||
default='INFO',
|
||||
action=SetLogLevel)
|
||||
parser.add_argument('-r', '--repo-path',
|
||||
help="Git Repository to be imported",
|
||||
default=None,
|
||||
required=True)
|
||||
required=True,
|
||||
action=StoreDir)
|
||||
parser.add_argument('-c', '--config-file',
|
||||
help="Configuration file",
|
||||
default=None,
|
||||
required=True)
|
||||
required=True,
|
||||
type=argparse.FileType('r'))
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def main():
|
||||
parser = get_parser()
|
||||
args = parser.parse_args()
|
||||
level = getattr(logging, args.log_level.upper())
|
||||
|
||||
# Set logging level
|
||||
logging.basicConfig(level=level)
|
||||
rel_log = logging.getLogger("Importer")
|
||||
|
||||
if (args.repo_path is None) or (args.config_file is None):
|
||||
rel_log.error("Repository path and config file required as input. Use \"--help\" for more info.")
|
||||
exit(1)
|
||||
|
||||
json_file = abspath(args.config_file)
|
||||
if not os.path.isfile(json_file):
|
||||
rel_log.error("%s not found.", args.config_file)
|
||||
exit(1)
|
||||
|
||||
repo = abspath(args.repo_path)
|
||||
if not os.path.exists(repo):
|
||||
rel_log.error("%s not found.", args.repo_path)
|
||||
exit(1)
|
||||
|
||||
sha = get_curr_sha(repo)
|
||||
if not sha:
|
||||
rel_log.error("Could not obtain latest SHA")
|
||||
exit(1)
|
||||
rel_log.info("%s SHA = %s", os.path.basename(repo), sha)
|
||||
|
||||
branch = 'feature_' + os.path.basename(repo) + '_' + sha
|
||||
commit_msg = "[" + os.path.basename(repo) + "]" + ": Updated to " + sha
|
||||
sha = get_curr_sha(args.repo_path)
|
||||
repo_dir = os.path.basename(args.repo_path)
|
||||
branch = 'feature_' + repo_dir + '_' + sha
|
||||
commit_msg = "[" + repo_dir + "]" + ": Updated to " + sha
|
||||
|
||||
# Read configuration data
|
||||
with open(json_file, 'r') as config:
|
||||
json_data = json.load(config)
|
||||
json_data = json.load(args.config_file)
|
||||
|
||||
'''
|
||||
Check if branch exists already, in case branch is present
|
||||
|
|
@ -211,38 +292,9 @@ if __name__ == "__main__":
|
|||
else:
|
||||
data_files = json_data["files"]
|
||||
data_folders = json_data["folders"]
|
||||
import_files(args.repo_path, data_files, data_folders)
|
||||
|
||||
## Remove all files listed in .json from mbed-os repo to avoid duplications
|
||||
for file in data_files:
|
||||
src_file = file['src_file']
|
||||
del_file(os.path.basename(src_file))
|
||||
dest_file = join(ROOT, file['dest_file'])
|
||||
if isfile(dest_file):
|
||||
os.remove(join(ROOT, dest_file))
|
||||
rel_log.debug("Deleted %s", file['dest_file'])
|
||||
|
||||
for folder in data_folders:
|
||||
dest_folder = folder['dest_folder']
|
||||
delete_dir_files(dest_folder)
|
||||
rel_log.debug("Deleted: %s", folder['dest_folder'])
|
||||
|
||||
rel_log.info("Removed files/folders listed in json file")
|
||||
|
||||
## Copy all the files listed in json file to mbed-os
|
||||
for file in data_files:
|
||||
repo_file = join(repo, file['src_file'])
|
||||
mbed_path = join(ROOT, file['dest_file'])
|
||||
mkdir(dirname(mbed_path))
|
||||
copy_file(repo_file, mbed_path)
|
||||
rel_log.debug("Copied %s to %s", normpath(repo_file), normpath(mbed_path))
|
||||
|
||||
for folder in data_folders:
|
||||
repo_folder = join(repo, folder['src_folder'])
|
||||
mbed_path = join(ROOT, folder['dest_folder'])
|
||||
copy_folder(repo_folder, mbed_path)
|
||||
rel_log.debug("Copied %s to %s", normpath(repo_folder), normpath(mbed_path))
|
||||
|
||||
## Create new branch with all changes
|
||||
# Create new branch with all changes
|
||||
create_branch = ['git', 'checkout', '-b', branch]
|
||||
run_cmd_with_output(create_branch, exit_on_failure=True)
|
||||
rel_log.info("Branch created: %s", branch)
|
||||
|
|
@ -254,25 +306,32 @@ if __name__ == "__main__":
|
|||
run_cmd_with_output(commit_branch, exit_on_failure=True)
|
||||
rel_log.info('Commit added: "%s"', commit_msg)
|
||||
|
||||
## Checkout the feature branch
|
||||
# Checkout the feature branch
|
||||
branch_checkout(branch)
|
||||
commit_sha = json_data["commit_sha"]
|
||||
last_sha = get_last_cherry_pick_sha(branch)
|
||||
if not last_sha:
|
||||
## Apply commits specific to mbed-os changes
|
||||
for sha in commit_sha:
|
||||
cherry_pick_sha = ['git', 'cherry-pick', '-x', sha]
|
||||
run_cmd_with_output(cherry_pick_sha, exit_on_failure=True)
|
||||
rel_log.info("Cherry-picked commit = %s", sha)
|
||||
## Few commits are already applied, check the next in sequence
|
||||
## and skip to last applied
|
||||
else:
|
||||
found = False
|
||||
for sha in commit_sha:
|
||||
if sha == last_sha:
|
||||
found = True
|
||||
continue
|
||||
if found is True:
|
||||
cherry_pick_sha = ['git', 'cherry-pick', '-x', sha]
|
||||
run_cmd_with_output(cherry_pick_sha, exit_on_failure=True)
|
||||
rel_log.info("Cherry-picked commit = %s", sha)
|
||||
commit_sha = normalize_commit_sha(json_data["commit_sha"])
|
||||
last_sha = get_last_cherry_pick_sha()
|
||||
|
||||
# Few commits are already applied, check the next in sequence
|
||||
# and skip to next commit
|
||||
if last_sha:
|
||||
assert last_sha in commit_sha, "%s not found in config file" % last_sha
|
||||
# Calculate the index of the next sha to be applied
|
||||
next_sha_idx = commit_sha.index(last_sha) + 1
|
||||
if next_sha_idx >= len(commit_sha):
|
||||
rel_log.info("No more commits to apply")
|
||||
sys.exit(0)
|
||||
# Skipping applied commits
|
||||
commit_sha = commit_sha[next_sha_idx:]
|
||||
|
||||
# Apply commits specific to mbed-os changes
|
||||
for sha in commit_sha:
|
||||
cherry_pick_sha = ['git', 'cherry-pick', '-x', sha]
|
||||
rel_log.info("Cherry-picking commit = %s", sha)
|
||||
run_cmd_with_output(cherry_pick_sha, exit_on_failure=True)
|
||||
|
||||
rel_log.info("Finished import successfully :)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with DirContext(ROOT):
|
||||
main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue