diff --git a/tools/importer/README.md b/tools/importer/README.md index b2fbf32318..a1e993bde4 100644 --- a/tools/importer/README.md +++ b/tools/importer/README.md @@ -1,6 +1,50 @@ -## Porting CMSIS/RTX Code in Mbed OS +## Importing repositories into mbed-os -This directory contains scripts to import latest CMSIS/RTX code into mbed-os local repository. -CMSIS/RTX fixes and new releases are imported into mbed-os on regular basis using the `cmsis_importer.py` python script input to which is `cmsis_importer.json` +importer.py script can be used to import code base from different repositories into mbed-os. -Verify the files and path in `cmsis_importer.json` +### Pre-requisties +1. Get the required repository clone and update it to commit required. Note: Repository should be placed outside the mbed-os. +2. Create json file as per template + +### JSON file template + +You can list all the files and folders which are required to be copied in config file in `file` and `folder` arrays respectively. Script will remove the files/folders specified in `src` section from mbed-os repository before copy operation is performed. +New commit is created on branch `feature_+repo_name+last_sha` with commit message "[REPO_NAME]: Updated to last_sha" + +Note: Only files present in folder will be copied, directories inside the folder will not be copied. + +`commit_sha` is list of commits present in mbed-os repo. These commits will be applied after copying files and folders listed above.Each commit in the commit_sha list is cherry-picked and applied with the -x option, which records the SHA of the source commit in the commit message. +Note: You must resolve any conflicts that arise during this cherry-pick process. Make sure that the "(cherry picked from commit ...)" statement is present in the commit message. Re-execute the python script to apply rest of the SHA commits. + +{ + "files" : [ + { + "src_file" : "CMSIS/Core/Template/ARMv8-M/tz_context.c", + "dest_file" : "cmsis/TARGET_CORTEX_M/mbed_tz_context.c" + }, + ... + { + "src_file" : "", + "dest_file" : "" + } + ], + "folders" : [ + { + "src_folder" : "CMSIS/Core/Include/", + "dest_folder" : "cmsis/TARGET_CORTEX_M/" + }, + ... + { + "src_folder" : "", + "dest_folder" : "" + } + ], + "commit_sha" : [ + "428acae1b2ac15c3ad523e8d40755a9301220822", + "d9d622afe0ca8c7ab9d24c17f9fe59b54dcc61c9", + ] +} + +### Input to importer.py +1. Repository: -r ( Example: CMSIS / Mbed-tls / uVisor) +2. `repo`_importer.json: -c (Example: cmsis_importer.json) diff --git a/tools/importer/cmsis_importer.json b/tools/importer/cmsis_importer.json index e3d5b1836a..e2571c259d 100644 --- a/tools/importer/cmsis_importer.json +++ b/tools/importer/cmsis_importer.json @@ -1,152 +1,154 @@ { - "config" : { - "cmsis_repo" : "https://github.com/ARM-software/CMSIS_5", - "cmsis_branch" : "develop" - }, "files" : [ { - "cmsis_file" : "CMSIS/Core/Template/ARMv8-M/tz_context.c", - "mbed_file" : "platform/mbed_tz_context.c" + "src_file" : "CMSIS/Core/Template/ARMv8-M/tz_context.c", + "dest_file" : "cmsis/TARGET_CORTEX_M/mbed_tz_context.c" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Config/handlers.c", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Config/TARGET_CORTEX_A/handlers.c" + "src_file" : "CMSIS/RTOS2/RTX/Config/handlers.c", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Config/TARGET_CORTEX_A/handlers.c" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Config/RTX_Config.h", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Config/RTX_Config.h" + "src_file" : "CMSIS/RTOS2/RTX/Config/RTX_Config.h", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Config/RTX_Config.h" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Config/RTX_Config.c", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Config/RTX_Config.c" + "src_file" : "CMSIS/RTOS2/RTX/Config/RTX_Config.c", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Config/RTX_Config.c" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm0.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M0/irq_cm0.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm0.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M0/irq_cm0.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm0.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M0P/irq_cm0.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm0.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M0P/irq_cm0.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_armv8mbl.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M23/irq_armv8mbl.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_armv8mbl.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M23/irq_armv8mbl.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm3.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M3/irq_cm3.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm3.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M3/irq_cm3.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_armv8mml.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M33/irq_armv8mml.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_armv8mml.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M33/irq_armv8mml.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm4f.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_RTOS_M4_M7/irq_cm4f.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm4f.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_RTOS_M4_M7/irq_cm4f.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_ca.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_CORTEX_A/irq_ca.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_ca.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_CORTEX_A/irq_ca.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm0.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M0/irq_cm0.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm0.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M0/irq_cm0.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm0.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M0P/irq_cm0.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm0.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M0P/irq_cm0.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_armv8mbl.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M23/irq_armv8mbl.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_armv8mbl.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M23/irq_armv8mbl.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm3.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M3/irq_cm3.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm3.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M3/irq_cm3.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_armv8mml.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M33/irq_armv8mml.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_armv8mml.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M33/irq_armv8mml.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm4f.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_RTOS_M4_M7/irq_cm4f.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm4f.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_RTOS_M4_M7/irq_cm4f.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_ca.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_CORTEX_A/irq_ca.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_ca.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_CORTEX_A/irq_ca.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm0.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M0/irq_cm0.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm0.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M0/irq_cm0.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm0.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M0P/irq_cm0.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm0.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M0P/irq_cm0.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_armv8mbl_common.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M23/irq_armv8mbl_common.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_armv8mbl_common.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M23/irq_armv8mbl_common.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm3.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M3/irq_cm3.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm3.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M3/irq_cm3.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_armv8mml_common.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M33/irq_armv8mml_common.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_armv8mml_common.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M33/irq_armv8mml_common.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm4f.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_RTOS_M4_M7/irq_cm4f.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm4f.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_RTOS_M4_M7/irq_cm4f.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_ca.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_CORTEX_A/irq_ca.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_ca.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_CORTEX_A/irq_ca.S" } ], "folders" : [ { - "cmsis_folder" : "CMSIS/Core/Include/", - "mbed_folder" : "cmsis/TARGET_CORTEX_M/" + "src_folder" : "CMSIS/Core/Include/", + "dest_folder" : "cmsis/TARGET_CORTEX_M/" }, { - "cmsis_folder" : "CMSIS/RTOS2/Include/", - "mbed_folder" : "rtos/TARGET_CORTEX/rtx5/Include/" + "src_folder" : "CMSIS/RTOS2/Include/", + "dest_folder" : "rtos/TARGET_CORTEX/rtx5/Include/" }, { - "cmsis_folder" : "CMSIS/RTOS2/RTX/Include1/", - "mbed_folder" : "rtos/TARGET_CORTEX/rtx4/" + "src_folder" : "CMSIS/RTOS2/RTX/Include1/", + "dest_folder" : "rtos/TARGET_CORTEX/rtx4/" }, { - "cmsis_folder" : "CMSIS/RTOS2/RTX/Include/", - "mbed_folder" : "rtos/TARGET_CORTEX/rtx5/RTX/Include/" + "src_folder" : "CMSIS/RTOS2/RTX/Include/", + "dest_folder" : "rtos/TARGET_CORTEX/rtx5/RTX/Include/" }, { - "cmsis_folder" : "CMSIS/RTOS2/RTX/Source/", - "mbed_folder" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/" + "src_folder" : "CMSIS/RTOS2/RTX/Source/", + "dest_folder" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/" }, { - "cmsis_folder" : "CMSIS/RTOS2/RTX/Source/", - "mbed_folder" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/" + "src_folder" : "CMSIS/RTOS2/RTX/Source/", + "dest_folder" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/" }, { - "cmsis_folder" : "CMSIS/Core_A/Include/", - "mbed_folder" : "cmsis/TARGET_CORTEX_A/" + "src_folder" : "CMSIS/Core_A/Include/", + "dest_folder" : "cmsis/TARGET_CORTEX_A/" }, { - "cmsis_folder" : "CMSIS/Core_A/Source/", - "mbed_folder" : "cmsis/TARGET_CORTEX_A/" + "src_folder" : "CMSIS/Core_A/Source/", + "dest_folder" : "cmsis/TARGET_CORTEX_A/" } ], - "Mbed_sha" : [ + "commit_sha" : [ "428acae1b2ac15c3ad523e8d40755a9301220822", "d9d622afe0ca8c7ab9d24c17f9fe59b54dcc61c9", "a1fcd36be8ee00aba2c9c1b079f5728368922bc8", "f3db103d481d8729950414868cfc8123b8055601", "c07cc6b0f42ff4fe215aa1641a043e205d9128a5", "dd8fdf4c768e5fef3a7ce2e014de4339dbafe5ce", - "2a837ea97900cc30f82e5a23b95b3f293d17eae1" + "2a837ea97900cc30f82e5a23b95b3f293d17eae1", + "c03b3f9eedab7cb0732d1519c4f1a8d90b08eede", + "314a9eb559752132a89b0dbd986db960b3ab9055", + "e83fd0099a69e6eb865e4e6fcadbfb1328c04c85", + "a019acaf8d6fb1f0512414d072f667cc2749b1d9", + "a884fdc0639ae4e17299838ec9de4fddd83cf93c", + "6c827cb5879bc096e45efd992dfadcb96c1d50bc" ] } diff --git a/tools/importer/importer.py b/tools/importer/importer.py index 4694b9ef30..f0c67a61cf 100644 --- a/tools/importer/importer.py +++ b/tools/importer/importer.py @@ -1,80 +1,135 @@ -import os, json, stat, sys, shutil, errno, subprocess, logging -from os.path import dirname, abspath, basename, join +import os, json, stat, sys, shutil, errno, subprocess, logging, re import argparse +from os.path import dirname, abspath, basename, join +# Be sure that the tools directory is in the search path ROOT = abspath(join(dirname(__file__), "../..")) -RTOS_UPDATE_BRANCH = "rtos_update" +sys.path.insert(0, ROOT) + +from tools.utils import run_cmd, delete_dir_files, mkdir, copy_file def del_file(name): + """ Delete the file in RTOS/CMSIS/features directory of mbed-os + Args: + name - name of the file + """ result = [] - search_path = [join(ROOT, 'rtos'), join(ROOT, 'cmsis')] + search_path = [join(ROOT, 'rtos'), join(ROOT, 'cmsis'), join(ROOT, 'features')] for path in search_path: for root, dirs, files in os.walk(path): if name in files: result.append(os.path.join(root, name)) for file in result: - print os.path.relpath(file, ROOT) os.remove(file) + rel_log.debug("Deleted: %s", os.path.relpath(file, ROOT)); -def rmtree(top): - for root, dirs, files in os.walk(top, topdown=False): - for name in files: - filename = os.path.join(root, name) - os.chmod(filename, stat.S_IWUSR) - os.remove(filename) - for name in dirs: - os.rmdir(os.path.join(root, name)) - os.rmdir(top) - -def copy_file(file, path): - try: - shutil.copy(file, path) - except IOError as e: - if e.errno != errno.ENOENT: - raise - ## Create directories - os.makedirs(os.path.dirname(path)) - shutil.copy(file, path) - print os.path.relpath(path, ROOT) - -def copy_folder(folder, path): - files = os.listdir(folder) +def copy_folder(src, dest): + """ Copy contents of folder in mbed-os listed path + Args: + src - src folder path + dest - destination folder path + """ + files = os.listdir(src) for file in files: - abs_src_file = os.path.join(folder, file) + abs_src_file = os.path.join(src, file) if os.path.isfile(abs_src_file): - abs_dst_file = os.path.join(path, file) + abs_dst_file = os.path.join(dest, file) + mkdir(os.path.dirname(abs_dst_file)) copy_file(abs_src_file, abs_dst_file) -def run_cmd(command, exit_on_failure=False): +def run_cmd_with_output(command, exit_on_failure=False): """ Passes a command to the system and returns a True/False result once the - command has been executed, indicating success/failure. Commands are passed - as a list of tokens. + command has been executed, indicating success/failure. If the command was + successful then the output from the command is returned to the caller. + Commands are passed as a list of tokens. E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v'] - """ - return_code = subprocess.call(command, shell=True) - if return_code: - print("The command %s failed with return code: %s" - %(' '.join(command), return_code)) + Args: + command - system command as a list of tokens + exit_on_failure - If True exit the program on failure (default = False) + + Returns: + result - True/False indicating the success/failure of the command + output - The output of the command if it was successful, else empty string + """ + rel_log.debug('[Exec] %s', ' '.join(command)) + returncode = 0 + output = "" + try: + output = subprocess.check_output(command, shell=True) + except subprocess.CalledProcessError as e: + returncode = e.returncode + if exit_on_failure: + rel_log.error("The command %s failed with return code: %s", + (' '.join(command)), returncode) sys.exit(1) + return returncode, output - return return_code +def get_curr_sha(repo_path): + """ Gets the latest SHA for the specified repo + Args: + repo_path - path to the repository -def remove_repo(folder): - os.chdir(abspath(dirname(__file__))) - if os.path.exists(folder): - rmtree(folder) - -def get_repo(repo, branch, folder): - """ Get the Repository files from git, at depth level 1 - repo - Git repository link - branch - repository branch - folder - folder at which repo will be cloned + Returns: + sha - last commit SHA """ - remove_repo(folder) - clone_cmd = ['git', 'clone', repo, "-b", branch, "--depth", '1', folder] - run_cmd(clone_cmd, exit_on_failure=True) + cwd = os.getcwd() + sha = None + os.chdir(abspath(repo_path)) + + cmd = "git log --pretty=format:%h -n 1" + ret, sha = run_cmd_with_output(cmd, exit_on_failure=True) + + os.chdir(cwd) + return sha + +def check_branch(name): + """ Check if branch already exists in mbed-os local repository. + It will not verify if branch is present in remote repository. + Args: + name - branch name + Returns: + True - If branch is already present + """ + branches = [] + cmd = "git branch" + _, output = run_cmd_with_output(cmd, exit_on_failure=False) + if name in output: + return True + return False + +def branch_checkout(name): + """ + Checkout the required branch + Args: + name - branch name + """ + cmd = "git checkout " + name + run_cmd_with_output(cmd, exit_on_failure=False) + +def get_last_cherry_pick_sha(branch): + """ + SHA of last cherry pick commit is returned. SHA should be added to all + cherry-pick commits with -x option. + + Args: + branch - Hash to be verified. + Returns - SHA if found, else None + """ + cmd = "git checkout " + branch + run_cmd_with_output(cmd, exit_on_failure=False) + + SHA = None + get_commit = "git log -n 1" + _, output = run_cmd_with_output(get_commit, exit_on_failure=True) + lines = output.split('\n') + for line in lines: + if 'cherry picked from' in line: + SHA = line.split(' ')[-1] + SHA = SHA[:-1] + return SHA + #for words in output.split('\n') if 'origin' in line and not '->' in line] if __name__ == "__main__": @@ -85,14 +140,14 @@ if __name__ == "__main__": default='INFO') parser.add_argument('-r', '--repo-path', help="Git Repository to be imported", - default=None) + default=None, + required=True) parser.add_argument('-c', '--config-file', help="Configuration file", - default=None) + default=None, + required=True) args = parser.parse_args() - - default = getattr(logging, 'INFO') - level = getattr(logging, args.log_level.upper(), default) + level = getattr(logging, args.log_level.upper()) # Set logging level logging.basicConfig(level=level) @@ -102,58 +157,96 @@ if __name__ == "__main__": rel_log.error("Repository path and config file required as input. Use \"--help\" for more info.") exit(1) - repo = os.path.abspath(args.repo_path) - if not os.path.exists(repo): - rel_log.error("%s not found.", args.repo_path) - exit(1) - json_file = os.path.abspath(args.config_file) if not os.path.isfile(json_file): rel_log.error("%s not found." , args.config_file) exit(1) + repo = os.path.abspath(args.repo_path) + if not os.path.exists(repo): + rel_log.error("%s not found.", args.repo_path) + exit(1) + + sha = get_curr_sha(repo) + if not sha: + rel_log.error("Could not obtain latest SHA") + exit(1) + rel_log.info("%s SHA = %s", os.path.basename(repo), sha) + + branch = 'feature_' + os.path.basename(repo) + '_' + sha + commit_msg = "[" + os.path.basename(repo) + "]" + ": Updated to " + sha + # Read configuration data with open(json_file, 'r') as config: - json_data = json.load(config) - data_files = json_data["files"] - data_folders = json_data["folders"] + json_data = json.load(config) - ## Remove all files listed in .json from mbed-os repo to avoid duplications - print "Cleaning up:" - for file in data_files: - cmsis_file = file['cmsis_file'] - del_file(os.path.basename(cmsis_file)) + ''' + Check if branch exists already, in case branch is present + we will skip all file transfer and merge operations and will + jump to cherry-pick + ''' + if check_branch(branch): + rel_log.info("Branch present = %s", branch) + else: + data_files = json_data["files"] + data_folders = json_data["folders"] - for folder in data_folders: - files = os.listdir(repo_path) - for file in files: - del_file(os.path.basename(file)) + ## Remove all files listed in .json from mbed-os repo to avoid duplications + for file in data_files: + src_file = file['src_file'] + del_file(os.path.basename(src_file)) - ## Copy all the CMSIS files listed in json file to mbed-os - print "Files Copied:" - for file in data_files: - repo_file = os.path.join(repo_path, file['cmsis_file']) - mbed_path = os.path.join(ROOT, file['mbed_file']) - copy_file(repo_file, mbed_path) + for folder in data_folders: + dest_folder = folder['dest_folder'] + delete_dir_files(dest_folder) + rel_log.debug("Deleted = %s", folder) - for folder in data_folders: - repo_folder = os.path.join(repo_path, folder['cmsis_folder']) - mbed_path = os.path.join(ROOT, folder['mbed_folder']) - copy_folder(repo_folder, mbed_path) + rel_log.info("Removed files/folders listed in json file") - ## Create new branch with all changes - create_branch = ['git', 'checkout', '-b', RTOS_UPDATE_BRANCH] - run_cmd(create_branch, exit_on_failure=True) + ## Copy all the CMSIS files listed in json file to mbed-os + for file in data_files: + repo_file = os.path.join(repo, file['src_file']) + mbed_path = os.path.join(ROOT, file['dest_file']) + mkdir(os.path.dirname(mbed_path)) + copy_file(repo_file, mbed_path) + rel_log.debug("Copied = %s", mbed_path) - add_files = ['git', 'add', '-A'] - run_cmd(add_files, exit_on_failure=True) + for folder in data_folders: + repo_folder = os.path.join(repo, folder['src_folder']) + mbed_path = os.path.join(ROOT, folder['dest_folder']) + copy_folder(repo_folder, mbed_path) + rel_log.debug("Copied = %s", mbed_path) - commit_branch = ['git', 'commit', '-m', "CMSIS/RTX: Update CMSIS/RTX"] - run_cmd(commit_branch, exit_on_failure=True) + ## Create new branch with all changes + create_branch = "git checkout -b "+ branch + run_cmd_with_output(create_branch, exit_on_failure=False) + rel_log.info("Branch created = %s", branch) - ## Apply commits specific to mbed-os changes - mbed_sha = json_data["Mbed_sha"] + add_files = "git add -A" + run_cmd_with_output(add_files, exit_on_failure=True) - for sha in mbed_sha: - cherry_pick_sha = ['git', 'cherry-pick', sha] - run_cmd(cherry_pick_sha, exit_on_failure=True) + commit_branch = "git commit -m \"" + commit_msg + "\"" + run_cmd_with_output(commit_branch, exit_on_failure=True) + rel_log.info("Commit added = %s", mbed_path) + + ## Checkout the feature branch + branch_checkout(branch) + commit_sha = json_data["commit_sha"] + last_sha = get_last_cherry_pick_sha(branch) + if not last_sha: + ## Apply commits specific to mbed-os changes + for sha in commit_sha: + cherry_pick_sha = "git cherry-pick -x " + sha + run_cmd_with_output(cherry_pick_sha, exit_on_failure=True) + rel_log.info("Commit added = %s", cherry_pick_sha) + ## Few commits are already applied, check the next in sequence + ## and skip to last applied + else: + found = False + for sha in commit_sha: + if sha == last_sha: + found = True + continue + if found is True: + cherry_pick_sha = "git cherry-pick -x " + sha + run_cmd_with_output(cherry_pick_sha, exit_on_failure=True)