From 021caa6dcf40cd953315d1719b7da0ef755d2c91 Mon Sep 17 00:00:00 2001 From: deepikabhavnani Date: Mon, 30 Oct 2017 16:57:28 -0500 Subject: [PATCH 1/6] Python script to add cmsis/rtx changes in mbed-os --- tools/importer/README.md | 6 ++ tools/importer/cmsis_imorter.py | 143 +++++++++++++++++++++++++++ tools/importer/cmsis_importer.json | 152 +++++++++++++++++++++++++++++ 3 files changed, 301 insertions(+) create mode 100644 tools/importer/README.md create mode 100644 tools/importer/cmsis_imorter.py create mode 100644 tools/importer/cmsis_importer.json diff --git a/tools/importer/README.md b/tools/importer/README.md new file mode 100644 index 0000000000..b2fbf32318 --- /dev/null +++ b/tools/importer/README.md @@ -0,0 +1,6 @@ +## Porting CMSIS/RTX Code in Mbed OS + +This directory contains scripts to import latest CMSIS/RTX code into mbed-os local repository. +CMSIS/RTX fixes and new releases are imported into mbed-os on regular basis using the `cmsis_importer.py` python script input to which is `cmsis_importer.json` + +Verify the files and path in `cmsis_importer.json` diff --git a/tools/importer/cmsis_imorter.py b/tools/importer/cmsis_imorter.py new file mode 100644 index 0000000000..3e8aff6332 --- /dev/null +++ b/tools/importer/cmsis_imorter.py @@ -0,0 +1,143 @@ +import os, json, stat, sys, shutil, errno, subprocess +from os.path import dirname, abspath, basename, join + +ROOT = abspath(join(dirname(__file__), "../..")) +CMSIS_REPO = "CMSIS_Repo" +CMSIS_PATH = abspath(join(dirname(__file__), CMSIS_REPO)) +RTOS_UPDATE_BRANCH = "rtos_update" + +def del_file(name): + result = [] + search_path = [join(ROOT, 'rtos'), join(ROOT, 'cmsis')] + for path in search_path: + for root, dirs, files in os.walk(path): + if name in files: + result.append(os.path.join(root, name)) + for file in result: + print os.path.relpath(file, ROOT) + os.remove(file) + +def rmtree(top): + for root, dirs, files in os.walk(top, topdown=False): + for name in files: + filename = os.path.join(root, name) + os.chmod(filename, stat.S_IWUSR) + os.remove(filename) + for name in dirs: + os.rmdir(os.path.join(root, name)) + os.rmdir(top) + +def copy_file(file, path): + try: + shutil.copy(file, path) + except IOError as e: + if e.errno != errno.ENOENT: + raise + ## Create directories + os.makedirs(os.path.dirname(path)) + shutil.copy(file, path) + print os.path.relpath(path, ROOT) + +def copy_folder(folder, path): + files = os.listdir(folder) + for file in files: + abs_src_file = os.path.join(folder, file) + if os.path.isfile(abs_src_file): + abs_dst_file = os.path.join(path, file) + copy_file(abs_src_file, abs_dst_file) + +def run_cmd(command, exit_on_failure=False): + """ Passes a command to the system and returns a True/False result once the + command has been executed, indicating success/failure. Commands are passed + as a list of tokens. + E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v'] + """ + return_code = subprocess.call(command, shell=True) + + if return_code: + print("The command %s failed with return code: %s" + %(' '.join(command), return_code)) + if exit_on_failure: + sys.exit(1) + + return return_code + +def remove_repo(folder): + os.chdir(abspath(dirname(__file__))) + if os.path.exists(folder): + rmtree(folder) + +def get_repo(repo, branch, folder): + """ Get the Repository files from git, at depth level 1 + repo - Git repository link + branch - repository branch + folder - folder at which repo will be cloned + """ + remove_repo(folder) + clone_cmd = ['git', 'clone', repo, "-b", branch, "--depth", '1', folder] + run_cmd(clone_cmd, exit_on_failure=True) + +if __name__ == "__main__": + + # Read configuration data + with open(os.path.join(os.path.dirname(__file__), "cmsis_importer.json"), 'r') as config: + json_data = json.load(config) + config = json_data["config"] + cmsis_repo = config['cmsis_repo'] + cmsis_branch = config['cmsis_branch'] + data_files = json_data["files"] + data_folders = json_data["folders"] + + print "Fetching git repo" + get_repo(cmsis_repo, cmsis_branch, CMSIS_REPO) + + ## Remove all files listed in .json from mbed-os repo to avoid duplications + print "Cleaning up:" + for file in data_files: + cmsis_file = file['cmsis_file'] + del_file(os.path.basename(cmsis_file)) + + for folder in data_folders: + cmsis_folder = os.path.join(CMSIS_PATH, folder['cmsis_folder']) + files = os.listdir(cmsis_folder) + for file in files: + del_file(os.path.basename(file)) + + ## Copy all the CMSIS files listed in json file to mbed-os + print "Files Copied:" + for file in data_files: + cmsis_file = os.path.join(CMSIS_PATH, file['cmsis_file']) + mbed_path = os.path.join(ROOT, file['mbed_file']) + copy_file(cmsis_file, mbed_path) + + for folder in data_folders: + cmsis_folder = os.path.join(CMSIS_PATH, folder['cmsis_folder']) + mbed_path = os.path.join(ROOT, folder['mbed_folder']) + copy_folder(cmsis_folder, mbed_path) + + #Remove CMSIS Repo + remove_repo(CMSIS_REPO) + + ## Create new branch with all changes + create_branch = ['git', 'checkout', '-b', RTOS_UPDATE_BRANCH] + run_cmd(create_branch, exit_on_failure=True) + + add_files = ['git', 'add', '-A'] + run_cmd(add_files, exit_on_failure=True) + + commit_branch = ['git', 'commit', '-m', "CMSIS/RTX: Update CMSIS/RTX"] + run_cmd(commit_branch, exit_on_failure=True) + + ## Apply commits specific to mbed-os changes + mbed_sha = json_data["Mbed_sha"] + + for sha in mbed_sha: + cherry_pick_sha = ['git', 'cherry-pick', sha] + run_cmd(cherry_pick_sha, exit_on_failure=True) + + + + + + + diff --git a/tools/importer/cmsis_importer.json b/tools/importer/cmsis_importer.json new file mode 100644 index 0000000000..e3d5b1836a --- /dev/null +++ b/tools/importer/cmsis_importer.json @@ -0,0 +1,152 @@ +{ + "config" : { + "cmsis_repo" : "https://github.com/ARM-software/CMSIS_5", + "cmsis_branch" : "develop" + }, + "files" : [ + { + "cmsis_file" : "CMSIS/Core/Template/ARMv8-M/tz_context.c", + "mbed_file" : "platform/mbed_tz_context.c" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Config/handlers.c", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Config/TARGET_CORTEX_A/handlers.c" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Config/RTX_Config.h", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Config/RTX_Config.h" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Config/RTX_Config.c", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Config/RTX_Config.c" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm0.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M0/irq_cm0.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm0.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M0P/irq_cm0.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_armv8mbl.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M23/irq_armv8mbl.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm3.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M3/irq_cm3.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_armv8mml.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M33/irq_armv8mml.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm4f.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_RTOS_M4_M7/irq_cm4f.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_ca.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_CORTEX_A/irq_ca.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm0.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M0/irq_cm0.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm0.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M0P/irq_cm0.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_armv8mbl.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M23/irq_armv8mbl.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm3.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M3/irq_cm3.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_armv8mml.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M33/irq_armv8mml.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm4f.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_RTOS_M4_M7/irq_cm4f.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_ca.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_CORTEX_A/irq_ca.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm0.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M0/irq_cm0.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm0.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M0P/irq_cm0.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_armv8mbl_common.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M23/irq_armv8mbl_common.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm3.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M3/irq_cm3.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_armv8mml_common.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M33/irq_armv8mml_common.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm4f.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_RTOS_M4_M7/irq_cm4f.S" + }, + { + "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_ca.S", + "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_CORTEX_A/irq_ca.S" + } + ], + "folders" : [ + { + "cmsis_folder" : "CMSIS/Core/Include/", + "mbed_folder" : "cmsis/TARGET_CORTEX_M/" + }, + { + "cmsis_folder" : "CMSIS/RTOS2/Include/", + "mbed_folder" : "rtos/TARGET_CORTEX/rtx5/Include/" + }, + { + "cmsis_folder" : "CMSIS/RTOS2/RTX/Include1/", + "mbed_folder" : "rtos/TARGET_CORTEX/rtx4/" + }, + { + "cmsis_folder" : "CMSIS/RTOS2/RTX/Include/", + "mbed_folder" : "rtos/TARGET_CORTEX/rtx5/RTX/Include/" + }, + { + "cmsis_folder" : "CMSIS/RTOS2/RTX/Source/", + "mbed_folder" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/" + }, + { + "cmsis_folder" : "CMSIS/RTOS2/RTX/Source/", + "mbed_folder" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/" + }, + { + "cmsis_folder" : "CMSIS/Core_A/Include/", + "mbed_folder" : "cmsis/TARGET_CORTEX_A/" + }, + { + "cmsis_folder" : "CMSIS/Core_A/Source/", + "mbed_folder" : "cmsis/TARGET_CORTEX_A/" + } + ], + "Mbed_sha" : [ + "428acae1b2ac15c3ad523e8d40755a9301220822", + "d9d622afe0ca8c7ab9d24c17f9fe59b54dcc61c9", + "a1fcd36be8ee00aba2c9c1b079f5728368922bc8", + "f3db103d481d8729950414868cfc8123b8055601", + "c07cc6b0f42ff4fe215aa1641a043e205d9128a5", + "dd8fdf4c768e5fef3a7ce2e014de4339dbafe5ce", + "2a837ea97900cc30f82e5a23b95b3f293d17eae1" + ] +} + From 78fdb7bcd69673065b34b626a362a04b4970abcd Mon Sep 17 00:00:00 2001 From: deepikabhavnani Date: Tue, 31 Oct 2017 14:30:50 -0500 Subject: [PATCH 2/6] Renamed importer script file --- tools/importer/{cmsis_imorter.py => importer.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tools/importer/{cmsis_imorter.py => importer.py} (100%) diff --git a/tools/importer/cmsis_imorter.py b/tools/importer/importer.py similarity index 100% rename from tools/importer/cmsis_imorter.py rename to tools/importer/importer.py From d4e6decf3268da4ce53e3a5751b78bcffa44b81c Mon Sep 17 00:00:00 2001 From: deepikabhavnani Date: Tue, 31 Oct 2017 14:33:40 -0500 Subject: [PATCH 3/6] Added arguments to importer script Repository and json file will be mandotory inputs to script file. As part of this change we do not need to clone/remove CMSIS repository --- tools/importer/importer.py | 106 +++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 45 deletions(-) diff --git a/tools/importer/importer.py b/tools/importer/importer.py index 3e8aff6332..4694b9ef30 100644 --- a/tools/importer/importer.py +++ b/tools/importer/importer.py @@ -1,9 +1,8 @@ -import os, json, stat, sys, shutil, errno, subprocess +import os, json, stat, sys, shutil, errno, subprocess, logging from os.path import dirname, abspath, basename, join +import argparse ROOT = abspath(join(dirname(__file__), "../..")) -CMSIS_REPO = "CMSIS_Repo" -CMSIS_PATH = abspath(join(dirname(__file__), CMSIS_REPO)) RTOS_UPDATE_BRANCH = "rtos_update" def del_file(name): @@ -30,36 +29,36 @@ def rmtree(top): def copy_file(file, path): try: shutil.copy(file, path) - except IOError as e: + except IOError as e: if e.errno != errno.ENOENT: raise ## Create directories os.makedirs(os.path.dirname(path)) shutil.copy(file, path) print os.path.relpath(path, ROOT) - -def copy_folder(folder, path): - files = os.listdir(folder) + +def copy_folder(folder, path): + files = os.listdir(folder) for file in files: abs_src_file = os.path.join(folder, file) if os.path.isfile(abs_src_file): abs_dst_file = os.path.join(path, file) copy_file(abs_src_file, abs_dst_file) - + def run_cmd(command, exit_on_failure=False): - """ Passes a command to the system and returns a True/False result once the + """ Passes a command to the system and returns a True/False result once the command has been executed, indicating success/failure. Commands are passed - as a list of tokens. + as a list of tokens. E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v'] """ return_code = subprocess.call(command, shell=True) - + if return_code: print("The command %s failed with return code: %s" %(' '.join(command), return_code)) if exit_on_failure: sys.exit(1) - + return return_code def remove_repo(folder): @@ -76,47 +75,71 @@ def get_repo(repo, branch, folder): remove_repo(folder) clone_cmd = ['git', 'clone', repo, "-b", branch, "--depth", '1', folder] run_cmd(clone_cmd, exit_on_failure=True) - + if __name__ == "__main__": - + + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('-l', '--log-level', + help="Level for providing logging output", + default='INFO') + parser.add_argument('-r', '--repo-path', + help="Git Repository to be imported", + default=None) + parser.add_argument('-c', '--config-file', + help="Configuration file", + default=None) + args = parser.parse_args() + + default = getattr(logging, 'INFO') + level = getattr(logging, args.log_level.upper(), default) + + # Set logging level + logging.basicConfig(level=level) + rel_log = logging.getLogger("Importer") + + if (args.repo_path is None) or (args.config_file is None) : + rel_log.error("Repository path and config file required as input. Use \"--help\" for more info.") + exit(1) + + repo = os.path.abspath(args.repo_path) + if not os.path.exists(repo): + rel_log.error("%s not found.", args.repo_path) + exit(1) + + json_file = os.path.abspath(args.config_file) + if not os.path.isfile(json_file): + rel_log.error("%s not found." , args.config_file) + exit(1) + # Read configuration data - with open(os.path.join(os.path.dirname(__file__), "cmsis_importer.json"), 'r') as config: + with open(json_file, 'r') as config: json_data = json.load(config) - config = json_data["config"] - cmsis_repo = config['cmsis_repo'] - cmsis_branch = config['cmsis_branch'] data_files = json_data["files"] data_folders = json_data["folders"] - - print "Fetching git repo" - get_repo(cmsis_repo, cmsis_branch, CMSIS_REPO) - + ## Remove all files listed in .json from mbed-os repo to avoid duplications print "Cleaning up:" for file in data_files: cmsis_file = file['cmsis_file'] del_file(os.path.basename(cmsis_file)) - + for folder in data_folders: - cmsis_folder = os.path.join(CMSIS_PATH, folder['cmsis_folder']) - files = os.listdir(cmsis_folder) - for file in files: + files = os.listdir(repo_path) + for file in files: del_file(os.path.basename(file)) - ## Copy all the CMSIS files listed in json file to mbed-os - print "Files Copied:" - for file in data_files: - cmsis_file = os.path.join(CMSIS_PATH, file['cmsis_file']) + ## Copy all the CMSIS files listed in json file to mbed-os + print "Files Copied:" + for file in data_files: + repo_file = os.path.join(repo_path, file['cmsis_file']) mbed_path = os.path.join(ROOT, file['mbed_file']) - copy_file(cmsis_file, mbed_path) + copy_file(repo_file, mbed_path) for folder in data_folders: - cmsis_folder = os.path.join(CMSIS_PATH, folder['cmsis_folder']) - mbed_path = os.path.join(ROOT, folder['mbed_folder']) - copy_folder(cmsis_folder, mbed_path) - - #Remove CMSIS Repo - remove_repo(CMSIS_REPO) + repo_folder = os.path.join(repo_path, folder['cmsis_folder']) + mbed_path = os.path.join(ROOT, folder['mbed_folder']) + copy_folder(repo_folder, mbed_path) ## Create new branch with all changes create_branch = ['git', 'checkout', '-b', RTOS_UPDATE_BRANCH] @@ -127,17 +150,10 @@ if __name__ == "__main__": commit_branch = ['git', 'commit', '-m', "CMSIS/RTX: Update CMSIS/RTX"] run_cmd(commit_branch, exit_on_failure=True) - + ## Apply commits specific to mbed-os changes mbed_sha = json_data["Mbed_sha"] for sha in mbed_sha: cherry_pick_sha = ['git', 'cherry-pick', sha] run_cmd(cherry_pick_sha, exit_on_failure=True) - - - - - - - From 43251e162d2bdaffe8228c4b67b12ca597d0dadc Mon Sep 17 00:00:00 2001 From: deepikabhavnani Date: Tue, 31 Oct 2017 23:48:31 -0500 Subject: [PATCH 4/6] Updated the script to be generic for any repo --- tools/importer/README.md | 52 +++++- tools/importer/cmsis_importer.json | 146 +++++++-------- tools/importer/importer.py | 285 +++++++++++++++++++---------- 3 files changed, 311 insertions(+), 172 deletions(-) diff --git a/tools/importer/README.md b/tools/importer/README.md index b2fbf32318..a1e993bde4 100644 --- a/tools/importer/README.md +++ b/tools/importer/README.md @@ -1,6 +1,50 @@ -## Porting CMSIS/RTX Code in Mbed OS +## Importing repositories into mbed-os -This directory contains scripts to import latest CMSIS/RTX code into mbed-os local repository. -CMSIS/RTX fixes and new releases are imported into mbed-os on regular basis using the `cmsis_importer.py` python script input to which is `cmsis_importer.json` +importer.py script can be used to import code base from different repositories into mbed-os. -Verify the files and path in `cmsis_importer.json` +### Pre-requisties +1. Get the required repository clone and update it to commit required. Note: Repository should be placed outside the mbed-os. +2. Create json file as per template + +### JSON file template + +You can list all the files and folders which are required to be copied in config file in `file` and `folder` arrays respectively. Script will remove the files/folders specified in `src` section from mbed-os repository before copy operation is performed. +New commit is created on branch `feature_+repo_name+last_sha` with commit message "[REPO_NAME]: Updated to last_sha" + +Note: Only files present in folder will be copied, directories inside the folder will not be copied. + +`commit_sha` is list of commits present in mbed-os repo. These commits will be applied after copying files and folders listed above.Each commit in the commit_sha list is cherry-picked and applied with the -x option, which records the SHA of the source commit in the commit message. +Note: You must resolve any conflicts that arise during this cherry-pick process. Make sure that the "(cherry picked from commit ...)" statement is present in the commit message. Re-execute the python script to apply rest of the SHA commits. + +{ + "files" : [ + { + "src_file" : "CMSIS/Core/Template/ARMv8-M/tz_context.c", + "dest_file" : "cmsis/TARGET_CORTEX_M/mbed_tz_context.c" + }, + ... + { + "src_file" : "", + "dest_file" : "" + } + ], + "folders" : [ + { + "src_folder" : "CMSIS/Core/Include/", + "dest_folder" : "cmsis/TARGET_CORTEX_M/" + }, + ... + { + "src_folder" : "", + "dest_folder" : "" + } + ], + "commit_sha" : [ + "428acae1b2ac15c3ad523e8d40755a9301220822", + "d9d622afe0ca8c7ab9d24c17f9fe59b54dcc61c9", + ] +} + +### Input to importer.py +1. Repository: -r ( Example: CMSIS / Mbed-tls / uVisor) +2. `repo`_importer.json: -c (Example: cmsis_importer.json) diff --git a/tools/importer/cmsis_importer.json b/tools/importer/cmsis_importer.json index e3d5b1836a..e2571c259d 100644 --- a/tools/importer/cmsis_importer.json +++ b/tools/importer/cmsis_importer.json @@ -1,152 +1,154 @@ { - "config" : { - "cmsis_repo" : "https://github.com/ARM-software/CMSIS_5", - "cmsis_branch" : "develop" - }, "files" : [ { - "cmsis_file" : "CMSIS/Core/Template/ARMv8-M/tz_context.c", - "mbed_file" : "platform/mbed_tz_context.c" + "src_file" : "CMSIS/Core/Template/ARMv8-M/tz_context.c", + "dest_file" : "cmsis/TARGET_CORTEX_M/mbed_tz_context.c" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Config/handlers.c", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Config/TARGET_CORTEX_A/handlers.c" + "src_file" : "CMSIS/RTOS2/RTX/Config/handlers.c", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Config/TARGET_CORTEX_A/handlers.c" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Config/RTX_Config.h", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Config/RTX_Config.h" + "src_file" : "CMSIS/RTOS2/RTX/Config/RTX_Config.h", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Config/RTX_Config.h" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Config/RTX_Config.c", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Config/RTX_Config.c" + "src_file" : "CMSIS/RTOS2/RTX/Config/RTX_Config.c", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Config/RTX_Config.c" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm0.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M0/irq_cm0.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm0.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M0/irq_cm0.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm0.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M0P/irq_cm0.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm0.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M0P/irq_cm0.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_armv8mbl.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M23/irq_armv8mbl.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_armv8mbl.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M23/irq_armv8mbl.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm3.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M3/irq_cm3.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm3.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M3/irq_cm3.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_armv8mml.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M33/irq_armv8mml.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_armv8mml.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_M33/irq_armv8mml.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm4f.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_RTOS_M4_M7/irq_cm4f.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_cm4f.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_RTOS_M4_M7/irq_cm4f.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_ca.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_CORTEX_A/irq_ca.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/ARM/irq_ca.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_ARM/TARGET_CORTEX_A/irq_ca.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm0.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M0/irq_cm0.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm0.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M0/irq_cm0.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm0.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M0P/irq_cm0.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm0.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M0P/irq_cm0.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_armv8mbl.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M23/irq_armv8mbl.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_armv8mbl.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M23/irq_armv8mbl.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm3.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M3/irq_cm3.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm3.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M3/irq_cm3.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_armv8mml.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M33/irq_armv8mml.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_armv8mml.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_M33/irq_armv8mml.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm4f.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_RTOS_M4_M7/irq_cm4f.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_cm4f.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_RTOS_M4_M7/irq_cm4f.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_ca.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_CORTEX_A/irq_ca.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/GCC/irq_ca.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_GCC/TARGET_CORTEX_A/irq_ca.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm0.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M0/irq_cm0.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm0.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M0/irq_cm0.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm0.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M0P/irq_cm0.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm0.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M0P/irq_cm0.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_armv8mbl_common.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M23/irq_armv8mbl_common.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_armv8mbl_common.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M23/irq_armv8mbl_common.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm3.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M3/irq_cm3.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm3.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M3/irq_cm3.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_armv8mml_common.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M33/irq_armv8mml_common.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_armv8mml_common.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_M33/irq_armv8mml_common.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm4f.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_RTOS_M4_M7/irq_cm4f.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_cm4f.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_RTOS_M4_M7/irq_cm4f.S" }, { - "cmsis_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_ca.S", - "mbed_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_CORTEX_A/irq_ca.S" + "src_file" : "CMSIS/RTOS2/RTX/Source/IAR/irq_ca.S", + "dest_file" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/TOOLCHAIN_IAR/TARGET_CORTEX_A/irq_ca.S" } ], "folders" : [ { - "cmsis_folder" : "CMSIS/Core/Include/", - "mbed_folder" : "cmsis/TARGET_CORTEX_M/" + "src_folder" : "CMSIS/Core/Include/", + "dest_folder" : "cmsis/TARGET_CORTEX_M/" }, { - "cmsis_folder" : "CMSIS/RTOS2/Include/", - "mbed_folder" : "rtos/TARGET_CORTEX/rtx5/Include/" + "src_folder" : "CMSIS/RTOS2/Include/", + "dest_folder" : "rtos/TARGET_CORTEX/rtx5/Include/" }, { - "cmsis_folder" : "CMSIS/RTOS2/RTX/Include1/", - "mbed_folder" : "rtos/TARGET_CORTEX/rtx4/" + "src_folder" : "CMSIS/RTOS2/RTX/Include1/", + "dest_folder" : "rtos/TARGET_CORTEX/rtx4/" }, { - "cmsis_folder" : "CMSIS/RTOS2/RTX/Include/", - "mbed_folder" : "rtos/TARGET_CORTEX/rtx5/RTX/Include/" + "src_folder" : "CMSIS/RTOS2/RTX/Include/", + "dest_folder" : "rtos/TARGET_CORTEX/rtx5/RTX/Include/" }, { - "cmsis_folder" : "CMSIS/RTOS2/RTX/Source/", - "mbed_folder" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/" + "src_folder" : "CMSIS/RTOS2/RTX/Source/", + "dest_folder" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/" }, { - "cmsis_folder" : "CMSIS/RTOS2/RTX/Source/", - "mbed_folder" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/" + "src_folder" : "CMSIS/RTOS2/RTX/Source/", + "dest_folder" : "rtos/TARGET_CORTEX/rtx5/RTX/Source/" }, { - "cmsis_folder" : "CMSIS/Core_A/Include/", - "mbed_folder" : "cmsis/TARGET_CORTEX_A/" + "src_folder" : "CMSIS/Core_A/Include/", + "dest_folder" : "cmsis/TARGET_CORTEX_A/" }, { - "cmsis_folder" : "CMSIS/Core_A/Source/", - "mbed_folder" : "cmsis/TARGET_CORTEX_A/" + "src_folder" : "CMSIS/Core_A/Source/", + "dest_folder" : "cmsis/TARGET_CORTEX_A/" } ], - "Mbed_sha" : [ + "commit_sha" : [ "428acae1b2ac15c3ad523e8d40755a9301220822", "d9d622afe0ca8c7ab9d24c17f9fe59b54dcc61c9", "a1fcd36be8ee00aba2c9c1b079f5728368922bc8", "f3db103d481d8729950414868cfc8123b8055601", "c07cc6b0f42ff4fe215aa1641a043e205d9128a5", "dd8fdf4c768e5fef3a7ce2e014de4339dbafe5ce", - "2a837ea97900cc30f82e5a23b95b3f293d17eae1" + "2a837ea97900cc30f82e5a23b95b3f293d17eae1", + "c03b3f9eedab7cb0732d1519c4f1a8d90b08eede", + "314a9eb559752132a89b0dbd986db960b3ab9055", + "e83fd0099a69e6eb865e4e6fcadbfb1328c04c85", + "a019acaf8d6fb1f0512414d072f667cc2749b1d9", + "a884fdc0639ae4e17299838ec9de4fddd83cf93c", + "6c827cb5879bc096e45efd992dfadcb96c1d50bc" ] } diff --git a/tools/importer/importer.py b/tools/importer/importer.py index 4694b9ef30..f0c67a61cf 100644 --- a/tools/importer/importer.py +++ b/tools/importer/importer.py @@ -1,80 +1,135 @@ -import os, json, stat, sys, shutil, errno, subprocess, logging -from os.path import dirname, abspath, basename, join +import os, json, stat, sys, shutil, errno, subprocess, logging, re import argparse +from os.path import dirname, abspath, basename, join +# Be sure that the tools directory is in the search path ROOT = abspath(join(dirname(__file__), "../..")) -RTOS_UPDATE_BRANCH = "rtos_update" +sys.path.insert(0, ROOT) + +from tools.utils import run_cmd, delete_dir_files, mkdir, copy_file def del_file(name): + """ Delete the file in RTOS/CMSIS/features directory of mbed-os + Args: + name - name of the file + """ result = [] - search_path = [join(ROOT, 'rtos'), join(ROOT, 'cmsis')] + search_path = [join(ROOT, 'rtos'), join(ROOT, 'cmsis'), join(ROOT, 'features')] for path in search_path: for root, dirs, files in os.walk(path): if name in files: result.append(os.path.join(root, name)) for file in result: - print os.path.relpath(file, ROOT) os.remove(file) + rel_log.debug("Deleted: %s", os.path.relpath(file, ROOT)); -def rmtree(top): - for root, dirs, files in os.walk(top, topdown=False): - for name in files: - filename = os.path.join(root, name) - os.chmod(filename, stat.S_IWUSR) - os.remove(filename) - for name in dirs: - os.rmdir(os.path.join(root, name)) - os.rmdir(top) - -def copy_file(file, path): - try: - shutil.copy(file, path) - except IOError as e: - if e.errno != errno.ENOENT: - raise - ## Create directories - os.makedirs(os.path.dirname(path)) - shutil.copy(file, path) - print os.path.relpath(path, ROOT) - -def copy_folder(folder, path): - files = os.listdir(folder) +def copy_folder(src, dest): + """ Copy contents of folder in mbed-os listed path + Args: + src - src folder path + dest - destination folder path + """ + files = os.listdir(src) for file in files: - abs_src_file = os.path.join(folder, file) + abs_src_file = os.path.join(src, file) if os.path.isfile(abs_src_file): - abs_dst_file = os.path.join(path, file) + abs_dst_file = os.path.join(dest, file) + mkdir(os.path.dirname(abs_dst_file)) copy_file(abs_src_file, abs_dst_file) -def run_cmd(command, exit_on_failure=False): +def run_cmd_with_output(command, exit_on_failure=False): """ Passes a command to the system and returns a True/False result once the - command has been executed, indicating success/failure. Commands are passed - as a list of tokens. + command has been executed, indicating success/failure. If the command was + successful then the output from the command is returned to the caller. + Commands are passed as a list of tokens. E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v'] - """ - return_code = subprocess.call(command, shell=True) - if return_code: - print("The command %s failed with return code: %s" - %(' '.join(command), return_code)) + Args: + command - system command as a list of tokens + exit_on_failure - If True exit the program on failure (default = False) + + Returns: + result - True/False indicating the success/failure of the command + output - The output of the command if it was successful, else empty string + """ + rel_log.debug('[Exec] %s', ' '.join(command)) + returncode = 0 + output = "" + try: + output = subprocess.check_output(command, shell=True) + except subprocess.CalledProcessError as e: + returncode = e.returncode + if exit_on_failure: + rel_log.error("The command %s failed with return code: %s", + (' '.join(command)), returncode) sys.exit(1) + return returncode, output - return return_code +def get_curr_sha(repo_path): + """ Gets the latest SHA for the specified repo + Args: + repo_path - path to the repository -def remove_repo(folder): - os.chdir(abspath(dirname(__file__))) - if os.path.exists(folder): - rmtree(folder) - -def get_repo(repo, branch, folder): - """ Get the Repository files from git, at depth level 1 - repo - Git repository link - branch - repository branch - folder - folder at which repo will be cloned + Returns: + sha - last commit SHA """ - remove_repo(folder) - clone_cmd = ['git', 'clone', repo, "-b", branch, "--depth", '1', folder] - run_cmd(clone_cmd, exit_on_failure=True) + cwd = os.getcwd() + sha = None + os.chdir(abspath(repo_path)) + + cmd = "git log --pretty=format:%h -n 1" + ret, sha = run_cmd_with_output(cmd, exit_on_failure=True) + + os.chdir(cwd) + return sha + +def check_branch(name): + """ Check if branch already exists in mbed-os local repository. + It will not verify if branch is present in remote repository. + Args: + name - branch name + Returns: + True - If branch is already present + """ + branches = [] + cmd = "git branch" + _, output = run_cmd_with_output(cmd, exit_on_failure=False) + if name in output: + return True + return False + +def branch_checkout(name): + """ + Checkout the required branch + Args: + name - branch name + """ + cmd = "git checkout " + name + run_cmd_with_output(cmd, exit_on_failure=False) + +def get_last_cherry_pick_sha(branch): + """ + SHA of last cherry pick commit is returned. SHA should be added to all + cherry-pick commits with -x option. + + Args: + branch - Hash to be verified. + Returns - SHA if found, else None + """ + cmd = "git checkout " + branch + run_cmd_with_output(cmd, exit_on_failure=False) + + SHA = None + get_commit = "git log -n 1" + _, output = run_cmd_with_output(get_commit, exit_on_failure=True) + lines = output.split('\n') + for line in lines: + if 'cherry picked from' in line: + SHA = line.split(' ')[-1] + SHA = SHA[:-1] + return SHA + #for words in output.split('\n') if 'origin' in line and not '->' in line] if __name__ == "__main__": @@ -85,14 +140,14 @@ if __name__ == "__main__": default='INFO') parser.add_argument('-r', '--repo-path', help="Git Repository to be imported", - default=None) + default=None, + required=True) parser.add_argument('-c', '--config-file', help="Configuration file", - default=None) + default=None, + required=True) args = parser.parse_args() - - default = getattr(logging, 'INFO') - level = getattr(logging, args.log_level.upper(), default) + level = getattr(logging, args.log_level.upper()) # Set logging level logging.basicConfig(level=level) @@ -102,58 +157,96 @@ if __name__ == "__main__": rel_log.error("Repository path and config file required as input. Use \"--help\" for more info.") exit(1) - repo = os.path.abspath(args.repo_path) - if not os.path.exists(repo): - rel_log.error("%s not found.", args.repo_path) - exit(1) - json_file = os.path.abspath(args.config_file) if not os.path.isfile(json_file): rel_log.error("%s not found." , args.config_file) exit(1) + repo = os.path.abspath(args.repo_path) + if not os.path.exists(repo): + rel_log.error("%s not found.", args.repo_path) + exit(1) + + sha = get_curr_sha(repo) + if not sha: + rel_log.error("Could not obtain latest SHA") + exit(1) + rel_log.info("%s SHA = %s", os.path.basename(repo), sha) + + branch = 'feature_' + os.path.basename(repo) + '_' + sha + commit_msg = "[" + os.path.basename(repo) + "]" + ": Updated to " + sha + # Read configuration data with open(json_file, 'r') as config: - json_data = json.load(config) - data_files = json_data["files"] - data_folders = json_data["folders"] + json_data = json.load(config) - ## Remove all files listed in .json from mbed-os repo to avoid duplications - print "Cleaning up:" - for file in data_files: - cmsis_file = file['cmsis_file'] - del_file(os.path.basename(cmsis_file)) + ''' + Check if branch exists already, in case branch is present + we will skip all file transfer and merge operations and will + jump to cherry-pick + ''' + if check_branch(branch): + rel_log.info("Branch present = %s", branch) + else: + data_files = json_data["files"] + data_folders = json_data["folders"] - for folder in data_folders: - files = os.listdir(repo_path) - for file in files: - del_file(os.path.basename(file)) + ## Remove all files listed in .json from mbed-os repo to avoid duplications + for file in data_files: + src_file = file['src_file'] + del_file(os.path.basename(src_file)) - ## Copy all the CMSIS files listed in json file to mbed-os - print "Files Copied:" - for file in data_files: - repo_file = os.path.join(repo_path, file['cmsis_file']) - mbed_path = os.path.join(ROOT, file['mbed_file']) - copy_file(repo_file, mbed_path) + for folder in data_folders: + dest_folder = folder['dest_folder'] + delete_dir_files(dest_folder) + rel_log.debug("Deleted = %s", folder) - for folder in data_folders: - repo_folder = os.path.join(repo_path, folder['cmsis_folder']) - mbed_path = os.path.join(ROOT, folder['mbed_folder']) - copy_folder(repo_folder, mbed_path) + rel_log.info("Removed files/folders listed in json file") - ## Create new branch with all changes - create_branch = ['git', 'checkout', '-b', RTOS_UPDATE_BRANCH] - run_cmd(create_branch, exit_on_failure=True) + ## Copy all the CMSIS files listed in json file to mbed-os + for file in data_files: + repo_file = os.path.join(repo, file['src_file']) + mbed_path = os.path.join(ROOT, file['dest_file']) + mkdir(os.path.dirname(mbed_path)) + copy_file(repo_file, mbed_path) + rel_log.debug("Copied = %s", mbed_path) - add_files = ['git', 'add', '-A'] - run_cmd(add_files, exit_on_failure=True) + for folder in data_folders: + repo_folder = os.path.join(repo, folder['src_folder']) + mbed_path = os.path.join(ROOT, folder['dest_folder']) + copy_folder(repo_folder, mbed_path) + rel_log.debug("Copied = %s", mbed_path) - commit_branch = ['git', 'commit', '-m', "CMSIS/RTX: Update CMSIS/RTX"] - run_cmd(commit_branch, exit_on_failure=True) + ## Create new branch with all changes + create_branch = "git checkout -b "+ branch + run_cmd_with_output(create_branch, exit_on_failure=False) + rel_log.info("Branch created = %s", branch) - ## Apply commits specific to mbed-os changes - mbed_sha = json_data["Mbed_sha"] + add_files = "git add -A" + run_cmd_with_output(add_files, exit_on_failure=True) - for sha in mbed_sha: - cherry_pick_sha = ['git', 'cherry-pick', sha] - run_cmd(cherry_pick_sha, exit_on_failure=True) + commit_branch = "git commit -m \"" + commit_msg + "\"" + run_cmd_with_output(commit_branch, exit_on_failure=True) + rel_log.info("Commit added = %s", mbed_path) + + ## Checkout the feature branch + branch_checkout(branch) + commit_sha = json_data["commit_sha"] + last_sha = get_last_cherry_pick_sha(branch) + if not last_sha: + ## Apply commits specific to mbed-os changes + for sha in commit_sha: + cherry_pick_sha = "git cherry-pick -x " + sha + run_cmd_with_output(cherry_pick_sha, exit_on_failure=True) + rel_log.info("Commit added = %s", cherry_pick_sha) + ## Few commits are already applied, check the next in sequence + ## and skip to last applied + else: + found = False + for sha in commit_sha: + if sha == last_sha: + found = True + continue + if found is True: + cherry_pick_sha = "git cherry-pick -x " + sha + run_cmd_with_output(cherry_pick_sha, exit_on_failure=True) From 51430cdeb30a9f4d575963dd049732ae4c997bf0 Mon Sep 17 00:00:00 2001 From: Deepika Date: Mon, 13 Nov 2017 15:05:58 -0600 Subject: [PATCH 5/6] Updated code as per Python style guide https://www.python.org/dev/peps/pep-0008/ --- tools/importer/importer.py | 50 ++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/tools/importer/importer.py b/tools/importer/importer.py index f0c67a61cf..e3a6401d93 100644 --- a/tools/importer/importer.py +++ b/tools/importer/importer.py @@ -1,6 +1,10 @@ -import os, json, stat, sys, shutil, errno, subprocess, logging, re +import os +import json +import sys +import subprocess +import logging import argparse -from os.path import dirname, abspath, basename, join +from os.path import dirname, abspath, join # Be sure that the tools directory is in the search path ROOT = abspath(join(dirname(__file__), "../..")) @@ -21,7 +25,7 @@ def del_file(name): result.append(os.path.join(root, name)) for file in result: os.remove(file) - rel_log.debug("Deleted: %s", os.path.relpath(file, ROOT)); + rel_log.debug("Deleted: %s", os.path.relpath(file, ROOT)) def copy_folder(src, dest): """ Copy contents of folder in mbed-os listed path @@ -75,16 +79,15 @@ def get_curr_sha(repo_path): sha - last commit SHA """ cwd = os.getcwd() - sha = None os.chdir(abspath(repo_path)) cmd = "git log --pretty=format:%h -n 1" - ret, sha = run_cmd_with_output(cmd, exit_on_failure=True) + _, sha = run_cmd_with_output(cmd, exit_on_failure=True) os.chdir(cwd) return sha -def check_branch(name): +def branch_exists(name): """ Check if branch already exists in mbed-os local repository. It will not verify if branch is present in remote repository. Args: @@ -92,7 +95,7 @@ def check_branch(name): Returns: True - If branch is already present """ - branches = [] + cmd = "git branch" _, output = run_cmd_with_output(cmd, exit_on_failure=False) if name in output: @@ -120,16 +123,15 @@ def get_last_cherry_pick_sha(branch): cmd = "git checkout " + branch run_cmd_with_output(cmd, exit_on_failure=False) - SHA = None + sha = None get_commit = "git log -n 1" _, output = run_cmd_with_output(get_commit, exit_on_failure=True) lines = output.split('\n') for line in lines: if 'cherry picked from' in line: - SHA = line.split(' ')[-1] - SHA = SHA[:-1] - return SHA - #for words in output.split('\n') if 'origin' in line and not '->' in line] + sha = line.split(' ')[-1] + return sha[:-1] + return sha if __name__ == "__main__": @@ -138,7 +140,7 @@ if __name__ == "__main__": parser.add_argument('-l', '--log-level', help="Level for providing logging output", default='INFO') - parser.add_argument('-r', '--repo-path', + parser.add_argument('-r', '--repo-path', help="Git Repository to be imported", default=None, required=True) @@ -153,13 +155,13 @@ if __name__ == "__main__": logging.basicConfig(level=level) rel_log = logging.getLogger("Importer") - if (args.repo_path is None) or (args.config_file is None) : + if (args.repo_path is None) or (args.config_file is None): rel_log.error("Repository path and config file required as input. Use \"--help\" for more info.") exit(1) json_file = os.path.abspath(args.config_file) if not os.path.isfile(json_file): - rel_log.error("%s not found." , args.config_file) + rel_log.error("%s not found.", args.config_file) exit(1) repo = os.path.abspath(args.repo_path) @@ -178,14 +180,14 @@ if __name__ == "__main__": # Read configuration data with open(json_file, 'r') as config: - json_data = json.load(config) + json_data = json.load(config) ''' Check if branch exists already, in case branch is present we will skip all file transfer and merge operations and will jump to cherry-pick ''' - if check_branch(branch): + if branch_exists(branch): rel_log.info("Branch present = %s", branch) else: data_files = json_data["files"] @@ -193,11 +195,11 @@ if __name__ == "__main__": ## Remove all files listed in .json from mbed-os repo to avoid duplications for file in data_files: - src_file = file['src_file'] + src_file = file['src_file'] del_file(os.path.basename(src_file)) for folder in data_folders: - dest_folder = folder['dest_folder'] + dest_folder = folder['dest_folder'] delete_dir_files(dest_folder) rel_log.debug("Deleted = %s", folder) @@ -205,21 +207,21 @@ if __name__ == "__main__": ## Copy all the CMSIS files listed in json file to mbed-os for file in data_files: - repo_file = os.path.join(repo, file['src_file']) - mbed_path = os.path.join(ROOT, file['dest_file']) + repo_file = os.path.join(repo, file['src_file']) + mbed_path = os.path.join(ROOT, file['dest_file']) mkdir(os.path.dirname(mbed_path)) copy_file(repo_file, mbed_path) rel_log.debug("Copied = %s", mbed_path) for folder in data_folders: - repo_folder = os.path.join(repo, folder['src_folder']) - mbed_path = os.path.join(ROOT, folder['dest_folder']) + repo_folder = os.path.join(repo, folder['src_folder']) + mbed_path = os.path.join(ROOT, folder['dest_folder']) copy_folder(repo_folder, mbed_path) rel_log.debug("Copied = %s", mbed_path) ## Create new branch with all changes create_branch = "git checkout -b "+ branch - run_cmd_with_output(create_branch, exit_on_failure=False) + run_cmd_with_output(create_branch, exit_on_failure=True) rel_log.info("Branch created = %s", branch) add_files = "git add -A" From d4f7291b9abf8ffea9368bbec35d5c90826601e0 Mon Sep 17 00:00:00 2001 From: Deepika Date: Wed, 15 Nov 2017 11:31:30 -0600 Subject: [PATCH 6/6] Corrected the input to remove command --- tools/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/utils.py b/tools/utils.py index 68c9631ba1..7ab791f7f7 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -199,7 +199,7 @@ def delete_dir_files(directory): for element in listdir(directory): to_remove = join(directory, element) if not isdir(to_remove): - remove(file) + remove(to_remove) def get_caller_name(steps=2):