Merge pull request #10505 from orenc17/imporve_importer

Improve importer.py

tools/importer/README.md

@@ -16,6 +16,7 @@ Note: Only files present in folder will be copied, directories inside the folder
 `commit_sha` is list of commits present in mbed-os repo. These commits will be applied after copying files and folders listed above.Each commit in the commit_sha list is cherry-picked and applied with the -x option, which records the SHA of the source commit in the commit message.
 Note: You must resolve any conflicts that arise during this cherry-pick process. Make sure that the "(cherry picked from commit ...)" statement is present in the commit message. Re-execute the python script to apply rest of the SHA commits.
 
+```json
 {
   "files" : [
     {
@@ -41,9 +42,10 @@ Note: You must resolve any conflicts that arise during this cherry-pick process.
   ],
    "commit_sha" : [
       "428acae1b2ac15c3ad523e8d40755a9301220822",
-      "d9d622afe0ca8c7ab9d24c17f9fe59b54dcc61c9",
+      {"sha": "d9d622afe0ca8c7ab9d24c17f9fe59b54dcc61c9", "msg": "Fix ..."}
     ]
 }
+```
 
 ### Input to importer.py
 1. Repository: -r <repo_path> ( Example: CMSIS / Mbed-tls)
@@ -52,4 +54,3 @@ Note: You must resolve any conflicts that arise during this cherry-pick process.
 For example the command below can be used to update CMSIS:
 `python tools\importer\importer.py -c tools\importer\cmsis_importer.json -r <path to cmsis repo>`
 
-Note: This script must be run from the mbed-os directory to work correctly.

tools/importer/importer.py

@@ -1,3 +1,4 @@
+#!/usr/bin/python
 """
 Copyright (c) 2017-2019 ARM Limited. All rights reserved.
 
@@ -22,184 +23,264 @@ import sys
 import subprocess
 import logging
 import argparse
+import re
 from os.path import dirname, abspath, join, isfile, normpath
 
 # Be sure that the tools directory is in the search path
-ROOT = abspath(join(dirname(__file__), "../.."))
+ROOT = abspath(join(dirname(__file__), os.path.pardir, os.path.pardir))
 sys.path.insert(0, ROOT)
 
-from tools.utils import run_cmd, delete_dir_files, mkdir, copy_file
+from tools.utils import delete_dir_files, mkdir, copy_file
+
+rel_log = logging.getLogger("Importer")
+
+
+class StoreDir(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        directory = abspath(values)
+        if not os.path.isdir(directory):
+            raise argparse.ArgumentError(
+                None, "The directory %s does not exist!" % directory)
+        setattr(namespace, self.dest, directory)
+
+
+class SetLogLevel(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        logging.basicConfig(level=values)
+
+
+class DirContext(object):
+    def __init__(self, dir):
+        self.dir = dir
+
+    def __enter__(self):
+        _backup_dir = os.getcwd()
+        os.chdir(self.dir)
+        self.dir = _backup_dir
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        os.chdir(self.dir)
+
 
 def del_file(name):
-    """ Delete the file in RTOS/CMSIS/features directory of mbed-os
-    Args:
-    name - name of the file
     """
+    Delete the file in RTOS/CMSIS/features directory of mbed-os.
+
+    :param name: Name of the file.
+    :return: None.
+    """
+
     result = []
-    search_path = [join(ROOT, 'rtos'), join(ROOT, 'cmsis'), join(ROOT, 'features')]
+    search_path = [join(ROOT, 'rtos'), join(ROOT, 'cmsis'),
+                   join(ROOT, 'features')]
     for path in search_path:
         for root, dirs, files in os.walk(path):
             if name in files:
                 result.append(join(root, name))
-    for file in result:
+    for f in result:
-        os.remove(file)
+        os.remove(f)
-        rel_log.debug("Deleted %s", os.path.relpath(file, ROOT))
+        rel_log.debug("Deleted %s", os.path.relpath(f, ROOT))
 
-def copy_folder(src, dest):
+
-    """ Copy contents of folder in mbed-os listed path
+def copy_folder(src, dst):
-    Args:
-    src - src folder path
-    dest - destination folder path
     """
+    Copy contents of folder in mbed-os listed path.
+
+    :param src: Source folder path.
+    :param dst: Destination folder path.
+    :return: None.
+    """
+
     files = os.listdir(src)
-    for file in files:
+    for f in files:
-        abs_src_file = join(src, file)
+        abs_src_file = join(src, f)
-        if os.path.isfile(abs_src_file):
+        if isfile(abs_src_file):
-            abs_dst_file = join(dest, file)
+            abs_dst_file = join(dst, f)
             mkdir(dirname(abs_dst_file))
             copy_file(abs_src_file, abs_dst_file)
 
+
+def import_files(repo_path, data_files, data_folders):
+    """
+    Imports files and directories to mbed-os
+
+    :param repo_path: Path to the repo copying from.
+    :param data_files: List of files to be copied. (with destination)
+    :param data_folders: List of directories to be copied. (with destination)
+    :return: None
+    """
+
+    # Remove all files listed in .json from mbed-os repo to avoid duplications
+    for fh in data_files:
+        src_file = fh['src_file']
+        del_file(os.path.basename(src_file))
+        dest_file = join(ROOT, fh['dest_file'])
+        if isfile(dest_file):
+            os.remove(join(ROOT, dest_file))
+            rel_log.debug("Deleted %s", fh['dest_file'])
+    for folder in data_folders:
+        dest_folder = folder['dest_folder']
+        delete_dir_files(dest_folder)
+        rel_log.debug("Deleted: %s", folder['dest_folder'])
+    rel_log.info("Removed files/folders listed in json file")
+
+    # Copy all the files listed in json file to mbed-os
+    for fh in data_files:
+        repo_file = join(repo_path, fh['src_file'])
+        mbed_path = join(ROOT, fh['dest_file'])
+        mkdir(dirname(mbed_path))
+        copy_file(repo_file, mbed_path)
+        rel_log.debug("Copied %s to %s", normpath(repo_file),
+                      normpath(mbed_path))
+    for folder in data_folders:
+        repo_folder = join(repo_path, folder['src_folder'])
+        mbed_path = join(ROOT, folder['dest_folder'])
+        copy_folder(repo_folder, mbed_path)
+        rel_log.debug("Copied %s to %s", normpath(repo_folder),
+                      normpath(mbed_path))
+
+
 def run_cmd_with_output(command, exit_on_failure=False):
-    """ Passes a command to the system and returns a True/False result once the
+    """
+    Passes a command to the system and returns a True/False result once the
     command has been executed, indicating success/failure. If the command was
     successful then the output from the command is returned to the caller.
     Commands are passed as a list of tokens.
-        E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v']
+    E.g. The command 'git remote -v' would be passed in as:
+     ['git', 'remote', '-v']
 
-    Args:
+    :param command: System command as a list of tokens.
-    command - system command as a list of tokens
+    :param exit_on_failure: Exit the program on failure (default=False)
-    exit_on_failure - If True exit the program on failure (default = False)
+    :return: Command return status code and output as tuple.
-
-    Returns:
-    result - True/False indicating the success/failure of the command
-    output - The output of the command if it was successful, else empty string
     """
+
     rel_log.debug('[Exec] %s', ' '.join(command))
-    returncode = 0
+    return_code = 0
     output = ""
     try:
         output = subprocess.check_output(command)
     except subprocess.CalledProcessError as e:
-        returncode = e.returncode
+        return_code = e.returncode
 
         if exit_on_failure:
             rel_log.error("The command %s failed with return code: %s",
-                        (' '.join(command)), returncode)
+                          (' '.join(command)), return_code)
             sys.exit(1)
-    return returncode, output
+    return return_code, output
+
 
 def get_curr_sha(repo_path):
-    """ Gets the latest SHA for the specified repo
-    Args:
-    repo_path - path to the repository
-
-    Returns:
-    sha - last commit SHA
     """
-    cwd = os.getcwd()
+    Gets the latest SHA for the specified repo.
-    os.chdir(abspath(repo_path))
 
-    cmd = ['git', 'log', '--pretty=format:%h', '-n', '1']
+    :param repo_path: Path to a git repository.
-    _, sha = run_cmd_with_output(cmd, exit_on_failure=True)
+    :return: Last commit SHA.
+    """
+
+    cmd = ['git', '-C', repo_path, 'log', '--pretty=format:%h', '-n', '1']
+    _, _sha = run_cmd_with_output(cmd, exit_on_failure=True)
+
+    if not _sha:
+        rel_log.error("Could not obtain latest SHA")
+        sys.exit(1)
+
+    rel_log.info("%s SHA = %s", repo_path, _sha)
+    return _sha
 
-    os.chdir(cwd)
-    return sha
 
 def branch_exists(name):
-    """ Check if branch already exists in mbed-os local repository.
+    """
-    It will not verify if branch is present in remote repository.
+    Check if branch already exists in mbed-os local repository.
-    Args:
+
-    name - branch name
+    :param name: Branch name.
-    Returns:
+    :return: True if branch is already present, False otherwise.
-    True - If branch is already present
     """
 
     cmd = ['git', 'branch']
     _, output = run_cmd_with_output(cmd, exit_on_failure=False)
-    if name in output:
+
-        return True
+    return name in output
-    return False
+
 
 def branch_checkout(name):
     """
-    Checkout the required branch
+    Checkout the required git branch.
-    Args:
+
-    name - branch name
+    :param name: Branch to checkout.
+    :return: None.
     """
+
     cmd = ['git', 'checkout', name]
     _, _ = run_cmd_with_output(cmd, exit_on_failure=False)
     rel_log.info("Checkout to branch %s", name)
 
-def get_last_cherry_pick_sha(branch):
-    """
-    SHA of last cherry pick commit is returned. SHA should be added to all
-    cherry-pick commits with -x option.
 
-    Args:
+def get_last_cherry_pick_sha():
-    branch - Hash to be verified.
+    """
-    Returns - SHA if found, else None
+    Finds the SHA of last cherry picked commit.
+    SHA should be added to cherry-pick commits with -x option.
+
+    :return: SHA if found, None otherwise.
     """
-    cmd = ['git', 'checkout', branch]
-    run_cmd_with_output(cmd, exit_on_failure=False)
 
-    sha = None
     get_commit = ['git', 'log', '-n', '1']
     _, output = run_cmd_with_output(get_commit, exit_on_failure=True)
-    lines = output.split('\n')
-    for line in lines:
-        if 'cherry picked from' in line:
-            sha = line.split(' ')[-1]
-            return sha[:-1]
-    return sha
 
-if __name__ == "__main__":
+    shas = re.findall(
+        pattern='^\s*\(cherry picked from commit ([0-9a-fA-F]+)\)$',
+        string=output,
+        flags=re.MULTILINE
+    )
 
-    parser = argparse.ArgumentParser(description=__doc__,
+    return shas[-1] if shas else None
+
+
+def normalize_commit_sha(sha_lst):
+    """
+    The commit_sha section of the config file can hold commits in 2 ways:
+    * "<SHA>" - E.g. "428acae1b2ac15c3ad523e8d40755a9301220822".
+    * {"sha": "<SHA>", "msg": "<HELP>"} - E.g.
+    {"sha": "d9d622afe0ca8c7ab9d24c17f9fe59b54dcc61c9", "msg": "Fix ..."}.
+
+    :param sha_lst: JSON data from config file.
+    :return: list of commit SHA.
+    """
+    return [_sha['sha'] if isinstance(_sha, dict) else _sha for _sha in sha_lst]
+
+
+def get_parser():
+    parser = argparse.ArgumentParser(
+        description=__doc__,
         formatter_class=argparse.RawDescriptionHelpFormatter)
+
     parser.add_argument('-l', '--log-level',
                         help="Level for providing logging output",
-                        default='INFO')
+                        choices=['DEBUG', 'INFO', 'ERROR'],
+                        default='INFO',
+                        action=SetLogLevel)
     parser.add_argument('-r', '--repo-path',
                         help="Git Repository to be imported",
-                        default=None,
+                        required=True,
-                        required=True)
+                        action=StoreDir)
     parser.add_argument('-c', '--config-file',
                         help="Configuration file",
-                        default=None,
+                        required=True,
-                        required=True)
+                        type=argparse.FileType('r'))
+
+    return parser
+
+
+def main():
+    parser = get_parser()
     args = parser.parse_args()
-    level = getattr(logging, args.log_level.upper())
+    sha = get_curr_sha(args.repo_path)
-
+    repo_dir = os.path.basename(args.repo_path)
-    # Set logging level
+    branch = 'feature_' + repo_dir + '_' + sha
-    logging.basicConfig(level=level)
+    commit_msg = "[" + repo_dir + "]" + ": Updated to " + sha
-    rel_log = logging.getLogger("Importer")
-
-    if (args.repo_path is None) or (args.config_file is None):
-        rel_log.error("Repository path and config file required as input. Use \"--help\" for more info.")
-        exit(1)
-
-    json_file = abspath(args.config_file)
-    if not os.path.isfile(json_file):
-        rel_log.error("%s not found.", args.config_file)
-        exit(1)
-
-    repo = abspath(args.repo_path)
-    if not os.path.exists(repo):
-        rel_log.error("%s not found.", args.repo_path)
-        exit(1)
-
-    sha = get_curr_sha(repo)
-    if not sha:
-        rel_log.error("Could not obtain latest SHA")
-        exit(1)
-    rel_log.info("%s SHA = %s", os.path.basename(repo), sha)
-
-    branch = 'feature_' + os.path.basename(repo) + '_' + sha
-    commit_msg = "[" + os.path.basename(repo) + "]" + ": Updated to " + sha
 
     # Read configuration data
-    with open(json_file, 'r') as config:
+    json_data = json.load(args.config_file)
-        json_data = json.load(config)
 
     '''
     Check if branch exists already, in case branch is present
@@ -211,38 +292,9 @@ if __name__ == "__main__":
     else:
         data_files = json_data["files"]
         data_folders = json_data["folders"]
+        import_files(args.repo_path, data_files, data_folders)
 
-        ## Remove all files listed in .json from mbed-os repo to avoid duplications
+        # Create new branch with all changes
-        for file in data_files:
-            src_file = file['src_file']
-            del_file(os.path.basename(src_file))
-            dest_file = join(ROOT, file['dest_file'])
-            if isfile(dest_file):
-                os.remove(join(ROOT, dest_file))
-                rel_log.debug("Deleted %s", file['dest_file'])
-
-        for folder in data_folders:
-            dest_folder = folder['dest_folder']
-            delete_dir_files(dest_folder)
-            rel_log.debug("Deleted: %s", folder['dest_folder'])
-
-        rel_log.info("Removed files/folders listed in json file")
-
-        ## Copy all the files listed in json file to mbed-os
-        for file in data_files:
-            repo_file = join(repo, file['src_file'])
-            mbed_path = join(ROOT, file['dest_file'])
-            mkdir(dirname(mbed_path))
-            copy_file(repo_file, mbed_path)
-            rel_log.debug("Copied %s to %s", normpath(repo_file), normpath(mbed_path))
-
-        for folder in data_folders:
-            repo_folder = join(repo, folder['src_folder'])
-            mbed_path = join(ROOT, folder['dest_folder'])
-            copy_folder(repo_folder, mbed_path)
-            rel_log.debug("Copied %s to %s", normpath(repo_folder), normpath(mbed_path))
-
-        ## Create new branch with all changes
         create_branch = ['git', 'checkout', '-b', branch]
         run_cmd_with_output(create_branch, exit_on_failure=True)
         rel_log.info("Branch created: %s", branch)
@@ -254,25 +306,32 @@ if __name__ == "__main__":
         run_cmd_with_output(commit_branch, exit_on_failure=True)
         rel_log.info('Commit added: "%s"', commit_msg)
 
-    ## Checkout the feature branch
+    # Checkout the feature branch
     branch_checkout(branch)
-    commit_sha = json_data["commit_sha"]
+    commit_sha = normalize_commit_sha(json_data["commit_sha"])
-    last_sha = get_last_cherry_pick_sha(branch)
+    last_sha = get_last_cherry_pick_sha()
-    if not last_sha:
+
-        ## Apply commits specific to mbed-os changes
+    # Few commits are already applied, check the next in sequence
+    # and skip to next commit
+    if last_sha:
+        assert last_sha in commit_sha, "%s not found in config file" % last_sha
+        # Calculate the index of the next sha to be applied
+        next_sha_idx = commit_sha.index(last_sha) + 1
+        if next_sha_idx >= len(commit_sha):
+            rel_log.info("No more commits to apply")
+            sys.exit(0)
+        # Skipping applied commits
+        commit_sha = commit_sha[next_sha_idx:]
+
+    # Apply commits specific to mbed-os changes
     for sha in commit_sha:
         cherry_pick_sha = ['git', 'cherry-pick', '-x', sha]
+        rel_log.info("Cherry-picking commit = %s", sha)
         run_cmd_with_output(cherry_pick_sha, exit_on_failure=True)
-            rel_log.info("Cherry-picked commit = %s", sha)
+
-    ## Few commits are already applied, check the next in sequence
+    rel_log.info("Finished import successfully :)")
-    ## and skip to last applied
+
-    else:
+
-        found = False
+if __name__ == "__main__":
-        for sha in commit_sha:
+    with DirContext(ROOT):
-            if sha == last_sha:
+        main()
-                found = True
-                continue
-            if found is True:
-                cherry_pick_sha = ['git', 'cherry-pick', '-x', sha]
-                run_cmd_with_output(cherry_pick_sha, exit_on_failure=True)
-                rel_log.info("Cherry-picked commit = %s", sha)