Allow Mbed config files to be json5 in addition to json (#194)

* Allow Mbed config files to be json5 in addition to json

* Also handle custom_targets.json

* Add missing "install python packages" step

.github/workflows/basic_checks.yml

@@ -214,6 +214,11 @@ jobs:
         name: Checkout repo
         uses: actions/checkout@v3
 
+
+      - name: Install Python packages
+        run: |
+          python3 -m pip install -r tools/requirements.txt
+
       -
         name: cmake build
         run: |

drivers/mbed_lib.json5

@@ -1,6 +1,8 @@
 {
     "name": "drivers",
     "config": {
+        // Note: These are called "uart-serial" because that was the old name
+        // of BufferedSerial before it was officially merged
         "uart-serial-txbuf-size": {
             "help": "Default TX buffer size for a BufferedSerial instance (unit Bytes))",
             "value": 256

targets/targets.json5

@@ -5126,6 +5126,8 @@
         "mbed_ram_start": "0x20000000",
         "mbed_ram_size": "0x10000"
     },
+    // Note: "MIMXRT105X" target is for the MIMXRT1051, MIMXRT1061=2, MIMXRT1061, and MIMXRT1062
+    // See here for details: https://github.com/mbed-ce/mbed-os/wiki/MCU-Info-Page:-MIMXRT105x-and-106x
     "MIMXRT105X": {
         "core": "Cortex-M7FD",
         "supported_toolchains": [

tools/python/mbed_tools/build/_internal/config/assemble_build_config.py

@@ -32,14 +32,13 @@ def assemble_config(target_attributes: dict, search_paths: Iterable[Path], mbed_
         search_paths: Iterable of paths to search for mbed_lib.json files.
         mbed_app_file: The path to mbed_app.json. This can be None.
     """
-    mbed_lib_files = list(
+    mbed_lib_files: Set[Path] = set()
-        set(
+
-            itertools.chain.from_iterable(
+    for path in search_paths:
-                find_files("mbed_lib.json", path.absolute().resolve()) for path in search_paths
+        mbed_lib_files.update(find_files("mbed_lib.json", path.absolute().resolve()))
-            )
+        mbed_lib_files.update(find_files("mbed_lib.json5", path.absolute().resolve()))
-        )
+
-    )
+    return _assemble_config_from_sources(target_attributes, list(mbed_lib_files), mbed_app_file)
-    return _assemble_config_from_sources(target_attributes, mbed_lib_files, mbed_app_file)
 
 
 def _assemble_config_from_sources(

tools/python/mbed_tools/lib/json_helpers.py

@@ -4,6 +4,7 @@
 #
 """Helpers for json related functions."""
 import json
+import json5
 import logging
 
 from pathlib import Path
@@ -14,9 +15,19 @@ logger = logging.getLogger(__name__)
 
 def decode_json_file(path: Path) -> Any:
     """Return the contents of json file."""
-    try:
+    if path.suffix == '.json':
-        logger.debug(f"Loading JSON file {path}")
+        try:
-        return json.loads(path.read_text())
+            logger.debug(f"Loading JSON file {path}")
-    except json.JSONDecodeError:
+            return json.loads(path.read_text())
-        logger.error(f"Failed to decode JSON data in the file located at '{path}'")
+        except json.JSONDecodeError:
-        raise
+            logger.error(f"Failed to decode JSON data in the file located at '{path}'")
+            raise
+    elif path.suffix == '.json5':
+        try:
+            logger.debug(f"Loading JSON file {path}")
+            return json5.loads(path.read_text())
+        except ValueError:
+            logger.error(f"Failed to decode JSON data in the file located at '{path}'")
+            raise
+    else:
+        raise ValueError(f"Unknown JSON file extension {path.suffix}")

tools/python/mbed_tools/project/_internal/project_data.py

@@ -19,14 +19,16 @@ from mbed_tools.project._internal.render_templates import (
 logger = logging.getLogger(__name__)
 
 # Mbed program file names and constants.
-APP_CONFIG_FILE_NAME = "mbed_app.json"
+APP_CONFIG_FILE_NAME_JSON = "mbed_app.json"
+APP_CONFIG_FILE_NAME_JSON5 = "mbed_app.json5"
 BUILD_DIR = "cmake_build"
 CMAKELISTS_FILE_NAME = "CMakeLists.txt"
 MAIN_CPP_FILE_NAME = "main.cpp"
 MBED_OS_REFERENCE_FILE_NAME = "mbed-os.lib"
 MBED_OS_DIR_NAME = "mbed-os"
-TARGETS_JSON_FILE_PATH = Path("targets", "targets.json")
+TARGETS_JSON_FILE_PATH = Path("targets", "targets.json5")
 CUSTOM_TARGETS_JSON_FILE_NAME = "custom_targets.json"
+CUSTOM_TARGETS_JSON5_FILE_NAME = "custom_targets.json5"
 
 # Information written to mbed-os.lib
 MBED_OS_REFERENCE_URL = "https://github.com/ARMmbed/mbed-os"
@@ -71,7 +73,7 @@ class MbedProgramFiles:
         Raises:
             ValueError: A program .mbed or mbed-os.lib file already exists at this path.
         """
-        app_config = root_path / APP_CONFIG_FILE_NAME
+        app_config = root_path / APP_CONFIG_FILE_NAME_JSON5
         mbed_os_ref = root_path / MBED_OS_REFERENCE_FILE_NAME
         cmakelists_file = root_path / CMAKELISTS_FILE_NAME
         main_cpp = root_path / MAIN_CPP_FILE_NAME
@@ -103,13 +105,21 @@ class MbedProgramFiles:
             root_path: The path containing the MbedProgramFiles.
             build_subdir: The subdirectory of BUILD_DIR to use for CMake build.
         """
-        app_config: Optional[Path]
+        app_config: Optional[Path] = None
-        app_config = root_path / APP_CONFIG_FILE_NAME
+        if (root_path / APP_CONFIG_FILE_NAME_JSON5).exists():
-        if not app_config.exists():
+            app_config = root_path / APP_CONFIG_FILE_NAME_JSON5
+        elif (root_path / APP_CONFIG_FILE_NAME_JSON).exists():
+            app_config = root_path / APP_CONFIG_FILE_NAME_JSON
+        else:
             logger.info("This program does not contain an mbed_app.json config file.")
-            app_config = None
 
-        custom_targets_json = root_path / CUSTOM_TARGETS_JSON_FILE_NAME
+        # If there's already a custom_targets.json5, use that.
+        # Otherwise, assume json.
+        if (root_path / CUSTOM_TARGETS_JSON5_FILE_NAME).exists():
+            custom_targets_json = root_path / CUSTOM_TARGETS_JSON5_FILE_NAME
+        else:
+            custom_targets_json = root_path / CUSTOM_TARGETS_JSON_FILE_NAME
+
         mbed_os_file = root_path / MBED_OS_REFERENCE_FILE_NAME
 
         cmakelists_file = root_path / CMAKELISTS_FILE_NAME

tools/python/python_tests/mbed_tools/lib/test_json_helpers.py

@@ -15,3 +15,11 @@ def test_invalid_json(tmp_path):
 
     with pytest.raises(json.JSONDecodeError):
         decode_json_file(lib_json_path)
+
+
+def test_invalid_json5(tmp_path):
+    lib_json_path = tmp_path / "mbed_lib.json5"
+    lib_json_path.write_text("name")
+
+    with pytest.raises(ValueError):
+        decode_json_file(lib_json_path)

tools/python/python_tests/mbed_tools/project/_internal/test_project_data.py

@@ -12,6 +12,7 @@ from mbed_tools.project._internal.project_data import (
     MbedProgramFiles,
     MbedOS,
     MAIN_CPP_FILE_NAME,
+    APP_CONFIG_FILE_NAME_JSON
 )
 from python_tests.mbed_tools.project.factories import make_mbed_lib_reference, make_mbed_program_files, make_mbed_os_files
 
@@ -59,6 +60,22 @@ class TestMbedProgramFiles:
         assert program.mbed_os_ref.exists()
         assert program.cmakelists_file.exists()
 
+    def test_from_existing_finds_existing_program_data_app_json(self, tmp_path):
+        """
+        Same as test_from_existing_finds_existing_program_data() except the app config
+        is json instead of json5
+        """
+
+        root = pathlib.Path(tmp_path, "foo")
+        make_mbed_program_files(root, APP_CONFIG_FILE_NAME_JSON)
+
+        program = MbedProgramFiles.from_existing(root, pathlib.Path("K64F", "develop", "GCC_ARM"))
+
+        assert program.app_config_file.exists()
+        assert program.mbed_os_ref.exists()
+        assert program.cmakelists_file.exists()
+
+
 
 class TestMbedLibReference:
     def test_is_resolved_returns_true_if_source_code_dir_exists(self, tmp_path):
@@ -95,7 +112,7 @@ class TestMbedOS:
 
         mbed_os = MbedOS.from_existing(root_path)
 
-        assert mbed_os.targets_json_file == root_path / "targets" / "targets.json"
+        assert mbed_os.targets_json_file == root_path / "targets" / "targets.json5"
 
     def test_raises_if_files_missing(self, tmp_path):
         root_path = pathlib.Path(tmp_path, "my-version-of-mbed-os")

tools/python/python_tests/mbed_tools/project/factories.py

@@ -5,12 +5,12 @@
 from mbed_tools.project._internal.libraries import MbedLibReference
 from mbed_tools.project._internal.project_data import (
     CMAKELISTS_FILE_NAME,
-    APP_CONFIG_FILE_NAME,
+    APP_CONFIG_FILE_NAME_JSON5,
     MBED_OS_REFERENCE_FILE_NAME,
 )
 
 
-def make_mbed_program_files(root, config_file_name=APP_CONFIG_FILE_NAME):
+def make_mbed_program_files(root, config_file_name=APP_CONFIG_FILE_NAME_JSON5):
     if not root.exists():
         root.mkdir()
 
@@ -42,4 +42,4 @@ def make_mbed_os_files(root):
 
     targets_dir = root / "targets"
     targets_dir.mkdir()
-    (targets_dir / "targets.json").touch()
+    (targets_dir / "targets.json5").touch()

tools/python/python_tests/mbed_tools/regression/test_configure.py

@@ -51,7 +51,7 @@ class TestConfigureRegression(TestCase):
             pathlib.Path(tmpDirPath / "mbed_app.json").write_text(mbed_app_json)
             pathlib.Path(tmpDirPath / "mbed-os").mkdir()
             pathlib.Path(tmpDirPath / "mbed-os" / "targets").mkdir()
-            pathlib.Path(tmpDirPath / "mbed-os" / "targets" / "targets.json").write_text(target_json)
+            pathlib.Path(tmpDirPath / "mbed-os" / "targets" / "targets.json5").write_text(target_json)
 
             result = CliRunner().invoke(
                 configure, ["-m", "Target", "-t", "gcc_arm", "-p", tmpDir], catch_exceptions=False

tools/requirements.apt.txt

@@ -18,3 +18,4 @@ python3-lockfile
 python3-junit.xml
 python3-cryptography
 python3-cbor
+python3-json5

tools/requirements.txt

@@ -24,6 +24,7 @@ junit-xml>=1.0,<2.0
 lockfile
 six>=1.0,<2.0
 colorama>=0.3,<0.5
+json5
 
 # beautifulsoup only needed for USB device detection on Mac
 beautifulsoup4; sys_platform == 'darwin'