Rework config_test to use py.test

.travis.yml

@@ -19,11 +19,10 @@ script:
   - |
     find  -name "*.s" | tee BUILD/badasm | sed -e "s/^/Bad Assembler file name found: /" && [ ! -s BUILD/badasm ]
   - make -C events/equeue test clean
-  - PYTHONPATH=. coverage run tools/test/config_test/config_test.py
+  - PYTHONPATH=. coverage run -m pytest tools/test/config_test/config_test.py tools/test/toolchains/api.py
   - PYTHONPATH=. coverage run tools/test/build_api/build_api_test.py
   - PYTHONPATH=. coverage run tools/test/targets/target_test.py
   - coverage run tools/test/pylint.py
-  - coverage run -m pytest tools/test/toolchains/api.py
   - coverage run tools/test/memap/memap_test.py
   - coverage run tools/project.py -S
   - coverage run tools/build_travis.py

tools/test/config_test/app_override_libs/test_data.json

@@ -0,0 +1,10 @@
+
+ {
+   "test_target": {
+        "lib1.p1": "v_p1_lib1_app",
+        "lib1.p2": "v_p2_lib1",
+        "lib1.p3": "v_p3_lib1",
+        "lib2.p1": "v_p1_lib2_app",
+        "lib2.p2": "v_p2_lib2"
+    }
+}

tools/test/config_test/bootloader_missing/test_data.json

@@ -0,0 +1,5 @@
+ {
+    "K64F": {
+        "exception_msg": "not found"
+    }
+}

tools/test/config_test/compound_inheritance/test_data.json

@@ -1,13 +1,6 @@
-# This is similar to test1, but this time B2 also inherits from B1, which allows it to override its config data.
-# B2 also overrides base1_2, like D1.
-# The order of inheritace in F is also reversed ([D1, B2] instead of [B2, D1])
-# Since the last override of base1_2 in inheritance order is in B2, base1_2 must now
-# have the value that was overriden in B2, not in D1.
-# This test also shows that multiple inheritance works for a simple diamond shaped inheritance pattern
 
-expected_results = {
+ {
     "f": {
-        "desc": "test multiple target inheritance (diamond shape)",
         "target.base1_1": "v_base1_1_f",
         "target.base1_2": "v_base1_2_b2",
         "target.base1_3": "v_base1_3_b1",
@@ -19,7 +12,6 @@ expected_results = {
         "target.f1_2": "v_f1_2_f"
     },
     "b2": {
-        "desc": "another single inheritance test",
         "target.base1_1": "v_base1_1_b1",
         "target.base1_2": "v_base1_2_b2",
         "target.base1_3": "v_base1_3_b1",

tools/test/config_test/config_test.py

@@ -15,10 +15,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
+import os
+import sys
+import json
+import pytest
+from os.path import join, isfile, dirname, abspath
 from tools.build_api import get_config
 from tools.targets import set_targets_json_location, Target
 from tools.config import ConfigException, Config
-import os, sys
 
 # Compare the output of config against a dictionary of known good results
 def compare_config(cfg, expected):
@@ -27,87 +31,43 @@ def compare_config(cfg, expected):
             if cfg[k].value != expected[k]:
                 return "'%s': expected '%s', got '%s'" % (k, expected[k], cfg[k].value)
     except KeyError:
-        raise
         return "Unexpected key '%s' in configuration data" % k
     for k in expected:
-        if k not in ["desc", "expected_macros", "expected_features"] + cfg.keys():
+        if k not in ["expected_macros", "expected_features"] + cfg.keys():
             return "Expected key '%s' was not found in configuration data" % k
     return ""
 
-def test_tree(full_name, name):
-    failed = 0
-    sys.path.append(full_name)
-    if "test_data" in sys.modules:
-       del sys.modules["test_data"]
-    import test_data
-    # If the test defines custom targets, they must exist in a file called
-    # "targets.json" in the test's directory.
-    if os.path.isfile(os.path.join(full_name, "targets.json")):
-        set_targets_json_location(os.path.join(full_name, "targets.json"))
-    else: # uset the regular set of targets
-        set_targets_json_location()
-    for target, expected in test_data.expected_results.items():
-        sys.stdout.write("%s:'%s'(%s) " % (name, expected["desc"], target))
-        sys.stdout.flush()
-        err_msg = None
+def data_path(path):
+    return join(path, "test_data.json")
+
+def is_test(path):
+    return isfile(data_path(path))
+
+root_dir = abspath(dirname(__file__))
+
+@pytest.mark.parametrize("name", filter(lambda d: is_test(join(root_dir, d)),
+                                             os.listdir(root_dir)))
+def test_config(name):
+    test_dir = join(root_dir, name)
+    test_data = json.load(open(data_path(test_dir)))
+    targets_json = os.path.join(test_dir, "targets.json")
+    set_targets_json_location(targets_json if isfile(targets_json) else None)
+    for target, expected in test_data.items():
         try:
-            cfg, macros, features = get_config(full_name, target, "GCC_ARM")
-            macros = Config.config_macros_to_macros(macros)
-        except ConfigException as e:
-            err_msg = e.message
-        if err_msg:
-            if expected.has_key("exception_msg"):
-                if err_msg.find(expected["exception_msg"]) == -1:
-                    print "FAILED!"
-                    sys.stderr.write("    Unexpected error message!\n")
-                    sys.stderr.write("    Expected: '%s'\n" % expected["exception_msg"])
-                    sys.stderr.write("    Got: '%s'\n" % err_msg)
-                    failed += 1
-                else:
-                    print "OK"
-            else:
-                print "FAILED!"
-                sys.stderr.write("    Error while getting configuration!\n")
-                sys.stderr.write("    " + err_msg + "\n")
-                failed += 1
-        else:
+            cfg, macros, features = get_config(test_dir, target, "GCC_ARM")
             res = compare_config(cfg, expected)
+            assert not(res), res
             expected_macros = expected.get("expected_macros", None)
             expected_features = expected.get("expected_features", None)
 
-            if res:
-                print "FAILED!"
-                sys.stdout.write("    " + res + "\n")
-                failed += 1
-            elif expected_macros is not None:
-                if sorted(expected_macros) != sorted(macros):
-                    print "FAILED!"
-                    sys.stderr.write("    List of macros doesn't match\n")
-                    sys.stderr.write("    Expected: '%s'\n" % ",".join(sorted(expected_macros)))
-                    sys.stderr.write("    Got: '%s'\n" % ",".join(sorted(expected_macros)))
-                    failed += 1
-                else:
-                    print "OK"
-            elif expected_features is not None:
-                if sorted(expected_features) != sorted(features):
-                    print "FAILED!"
-                    sys.stderr.write("    List of features doesn't match\n")
-                    sys.stderr.write("    Expected: '%s'\n" % ",".join(sorted(expected_features)))
-                    sys.stderr.write("    Got: '%s'\n" % ",".join(sorted(expected_features)))
-                    failed += 1
-                else:
-                    print "OK"
+            if expected_macros is not None:
+                macros = Config.config_macros_to_macros(macros)
+                assert sorted(expected_macros) == sorted(macros)
+            if expected_features is not None:
+                assert sorted(expected_features) == sorted(features)
+        except ConfigException as e:
+            err_msg = e.message
+            if "exception_msg" not in expected:
+                assert not(err_msg), "Unexpected Error: %s" % e
             else:
-                print "OK"
-    sys.path.remove(full_name)
-    return failed
-
-failed = 0
-root_dir = os.path.abspath(os.path.dirname(__file__))
-tlist = sorted(os.listdir(root_dir), key = lambda e: int(e[4:]) if e.startswith('test') else -1)
-for test_name in tlist:
-    full_name = os.path.join(root_dir, test_name)
-    if os.path.isdir(full_name) and test_name.startswith('test'):
-        failed += test_tree(full_name, test_name)
-sys.exit(failed)
-
+                assert expected["exception_msg"] in err_msg

tools/test/config_test/double_define/test_data.json

@@ -0,0 +1,12 @@
+
+ {
+    "f": {
+        "exception_msg": "Parameter name 'base1_1' defined in both 'target:b2' and 'target:b1'"
+    },
+    "b2": {
+        "target.base2_1": "v_base2_1_b2",
+        "target.base2_2": "v_base2_2_b2",
+        "target.base1_1": "v_base1_1_b2"
+    }
+}
+

tools/test/config_test/duplicate_lib_name/test_data.json

@@ -0,0 +1,6 @@
+
+ {
+    "K64F": {
+        "exception_msg": "Library name 'lib1' is not unique"
+    }
+}

tools/test/config_test/feature_compesition/test_data.json

@@ -0,0 +1,6 @@
+
+ {
+    "test_target": {
+        "expected_features": ["IPV4", "STORAGE"]
+    }
+}

tools/test/config_test/feature_conflict/test_data.json

@@ -1,8 +1,6 @@
-# Testing when two features collide
 
-expected_results = {
+ {
     "K64F": {
-        "desc": "test feature collisions",
         "exception_msg": "Configuration conflict. The feature IPV4 both added and removed." 
     }
 }

tools/test/config_test/feature_recursive_add/test_data.json

@@ -0,0 +1,6 @@
+
+ {
+    "test_target": {
+        "expected_features": ["IPV4", "STORAGE", "UVISOR"]
+    }
+}

tools/test/config_test/feature_recursive_complex/test_data.json

@@ -0,0 +1,6 @@
+
+ {
+    "test_target": {
+        "lib2.test": "GOOD"
+    }
+}

tools/test/config_test/feature_remove/test_data.json

@@ -0,0 +1,6 @@
+
+ {
+    "test_target": {
+        "expected_features": []
+    }
+}

tools/test/config_test/feature_uvisor/test_data.json

@@ -0,0 +1,6 @@
+
+ {
+    "test_target": {
+        "expected_features": ["UVISOR"]
+    }
+}

tools/test/config_test/fetaure_recursive_conflict/test_data.json

@@ -0,0 +1,6 @@
+
+ {
+    "K64F": {
+        "exception_msg": "Configuration conflict. The feature UVISOR both added and removed." 
+    }
+}

tools/test/config_test/invalid_app_macro_define/test_data.json

@@ -0,0 +1,6 @@
+
+ {
+    "K64F": {
+        "exception_msg": "Macro 'LIB2_1' defined in both 'library:lib2' and 'application' with incompatible values"
+    }
+}

tools/test/config_test/invalid_key/test_data.json

@@ -0,0 +1,6 @@
+
+ {
+    "K64F": {
+        "exception_msg": "Unknown key(s)"
+    }
+}

tools/test/config_test/invalid_key_lib/test_data.json

@@ -0,0 +1,6 @@
+
+ {
+    "K64F": {
+        "exception_msg": "Unknown key(s)"
+    }
+}

tools/test/config_test/invalid_lib_macro_define/test_data.json

@@ -0,0 +1,6 @@
+
+ {
+    "K64F": {
+        "exception_msg": "defined in both"
+    }
+}

tools/test/config_test/macro_inheritance/test_data.json

@@ -0,0 +1,6 @@
+
+ {
+    "test_target": {
+        "expected_macros": ["APP1=10", "APP2", "LIB1_1=1","LIB1_2", "LIB2_1=5"]
+    }
+}

tools/test/config_test/override_labels_libs/test_data.json

@@ -1,9 +1,6 @@
-# This build on top of test5 by adding a few libraries with their own configurations
-# and overrides. The same targets are used for building and testing (base, b1, b2, both)
 
-expected_results = {
+ {
    "base": {
-        "desc": "override values based on labels with libs (no labels)",
         "app.app1": "v_app1",
         "app.app2": "v_app2",
         "lib1.p1": "v_p1_lib1",
@@ -13,7 +10,6 @@ expected_results = {
         "lib2.p2": "v_p2_lib2"
     },
     "b1": {
-        "desc": "override values based on labels with libs (first label)",
         "app.app1": "v_app1[b1_label]",
         "app.app2": "v_app2",
         "lib1.p1": "v_p1_lib1[b1_label]",
@@ -23,7 +19,6 @@ expected_results = {
         "lib2.p2": "v_p2_lib2[b1_label]"
     },
     "b2": {
-        "desc": "override values based on labels with libs (second label)",
         "app.app1": "v_app1",
         "app.app2": "v_app2[b2_label]",
         "lib1.p1": "v_p1_lib1",
@@ -32,12 +27,7 @@ expected_results = {
         "lib2.p1": "v_p1_lib2[b2_label]",
         "lib2.p2": "v_p2_lib2[b2_label]"
     },
-    # The values for lib2.p1 and lib2.p2 illustrate how overriding on multiple
-    # labels work. In lib2, both lib2.p1 and lib2.p2 are overriden for both
-    # labels (b1_label and b2_label). However, since "b2_label" is specified
-    # after "b1_label", it sets the final values of the overrides.
     "both": {
-        "desc": "override values based on labels with libs (both labels)",
         "app.app1": "v_app1[b1_label]",
         "app.app2": "v_app2[b2_label]",
         "lib1.p1": "v_p1_lib1[b1_label]",
@@ -45,5 +35,5 @@ expected_results = {
         "lib1.p3": "v_p3_lib1",
         "lib2.p1": "v_p1_lib2[b2_label]",
         "lib2.p2": "v_p2_lib2[b2_label]"
-    },
+    }
 }

tools/test/config_test/override_labels_libs_more/test_data.json

@@ -1,8 +1,6 @@
-# This build on top of test6 by adding overrides for libs in the application
 
-expected_results = {
+ {
    "base": {
-        "desc": "override values based on labels with libs (no labels)",
         "app.app1": "v_app1",
         "app.app2": "v_app2",
         "lib1.p1": "v_p1_lib1_app",
@@ -12,7 +10,6 @@ expected_results = {
         "lib2.p2": "v_p2_lib2"
     },
     "b1": {
-        "desc": "override values based on labels with libs (first label)",
         "app.app1": "v_app1[b1_label]",
         "app.app2": "v_app2",
         "lib1.p1": "v_p1_lib1_app",
@@ -22,7 +19,6 @@ expected_results = {
         "lib2.p2": "v_p2_lib2[b1_label]"
     },
     "b2": {
-        "desc": "override values based on labels with libs (second label)",
         "app.app1": "v_app1",
         "app.app2": "v_app2[b2_label]",
         "lib1.p1": "v_p1_lib1_app",
@@ -31,12 +27,7 @@ expected_results = {
         "lib2.p1": "v_p1_lib2_app",
         "lib2.p2": "v_p2_lib2[b2_label]"
     },
-    # The values for lib2.p1 and lib2.p2 illustrate how overriding on multiple
-    # labels work. In lib2, both lib2.p1 and lib2.p2 are overriden for both
-    # labels (b1_label and b2_label). However, since "b2_label" is specified
-    # after "b1_label", it sets the final values of the overrides.
     "both": {
-        "desc": "override values based on labels with libs (both labels)",
         "app.app1": "v_app1[b1_label]",
         "app.app2": "v_app2[b2_label]",
         "lib1.p1": "v_p1_lib1_app",

tools/test/config_test/override_labels_targets/test_data.json

@@ -1,9 +1,6 @@
-# This build on top of test7 by adding some configuration values in targets
-# and overriding them in the application
 
-expected_results = {
+ {
    "base": {
-        "desc": "override values based on labels with libs and target params (no labels)",
         "app.app1": "v_app1",
         "app.app2": "v_app2",
         "lib1.p1": "v_p1_lib1_app",
@@ -16,7 +13,6 @@ expected_results = {
         "target.par3": "v_par3_base"
     },
     "b1": {
-        "desc": "override values based on labels with libs and target params (first label)",
         "app.app1": "v_app1[b1_label]",
         "app.app2": "v_app2",
         "lib1.p1": "v_p1_lib1_app",
@@ -29,7 +25,6 @@ expected_results = {
         "target.par3": "v_par3_base"
     },
     "b2": {
-        "desc": "override values based on labels with libs and target params (second label)",
         "app.app1": "v_app1",
         "app.app2": "v_app2[b2_label]",
         "lib1.p1": "v_p1_lib1_app",
@@ -42,7 +37,6 @@ expected_results = {
         "target.par3": "v_par3_base"
     },
     "both": {
-        "desc": "override values based on labels with libs and target params (both labels)",
         "app.app1": "v_app1[b1_label]",
         "app.app2": "v_app2[b2_label]",
         "lib1.p1": "v_p1_lib1_app",

tools/test/config_test/override_precidence/test_data.json

@@ -1,8 +1,6 @@
-# This builds on top of test8 by adding target-conditional overrides in mbed_app_config.json.
 
-expected_results = {
+ {
    "base": {
-        "desc": "override values based on labels with libs, target params and target overrides (no labels)",
         "app.app1": "v_app1",
         "app.app2": "v_app2",
         "lib1.p1": "v_p1_lib1_app",
@@ -15,7 +13,6 @@ expected_results = {
         "target.par3": "v_par3_base"
     },
     "b1": {
-        "desc": "override values based on labels with libs, target params and target overrides (first label)",
         "app.app1": "v_app1[b1_label_label]",
         "app.app2": "v_app2",
         "lib1.p1": "v_p1_lib1_app",
@@ -28,7 +25,6 @@ expected_results = {
         "target.par3": "v_par3_base"
     },
     "b2": {
-        "desc": "override values based on labels with libs, target params and target overrides (second label)",
         "app.app1": "v_app1",
         "app.app2": "v_app2[b2_label]",
         "lib1.p1": "v_p1_lib1_app",
@@ -41,7 +37,6 @@ expected_results = {
         "target.par3": "v_par3_base"
     },
     "both": {
-        "desc": "override values based on labels with libs, target params and target overrides (both labels)",
         "app.app1": "v_app1[b1_label_label]",
         "app.app2": "v_app2[b2_label]",
         "lib1.p1": "v_p1_lib1_app",

tools/test/config_test/override_undefined/test_data.json

@@ -1,13 +1,9 @@
-# Similar to test1, but this time B2 attempt to override base1_1. Since B2 doesn't directly inherit
-# from B1, this must raise an error
 
-expected_results = {
+ {
     "f": {
-        "desc": "attempt to override undefined parameter in inherited target",
         "exception_msg": "Attempt to override undefined parameter 'base1_1' in 'target:b2'"
     },
     "d1": {
-        "desc": "single target inheritance again",
         "target.base1_1": "v_base1_1_d1",
         "target.base1_2": "v_base1_2_d1",
         "target.base1_3": "v_base1_3_b1",

tools/test/config_test/override_undefined_libs/test_data.json

@@ -1,10 +1,6 @@
-# This build on top of test6 by adding an invalid override in mbed_app_override.json for b1_label.
-# This will prevent the configuration for working for b1 and both, but it should still
-# work for base and b2.
 
-expected_results = {
+ {
    "base": {
-        "desc": "override values based on labels with libs (no labels)",
         "app.app1": "v_app1",
         "app.app2": "v_app2",
         "lib1.p1": "v_p1_lib1",
@@ -14,11 +10,9 @@ expected_results = {
         "lib2.p2": "v_p2_lib2"
     },
     "b1": {
-        "desc": "override values based on labels with libs - invalid override (first label)",
         "exception_msg": "Attempt to override undefined parameter 'app.app_wrong' in 'application[b1_label]"
     },
     "b2": {
-        "desc": "override values based on labels with libs (second label)",
         "app.app1": "v_app1",
         "app.app2": "v_app2[b2_label]",
         "lib1.p1": "v_p1_lib1",
@@ -28,7 +22,6 @@ expected_results = {
         "lib2.p2": "v_p2_lib2[b2_label]"
     },
     "both": {
-        "desc": "override values based on labels with libs - invalid override (both labels)",
         "exception_msg": "Attempt to override undefined parameter 'app.app_wrong' in 'application[b1_label]"
-    },
+    }
 }

tools/test/config_test/override_with_labels/test_data.json

@@ -0,0 +1,19 @@
+
+ {
+    "b1": {
+        "app.app1": "v_app1[b1_label]",
+        "app.app2": "v_app2"
+    },
+    "b2": {
+        "app.app1": "v_app1",
+        "app.app2": "v_app2[b2_label]"
+    },
+    "both": {
+        "app.app1": "v_app1[b1_label]",
+        "app.app2": "v_app2[b2_label]"
+    },
+    "base": {
+        "app.app1": "v_app1",
+        "app.app2": "v_app2"
+    }
+}

tools/test/config_test/parameter_outside_lib/test_data.json

@@ -0,0 +1,6 @@
+
+ {
+   "K64F": {
+        "exception_msg": "Invalid prefix 'lib2' for parameter name 'lib2.p1' in 'library:lib1[K64F]'"
+    }
+}

tools/test/config_test/simple_features/test_data.json

@@ -0,0 +1,6 @@
+
+ {
+    "test_target": {
+        "expected_features": ["IPV4"]
+    }
+}

tools/test/config_test/simple_iheritance/test_data.json

@@ -0,0 +1,27 @@
+
+ {
+    "f": {
+        "target.base1_1": "v_base1_1_f",
+        "target.base1_2": "v_base1_2_d1",
+        "target.base1_3": "v_base1_3_b1",
+        "target.derived1": "v_derived1_d1",
+        "target.derived2": "v_derived2_f",
+        "target.base2_1": "v_base2_1_f",
+        "target.base2_2": "v_base2_2_b2",
+        "target.f1_1": "v_f1_1_f_override",
+        "target.f1_2": "v_f1_2_f"
+    },
+    "b1": {
+        "target.base1_1": "v_base1_1_b1",
+        "target.base1_2": "v_base1_2_b1",
+        "target.base1_3": "v_base1_3_b1"
+    },
+    "d1": {
+        "target.base1_1": "v_base1_1_d1",
+        "target.base1_2": "v_base1_2_d1",
+        "target.base1_3": "v_base1_3_b1",
+        "target.derived1": "v_derived1_d1",
+        "target.derived2": "v_derived2_d1"
+    }
+}
+

tools/test/config_test/test01/test_data.py

@@ -1,46 +0,0 @@
-# Test for configuration data defined in targets
-# A base target (B1) defines 3 configuration parameters (base1_1, base1_2 and base1_3)
-# A derived target (D1) inherits drom B1 and defines one configuration parameters (derived1 and derived2) and overrides base1_1 and base1_2
-# Another base target (B2) defines its own configuration parameters (base2_1 and base2_2)
-# The final target F derives from B2 and D1, defines two configuration paramaters (f1_1 and f1_2)
-# and overrides base2_1, base1_1, derived2 and its own configuration parameter f1_1 (which is legal)
-# Final result:
-#     base1_1 must have the value defined in F
-#     base1_2 must have the value defined in D1
-#     base1_3 must have the value defined in B1
-#     derived1 must have the value defined in D1
-#     derived2 must have the value defined in F
-#     base2_1 must have the value defined in F
-#     base2_2 must have the value defined in B2
-#     f1_1 must have the value defined and then overriden in F
-#     f1_2 must have the value defined in F
-
-expected_results = {
-    "f": {
-        "desc": "test multiple target inheritance",
-        "target.base1_1": "v_base1_1_f",
-        "target.base1_2": "v_base1_2_d1",
-        "target.base1_3": "v_base1_3_b1",
-        "target.derived1": "v_derived1_d1",
-        "target.derived2": "v_derived2_f",
-        "target.base2_1": "v_base2_1_f",
-        "target.base2_2": "v_base2_2_b2",
-        "target.f1_1": "v_f1_1_f_override",
-        "target.f1_2": "v_f1_2_f"
-    },
-    "b1": {
-        "desc": "test with a single base target, no inheritance",
-        "target.base1_1": "v_base1_1_b1",
-        "target.base1_2": "v_base1_2_b1",
-        "target.base1_3": "v_base1_3_b1"
-    },
-    "d1": {
-        "desc": "test single target inheritance",
-        "target.base1_1": "v_base1_1_d1",
-        "target.base1_2": "v_base1_2_d1",
-        "target.base1_3": "v_base1_3_b1",
-        "target.derived1": "v_derived1_d1",
-        "target.derived2": "v_derived2_d1"
-    }
-}
-