Merge pull request #1568 from bridadan/not-supported-test-result

Adding the ability for tests to report they aren't supported on a target

workspace_tools/build_api.py

@@ -25,7 +25,7 @@ from shutil import rmtree
 from os.path import join, exists, basename
 from time import time
 
-from workspace_tools.utils import mkdir, run_cmd, run_cmd_ext
+from workspace_tools.utils import mkdir, run_cmd, run_cmd_ext, NotSupportedException
 from workspace_tools.paths import MBED_TARGETS_PATH, MBED_LIBRARIES, MBED_API, MBED_HAL, MBED_COMMON
 from workspace_tools.targets import TARGET_NAMES, TARGET_MAP
 from workspace_tools.libraries import Library
@@ -139,13 +139,13 @@ def build_project(src_path, build_path, target, toolchain_name,
                 resources.inc_dirs.extend(inc_dirs)
             else:
                 resources.inc_dirs.append(inc_dirs)
-
         # Compile Sources
         for path in src_paths:
             src = toolchain.scan_resources(path)
             objects = toolchain.compile_sources(src, build_path, resources.inc_dirs)
             resources.objects.extend(objects)
 
+
         # Link Program
         res, needed_update = toolchain.link_program(resources, build_path, name)
 
@@ -162,7 +162,12 @@ def build_project(src_path, build_path, target, toolchain_name,
     except Exception, e:
         if report != None:
             end = time()
-            cur_result["result"] = "FAIL"
+
+            if isinstance(e, NotSupportedException):
+                cur_result["result"] = "NOT_SUPPORTED"
+            else:
+                cur_result["result"] = "FAIL"
+
             cur_result["elapsed_time"] = end - start
 
             toolchain_output = toolchain.get_output()

workspace_tools/test_api.py

@@ -44,6 +44,7 @@ from workspace_tools.tests import TEST_MAP
 from workspace_tools.paths import BUILD_DIR
 from workspace_tools.paths import HOST_TESTS
 from workspace_tools.utils import ToolException
+from workspace_tools.utils import NotSupportedException
 from workspace_tools.utils import construct_enum
 from workspace_tools.targets import TARGET_MAP
 from workspace_tools.test_db import BaseDBAccess
@@ -130,6 +131,7 @@ class SingleTestRunner(object):
     TEST_RESULT_NO_IMAGE = "NO_IMAGE"
     TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
     TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"
+    TEST_RESULT_NOT_SUPPORTED = "NOT_SUPPORTED"
 
     GLOBAL_LOOPS_COUNT = 1  # How many times each test should be repeated
     TEST_LOOPS_LIST = []    # We redefine no.of loops per test_id
@@ -149,7 +151,8 @@ class SingleTestRunner(object):
                            "no_image" : TEST_RESULT_NO_IMAGE,
                            "end" : TEST_RESULT_UNDEF,
                            "mbed_assert" : TEST_RESULT_MBED_ASSERT,
-                           "build_failed" : TEST_RESULT_BUILD_FAILED
+                           "build_failed" : TEST_RESULT_BUILD_FAILED,
+                           "not_supproted" : TEST_RESULT_NOT_SUPPORTED
     }
 
     def __init__(self,
@@ -476,13 +479,23 @@ class SingleTestRunner(object):
                                      project_id=test_id,
                                      project_description=test.get_description())
 
-                except ToolException:
+                except Exception, e:
                     project_name_str = project_name if project_name is not None else test_id
-                    print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building project %s'% (project_name_str))
+
+
+                    test_result = self.TEST_RESULT_FAIL
+
+                    if isinstance(e, ToolException):
+                        print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building project %s'% (project_name_str))
+                        test_result = self.TEST_RESULT_BUILD_FAILED
+                    elif isinstance(e, NotSupportedException):
+                        print self.logger.log_line(self.logger.LogType.INFO, 'The project %s is not supported'% (project_name_str))
+                        test_result = self.TEST_RESULT_NOT_SUPPORTED
+
 
                     # Append test results to global test summary
                     self.test_summary.append(
-                        (self.TEST_RESULT_BUILD_FAILED, target, toolchain, test_id, 'Toolchain build failed', 0, 0, '-')
+                        (test_result, target, toolchain, test_id, test.get_description(), 0, 0, '-')
                     )
 
                     # Add detailed test result to test summary structure
@@ -490,13 +503,13 @@ class SingleTestRunner(object):
                         self.test_summary_ext[target][toolchain][test_id] = []
 
                     self.test_summary_ext[target][toolchain][test_id].append({ 0: {
-                        'result' : self.TEST_RESULT_BUILD_FAILED,
+                        'result' : test_result,
                         'output' : '',
                         'target_name' : target,
                         'target_name_unique': target,
                         'toolchain_name' : toolchain,
                         'id' : test_id,
-                        'description' : 'Toolchain build failed',
+                        'description' : test.get_description(),
                         'elapsed_time' : 0,
                         'duration' : 0,
                         'copy_method' : None
@@ -736,7 +749,8 @@ class SingleTestRunner(object):
                        self.TEST_RESULT_NO_IMAGE : 0,
                        self.TEST_RESULT_TIMEOUT : 0,
                        self.TEST_RESULT_MBED_ASSERT : 0,
-                       self.TEST_RESULT_BUILD_FAILED : 0
+                       self.TEST_RESULT_BUILD_FAILED : 0,
+                       self.TEST_RESULT_NOT_SUPPORTED : 0
         }
 
         for test in test_summary:

workspace_tools/test_exporters.py

@@ -238,7 +238,7 @@ class ReportExporter():
                     tc.add_failure_info(description, _stdout)
                 elif result == 'ERROR':
                     tc.add_error_info(description, _stdout)
-                elif result == 'SKIP':
+                elif result == 'SKIP' or result == 'NOT_SUPPORTED':
                     tc.add_skipped_info(description, _stdout)
 
                 test_cases.append(tc)
@@ -282,7 +282,7 @@ class ReportExporter():
                             message = test_result['result']
                             if test_result['result'] == 'FAIL':
                                 tc.add_failure_info(message, _stdout)
-                            elif test_result['result'] == 'SKIP':
+                            elif test_result['result'] == 'SKIP' or test_result["result"] == 'NOT_SUPPORTED':
                                 tc.add_skipped_info(message, _stdout)
                             elif test_result['result'] != 'OK':
                                 tc.add_error_info(message, _stdout)
@@ -319,7 +319,7 @@ class ReportExporter():
 
                         if test_run["result"] == "FAIL":
                             failures.append(test_run)
-                        elif test_run["result"] == "SKIP":
+                        elif test_run["result"] == "SKIP" or test_run["result"] == "NOT_SUPPORTED":
                             skips.append(test_run)
                         elif test_run["result"] == "OK":
                             successes.append(test_run)

workspace_tools/toolchains/__init__.py

@@ -26,7 +26,7 @@ from os.path import join, splitext, exists, relpath, dirname, basename, split
 from inspect import getmro
 
 from multiprocessing import Pool, cpu_count
-from workspace_tools.utils import run_cmd, mkdir, rel_path, ToolException, split_path
+from workspace_tools.utils import run_cmd, mkdir, rel_path, ToolException, NotSupportedException, split_path
 from workspace_tools.settings import BUILD_OPTIONS, MBED_ORG_USER
 import workspace_tools.hooks as hooks
 
@@ -604,6 +604,9 @@ class mbedToolchain:
 
         return None
 
+    def is_not_supported_error(self, output):
+        return "#error directive: [NOT_SUPPORTED]" in output
+
     def compile_output(self, output=[]):
         _rc = output[0]
         _stderr = output[1]
@@ -621,7 +624,10 @@ class mbedToolchain:
             for line in _stderr.splitlines():
                 self.tool_error(line)
 
-            raise ToolException(_stderr)
+            if self.is_not_supported_error(_stderr):
+                raise NotSupportedException(_stderr)
+            else:
+                raise ToolException(_stderr)
 
     def compile(self, cc, source, object, includes):
         _, ext = splitext(source)

workspace_tools/toolchains/gcc.py

@@ -118,6 +118,9 @@ class GCC(mbedToolchain):
                     dependencies = dependencies + [f.replace('\a', ' ') for f in file.split(" ")]
         return dependencies
 
+    def is_not_supported_error(self, output):
+        return "error: #error [NOT_SUPPORTED]" in output
+
     def parse_output(self, output):
         # The warning/error notification is multiline
         WHERE, WHAT = 0, 1

workspace_tools/utils.py

@@ -123,6 +123,8 @@ def rel_path(path, base, dot=False):
 class ToolException(Exception):
     pass
 
+class NotSupportedException(Exception):
+    pass
 
 def split_path(path):
     base, file = split(path)