Compiling tests stops on first failure by default, with option to continue

tools/test.py

@@ -66,6 +66,9 @@ if __name__ == '__main__':
         parser.add_option("-f", "--format", type="choice", dest="format",
                           choices=format_choices, default=format_default_choice, help=format_help)
         
+        parser.add_option("--continue-on-build-fail", action="store_true", dest="continue_on_build_fail",
+                          default=None, help="Continue trying to build all tests if a build failure occurs")
+
         parser.add_option("-n", "--names", dest="names",
                           default=None, help="Limit the tests to a comma separated list of names")
                           
@@ -157,7 +160,8 @@ if __name__ == '__main__':
                         properties=build_properties,
                         macros=options.macros,
                         verbose=options.verbose,
-                        jobs=options.jobs)
+                        jobs=options.jobs,
+                        continue_on_build_fail=options.continue_on_build_fail)
                 
                 # If a path to a test spec is provided, write it to a file
                 if options.test_spec:

tools/test_api.py

@@ -2027,7 +2027,8 @@ def print_tests(tests, format="list"):
 
 def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
         options=None, clean=False, notify=None, verbose=False, jobs=1,
-        macros=None, silent=False, report=None, properties=None):
+        macros=None, silent=False, report=None, properties=None,
+        continue_on_build_fail=False):
     """Given the data structure from 'find_tests' and the typical build parameters,
     build all the tests
     
@@ -2062,7 +2063,11 @@ def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
 
         except Exception, e:
             result = False
-            continue
+            
+            if continue_on_build_fail:
+                continue
+            else:
+                break
         
         # If a clean build was carried out last time, disable it for the next build.
         # Otherwise the previously built test will be deleted.

tools/test_exporters.py

@@ -20,7 +20,6 @@ Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
 from tools.utils import construct_enum, mkdir
 import os
 
-
 ResultExporterType = construct_enum(HTML='Html_Exporter',
                                     JUNIT='JUnit_Exporter',
                                     JUNIT_OPER='JUnit_Exporter_Interoperability',
@@ -73,7 +72,8 @@ class ReportExporter():
         self.result_exporter_type = result_exporter_type
         self.package = package
 
-    def report(self, test_summary_ext, test_suite_properties=None):
+    def report(self, test_summary_ext, test_suite_properties=None,
+               print_log_for_failures=True):
         """ Invokes report depending on exporter_type set in constructor
         """
         if self.result_exporter_type == ResultExporterType.HTML:
@@ -87,7 +87,7 @@ class ReportExporter():
             return self.exporter_junit_ioper(test_summary_ext, test_suite_properties)
         elif self.result_exporter_type == ResultExporterType.PRINT:
             # JUNIT exporter for interoperability test
-            return self.exporter_print(test_summary_ext)
+            return self.exporter_print(test_summary_ext, print_log_for_failures=print_log_for_failures)
         return None
 
     def report_to_file(self, test_summary_ext, file_name, test_suite_properties=None):
@@ -297,11 +297,15 @@ class ReportExporter():
                 test_suites.append(ts)
         return TestSuite.to_xml_string(test_suites)
 
-    def exporter_print_helper(self, array):
+    def exporter_print_helper(self, array, print_log=False):
         for item in array:
             print "  * %s::%s::%s" % (item["target_name"], item["toolchain_name"], item["id"])
+            if print_log:
+                log_lines = item["output"].split("\n")
+                for log_line in log_lines:
+                    print "        %s" % log_line
 
-    def exporter_print(self, test_result_ext):
+    def exporter_print(self, test_result_ext, print_log_for_failures=False):
         """ Export test results in print format.
         """
         failures = []
@@ -340,7 +344,7 @@ class ReportExporter():
 
         if failures:
             print "\n\nBuild failures:"
-            self.exporter_print_helper(failures)
+            self.exporter_print_helper(failures, print_log=print_log_for_failures)
             return False
         else:
             return True