Merge pull request #966 from PrzemekWirkus/devel_parallel_exec_2

Experimental parallel test execution on MUTs (option --parallel), v2

workspace_tools/singletest.py

@@ -221,6 +221,7 @@ if __name__ == '__main__':
                                    _opts_verbose=opts.verbose,
                                    _opts_firmware_global_name=opts.firmware_global_name,
                                    _opts_only_build_tests=opts.only_build_tests,
+                                   _opts_parallel_test_exec=opts.parallel_test_exec,
                                    _opts_suppress_summary=opts.suppress_summary,
                                    _opts_test_x_toolchain_summary=opts.test_x_toolchain_summary,
                                    _opts_copy_method=opts.copy_method,

workspace_tools/test_api.py

@@ -34,7 +34,7 @@ from prettytable import PrettyTable
 from time import sleep, time
 from Queue import Queue, Empty
 from os.path import join, exists, basename
-from threading import Thread
+from threading import Thread, Lock
 from subprocess import Popen, PIPE
 
 # Imports related to mbed build api
@@ -167,6 +167,7 @@ class SingleTestRunner(object):
                  _opts_verbose=False,
                  _opts_firmware_global_name=None,
                  _opts_only_build_tests=False,
+                 _opts_parallel_test_exec=False,
                  _opts_suppress_summary=False,
                  _opts_test_x_toolchain_summary=False,
                  _opts_copy_method=None,
@@ -217,6 +218,7 @@ class SingleTestRunner(object):
         self.opts_verbose = _opts_verbose
         self.opts_firmware_global_name = _opts_firmware_global_name
         self.opts_only_build_tests = _opts_only_build_tests
+        self.opts_parallel_test_exec = _opts_parallel_test_exec
         self.opts_suppress_summary = _opts_suppress_summary
         self.opts_test_x_toolchain_summary = _opts_test_x_toolchain_summary
         self.opts_copy_method = _opts_copy_method
@@ -284,26 +286,17 @@ class SingleTestRunner(object):
             result = False
         return result
 
-    def execute(self):
-        clean = self.test_spec.get('clean', False)
-        test_ids = self.test_spec.get('test_ids', [])
-
     # This will store target / toolchain specific properties
     test_suite_properties_ext = {}  # target : toolchain
-
     # Here we store test results
     test_summary = []
     # Here we store test results in extended data structure
     test_summary_ext = {}
+    execute_thread_slice_lock = Lock()
 
-        # Generate seed for shuffle if seed is not provided in
-        self.shuffle_random_seed = round(random.random(), self.SHUFFLE_SEED_ROUND)
-        if self.opts_shuffle_test_seed is not None and self.is_shuffle_seed_float():
-            self.shuffle_random_seed = round(float(self.opts_shuffle_test_seed), self.SHUFFLE_SEED_ROUND)
-
-        for target, toolchains in self.test_spec['targets'].iteritems():
-            test_suite_properties_ext[target] = {}
+    def execute_thread_slice(self, q, target, toolchains, clean, test_ids):
         for toolchain in toolchains:
+            # print target, toolchain
             # Test suite properties returned to external tools like CI
             test_suite_properties = {}
             test_suite_properties['jobs'] = self.opts_jobs
@@ -335,7 +328,9 @@ class SingleTestRunner(object):
                     continue
             except ToolException:
                 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building MBED libs for %s using %s'% (target, toolchain))
-                    return test_summary, self.shuffle_random_seed, test_summary_ext, test_suite_properties_ext
+                #return self.test_summary, self.shuffle_random_seed, self.test_summary_ext, self.test_suite_properties_ext
+                q.put(target + '_'.join(toolchains))
+                return
 
             build_dir = join(BUILD_DIR, "test", target, toolchain)
 
@@ -363,10 +358,8 @@ class SingleTestRunner(object):
                     self.db_logger.update_build_id_info(self.db_logger_build_id, _extra=json.dumps(self.dump_options()))
                     self.db_logger.disconnect();
 
-
             for test_id in test_map_keys:
                 test = TEST_MAP[test_id]
-
                 if self.opts_test_by_names and test_id not in self.opts_test_by_names.split(','):
                     continue
 
@@ -431,7 +424,9 @@ class SingleTestRunner(object):
                                       jobs=self.opts_jobs)
                         except ToolException:
                             print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building library %s'% (lib_id))
-                                return test_summary, self.shuffle_random_seed, test_summary_ext, test_suite_properties_ext
+                            #return self.test_summary, self.shuffle_random_seed, self.test_summary_ext, self.test_suite_properties_ext
+                            q.put(target + '_'.join(toolchains))
+                            return
 
                     test_suite_properties['test.libs.%s.%s.%s'% (target, toolchain, test_id)] = ', '.join(libraries)
 
@@ -467,7 +462,9 @@ class SingleTestRunner(object):
                     except ToolException:
                         project_name_str = project_name if project_name is not None else test_id
                         print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building project %s'% (project_name_str))
-                            return test_summary, self.shuffle_random_seed, test_summary_ext, test_suite_properties_ext
+                        # return self.test_summary, self.shuffle_random_seed, self.test_summary_ext, self.test_suite_properties_ext
+                        q.put(target + '_'.join(toolchains))
+                        return
                     if self.opts_only_build_tests:
                         # With this option we are skipping testing phase
                         continue
@@ -491,18 +488,58 @@ class SingleTestRunner(object):
 
                     # Append test results to global test summary
                     if single_test_result is not None:
-                            test_summary.append(single_test_result)
+                        self.test_summary.append(single_test_result)
 
                     # Prepare extended test results data structure (it can be used to generate detailed test report)
-                        if toolchain not in test_summary_ext:
-                            test_summary_ext[toolchain] = {}  # test_summary_ext : toolchain
-                        if target not in test_summary_ext[toolchain]:
-                            test_summary_ext[toolchain][target] = {}    # test_summary_ext : toolchain : target
-                        if target not in test_summary_ext[toolchain][target]:
-                            test_summary_ext[toolchain][target][test_id] = detailed_test_results    # test_summary_ext : toolchain : target : test_it
+                    if toolchain not in self.test_summary_ext:
+                        self.test_summary_ext[toolchain] = {}  # test_summary_ext : toolchain
+                    if target not in self.test_summary_ext[toolchain]:
+                        self.test_summary_ext[toolchain][target] = {}    # test_summary_ext : toolchain : target
+                    if target not in self.test_summary_ext[toolchain][target]:
+                        self.test_summary_ext[toolchain][target][test_id] = detailed_test_results    # test_summary_ext : toolchain : target : test_it
 
             test_suite_properties['skipped'] = ', '.join(test_suite_properties['skipped'])
-                test_suite_properties_ext[target][toolchain] = test_suite_properties
+            self.test_suite_properties_ext[target][toolchain] = test_suite_properties
+        # return self.test_summary, self.shuffle_random_seed, test_summary_ext, self.test_suite_properties_ext
+        q.put(target + '_'.join(toolchains))
+        return
+
+    def execute(self):
+        clean = self.test_spec.get('clean', False)
+        test_ids = self.test_spec.get('test_ids', [])
+        q = Queue()
+
+        # Generate seed for shuffle if seed is not provided in
+        self.shuffle_random_seed = round(random.random(), self.SHUFFLE_SEED_ROUND)
+        if self.opts_shuffle_test_seed is not None and self.is_shuffle_seed_float():
+            self.shuffle_random_seed = round(float(self.opts_shuffle_test_seed), self.SHUFFLE_SEED_ROUND)
+
+        if self.opts_parallel_test_exec:
+            ###################################################################
+            # Experimental, parallel test execution per singletest instance.
+            ###################################################################
+            execute_threads = []    # Threads used to build mbed SDL, libs, test cases and execute tests
+            # Note: We are building here in parallel for each target separately!
+            # So we are not building the same thing multiple times and compilers
+            # in separate threads do not collide.
+            # Inside execute_thread_slice() function function handle() will be called to
+            # get information about available MUTs (per target).
+            for target, toolchains in self.test_spec['targets'].iteritems():
+                self.test_suite_properties_ext[target] = {}
+                t = threading.Thread(target=self.execute_thread_slice, args = (q, target, toolchains, clean, test_ids))
+                t.daemon = True
+                t.start()
+                execute_threads.append(t)
+
+            for t in execute_threads:
+                q.get() # t.join() would block some threads because we should not wait in any order for thread end
+        else:
+            # Serialized (not parallel) test execution
+            for target, toolchains in self.test_spec['targets'].iteritems():
+                if target not in self.test_suite_properties_ext:
+                    self.test_suite_properties_ext[target] = {}
+                self.execute_thread_slice(q, target, toolchains, clean, test_ids)
+                q.get()
 
         if self.db_logger:
             self.db_logger.reconnect();
@@ -510,7 +547,7 @@ class SingleTestRunner(object):
                 self.db_logger.update_build_id_info(self.db_logger_build_id, _status_fk=self.db_logger.BUILD_ID_STATUS_COMPLETED)
                 self.db_logger.disconnect();
 
-        return test_summary, self.shuffle_random_seed, test_summary_ext, test_suite_properties_ext
+        return self.test_summary, self.shuffle_random_seed, self.test_summary_ext, self.test_suite_properties_ext
 
     def generate_test_summary_by_target(self, test_summary, shuffle_seed=None):
         """ Prints well-formed summary with results (SQL table like)
@@ -641,7 +678,8 @@ class SingleTestRunner(object):
 
     def handle(self, test_spec, target_name, toolchain_name, test_loops=1):
         """ Function determines MUT's mbed disk/port and copies binary to
-            target. Test is being invoked afterwards.
+            target.
+            Test is being invoked afterwards.
         """
         data = json.loads(test_spec)
         # Get test information, image and test timeout
@@ -1592,6 +1630,12 @@ def get_default_test_options_parser():
                       default=False,
                       help="Only build tests, skips actual test procedures (flashing etc.)")
 
+    parser.add_option('', '--parallel',
+                      dest='parallel_test_exec',
+                      default=False,
+                      action="store_true",
+                      help='Experimental, you execute test runners for connected to your host MUTs in parallel (speeds up test result collection)')
+
     parser.add_option('', '--config',
                       dest='verbose_test_configuration_only',
                       default=False,