From 87103abd56dca24bccb8c68aabbf031eccbc0a46 Mon Sep 17 00:00:00 2001
From: Brian Daniels <brianddaniels@gmail.com>
Date: Mon, 10 Oct 2016 15:09:50 -0500
Subject: [PATCH] Making building and linking tests fully parallel.

This uses similar code that is used withing the toolchains to parallelize
the linking process of all the tests accross all the available CPUs. It
also respects the `-j` parameter if you wish to limit the number of cores
used.
---
 tools/test_api.py | 128 ++++++++++++++++++++++++++++++++--------------
 1 file changed, 89 insertions(+), 39 deletions(-)

diff --git a/tools/test_api.py b/tools/test_api.py
index 126328f3ad..9b0e01aafc 100644
--- a/tools/test_api.py
+++ b/tools/test_api.py
@@ -37,6 +37,7 @@ from time import sleep, time
 from Queue import Queue, Empty
 from os.path import join, exists, basename, relpath
 from threading import Thread, Lock
+from multiprocessing import Pool, cpu_count
 from subprocess import Popen, PIPE
 
 # Imports related to mbed build api
@@ -2068,6 +2069,27 @@ def norm_relative_path(path, start):
     path = path.replace("\\", "/")
     return path
 
+
+def build_test_worker(*args, **kwargs):
+    bin_file = None
+    ret = {
+        'result': False,
+        'args': args,
+        'kwargs': kwargs
+    }
+
+    try:
+        bin_file = build_project(*args, **kwargs)
+        ret['result'] = True
+        ret['bin_file'] = bin_file
+        ret['kwargs'] = kwargs
+
+    except:
+        ret['reason'] = sys.exc_info()[1]
+
+    return ret
+
+
 def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
                 clean=False, notify=None, verbose=False, jobs=1, macros=None,
                 silent=False, report=None, properties=None,
@@ -2095,58 +2117,86 @@ def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
 
     result = True
 
-    map_outputs_total = list()
+    jobs_count = int(jobs if jobs else cpu_count())
+    p = Pool(processes=jobs_count)
+
+    results = []
     for test_name, test_path in tests.iteritems():
         test_build_path = os.path.join(build_path, test_path)
         src_path = base_source_paths + [test_path]
         bin_file = None
         test_case_folder_name = os.path.basename(test_path)
         
+        args = (src_path, test_build_path, target, toolchain_name)
+        kwargs = {
+            'jobs': jobs,
+            'clean': clean,
+            'macros': macros,
+            'name': test_case_folder_name,
+            'project_id': test_name,
+            'report': report,
+            'properties': properties,
+            'verbose': verbose,
+            'app_config': app_config,
+            'build_profile': build_profile
+        }
         
-        try:
-            bin_file = build_project(src_path, test_build_path, target, toolchain_name,
-                                     jobs=jobs,
-                                     clean=clean,
-                                     macros=macros,
-                                     name=test_case_folder_name,
-                                     project_id=test_name,
-                                     report=report,
-                                     properties=properties,
-                                     verbose=verbose,
-                                     app_config=app_config,
-                                     build_profile=build_profile)
+        results.append(p.apply_async(build_test_worker, args, kwargs))
 
-        except NotSupportedException:
-            pass
-        except ToolException:
-            result = False
-            if continue_on_build_fail:
-                continue
+    p.close()
+    result = True
+    itr = 0
+    while len(results):
+        itr += 1
+        if itr > 360000:
+            p.terminate()
+            p.join()
+            raise ToolException("Compile did not finish in 10 minutes")
+
+        sleep(0.01)
+        pending = 0
+        for r in results:
+            if r.ready() is True:
+                try:
+                    worker_result = r.get()
+                    results.remove(r)
+
+
+                    for test_key in worker_result['kwargs']['report'][target_name][toolchain_name].keys():
+                        report[target_name][toolchain_name][test_key] = worker_result['kwargs']['report'][target_name][toolchain_name][test_key]
+
+                    if not worker_result['result'] and not isinstance(worker_result['reason'], NotSupportedException):
+                        result = False
+
+                    if worker_result['result'] and 'bin_file' in worker_result:
+                        bin_file = norm_relative_path(worker_result['bin_file'], execution_directory)
+
+                        test_build['tests'][worker_result['kwargs']['project_id']] = {
+                            "binaries": [
+                                {
+                                    "path": bin_file
+                                }
+                            ]
+                        }
+                        
+                        # TODO: add 'Image: bin_file' print statement here
+
+                except ToolException, err:
+                    if p._taskqueue.queue:
+                        p._taskqueue.queue.clear()
+                        sleep(0.5)
+                    p.terminate()
+                    p.join()
+                    raise ToolException(err)
             else:
-                break
+                pending += 1
+                if pending >= jobs_count:
+                    break
 
-        # If a clean build was carried out last time, disable it for the next build.
-        # Otherwise the previously built test will be deleted.
-        if clean:
-            clean = False
-
-        # Normalize the path
-        if bin_file:
-            bin_file = norm_relative_path(bin_file, execution_directory)
-
-            test_build['tests'][test_name] = {
-                "binaries": [
-                    {
-                        "path": bin_file
-                    }
-                ]
-            }
-
-            print 'Image: %s'% bin_file
+    p.join()
 
     test_builds = {}
     test_builds["%s-%s" % (target_name, toolchain_name)] = test_build
-    
 
     return result, test_builds