From 8b8606b98d4c0168a28f03fcbe33dac503328f16 Mon Sep 17 00:00:00 2001
From: Christopher Haster <chaster@utexas.edu>
Date: Fri, 22 Apr 2016 01:03:53 -0500
Subject: [PATCH 01/14] Add start function for separating object allocation
 from thread initialization

Allows threads to started separately from when they are declared,
avoiding the need to dynamically allocate threads at runtime.
---
 core/mbed-rtos/rtos/Thread.cpp | 19 +++++++++++++++++++
 core/mbed-rtos/rtos/Thread.h   | 17 +++++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/core/mbed-rtos/rtos/Thread.cpp b/core/mbed-rtos/rtos/Thread.cpp
index ed8270a36a..5ebb72d005 100644
--- a/core/mbed-rtos/rtos/Thread.cpp
+++ b/core/mbed-rtos/rtos/Thread.cpp
@@ -26,8 +26,22 @@
 
 namespace rtos {
 
+Thread::Thread() {
+    _tid = NULL;
+}
+
 Thread::Thread(void (*task)(void const *argument), void *argument,
         osPriority priority, uint32_t stack_size, unsigned char *stack_pointer) {
+    _tid = NULL;
+    start(task, argument, priority, stack_size, stack_pointer);
+}
+
+osStatus Thread::start(void (*task)(void const *argument), void *argument,
+        osPriority priority, uint32_t stack_size, unsigned char *stack_pointer) {
+    if (_tid != NULL) {
+        return osErrorResource;
+    }
+
 #ifdef __MBED_CMSIS_RTOS_CM
     _thread_def.pthread = task;
     _thread_def.tpriority = priority;
@@ -48,6 +62,11 @@ Thread::Thread(void (*task)(void const *argument), void *argument,
     }
 #endif
     _tid = osThreadCreate(&_thread_def, argument);
+    if (_tid == NULL) {
+        return osErrorResource;
+    }
+
+    return osOK;
 }
 
 osStatus Thread::terminate() {
diff --git a/core/mbed-rtos/rtos/Thread.h b/core/mbed-rtos/rtos/Thread.h
index 3e787983ab..0da6764a00 100644
--- a/core/mbed-rtos/rtos/Thread.h
+++ b/core/mbed-rtos/rtos/Thread.h
@@ -30,6 +30,10 @@ namespace rtos {
 /** The Thread class allow defining, creating, and controlling thread functions in the system. */
 class Thread {
 public:
+    /** Allocate a new thread without starting execution
+    */
+    Thread();
+
     /** Create a new thread, and start it executing the specified function.
       @param   task           function to be executed by this thread.
       @param   argument       pointer that is passed to the thread function as start argument. (default: NULL).
@@ -42,6 +46,19 @@ public:
            uint32_t stack_size=DEFAULT_STACK_SIZE,
            unsigned char *stack_pointer=NULL);
 
+    /** Starts a thread executing the specified function.
+      @param   task           function to be executed by this thread.
+      @param   argument       pointer that is passed to the thread function as start argument. (default: NULL).
+      @param   priority       initial priority of the thread function. (default: osPriorityNormal).
+      @param   stack_size      stack size (in bytes) requirements for the thread function. (default: DEFAULT_STACK_SIZE).
+      @param   stack_pointer  pointer to the stack area to be used by this thread (default: NULL).
+      @return  status code that indicates the execution status of the function.
+    */
+    osStatus start(void (*task)(void const *argument), void *argument=NULL,
+                   osPriority priority=osPriorityNormal,
+                   uint32_t stack_size=DEFAULT_STACK_SIZE,
+                   unsigned char *stack_pointer=NULL);
+
     /** Terminate execution of a thread and remove it from Active Threads
       @return  status code that indicates the execution status of the function.
     */

From d15cd7826ade05f28499c84fb4f5c5ae2da9fcc4 Mon Sep 17 00:00:00 2001
From: Christopher Haster <chaster@utexas.edu>
Date: Fri, 22 Apr 2016 01:12:40 -0500
Subject: [PATCH 02/14] Add Thread::join call for exiting threads without
 forceful termination

Allows multiple threads to join without forceful termination. Allows
threads to synchronize on cleanup.
---
 core/mbed-rtos/rtos/Thread.cpp | 16 +++++++++++++++-
 core/mbed-rtos/rtos/Thread.h   |  5 +++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/core/mbed-rtos/rtos/Thread.cpp b/core/mbed-rtos/rtos/Thread.cpp
index 5ebb72d005..82ce688119 100644
--- a/core/mbed-rtos/rtos/Thread.cpp
+++ b/core/mbed-rtos/rtos/Thread.cpp
@@ -39,7 +39,7 @@ Thread::Thread(void (*task)(void const *argument), void *argument,
 osStatus Thread::start(void (*task)(void const *argument), void *argument,
         osPriority priority, uint32_t stack_size, unsigned char *stack_pointer) {
     if (_tid != NULL) {
-        return osErrorResource;
+        return osErrorParameter;
     }
 
 #ifdef __MBED_CMSIS_RTOS_CM
@@ -73,6 +73,20 @@ osStatus Thread::terminate() {
     return osThreadTerminate(_tid);
 }
 
+osStatus Thread::join() {
+    while (true) {
+        uint8_t state = get_state();
+        if (state == Thread::Inactive || state == osErrorParameter) {
+            return osOK;
+        }
+
+        osStatus status = yield();
+        if (status != osOK) {
+            return status;
+        }
+    }
+}
+
 osStatus Thread::set_priority(osPriority priority) {
     return osThreadSetPriority(_tid, priority);
 }
diff --git a/core/mbed-rtos/rtos/Thread.h b/core/mbed-rtos/rtos/Thread.h
index 0da6764a00..c172f88930 100644
--- a/core/mbed-rtos/rtos/Thread.h
+++ b/core/mbed-rtos/rtos/Thread.h
@@ -59,6 +59,11 @@ public:
                    uint32_t stack_size=DEFAULT_STACK_SIZE,
                    unsigned char *stack_pointer=NULL);
 
+    /** Wait for thread to terminate
+      @return  status code that indicates the execution status of the function.
+    */
+    osStatus join();
+
     /** Terminate execution of a thread and remove it from Active Threads
       @return  status code that indicates the execution status of the function.
     */

From 21e68f3c9116c4fa4e62bfb9eb88247845a70fc7 Mon Sep 17 00:00:00 2001
From: Christopher Haster <chaster@utexas.edu>
Date: Sat, 23 Apr 2016 03:40:05 -0500
Subject: [PATCH 03/14] Move configuration arguments to only constructor

---
 core/mbed-rtos/rtos/Thread.cpp | 30 +++++++++++++++++-------------
 core/mbed-rtos/rtos/Thread.h   | 20 +++++++++++---------
 2 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/core/mbed-rtos/rtos/Thread.cpp b/core/mbed-rtos/rtos/Thread.cpp
index 82ce688119..70ffa9e30c 100644
--- a/core/mbed-rtos/rtos/Thread.cpp
+++ b/core/mbed-rtos/rtos/Thread.cpp
@@ -26,38 +26,42 @@
 
 namespace rtos {
 
-Thread::Thread() {
+Thread::Thread(osPriority priority,
+        uint32_t stack_size, unsigned char *stack_pointer) {
     _tid = NULL;
+    _priority = priority;
+    _stack_size = stack_size;
+    _stack_pointer = stack_pointer;
 }
 
 Thread::Thread(void (*task)(void const *argument), void *argument,
         osPriority priority, uint32_t stack_size, unsigned char *stack_pointer) {
     _tid = NULL;
-    start(task, argument, priority, stack_size, stack_pointer);
+    _priority = priority;
+    _stack_size = stack_size;
+    _stack_pointer = stack_pointer;
+    start(task, argument);
 }
 
-osStatus Thread::start(void (*task)(void const *argument), void *argument,
-        osPriority priority, uint32_t stack_size, unsigned char *stack_pointer) {
+osStatus Thread::start(void (*task)(void const *argument), void *argument) {
     if (_tid != NULL) {
         return osErrorParameter;
     }
 
 #ifdef __MBED_CMSIS_RTOS_CM
     _thread_def.pthread = task;
-    _thread_def.tpriority = priority;
-    _thread_def.stacksize = stack_size;
-    if (stack_pointer != NULL) {
-        _thread_def.stack_pointer = (uint32_t*)stack_pointer;
-        _dynamic_stack = false;
+    _thread_def.tpriority = _priority;
+    _thread_def.stacksize = _stack_size;
+    if (_stack_pointer != NULL) {
+        _thread_def.stack_pointer = (uint32_t*)_stack_pointer;
     } else {
-        _thread_def.stack_pointer = new uint32_t[stack_size/sizeof(uint32_t)];
+        _thread_def.stack_pointer = new uint32_t[_stack_size/sizeof(uint32_t)];
         if (_thread_def.stack_pointer == NULL)
             error("Error allocating the stack memory\n");
-        _dynamic_stack = true;
     }
     
     //Fill the stack with a magic word for maximum usage checking
-    for (uint32_t i = 0; i < (stack_size / sizeof(uint32_t)); i++) {
+    for (uint32_t i = 0; i < (_stack_size / sizeof(uint32_t)); i++) {
         _thread_def.stack_pointer[i] = 0xE25A2EA5;
     }
 #endif
@@ -191,7 +195,7 @@ void Thread::attach_idle_hook(void (*fptr)(void)) {
 Thread::~Thread() {
     terminate();
 #ifdef __MBED_CMSIS_RTOS_CM
-    if (_dynamic_stack) {
+    if (_stack_pointer == NULL) {
         delete[] (_thread_def.stack_pointer);
     }
 #endif
diff --git a/core/mbed-rtos/rtos/Thread.h b/core/mbed-rtos/rtos/Thread.h
index c172f88930..1cbc93afa4 100644
--- a/core/mbed-rtos/rtos/Thread.h
+++ b/core/mbed-rtos/rtos/Thread.h
@@ -31,8 +31,13 @@ namespace rtos {
 class Thread {
 public:
     /** Allocate a new thread without starting execution
+      @param   priority       initial priority of the thread function. (default: osPriorityNormal).
+      @param   stack_size      stack size (in bytes) requirements for the thread function. (default: DEFAULT_STACK_SIZE).
+      @param   stack_pointer  pointer to the stack area to be used by this thread (default: NULL).
     */
-    Thread();
+    Thread(osPriority priority=osPriorityNormal,
+           uint32_t stack_size=DEFAULT_STACK_SIZE,
+           unsigned char *stack_pointer=NULL);
 
     /** Create a new thread, and start it executing the specified function.
       @param   task           function to be executed by this thread.
@@ -49,15 +54,9 @@ public:
     /** Starts a thread executing the specified function.
       @param   task           function to be executed by this thread.
       @param   argument       pointer that is passed to the thread function as start argument. (default: NULL).
-      @param   priority       initial priority of the thread function. (default: osPriorityNormal).
-      @param   stack_size      stack size (in bytes) requirements for the thread function. (default: DEFAULT_STACK_SIZE).
-      @param   stack_pointer  pointer to the stack area to be used by this thread (default: NULL).
       @return  status code that indicates the execution status of the function.
     */
-    osStatus start(void (*task)(void const *argument), void *argument=NULL,
-                   osPriority priority=osPriorityNormal,
-                   uint32_t stack_size=DEFAULT_STACK_SIZE,
-                   unsigned char *stack_pointer=NULL);
+    osStatus start(void (*task)(void const *argument), void *argument=NULL);
 
     /** Wait for thread to terminate
       @return  status code that indicates the execution status of the function.
@@ -164,7 +163,10 @@ public:
 private:
     osThreadId _tid;
     osThreadDef_t _thread_def;
-    bool _dynamic_stack;
+
+    osPriority _priority;
+    uint32_t _stack_size;
+    unsigned char *_stack_pointer;
 };
 
 }

From da6571cc249a09ad4968cb78c344c0c933e7912b Mon Sep 17 00:00:00 2001
From: Christopher Haster <chaster@utexas.edu>
Date: Sat, 23 Apr 2016 12:10:47 -0500
Subject: [PATCH 04/14] Add documentation on invalid thread functions in irq

---
 core/mbed-rtos/rtos/Thread.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/mbed-rtos/rtos/Thread.h b/core/mbed-rtos/rtos/Thread.h
index 1cbc93afa4..d9a5cbe68d 100644
--- a/core/mbed-rtos/rtos/Thread.h
+++ b/core/mbed-rtos/rtos/Thread.h
@@ -60,6 +60,7 @@ public:
 
     /** Wait for thread to terminate
       @return  status code that indicates the execution status of the function.
+      @note not callable from interrupt
     */
     osStatus join();
 
@@ -134,17 +135,20 @@ public:
       @param   signals   wait until all specified signal flags set or 0 for any single signal flag.
       @param   millisec  timeout value or 0 in case of no time-out. (default: osWaitForever).
       @return  event flag information or error code.
+      @note not callable from interrupt
     */
     static osEvent signal_wait(int32_t signals, uint32_t millisec=osWaitForever);
 
     /** Wait for a specified time period in millisec:
       @param   millisec  time delay value
       @return  status code that indicates the execution status of the function.
+      @note not callable from interrupt
     */
     static osStatus wait(uint32_t millisec);
 
     /** Pass control to next thread that is in state READY.
       @return  status code that indicates the execution status of the function.
+      @note not callable from interrupt
     */
     static osStatus yield();
 

From ef291e79d969d896971922c49a5f79e66c476766 Mon Sep 17 00:00:00 2001
From: Christopher Haster <chaster@utexas.edu>
Date: Sat, 23 Apr 2016 15:32:50 -0500
Subject: [PATCH 05/14] Add unit tests over spawning/joining threads with
 thread functions

---
 .../TESTS/mbed-rtos/threads/main.cpp          | 110 ++++++++++++++++++
 1 file changed, 110 insertions(+)
 create mode 100644 core/mbed-rtos/TESTS/mbed-rtos/threads/main.cpp

diff --git a/core/mbed-rtos/TESTS/mbed-rtos/threads/main.cpp b/core/mbed-rtos/TESTS/mbed-rtos/threads/main.cpp
new file mode 100644
index 0000000000..030beec2da
--- /dev/null
+++ b/core/mbed-rtos/TESTS/mbed-rtos/threads/main.cpp
@@ -0,0 +1,110 @@
+#include "mbed.h"
+#include "test_env.h"
+#include "unity.h"
+#include "utest.h"
+#include "rtos.h"
+
+
+using namespace utest::v1;
+
+
+// Tasks with different functions to test on threads
+void increment(const void *var) {
+    (*(int *)var)++;
+}
+
+void increment_with_yield(const void *var) {
+    Thread::yield();
+    (*(int *)var)++;
+}
+
+void increment_with_wait(const void *var) {
+    Thread::wait(100);
+    (*(int *)var)++;
+}
+
+void increment_with_child(const void *var) {
+    Thread child(increment, (void*)var);
+    child.join();
+}
+
+void increment_with_murder(const void *var) {
+    Thread child(increment_with_wait, (void*)var);
+    // Kill child before it can increment var
+    child.terminate();
+    (*(int *)var)++;
+}
+
+
+// Tests that spawn tasks in different configurations
+template <void (*F)(const void *)>
+void test_single_thread() {
+    int var = 0;
+    Thread thread(F, &var);
+    thread.join();
+    TEST_ASSERT_EQUAL(var, 1);
+}
+
+template <int N, void (*F)(const void *)>
+void test_parallel_threads() {
+    int var = 0;
+    Thread *threads[N];
+
+    for (int i = 0; i < N; i++) {
+        threads[i] = new Thread(F, &var);
+    }
+
+    for (int i = 0; i < N; i++) {
+        threads[i]->join();
+        delete threads[i];
+    }
+
+    TEST_ASSERT_EQUAL(var, N);
+}
+
+template <int N, void (*F)(const void *)>
+void test_serial_threads() {
+    int var = 0;
+
+    for (int i = 0; i < N; i++) {
+        Thread thread(F, &var);
+        thread.join();
+    }
+
+    TEST_ASSERT_EQUAL(var, N);
+}
+    
+
+status_t test_setup(const size_t number_of_cases) {
+    GREENTEA_SETUP(40, "default_auto");
+    return verbose_test_setup_handler(number_of_cases);
+}
+
+// Test cases
+Case cases[] = {
+    Case("Testing single thread", test_single_thread<increment>),
+    Case("Testing parallel threads", test_parallel_threads<3, increment>),
+    Case("Testing serial threads", test_serial_threads<10, increment>),
+
+    Case("Testing single thread with yield", test_single_thread<increment_with_yield>),
+    Case("Testing parallel threads with yield", test_parallel_threads<3, increment_with_yield>),
+    Case("Testing serial threads with yield", test_serial_threads<10, increment_with_yield>),
+
+    Case("Testing single thread with wait", test_single_thread<increment_with_wait>),
+    Case("Testing parallel threads with wait", test_parallel_threads<3, increment_with_wait>),
+    Case("Testing serial threads with wait", test_serial_threads<10, increment_with_wait>),
+    
+    Case("Testing single thread with child", test_single_thread<increment_with_child>),
+    Case("Testing parallel threads with child", test_parallel_threads<3, increment_with_child>),
+    Case("Testing serial threads with child", test_serial_threads<10, increment_with_child>),
+
+    Case("Testing single thread with murder", test_single_thread<increment_with_murder>),
+    Case("Testing parallel threads with murder", test_parallel_threads<3, increment_with_murder>),
+    Case("Testing serial threads with murder", test_serial_threads<10, increment_with_murder>),
+};
+
+Specification specification(test_setup, cases);
+
+int main() {
+    return !Harness::run(specification);
+}

From 2be0385f73056c41c4f4d4ffc2e3af441a4c720e Mon Sep 17 00:00:00 2001
From: Mihail Stoyanov <wm@smode.org>
Date: Wed, 4 May 2016 18:35:29 +0100
Subject: [PATCH 06/14] Response files for includes during assemble() and
 compile() Moved unified compile to toolchains to enable specific toolchain
 support for response files

---
 tools/toolchains/__init__.py | 25 ++-------------
 tools/toolchains/arm.py      | 60 ++++++++++++++++++++++++++++++++++--
 tools/toolchains/gcc.py      | 53 +++++++++++++++++++++++++++++--
 tools/toolchains/iar.py      | 39 ++++++++++++++++++++++-
 4 files changed, 149 insertions(+), 28 deletions(-)

diff --git a/tools/toolchains/__init__.py b/tools/toolchains/__init__.py
index 4683b691a4..8534772fed 100644
--- a/tools/toolchains/__init__.py
+++ b/tools/toolchains/__init__.py
@@ -24,6 +24,7 @@ from types import ListType
 from shutil import copyfile
 from os.path import join, splitext, exists, relpath, dirname, basename, split
 from inspect import getmro
+from tempfile import mkdtemp
 
 from multiprocessing import Pool, cpu_count
 from tools.utils import run_cmd, mkdir, rel_path, ToolException, NotSupportedException, split_path
@@ -211,6 +212,7 @@ class mbedToolchain:
 
         self.build_all = False
         self.timestamp = time()
+        self.temp_dir = None
         self.jobs = 1
 
         self.CHROOT = None
@@ -481,6 +483,7 @@ class mbedToolchain:
         files_to_compile = resources.s_sources + resources.c_sources + resources.cpp_sources
         self.to_be_compiled = len(files_to_compile)
         self.compiled = 0
+        self.temp_dir = build_path
 
         #for i in self.build_params:
         #    self.debug(i)
@@ -641,28 +644,6 @@ class mbedToolchain:
             else:
                 raise ToolException(_stderr)
 
-    def compile(self, cc, source, object, includes):
-        _, ext = splitext(source)
-        ext = ext.lower()
-
-        command = cc + ['-D%s' % s for s in self.get_symbols()] + ["-I%s" % i for i in includes] + ["-o", object, source]
-
-        if hasattr(self, "get_dep_opt"):
-            base, _ = splitext(object)
-            dep_path = base + '.d'
-            command.extend(self.get_dep_opt(dep_path))
-
-        if hasattr(self, "cc_extra"):
-            command.extend(self.cc_extra(base))
-
-        return [command]
-
-    def compile_c(self, source, object, includes):
-        return self.compile(self.cc, source, object, includes)
-
-    def compile_cpp(self, source, object, includes):
-        return self.compile(self.cppc, source, object, includes)
-
     def build_library(self, objects, dir, name):
         needed_update = False
         lib = self.STD_LIB_NAME % name
diff --git a/tools/toolchains/arm.py b/tools/toolchains/arm.py
index 1e524a7b03..323418a39e 100644
--- a/tools/toolchains/arm.py
+++ b/tools/toolchains/arm.py
@@ -15,7 +15,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 import re
-from os.path import join, dirname, basename
+from os.path import join, dirname, splitext, basename, exists
+from hashlib import md5
 
 from tools.toolchains import mbedToolchain
 from tools.settings import ARM_BIN, ARM_INC, ARM_LIB, MY_ARM_CLIB, ARM_CPPLIB, GOANNA_PATH
@@ -123,8 +124,27 @@ class ARM(mbedToolchain):
         tempfile = join(dir, basename(object) + '.E.s')
         
         # Build preprocess assemble command
-        cmd_pre = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros] + ["-I%s" % i for i in includes] + ["-E", "-o", tempfile, source]
-        
+        cmd_pre = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros]
+
+        # Response file
+        inc_str = ' '.join(includes)
+        if len(inc_str) > 16000:
+            sum = md5(inc_str).hexdigest()
+            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
+            if not exists(include_files):
+                with open(include_files, "wb") as f:
+                    cmd_list = []
+                    for c in includes:
+                        if c:
+                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)
+                    string = " ".join(cmd_list).replace("\\", "/")
+                    f.write(string)
+            cmd_pre.extend(['--via', include_files])
+        else:
+            cmd_pre.extend(['-I"%s"' % i for i in includes])
+
+        cmd_pre.extend(["-E", "-o", tempfile, source])
+
         # Build main assemble command
         cmd = self.asm + ["-o", object, tempfile]
 
@@ -135,6 +155,40 @@ class ARM(mbedToolchain):
         # Return command array, don't execute
         return [cmd_pre, cmd]
 
+    @hook_tool
+    def compile(self, cc, source, object, includes):
+        cmd = cc + ['-D %s' % s for s in self.get_symbols()]
+
+        inc_str = ' '.join(includes)
+        if len(inc_str) > 16000:
+            sum = md5(inc_str).hexdigest()
+            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
+            if not exists(include_files):
+                with open(include_files, "wb") as f:
+                    cmd_list = []
+                    for c in includes:
+                        if c:
+                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)
+                    string = " ".join(cmd_list).replace("\\", "/")
+                    f.write(string)
+            cmd.extend(['--via', include_files])
+        else:
+            cmd.extend(['-I"%s"' % i for i in includes])
+
+        
+        base, _ = splitext(object)
+        dep_path = base + '.d'
+        cmd.extend(self.get_dep_opt(dep_path))
+            
+        cmd.extend(["-o", object, source])
+
+        return [cmd]
+
+    def compile_c(self, source, object, includes):
+        return self.compile(self.cc, source, object, includes)
+
+    def compile_cpp(self, source, object, includes):
+        return self.compile(self.cppc, source, object, includes)
 
     @hook_tool
     def link(self, output, objects, libraries, lib_dirs, mem_map):
diff --git a/tools/toolchains/gcc.py b/tools/toolchains/gcc.py
index 4bc81a7b8e..4a54b36c06 100644
--- a/tools/toolchains/gcc.py
+++ b/tools/toolchains/gcc.py
@@ -15,7 +15,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 import re
-from os.path import join, basename, splitext, dirname
+from os.path import join, basename, splitext, dirname, exists
+from hashlib import md5
 
 from tools.toolchains import mbedToolchain
 from tools.settings import GCC_ARM_PATH, GCC_CR_PATH
@@ -164,7 +165,25 @@ class GCC(mbedToolchain):
     @hook_tool
     def assemble(self, source, object, includes):
         # Build assemble command
-        cmd = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros] + ["-I%s" % i for i in includes] + ["-o", object, source]
+        cmd = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros]
+        
+        inc_str = ' '.join(includes)
+        if len(inc_str) > 16000:
+            sum = md5(inc_str).hexdigest()
+            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
+            if not exists(include_files):
+                with open(include_files, "wb") as f:
+                    cmd_list = []
+                    for c in includes:
+                        if c:
+                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)                    
+                    string = " ".join(cmd_list).replace("\\", "/")
+                    f.write(string)
+            cmd.extend(['@%s' % include_files])
+        else:
+            cmd.extend(['-I"%s"' % i for i in includes])
+        
+        cmd.extend(["-o", object, source])
 
         # Call cmdline hook
         cmd = self.hook.get_cmdline_assembler(cmd)
@@ -172,6 +191,36 @@ class GCC(mbedToolchain):
         # Return command array, don't execute
         return [cmd]
 
+    @hook_tool
+    def compile(self, cc, source, object, includes):
+        cmd = cc + ['-D%s' % s for s in self.get_symbols()]
+
+        inc_str = ' '.join(includes)
+        if len(inc_str) > 16000:
+            sum = md5(inc_str).hexdigest()
+            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
+            if not exists(include_files):
+                with open(include_files, "wb") as f:
+                    cmd_list = []
+                    for c in includes:
+                        if c:
+                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)                    
+                    string = " ".join(cmd_list).replace("\\", "/")
+                    f.write(string)
+            cmd.extend(['@%s' % include_files])
+        else:
+            cmd.extend(['-I"%s"' % i for i in includes])
+
+        cmd.extend(["-o", object, source])
+
+        return [cmd]
+
+    def compile_c(self, source, object, includes):
+        return self.compile(self.cc, source, object, includes)
+
+    def compile_cpp(self, source, object, includes):
+        return self.compile(self.cppc, source, object, includes)
+
     @hook_tool
     def link(self, output, objects, libraries, lib_dirs, mem_map):
         libs = []
diff --git a/tools/toolchains/iar.py b/tools/toolchains/iar.py
index 3d297a80f3..a9eb4deadf 100644
--- a/tools/toolchains/iar.py
+++ b/tools/toolchains/iar.py
@@ -16,7 +16,8 @@ limitations under the License.
 """
 import re
 from os import remove
-from os.path import join, exists, dirname
+from os.path import join, exists, dirname, splitext, exists
+from hashlib import md5
 
 from tools.toolchains import mbedToolchain
 from tools.settings import IAR_PATH
@@ -114,6 +115,42 @@ class IAR(mbedToolchain):
         # Return command array, don't execute
         return [cmd]
 
+    @hook_tool
+    def compile(self, cc, source, object, includes):
+        cmd = cc + ['-D%s' % s for s in self.get_symbols()]
+
+        inc_str = ' '.join(includes)
+        if len(inc_str) > 1000:
+            sum = md5(inc_str).hexdigest()
+            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
+            if not exists(include_files):
+                with open(include_files, "wb") as f:
+                    cmd_list = []
+                    for c in includes:
+                        if c:
+                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)                    
+                    string = " ".join(cmd_list).replace("\\", "/")
+                    f.write(string)
+            cmd.extend(['-f', include_files])
+        else:
+            cmd.extend(['-I"%s"' % i for i in includes])
+
+        base, _ = splitext(object)
+        dep_path = base + '.d'
+        cmd.extend(self.get_dep_opt(dep_path))
+
+        cmd.extend(self.cc_extra(base))
+        
+        cmd.extend(["-o", object, source])
+
+        return [cmd]
+
+    def compile_c(self, source, object, includes):
+        return self.compile(self.cc, source, object, includes)
+
+    def compile_cpp(self, source, object, includes):
+        return self.compile(self.cppc, source, object, includes)
+
     @hook_tool
     def link(self, output, objects, libraries, lib_dirs, mem_map):
         # Build linker command

From ecb9651873910cf3cc942097976ed8ba5f8950b9 Mon Sep 17 00:00:00 2001
From: Mihail Stoyanov <wm@smode.org>
Date: Thu, 5 May 2016 13:13:38 +0100
Subject: [PATCH 07/14] Unified command-line handling of defines and includes
 with response files for all toolchains (ARM, GCC, IAR) Capture system/OS
 error during command execution and report the command Report command before
 executing it in default_cmd()

---
 tools/toolchains/__init__.py |  2 +-
 tools/toolchains/arm.py      | 65 +++++++++++++++---------------------
 tools/toolchains/gcc.py      | 63 +++++++++++++++-------------------
 tools/toolchains/iar.py      | 45 +++++++++++++++----------
 tools/utils.py               |  8 +++--
 5 files changed, 87 insertions(+), 96 deletions(-)

diff --git a/tools/toolchains/__init__.py b/tools/toolchains/__init__.py
index 8534772fed..3fe0e67796 100644
--- a/tools/toolchains/__init__.py
+++ b/tools/toolchains/__init__.py
@@ -688,12 +688,12 @@ class mbedToolchain:
         return bin, needed_update
 
     def default_cmd(self, command):
+        self.debug("Command: %s"% ' '.join(command))
         _stdout, _stderr, _rc = run_cmd(command)
         # Print all warning / erros from stderr to console output
         for error_line in _stderr.splitlines():
             print error_line
 
-        self.debug("Command: %s"% ' '.join(command))
         self.debug("Return: %s"% _rc)
 
         for output_line in _stdout.splitlines():
diff --git a/tools/toolchains/arm.py b/tools/toolchains/arm.py
index 323418a39e..8f89d986ae 100644
--- a/tools/toolchains/arm.py
+++ b/tools/toolchains/arm.py
@@ -116,6 +116,27 @@ class ARM(mbedToolchain):
     def get_dep_opt(self, dep_path):
         return ["--depend", dep_path]
 
+    def get_compile_options(self, defines, includes):
+        cmd = []
+        
+        str = (' '.join(defines))+"|"+(' '.join(includes))
+        if len(str) > 160:
+            sum = md5(str).hexdigest()
+            options_file = join(self.temp_dir, "options_%s.txt" % sum)
+            if not exists(options_file):
+                with open(options_file, "wb") as f:
+                    cmd_list = ['-D%s' % d for d in defines]
+                    for c in includes:
+                        if c:
+                            cmd_list.append(('-I%s' % c) if not c.startswith('-') else c)                    
+                    string = " ".join(cmd_list).replace("\\", "/")
+                    f.write(string)
+            cmd.extend(['--via', options_file])
+        else:
+            cmd.extend(['-D%s' % d for d in defines] + ['-I%s' % i for i in includes])
+        
+        return cmd
+
     @hook_tool
     def assemble(self, source, object, includes):
         # Preprocess first, then assemble
@@ -124,26 +145,7 @@ class ARM(mbedToolchain):
         tempfile = join(dir, basename(object) + '.E.s')
         
         # Build preprocess assemble command
-        cmd_pre = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros]
-
-        # Response file
-        inc_str = ' '.join(includes)
-        if len(inc_str) > 16000:
-            sum = md5(inc_str).hexdigest()
-            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
-            if not exists(include_files):
-                with open(include_files, "wb") as f:
-                    cmd_list = []
-                    for c in includes:
-                        if c:
-                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)
-                    string = " ".join(cmd_list).replace("\\", "/")
-                    f.write(string)
-            cmd_pre.extend(['--via', include_files])
-        else:
-            cmd_pre.extend(['-I"%s"' % i for i in includes])
-
-        cmd_pre.extend(["-E", "-o", tempfile, source])
+        cmd_pre = self.asm + self.get_compile_options(self.get_symbols(), includes) + ["-E", "-o", tempfile, source]
 
         # Build main assemble command
         cmd = self.asm + ["-o", object, tempfile]
@@ -157,24 +159,8 @@ class ARM(mbedToolchain):
 
     @hook_tool
     def compile(self, cc, source, object, includes):
-        cmd = cc + ['-D %s' % s for s in self.get_symbols()]
-
-        inc_str = ' '.join(includes)
-        if len(inc_str) > 16000:
-            sum = md5(inc_str).hexdigest()
-            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
-            if not exists(include_files):
-                with open(include_files, "wb") as f:
-                    cmd_list = []
-                    for c in includes:
-                        if c:
-                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)
-                    string = " ".join(cmd_list).replace("\\", "/")
-                    f.write(string)
-            cmd.extend(['--via', include_files])
-        else:
-            cmd.extend(['-I"%s"' % i for i in includes])
-
+        # Build compile command
+        cmd = cc + self.get_compile_options(self.get_symbols(), includes)
         
         base, _ = splitext(object)
         dep_path = base + '.d'
@@ -182,6 +168,9 @@ class ARM(mbedToolchain):
             
         cmd.extend(["-o", object, source])
 
+        # Call cmdline hook
+        cmd = self.hook.get_cmdline_compiler(cmd)
+
         return [cmd]
 
     def compile_c(self, source, object, includes):
diff --git a/tools/toolchains/gcc.py b/tools/toolchains/gcc.py
index 4a54b36c06..8120f669d5 100644
--- a/tools/toolchains/gcc.py
+++ b/tools/toolchains/gcc.py
@@ -162,28 +162,31 @@ class GCC(mbedToolchain):
                     message + match.group('message')
                 )
 
+    def get_compile_options(self, defines, includes):
+        cmd = []
+        
+        str = (' '.join(defines))+"|"+(' '.join(includes))
+        if len(str) > 160:
+            sum = md5(str).hexdigest()
+            options_file = join(self.temp_dir, "options_%s.txt" % sum)
+            if not exists(options_file):
+                with open(options_file, "wb") as f:
+                    cmd_list = ['-D%s' % d for d in defines]
+                    for c in includes:
+                        if c:
+                            cmd_list.append(('-I%s' % c) if not c.startswith('-') else c)                    
+                    string = " ".join(cmd_list).replace("\\", "/")
+                    f.write(string)
+            cmd.extend(['@%s' % options_file])
+        else:
+            cmd.extend(['-D%s' % d for d in defines] + ['-I%s' % i for i in includes])
+        
+        return cmd
+
     @hook_tool
     def assemble(self, source, object, includes):
         # Build assemble command
-        cmd = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros]
-        
-        inc_str = ' '.join(includes)
-        if len(inc_str) > 16000:
-            sum = md5(inc_str).hexdigest()
-            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
-            if not exists(include_files):
-                with open(include_files, "wb") as f:
-                    cmd_list = []
-                    for c in includes:
-                        if c:
-                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)                    
-                    string = " ".join(cmd_list).replace("\\", "/")
-                    f.write(string)
-            cmd.extend(['@%s' % include_files])
-        else:
-            cmd.extend(['-I"%s"' % i for i in includes])
-        
-        cmd.extend(["-o", object, source])
+        cmd = self.asm + self.get_compile_options(self.get_symbols(), includes) + ["-o", object, source]
 
         # Call cmdline hook
         cmd = self.hook.get_cmdline_assembler(cmd)
@@ -193,25 +196,11 @@ class GCC(mbedToolchain):
 
     @hook_tool
     def compile(self, cc, source, object, includes):
-        cmd = cc + ['-D%s' % s for s in self.get_symbols()]
+        # Build compile command
+        cmd = cc + self.get_compile_options(self.get_symbols(), includes) + ["-o", object, source]
 
-        inc_str = ' '.join(includes)
-        if len(inc_str) > 16000:
-            sum = md5(inc_str).hexdigest()
-            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
-            if not exists(include_files):
-                with open(include_files, "wb") as f:
-                    cmd_list = []
-                    for c in includes:
-                        if c:
-                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)                    
-                    string = " ".join(cmd_list).replace("\\", "/")
-                    f.write(string)
-            cmd.extend(['@%s' % include_files])
-        else:
-            cmd.extend(['-I"%s"' % i for i in includes])
-
-        cmd.extend(["-o", object, source])
+        # Call cmdline hook
+        cmd = self.hook.get_cmdline_compiler(cmd)
 
         return [cmd]
 
diff --git a/tools/toolchains/iar.py b/tools/toolchains/iar.py
index a9eb4deadf..258830eaa0 100644
--- a/tools/toolchains/iar.py
+++ b/tools/toolchains/iar.py
@@ -104,10 +104,31 @@ class IAR(mbedToolchain):
         return [path.strip() for path in open(dep_path).readlines()
                 if (path and not path.isspace())]
 
+    def get_compile_options(self, defines, includes):
+        cmd = []
+        
+        str = (' '.join(defines))+"|"+(' '.join(includes))
+        if len(str) > 160:
+            sum = md5(str).hexdigest()
+            options_file = join(self.temp_dir, "options_%s.txt" % sum)
+            if not exists(options_file):
+                with open(options_file, "wb") as f:
+                    cmd_list = ['-D%s' % d for d in defines]
+                    for c in includes:
+                        if c:
+                            cmd_list.append(('-I%s' % c) if not c.startswith('-') else c)                    
+                    string = " ".join(cmd_list).replace("\\", "/")
+                    f.write(string)
+            cmd.extend(['-f', options_file])
+        else:
+            cmd.extend(['-D%s' % d for d in defines] + ['-I%s' % i for i in includes])
+        
+        return cmd
+
     @hook_tool
     def assemble(self, source, object, includes):
         # Build assemble command
-        cmd = self.asm + ['-D%s' % s for s in self.get_symbols() + self.macros] + ["-I%s" % i for i in includes] + ["-o", object, source]
+        cmd = self.asm + self.get_compile_options(self.get_symbols(), includes) + ["-o", object, source]
 
         # Call cmdline hook
         cmd = self.hook.get_cmdline_assembler(cmd)
@@ -117,23 +138,8 @@ class IAR(mbedToolchain):
 
     @hook_tool
     def compile(self, cc, source, object, includes):
-        cmd = cc + ['-D%s' % s for s in self.get_symbols()]
-
-        inc_str = ' '.join(includes)
-        if len(inc_str) > 1000:
-            sum = md5(inc_str).hexdigest()
-            include_files = join(self.temp_dir, "includes_%s.txt" % sum)
-            if not exists(include_files):
-                with open(include_files, "wb") as f:
-                    cmd_list = []
-                    for c in includes:
-                        if c:
-                            cmd_list.append(('-I"%s"' % c) if not c.startswith('-') else c)                    
-                    string = " ".join(cmd_list).replace("\\", "/")
-                    f.write(string)
-            cmd.extend(['-f', include_files])
-        else:
-            cmd.extend(['-I"%s"' % i for i in includes])
+        # Build compile command
+        cmd = cc +  self.get_compile_options(self.get_symbols(), includes)
 
         base, _ = splitext(object)
         dep_path = base + '.d'
@@ -143,6 +149,9 @@ class IAR(mbedToolchain):
         
         cmd.extend(["-o", object, source])
 
+        # Call cmdline hook
+        cmd = self.hook.get_cmdline_compiler(cmd)
+
         return [cmd]
 
     def compile_c(self, source, object, includes):
diff --git a/tools/utils.py b/tools/utils.py
index 21f0e1496b..a03558a019 100644
--- a/tools/utils.py
+++ b/tools/utils.py
@@ -34,8 +34,12 @@ def cmd(l, check=True, verbose=False, shell=False, cwd=None):
 
 def run_cmd(command, wd=None, redirect=False):
     assert is_cmd_valid(command[0])
-    p = Popen(command, stdout=PIPE, stderr=STDOUT if redirect else PIPE, cwd=wd)
-    _stdout, _stderr = p.communicate()
+    try:
+        p = Popen(command, stdout=PIPE, stderr=STDOUT if redirect else PIPE, cwd=wd)
+        _stdout, _stderr = p.communicate()
+    except:
+        print "[OS ERROR] Command: "+(' '.join(command))
+        raise
     return _stdout, _stderr, p.returncode
 
 

From 868942a14878f09c6b831446875ade22ba3514d3 Mon Sep 17 00:00:00 2001
From: Mihail Stoyanov <wm@smode.org>
Date: Thu, 5 May 2016 19:26:19 +0100
Subject: [PATCH 08/14] Build system response files for all macros and
 includes. This also allows developers to check the response file after/during
 compile for the list of all macros and includes being used.

---
 tools/toolchains/arm.py | 26 +++++++++++---------------
 tools/toolchains/gcc.py | 26 +++++++++++---------------
 tools/toolchains/iar.py | 26 +++++++++++---------------
 3 files changed, 33 insertions(+), 45 deletions(-)

diff --git a/tools/toolchains/arm.py b/tools/toolchains/arm.py
index 8f89d986ae..462369f591 100644
--- a/tools/toolchains/arm.py
+++ b/tools/toolchains/arm.py
@@ -119,21 +119,17 @@ class ARM(mbedToolchain):
     def get_compile_options(self, defines, includes):
         cmd = []
         
-        str = (' '.join(defines))+"|"+(' '.join(includes))
-        if len(str) > 160:
-            sum = md5(str).hexdigest()
-            options_file = join(self.temp_dir, "options_%s.txt" % sum)
-            if not exists(options_file):
-                with open(options_file, "wb") as f:
-                    cmd_list = ['-D%s' % d for d in defines]
-                    for c in includes:
-                        if c:
-                            cmd_list.append(('-I%s' % c) if not c.startswith('-') else c)                    
-                    string = " ".join(cmd_list).replace("\\", "/")
-                    f.write(string)
-            cmd.extend(['--via', options_file])
-        else:
-            cmd.extend(['-D%s' % d for d in defines] + ['-I%s' % i for i in includes])
+        sum = md5(' '.join(includes)).hexdigest()
+        options_file = join(self.temp_dir, "options_%s.txt" % sum)
+        if not exists(options_file):
+            with open(options_file, "wb") as f:
+                cmd_list = ['-D%s' % d for d in defines]
+                for c in includes:
+                    if c:
+                        cmd_list.append(('-I%s' % c) if not c.startswith('-') else c)                    
+                string = " ".join(cmd_list).replace("\\", "/")
+                f.write(string)
+        cmd.extend(['--via', options_file])
         
         return cmd
 
diff --git a/tools/toolchains/gcc.py b/tools/toolchains/gcc.py
index 8120f669d5..8fae2fb667 100644
--- a/tools/toolchains/gcc.py
+++ b/tools/toolchains/gcc.py
@@ -165,21 +165,17 @@ class GCC(mbedToolchain):
     def get_compile_options(self, defines, includes):
         cmd = []
         
-        str = (' '.join(defines))+"|"+(' '.join(includes))
-        if len(str) > 160:
-            sum = md5(str).hexdigest()
-            options_file = join(self.temp_dir, "options_%s.txt" % sum)
-            if not exists(options_file):
-                with open(options_file, "wb") as f:
-                    cmd_list = ['-D%s' % d for d in defines]
-                    for c in includes:
-                        if c:
-                            cmd_list.append(('-I%s' % c) if not c.startswith('-') else c)                    
-                    string = " ".join(cmd_list).replace("\\", "/")
-                    f.write(string)
-            cmd.extend(['@%s' % options_file])
-        else:
-            cmd.extend(['-D%s' % d for d in defines] + ['-I%s' % i for i in includes])
+        sum = md5(' '.join(includes)).hexdigest()
+        options_file = join(self.temp_dir, "options_%s.txt" % sum)
+        if not exists(options_file):
+            with open(options_file, "wb") as f:
+                cmd_list = ['-D%s' % d for d in defines]
+                for c in includes:
+                    if c:
+                        cmd_list.append(('-I%s' % c) if not c.startswith('-') else c)                    
+                string = " ".join(cmd_list).replace("\\", "/")
+                f.write(string)
+        cmd.extend(['@%s' % options_file])
         
         return cmd
 
diff --git a/tools/toolchains/iar.py b/tools/toolchains/iar.py
index 258830eaa0..c449d560a6 100644
--- a/tools/toolchains/iar.py
+++ b/tools/toolchains/iar.py
@@ -107,21 +107,17 @@ class IAR(mbedToolchain):
     def get_compile_options(self, defines, includes):
         cmd = []
         
-        str = (' '.join(defines))+"|"+(' '.join(includes))
-        if len(str) > 160:
-            sum = md5(str).hexdigest()
-            options_file = join(self.temp_dir, "options_%s.txt" % sum)
-            if not exists(options_file):
-                with open(options_file, "wb") as f:
-                    cmd_list = ['-D%s' % d for d in defines]
-                    for c in includes:
-                        if c:
-                            cmd_list.append(('-I%s' % c) if not c.startswith('-') else c)                    
-                    string = " ".join(cmd_list).replace("\\", "/")
-                    f.write(string)
-            cmd.extend(['-f', options_file])
-        else:
-            cmd.extend(['-D%s' % d for d in defines] + ['-I%s' % i for i in includes])
+        sum = md5(' '.join(includes)).hexdigest()
+        options_file = join(self.temp_dir, "options_%s.txt" % sum)
+        if not exists(options_file):
+            with open(options_file, "wb") as f:
+                cmd_list = ['-D%s' % d for d in defines]
+                for c in includes:
+                    if c:
+                        cmd_list.append(('-I%s' % c) if not c.startswith('-') else c)                    
+                string = " ".join(cmd_list).replace("\\", "/")
+                f.write(string)
+        cmd.extend(['-f', options_file])
         
         return cmd
 

From f01e1363d15715ba2a94f4c2b80eb78f5b174d4e Mon Sep 17 00:00:00 2001
From: Mihail Stoyanov <wm@smode.org>
Date: Fri, 6 May 2016 12:18:07 +0100
Subject: [PATCH 09/14] De-duplicate the include paths. Without this all
 compiles are passed include paths twice

---
 tools/toolchains/__init__.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tools/toolchains/__init__.py b/tools/toolchains/__init__.py
index 3fe0e67796..441630dba6 100644
--- a/tools/toolchains/__init__.py
+++ b/tools/toolchains/__init__.py
@@ -485,13 +485,11 @@ class mbedToolchain:
         self.compiled = 0
         self.temp_dir = build_path
 
-        #for i in self.build_params:
-        #    self.debug(i)
-        #    self.debug("%s" % self.build_params[i])
-
         inc_paths = resources.inc_dirs
         if inc_dirs is not None:
             inc_paths.extend(inc_dirs)
+        # De-duplicate include paths and sort for consistency
+        inc_paths = sorted(set(inc_paths))
 
         objects = []
         queue = []
@@ -499,6 +497,7 @@ class mbedToolchain:
 
         # The dependency checking for C/C++ is delegated to the compiler
         base_path = resources.base_path
+        # Sort compile queue for consistency
         files_to_compile.sort()
         work_dir = getcwd()
 

From 7e11174a0065a9f348a583e0e4ba2813277ccb25 Mon Sep 17 00:00:00 2001
From: Mihail Stoyanov <wm@smode.org>
Date: Fri, 6 May 2016 13:06:48 +0100
Subject: [PATCH 10/14] Calculate md5 of all include paths in compile_sources()
 and remove calculation from <toolchaon>get_compile_options(), thus
 significantly reduce repetitive md5 calculations Unify handling of the
 include response file in mbedToolchain::get_inc_file() Sanitize obsolete
 no-longer needed methods

---
 tools/toolchains/__init__.py | 22 +++++++++++++++++++--
 tools/toolchains/arm.py      | 32 ++++++------------------------
 tools/toolchains/gcc.py      | 30 ++++++++++++----------------
 tools/toolchains/iar.py      | 38 ++++++++++++------------------------
 4 files changed, 50 insertions(+), 72 deletions(-)

diff --git a/tools/toolchains/__init__.py b/tools/toolchains/__init__.py
index 441630dba6..1a60ad0758 100644
--- a/tools/toolchains/__init__.py
+++ b/tools/toolchains/__init__.py
@@ -30,6 +30,7 @@ from multiprocessing import Pool, cpu_count
 from tools.utils import run_cmd, mkdir, rel_path, ToolException, NotSupportedException, split_path
 from tools.settings import BUILD_OPTIONS, MBED_ORG_USER
 import tools.hooks as hooks
+from hashlib import md5
 
 
 #Disables multiprocessing if set to higher number than the host machine CPUs
@@ -478,18 +479,35 @@ class mbedToolchain:
         mkdir(obj_dir)
         return join(obj_dir, name + '.o')
 
+    def get_inc_file(self, includes):
+        include_file = join(self.temp_dir, "includes_%s.txt" % self.inc_md5)
+        if not exists(include_file):
+            with open(include_file, "wb") as f:
+                cmd_list = []
+                for c in includes:
+                    if c:
+                        cmd_list.append(('-I%s' % c).replace("\\", "/"))                    
+                string = " ".join(cmd_list)
+                f.write(string)
+        return include_file
+
     def compile_sources(self, resources, build_path, inc_dirs=None):
         # Web IDE progress bar for project build
         files_to_compile = resources.s_sources + resources.c_sources + resources.cpp_sources
         self.to_be_compiled = len(files_to_compile)
         self.compiled = 0
-        self.temp_dir = build_path
 
         inc_paths = resources.inc_dirs
         if inc_dirs is not None:
             inc_paths.extend(inc_dirs)
-        # De-duplicate include paths and sort for consistency
+        # De-duplicate include paths
+        inc_paths = set(inc_paths)
+        # Sort include paths for consistency
         inc_paths = sorted(set(inc_paths))
+        # Unique id of all include paths
+        self.inc_md5 = md5(' '.join(inc_paths)).hexdigest()
+        # Where to store response files
+        self.temp_dir = build_path
 
         objects = []
         queue = []
diff --git a/tools/toolchains/arm.py b/tools/toolchains/arm.py
index 462369f591..c4633debf7 100644
--- a/tools/toolchains/arm.py
+++ b/tools/toolchains/arm.py
@@ -16,7 +16,6 @@ limitations under the License.
 """
 import re
 from os.path import join, dirname, splitext, basename, exists
-from hashlib import md5
 
 from tools.toolchains import mbedToolchain
 from tools.settings import ARM_BIN, ARM_INC, ARM_LIB, MY_ARM_CLIB, ARM_CPPLIB, GOANNA_PATH
@@ -79,11 +78,6 @@ class ARM(mbedToolchain):
         self.ar = join(ARM_BIN, "armar")
         self.elf2bin = join(ARM_BIN, "fromelf")
 
-    def remove_option(self, option):
-        for tool in [self.asm, self.cc, self.cppc]:
-            if option in tool:
-                tool.remove(option)
-
     def parse_dependencies(self, dep_path):
         dependencies = []
         for line in open(dep_path).readlines():
@@ -113,25 +107,13 @@ class ARM(mbedToolchain):
                     match.group('message')
                 )
 
-    def get_dep_opt(self, dep_path):
+    def get_dep_option(self, object):
+        base, _ = splitext(object)
+        dep_path = base + '.d'
         return ["--depend", dep_path]
 
-    def get_compile_options(self, defines, includes):
-        cmd = []
-        
-        sum = md5(' '.join(includes)).hexdigest()
-        options_file = join(self.temp_dir, "options_%s.txt" % sum)
-        if not exists(options_file):
-            with open(options_file, "wb") as f:
-                cmd_list = ['-D%s' % d for d in defines]
-                for c in includes:
-                    if c:
-                        cmd_list.append(('-I%s' % c) if not c.startswith('-') else c)                    
-                string = " ".join(cmd_list).replace("\\", "/")
-                f.write(string)
-        cmd.extend(['--via', options_file])
-        
-        return cmd
+    def get_compile_options(self, defines, includes):        
+        return ['-D%s' % d for d in defines] + ['--via', self.get_inc_file(includes)]
 
     @hook_tool
     def assemble(self, source, object, includes):
@@ -158,9 +140,7 @@ class ARM(mbedToolchain):
         # Build compile command
         cmd = cc + self.get_compile_options(self.get_symbols(), includes)
         
-        base, _ = splitext(object)
-        dep_path = base + '.d'
-        cmd.extend(self.get_dep_opt(dep_path))
+        cmd.extend(self.get_dep_option(object))
             
         cmd.extend(["-o", object, source])
 
diff --git a/tools/toolchains/gcc.py b/tools/toolchains/gcc.py
index 8fae2fb667..df4be262ac 100644
--- a/tools/toolchains/gcc.py
+++ b/tools/toolchains/gcc.py
@@ -16,7 +16,6 @@ limitations under the License.
 """
 import re
 from os.path import join, basename, splitext, dirname, exists
-from hashlib import md5
 
 from tools.toolchains import mbedToolchain
 from tools.settings import GCC_ARM_PATH, GCC_CR_PATH
@@ -69,7 +68,7 @@ class GCC(mbedToolchain):
             "-Wno-unused-parameter", "-Wno-missing-field-initializers",
             "-fmessage-length=0", "-fno-exceptions", "-fno-builtin",
             "-ffunction-sections", "-fdata-sections",
-            "-MMD", "-fno-delete-null-pointer-checks", "-fomit-frame-pointer"
+            "-fno-delete-null-pointer-checks", "-fomit-frame-pointer"
             ] + self.cpu
 
         if "save-asm" in self.options:
@@ -162,22 +161,13 @@ class GCC(mbedToolchain):
                     message + match.group('message')
                 )
 
+    def get_dep_option(self, object):
+        base, _ = splitext(object)
+        dep_path = base + '.d'
+        return ["-MD", "-MF", dep_path]
+
     def get_compile_options(self, defines, includes):
-        cmd = []
-        
-        sum = md5(' '.join(includes)).hexdigest()
-        options_file = join(self.temp_dir, "options_%s.txt" % sum)
-        if not exists(options_file):
-            with open(options_file, "wb") as f:
-                cmd_list = ['-D%s' % d for d in defines]
-                for c in includes:
-                    if c:
-                        cmd_list.append(('-I%s' % c) if not c.startswith('-') else c)                    
-                string = " ".join(cmd_list).replace("\\", "/")
-                f.write(string)
-        cmd.extend(['@%s' % options_file])
-        
-        return cmd
+        return ['-D%s' % d for d in defines] + ['@%s' % self.get_inc_file(includes)]
 
     @hook_tool
     def assemble(self, source, object, includes):
@@ -193,8 +183,12 @@ class GCC(mbedToolchain):
     @hook_tool
     def compile(self, cc, source, object, includes):
         # Build compile command
-        cmd = cc + self.get_compile_options(self.get_symbols(), includes) + ["-o", object, source]
+        cmd = cc + self.get_compile_options(self.get_symbols(), includes)
 
+        cmd.extend(self.get_dep_option(object))
+
+        cmd.extend(["-o", object, source])
+        
         # Call cmdline hook
         cmd = self.hook.get_cmdline_compiler(cmd)
 
diff --git a/tools/toolchains/iar.py b/tools/toolchains/iar.py
index c449d560a6..7d2fb60992 100644
--- a/tools/toolchains/iar.py
+++ b/tools/toolchains/iar.py
@@ -17,7 +17,6 @@ limitations under the License.
 import re
 from os import remove
 from os.path import join, exists, dirname, splitext, exists
-from hashlib import md5
 
 from tools.toolchains import mbedToolchain
 from tools.settings import IAR_PATH
@@ -73,6 +72,10 @@ class IAR(mbedToolchain):
         self.ar = join(IAR_BIN, "iarchive")
         self.elf2bin = join(IAR_BIN, "ielftool")
 
+    def parse_dependencies(self, dep_path):
+        return [path.strip() for path in open(dep_path).readlines()
+                if (path and not path.isspace())]
+
     def parse_output(self, output):
         for line in output.splitlines():
             match = IAR.DIAGNOSTIC_PATTERN.match(line)
@@ -94,32 +97,17 @@ class IAR(mbedToolchain):
                     match.group('message')
                 )
 
-    def get_dep_opt(self, dep_path):
+    def get_dep_option(self, object):
+        base, _ = splitext(object)
+        dep_path = base + '.d'
         return ["--dependencies", dep_path]
 
-    def cc_extra(self, base):
+    def cc_extra(self, object):
+        base, _ = splitext(object)
         return ["-l", base + '.s']
 
-    def parse_dependencies(self, dep_path):
-        return [path.strip() for path in open(dep_path).readlines()
-                if (path and not path.isspace())]
-
     def get_compile_options(self, defines, includes):
-        cmd = []
-        
-        sum = md5(' '.join(includes)).hexdigest()
-        options_file = join(self.temp_dir, "options_%s.txt" % sum)
-        if not exists(options_file):
-            with open(options_file, "wb") as f:
-                cmd_list = ['-D%s' % d for d in defines]
-                for c in includes:
-                    if c:
-                        cmd_list.append(('-I%s' % c) if not c.startswith('-') else c)                    
-                string = " ".join(cmd_list).replace("\\", "/")
-                f.write(string)
-        cmd.extend(['-f', options_file])
-        
-        return cmd
+        return ['-D%s' % d for d in defines] + ['-f', self.get_inc_file(includes)]
 
     @hook_tool
     def assemble(self, source, object, includes):
@@ -137,11 +125,9 @@ class IAR(mbedToolchain):
         # Build compile command
         cmd = cc +  self.get_compile_options(self.get_symbols(), includes)
 
-        base, _ = splitext(object)
-        dep_path = base + '.d'
-        cmd.extend(self.get_dep_opt(dep_path))
+        cmd.extend(self.get_dep_option(object))
 
-        cmd.extend(self.cc_extra(base))
+        cmd.extend(self.cc_extra(object))
         
         cmd.extend(["-o", object, source])
 

From d33f890fdd27c7849a7b7811a8d0a0db377024a5 Mon Sep 17 00:00:00 2001
From: Mihail Stoyanov <wm@smode.org>
Date: Fri, 6 May 2016 13:10:25 +0100
Subject: [PATCH 11/14] Minor code cleanup

---
 tools/toolchains/__init__.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tools/toolchains/__init__.py b/tools/toolchains/__init__.py
index 1a60ad0758..9880b2a616 100644
--- a/tools/toolchains/__init__.py
+++ b/tools/toolchains/__init__.py
@@ -24,7 +24,6 @@ from types import ListType
 from shutil import copyfile
 from os.path import join, splitext, exists, relpath, dirname, basename, split
 from inspect import getmro
-from tempfile import mkdtemp
 
 from multiprocessing import Pool, cpu_count
 from tools.utils import run_cmd, mkdir, rel_path, ToolException, NotSupportedException, split_path
@@ -212,8 +211,8 @@ class mbedToolchain:
         self.has_config = False
 
         self.build_all = False
+        self.build_dir = None
         self.timestamp = time()
-        self.temp_dir = None
         self.jobs = 1
 
         self.CHROOT = None
@@ -480,7 +479,7 @@ class mbedToolchain:
         return join(obj_dir, name + '.o')
 
     def get_inc_file(self, includes):
-        include_file = join(self.temp_dir, "includes_%s.txt" % self.inc_md5)
+        include_file = join(self.build_dir, "includes_%s.txt" % self.inc_md5)
         if not exists(include_file):
             with open(include_file, "wb") as f:
                 cmd_list = []
@@ -507,7 +506,7 @@ class mbedToolchain:
         # Unique id of all include paths
         self.inc_md5 = md5(' '.join(inc_paths)).hexdigest()
         # Where to store response files
-        self.temp_dir = build_path
+        self.build_dir = build_path
 
         objects = []
         queue = []

From 7e3b69ec13fba7d451e8563219f10af8d95bb040 Mon Sep 17 00:00:00 2001
From: Mihail Stoyanov <wm@smode.org>
Date: Fri, 6 May 2016 13:35:48 +0100
Subject: [PATCH 12/14] Add dot to include response file name, which follows
 the naming convention with .link_totals.txt and .link_files.txt

---
 tools/toolchains/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/toolchains/__init__.py b/tools/toolchains/__init__.py
index 9880b2a616..bce80cdb4b 100644
--- a/tools/toolchains/__init__.py
+++ b/tools/toolchains/__init__.py
@@ -479,7 +479,7 @@ class mbedToolchain:
         return join(obj_dir, name + '.o')
 
     def get_inc_file(self, includes):
-        include_file = join(self.build_dir, "includes_%s.txt" % self.inc_md5)
+        include_file = join(self.build_dir, ".includes_%s.txt" % self.inc_md5)
         if not exists(include_file):
             with open(include_file, "wb") as f:
                 cmd_list = []

From 4ae7d54830286328373afb7318ac9597c64a5936 Mon Sep 17 00:00:00 2001
From: Brian Daniels <brianddaniels@gmail.com>
Date: Tue, 10 May 2016 12:05:48 +0100
Subject: [PATCH 13/14] Fixing build issues for tests. Build artifacts from
 previous builds were polluting the current build. Now shared sources are
 built as a library, then tests are built into their own folder.

---
 tools/test.py     | 11 +++++++++--
 tools/test_api.py |  6 +++---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/tools/test.py b/tools/test.py
index 136a55fda6..e79c1dc6ac 100644
--- a/tools/test.py
+++ b/tools/test.py
@@ -27,7 +27,7 @@ sys.path.insert(0, ROOT)
 
 from tools.test_api import test_path_to_name, find_tests, print_tests, build_tests, test_spec_from_test_build
 from tools.options import get_default_options_parser
-from tools.build_api import build_project
+from tools.build_api import build_project, build_library
 from tools.targets import TARGET_MAP
 from tools.utils import mkdir
 
@@ -115,10 +115,17 @@ if __name__ == '__main__':
             if not base_source_paths:
                 base_source_paths = ['.']
             
+            
             target = TARGET_MAP[options.mcu]
             
+            lib_build_res = build_library(base_source_paths, options.build_dir, target, options.tool,
+                                            options=options.options,
+                                            jobs=options.jobs,
+                                            clean=options.clean,
+                                            archive=False)
+            
             # Build all the tests
-            test_build = build_tests(tests, base_source_paths, options.build_dir, target, options.tool,
+            test_build = build_tests(tests, [options.build_dir], options.build_dir, target, options.tool,
                     options=options.options,
                     clean=options.clean,
                     jobs=options.jobs)
diff --git a/tools/test_api.py b/tools/test_api.py
index 44a6b6aed6..812d5667d3 100644
--- a/tools/test_api.py
+++ b/tools/test_api.py
@@ -2041,13 +2041,13 @@ def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
     }
     
     for test_name, test_path in tests.iteritems():
+        test_build_path = os.path.join(build_path, test_path)
         src_path = base_source_paths + [test_path]
-        artifact_name = os.path.join(test_path, test_name)
-        bin_file = build_project(src_path, build_path, target, toolchain_name,
+        bin_file = build_project(src_path, test_build_path, target, toolchain_name,
                                  options=options,
                                  jobs=jobs,
                                  clean=clean,
-                                 name=artifact_name,
+                                 name=test_name,
                                  report=report,
                                  properties=properties,
                                  verbose=verbose)

From 8a37762866a1820c0a5333943b088056bbd2a063 Mon Sep 17 00:00:00 2001
From: Niklas Hauser <niklas.hauser@arm.com>
Date: Tue, 10 May 2016 14:02:58 +0100
Subject: [PATCH 14/14] Smaller Thread class with clearer error messages.

---
 core/mbed-rtos/rtos/Thread.cpp | 55 ++++++++++++++++++++--------------
 core/mbed-rtos/rtos/Thread.h   |  5 +---
 2 files changed, 34 insertions(+), 26 deletions(-)

diff --git a/core/mbed-rtos/rtos/Thread.cpp b/core/mbed-rtos/rtos/Thread.cpp
index 70ffa9e30c..402295e8d7 100644
--- a/core/mbed-rtos/rtos/Thread.cpp
+++ b/core/mbed-rtos/rtos/Thread.cpp
@@ -27,20 +27,35 @@
 namespace rtos {
 
 Thread::Thread(osPriority priority,
-        uint32_t stack_size, unsigned char *stack_pointer) {
-    _tid = NULL;
-    _priority = priority;
-    _stack_size = stack_size;
-    _stack_pointer = stack_pointer;
+        uint32_t stack_size, unsigned char *stack_pointer):
+        _tid(NULL), _dynamic_stack(stack_pointer == NULL) {
+#ifdef __MBED_CMSIS_RTOS_CM
+    _thread_def.tpriority = priority;
+    _thread_def.stacksize = stack_size;
+    _thread_def.stack_pointer = (uint32_t*)stack_pointer;
+#endif
 }
 
 Thread::Thread(void (*task)(void const *argument), void *argument,
-        osPriority priority, uint32_t stack_size, unsigned char *stack_pointer) {
-    _tid = NULL;
-    _priority = priority;
-    _stack_size = stack_size;
-    _stack_pointer = stack_pointer;
-    start(task, argument);
+        osPriority priority, uint32_t stack_size, unsigned char *stack_pointer):
+        _tid(NULL), _dynamic_stack(stack_pointer == NULL) {
+#ifdef __MBED_CMSIS_RTOS_CM
+    _thread_def.tpriority = priority;
+    _thread_def.stacksize = stack_size;
+    _thread_def.stack_pointer = (uint32_t*)stack_pointer;
+#endif
+    switch(start(task, argument)) {
+        case osErrorResource:
+            error("OS ran out of threads!\n");
+            break;
+        case osErrorParameter:
+            error("Thread already running!\n");
+            break;
+        case osErrorNoMemory:
+            error("Error allocating the stack memory\n");
+        default:
+            break;
+    }
 }
 
 osStatus Thread::start(void (*task)(void const *argument), void *argument) {
@@ -50,26 +65,22 @@ osStatus Thread::start(void (*task)(void const *argument), void *argument) {
 
 #ifdef __MBED_CMSIS_RTOS_CM
     _thread_def.pthread = task;
-    _thread_def.tpriority = _priority;
-    _thread_def.stacksize = _stack_size;
-    if (_stack_pointer != NULL) {
-        _thread_def.stack_pointer = (uint32_t*)_stack_pointer;
-    } else {
-        _thread_def.stack_pointer = new uint32_t[_stack_size/sizeof(uint32_t)];
+    if (_thread_def.stack_pointer == NULL) {
+        _thread_def.stack_pointer = new uint32_t[_thread_def.stacksize/sizeof(uint32_t)];
         if (_thread_def.stack_pointer == NULL)
-            error("Error allocating the stack memory\n");
+            return osErrorNoMemory;
     }
-    
+
     //Fill the stack with a magic word for maximum usage checking
-    for (uint32_t i = 0; i < (_stack_size / sizeof(uint32_t)); i++) {
+    for (uint32_t i = 0; i < (_thread_def.stacksize / sizeof(uint32_t)); i++) {
         _thread_def.stack_pointer[i] = 0xE25A2EA5;
     }
 #endif
     _tid = osThreadCreate(&_thread_def, argument);
     if (_tid == NULL) {
+        if (_dynamic_stack) delete[] (_thread_def.stack_pointer);
         return osErrorResource;
     }
-
     return osOK;
 }
 
@@ -195,7 +206,7 @@ void Thread::attach_idle_hook(void (*fptr)(void)) {
 Thread::~Thread() {
     terminate();
 #ifdef __MBED_CMSIS_RTOS_CM
-    if (_stack_pointer == NULL) {
+    if (_dynamic_stack) {
         delete[] (_thread_def.stack_pointer);
     }
 #endif
diff --git a/core/mbed-rtos/rtos/Thread.h b/core/mbed-rtos/rtos/Thread.h
index d9a5cbe68d..275f6fe5c9 100644
--- a/core/mbed-rtos/rtos/Thread.h
+++ b/core/mbed-rtos/rtos/Thread.h
@@ -167,10 +167,7 @@ public:
 private:
     osThreadId _tid;
     osThreadDef_t _thread_def;
-
-    osPriority _priority;
-    uint32_t _stack_size;
-    unsigned char *_stack_pointer;
+    bool _dynamic_stack;
 };
 
 }