Further enhancements:

* Added "--simple" mode, to get the old cli behavior
* Rewrote to not use tail (works better with multiple log files)
* Added Ctrl+PgUp/Dn support for scrolling back in logs
* Added filtering
* Refined look and log coloring
* Added :help screen
* Added support for terminal resizing

mycroft/client/text/main.py

@@ -1,4 +1,4 @@
-# Copyright 2016 Mycroft AI, Inc.
+# Copyright 2017 Mycroft AI, Inc.
 #
 # This file is part of Mycroft Core.
 #
@@ -15,21 +15,28 @@
 # You should have received a copy of the GNU General Public License
 # along with Mycroft Core.  If not, see <http://www.gnu.org/licenses/>.
 
-import os
-import os.path
 import sys
-import time
-import subprocess
 from cStringIO import StringIO
-from threading import Thread, Lock
 
-import curses
+# NOTE: If this script has errors, the following two lines might need to
-import curses.ascii
+# be commented out for them to be displayed (depending on the type of
+# error).  But normally we want this to prevent extra messages from the
+# messagebus setup from appearing during startup.
+sys.stdout = StringIO()  # capture any output
+sys.stderr = StringIO()  # capture any output
 
-from mycroft.messagebus.client.ws import WebsocketClient
+# All of the nopep8 comments below are to avoid E402 errors
-from mycroft.messagebus.message import Message
+import os                                                   # nopep8
-from mycroft.tts import TTSFactory
+import os.path                                              # nopep8
-from mycroft.util.log import getLogger
+import time                                                 # nopep8
+import subprocess                                           # nopep8
+import curses                                               # nopep8
+import curses.ascii                                         # nopep8
+from threading import Thread, Lock                          # nopep8
+from mycroft.messagebus.client.ws import WebsocketClient    # nopep8
+from mycroft.messagebus.message import Message              # nopep8
+from mycroft.tts import TTSFactory                          # nopep8
+from mycroft.util.log import getLogger                      # nopep8
 
 tts = None
 ws = None
@@ -38,15 +45,19 @@ logger = getLogger("CLIClient")
 
 utterances = []
 chat = []
-mergedLog = []
 line = "What time is it"
+bSimple = '--simple' in sys.argv
 bQuiet = '--quiet' in sys.argv
 scr = None
+log_line_offset = 0  # num lines back in logs to show
+
+mergedLog = []
+log_filters = ["enclosure.mouth.viseme"]
+
 
 ##############################################################################
 # Helper functions
 
-
 def clamp(n, smallest, largest):
     return max(smallest, min(n, largest))
 
@@ -59,31 +70,48 @@ def stripNonAscii(text):
 # Log file monitoring
 
 class LogMonitorThread(Thread):
-    def __init__(self, filename):
+    def __init__(self, filename, logid):
         Thread.__init__(self)
         self.filename = filename
+        self.st_results = os.stat(filename)
+        self.logid = logid
 
     def run(self):
         global mergedLog
 
-        proc = subprocess.Popen(["tail", "-f", self.filename],
-                                stdout=subprocess.PIPE)
         while True:
-            output = proc.stdout.readline().strip()
+            st_results = os.stat(self.filename)
-            if output == "" and proc.poll() is not None:
+            if not st_results.st_mtime == self.st_results.st_mtime:
+                self.read_file_from(self.st_results.st_size)
+                self.st_results = st_results
+                draw_screen()
+            time.sleep(0.1)
+
+    def read_file_from(self, bytefrom):
+        with open(self.filename, 'rb') as fh:
+            fh.seek(bytefrom)
+            while True:
+                line = fh.readline()
+                if line == "":
                     break
 
-            # TODO: Allow user to filter log output (blacklist and whitelist)
+                # Allow user to filter log output
-            if "enclosure.mouth.viseme" in output:
+                ignore = False
-                continue
+                for filtered_text in log_filters:
+                    if filtered_text in line:
+                        ignore = True
+                        break
 
-            if output:
+                if not ignore:
-                mergedLog.append(output)
+                    if bSimple:
-                draw_screen()
+                        print line.strip()
+                    else:
+                        mergedLog.append(self.logid+line.strip())
 
 
-def startLogMonitor(filename):
+def startLogMonitor(filename, logid):
-    thread = LogMonitorThread(filename)
+    if os.path.isfile(filename):
+        thread = LogMonitorThread(filename, logid)
         thread.setDaemon(True)  # this thread won't prevent prog from exiting
         thread.start()
 
@@ -93,11 +121,15 @@ def startLogMonitor(filename):
 
 def handle_speak(event):
     global chat
+    global tts
     mutex.acquire()
     if not bQuiet:
         ws.emit(Message("recognizer_loop:audio_output_start"))
     try:
         utterance = event.data.get('utterance')
+        if bSimple:
+            print(">> " + utterance)
+        else:
             chat.append(">> " + utterance)
         draw_screen()
         if not bQuiet:
@@ -119,69 +151,97 @@ def connect():
 ##############################################################################
 # Screen handling
 
-
 def init_screen():
-    global CLR_CHAT_HEADING
+    global CLR_HEADING
     global CLR_CHAT_RESP
     global CLR_CHAT_QUERY
     global CLR_CMDLINE
     global CLR_INPUT
-    global CLR_LOG
+    global CLR_LOG1
+    global CLR_LOG2
     global CLR_LOG_DEBUG
 
     if curses.has_colors():
-        bg = curses.COLOR_BLACK
+        bg = curses.COLOR_WHITE
         for i in range(0, curses.COLORS):
             curses.init_pair(i + 1, i, bg)
+            bg = curses.COLOR_BLACK
 
-        # Colors
+        # Colors:
-        # 1 = black on black
+        # 1 = black on white    9 = dk gray
-        # 2 = dk red
+        # 2 = dk red           10 = red
-        # 3 = dk green
+        # 3 = dk green         11 = green
-        # 4 = dk yellow
+        # 4 = dk yellow        12 = yellow
-        # 5 = dk blue
+        # 5 = dk blue          13 = blue
-        # 6 = dk purple
+        # 6 = dk purple        14 = purple
-        # 7 = dk cyan
+        # 7 = dk cyan          15 = cyan
-        # 8 = lt gray
+        # 8 = lt gray          16 = white
-        # 9 = dk gray
+        CLR_HEADING = curses.color_pair(16)
-        # 10= red
-        # 11= green
-        # 12= yellow
-        # 13= blue
-        # 14= purple
-        # 15= cyan
-        # 16= white
-
-        CLR_CHAT_HEADING = curses.color_pair(3)
         CLR_CHAT_RESP = curses.color_pair(7)
         CLR_CHAT_QUERY = curses.color_pair(8)
         CLR_CMDLINE = curses.color_pair(15)
         CLR_INPUT = curses.color_pair(16)
-        CLR_LOG = curses.color_pair(8)
+        CLR_LOG1 = curses.color_pair(8)
+        CLR_LOG2 = curses.color_pair(6)
         CLR_LOG_DEBUG = curses.color_pair(4)
 
 
+def page_log(page_up):
+    global log_line_offset
+    if page_up:
+        log_line_offset += 10
+    else:
+        log_line_offset -= 10
+    if log_line_offset > len(mergedLog):
+        log_line_offset = len(mergedLog)-10
+    if log_line_offset < 0:
+        log_line_offset = 0
+    draw_screen()
+
+
 def draw_screen():
     global scr
+    global log_line_offset
+    if not scr:
+        return
+
     scr.clear()
 
     # Display log output at the top
-    scr.addstr(0, 0, "Log Output", curses.A_REVERSE)
-    scr.addstr(1, 0,  "=" * (curses.COLS-1), CLR_LOG)
-    cLogLines = curses.LINES-13
-
     cLogs = len(mergedLog)
+    cLogLinesToShow = curses.LINES-13
+    start = clamp(cLogs - cLogLinesToShow, 0, cLogs - 1) - log_line_offset
+    end = cLogs - log_line_offset
+    if start < 0:
+        end -= start
+        start = 0
+    if end > cLogs:
+        end = cLogs
+
+    # adjust the line offset (prevents paging up too far)
+    log_line_offset = cLogs - end
+
+    scr.addstr(0, 0, "Log Output:" + " " * (curses.COLS-31) + str(start) +
+               "-" + str(end) + " of " + str(cLogs), CLR_HEADING)
+    scr.addstr(1, 0,  "=" * (curses.COLS-1), CLR_HEADING)
     y = 2
-    for i in range(clamp(cLogs-cLogLines, 0, cLogs-1), cLogs):
+    for i in range(start, end):
         log = mergedLog[i]
-        log = log[26:]  # skip date/time at the front of log line
+        logid = log[0]
+        if log[5] == '-' and log[8] == '-':  # matches 'YYYY-MM-DD HH:MM:SS'
+            log = log[27:]  # skip logid & date/time at the front of log line
+        else:
+            log = log[1:]   # just skip the logid
 
         # Categorize log line
         if "Skills - DEBUG - " in log:
             log = log.replace("Skills - DEBUG - ", "")
             clr = CLR_LOG_DEBUG
         else:
-            clr = CLR_LOG
+            if logid == "1":
+                clr = CLR_LOG1
+            else:
+                clr = CLR_LOG2
 
         # limit line to screen width (show tail end)
         log = ("..."+log[-(curses.COLS-3):]) if len(log) > curses.COLS else log
@@ -190,20 +250,21 @@ def draw_screen():
 
     # Log legend in the lower-right
     scr.addstr(curses.LINES-10, curses.COLS/2 + 2, "Log Output Legend",
-               curses.A_REVERSE)
+               CLR_HEADING)
-    scr.addstr(curses.LINES-9, curses.COLS/2 + 2, "=" * (curses.COLS/2 - 4))
+    scr.addstr(curses.LINES-9, curses.COLS/2 + 2, "=" * (curses.COLS/2 - 4),
+               CLR_HEADING)
     scr.addstr(curses.LINES-8, curses.COLS/2 + 2,
-               "mycroft-skills.log, debug info",
+               "mycroft-skills.log, system debug",
                CLR_LOG_DEBUG)
     scr.addstr(curses.LINES-7, curses.COLS/2 + 2,
-               "mycroft-skills.log, non debug",
+               "mycroft-skills.log, other",
-               CLR_LOG)
+               CLR_LOG1)
     scr.addstr(curses.LINES-6, curses.COLS/2 + 2, "mycroft-voice.log",
-               CLR_LOG)
+               CLR_LOG2)
 
     # History log in the middle
-    scr.addstr(curses.LINES-10, 0, "History", CLR_CHAT_HEADING)
+    scr.addstr(curses.LINES-10, 0, "History", CLR_HEADING)
-    scr.addstr(curses.LINES-9, 0,  "=" * (curses.COLS/2), CLR_CHAT_HEADING)
+    scr.addstr(curses.LINES-9, 0,  "=" * (curses.COLS/2), CLR_HEADING)
 
     cChat = len(chat)
     if cChat:
@@ -225,21 +286,61 @@ def draw_screen():
         scr.addstr(curses.LINES-1, 0, ":", CLR_CMDLINE)
         l = line[1:]
     else:
-        scr.addstr(curses.LINES-2, 0, "Input (Ctrl+C to quit):", CLR_CMDLINE)
+        scr.addstr(curses.LINES-2, 0,
+                   "Input (':' for command mode, Ctrl+C to quit):",
+                   CLR_CMDLINE)
         scr.addstr(curses.LINES-1, 0, ">", CLR_CMDLINE)
     scr.addstr(curses.LINES-1, 2, l, CLR_INPUT)
     scr.refresh()
 
 
+def show_help():
+    global scr
+    if not scr:
+        return
+
+    scr.clear()
+    scr.addstr(0, 0,  center(25) + "Mycroft Command Line Help",
+               CLR_CMDLINE)
+    scr.addstr(1, 0,  "=" * (curses.COLS-1),
+               CLR_CMDLINE)
+    scr.addstr(2, 0,  "Up / Down         scroll thru query history")
+    scr.addstr(3, 0,  "Ctrl+PgUp / PgDn  scroll thru log history")
+
+    scr.addstr(10, 0,  "Commands (type ':' to enter command mode)",
+               CLR_CMDLINE)
+    scr.addstr(11, 0,  "=" * (curses.COLS-1),
+               CLR_CMDLINE)
+    scr.addstr(12, 0,  ":help             this screen")
+    scr.addstr(13, 0,  ":quit or :exit    exit the program")
+
+    scr.addstr(curses.LINES-1, 0,  center(23) + "Press any key to return",
+               CLR_HEADING)
+
+    scr.refresh()
+    c = scr.getch()
+
+
+def center(str_len):
+    # generate number of characters needed to center a string
+    # of the given length
+    return " " * ((curses.COLS - str_len) / 2)
+
+
 ##############################################################################
 # Main UI
 
 def handle_cmd(cmd):
     if "show" in cmd and "log" in cmd:
         pass
-    elif "errors" in cmd:
+    elif "help" in cmd:
-        # Look in all logs for error messages, print here
+        show_help()
-        pass
+    elif "exit" in cmd or "quit" in cmd:
+        return 1
+    # TODO: More commands
+    # elif "find" in cmd:
+    #    ... search logs for given string
+    return 0  # do nothing upon return
 
 
 def main(stdscr):
@@ -251,7 +352,6 @@ def main(stdscr):
     init_screen()
 
     ws = WebsocketClient()
-
     ws.on('speak', handle_speak)
     event_thread = Thread(target=connect)
     event_thread.setDaemon(True)
@@ -263,70 +363,113 @@ def main(stdscr):
         input = ""
         while True:
             draw_screen()
-            # TODO: Change this mechanism
-            # Sleep for a while so all the output that results
-            # from the previous command finishes before we print.
-            # time.sleep(1.5)
 
-            # print("Input (Ctrl+C to quit):")
             c = scr.getch()
             if c == curses.KEY_ENTER or c == 10 or c == 13:
                 if line == "":
                     continue
 
                 if line[:1] == ":":
-                    handle_cmd(line[1:])
+                    # Lines typed like ":help" are 'commands'
+                    if handle_cmd(line[1:]) == 1:
+                        break
                 else:
+                    # Treat this as an utterance
                     history.append(line)
                     chat.append(line)
-                    ws.emit(
+                    ws.emit(Message("recognizer_loop:utterance",
-                        Message("recognizer_loop:utterance",
+                                    {'utterances': [line.strip()],
-                                {'utterances': [line.strip()]}))
+                                     'lang': 'en-us'}))
                 hist_idx = -1
                 line = ""
             elif c == curses.KEY_UP:
+                # Move up the history stack
                 hist_idx = clamp(hist_idx+1, -1, len(history)-1)
                 if hist_idx >= 0:
                     line = history[len(history)-hist_idx-1]
                 else:
                     line = ""
             elif c == curses.KEY_DOWN:
+                # Move down the history stack
                 hist_idx = clamp(hist_idx-1, -1, len(history)-1)
                 if hist_idx >= 0:
                     line = history[len(history)-hist_idx-1]
                 else:
                     line = ""
             elif curses.ascii.isascii(c):
+                # Accept typed character
                 line += chr(c)
             elif c == curses.KEY_BACKSPACE:
                 line = line[:-1]
-            else:
+            elif c == 555:  # Ctrl+PgUp
-                line += str(c)
+                page_log(True)
-                pass
+                draw_screen()
-            # if line.startswith("*"):
+            elif c == 550:  # Ctrl+PgUp
-            #     handle_cmd(line.strip("*"))
+                page_log(False)
+            elif c == curses.KEY_RESIZE:
+                y, x = scr.getmaxyx()
+                curses.resizeterm(y, x)
+
+                # resizeterm() causes another curses.KEY_RESIZE, so
+                # we need to capture that to prevent a loop of resizes
+                c = scr.getch()
+
+            # DEBUG: Uncomment the following code to see what key codes
+            #        are generated when an unknown key is pressed.
             # else:
+            #    line += str(c)
 
     except KeyboardInterrupt, e:
         # User hit Ctrl+C to quit
         pass
+    except KeyboardInterrupt, e:
+        logger.exception(e)
+    finally:
+        print "quitting"
+        pass
+
+
+def simple_cli():
+    global ws
+    ws = WebsocketClient()
+    event_thread = Thread(target=connect)
+    event_thread.setDaemon(True)
+    event_thread.start()
+    try:
+        while True:
+            # TODO: Change this mechanism
+            # Sleep for a while so all the output that results
+            # from the previous command finishes before we print.
+            time.sleep(1.5)
+            print("Input (Ctrl+C to quit):")
+            line = sys.stdin.readline()
+            ws.emit(
+                Message("recognizer_loop:utterance",
+                        {'utterances': [line.strip()]}))
+    except KeyboardInterrupt, e:
+        # User hit Ctrl+C to quit
+        print("")
     except KeyboardInterrupt, e:
         logger.exception(e)
         event_thread.exit()
         sys.exit()
 
-
 # Find the correct log path relative to this script
 scriptPath = os.path.dirname(os.path.realpath(__file__))
 localLogPath = os.path.realpath(scriptPath+"/../../../scripts/logs")
 
 # Monitor relative logs (for Github installs)
-startLogMonitor(localLogPath + "/mycroft-skills.log")
+startLogMonitor(localLogPath + "/mycroft-skills.log", "1")
-startLogMonitor(localLogPath + "/mycroft-voice.log")
+startLogMonitor(localLogPath + "/mycroft-voice.log", "2")
 
-# Also monitor system logs (for apt-get installs)
+# Also monitor system logs (for package installs)
-startLogMonitor("/var/log/mycroft-skills.log")
+startLogMonitor("/var/log/mycroft-skills.log", "1")
-startLogMonitor("/var/log/mycroft-voice.log")
+startLogMonitor("/var/log/mycroft-voice.log", "2")
 
 if __name__ == "__main__":
+    if bSimple:
+        sys.stdout = sys.__stdout__
+        sys.stderr = sys.__stderr__
+        simple_cli()
+    else:
         curses.wrapper(main)