Merge pull request #838 from MycroftAI/feature/display_state_manager

Feature/display state manager

mycroft/client/enclosure/__init__.py

@@ -36,12 +36,17 @@ from mycroft.util import play_wav, create_signal, connected, \
     wait_while_speaking
 from mycroft.util.audio_test import record
 from mycroft.util.log import getLogger
+from mycroft.client.enclosure.display_manager import \
+    initiate_display_manager_ws
 from mycroft.api import is_paired, has_been_paired
 
 __author__ = 'aatchison', 'jdorleans', 'iward'
 
 LOG = getLogger("EnclosureClient")
 
+# initiates the web sockets on display manager
+initiate_display_manager_ws()
+
 
 class EnclosureReader(Thread):
     """

mycroft/client/enclosure/api.py

@@ -15,15 +15,23 @@
 # You should have received a copy of the GNU General Public License
 # along with Mycroft Core.  If not, see <http://www.gnu.org/licenses/>.
 
-
+import mycroft.client.enclosure.display_manager as DisplayManager
 from mycroft.messagebus.message import Message
 from mycroft.util.log import getLogger
+from PIL import Image
 
 __author__ = 'jdorleans'
 
 LOGGER = getLogger(__name__)
 
 
+'''
+API for the functions that affect the Mark I device.
+NOTE: current state management is poorly implemented,
+will be changed in the future.
+'''
+
+
 class EnclosureAPI:
     """
     This API is intended to be used to interface with the hardware
@@ -37,8 +45,18 @@ class EnclosureAPI:
     where there is no face at all.
     """
 
-    def __init__(self, ws):
+    def __init__(self, ws, name=""):
         self.ws = ws
+        self.name = name
+
+    def register(self, skill_name=""):
+        """Registers a skill as active. Used for speak() and speak_dialog()
+        to 'patch' a previous implementation. Somewhat hacky.
+        """
+        if self.name != "":
+            DisplayManager.set_active(self.name)
+        else:
+            DisplayManager.set_active(skill_name)
 
     def reset(self):
         """The enclosure should restore itself to a started state.
@@ -135,22 +153,27 @@ class EnclosureAPI:
     def mouth_reset(self):
         """Restore the mouth display to normal (blank)"""
         self.ws.emit(Message("enclosure.mouth.reset"))
+        DisplayManager.set_active(self.name)
 
     def mouth_talk(self):
         """Show a generic 'talking' animation for non-synched speech"""
         self.ws.emit(Message("enclosure.mouth.talk"))
+        DisplayManager.set_active(self.name)
 
     def mouth_think(self):
         """Show a 'thinking' image or animation"""
         self.ws.emit(Message("enclosure.mouth.think"))
+        DisplayManager.set_active(self.name)
 
     def mouth_listen(self):
         """Show a 'thinking' image or animation"""
         self.ws.emit(Message("enclosure.mouth.listen"))
+        DisplayManager.set_active(self.name)
 
     def mouth_smile(self):
         """Show a 'smile' image or animation"""
         self.ws.emit(Message("enclosure.mouth.smile"))
+        DisplayManager.set_active(self.name)
 
     def mouth_viseme(self, code):
         """Display a viseme mouth shape for synched speech
@@ -170,8 +193,153 @@ class EnclosureAPI:
         Args:
             text (str): text string to display
         """
+        DisplayManager.set_active(self.name)
         self.ws.emit(Message("enclosure.mouth.text", {'text': text}))
 
+    def mouth_display(self, img_code="", x=0, y=0, refresh=True):
+        """Display images on faceplate. Currently supports images up to 16x8,
+           or half the face. You can use the 'x' parameter to cover the other
+           half of the faceplate.
+        Args:
+            img_code (str): text string that encodes a black and white image
+            x (int): x offset for image
+            y (int): y offset for image
+            refresh (bool): specify whether to clear the faceplate before
+                            displaying the new image or not.
+                            Useful if you'd like to display muliple images
+                            on the faceplate at once.
+        """
+        DisplayManager.set_active(self.name)
+        self.ws.emit(Message('enclosure.mouth.display',
+                             {'img_code': img_code,
+                              'xOffset': x,
+                              'yOffset': y,
+                              'clearPrev': refresh}))
+
+    def mouth_display_png(self, image_absolute_path, threshold=70,
+                          invert=False, x=0, y=0, refresh=True):
+        """Converts a png image into the appropriate encoding for the
+            Arduino Mark I enclosure.
+
+            NOTE: extract this out of api.py when re structuing the
+                  enclosure folder
+
+            Args:
+                image_absolute_path (string): The absolute path of the image
+                threshold (int): The value ranges from 0 to 255. The pixel will
+                                 draw on the faceplate it the value is below a
+                                 threshold
+                invert (bool): inverts the image being drawn.
+                x (int): x offset for image
+                y (int): y offset for image
+                refresh (bool): specify whether to clear the faceplate before
+                                displaying the new image or not.
+                                Useful if you'd like to display muliple images
+                                on the faceplate at once.
+            """
+        DisplayManager.set_active(self.name)
+
+        # to understand how this funtion works you need to understand how the
+        # Mark I arduino proprietary encoding works to display to the faceplate
+        img = Image.open(image_absolute_path).convert("RGBA")
+        img2 = Image.new('RGBA', img.size, (255, 255, 255))
+        width = img.size[0]
+        height = img.size[1]
+
+        # strips out alpha value and blends it with the RGB values
+        img = Image.alpha_composite(img2, img)
+        img = img.convert("L")
+
+        # crop image to only allow a max width of 16
+        if width > 32:
+            img = img.crop((0, 0, 32, height))
+            width = img.size[0]
+            height = img.size[1]
+
+        # crop the image to limit the max height of 8
+        if height > 8:
+            img = img.crop((0, 0, width, 8))
+            width = img.size[0]
+            height = img.size[1]
+
+        encode = ""
+
+        # Each char value represents a width number starting with B=1
+        # then increment 1 for the next. ie C=2
+        width_codes = ['B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L',
+                       'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W',
+                       'X', 'Y', 'Z', '[', '\\', ']', '^', '_', '`', 'a']
+
+        height_codes = ['B', 'C', 'D', 'E', 'F', 'G', 'H', 'I']
+
+        encode += width_codes[width - 1]
+        encode += height_codes[height - 1]
+
+        # Turn the image pixels into binary values 1's and 0's
+        # the Mark I face plate encoding uses binary values to
+        # binary_values returns a list of 1's and 0s'. ie ['1', '1', '0', ...]
+        binary_values = []
+        for i in range(width):
+            for j in range(height):
+                if img.getpixel((i, j)) < threshold:
+                    if invert is False:
+                        binary_values.append('1')
+                    else:
+                        binary_values.append('0')
+                else:
+                    if invert is False:
+                        binary_values.append('0')
+                    else:
+                        binary_values.append('1')
+
+        # these values are used to determine how binary values
+        # needs to be grouped together
+        number_of_top_pixel = 0
+        number_of_bottom_pixel = 0
+
+        if height > 4:
+            number_of_top_pixel = 4
+            number_of_bottom_pixel = height - 4
+        else:
+            number_of_top_pixel = height
+
+        # this loop will group together the individual binary values
+        # ie. binary_list = ['1111', '001', '0101', '100']
+        binary_list = []
+        binary_code = ''
+        increment = 0
+        alternate = False
+        for val in binary_values:
+            binary_code += val
+            increment += 1
+            if increment == number_of_top_pixel and alternate is False:
+                # binary code is reversed for encoding
+                binary_list.append(binary_code[::-1])
+                increment = 0
+                binary_code = ''
+                alternate = True
+            elif increment == number_of_bottom_pixel and alternate is True:
+                binary_list.append(binary_code[::-1])
+                increment = 0
+                binary_code = ''
+                alternate = False
+
+        # Code to let the Makrk I arduino know where to place the
+        # pixels on the faceplate
+        pixel_codes = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
+                       'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P']
+
+        for binary_values in binary_list:
+            number = int(binary_values, 2)
+            pixel_code = pixel_codes[number]
+            encode += pixel_code
+
+        self.ws.emit(Message("enclosure.mouth.display",
+                             {'img_code': encode,
+                              'xOffset': x,
+                              'yOffset': y,
+                              'clearPrev': refresh}))
+
     def weather_display(self, img_code, temp):
         """Show a the temperature and a weather icon
 
@@ -187,6 +355,7 @@ class EnclosureAPI:
                          7 = wind/mist
             temp (int): the temperature (either C or F, not indicated)
         """
+        DisplayManager.set_active(self.name)
         self.ws.emit(Message("enclosure.weather.display",
                              {'img_code': img_code, 'temp': temp}))
 

mycroft/client/enclosure/display_manager.py

@@ -0,0 +1,164 @@
+
+# Copyright 2017 Mycroft AI, Inc.
+#
+# This file is part of Mycroft Core.
+#
+# Mycroft Core is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Mycroft Core is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Mycroft Core.  If not, see <http://www.gnu.org/licenses/>.
+from threading import Thread, Timer
+from mycroft.messagebus.client.ws import WebsocketClient
+from mycroft.configuration import ConfigurationManager
+from mycroft.util import get_ipc_directory
+import json
+import os
+from logging import getLogger
+
+__author__ = 'connorpenrod', 'michaelnguyen'
+
+
+LOG = getLogger("Display Manager (mycroft.client.enclosure)")
+managerIPCDir = os.path.join(get_ipc_directory(), "managers")
+
+
+def _write_data(dictionary):
+    """Writes the parama as JSON to the
+        IPC dir (/tmp/mycroft/ipc/managers)
+        args:
+            dict: dictionary
+    """
+
+    # change read/write permissions based on if file exists or not
+    path = os.path.join(managerIPCDir, "disp_info")
+    permission = "r+" if os.path.isfile(path) else "w+"
+
+    if permission == "w+" and os.path.isdir(managerIPCDir) is False:
+        os.makedirs(managerIPCDir)
+
+    try:
+        with open(path, permission) as dispFile:
+
+            # check if file is empty
+            if os.stat(str(dispFile.name)).st_size != 0:
+                data = json.load(dispFile)
+
+            else:
+                data = {}
+                LOG.info("Display Manager is creating " + dispFile.name)
+
+            for key in dictionary:
+                data[key] = dictionary[key]
+
+            dispFile.seek(0)
+            dispFile.write(json.dumps(data))
+            dispFile.truncate()
+
+    except Exception as e:
+        LOG.error(e)
+        LOG.error("Error found in display manager file, deleting...")
+        os.remove(path)
+        _write_data(dictionary)
+
+
+def _read_data():
+    """ Reads the file in (/tmp/mycroft/ipc/managers/disp_info)
+        and returns the the data as python dict
+    """
+
+    path = os.path.join(managerIPCDir, "disp_info")
+    permission = "r" if os.path.isfile(path) else "w+"
+
+    if permission == "w+" and os.path.isdir(managerIPCDir) is False:
+        os.makedirs(managerIPCDir)
+
+    data = {}
+    try:
+        with open(path, permission) as dispFile:
+
+            if os.stat(str(dispFile.name)).st_size != 0:
+                data = json.load(dispFile)
+
+    except Exception as e:
+        LOG.error(e)
+        os.remove(path)
+        _read_data()
+
+    return data
+
+
+def set_active(skill_name):
+    """ Sets skill name as active in the display Manager
+        args:
+            string: skill_name
+    """
+    _write_data({"active_skill": skill_name})
+    LOG.info("Setting active skill to " + skill_name)
+
+
+def get_active():
+    """ Get active skill in the display manager
+    """
+    data = _read_data()
+    active_skill = ""
+
+    if "active_skill" in data:
+        active_skill = data["active_skill"]
+
+    return active_skill
+
+
+def remove_active():
+    """ Remove the active skill in the skill manager
+    """
+    LOG.error("Removing active skill...")
+    _write_data({"active_skill": ""})
+
+
+def initiate_display_manager_ws():
+    """ Initiates the web sockets on the display_manager
+    """
+
+    # Should remove needs to be an object so it can be referenced in functions
+    # [https://stackoverflow.com/questions/986006/how-do-i-pass-a-variable-by-reference]
+    should_remove = [True]
+
+    def check_flag(flag):
+        if flag[0] is True:
+            remove_active()
+
+    def set_delay(event=None):
+        should_remove[0] = True
+        Timer(2, check_flag, [should_remove]).start()
+
+    def set_remove_flag(event=None):
+        should_remove[0] = False
+
+    def connect():
+        ws.run_forever()
+
+    def remove_wake_word():
+        data = _read_data()
+        if "active_skill" in data and data["active_skill"] == "wakeword":
+            remove_active()
+
+    def set_wakeword_skill(event=None):
+        set_active("wakeword")
+        Timer(10, remove_wake_word).start()
+
+    ws = WebsocketClient()
+    ws.on('recognizer_loop:audio_output_end', set_delay)
+    ws.on('recognizer_loop:audio_output_start', set_remove_flag)
+    ws.on('recognizer_loop:record_begin', set_wakeword_skill)
+
+    event_thread = Thread(target=connect)
+    event_thread.setDaemon(True)
+    event_thread.start()

mycroft/client/enclosure/main.py

@@ -17,7 +17,6 @@
 
 
 import sys
-
 from mycroft.client.enclosure import Enclosure
 
 

mycroft/client/enclosure/mouth.py

@@ -17,6 +17,8 @@
 
 
 from mycroft.util.log import getLogger
+from threading import Timer
+import time
 
 __author__ = 'jdorleans'
 
@@ -33,6 +35,7 @@ class EnclosureMouth:
     def __init__(self, ws, writer):
         self.ws = ws
         self.writer = writer
+        self.is_timer_on = False
         self.__init_events()
 
     def __init_events(self):
@@ -43,6 +46,7 @@ class EnclosureMouth:
         self.ws.on('enclosure.mouth.smile', self.smile)
         self.ws.on('enclosure.mouth.viseme', self.viseme)
         self.ws.on('enclosure.mouth.text', self.text)
+        self.ws.on('enclosure.mouth.display', self.display)
 
     def reset(self, event=None):
         self.writer.write("mouth.reset")
@@ -70,3 +74,34 @@ class EnclosureMouth:
         if event and event.data:
             text = event.data.get("text", text)
         self.writer.write("mouth.text=" + text)
+
+    def display(self, event=None):
+        code = ""
+        xOffset = ""
+        yOffset = ""
+        clearPrevious = ""
+        if event and event.data:
+            code = event.data.get("img_code", code)
+            xOffset = event.data.get("xOffset", xOffset)
+            yOffset = event.data.get("yOffset", yOffset)
+            clearPrevious = event.data.get("clearPrev", clearPrevious)
+
+        clearPrevious = int(str(clearPrevious) == "True")
+        clearPrevious = "cP=" + str(clearPrevious) + ","
+        x_offset = "x=" + str(xOffset) + ","
+        y_offset = "y=" + str(yOffset) + ","
+
+        message = "mouth.icon=" + x_offset + y_offset + clearPrevious + code
+        # Check if message exceeds Arduino's serial buffer input limit 64 bytes
+        if len(message) > 60:
+            message1 = message[:31]
+            message2 = message[31:]
+            message1 += "$"
+            message2 += "$"
+            message2 = "mouth.icon=" + message2
+            self.writer.write(message1)
+            time.sleep(0.25)  # writer bugs out if sending messages too rapidly
+            self.writer.write(message2)
+        else:
+            time.sleep(0.1)
+            self.writer.write(message)

mycroft/skills/core.py

@@ -36,6 +36,7 @@ from mycroft.filesystem import FileSystemAccess
 from mycroft.messagebus.message import Message
 from mycroft.util.log import getLogger
 from mycroft.skills.settings import SkillSettings
+
 __author__ = 'seanfitz'
 
 skills_config = ConfigurationManager.instance().get("skills")
@@ -239,7 +240,7 @@ class MycroftSkill(object):
     def bind(self, emitter):
         if emitter:
             self.emitter = emitter
-            self.enclosure = EnclosureAPI(emitter)
+            self.enclosure = EnclosureAPI(emitter, self.name)
             self.__register_stop()
 
     def __register_stop(self):
@@ -325,6 +326,8 @@ class MycroftSkill(object):
         self.emitter.emit(Message('register_vocab', {'regex': regex_str}))
 
     def speak(self, utterance, expect_response=False):
+        # registers the skill as being active
+        self.enclosure.register(self.name)
         data = {'utterance': utterance,
                 'expect_response': expect_response}
         self.emitter.emit(Message("speak", data))

requirements.txt

@@ -32,5 +32,6 @@ wifi==0.3.8
 pyric==0.1.6
 inflection==0.3.1
 pytz==2017.2
+pillow==4.1.1
 mock
 python-dateutil==2.6.0