Merge branch 'dev' into feature/display_state_manager

pull/838/head^2
Michael Nguyen 2017-06-30 10:30:40 -05:00 committed by GitHub
commit eb03c3592c
16 changed files with 399 additions and 139 deletions

View File

@ -0,0 +1,28 @@
#!/usr/bin/env bash
#Docker doesn't use sudo so need one without sudo for automation of image.
apt-get install -y \
git \
python \
python-dev \
python-setuptools \
python-virtualenv \
python-gobject-dev \
virtualenvwrapper \
libtool \
libffi-dev \
libssl-dev \
autoconf \
automake \
bison \
swig \
libglib2.0-dev \
s3cmd \
portaudio19-dev \
mpg123 \
screen \
flac \
curl \
libicu-dev \
pkg-config \
automake

View File

@ -41,28 +41,33 @@ else
fi
# create virtualenv, consistent with virtualenv-wrapper conventions
if [ ! -d ${VIRTUALENV_ROOT} ]; then
mkdir -p $(dirname ${VIRTUALENV_ROOT})
virtualenv -p python2.7 ${VIRTUALENV_ROOT}
if [ ! -d "${VIRTUALENV_ROOT}" ]; then
mkdir -p $(dirname "${VIRTUALENV_ROOT}")
virtualenv -p python2.7 "${VIRTUALENV_ROOT}"
fi
source ${VIRTUALENV_ROOT}/bin/activate
cd ${TOP}
source "${VIRTUALENV_ROOT}/bin/activate"
cd "${TOP}"
easy_install pip==7.1.2 # force version of pip
pip install --upgrade virtualenv
# install requirements (except pocketsphinx)
pip2 install -r requirements.txt
CORES=$(nproc)
echo Building with $CORES cores.
if [[ $(free|awk '/^Mem:/{print $2}') -lt 1572864 ]] ; then
CORES=1
else
CORES=$(nproc)
fi
echo "Building with $CORES cores."
#build and install pocketsphinx
#cd ${TOP}
#${TOP}/scripts/install-pocketsphinx.sh -q
#build and install mimic
cd ${TOP}
${TOP}/scripts/install-mimic.sh
cd "${TOP}"
echo "WARNING: The following can take a long time to run!"
"${TOP}/scripts/install-mimic.sh"
# install pygtk for desktop_launcher skill
${TOP}/scripts/install-pygtk.sh
"${TOP}/scripts/install-pygtk.sh"

272
msm/msm
View File

@ -23,114 +23,188 @@
# This script assists in the installation and management of
# skills loaded from Github.
set -e
mycroft_skill_folder=${mycroft_skill_folder:-"/opt/mycroft/skills"}
if [[ ! -d "${mycroft_skill_folder}" ]] ; then
echo "Unable to access ${mycroft_skill_folder}!"
exit 101
fi
mycroft_skill_folder="/opt/mycroft/skills"
mycroft_virtualenv=~/.virtualenvs/mycroft/bin/activate
# picroft?
if [[ "$(hostname)" == 'picroft' ]] || [[ -x /home/pi/bin/cli ]] ; then
picroft='true'
else
picroft='false'
if [[ -r /etc/bash_completion.d/virtualenvwrapper ]]; then
source /etc/bash_completion.d/virtualenvwrapper
else
if locate virtualenvwrapper ; then
if ! source $(locate virtualenvwrapper) ; then
echo "Unable to locate virtualenvwrapper.sh, will not be able to install skills!"
vwrap='false'
fi
fi
fi
fi
default_skills="skill-alarm skill-audio-record skill-configuration skill-date-time skill-desktop-launcher skill-ip skill-joke skill-hello-world skill-media skill-npr-news skill-naptime skill-pairing skill-personal skill-reminder skill-installer skill-singing skill-speak skill-spelling skill-stop skill-stock skill-volume skill-weather skill-wiki skill-wolfram-alpha skill-mark1-demo"
echo "####### Mycroft Skill Manager #######"
function help() {
echo "msm: Mycroft Skill Manager"
echo -e " Copyright (c) 2017 Mycroft AI, Inc. All rights reserved.\n"
echo "usage: msm install <repository> or <name>"
echo " Installs the given Skill into the /opt/mycroft/skills directory"
echo " where <repository> is the address of the skill in Github."
echo -e "example: msm install https://github.com/ethanaward/demo_skill.git\n"
}
function goto_skills_dir {
if [ ! -d $mycroft_skill_folder ]; then
echo "Couldn't install skill, $mycroft_skill_folder does not exist"
exit 1
fi
cd $mycroft_skill_folder
}
function install() {
goto_skills_dir
if [ -z "$2" ]; then
echo "You must pass the git url or skill name"
exit 1
fi
if [[ "$2" == "git@"* || "$2" == "https://"* || "$2" == "http://"* ]]; then
repo=$2
else
skill_list="`curl -s "https://raw.githubusercontent.com/MycroftAI/mycroft-skills/master/.gitmodules"`"
skills=`echo "$skill_list" | grep -n 'submodule' | sed 's/[[:space:]]//g' | sed 's/\[submodule"//g' | sed 's/"\]//g'`
exact_match=`echo "$skills" | grep -i ".*:$2$"`
skill=`echo "$skills" | grep -i ".*:.*$2.*"`
if [ ! -z $exact_match ]; then
skill=$exact_match
fi
git_line=`echo "$skill" | sed 's/\:.*//'`
if [[ $skill == *$'\n'* ]]; then
echo -e "Your search has multiple choices\n\n$skill" | sed 's/.*://g'
exit 2
else
if [ -z $git_line ]; then
echo "Skill not found"
exit 3
fi
repo_line=$(($git_line + 2))
repo=`echo "$skill_list" | sed -n $repo_line'{p;q;}' | sed 's/[[:space:]]//g' | sed 's/url=//g'`
fi
fi
git_name=`echo "$repo" | sed 's/.*\///'`
name=`echo "$git_name" | sed 's/.git//'`
echo "Cloning repository"
git clone $repo >> /dev/null
cd $name
if [ -f "requirements.txt" ]; then
echo "Installing libraries requirements"
pip install -r requirements.txt
fi
echo "Skill installed!"
}
function update() {
goto_skills_dir
for d in $(ls -d */); do
if git -C "$d" rev-parse --git-dir > /dev/null 2>&1; then
cd $d
if [[ -z $(git status --porcelain) ]]; then
git fetch
git reset --hard origin/master
fi
cd ..
fi
done
}
function install_defaults() {
skills=( "alarm" "audio-record" "configuration" "date-time" "desktop-launcher" "ip" "joke" "hello-world" "media" "npr-news" "naptime" "pairing" "personal" "reminder" "installer" "singing" "speak" "spelling" "stop" "stock" "volume" "weather" "wiki" "wolfram-alpha" "mark1-demo" )
for i in "${skills[@]}"
do
if [ ! -d "$mycroft_skill_folder/skill-$i" ]; then
install "" "https://github.com/MycroftAI/skill-$i.git"
fi
done
update
echo "Installed!"
exit 0
echo "msm: Mycroft Skill Manager"
echo -e " Copyright (c) 2017 Mycroft AI, Inc. All rights reserved.\n"
echo "usage: msm install <repository> or <name>"
echo " Installs the given Skill into the ${mycroft_skill_folder}"
echo " where <repository> is the address of the skill in Github."
echo "example: msm search rss-skill"
echo -e "example: msm install https://github.com/ethanaward/demo_skill.git\n"
exit 1
}
function list() {
curl -s "https://raw.githubusercontent.com/MycroftAI/mycroft-skills/master/.gitmodules" | grep 'submodule "' | sed 's/\[submodule "//g'| sed 's/"\]//g'
if hash curl ; then
if ! curl -s "https://raw.githubusercontent.com/MycroftAI/mycroft-skills/master/.gitmodules" ; then
echo "Unable to pull master skills list!"
exit 111
fi
else
if ! wget -qO- "https://raw.githubusercontent.com/MycroftAI/mycroft-skills/master/.gitmodules" ; then
echo "Unable to pull master skills list!"
exit 112
fi
fi
}
if [ "$1" = "install" ]; then
install $*
elif [ "$1" = "list" ]; then
echo -e "Searching...\n"
list
elif [ "$1" = "update" ]; then
update
elif [ "$1" = "default" ]; then
install_defaults
else
help
function install() {
cd "${mycroft_skill_folder}"
if [[ "${vwrap}" == 'false' ]] ; then
echo "Missing virtualwrapper, cowardly refusing to install skills."
return 5
fi
# loop through list of arguments
while [[ $# -gt 0 ]] ; do
cd "${mycroft_skill_folder}"
iskill="${1}";
shift;
echo "Attempting to install ${iskill}..."
if [[ "${iskill}" == "git@"* || "${iskill}" == "https://"* || "${iskill}" == "http://"* ]]; then
repo="${iskill}"
else
skills=$(list | grep -n 'submodule' | sed 's/[[:space:]]//g' | sed 's/\[submodule"//g' | sed 's/"\]//g')
exact_match=$(echo "$skills" | grep -i ".*:${iskill}$")
skill=$(echo "$skills" | grep -i ".*:.*${iskill}.*")
if [[ ! -z "${exact_match}" ]]; then
skill=${exact_match}
fi
git_line=$(echo "$skill" | sed 's/\:.*//')
if [[ "${skill}" == *$'\n'* ]]; then
echo -e "Your search has multiple choices\n\n$skill" | sed 's/.*://g'
return 3
else
if [[ -z "${git_line}" ]]; then
echo "A ${iskill} skill was not found"
return 3
fi
repo_line=$(($git_line + 2))
repo=$(list | sed -n $repo_line'{p;q;}' | sed 's/[[:space:]]//g' | sed 's/url=//g')
fi
fi
git_name=$(echo "${repo}" | sed 's/.*\///')
name=$(echo "$git_name" | sed 's/.git//')
if [[ -d "${mycroft_skill_folder}/${name}" ]] ; then
echo "Skill appears to exist already. Perhaps you meant to use update?"
continue 169
fi
echo "Cloning repository"
git clone "${repo}" >> /dev/null
if ! cd "${name}" ; then
echo "Unable to access directory ${name}!"
return 102
fi
if [[ "${picroft}" == "true" ]] ; then
if ! sudo chown -R mycroft:mycroft "${mycroft_skill_folder}/${name}" ; then
echo "Unable to chown install directory ${name}!"
return 123
fi
fi
if [[ -f "requirements.txt" ]]; then
echo "Installing libraries requirements"
if [[ "${picroft}" == 'false' ]]; then
if [[ "${VIRTUAL_ENV}" =~ .mycroft$ ]] ; then
if ! pip install -r requirements.txt ; then
echo "Unable to install requirements for skill ${iskill}!"
return 121
fi
else
if workon mycroft ; then
if ! pip install -r requirements.txt ; then
echo "Unable to install requirements for skill ${iskill}!"
deactivate mycroft
return 121
fi
else
echo "Unable to activate mycroft virtualenv!"
deactivate
return 120
fi
fi
else
if ! sudo pip install -r requirements.txt ; then
echo "Unable to install requirements for skill ${iskill}!"
return 121
fi
fi
fi
echo "The ${iskill} skill has been installed!"
done
}
function update() {
cd "${mycroft_skill_folder}"
for d in $(find "${mycroft_skill_folder}" -mindepth 1 -maxdepth 1 -type d |grep -v '.git'$ ); do
if git -C "$d" rev-parse --git-dir > /dev/null 2>&1; then
cd "${d}"
if [[ -z $(git status --porcelain) ]]; then
git fetch
git reset --hard origin/master
fi
fi
done
}
function search() {
search_list=$(list | grep 'submodule "' | sed 's/\[submodule "//g'| sed 's/"\]//g')
while [[ $# -gt 0 ]] ; do
search_string=$1
shift
while read -r matches; do
if [[ "${search_string}" == "${matches}" ]] ; then
echo "Exact match found: ${matches}"
else
echo "Possible match: ${matches}"
fi
done < <(grep -i "${search_string}" <<< "${search_list}")
done
}
#### Main
OPT=$1
shift
case ${OPT} in
"install") if [[ $# -gt 0 ]] ; then install $(echo "$*") ; else help ; fi;;
"list") list | grep 'submodule "' | sed 's/\[submodule "//g'| sed 's/"\]//g' ;;
"update") update ;;
"default") install $(echo ${default_skills}) ;;
"search") if [[ $# -gt 0 ]] ; then search $(echo "$*") ; else help ; fi;;
*) help ;;
esac
exit_code=$?
if [[ ${exit_code} -gt 0 ]] ; then
echo "Sorry I'm unable to complete the request! Check the error messages above for why. Err ${exit_code}"
fi
exit 0

View File

@ -248,30 +248,45 @@ def rebuild_filtered_log():
##############################################################################
# Capturing output from Mycroft
def handle_speak(event):
global chat
tts_threads = []
def start_tts(utterance):
"""
Begin speaking in another thread to redirect output
Otherwise, the CLI get's polluted with text to speech debug
"""
global tts
mutex.acquire()
if not bQuiet:
ws.emit(Message("recognizer_loop:audio_output_start"))
try:
utterance = event.data.get('utterance')
if bSimple:
print(">> " + utterance)
else:
chat.append(">> " + utterance)
draw_screen()
if not bQuiet:
if not tts:
tts = TTSFactory.create()
tts.init(ws)
tts.execute(utterance)
if not tts:
tts = TTSFactory.create()
tts.init(ws)
tts.execute(utterance)
finally:
mutex.release()
if not bQuiet:
ws.emit(Message("recognizer_loop:audio_output_end"))
def handle_speak(event):
global chat
global tts_threads
utterance = event.data.get('utterance')
if bSimple:
print(">> " + utterance)
else:
chat.append(">> " + utterance)
draw_screen()
if not bQuiet:
t = Thread(start_tts, utterance)
t.start()
tts_threads.append(t)
def connect():
# Once the websocket has connected, just watch it for speak events
ws.run_forever()
@ -749,7 +764,7 @@ def main(stdscr):
# resizeterm() causes another curses.KEY_RESIZE, so
# we need to capture that to prevent a loop of resizes
c = scr.getch()
elif c == curses.KEY_BACKSPACE:
elif c == curses.KEY_BACKSPACE or c == 127:
# Backspace to erase a character in the utterance
line = line[:-1]
elif curses.ascii.isascii(c):

View File

@ -302,7 +302,7 @@ class WiFi:
if event and event.data.get("msg"):
self.intro_msg = event.data.get("msg")
self.allow_timeout = True
if event and event.data.get("allow_timeout"):
if event and event.data.get("allow_timeout") is False:
self.allow_timeout = event.data.get("allow_timeout")
# Fire up our access point
@ -417,7 +417,7 @@ class WiFi:
# has disconnected
if not self._is_ARP_filled():
cARPFailures += 1
if cARPFailures > 2:
if cARPFailures > 5:
self._connection_prompt("Connection lost.")
bHasConnected = False
else:

View File

@ -91,7 +91,6 @@ class ConfigurationLoader(object):
locations = ConfigurationLoader.init_locations(locations,
keep_user_config)
ConfigurationLoader.validate(config, locations)
for location in locations:
config = ConfigurationLoader.__load(config, location)
@ -135,6 +134,7 @@ class RemoteConfiguration(object):
config in the [core] config section
"""
IGNORED_SETTINGS = ["uuid", "@type", "active", "user", "device"]
WEB_CONFIG_CACHE = '/opt/mycroft/web_config_cache.json'
@staticmethod
def validate(config):
@ -156,8 +156,11 @@ class RemoteConfiguration(object):
if location:
setting["location"] = location
RemoteConfiguration.__load(config, setting)
RemoteConfiguration.__store_cache(setting)
except Exception as e:
LOG.warn("Failed to fetch remote configuration: %s" % repr(e))
RemoteConfiguration.__load_cache(config)
else:
LOG.debug("Remote configuration not activated.")
return config
@ -173,10 +176,35 @@ class RemoteConfiguration(object):
config[key] = config.get(key, {})
RemoteConfiguration.__load(config[key], v)
elif isinstance(v, list):
if key not in config:
config[key] = {}
RemoteConfiguration.__load_list(config[key], v)
else:
config[key] = v
@staticmethod
def __store_cache(setting):
"""
Cache the received settings locally. The cache will be used if
the remote is unreachable to load settings that are as close
to the user's as possible
"""
config = {}
# Remove server specific entries
RemoteConfiguration.__load(config, setting)
with open(RemoteConfiguration.WEB_CONFIG_CACHE, 'w') as f:
json.dump(config, f)
@staticmethod
def __load_cache(config):
"""
Load cache from file
"""
LOG.info("Using cached configuration if available")
ConfigurationLoader.load(config,
[RemoteConfiguration.WEB_CONFIG_CACHE],
False)
@staticmethod
def __load_list(config, values):
for v in values:

View File

@ -187,8 +187,11 @@ class Lock(object): # python 3+ 'class Lock'
*args: Ignored. Required as this fuction is called as a signel
handler.
'''
with open(self.path, 'r') as L:
if self.__pid == L.read():
os.unlink(self.path)
try:
with open(self.path, 'r') as L:
pid = int(L.read())
if self.__pid == pid:
os.unlink(self.path)
except IOError:
pass
# End class Lock

View File

@ -100,6 +100,17 @@ class WebsocketClient(object):
def remove(self, event_name, func):
self.emitter.remove_listener(event_name, func)
def remove_all_listeners(self, event_name):
'''
Remove all listeners connected to event_name.
Args:
event_name: event from which to remove listeners
'''
if event_name is None:
raise ValueError
self.emitter.remove_all_listeners(event_name)
def run_forever(self):
self.client.run_forever()

View File

@ -17,6 +17,7 @@
import tornado.ioloop as ioloop
import tornado.web as web
import tornado.autoreload as autoreload
from mycroft.configuration import ConfigurationManager
from mycroft.messagebus.service.ws import WebsocketEventHandler
@ -35,6 +36,13 @@ def main():
import tornado.options
lock = Lock("service")
tornado.options.parse_command_line()
def reload_hook():
""" Hook to release lock when autoreload is triggered. """
lock.delete()
tornado.autoreload.add_reload_hook(reload_hook)
config = ConfigurationManager.get().get("websocket")
host = config.get("host")

View File

@ -44,7 +44,6 @@ class SkillSettings(dict):
def __init__(self, settings_file):
super(SkillSettings, self).__init__()
self._path = settings_file
# if file exist, open and read stored values into self
if isfile(self._path):
with open(self._path) as f:
@ -52,7 +51,11 @@ class SkillSettings(dict):
for key in json_data:
self.__setitem__(key, json_data[key])
self._is_stored = True
self.loaded_hash = hash(str(self))
@property
def _is_stored(self):
return hash(str(self)) == self.loaded_hash
def __getitem__(self, key):
return super(SkillSettings, self).__getitem__(key)
@ -61,7 +64,6 @@ class SkillSettings(dict):
"""
Add/Update key and note that the file needs saving.
"""
self._is_stored = False
return super(SkillSettings, self).__setitem__(key, value)
def store(self):
@ -71,3 +73,4 @@ class SkillSettings(dict):
if not self._is_stored:
with open(self._path, 'w')as f:
json.dump(self, f)
self.loaded_hash = hash(str(self))

View File

@ -18,6 +18,7 @@ import json
import logging
from os.path import isfile
from mycroft.util.json_helper import load_commented_json
SYSTEM_CONFIG = '/etc/mycroft/mycroft.conf'
@ -26,9 +27,8 @@ __author__ = 'seanfitz'
log_level = "DEBUG"
if isfile(SYSTEM_CONFIG):
with open(SYSTEM_CONFIG) as f:
config = json.load(f)
log_level = config.get("log_level", "DEBUG")
config = load_commented_json(SYSTEM_CONFIG)
log_level = config.get("log_level", "DEBUG")
FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(format=FORMAT, level=logging.getLevelName(log_level))

View File

@ -26,7 +26,7 @@ __author__ = 'augustnmonteiro'
# START_VERSION_BLOCK
CORE_VERSION_MAJOR = 0
CORE_VERSION_MINOR = 8
CORE_VERSION_BUILD = 16
CORE_VERSION_BUILD = 17
# END_VERSION_BLOCK
CORE_VERSION_STR = (str(CORE_VERSION_MAJOR) + "." +

View File

@ -42,4 +42,5 @@ pyric==0.1.6
inflection==0.3.1
uuid==1.30
pytz==2017.2
pillow==4.1.1
pillow==4.1.1
mock

0
test/lock/__init__.py Normal file
View File

46
test/lock/test_lock.py Normal file
View File

@ -0,0 +1,46 @@
from os.path import dirname, join, exists, isfile
import os
import signal
import unittest
import mock
from shutil import rmtree
from mycroft.lock import Lock
class TestLock(unittest.TestCase):
def setUp(self):
if exists('/tmp/mycroft'):
rmtree('/tmp/mycroft')
def test_create_lock(self):
l1 = Lock('test')
self.assertTrue(isfile('/tmp/mycroft/test.pid'))
def test_delete_lock(self):
l1 = Lock('test')
self.assertTrue(isfile('/tmp/mycroft/test.pid'))
l1.delete()
self.assertFalse(isfile('/tmp/mycroft/test.pid'))
@mock.patch('os.kill')
def test_existing_lock(self, mock_kill):
""" Test that an existing lock will kill the old pid. """
l1 = Lock('test')
self.assertTrue(isfile('/tmp/mycroft/test.pid'))
l2 = Lock('test2')
self.assertFalse(mock_kill.called)
l2 = Lock('test')
self.assertTrue(mock_kill.called)
def test_keyboard_interrupt(self):
l1 = Lock('test')
self.assertTrue(isfile('/tmp/mycroft/test.pid'))
try:
os.kill(os.getpid(), signal.SIGINT)
except KeyboardInterrupt:
pass
self.assertFalse(isfile('/tmp/mycroft/test.pid'))
if __name__ == '__main__':
unittest.main()

View File

@ -36,3 +36,41 @@ class SkillSettingsTest(unittest.TestCase):
s2 = SkillSettings(join(dirname(__file__), 'settings', 'store.json'))
for key in s:
self.assertEqual(s[key], s2[key])
def test_update_list(self):
s = SkillSettings(join(dirname(__file__), 'settings', 'store.json'))
s['l'] = ['a', 'b', 'c']
s.store()
s2 = SkillSettings(join(dirname(__file__), 'settings', 'store.json'))
self.assertEqual(s['l'], s2['l'])
# Update list
s2['l'].append('d')
s2.store()
s3 = SkillSettings(join(dirname(__file__), 'settings', 'store.json'))
self.assertEqual(s2['l'], s3['l'])
def test_update_dict(self):
s = SkillSettings(join(dirname(__file__), 'settings', 'store.json'))
s['d'] = {'a': 1, 'b': 2}
s.store()
s2 = SkillSettings(join(dirname(__file__), 'settings', 'store.json'))
self.assertEqual(s['d'], s2['d'])
# Update dict
s2['d']['c'] = 3
s2.store()
s3 = SkillSettings(join(dirname(__file__), 'settings', 'store.json'))
self.assertEqual(s2['d'], s3['d'])
def test_no_change(self):
s = SkillSettings(join(dirname(__file__), 'settings', 'store.json'))
s['d'] = {'a': 1, 'b': 2}
s.store()
s = SkillSettings(join(dirname(__file__), 'settings', 'store.json'))
self.assertTrue(s._is_stored)
if __name__ == '__main__':
unittest.main()