Issues 108 - Change regex to be localized (#222)

* Issues 108 - Add load_regex_files

* Issues 108 - Update weather skill with new regex syntax

* Issues 108 - Update stock skill with new regex syntax

* Issues 108 - Add stock regex file

* Issues 108 - Update time skill to use new regex syntax

* Issues 108 - Update desktop skill to use new regex syntax

* Issues 108 - Update volume skill to use new regex syntax

* Issues 108 - Update wikipedia skill to use new regex syntax

* Issues 108 - Update spelling skill to use new regex syntax

* Issues 108 - Update sms skill to use new regex syntax

* Issues 108 - Update calling skill to use new regex syntax

* Issues 108 - Remove unused argument

* Issues 108 - Add unit tests

* Issues 108 - Minor changes to fix tests

* Issues 108 - Preserve intended test logic

* Issues 108 - Address feedback

* Issues 108 - Update test formatting

* Issues 108 - Change file location
pull/248/head
Ethan Ward 2016-06-28 15:20:48 -05:00 committed by Jonathan D'Orleans
parent eed03f3036
commit 90905d526c
25 changed files with 147 additions and 55 deletions

View File

@ -60,12 +60,29 @@ def load_vocab_from_file(path, vocab_type, emitter):
'alias_of': entity})) 'alias_of': entity}))
def load_regex_from_file(path, emitter):
if(path.endswith('.rx')):
with open(path, 'r') as reg_file:
for line in reg_file.readlines():
re.compile(line.strip())
emitter.emit(
Message("register_vocab",
metadata={'regex': line.strip()}))
def load_vocabulary(basedir, emitter): def load_vocabulary(basedir, emitter):
for vocab_type in os.listdir(basedir): for vocab_type in os.listdir(basedir):
load_vocab_from_file( load_vocab_from_file(
join(basedir, vocab_type), splitext(vocab_type)[0], emitter) join(basedir, vocab_type), splitext(vocab_type)[0], emitter)
def load_regex(basedir, emitter):
for regex_type in os.listdir(basedir):
if regex_type.endswith(".rx"):
load_regex_from_file(
join(basedir, regex_type), emitter)
def create_intent_envelope(intent): def create_intent_envelope(intent):
return Message(None, metadata=intent.__dict__, context={}) return Message(None, metadata=intent.__dict__, context={})
@ -225,6 +242,9 @@ class MycroftSkill(object):
def load_vocab_files(self, vocab_dir): def load_vocab_files(self, vocab_dir):
load_vocabulary(vocab_dir, self.emitter) load_vocabulary(vocab_dir, self.emitter)
def load_regex_files(self, regex_dir):
load_regex(regex_dir, self.emitter)
def __handle_stop(self, event): def __handle_stop(self, event):
self.stop_time = time.time() self.stop_time = time.time()
self.stop() self.stop()

View File

@ -37,9 +37,7 @@ class TimeSkill(MycroftSkill):
def initialize(self): def initialize(self):
self.load_vocab_files(join(dirname(__file__), 'vocab', 'en-us')) self.load_vocab_files(join(dirname(__file__), 'vocab', 'en-us'))
self.load_regex_files(join(dirname(__file__), 'regex', 'en-us'))
self.register_regex("in (?P<Location>.*)")
self.register_regex("at (?P<Location>.*)")
intent = IntentBuilder("TimeIntent").require( intent = IntentBuilder("TimeIntent").require(
"TimeKeyword").optionally("Location").build() "TimeKeyword").optionally("Location").build()

View File

@ -0,0 +1 @@
(at|in) (?P<Location>.*)

View File

@ -16,7 +16,7 @@
# along with Mycroft Core. If not, see <http://www.gnu.org/licenses/>. # along with Mycroft Core. If not, see <http://www.gnu.org/licenses/>.
import os from os.path import dirname, join
import sys import sys
import urllib2 import urllib2
import webbrowser import webbrowser
@ -48,9 +48,8 @@ class DesktopLauncherSkill(MycroftSkill):
logger.error("Could not import gio") logger.error("Could not import gio")
return return
vocab_dir = os.path.join(os.path.dirname(__file__), 'vocab', 'en-us') self.load_vocab_files(join(dirname(__file__), 'vocab', self.lang))
self.load_regex_files(join(dirname(__file__), 'regex', self.lang))
self.load_vocab_files(vocab_dir)
tokenizer = EnglishTokenizer() tokenizer = EnglishTokenizer()
for app in gio.app_info_get_all(): for app in gio.app_info_get_all():
@ -71,10 +70,6 @@ class DesktopLauncherSkill(MycroftSkill):
else: else:
self.appmap[tokenized_name] = entry self.appmap[tokenized_name] = entry
self.register_regex("for (?P<SearchTerms>.*)")
self.register_regex("for (?P<SearchTerms>.*) on")
self.register_regex("(?P<SearchTerms>.*) on")
launch_intent = IntentBuilder( launch_intent = IntentBuilder(
"LaunchDesktopApplicationIntent").require("LaunchKeyword").require( "LaunchDesktopApplicationIntent").require("LaunchKeyword").require(
"Application").build() "Application").build()

View File

@ -0,0 +1,3 @@
for (?P<SearchTerms>.*)
for (?P<SearchTerms>.*) on
(?P<SearchTerms>.*) on

View File

@ -44,19 +44,13 @@ class DialCallSkill(MycroftSkill):
'sean': '34567890'} # TODO - Use API 'sean': '34567890'} # TODO - Use API
def initialize(self): def initialize(self):
self.load_vocab_files(join(dirname(__file__), 'vocab', 'en-us')) self.load_vocab_files(join(dirname(__file__), 'vocab', self.lang))
self.load_regex_files(join(dirname(__file__), 'regex', self.lang))
prefixes = ['call', 'phone'] # TODO - i10n
self.__register_prefixed_regex(prefixes, "(?P<Contact>.*)")
intent = IntentBuilder("DialCallIntent").require( intent = IntentBuilder("DialCallIntent").require(
"DialCallKeyword").require("Contact").build() "DialCallKeyword").require("Contact").build()
self.register_intent(intent, self.handle_intent) self.register_intent(intent, self.handle_intent)
def __register_prefixed_regex(self, prefixes, suffix_regex):
for prefix in prefixes:
self.register_regex(prefix + ' ' + suffix_regex)
def handle_intent(self, message): def handle_intent(self, message):
try: try:
contact = message.metadata.get("Contact").lower() contact = message.metadata.get("Contact").lower()

View File

@ -0,0 +1 @@
(call|phone) (?P<Contact>.*)

View File

@ -41,20 +41,13 @@ class SendSMSSkill(MycroftSkill):
'sean': '34567890'} # TODO - Use API 'sean': '34567890'} # TODO - Use API
def initialize(self): def initialize(self):
self.load_vocab_files(join(dirname(__file__), 'vocab', 'en-us')) self.load_vocab_files(join(dirname(__file__), 'vocab', self.lang))
self.load_regex_files(join(dirname(__file__), 'regex', self.lang))
prefixes = ['tell', 'text', 'message'] # TODO - i10n
self.__register_prefixed_regex(
prefixes, "(?P<Contact>\w*) (?P<Message>.*)")
intent = IntentBuilder("SendSMSIntent").require( intent = IntentBuilder("SendSMSIntent").require(
"SendSMSKeyword").require("Contact").require("Message").build() "SendSMSKeyword").require("Contact").require("Message").build()
self.register_intent(intent, self.handle_intent) self.register_intent(intent, self.handle_intent)
def __register_prefixed_regex(self, prefixes, suffix_regex):
for prefix in prefixes:
self.register_regex(prefix + ' ' + suffix_regex)
def handle_intent(self, message): def handle_intent(self, message):
try: try:
contact = message.metadata.get("Contact").lower() contact = message.metadata.get("Contact").lower()

View File

@ -0,0 +1 @@
(tell|text|message) (?P<Contact>\w*) (?P<Message>.*)

View File

@ -0,0 +1 @@
(tell|text|message)

View File

@ -35,20 +35,13 @@ class SpellingSkill(MycroftSkill):
super(SpellingSkill, self).__init__(name="SpellingSkill") super(SpellingSkill, self).__init__(name="SpellingSkill")
def initialize(self): def initialize(self):
self.load_vocab_files(join(dirname(__file__), 'vocab', 'en-us')) self.load_vocab_files(join(dirname(__file__), 'vocab', self.lang))
self.load_regex_files(join(dirname(__file__), 'regex', self.lang))
prefixes = [
'spell', 'spell the word', 'spelling of', 'spelling of the word']
self.__register_prefixed_regex(prefixes, "(?P<Word>\w+)")
intent = IntentBuilder("SpellingIntent").require( intent = IntentBuilder("SpellingIntent").require(
"SpellingKeyword").require("Word").build() "SpellingKeyword").require("Word").build()
self.register_intent(intent, self.handle_intent) self.register_intent(intent, self.handle_intent)
def __register_prefixed_regex(self, prefixes, suffix_regex):
for prefix in prefixes:
self.register_regex(prefix + ' ' + suffix_regex)
def handle_intent(self, message): def handle_intent(self, message):
word = message.metadata.get("Word") word = message.metadata.get("Word")
self.emitter.once("recognizer_loop:audio_output_start", self.emitter.once("recognizer_loop:audio_output_start",

View File

@ -0,0 +1 @@
(spell the word|spelling of the word|spelling of|spell) (?P<Word>\w+)

View File

@ -15,7 +15,7 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Mycroft Core. If not, see <http://www.gnu.org/licenses/>. # along with Mycroft Core. If not, see <http://www.gnu.org/licenses/>.
from os.path import dirname from os.path import dirname, join
import requests import requests
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
@ -33,9 +33,7 @@ class StockSkill(MycroftSkill):
def initialize(self): def initialize(self):
self.load_data_files(dirname(__file__)) self.load_data_files(dirname(__file__))
self.load_regex_files(join(dirname(__file__), 'regex', self.lang))
self.register_regex("price of (?P<Company>.*)")
self.register_regex("is (?P<Company>.*) trading at")
stock_price_intent = IntentBuilder("StockPriceIntent")\ stock_price_intent = IntentBuilder("StockPriceIntent")\
.require("StockPriceKeyword")\ .require("StockPriceKeyword")\

View File

@ -0,0 +1,2 @@
price of (?P<Company>.*)
is (?P<Company>.*) trading at

View File

@ -18,7 +18,7 @@
import time import time
from alsaaudio import Mixer from alsaaudio import Mixer
from os.path import dirname from os.path import dirname, join
from adapt.intent import IntentBuilder from adapt.intent import IntentBuilder
from mycroft.skills.core import MycroftSkill from mycroft.skills.core import MycroftSkill
@ -39,7 +39,7 @@ class VolumeSkill(MycroftSkill):
def initialize(self): def initialize(self):
self.load_data_files(dirname(__file__)) self.load_data_files(dirname(__file__))
self.register_regex("(?P<VolumeAmount>\d+)") self.load_regex_files(join(dirname(__file__), 'regex', self.lang))
self.__build_set_volume() self.__build_set_volume()
def __build_set_volume(self): def __build_set_volume(self):

View File

@ -0,0 +1 @@
(?P<VolumeAmount>\d+)

View File

@ -17,7 +17,7 @@
from adapt.intent import IntentBuilder from adapt.intent import IntentBuilder
from os.path import dirname from os.path import dirname, join
from pyowm.exceptions.api_call_error import APICallError from pyowm.exceptions.api_call_error import APICallError
from mycroft.identity import IdentityManager from mycroft.identity import IdentityManager
@ -42,8 +42,7 @@ class WeatherSkill(MycroftSkill):
def initialize(self): def initialize(self):
self.load_data_files(dirname(__file__)) self.load_data_files(dirname(__file__))
self.register_regex("at (?P<Location>.*)") self.load_regex_files(join(dirname(__file__), 'regex', self.lang))
self.register_regex("in (?P<Location>.*)")
self.__build_current_intent() self.__build_current_intent()
self.__build_next_hour_intent() self.__build_next_hour_intent()
self.__build_next_day_intent() self.__build_next_day_intent()

View File

@ -0,0 +1 @@
(at|in) (?P<Location>.*)

View File

@ -46,20 +46,13 @@ class WikipediaSkill(MycroftSkill):
'FeedbackSearch.dialog')) 'FeedbackSearch.dialog'))
def initialize(self): def initialize(self):
self.load_vocab_files(join(dirname(__file__), 'vocab', 'en-us')) self.load_vocab_files(join(dirname(__file__), 'vocab', self.lang))
self.load_regex_files(join(dirname(__file__), 'regex', self.lang))
prefixes = ['wiki', 'wikipedia', 'tell me about', 'tell us about',
'what does wikipedia say about'] # TODO - i10n
self.__register_prefixed_regex(prefixes, "(?P<ArticleTitle>.*)")
intent = IntentBuilder("WikipediaIntent").require( intent = IntentBuilder("WikipediaIntent").require(
"WikipediaKeyword").require("ArticleTitle").build() "WikipediaKeyword").require("ArticleTitle").build()
self.register_intent(intent, self.handle_intent) self.register_intent(intent, self.handle_intent)
def __register_prefixed_regex(self, prefixes, suffix_regex):
for prefix in prefixes:
self.register_regex(prefix + ' ' + suffix_regex)
def handle_intent(self, message): def handle_intent(self, message):
try: try:
title = message.metadata.get("ArticleTitle") title = message.metadata.get("ArticleTitle")

View File

@ -0,0 +1 @@
(tell us about|tell me about|what does wikipedia say about|wiki|wikipedia) (?P<ArticleTitle>.*)

View File

@ -0,0 +1 @@
[

View File

View File

@ -0,0 +1,2 @@
(?P<MultipleTest1>.*)
(?P<MultipleTest2>.*)

View File

@ -0,0 +1 @@
(?P<SingleTest>.*)

92
test/skills/core.py Normal file
View File

@ -0,0 +1,92 @@
import unittest
from re import error
from os.path import join, dirname, abspath
from mycroft.skills.core import load_regex_from_file, load_regex
from mycroft.util.log import getLogger
__author__ = 'eward'
logger = getLogger(__name__)
class MockEmitter(object):
def __init__(self):
self.reset()
def emit(self, message):
self.types.append(message.message_type)
self.results.append(message.metadata['regex'])
def get_types(self):
return self.types
def get_results(self):
return self.results
def reset(self):
self.types = []
self.results = []
class MycroftSkillTest(unittest.TestCase):
emitter = MockEmitter()
path = abspath(join(dirname(__file__), '../regex_test'))
def check_load_from_file(self, filename, regex_list=[]):
load_regex_from_file(join(self.path, filename), self.emitter)
self.check_emitter(self.emitter, regex_list)
def check_load(self, path, regex_list=[]):
load_regex(path, self.emitter)
self.check_emitter(self.emitter, regex_list)
def check_emitter(self, emitter, regex_list):
for regex_type in emitter.get_types():
self.assertEquals(regex_type, 'register_vocab')
if not regex_list:
self.assertEquals(emitter.get_results(), regex_list)
for value in regex_list:
self.assertTrue(value in emitter.get_results())
self.emitter.reset()
def test_load_regex_from_file_single(self):
self.check_load_from_file('valid/single.rx',
['(?P<SingleTest>.*)'])
def test_load_regex_from_file_multiple(self):
self.check_load_from_file('valid/multiple.rx',
['(?P<MultipleTest1>.*)',
'(?P<MultipleTest2>.*)'])
def test_load_regex_from_file_none(self):
self.check_load_from_file('invalid/none.rx')
def test_load_regex_from_file_invalid(self):
try:
self.check_load_from_file('invalid/invalid.rx')
except error as e:
self.assertEquals(e.__str__(),
'unexpected end of regular expression')
def test_load_regex_from_file_does_not_exist(self):
try:
self.check_load_from_file('does_not_exist.rx')
except IOError as e:
self.assertEquals(e.strerror, 'No such file or directory')
def test_load_regex_full(self):
self.check_load(join(self.path, 'valid'),
['(?P<MultipleTest1>.*)',
'(?P<MultipleTest2>.*)',
'(?P<SingleTest>.*)'])
def test_load_regex_empty(self):
self.check_load(join(dirname(__file__),
'wolfram_alpha'))
def test_load_regex_fail(self):
try:
self.check_load(join(dirname(__file__),
'regex_test_fail'))
except OSError as e:
self.assertEquals(e.strerror, 'No such file or directory')