Issues 108 - Change regex to be localized (#222)

* Issues 108 - Add load_regex_files

* Issues 108 - Update weather skill with new regex syntax

* Issues 108 - Update stock skill with new regex syntax

* Issues 108 - Add stock regex file

* Issues 108 - Update time skill to use new regex syntax

* Issues 108 - Update desktop skill to use new regex syntax

* Issues 108 - Update volume skill to use new regex syntax

* Issues 108 - Update wikipedia skill to use new regex syntax

* Issues 108 - Update spelling skill to use new regex syntax

* Issues 108 - Update sms skill to use new regex syntax

* Issues 108 - Update calling skill to use new regex syntax

* Issues 108 - Remove unused argument

* Issues 108 - Add unit tests

* Issues 108 - Minor changes to fix tests

* Issues 108 - Preserve intended test logic

* Issues 108 - Address feedback

* Issues 108 - Update test formatting

* Issues 108 - Change file location

mycroft/skills/core.py

@@ -60,12 +60,29 @@ def load_vocab_from_file(path, vocab_type, emitter):
                                       'alias_of': entity}))
 
 
+def load_regex_from_file(path, emitter):
+    if(path.endswith('.rx')):
+        with open(path, 'r') as reg_file:
+            for line in reg_file.readlines():
+                re.compile(line.strip())
+                emitter.emit(
+                    Message("register_vocab",
+                            metadata={'regex': line.strip()}))
+
+
 def load_vocabulary(basedir, emitter):
     for vocab_type in os.listdir(basedir):
         load_vocab_from_file(
             join(basedir, vocab_type), splitext(vocab_type)[0], emitter)
 
 
+def load_regex(basedir, emitter):
+    for regex_type in os.listdir(basedir):
+        if regex_type.endswith(".rx"):
+            load_regex_from_file(
+                join(basedir, regex_type), emitter)
+
+
 def create_intent_envelope(intent):
     return Message(None, metadata=intent.__dict__, context={})
 
@@ -225,6 +242,9 @@ class MycroftSkill(object):
     def load_vocab_files(self, vocab_dir):
         load_vocabulary(vocab_dir, self.emitter)
 
+    def load_regex_files(self, regex_dir):
+        load_regex(regex_dir, self.emitter)
+
     def __handle_stop(self, event):
         self.stop_time = time.time()
         self.stop()

mycroft/skills/date_time/__init__.py

@@ -37,9 +37,7 @@ class TimeSkill(MycroftSkill):
 
     def initialize(self):
         self.load_vocab_files(join(dirname(__file__), 'vocab', 'en-us'))
-
-        self.register_regex("in (?P<Location>.*)")
-        self.register_regex("at (?P<Location>.*)")
+        self.load_regex_files(join(dirname(__file__), 'regex', 'en-us'))
 
         intent = IntentBuilder("TimeIntent").require(
             "TimeKeyword").optionally("Location").build()

mycroft/skills/date_time/regex/en-us/location.rx

@@ -0,0 +1 @@
+(at|in) (?P<Location>.*)

mycroft/skills/desktop_launcher/__init__.py

@@ -16,7 +16,7 @@
 # along with Mycroft Core.  If not, see <http://www.gnu.org/licenses/>.
 
 
-import os
+from os.path import dirname, join
 import sys
 import urllib2
 import webbrowser
@@ -48,9 +48,8 @@ class DesktopLauncherSkill(MycroftSkill):
                 logger.error("Could not import gio")
                 return
 
-        vocab_dir = os.path.join(os.path.dirname(__file__), 'vocab', 'en-us')
-
-        self.load_vocab_files(vocab_dir)
+        self.load_vocab_files(join(dirname(__file__), 'vocab', self.lang))
+        self.load_regex_files(join(dirname(__file__), 'regex', self.lang))
         tokenizer = EnglishTokenizer()
 
         for app in gio.app_info_get_all():
@@ -71,10 +70,6 @@ class DesktopLauncherSkill(MycroftSkill):
                 else:
                     self.appmap[tokenized_name] = entry
 
-        self.register_regex("for (?P<SearchTerms>.*)")
-        self.register_regex("for (?P<SearchTerms>.*) on")
-        self.register_regex("(?P<SearchTerms>.*) on")
-
         launch_intent = IntentBuilder(
             "LaunchDesktopApplicationIntent").require("LaunchKeyword").require(
                 "Application").build()

mycroft/skills/desktop_launcher/regex/en-us/search.terms.rx

@@ -0,0 +1,3 @@
+for (?P<SearchTerms>.*)
+for (?P<SearchTerms>.*) on
+(?P<SearchTerms>.*) on

mycroft/skills/dial_call/__init__.py

@@ -44,19 +44,13 @@ class DialCallSkill(MycroftSkill):
             'sean': '34567890'}  # TODO - Use API
 
     def initialize(self):
-        self.load_vocab_files(join(dirname(__file__), 'vocab', 'en-us'))
-
-        prefixes = ['call', 'phone']  # TODO - i10n
-        self.__register_prefixed_regex(prefixes, "(?P<Contact>.*)")
+        self.load_vocab_files(join(dirname(__file__), 'vocab', self.lang))
+        self.load_regex_files(join(dirname(__file__), 'regex', self.lang))
 
         intent = IntentBuilder("DialCallIntent").require(
             "DialCallKeyword").require("Contact").build()
         self.register_intent(intent, self.handle_intent)
 
-    def __register_prefixed_regex(self, prefixes, suffix_regex):
-        for prefix in prefixes:
-            self.register_regex(prefix + ' ' + suffix_regex)
-
     def handle_intent(self, message):
         try:
             contact = message.metadata.get("Contact").lower()

mycroft/skills/dial_call/regex/en-us/contact.rx

@@ -0,0 +1 @@
+(call|phone) (?P<Contact>.*)

mycroft/skills/send_sms/__init__.py

@@ -41,20 +41,13 @@ class SendSMSSkill(MycroftSkill):
                          'sean': '34567890'}  # TODO - Use API
 
     def initialize(self):
-        self.load_vocab_files(join(dirname(__file__), 'vocab', 'en-us'))
-
-        prefixes = ['tell', 'text', 'message']  # TODO - i10n
-        self.__register_prefixed_regex(
-            prefixes, "(?P<Contact>\w*) (?P<Message>.*)")
+        self.load_vocab_files(join(dirname(__file__), 'vocab', self.lang))
+        self.load_regex_files(join(dirname(__file__), 'regex', self.lang))
 
         intent = IntentBuilder("SendSMSIntent").require(
             "SendSMSKeyword").require("Contact").require("Message").build()
         self.register_intent(intent, self.handle_intent)
 
-    def __register_prefixed_regex(self, prefixes, suffix_regex):
-        for prefix in prefixes:
-            self.register_regex(prefix + ' ' + suffix_regex)
-
     def handle_intent(self, message):
         try:
             contact = message.metadata.get("Contact").lower()

mycroft/skills/send_sms/regex/en-us/contact.message.rx

@@ -0,0 +1 @@
+(tell|text|message) (?P<Contact>\w*) (?P<Message>.*)

mycroft/skills/send_sms/regex/en-us/message.rx

@@ -0,0 +1 @@
+(tell|text|message)

mycroft/skills/spelling/__init__.py

@@ -35,20 +35,13 @@ class SpellingSkill(MycroftSkill):
         super(SpellingSkill, self).__init__(name="SpellingSkill")
 
     def initialize(self):
-        self.load_vocab_files(join(dirname(__file__), 'vocab', 'en-us'))
-
-        prefixes = [
-            'spell', 'spell the word', 'spelling of', 'spelling of the word']
-        self.__register_prefixed_regex(prefixes, "(?P<Word>\w+)")
+        self.load_vocab_files(join(dirname(__file__), 'vocab', self.lang))
+        self.load_regex_files(join(dirname(__file__), 'regex', self.lang))
 
         intent = IntentBuilder("SpellingIntent").require(
             "SpellingKeyword").require("Word").build()
         self.register_intent(intent, self.handle_intent)
 
-    def __register_prefixed_regex(self, prefixes, suffix_regex):
-        for prefix in prefixes:
-            self.register_regex(prefix + ' ' + suffix_regex)
-
     def handle_intent(self, message):
         word = message.metadata.get("Word")
         self.emitter.once("recognizer_loop:audio_output_start",

mycroft/skills/spelling/regex/en-us/word.rx

@@ -0,0 +1 @@
+(spell the word|spelling of the word|spelling of|spell) (?P<Word>\w+)

mycroft/skills/stock/__init__.py

@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with Mycroft Core.  If not, see <http://www.gnu.org/licenses/>.
 
-from os.path import dirname
+from os.path import dirname, join
 import requests
 import xml.etree.ElementTree as ET
 
@@ -33,9 +33,7 @@ class StockSkill(MycroftSkill):
 
     def initialize(self):
         self.load_data_files(dirname(__file__))
-
-        self.register_regex("price of (?P<Company>.*)")
-        self.register_regex("is (?P<Company>.*) trading at")
+        self.load_regex_files(join(dirname(__file__), 'regex', self.lang))
 
         stock_price_intent = IntentBuilder("StockPriceIntent")\
             .require("StockPriceKeyword")\

mycroft/skills/stock/regex/en-us/company.rx

@@ -0,0 +1,2 @@
+price of (?P<Company>.*)
+is (?P<Company>.*) trading at

mycroft/skills/volume/__init__.py

@@ -18,7 +18,7 @@
 
 import time
 from alsaaudio import Mixer
-from os.path import dirname
+from os.path import dirname, join
 
 from adapt.intent import IntentBuilder
 from mycroft.skills.core import MycroftSkill
@@ -39,7 +39,7 @@ class VolumeSkill(MycroftSkill):
 
     def initialize(self):
         self.load_data_files(dirname(__file__))
-        self.register_regex("(?P<VolumeAmount>\d+)")
+        self.load_regex_files(join(dirname(__file__), 'regex', self.lang))
         self.__build_set_volume()
 
     def __build_set_volume(self):

mycroft/skills/volume/regex/en-us/volume.amount.rx

@@ -0,0 +1 @@
+(?P<VolumeAmount>\d+)

mycroft/skills/weather/__init__.py

@@ -17,7 +17,7 @@
 
 
 from adapt.intent import IntentBuilder
-from os.path import dirname
+from os.path import dirname, join
 from pyowm.exceptions.api_call_error import APICallError
 
 from mycroft.identity import IdentityManager
@@ -42,8 +42,7 @@ class WeatherSkill(MycroftSkill):
 
     def initialize(self):
         self.load_data_files(dirname(__file__))
-        self.register_regex("at (?P<Location>.*)")
-        self.register_regex("in (?P<Location>.*)")
+        self.load_regex_files(join(dirname(__file__), 'regex', self.lang))
         self.__build_current_intent()
         self.__build_next_hour_intent()
         self.__build_next_day_intent()

mycroft/skills/weather/regex/en-us/location.rx

@@ -0,0 +1 @@
+(at|in) (?P<Location>.*)

mycroft/skills/wiki/__init__.py

@@ -46,20 +46,13 @@ class WikipediaSkill(MycroftSkill):
                  'FeedbackSearch.dialog'))
 
     def initialize(self):
-        self.load_vocab_files(join(dirname(__file__), 'vocab', 'en-us'))
-
-        prefixes = ['wiki', 'wikipedia', 'tell me about', 'tell us about',
-                    'what does wikipedia say about']  # TODO - i10n
-        self.__register_prefixed_regex(prefixes, "(?P<ArticleTitle>.*)")
+        self.load_vocab_files(join(dirname(__file__), 'vocab', self.lang))
+        self.load_regex_files(join(dirname(__file__), 'regex', self.lang))
 
         intent = IntentBuilder("WikipediaIntent").require(
             "WikipediaKeyword").require("ArticleTitle").build()
         self.register_intent(intent, self.handle_intent)
 
-    def __register_prefixed_regex(self, prefixes, suffix_regex):
-        for prefix in prefixes:
-            self.register_regex(prefix + ' ' + suffix_regex)
-
     def handle_intent(self, message):
         try:
             title = message.metadata.get("ArticleTitle")

mycroft/skills/wiki/regex/en-us/article.title.rx

@@ -0,0 +1 @@
+(tell us about|tell me about|what does wikipedia say about|wiki|wikipedia) (?P<ArticleTitle>.*)

test/regex_test/invalid/invalid.rx

@@ -0,0 +1 @@
+[

test/regex_test/valid/multiple.rx

@@ -0,0 +1,2 @@
+(?P<MultipleTest1>.*)
+(?P<MultipleTest2>.*)

test/regex_test/valid/single.rx

@@ -0,0 +1 @@
+(?P<SingleTest>.*)

test/skills/core.py

@@ -0,0 +1,92 @@
+import unittest
+from re import error
+from os.path import join, dirname, abspath
+
+from mycroft.skills.core import load_regex_from_file, load_regex
+from mycroft.util.log import getLogger
+
+__author__ = 'eward'
+logger = getLogger(__name__)
+
+
+class MockEmitter(object):
+    def __init__(self):
+        self.reset()
+
+    def emit(self, message):
+        self.types.append(message.message_type)
+        self.results.append(message.metadata['regex'])
+
+    def get_types(self):
+        return self.types
+
+    def get_results(self):
+        return self.results
+
+    def reset(self):
+        self.types = []
+        self.results = []
+
+
+class MycroftSkillTest(unittest.TestCase):
+    emitter = MockEmitter()
+    path = abspath(join(dirname(__file__), '../regex_test'))
+
+    def check_load_from_file(self, filename, regex_list=[]):
+        load_regex_from_file(join(self.path, filename), self.emitter)
+        self.check_emitter(self.emitter, regex_list)
+
+    def check_load(self, path, regex_list=[]):
+        load_regex(path, self.emitter)
+        self.check_emitter(self.emitter, regex_list)
+
+    def check_emitter(self, emitter, regex_list):
+        for regex_type in emitter.get_types():
+            self.assertEquals(regex_type, 'register_vocab')
+        if not regex_list:
+            self.assertEquals(emitter.get_results(), regex_list)
+        for value in regex_list:
+            self.assertTrue(value in emitter.get_results())
+        self.emitter.reset()
+
+    def test_load_regex_from_file_single(self):
+        self.check_load_from_file('valid/single.rx',
+                                  ['(?P<SingleTest>.*)'])
+
+    def test_load_regex_from_file_multiple(self):
+        self.check_load_from_file('valid/multiple.rx',
+                                  ['(?P<MultipleTest1>.*)',
+                                   '(?P<MultipleTest2>.*)'])
+
+    def test_load_regex_from_file_none(self):
+        self.check_load_from_file('invalid/none.rx')
+
+    def test_load_regex_from_file_invalid(self):
+        try:
+            self.check_load_from_file('invalid/invalid.rx')
+        except error as e:
+            self.assertEquals(e.__str__(),
+                              'unexpected end of regular expression')
+
+    def test_load_regex_from_file_does_not_exist(self):
+        try:
+            self.check_load_from_file('does_not_exist.rx')
+        except IOError as e:
+            self.assertEquals(e.strerror, 'No such file or directory')
+
+    def test_load_regex_full(self):
+        self.check_load(join(self.path, 'valid'),
+                        ['(?P<MultipleTest1>.*)',
+                         '(?P<MultipleTest2>.*)',
+                         '(?P<SingleTest>.*)'])
+
+    def test_load_regex_empty(self):
+        self.check_load(join(dirname(__file__),
+                             'wolfram_alpha'))
+
+    def test_load_regex_fail(self):
+        try:
+            self.check_load(join(dirname(__file__),
+                                 'regex_test_fail'))
+        except OSError as e:
+            self.assertEquals(e.strerror, 'No such file or directory')