MycroftAI
/
mycroft-core
mirror of https://github.com/MycroftAI/mycroft-core.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Added message tester utility. Reduced line length to <80 chars in general

pull/1527/head
CarstenAgerskov 2018-02-25 19:12:00 +01:00 committed by Matthew D. Scholefield
parent a16c2a0ecc
commit 6c226ea4d9
3 changed files with 128 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
test/integrationtests/skills/message_tester.py Executable file
View File

@ -0,0 +1,87 @@
#!/usr/bin/env python
# Copyright 2017 Mycroft AI Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from Tkinter import *
import ScrolledText
import skill_tester
import ast
EXAMPLE_EVENT = "{'expect_response': False, \
'utterance': u'Recording audio for 600 seconds'}"
EXAMPLE_TEST_CASE = '{ \n\
"utterance": "record", \n\
"intent_type": "AudioRecordSkillIntent", \n\
"intent": { \n\
"AudioRecordSkillKeyword": "record" \n\
}, \n\
"expected_response": ".*(recording|audio)" \n\
}'
class MessageTester:
"""
This message tester lets a user input a Message event and a json test
case, and allows the evaluation of the Message event based on the
test case
It is supposed to be run in the Mycroft virtualenv, and python-tk
must be installed (on Ubuntu: apt-get install python-tk)
"""
def __init__(self, root):
root.title("Message tester")
Label(root, text="Enter message event below", bg="light green").pack()
self.event_field = ScrolledText.ScrolledText(root,
width=180, height=10)
self.event_field.pack()
Label(root, text="Enter test case below", bg="light green").pack()
self.test_case_field = ScrolledText.ScrolledText(root,
width=180, height=20)
self.test_case_field.pack()
Label(root, text="Test result:", bg="light green").pack()
self.result_field = ScrolledText.ScrolledText(root,
width=180, height=10)
self.result_field.pack()
self.result_field.config(state=DISABLED)
self.button = Button(root, text="Evaluate", fg="red",
command=self.clicked)
self.button.pack()
self.event_field.delete('1.0', END)
self.event_field.insert('insert', EXAMPLE_EVENT)
self.test_case_field.delete('1.0', END)
self.test_case_field.insert('insert', EXAMPLE_TEST_CASE)
def clicked(self):
event = self.event_field.get('1.0', END)
test_case = self.test_case_field.get('1.0', END)
evaluation = skill_tester.EvaluationRule(ast.literal_eval(test_case))
evaluation.evaluate(ast.literal_eval(event))
self.result_field.config(state=NORMAL)
self.result_field.delete('1.0', END)
self.result_field.insert('insert', evaluation.rule)
self.result_field.config(state=DISABLED)
r = Tk()
app = MessageTester(r)
r.mainloop()

15
test/integrationtests/skills/skill_developers_testrunner.py
View File

@ -13,7 +13,6 @@
# limitations under the License.
#
import glob
import sys
import unittest
import os
@ -23,6 +22,7 @@ from test.integrationtests.skills.skill_tester import SkillTest
HOME_DIR = os.path.dirname(os.path.abspath(__file__))
def discover_tests():
"""
Find test files starting from the directory where this file resides.
@ -54,6 +54,7 @@ class IntentTestSequenceMeta(type):
def gen_test(a, b):
def test(self):
SkillTest(a, b, self.emitter).run(self.loader)
return test
tests = discover_tests()
@ -80,17 +81,17 @@ class IntentTestSequence(unittest.TestCase):
"""
__metaclass__ = IntentTestSequenceMeta
loader = None
@classmethod
def setUpClass(self):
self.loader = MockSkillsLoader(HOME_DIR)
self.emitter = self.loader.load_skills()
def setUpClass(cls):
cls.loader = MockSkillsLoader(HOME_DIR)
cls.emitter = cls.loader.load_skills()
@classmethod
def tearDownClass(self):
self.loader.unload_skills()
def tearDownClass(cls):
cls.loader.unload_skills()
if __name__ == '__main__':
unittest.main()

56
test/integrationtests/skills/skill_tester.py
View File

@ -24,7 +24,8 @@ from os.path import join, isdir
from pyee import EventEmitter
from mycroft.messagebus.message import Message
from mycroft.skills.core import create_skill_descriptor, load_skill, MycroftSkill
from mycroft.skills.core import create_skill_descriptor, load_skill, \
MycroftSkill
MainModule = '__init__'
@ -38,7 +39,8 @@ def get_skills(skills_folder):
is discovered
Args:
skills_folder: Folder to start a search for skills __init__.py files
skills_folder: Folder to start a search for skills __init__.py
files
"""
skills = []
@ -85,8 +87,8 @@ def unload_skills(skills):
class InterceptEmitter(object):
"""
This class intercepts and allows emitting events between the skill_tester and
the skill being tested.
This class intercepts and allows emitting events between the
skill_tester and the skill being tested.
When a test is running emitted communication is intercepted for analysis
"""
@ -136,8 +138,8 @@ class MockSkillsLoader(object):
class SkillTest(object):
"""
This class is instantiated for each skill being tested. It holds the data
needed for the test, and contains the methods doing the test
This class is instantiated for each skill being tested. It holds the
data needed for the test, and contains the methods doing the test
"""
@ -172,7 +174,8 @@ class SkillTest(object):
s.emitter.q = q
# Set up context before calling intent
# This option makes it possible to better isolate (reduce dependance) between test_cases
# This option makes it possible to better isolate (reduce dependance)
# between test_cases
cxt = test_case.get('remove_context', None)
if cxt:
if isinstance(cxt, list):
@ -196,7 +199,8 @@ class SkillTest(object):
# Wait up to X seconds for the test_case to complete
timeout = time.time() + int(test_case.get('evaluation_timeout', None)) \
if test_case.get('evaluation_timeout', None) and isinstance(test_case['evaluation_timeout'], int) \
if test_case.get('evaluation_timeout', None) and \
isinstance(test_case['evaluation_timeout'], int) \
else time.time() + DEFAULT_EVALUAITON_TIMEOUT
while not evaluation_rule.all_succeeded():
try:
@ -222,19 +226,20 @@ class SkillTest(object):
assert False
# TODO: Add command line utility to test an event against a test_case, allow for debugging tests
# TODO: Add command line utility to test an event against a test_case, allow
# for debugging tests
class EvaluationRule(object):
"""
This class initially convert the test_case json file to internal rule format, which is
stored throughout the testcase run. All Messages on the event bus can be evaluated against the
rules (test_case)
This class initially convert the test_case json file to internal rule
format, which is stored throughout the testcase run. All Messages on
the event bus can be evaluated against the rules (test_case)
This approach makes it easier to add new tests, since Message and rule traversal is already
set up for the internal rule format.
The test writer can use the internal rule format directly in the test_case
using the assert keyword, which allows for more powerfull/individual
test cases than the standard dictionaly
This approach makes it easier to add new tests, since Message and rule
traversal is already set up for the internal rule format.
The test writer can use the internal rule format directly in the
test_case using the assert keyword, which allows for more
powerfull/individual test cases than the standard dictionaly
"""
def __init__(self, test_case):
@ -248,7 +253,8 @@ class EvaluationRule(object):
_x = ['and']
if test_case.get('utterance', None):
_x.append(['endsWith', 'intent_type', str(test_case['intent_type'])])
_x.append(['endsWith', 'intent_type',
str(test_case['intent_type'])])
if test_case.get('intent', None):
for item in test_case['intent'].items():
@ -258,7 +264,8 @@ class EvaluationRule(object):
self.rule.append(_x)
if test_case.get('expected_response', None):
self.rule.append(['match', 'utterance', str(test_case['expected_response'])])
self.rule.append(['match', 'utterance',
str(test_case['expected_response'])])
if test_case.get('changed_context', None):
ctx = test_case['changed_context']
@ -304,7 +311,8 @@ class EvaluationRule(object):
def _partial_evaluate(self, rule, msg):
"""
Evaluate the message against a part of the rules (recursive over rules)
Evaluate the message against a part of the rules (recursive over
rules)
Args:
rule: A rule or a part of the rules to be broken down further
@ -323,11 +331,13 @@ class EvaluationRule(object):
return False
if rule[0] == 'endsWith':
if not (self._get_field_value(rule[1], msg) and self._get_field_value(rule[1], msg).endswith(rule[2])):
if not (self._get_field_value(rule[1], msg) and
self._get_field_value(rule[1], msg).endswith(rule[2])):
return False
if rule[0] == 'match':
if not (self._get_field_value(rule[1], msg) and re.match(rule[2], self._get_field_value(rule[1], msg))):
if not (self._get_field_value(rule[1], msg) and
re.match(rule[2], self._get_field_value(rule[1], msg))):
return False
if rule[0] == 'and':
@ -350,7 +360,7 @@ class EvaluationRule(object):
Test if all rules succeeded
Returns:
bool: True is all rules succeeded
bool: True if all rules succeeded
"""
# return len(filter(lambda x: x[-1] != 'succeeded', self.rule)) == 0
return len([x for x in self.rule if x[-1] != 'succeeded']) == 0