From 03ef7f4b219533507f016c64a45487bc827ec78c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85ke=20Forslund?= Date: Tue, 18 Jan 2022 13:12:30 +0100 Subject: [PATCH 1/4] VK: Include locale folders in dialog_from_sentence Only dialog folder was used previously this adds globbing through the locale/lang/ folder and it's subfolders --- .../features/steps/utterance_responses.py | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/test/integrationtests/voight_kampff/features/steps/utterance_responses.py b/test/integrationtests/voight_kampff/features/steps/utterance_responses.py index e7422712b2..30c051765d 100644 --- a/test/integrationtests/voight_kampff/features/steps/utterance_responses.py +++ b/test/integrationtests/voight_kampff/features/steps/utterance_responses.py @@ -17,7 +17,7 @@ Predefined step definitions for handling dialog interaction with Mycroft for use with behave. """ from os.path import join, exists, basename -from glob import glob +from pathlib import Path import re import time @@ -65,6 +65,26 @@ def load_dialog_list(skill_path, dialog): return load_dialog_file(dialog_path), debug +def _get_dialog_files(skill_path, lang): + """Generator expression returning all dialog files. + + This includes both the 'locale' and the older style 'dialog' folder. + + Args: + skill_path (str): skill root folder + lang (str): language code to check + + yields: + (Path) path of each found dialog file + """ + in_dialog_dir = Path(skill_path, 'dialog', lang).rglob('*.dialog') + for dialog_path in in_dialog_dir: + yield dialog_path + in_locale_dir = Path(skill_path, 'locale', lang).rglob('*.dialog') + for dialog_path in in_locale_dir: + yield dialog_path + + def dialog_from_sentence(sentence, skill_path, lang): """Find dialog file from example sentence. @@ -75,9 +95,8 @@ def dialog_from_sentence(sentence, skill_path, lang): Returns (str): Dialog file best matching the sentence. """ - dialog_paths = join(skill_path, 'dialog', lang, '*.dialog') best = (None, 0) - for path in glob(dialog_paths): + for path in _get_dialog_files(skill_path, lang): patterns = load_dialog_file(path) match, _ = _match_dialog_patterns(patterns, sentence.lower()) if match is not False: From 825b3879a51c15513e069e61872266bfa55ecd99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85ke=20Forslund?= Date: Tue, 18 Jan 2022 13:13:50 +0100 Subject: [PATCH 2/4] VK: Fix regex used in _match_dialog_patterns A redundant step caused issue when performing tests in the tv-remove-control-skill --- .../voight_kampff/features/steps/utterance_responses.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/integrationtests/voight_kampff/features/steps/utterance_responses.py b/test/integrationtests/voight_kampff/features/steps/utterance_responses.py index 30c051765d..7cb90c0afd 100644 --- a/test/integrationtests/voight_kampff/features/steps/utterance_responses.py +++ b/test/integrationtests/voight_kampff/features/steps/utterance_responses.py @@ -117,7 +117,6 @@ def _match_dialog_patterns(dialogs, sentence): dialogs = [re.sub(r'{.*?\}', r'.*', dia) for dia in dialogs] # Remove left over '}' dialogs = [re.sub(r'\}', r'', dia) for dia in dialogs] - dialogs = [re.sub(r' .* ', r' .*', dia) for dia in dialogs] # Merge consequtive .*'s into a single .* dialogs = [re.sub(r'\.\*( \.\*)+', r'.*', dia) for dia in dialogs] # Remove double whitespaces From a3fd830cb0ae3deef01d0e4d3cfc4512fd55f269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85ke=20Forslund?= Date: Wed, 26 Jan 2022 12:55:36 +0100 Subject: [PATCH 3/4] VK: Expand the parentheses from dialog files --- .../features/steps/utterance_responses.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/integrationtests/voight_kampff/features/steps/utterance_responses.py b/test/integrationtests/voight_kampff/features/steps/utterance_responses.py index 7cb90c0afd..ca4d6ed95d 100644 --- a/test/integrationtests/voight_kampff/features/steps/utterance_responses.py +++ b/test/integrationtests/voight_kampff/features/steps/utterance_responses.py @@ -25,6 +25,7 @@ from behave import given, when, then from mycroft.messagebus import Message from mycroft.audio import wait_while_speaking +from mycroft.util.format import expand_options from test.integrationtests.voight_kampff import (mycroft_responses, then_wait, then_wait_fail) @@ -45,8 +46,14 @@ def load_dialog_file(dialog_path): """Load dialog files and get the contents.""" with open(dialog_path) as f: lines = f.readlines() - return [l.strip().lower() for l in lines - if l.strip() != '' and l.strip()[0] != '#'] + + # Expand parentheses in lines + expanded_lines = [] + for line in lines: + expanded_lines += expand_options(line) + + return [line.strip().lower() for line in expanded_lines + if line.strip() != '' and line.strip()[0] != '#'] def load_dialog_list(skill_path, dialog): From f8aa77c8e7ccdb39bafab9bdf12e86b07dbb2310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85ke=20Forslund?= Date: Wed, 26 Jan 2022 21:01:46 +0100 Subject: [PATCH 4/4] VK simplify dialog matching This uses the existing dialog renderer and the standard format library to in two steps create a regex where the {elements} in a dialog is replaced with ".*" to match the given sentence. --- .../features/steps/utterance_responses.py | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/test/integrationtests/voight_kampff/features/steps/utterance_responses.py b/test/integrationtests/voight_kampff/features/steps/utterance_responses.py index ca4d6ed95d..6277d9922b 100644 --- a/test/integrationtests/voight_kampff/features/steps/utterance_responses.py +++ b/test/integrationtests/voight_kampff/features/steps/utterance_responses.py @@ -19,10 +19,12 @@ use with behave. from os.path import join, exists, basename from pathlib import Path import re +from string import Formatter import time from behave import given, when, then +from mycroft.dialog import MustacheDialogRenderer from mycroft.messagebus import Message from mycroft.audio import wait_while_speaking from mycroft.util.format import expand_options @@ -44,14 +46,13 @@ def find_dialog(skill_path, dialog, lang): def load_dialog_file(dialog_path): """Load dialog files and get the contents.""" - with open(dialog_path) as f: - lines = f.readlines() - - # Expand parentheses in lines + renderer = MustacheDialogRenderer() + renderer.load_template_file('template', dialog_path) expanded_lines = [] - for line in lines: - expanded_lines += expand_options(line) - + for template in renderer.templates: + # Expand parentheses in lines + for line in renderer.templates[template]: + expanded_lines += expand_options(line) return [line.strip().lower() for line in expanded_lines if line.strip() != '' and line.strip()[0] != '#'] @@ -118,18 +119,25 @@ def dialog_from_sentence(sentence, skill_path, lang): def _match_dialog_patterns(dialogs, sentence): """Match sentence against a list of dialog patterns. - Returns index of found match. + dialogs (list of str): dialog file entries to match against + sentence (str): string to match. + + Returns: + (tup) index of found match, debug text """ # Allow custom fields to be anything - dialogs = [re.sub(r'{.*?\}', r'.*', dia) for dia in dialogs] - # Remove left over '}' - dialogs = [re.sub(r'\}', r'', dia) for dia in dialogs] - # Merge consequtive .*'s into a single .* - dialogs = [re.sub(r'\.\*( \.\*)+', r'.*', dia) for dia in dialogs] - # Remove double whitespaces - dialogs = ['^' + ' '.join(dia.split()) for dia in dialogs] + # i.e {field} gets turned into ".*" + regexes = [] + for dialog in dialogs: + data = {element[1]: '.*' + for element in Formatter().parse(dialog)} + regexes.append(dialog.format(**data)) + + # Remove double whitespaces and ensure that it matches from + # the beginning of the line. + regexes = ['^' + ' '.join(reg.split()) for reg in regexes] debug = 'MATCHING: {}\n'.format(sentence) - for index, regex in enumerate(dialogs): + for index, regex in enumerate(regexes): match = re.match(regex, sentence) debug += '---------------\n' debug += '{} {}\n'.format(regex, match is not None)