Home-Assistant
/
core
mirror of https://github.com/home-assistant/core.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix intent loading and incorporate unmatched entities more (#108423) 

* Incorporate unmatched entities more

* Don't list targets when match is incomplete

* Add test for out of range
pull/108750/head
Michael Hansen 2024-01-23 19:31:57 -06:00 committed by GitHub
parent c725238c20
commit d8a1c58b12
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 382 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
homeassistant/components/conversation/__init__.py
View File

@ -9,7 +9,12 @@ import re
from typing import Any, Literal
from aiohttp import web
from hassil.recognize import RecognizeResult
from hassil.recognize import (
MISSING_ENTITY,
RecognizeResult,
UnmatchedRangeEntity,
UnmatchedTextEntity,
)
import voluptuous as vol
from homeassistant import core
@ -317,37 +322,55 @@ async def websocket_hass_agent_debug(
]
# Return results for each sentence in the same order as the input.
connection.send_result(
msg["id"],
{
"results": [
{
"intent": {
"name": result.intent.name,
},
"slots": { # direct access to values
entity_key: entity.value
for entity_key, entity in result.entities.items()
},
"details": {
entity_key: {
"name": entity.name,
"value": entity.value,
"text": entity.text,
}
for entity_key, entity in result.entities.items()
},
"targets": {
state.entity_id: {"matched": is_matched}
for state, is_matched in _get_debug_targets(hass, result)
},
result_dicts: list[dict[str, Any] | None] = []
for result in results:
if result is None:
# Indicate that a recognition failure occurred
result_dicts.append(None)
continue
successful_match = not result.unmatched_entities
result_dict = {
# Name of the matching intent (or the closest)
"intent": {
"name": result.intent.name,
},
# Slot values that would be received by the intent
"slots": { # direct access to values
entity_key: entity.value
for entity_key, entity in result.entities.items()
},
# Extra slot details, such as the originally matched text
"details": {
entity_key: {
"name": entity.name,
"value": entity.value,
"text": entity.text,
}
if result is not None
else None
for result in results
]
},
)
for entity_key, entity in result.entities.items()
},
# Entities/areas/etc. that would be targeted
"targets": {},
# True if match was successful
"match": successful_match,
# Text of the sentence template that matched (or was closest)
"sentence_template": "",
# When match is incomplete, this will contain the best slot guesses
"unmatched_slots": _get_unmatched_slots(result),
}
if successful_match:
result_dict["targets"] = {
state.entity_id: {"matched": is_matched}
for state, is_matched in _get_debug_targets(hass, result)
}
if result.intent_sentence is not None:
result_dict["sentence_template"] = result.intent_sentence.text
result_dicts.append(result_dict)
connection.send_result(msg["id"], {"results": result_dicts})
def _get_debug_targets(
@ -393,6 +416,25 @@ def _get_debug_targets(
yield state, is_matched
def _get_unmatched_slots(
result: RecognizeResult,
) -> dict[str, str | int]:
"""Return a dict of unmatched text/range slot entities."""
unmatched_slots: dict[str, str | int] = {}
for entity in result.unmatched_entities_list:
if isinstance(entity, UnmatchedTextEntity):
if entity.text == MISSING_ENTITY:
# Don't report <missing> since these are just missing context
# slots.
continue
unmatched_slots[entity.name] = entity.text
elif isinstance(entity, UnmatchedRangeEntity):
unmatched_slots[entity.name] = entity.value
return unmatched_slots
class ConversationProcessView(http.HomeAssistantView):
"""View to process text."""

161
homeassistant/components/conversation/default_agent.py
View File

@ -6,6 +6,7 @@ from collections import defaultdict
from collections.abc import Awaitable, Callable, Iterable
from dataclasses import dataclass
import functools
import itertools
import logging
from pathlib import Path
import re
@ -20,6 +21,7 @@ from hassil.intents import (
WildcardSlotList,
)
from hassil.recognize import (
MISSING_ENTITY,
RecognizeResult,
UnmatchedEntity,
UnmatchedTextEntity,
@ -75,7 +77,7 @@ class LanguageIntents:
intents_dict: dict[str, Any]
intent_responses: dict[str, Any]
error_responses: dict[str, Any]
loaded_components: set[str]
language_variant: str | None
@dataclass(slots=True)
@ -181,9 +183,7 @@ class DefaultAgent(AbstractConversationAgent):
lang_intents = self._lang_intents.get(language)
# Reload intents if missing or new components
if lang_intents is None or (
lang_intents.loaded_components - self.hass.config.components
):
if lang_intents is None:
# Load intents in executor
lang_intents = await self.async_get_or_load_intents(language)
@ -357,6 +357,13 @@ class DefaultAgent(AbstractConversationAgent):
intent_context=intent_context,
allow_unmatched_entities=True,
):
# Remove missing entities that couldn't be filled from context
for entity_key, entity in list(result.unmatched_entities.items()):
if isinstance(entity, UnmatchedTextEntity) and (
entity.text == MISSING_ENTITY
):
result.unmatched_entities.pop(entity_key)
if maybe_result is None:
# First result
maybe_result = result
@ -364,8 +371,11 @@ class DefaultAgent(AbstractConversationAgent):
# Fewer unmatched entities
maybe_result = result
elif len(result.unmatched_entities) == len(maybe_result.unmatched_entities):
if result.text_chunks_matched > maybe_result.text_chunks_matched:
# More literal text chunks matched
if (result.text_chunks_matched > maybe_result.text_chunks_matched) or (
(result.text_chunks_matched == maybe_result.text_chunks_matched)
and ("name" in result.unmatched_entities) # prefer entities
):
# More literal text chunks matched, but prefer entities to areas, etc.
maybe_result = result
if (maybe_result is not None) and maybe_result.unmatched_entities:
@ -484,84 +494,93 @@ class DefaultAgent(AbstractConversationAgent):
if lang_intents is None:
intents_dict: dict[str, Any] = {}
loaded_components: set[str] = set()
language_variant: str | None = None
else:
intents_dict = lang_intents.intents_dict
loaded_components = lang_intents.loaded_components
language_variant = lang_intents.language_variant
# en-US, en_US, en, ...
language_variations = list(_get_language_variations(language))
domains_langs = get_domains_and_languages()
# Check if any new components have been loaded
intents_changed = False
for component in hass_components:
if component in loaded_components:
continue
if not language_variant:
# Choose a language variant upfront and commit to it for custom
# sentences, etc.
all_language_variants = {
lang.lower(): lang for lang in itertools.chain(*domains_langs.values())
}
# Don't check component again
loaded_components.add(component)
# Check for intents for this component with the target language.
# Try en-US, en, etc.
for language_variation in language_variations:
component_intents = get_intents(
component, language_variation, json_load=json_load
)
if component_intents:
# Merge sentences into existing dictionary
merge_dict(intents_dict, component_intents)
# Will need to recreate graph
intents_changed = True
_LOGGER.debug(
"Loaded intents component=%s, language=%s (%s)",
component,
language,
language_variation,
)
# en-US, en_US, en, ...
for maybe_variant in _get_language_variations(language):
matching_variant = all_language_variants.get(maybe_variant.lower())
if matching_variant:
language_variant = matching_variant
break
if not language_variant:
_LOGGER.warning(
"Unable to find supported language variant for %s", language
)
return None
# Load intents for all domains supported by this language variant
for domain in domains_langs:
domain_intents = get_intents(
domain, language_variant, json_load=json_load
)
if not domain_intents:
continue
# Merge sentences into existing dictionary
merge_dict(intents_dict, domain_intents)
# Will need to recreate graph
intents_changed = True
_LOGGER.debug(
"Loaded intents domain=%s, language=%s (%s)",
domain,
language,
language_variant,
)
# Check for custom sentences in <config>/custom_sentences/<language>/
if lang_intents is None:
# Only load custom sentences once, otherwise they will be re-loaded
# when components change.
for language_variation in language_variations:
custom_sentences_dir = Path(
self.hass.config.path("custom_sentences", language_variation)
)
if custom_sentences_dir.is_dir():
for custom_sentences_path in custom_sentences_dir.rglob("*.yaml"):
with custom_sentences_path.open(
encoding="utf-8"
) as custom_sentences_file:
# Merge custom sentences
if isinstance(
custom_sentences_yaml := yaml.safe_load(
custom_sentences_file
),
dict,
):
merge_dict(intents_dict, custom_sentences_yaml)
else:
_LOGGER.warning(
"Custom sentences file does not match expected format path=%s",
custom_sentences_file.name,
)
custom_sentences_dir = Path(
self.hass.config.path("custom_sentences", language_variant)
)
if custom_sentences_dir.is_dir():
for custom_sentences_path in custom_sentences_dir.rglob("*.yaml"):
with custom_sentences_path.open(
encoding="utf-8"
) as custom_sentences_file:
# Merge custom sentences
if isinstance(
custom_sentences_yaml := yaml.safe_load(
custom_sentences_file
),
dict,
):
merge_dict(intents_dict, custom_sentences_yaml)
else:
_LOGGER.warning(
"Custom sentences file does not match expected format path=%s",
custom_sentences_file.name,
)
# Will need to recreate graph
intents_changed = True
_LOGGER.debug(
"Loaded custom sentences language=%s (%s), path=%s",
language,
language_variation,
custom_sentences_path,
)
# Stop after first matched language variation
break
# Will need to recreate graph
intents_changed = True
_LOGGER.debug(
"Loaded custom sentences language=%s (%s), path=%s",
language,
language_variant,
custom_sentences_path,
)
# Load sentences from HA config for default language only
if self._config_intents and (language == self.hass.config.language):
if self._config_intents and (
self.hass.config.language in (language, language_variant)
):
merge_dict(
intents_dict,
{
@ -598,7 +617,7 @@ class DefaultAgent(AbstractConversationAgent):
intents_dict,
intent_responses,
error_responses,
loaded_components,
language_variant,
)
self._lang_intents[language] = lang_intents
else:

2
homeassistant/components/conversation/manifest.json
View File

@ -7,5 +7,5 @@
"integration_type": "system",
"iot_class": "local_push",
"quality_scale": "internal",
"requirements": ["hassil==1.5.2", "home-assistant-intents==2024.1.2"]
"requirements": ["hassil==1.5.3", "home-assistant-intents==2024.1.2"]
}

2
homeassistant/package_constraints.txt
View File

@ -26,7 +26,7 @@ ha-av==10.1.1
ha-ffmpeg==3.1.0
habluetooth==2.4.0
hass-nabucasa==0.75.1
hassil==1.5.2
hassil==1.5.3
home-assistant-bluetooth==1.12.0
home-assistant-frontend==20240112.0
home-assistant-intents==2024.1.2

2
requirements_all.txt
View File

@ -1019,7 +1019,7 @@ hass-nabucasa==0.75.1
hass-splunk==0.1.1
# homeassistant.components.conversation
hassil==1.5.2
hassil==1.5.3
# homeassistant.components.jewish_calendar
hdate==0.10.4

2
requirements_test_all.txt
View File

@ -821,7 +821,7 @@ habluetooth==2.4.0
hass-nabucasa==0.75.1
# homeassistant.components.conversation
hassil==1.5.2
hassil==1.5.3
# homeassistant.components.jewish_calendar
hdate==0.10.4

108
tests/components/conversation/snapshots/test_init.ambr
View File

@ -539,7 +539,7 @@
'speech': dict({
'plain': dict({
'extra_data': None,
'speech': 'No area named kitchen',
'speech': 'No device or entity named kitchen light',
}),
}),
}),
@ -679,7 +679,7 @@
'speech': dict({
'plain': dict({
'extra_data': None,
'speech': 'No area named late added',
'speech': 'No device or entity named late added light',
}),
}),
}),
@ -759,7 +759,7 @@
'speech': dict({
'plain': dict({
'extra_data': None,
'speech': 'No area named kitchen',
'speech': 'No device or entity named kitchen light',
}),
}),
}),
@ -779,7 +779,7 @@
'speech': dict({
'plain': dict({
'extra_data': None,
'speech': 'No area named my cool',
'speech': 'No device or entity named my cool light',
}),
}),
}),
@ -919,7 +919,7 @@
'speech': dict({
'plain': dict({
'extra_data': None,
'speech': 'No area named kitchen',
'speech': 'No device or entity named kitchen light',
}),
}),
}),
@ -969,7 +969,7 @@
'speech': dict({
'plain': dict({
'extra_data': None,
'speech': 'No area named renamed',
'speech': 'No device or entity named renamed light',
}),
}),
}),
@ -1403,6 +1403,8 @@
'intent': dict({
'name': 'HassTurnOn',
}),
'match': True,
'sentence_template': '<turn> on (<area> <name>|<name> [in <area>])',
'slots': dict({
'name': 'my cool light',
}),
@ -1411,6 +1413,8 @@
'matched': True,
}),
}),
'unmatched_slots': dict({
}),
}),
dict({
'details': dict({
@ -1423,6 +1427,8 @@
'intent': dict({
'name': 'HassTurnOff',
}),
'match': True,
'sentence_template': '[<turn>] (<area> <name>|<name> [in <area>]) [to] off',
'slots': dict({
'name': 'my cool light',
}),
@ -1431,6 +1437,8 @@
'matched': True,
}),
}),
'unmatched_slots': dict({
}),
}),
dict({
'details': dict({
@ -1448,6 +1456,8 @@
'intent': dict({
'name': 'HassTurnOn',
}),
'match': True,
'sentence_template': '<turn> on [all] <light> in <area>',
'slots': dict({
'area': 'kitchen',
'domain': 'light',
@ -1457,6 +1467,8 @@
'matched': True,
}),
}),
'unmatched_slots': dict({
}),
}),
dict({
'details': dict({
@ -1479,6 +1491,8 @@
'intent': dict({
'name': 'HassGetState',
}),
'match': True,
'sentence_template': '[tell me] how many {on_off_domains:domain} (is|are) {on_off_states:state} [in <area>]',
'slots': dict({
'area': 'kitchen',
'domain': 'light',
@ -1489,23 +1503,30 @@
'matched': False,
}),
}),
'unmatched_slots': dict({
}),
}),
dict({
'details': dict({
'domain': dict({
'name': 'domain',
'text': '',
'value': 'script',
'value': 'scene',
}),
}),
'intent': dict({
'name': 'HassTurnOn',
}),
'match': False,
'sentence_template': '[activate|<turn>] <name> [scene] [on]',
'slots': dict({
'domain': 'script',
'domain': 'scene',
}),
'targets': dict({
}),
'unmatched_slots': dict({
'name': 'this will not match anything',
}),
}),
]),
})
@ -1519,3 +1540,74 @@
}),
})
# ---
# name: test_ws_hass_agent_debug_null_result
dict({
'results': list([
None,
]),
})
# ---
# name: test_ws_hass_agent_debug_out_of_range
dict({
'results': list([
dict({
'details': dict({
'brightness': dict({
'name': 'brightness',
'text': '100%',
'value': 100,
}),
'name': dict({
'name': 'name',
'text': 'test light',
'value': 'test light',
}),
}),
'intent': dict({
'name': 'HassLightSet',
}),
'match': True,
'sentence_template': '[<numeric_value_set>] <name> brightness [to] <brightness>',
'slots': dict({
'brightness': 100,
'name': 'test light',
}),
'targets': dict({
'light.demo_1234': dict({
'matched': True,
}),
}),
'unmatched_slots': dict({
}),
}),
]),
})
# ---
# name: test_ws_hass_agent_debug_out_of_range.1
dict({
'results': list([
dict({
'details': dict({
'name': dict({
'name': 'name',
'text': 'test light',
'value': 'test light',
}),
}),
'intent': dict({
'name': 'HassLightSet',
}),
'match': False,
'sentence_template': '[<numeric_value_set>] <name> brightness [to] <brightness>',
'slots': dict({
'name': 'test light',
}),
'targets': dict({
}),
'unmatched_slots': dict({
'brightness': 1001,
}),
}),
]),
})
# ---

21
tests/components/conversation/test_default_agent.py
View File

@ -577,3 +577,24 @@ async def test_empty_aliases(
names = slot_lists["name"]
assert len(names.values) == 1
assert names.values[0].value_out == "kitchen light"
async def test_all_domains_loaded(
hass: HomeAssistant, init_components, area_registry: ar.AreaRegistry
) -> None:
"""Test that sentences for all domains are always loaded."""
# light domain is not loaded
assert "light" not in hass.config.components
result = await conversation.async_converse(
hass, "set brightness of test light to 100%", None, Context(), None
)
# Invalid target vs. no intent recognized
assert result.response.response_type == intent.IntentResponseType.ERROR
assert result.response.error_code == intent.IntentResponseErrorCode.NO_VALID_TARGETS
assert (
result.response.speech["plain"]["speech"]
== "No device or entity named test light"
)

121
tests/components/conversation/test_init.py
View File

@ -913,32 +913,6 @@ async def test_language_region(hass: HomeAssistant, init_components) -> None:
assert call.data == {"entity_id": ["light.kitchen"]}
async def test_reload_on_new_component(hass: HomeAssistant) -> None:
"""Test intents being reloaded when a new component is loaded."""
language = hass.config.language
assert await async_setup_component(hass, "homeassistant", {})
assert await async_setup_component(hass, "conversation", {})
# Load intents
agent = await conversation._get_agent_manager(hass).async_get_agent()
assert isinstance(agent, conversation.DefaultAgent)
await agent.async_prepare()
lang_intents = agent._lang_intents.get(language)
assert lang_intents is not None
loaded_components = set(lang_intents.loaded_components)
# Load another component
assert await async_setup_component(hass, "light", {})
# Intents should reload
await agent.async_prepare()
lang_intents = agent._lang_intents.get(language)
assert lang_intents is not None
assert {"light"} == (lang_intents.loaded_components - loaded_components)
async def test_non_default_response(hass: HomeAssistant, init_components) -> None:
"""Test intent response that is not the default."""
hass.states.async_set("cover.front_door", "closed")
@ -1206,7 +1180,7 @@ async def test_ws_hass_agent_debug(
"turn my cool light off",
"turn on all lights in the kitchen",
"how many lights are on in the kitchen?",
"this will not match anything", # null in results
"this will not match anything", # unmatched in results
],
}
)
@ -1219,3 +1193,96 @@ async def test_ws_hass_agent_debug(
# Light state should not have been changed
assert len(on_calls) == 0
assert len(off_calls) == 0
async def test_ws_hass_agent_debug_null_result(
hass: HomeAssistant,
init_components,
hass_ws_client: WebSocketGenerator,
snapshot: SnapshotAssertion,
) -> None:
"""Test homeassistant agent debug websocket command with a null result."""
client = await hass_ws_client(hass)
async def async_recognize(self, user_input, *args, **kwargs):
if user_input.text == "bad sentence":
return None
return await self.async_recognize(user_input, *args, **kwargs)
with patch(
"homeassistant.components.conversation.default_agent.DefaultAgent.async_recognize",
async_recognize,
):
await client.send_json_auto_id(
{
"type": "conversation/agent/homeassistant/debug",
"sentences": [
"bad sentence",
],
}
)
msg = await client.receive_json()
assert msg["success"]
assert msg["result"] == snapshot
assert msg["result"]["results"] == [None]
async def test_ws_hass_agent_debug_out_of_range(
hass: HomeAssistant,
init_components,
hass_ws_client: WebSocketGenerator,
snapshot: SnapshotAssertion,
entity_registry: er.EntityRegistry,
) -> None:
"""Test homeassistant agent debug websocket command with an out of range entity."""
test_light = entity_registry.async_get_or_create("light", "demo", "1234")
hass.states.async_set(
test_light.entity_id, "off", attributes={ATTR_FRIENDLY_NAME: "test light"}
)
client = await hass_ws_client(hass)
# Brightness is in range (0-100)
await client.send_json_auto_id(
{
"type": "conversation/agent/homeassistant/debug",
"sentences": [
"set test light brightness to 100%",
],
}
)
msg = await client.receive_json()
assert msg["success"]
assert msg["result"] == snapshot
results = msg["result"]["results"]
assert len(results) == 1
assert results[0]["match"]
# Brightness is out of range
await client.send_json_auto_id(
{
"type": "conversation/agent/homeassistant/debug",
"sentences": [
"set test light brightness to 1001%",
],
}
)
msg = await client.receive_json()
assert msg["success"]
assert msg["result"] == snapshot
results = msg["result"]["results"]
assert len(results) == 1
assert not results[0]["match"]
# Name matched, but brightness didn't
assert results[0]["slots"] == {"name": "test light"}
assert results[0]["unmatched_slots"] == {"brightness": 1001}