Home-Assistant
/
core
mirror of https://github.com/home-assistant/core.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reduce overhead to load multiple languages in translations (#111028) 

* Reduce overhead to load multiple languages in translations

Instead of loading in a task, we now group everything
to be loaded into a single executor job

* fixes

* fixes

* fixes

* fixes

* fixes

* update tests

* add missing coverage (was existing)
pull/111054/head
J. Nick Koston 2024-02-20 20:52:28 -06:00 committed by GitHub
parent 9c145b5faa
commit 9ce1ec414e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 109 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
homeassistant/helpers/translation.py
View File

@ -72,23 +72,27 @@ def component_translation_path(
return str(translation_path / filename)
def load_translations_files(
translation_files: dict[str, str],
def _load_translations_files_by_language(
translation_files: dict[str, dict[str, str]],
) -> dict[str, dict[str, Any]]:
"""Load and parse translation.json files."""
loaded = {}
for component, translation_file in translation_files.items():
loaded_json = load_json(translation_file)
loaded: dict[str, dict[str, Any]] = {}
for language, component_translation_file in translation_files.items():
loaded_for_language: dict[str, Any] = {}
loaded[language] = loaded_for_language
if not isinstance(loaded_json, dict):
_LOGGER.warning(
"Translation file is unexpected type %s. Expected dict for %s",
type(loaded_json),
translation_file,
)
continue
for component, translation_file in component_translation_file.items():
loaded_json = load_json(translation_file)
loaded[component] = loaded_json
if not isinstance(loaded_json, dict):
_LOGGER.warning(
"Translation file is unexpected type %s. Expected dict for %s",
type(loaded_json),
translation_file,
)
continue
loaded_for_language[component] = loaded_json
return loaded
@ -151,47 +155,53 @@ def build_resources(
async def _async_get_component_strings(
hass: HomeAssistant,
language: str,
languages: Iterable[str],
components: set[str],
integrations: dict[str, Integration],
) -> dict[str, Any]:
) -> dict[str, dict[str, Any]]:
"""Load translations."""
translations: dict[str, Any] = {}
translations_by_language: dict[str, dict[str, Any]] = {}
# Determine paths of missing components/platforms
files_to_load: dict[str, str] = {}
for loaded in components:
domain = loaded.partition(".")[0]
if not (integration := integrations.get(domain)):
continue
files_to_load_by_language: dict[str, dict[str, str]] = {}
for language in languages:
files_to_load: dict[str, str] = {}
files_to_load_by_language[language] = files_to_load
path = component_translation_path(loaded, language, integration)
# No translation available
if path is None:
translations[loaded] = {}
else:
files_to_load[loaded] = path
loaded_translations: dict[str, Any] = {}
translations_by_language[language] = loaded_translations
for loaded in components:
domain = loaded.partition(".")[0]
if not (integration := integrations.get(domain)):
continue
path = component_translation_path(loaded, language, integration)
# No translation available
if path is None:
loaded_translations[loaded] = {}
else:
files_to_load[loaded] = path
if not files_to_load:
return translations
return translations_by_language
# Load files
load_translations_job = hass.async_add_executor_job(
load_translations_files, files_to_load
loaded_translations_by_language = await hass.async_add_executor_job(
_load_translations_files_by_language, files_to_load_by_language
)
assert load_translations_job is not None
loaded_translations = await load_translations_job
# Translations that miss "title" will get integration put in.
for loaded, loaded_translation in loaded_translations.items():
if "." in loaded:
continue
for language, loaded_translations in loaded_translations_by_language.items():
for loaded, loaded_translation in loaded_translations.items():
if "." in loaded:
continue
if "title" not in loaded_translation:
loaded_translation["title"] = integrations[loaded].name
if "title" not in loaded_translation:
loaded_translation["title"] = integrations[loaded].name
translations.update(loaded_translations)
translations_by_language[language].update(loaded_translations)
return translations
return translations_by_language
class _TranslationCache:
@ -280,13 +290,29 @@ class _TranslationCache:
continue
integrations[domain] = int_or_exc
for translation_strings in await asyncio.gather(
*(
_async_get_component_strings(self.hass, lang, components, integrations)
for lang in languages
translation_by_language_strings = await _async_get_component_strings(
self.hass, languages, components, integrations
)
# English is always the fallback language so we load them first
self._build_category_cache(
language, components, translation_by_language_strings[LOCALE_EN]
)
if language != LOCALE_EN:
# Now overlay the requested language on top of the English
self._build_category_cache(
language, components, translation_by_language_strings[language]
)
):
self._build_category_cache(language, components, translation_strings)
loaded_english_components = self.loaded.setdefault(LOCALE_EN, set())
# Since we just loaded english anyway we can avoid loading
# again if they switch back to english.
if loaded_english_components.isdisjoint(components):
self._build_category_cache(
LOCALE_EN, components, translation_by_language_strings[LOCALE_EN]
)
loaded_english_components.update(components)
self.loaded[language].update(components)

62
tests/helpers/test_translation.py
View File

@ -2,6 +2,7 @@
import asyncio
from os import path
import pathlib
from typing import Any
from unittest.mock import Mock, call, patch
import pytest
@ -79,7 +80,9 @@ async def test_component_translation_path(
)
def test_load_translations_files(hass: HomeAssistant) -> None:
def test__load_translations_files_by_language(
hass: HomeAssistant, caplog: pytest.LogCaptureFixture
) -> None:
"""Test the load translation files function."""
# Test one valid and one invalid file
file1 = hass.config.path(
@ -88,15 +91,22 @@ def test_load_translations_files(hass: HomeAssistant) -> None:
file2 = hass.config.path(
"custom_components", "test", "translations", "invalid.json"
)
assert translation.load_translations_files(
{"switch.test": file1, "invalid": file2}
file3 = hass.config.path(
"custom_components", "test", "translations", "_broken.en.json"
)
assert translation._load_translations_files_by_language(
{"en": {"switch.test": file1, "invalid": file2, "broken": file3}}
) == {
"switch.test": {
"state": {"string1": "Value 1", "string2": "Value 2"},
"something": "else",
},
"invalid": {},
"en": {
"switch.test": {
"state": {"string1": "Value 1", "string2": "Value 2"},
"something": "else",
},
"invalid": {},
}
}
assert "Translation file is unexpected type" in caplog.text
assert "_broken.en.json" in caplog.text
@pytest.mark.parametrize(
@ -215,8 +225,8 @@ async def test_get_translations_loads_config_flows(
"homeassistant.helpers.translation.component_translation_path",
return_value="bla.json",
), patch(
"homeassistant.helpers.translation.load_translations_files",
return_value={"component1": {"title": "world"}},
"homeassistant.helpers.translation._load_translations_files_by_language",
return_value={"en": {"component1": {"title": "world"}}},
), patch(
"homeassistant.helpers.translation.async_get_integrations",
return_value={"component1": integration},
@ -244,8 +254,8 @@ async def test_get_translations_loads_config_flows(
"homeassistant.helpers.translation.component_translation_path",
return_value="bla.json",
), patch(
"homeassistant.helpers.translation.load_translations_files",
return_value={"component2": {"title": "world"}},
"homeassistant.helpers.translation._load_translations_files_by_language",
return_value={"en": {"component2": {"title": "world"}}},
), patch(
"homeassistant.helpers.translation.async_get_integrations",
return_value={"component2": integration},
@ -280,19 +290,21 @@ async def test_get_translations_while_loading_components(hass: HomeAssistant) ->
hass.config.components.add("component1")
load_count = 0
def mock_load_translation_files(files):
def mock_load_translation_files(
files: dict[str, dict[str, Any]],
) -> dict[str, dict[str, Any]]:
"""Mock load translation files."""
nonlocal load_count
load_count += 1
# Mimic race condition by loading a component during setup
return {"component1": {"title": "world"}}
return {language: {"component1": {"title": "world"}} for language in files}
with patch(
"homeassistant.helpers.translation.component_translation_path",
return_value="bla.json",
), patch(
"homeassistant.helpers.translation.load_translations_files",
"homeassistant.helpers.translation._load_translations_files_by_language",
mock_load_translation_files,
), patch(
"homeassistant.helpers.translation.async_get_integrations",
@ -330,18 +342,18 @@ async def test_translation_merging(
hass.config.components.add("moon.sensor")
hass.config.components.add("sensor")
orig_load_translations = translation.load_translations_files
orig_load_translations = translation._load_translations_files_by_language
def mock_load_translations_files(files):
"""Mock loading."""
result = orig_load_translations(files)
result["moon.sensor"] = {
result["en"]["moon.sensor"] = {
"state": {"moon__phase": {"first_quarter": "First Quarter"}}
}
return result
with patch(
"homeassistant.helpers.translation.load_translations_files",
"homeassistant.helpers.translation._load_translations_files_by_language",
side_effect=mock_load_translations_files,
):
translations = await translation.async_get_translations(hass, "en", "state")
@ -354,11 +366,11 @@ async def test_translation_merging(
def mock_load_bad_translations_files(files):
"""Mock loading."""
result = orig_load_translations(files)
result["season.sensor"] = {"state": "bad data"}
result["en"]["season.sensor"] = {"state": "bad data"}
return result
with patch(
"homeassistant.helpers.translation.load_translations_files",
"homeassistant.helpers.translation._load_translations_files_by_language",
side_effect=mock_load_bad_translations_files,
):
translations = await translation.async_get_translations(hass, "en", "state")
@ -375,12 +387,12 @@ async def test_translation_merging_loaded_apart(
hass: HomeAssistant, caplog: pytest.LogCaptureFixture
) -> None:
"""Test we merge translations of two integrations when they are not loaded at the same time."""
orig_load_translations = translation.load_translations_files
orig_load_translations = translation._load_translations_files_by_language
def mock_load_translations_files(files):
"""Mock loading."""
result = orig_load_translations(files)
result["moon.sensor"] = {
result["en"]["moon.sensor"] = {
"state": {"moon__phase": {"first_quarter": "First Quarter"}}
}
return result
@ -388,7 +400,7 @@ async def test_translation_merging_loaded_apart(
hass.config.components.add("sensor")
with patch(
"homeassistant.helpers.translation.load_translations_files",
"homeassistant.helpers.translation._load_translations_files_by_language",
side_effect=mock_load_translations_files,
):
translations = await translation.async_get_translations(hass, "en", "state")
@ -398,7 +410,7 @@ async def test_translation_merging_loaded_apart(
hass.config.components.add("moon.sensor")
with patch(
"homeassistant.helpers.translation.load_translations_files",
"homeassistant.helpers.translation._load_translations_files_by_language",
side_effect=mock_load_translations_files,
):
translations = await translation.async_get_translations(hass, "en", "state")
@ -406,7 +418,7 @@ async def test_translation_merging_loaded_apart(
assert "component.sensor.state.moon__phase.first_quarter" in translations
with patch(
"homeassistant.helpers.translation.load_translations_files",
"homeassistant.helpers.translation._load_translations_files_by_language",
side_effect=mock_load_translations_files,
):
translations = await translation.async_get_translations(

1
tests/testing_config/custom_components/test/translations/_broken.en.json Normal file
View File

@ -0,0 +1 @@
[]