Home-Assistant
/
core
mirror of https://github.com/home-assistant/core.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reduce overhead to load multiple languages in translations (#111028) 

* Reduce overhead to load multiple languages in translations

Instead of loading in a task, we now group everything
to be loaded into a single executor job

* fixes

* fixes

* fixes

* fixes

* fixes

* update tests

* add missing coverage (was existing)
pull/111054/head
J. Nick Koston 2024-02-20 20:52:28 -06:00 committed by GitHub
parent 9c145b5faa
commit 9ce1ec414e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 109 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
homeassistant/helpers/translation.py
View File

@ -72,23 +72,27 @@ def component_translation_path(
return str(translation_path / filename) return str(translation_path / filename)
def load_translations_files( def _load_translations_files_by_language(
translation_files: dict[str, str], translation_files: dict[str, dict[str, str]],
) -> dict[str, dict[str, Any]]: ) -> dict[str, dict[str, Any]]:
"""Load and parse translation.json files.""" """Load and parse translation.json files."""
loaded = {} loaded: dict[str, dict[str, Any]] = {}
for component, translation_file in translation_files.items(): for language, component_translation_file in translation_files.items():
loaded_json = load_json(translation_file) loaded_for_language: dict[str, Any] = {}
loaded[language] = loaded_for_language
if not isinstance(loaded_json, dict): for component, translation_file in component_translation_file.items():
_LOGGER.warning( loaded_json = load_json(translation_file)
"Translation file is unexpected type %s. Expected dict for %s",
type(loaded_json),
translation_file,
)
continue
loaded[component] = loaded_json if not isinstance(loaded_json, dict):
_LOGGER.warning(
"Translation file is unexpected type %s. Expected dict for %s",
type(loaded_json),
translation_file,
)
continue
loaded_for_language[component] = loaded_json
return loaded return loaded
@ -151,47 +155,53 @@ def build_resources(
async def _async_get_component_strings( async def _async_get_component_strings(
hass: HomeAssistant, hass: HomeAssistant,
language: str, languages: Iterable[str],
components: set[str], components: set[str],
integrations: dict[str, Integration], integrations: dict[str, Integration],
) -> dict[str, Any]: ) -> dict[str, dict[str, Any]]:
"""Load translations.""" """Load translations."""
translations: dict[str, Any] = {} translations_by_language: dict[str, dict[str, Any]] = {}
# Determine paths of missing components/platforms # Determine paths of missing components/platforms
files_to_load: dict[str, str] = {} files_to_load_by_language: dict[str, dict[str, str]] = {}
for loaded in components: for language in languages:
domain = loaded.partition(".")[0] files_to_load: dict[str, str] = {}
if not (integration := integrations.get(domain)): files_to_load_by_language[language] = files_to_load
continue
path = component_translation_path(loaded, language, integration) loaded_translations: dict[str, Any] = {}
# No translation available translations_by_language[language] = loaded_translations
if path is None:
translations[loaded] = {} for loaded in components:
else: domain = loaded.partition(".")[0]
files_to_load[loaded] = path if not (integration := integrations.get(domain)):
continue
path = component_translation_path(loaded, language, integration)
# No translation available
if path is None:
loaded_translations[loaded] = {}
else:
files_to_load[loaded] = path
if not files_to_load: if not files_to_load:
return translations return translations_by_language
# Load files # Load files
load_translations_job = hass.async_add_executor_job( loaded_translations_by_language = await hass.async_add_executor_job(
load_translations_files, files_to_load _load_translations_files_by_language, files_to_load_by_language
) )
assert load_translations_job is not None
loaded_translations = await load_translations_job
# Translations that miss "title" will get integration put in. # Translations that miss "title" will get integration put in.
for loaded, loaded_translation in loaded_translations.items(): for language, loaded_translations in loaded_translations_by_language.items():
if "." in loaded: for loaded, loaded_translation in loaded_translations.items():
continue if "." in loaded:
continue
if "title" not in loaded_translation: if "title" not in loaded_translation:
loaded_translation["title"] = integrations[loaded].name loaded_translation["title"] = integrations[loaded].name
translations.update(loaded_translations) translations_by_language[language].update(loaded_translations)
return translations return translations_by_language
class _TranslationCache: class _TranslationCache:
@ -280,13 +290,29 @@ class _TranslationCache:
continue continue
integrations[domain] = int_or_exc integrations[domain] = int_or_exc
for translation_strings in await asyncio.gather( translation_by_language_strings = await _async_get_component_strings(
*( self.hass, languages, components, integrations
_async_get_component_strings(self.hass, lang, components, integrations) )
for lang in languages
# English is always the fallback language so we load them first
self._build_category_cache(
language, components, translation_by_language_strings[LOCALE_EN]
)
if language != LOCALE_EN:
# Now overlay the requested language on top of the English
self._build_category_cache(
language, components, translation_by_language_strings[language]
) )
):
self._build_category_cache(language, components, translation_strings) loaded_english_components = self.loaded.setdefault(LOCALE_EN, set())
# Since we just loaded english anyway we can avoid loading
# again if they switch back to english.
if loaded_english_components.isdisjoint(components):
self._build_category_cache(
LOCALE_EN, components, translation_by_language_strings[LOCALE_EN]
)
loaded_english_components.update(components)
self.loaded[language].update(components) self.loaded[language].update(components)

62
tests/helpers/test_translation.py
View File

@ -2,6 +2,7 @@
import asyncio import asyncio
from os import path from os import path
import pathlib import pathlib
from typing import Any
from unittest.mock import Mock, call, patch from unittest.mock import Mock, call, patch
import pytest import pytest
@ -79,7 +80,9 @@ async def test_component_translation_path(
) )
def test_load_translations_files(hass: HomeAssistant) -> None: def test__load_translations_files_by_language(
hass: HomeAssistant, caplog: pytest.LogCaptureFixture
) -> None:
"""Test the load translation files function.""" """Test the load translation files function."""
# Test one valid and one invalid file # Test one valid and one invalid file
file1 = hass.config.path( file1 = hass.config.path(
@ -88,15 +91,22 @@ def test_load_translations_files(hass: HomeAssistant) -> None:
file2 = hass.config.path( file2 = hass.config.path(
"custom_components", "test", "translations", "invalid.json" "custom_components", "test", "translations", "invalid.json"
) )
assert translation.load_translations_files( file3 = hass.config.path(
{"switch.test": file1, "invalid": file2} "custom_components", "test", "translations", "_broken.en.json"
)
assert translation._load_translations_files_by_language(
{"en": {"switch.test": file1, "invalid": file2, "broken": file3}}
) == { ) == {
"switch.test": { "en": {
"state": {"string1": "Value 1", "string2": "Value 2"}, "switch.test": {
"something": "else", "state": {"string1": "Value 1", "string2": "Value 2"},
}, "something": "else",
"invalid": {}, },
"invalid": {},
}
} }
assert "Translation file is unexpected type" in caplog.text
assert "_broken.en.json" in caplog.text
@pytest.mark.parametrize( @pytest.mark.parametrize(
@ -215,8 +225,8 @@ async def test_get_translations_loads_config_flows(
"homeassistant.helpers.translation.component_translation_path", "homeassistant.helpers.translation.component_translation_path",
return_value="bla.json", return_value="bla.json",
), patch( ), patch(
"homeassistant.helpers.translation.load_translations_files", "homeassistant.helpers.translation._load_translations_files_by_language",
return_value={"component1": {"title": "world"}}, return_value={"en": {"component1": {"title": "world"}}},
), patch( ), patch(
"homeassistant.helpers.translation.async_get_integrations", "homeassistant.helpers.translation.async_get_integrations",
return_value={"component1": integration}, return_value={"component1": integration},
@ -244,8 +254,8 @@ async def test_get_translations_loads_config_flows(
"homeassistant.helpers.translation.component_translation_path", "homeassistant.helpers.translation.component_translation_path",
return_value="bla.json", return_value="bla.json",
), patch( ), patch(
"homeassistant.helpers.translation.load_translations_files", "homeassistant.helpers.translation._load_translations_files_by_language",
return_value={"component2": {"title": "world"}}, return_value={"en": {"component2": {"title": "world"}}},
), patch( ), patch(
"homeassistant.helpers.translation.async_get_integrations", "homeassistant.helpers.translation.async_get_integrations",
return_value={"component2": integration}, return_value={"component2": integration},
@ -280,19 +290,21 @@ async def test_get_translations_while_loading_components(hass: HomeAssistant) ->
hass.config.components.add("component1") hass.config.components.add("component1")
load_count = 0 load_count = 0
def mock_load_translation_files(files): def mock_load_translation_files(
files: dict[str, dict[str, Any]],
) -> dict[str, dict[str, Any]]:
"""Mock load translation files.""" """Mock load translation files."""
nonlocal load_count nonlocal load_count
load_count += 1 load_count += 1
# Mimic race condition by loading a component during setup # Mimic race condition by loading a component during setup
return {"component1": {"title": "world"}} return {language: {"component1": {"title": "world"}} for language in files}
with patch( with patch(
"homeassistant.helpers.translation.component_translation_path", "homeassistant.helpers.translation.component_translation_path",
return_value="bla.json", return_value="bla.json",
), patch( ), patch(
"homeassistant.helpers.translation.load_translations_files", "homeassistant.helpers.translation._load_translations_files_by_language",
mock_load_translation_files, mock_load_translation_files,
), patch( ), patch(
"homeassistant.helpers.translation.async_get_integrations", "homeassistant.helpers.translation.async_get_integrations",
@ -330,18 +342,18 @@ async def test_translation_merging(
hass.config.components.add("moon.sensor") hass.config.components.add("moon.sensor")
hass.config.components.add("sensor") hass.config.components.add("sensor")
orig_load_translations = translation.load_translations_files orig_load_translations = translation._load_translations_files_by_language
def mock_load_translations_files(files): def mock_load_translations_files(files):
"""Mock loading.""" """Mock loading."""
result = orig_load_translations(files) result = orig_load_translations(files)
result["moon.sensor"] = { result["en"]["moon.sensor"] = {
"state": {"moon__phase": {"first_quarter": "First Quarter"}} "state": {"moon__phase": {"first_quarter": "First Quarter"}}
} }
return result return result
with patch( with patch(
"homeassistant.helpers.translation.load_translations_files", "homeassistant.helpers.translation._load_translations_files_by_language",
side_effect=mock_load_translations_files, side_effect=mock_load_translations_files,
): ):
translations = await translation.async_get_translations(hass, "en", "state") translations = await translation.async_get_translations(hass, "en", "state")
@ -354,11 +366,11 @@ async def test_translation_merging(
def mock_load_bad_translations_files(files): def mock_load_bad_translations_files(files):
"""Mock loading.""" """Mock loading."""
result = orig_load_translations(files) result = orig_load_translations(files)
result["season.sensor"] = {"state": "bad data"} result["en"]["season.sensor"] = {"state": "bad data"}
return result return result
with patch( with patch(
"homeassistant.helpers.translation.load_translations_files", "homeassistant.helpers.translation._load_translations_files_by_language",
side_effect=mock_load_bad_translations_files, side_effect=mock_load_bad_translations_files,
): ):
translations = await translation.async_get_translations(hass, "en", "state") translations = await translation.async_get_translations(hass, "en", "state")
@ -375,12 +387,12 @@ async def test_translation_merging_loaded_apart(
hass: HomeAssistant, caplog: pytest.LogCaptureFixture hass: HomeAssistant, caplog: pytest.LogCaptureFixture
) -> None: ) -> None:
"""Test we merge translations of two integrations when they are not loaded at the same time.""" """Test we merge translations of two integrations when they are not loaded at the same time."""
orig_load_translations = translation.load_translations_files orig_load_translations = translation._load_translations_files_by_language
def mock_load_translations_files(files): def mock_load_translations_files(files):
"""Mock loading.""" """Mock loading."""
result = orig_load_translations(files) result = orig_load_translations(files)
result["moon.sensor"] = { result["en"]["moon.sensor"] = {
"state": {"moon__phase": {"first_quarter": "First Quarter"}} "state": {"moon__phase": {"first_quarter": "First Quarter"}}
} }
return result return result
@ -388,7 +400,7 @@ async def test_translation_merging_loaded_apart(
hass.config.components.add("sensor") hass.config.components.add("sensor")
with patch( with patch(
"homeassistant.helpers.translation.load_translations_files", "homeassistant.helpers.translation._load_translations_files_by_language",
side_effect=mock_load_translations_files, side_effect=mock_load_translations_files,
): ):
translations = await translation.async_get_translations(hass, "en", "state") translations = await translation.async_get_translations(hass, "en", "state")
@ -398,7 +410,7 @@ async def test_translation_merging_loaded_apart(
hass.config.components.add("moon.sensor") hass.config.components.add("moon.sensor")
with patch( with patch(
"homeassistant.helpers.translation.load_translations_files", "homeassistant.helpers.translation._load_translations_files_by_language",
side_effect=mock_load_translations_files, side_effect=mock_load_translations_files,
): ):
translations = await translation.async_get_translations(hass, "en", "state") translations = await translation.async_get_translations(hass, "en", "state")
@ -406,7 +418,7 @@ async def test_translation_merging_loaded_apart(
assert "component.sensor.state.moon__phase.first_quarter" in translations assert "component.sensor.state.moon__phase.first_quarter" in translations
with patch( with patch(
"homeassistant.helpers.translation.load_translations_files", "homeassistant.helpers.translation._load_translations_files_by_language",
side_effect=mock_load_translations_files, side_effect=mock_load_translations_files,
): ):
translations = await translation.async_get_translations( translations = await translation.async_get_translations(

1
tests/testing_config/custom_components/test/translations/_broken.en.json Normal file
View File

@ -0,0 +1 @@
[]