diff --git a/.gitignore b/.gitignore index 0d55cae3c9d..b3774b06bc8 100644 --- a/.gitignore +++ b/.gitignore @@ -103,3 +103,6 @@ desktop.ini # mypy /.mypy_cache/* + +# Secrets +.lokalise_token diff --git a/.travis.yml b/.travis.yml index 027c1f25c62..c1d70d528b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,4 +28,13 @@ cache: install: pip install -U tox coveralls language: python script: travis_wait 30 tox --develop +services: + - docker +before_deploy: + - docker pull lokalise/lokalise-cli +deploy: + provider: script + script: script/travis_deploy + on: + branch: dev after_success: coveralls diff --git a/homeassistant/components/frontend/__init__.py b/homeassistant/components/frontend/__init__.py index e2b98c3e59c..9fbf936cccc 100644 --- a/homeassistant/components/frontend/__init__.py +++ b/homeassistant/components/frontend/__init__.py @@ -21,6 +21,7 @@ from homeassistant.components.http.const import KEY_AUTHENTICATED from homeassistant.config import find_config_file, load_yaml_config_file from homeassistant.const import CONF_NAME, EVENT_THEMES_UPDATED from homeassistant.core import callback +from homeassistant.helpers.translation import async_get_translations from homeassistant.loader import bind_hass REQUIREMENTS = ['home-assistant-frontend==20180228.1'] @@ -379,6 +380,8 @@ def async_setup(hass, config): async_setup_themes(hass, conf.get(CONF_THEMES)) + hass.http.register_view(TranslationsView) + return True @@ -541,6 +544,23 @@ class ThemesView(HomeAssistantView): }) +class TranslationsView(HomeAssistantView): + """View to return backend defined translations.""" + + url = '/api/translations/{language}' + name = 'api:translations' + + @asyncio.coroutine + def get(self, request, language): + """Return translations.""" + hass = request.app['hass'] + + resources = yield from async_get_translations(hass, language) + return self.json({ + 'resources': resources, + }) + + def _fingerprint(path): """Fingerprint a file.""" with open(path) as fil: diff --git a/homeassistant/components/sensor/.translations/season.en.json b/homeassistant/components/sensor/.translations/season.en.json new file mode 100644 index 00000000000..b42100215ca --- /dev/null +++ b/homeassistant/components/sensor/.translations/season.en.json @@ -0,0 +1,8 @@ +{ + "state": { + "autumn": "Autumn", + "spring": "Spring", + "summer": "Summer", + "winter": "Winter" + } +} \ No newline at end of file diff --git a/homeassistant/components/sensor/season.py b/homeassistant/components/sensor/season.py index e02f3cac2b0..b04b7727e40 100644 --- a/homeassistant/components/sensor/season.py +++ b/homeassistant/components/sensor/season.py @@ -21,10 +21,10 @@ _LOGGER = logging.getLogger(__name__) NORTHERN = 'northern' SOUTHERN = 'southern' EQUATOR = 'equator' -STATE_SPRING = 'Spring' -STATE_SUMMER = 'Summer' -STATE_AUTUMN = 'Autumn' -STATE_WINTER = 'Winter' +STATE_SPRING = 'spring' +STATE_SUMMER = 'summer' +STATE_AUTUMN = 'autumn' +STATE_WINTER = 'winter' TYPE_ASTRONOMICAL = 'astronomical' TYPE_METEOROLOGICAL = 'meteorological' VALID_TYPES = [TYPE_ASTRONOMICAL, TYPE_METEOROLOGICAL] diff --git a/homeassistant/components/sensor/strings.season.json b/homeassistant/components/sensor/strings.season.json new file mode 100644 index 00000000000..63136320d74 --- /dev/null +++ b/homeassistant/components/sensor/strings.season.json @@ -0,0 +1,8 @@ +{ + "state": { + "spring": "Spring", + "summer": "Summer", + "autumn": "Autumn", + "winter": "Winter" + } +} diff --git a/homeassistant/helpers/translation.py b/homeassistant/helpers/translation.py new file mode 100644 index 00000000000..9d1773de4d2 --- /dev/null +++ b/homeassistant/helpers/translation.py @@ -0,0 +1,126 @@ +"""Translation string lookup helpers.""" +import logging +# pylint: disable=unused-import +from typing import Optional # NOQA +from os import path + +from homeassistant.loader import get_component, bind_hass +from homeassistant.util.json import load_json + +_LOGGER = logging.getLogger(__name__) + +TRANSLATION_STRING_CACHE = 'translation_string_cache' + + +def recursive_flatten(prefix, data): + """Return a flattened representation of dict data.""" + output = {} + for key, value in data.items(): + if isinstance(value, dict): + output.update( + recursive_flatten('{}{}.'.format(prefix, key), value)) + else: + output['{}{}'.format(prefix, key)] = value + return output + + +def flatten(data): + """Return a flattened representation of dict data.""" + return recursive_flatten('', data) + + +def component_translation_file(component, language): + """Return the translation json file location for a component.""" + if '.' in component: + name = component.split('.', 1)[1] + else: + name = component + + module = get_component(component) + component_path = path.dirname(module.__file__) + + # If loading translations for the package root, (__init__.py), the + # prefix should be skipped. + if module.__name__ == module.__package__: + filename = '{}.json'.format(language) + else: + filename = '{}.{}.json'.format(name, language) + + return path.join(component_path, '.translations', filename) + + +def load_translations_files(translation_files): + """Load and parse translation.json files.""" + loaded = {} + for component, translation_file in translation_files.items(): + loaded[component] = load_json(translation_file) + + return loaded + + +def build_resources(translation_cache, components): + """Build the resources response for the given components.""" + # Build response + resources = {} + for component in components: + if '.' not in component: + domain = component + else: + domain = component.split('.', 1)[0] + + if domain not in resources: + resources[domain] = {} + + # Add the translations for this component to the domain resources. + # Since clients cannot determine which platform an entity belongs to, + # all translations for a domain will be returned together. + resources[domain].update(translation_cache[component]) + + return resources + + +@bind_hass +async def async_get_component_resources(hass, language): + """Return translation resources for all components.""" + if TRANSLATION_STRING_CACHE not in hass.data: + hass.data[TRANSLATION_STRING_CACHE] = {} + if language not in hass.data[TRANSLATION_STRING_CACHE]: + hass.data[TRANSLATION_STRING_CACHE][language] = {} + translation_cache = hass.data[TRANSLATION_STRING_CACHE][language] + + # Get the set of components + components = hass.config.components + + # Calculate the missing components + missing_components = components - set(translation_cache) + missing_files = {} + for component in missing_components: + missing_files[component] = component_translation_file( + component, language) + + # Load missing files + if missing_files: + loaded_translations = await hass.async_add_job( + load_translations_files, missing_files) + + # Update cache + for component, translation_data in loaded_translations.items(): + translation_cache[component] = translation_data + + resources = build_resources(translation_cache, components) + + # Return the component translations resources under the 'component' + # translation namespace + return flatten({'component': resources}) + + +@bind_hass +async def async_get_translations(hass, language): + """Return all backend translations.""" + resources = await async_get_component_resources(hass, language) + if language != 'en': + # Fetch the English resources, as a fallback for missing keys + base_resources = await async_get_component_resources(hass, 'en') + resources = {**base_resources, **resources} + + return resources diff --git a/homeassistant/util/json.py b/homeassistant/util/json.py index 7a326c34f15..b2577ff6be6 100644 --- a/homeassistant/util/json.py +++ b/homeassistant/util/json.py @@ -32,13 +32,13 @@ def load_json(filename: str, default: Union[List, Dict] = _UNDEFINED) \ return {} if default is _UNDEFINED else default -def save_json(filename: str, config: Union[List, Dict]): +def save_json(filename: str, data: Union[List, Dict]): """Save JSON data to a file. Returns True on success. """ try: - data = json.dumps(config, sort_keys=True, indent=4) + data = json.dumps(data, sort_keys=True, indent=4) with open(filename, 'w', encoding='utf-8') as fdesc: fdesc.write(data) return True diff --git a/script/translations_download b/script/translations_download new file mode 100755 index 00000000000..de1f4640988 --- /dev/null +++ b/script/translations_download @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Safe bash settings +# -e Exit on command fail +# -u Exit on unset variable +# -o pipefail Exit if piped command has error code +set -eu -o pipefail + +cd "$(dirname "$0")/.." + +if [ -z "${LOKALISE_TOKEN-}" ] && [ ! -f .lokalise_token ] ; then + echo "Lokalise API token is required to download the latest set of" \ + "translations. Please create an account by using the following link:" \ + "https://lokalise.co/signup/130246255a974bd3b5e8a1.51616605/all/" \ + "Place your token in a new file \".lokalise_token\" in the repo" \ + "root directory." + exit 1 +fi + +# Load token from file if not already in the environment +[ -z "${LOKALISE_TOKEN-}" ] && LOKALISE_TOKEN="$(<.lokalise_token)" + +PROJECT_ID="130246255a974bd3b5e8a1.51616605" +LOCAL_DIR="$(pwd)/build/translations-download" +FILE_FORMAT=json + +mkdir -p ${LOCAL_DIR} + +docker pull lokalise/lokalise-cli +docker run \ + -v ${LOCAL_DIR}:/opt/dest/locale \ + lokalise/lokalise-cli lokalise \ + --token ${LOKALISE_TOKEN} \ + export ${PROJECT_ID} \ + --export_empty skip \ + --type json \ + --unzip_to /opt/dest + +script/translations_download_split.py diff --git a/script/translations_download_split.py b/script/translations_download_split.py new file mode 100755 index 00000000000..08ea9fbcccc --- /dev/null +++ b/script/translations_download_split.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Merge all translation sources into a single JSON file.""" +import glob +import os +import re + +from homeassistant.util import json as json_util + +FILENAME_FORMAT = re.compile(r'strings\.(?P\w+)\.json') + + +def get_language(path): + """Get the language code for the given file path.""" + return os.path.splitext(os.path.basename(path))[0] + + +def get_component_path(lang, component): + """Get the component translation path.""" + if os.path.isdir(os.path.join("homeassistant", "components", component)): + return os.path.join( + "homeassistant", "components", component, ".translations", + "{}.json".format(lang)) + else: + return os.path.join( + "homeassistant", "components", ".translations", + "{}.{}.json".format(component, lang)) + + +def get_platform_path(lang, component, platform): + """Get the platform translation path.""" + if os.path.isdir(os.path.join( + "homeassistant", "components", component, platform)): + return os.path.join( + "homeassistant", "components", component, platform, + ".translations", "{}.json".format(lang)) + else: + return os.path.join( + "homeassistant", "components", component, ".translations", + "{}.{}.json".format(platform, lang)) + + +def get_component_translations(translations): + """Get the component level translations.""" + translations = translations.copy() + translations.pop('platform', None) + + return translations + + +def save_language_translations(lang, translations): + """Distribute the translations for this language.""" + components = translations.get('component', {}) + for component, component_translations in components.items(): + base_translations = get_component_translations(component_translations) + if base_translations: + path = get_component_path(lang, component) + os.makedirs(os.path.dirname(path), exist_ok=True) + json_util.save_json(path, base_translations) + + for platform, platform_translations in component_translations.get( + 'platform', {}).items(): + path = get_platform_path(lang, component, platform) + os.makedirs(os.path.dirname(path), exist_ok=True) + json_util.save_json(path, platform_translations) + + +def main(): + """Main section of the script.""" + if not os.path.isfile("requirements_all.txt"): + print("Run this from HA root dir") + return + + paths = glob.iglob("build/translations-download/*.json") + for path in paths: + lang = get_language(path) + translations = json_util.load_json(path) + save_language_translations(lang, translations) + + +if __name__ == '__main__': + main() diff --git a/script/translations_upload b/script/translations_upload new file mode 100755 index 00000000000..fcc12ef272f --- /dev/null +++ b/script/translations_upload @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# Safe bash settings +# -e Exit on command fail +# -u Exit on unset variable +# -o pipefail Exit if piped command has error code +set -eu -o pipefail + +cd "$(dirname "$0")/.." + +if [ -z "${LOKALISE_TOKEN-}" ] && [ ! -f .lokalise_token ] ; then + echo "Lokalise API token is required to download the latest set of" \ + "translations. Please create an account by using the following link:" \ + "https://lokalise.co/signup/130246255a974bd3b5e8a1.51616605/all/" \ + "Place your token in a new file \".lokalise_token\" in the repo" \ + "root directory." + exit 1 +fi + +# Load token from file if not already in the environment +[ -z "${LOKALISE_TOKEN-}" ] && LOKALISE_TOKEN="$(<.lokalise_token)" + +PROJECT_ID="130246255a974bd3b5e8a1.51616605" +LOCAL_FILE="$(pwd)/build/translations-upload.json" +LANG_ISO=en + +CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + +if [ "${CURRENT_BRANCH-}" != "dev" ] && [ "${TRAVIS_BRANCH-}" != "dev" ] ; then + echo "Please only run the translations upload script from a clean checkout of dev." + exit 1 +fi + +script/translations_upload_merge.py + +docker pull lokalise/lokalise-cli +docker run \ + -v ${LOCAL_FILE}:/opt/src/${LOCAL_FILE} \ + lokalise/lokalise-cli lokalise \ + --token ${LOKALISE_TOKEN} \ + import ${PROJECT_ID} \ + --file /opt/src/${LOCAL_FILE} \ + --lang_iso ${LANG_ISO} \ + --replace 1 diff --git a/script/translations_upload_merge.py b/script/translations_upload_merge.py new file mode 100755 index 00000000000..6382c8d9abe --- /dev/null +++ b/script/translations_upload_merge.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Merge all translation sources into a single JSON file.""" +import glob +import itertools +import os +import re + +from homeassistant.util import json as json_util + +FILENAME_FORMAT = re.compile(r'strings\.(?P\w+)\.json') + + +def find_strings_files(): + """Return the paths of the strings source files.""" + return itertools.chain( + glob.iglob("strings*.json"), + glob.iglob("*{}strings*.json".format(os.sep)), + ) + + +def get_component_platform(path): + """Get the component and platform name from the path.""" + directory, filename = os.path.split(path) + match = FILENAME_FORMAT.search(filename) + suffix = match.group('suffix') if match else None + if directory: + return directory, suffix + else: + return suffix, None + + +def get_translation_dict(translations, component, platform): + """Return the dict to hold component translations.""" + if not component: + return translations['component'] + + if component not in translations: + translations['component'][component] = {} + + if not platform: + return translations['component'][component] + + if 'platform' not in translations['component'][component]: + translations['component'][component]['platform'] = {} + + if platform not in translations['component'][component]['platform']: + translations['component'][component]['platform'][platform] = {} + + return translations['component'][component]['platform'][platform] + + +def main(): + """Main section of the script.""" + if not os.path.isfile("requirements_all.txt"): + print("Run this from HA root dir") + return + + root = os.getcwd() + os.chdir(os.path.join("homeassistant", "components")) + + translations = { + 'component': {} + } + + paths = find_strings_files() + for path in paths: + component, platform = get_component_platform(path) + parent = get_translation_dict(translations, component, platform) + strings = json_util.load_json(path) + parent.update(strings) + + os.chdir(root) + + os.makedirs("build", exist_ok=True) + + json_util.save_json( + os.path.join("build", "translations-upload.json"), translations) + + +if __name__ == '__main__': + main() diff --git a/script/travis_deploy b/script/travis_deploy new file mode 100755 index 00000000000..359f6a46077 --- /dev/null +++ b/script/travis_deploy @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +# Safe bash settings +# -e Exit on command fail +# -u Exit on unset variable +# -o pipefail Exit if piped command has error code +set -eu -o pipefail + +cd "$(dirname "$0")/.." + +script/translations_upload diff --git a/tests/helpers/test_translation.py b/tests/helpers/test_translation.py new file mode 100644 index 00000000000..840f665f410 --- /dev/null +++ b/tests/helpers/test_translation.py @@ -0,0 +1,108 @@ +"""Test the translation helper.""" +# pylint: disable=protected-access +from os import path + +import homeassistant.helpers.translation as translation +from homeassistant.setup import async_setup_component + + +def test_flatten(): + """Test the flatten function.""" + data = { + "parent1": { + "child1": "data1", + "child2": "data2", + }, + "parent2": "data3", + } + + flattened = translation.flatten(data) + + assert flattened == { + "parent1.child1": "data1", + "parent1.child2": "data2", + "parent2": "data3", + } + + +async def test_component_translation_file(hass): + """Test the component translation file function.""" + assert await async_setup_component(hass, 'switch', { + 'switch': {'platform': 'test'} + }) + assert await async_setup_component(hass, 'test_standalone', { + 'test_standalone' + }) + assert await async_setup_component(hass, 'test_package', { + 'test_package' + }) + + assert path.normpath(translation.component_translation_file( + 'switch.test', 'en')) == path.normpath(hass.config.path( + 'custom_components', 'switch', '.translations', 'test.en.json')) + + assert path.normpath(translation.component_translation_file( + 'test_standalone', 'en')) == path.normpath(hass.config.path( + 'custom_components', '.translations', 'test_standalone.en.json')) + + assert path.normpath(translation.component_translation_file( + 'test_package', 'en')) == path.normpath(hass.config.path( + 'custom_components', 'test_package', '.translations', 'en.json')) + + +def test_load_translations_files(hass): + """Test the load translation files function.""" + # Test one valid and one invalid file + file1 = hass.config.path( + 'custom_components', 'switch', '.translations', 'test.en.json') + file2 = hass.config.path( + 'custom_components', 'switch', '.translations', 'invalid.json') + assert translation.load_translations_files({ + 'switch.test': file1, + 'invalid': file2 + }) == { + 'switch.test': { + 'state': { + 'string1': 'Value 1', + 'string2': 'Value 2', + } + }, + 'invalid': {}, + } + + +async def test_get_translations(hass): + """Test the get translations helper.""" + translations = await translation.async_get_translations(hass, 'en') + assert translations == {} + + assert await async_setup_component(hass, 'switch', { + 'switch': {'platform': 'test'} + }) + + translations = await translation.async_get_translations(hass, 'en') + assert translations == { + 'component.switch.state.string1': 'Value 1', + 'component.switch.state.string2': 'Value 2', + } + + translations = await translation.async_get_translations(hass, 'de') + assert translations == { + 'component.switch.state.string1': 'German Value 1', + 'component.switch.state.string2': 'German Value 2', + } + + # Test a partial translation + translations = await translation.async_get_translations(hass, 'es') + assert translations == { + 'component.switch.state.string1': 'Spanish Value 1', + 'component.switch.state.string2': 'Value 2', + } + + # Test that an untranslated language falls back to English. + translations = await translation.async_get_translations( + hass, 'invalid-language') + assert translations == { + 'component.switch.state.string1': 'Value 1', + 'component.switch.state.string2': 'Value 2', + } diff --git a/tests/testing_config/custom_components/switch/.translations/test.de.json b/tests/testing_config/custom_components/switch/.translations/test.de.json new file mode 100644 index 00000000000..fad78b12d63 --- /dev/null +++ b/tests/testing_config/custom_components/switch/.translations/test.de.json @@ -0,0 +1,6 @@ +{ + "state": { + "string1": "German Value 1", + "string2": "German Value 2" + } +} diff --git a/tests/testing_config/custom_components/switch/.translations/test.en.json b/tests/testing_config/custom_components/switch/.translations/test.en.json new file mode 100644 index 00000000000..f4ce728af05 --- /dev/null +++ b/tests/testing_config/custom_components/switch/.translations/test.en.json @@ -0,0 +1,6 @@ +{ + "state": { + "string1": "Value 1", + "string2": "Value 2" + } +} diff --git a/tests/testing_config/custom_components/switch/.translations/test.es.json b/tests/testing_config/custom_components/switch/.translations/test.es.json new file mode 100644 index 00000000000..b3590a6d321 --- /dev/null +++ b/tests/testing_config/custom_components/switch/.translations/test.es.json @@ -0,0 +1,5 @@ +{ + "state": { + "string1": "Spanish Value 1" + } +} diff --git a/tests/testing_config/custom_components/test_package/__init__.py b/tests/testing_config/custom_components/test_package/__init__.py new file mode 100644 index 00000000000..528f056948b --- /dev/null +++ b/tests/testing_config/custom_components/test_package/__init__.py @@ -0,0 +1,7 @@ +"""Provide a mock package component.""" +DOMAIN = 'test_package' + + +def setup(hass, config): + """Mock a successful setup.""" + return True diff --git a/tests/testing_config/custom_components/test_standalone.py b/tests/testing_config/custom_components/test_standalone.py new file mode 100644 index 00000000000..f0d4ba7982b --- /dev/null +++ b/tests/testing_config/custom_components/test_standalone.py @@ -0,0 +1,7 @@ +"""Provide a mock standalone component.""" +DOMAIN = 'test_standalone' + + +def setup(hass, config): + """Mock a successful setup.""" + return True