Significantly improve yaml load times when the C loader is available (#73337)

pull/73424/head
J. Nick Koston 2022-06-13 08:44:46 -10:00 committed by GitHub
parent b84e844c76
commit dca4d3cd61
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 190 additions and 78 deletions

View File

@ -28,6 +28,7 @@ env:
PIP_CACHE: /tmp/pip-cache PIP_CACHE: /tmp/pip-cache
SQLALCHEMY_WARN_20: 1 SQLALCHEMY_WARN_20: 1
PYTHONASYNCIODEBUG: 1 PYTHONASYNCIODEBUG: 1
HASS_CI: 1
concurrency: concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

View File

@ -158,7 +158,7 @@ jobs:
wheels-key: ${{ secrets.WHEELS_KEY }} wheels-key: ${{ secrets.WHEELS_KEY }}
wheels-user: wheels wheels-user: wheels
env-file: true env-file: true
apk: "build-base;cmake;git;linux-headers;libexecinfo-dev;bluez-dev;libffi-dev;openssl-dev;glib-dev;eudev-dev;libxml2-dev;libxslt-dev;libpng-dev;libjpeg-turbo-dev;tiff-dev;autoconf;automake;cups-dev;gmp-dev;mpfr-dev;mpc1-dev;ffmpeg-dev;gammu-dev;cargo" apk: "build-base;cmake;git;linux-headers;libexecinfo-dev;bluez-dev;libffi-dev;openssl-dev;glib-dev;eudev-dev;libxml2-dev;libxslt-dev;libpng-dev;libjpeg-turbo-dev;tiff-dev;autoconf;automake;cups-dev;gmp-dev;mpfr-dev;mpc1-dev;ffmpeg-dev;gammu-dev;yaml-dev;cargo"
pip: "Cython;numpy;scikit-build" pip: "Cython;numpy;scikit-build"
skip-binary: aiohttp,grpcio skip-binary: aiohttp,grpcio
constraints: "homeassistant/package_constraints.txt" constraints: "homeassistant/package_constraints.txt"

View File

@ -18,6 +18,7 @@ RUN \
libavfilter-dev \ libavfilter-dev \
libpcap-dev \ libpcap-dev \
libturbojpeg0 \ libturbojpeg0 \
libyaml-dev \
libxml2 \ libxml2 \
git \ git \
cmake \ cmake \

View File

@ -191,7 +191,7 @@ def check(config_dir, secrets=False):
if secrets: if secrets:
# Ensure !secrets point to the patched function # Ensure !secrets point to the patched function
yaml_loader.SafeLineLoader.add_constructor("!secret", yaml_loader.secret_yaml) yaml_loader.add_constructor("!secret", yaml_loader.secret_yaml)
def secrets_proxy(*args): def secrets_proxy(*args):
secrets = Secrets(*args) secrets = Secrets(*args)
@ -219,9 +219,7 @@ def check(config_dir, secrets=False):
pat.stop() pat.stop()
if secrets: if secrets:
# Ensure !secrets point to the original function # Ensure !secrets point to the original function
yaml_loader.SafeLineLoader.add_constructor( yaml_loader.add_constructor("!secret", yaml_loader.secret_yaml)
"!secret", yaml_loader.secret_yaml
)
return res return res

View File

@ -4,6 +4,7 @@ from __future__ import annotations
from collections import OrderedDict from collections import OrderedDict
from collections.abc import Iterator from collections.abc import Iterator
import fnmatch import fnmatch
from io import StringIO
import logging import logging
import os import os
from pathlib import Path from pathlib import Path
@ -11,6 +12,14 @@ from typing import Any, TextIO, TypeVar, Union, overload
import yaml import yaml
try:
from yaml import CSafeLoader as FastestAvailableSafeLoader
HAS_C_LOADER = True
except ImportError:
HAS_C_LOADER = False
from yaml import SafeLoader as FastestAvailableSafeLoader # type: ignore[misc]
from homeassistant.exceptions import HomeAssistantError from homeassistant.exceptions import HomeAssistantError
from .const import SECRET_YAML from .const import SECRET_YAML
@ -88,6 +97,30 @@ class Secrets:
return secrets return secrets
class SafeLoader(FastestAvailableSafeLoader):
"""The fastest available safe loader."""
def __init__(self, stream: Any, secrets: Secrets | None = None) -> None:
"""Initialize a safe line loader."""
self.stream = stream
if isinstance(stream, str):
self.name = "<unicode string>"
elif isinstance(stream, bytes):
self.name = "<byte string>"
else:
self.name = getattr(stream, "name", "<file>")
super().__init__(stream)
self.secrets = secrets
def get_name(self) -> str:
"""Get the name of the loader."""
return self.name
def get_stream_name(self) -> str:
"""Get the name of the stream."""
return self.stream.name or ""
class SafeLineLoader(yaml.SafeLoader): class SafeLineLoader(yaml.SafeLoader):
"""Loader class that keeps track of line numbers.""" """Loader class that keeps track of line numbers."""
@ -103,6 +136,17 @@ class SafeLineLoader(yaml.SafeLoader):
node.__line__ = last_line + 1 # type: ignore[attr-defined] node.__line__ = last_line + 1 # type: ignore[attr-defined]
return node return node
def get_name(self) -> str:
"""Get the name of the loader."""
return self.name
def get_stream_name(self) -> str:
"""Get the name of the stream."""
return self.stream.name or ""
LoaderType = Union[SafeLineLoader, SafeLoader]
def load_yaml(fname: str, secrets: Secrets | None = None) -> JSON_TYPE: def load_yaml(fname: str, secrets: Secrets | None = None) -> JSON_TYPE:
"""Load a YAML file.""" """Load a YAML file."""
@ -114,60 +158,90 @@ def load_yaml(fname: str, secrets: Secrets | None = None) -> JSON_TYPE:
raise HomeAssistantError(exc) from exc raise HomeAssistantError(exc) from exc
def parse_yaml(content: str | TextIO, secrets: Secrets | None = None) -> JSON_TYPE: def parse_yaml(
"""Load a YAML file.""" content: str | TextIO | StringIO, secrets: Secrets | None = None
) -> JSON_TYPE:
"""Parse YAML with the fastest available loader."""
if not HAS_C_LOADER:
return _parse_yaml_pure_python(content, secrets)
try: try:
# If configuration file is empty YAML returns None return _parse_yaml(SafeLoader, content, secrets)
# We convert that to an empty dict except yaml.YAMLError:
return ( # Loading failed, so we now load with the slow line loader
yaml.load(content, Loader=lambda stream: SafeLineLoader(stream, secrets)) # since the C one will not give us line numbers
or OrderedDict() if isinstance(content, (StringIO, TextIO)):
) # Rewind the stream so we can try again
content.seek(0, 0)
return _parse_yaml_pure_python(content, secrets)
def _parse_yaml_pure_python(
content: str | TextIO | StringIO, secrets: Secrets | None = None
) -> JSON_TYPE:
"""Parse YAML with the pure python loader (this is very slow)."""
try:
return _parse_yaml(SafeLineLoader, content, secrets)
except yaml.YAMLError as exc: except yaml.YAMLError as exc:
_LOGGER.error(str(exc)) _LOGGER.error(str(exc))
raise HomeAssistantError(exc) from exc raise HomeAssistantError(exc) from exc
def _parse_yaml(
loader: type[SafeLoader] | type[SafeLineLoader],
content: str | TextIO,
secrets: Secrets | None = None,
) -> JSON_TYPE:
"""Load a YAML file."""
# If configuration file is empty YAML returns None
# We convert that to an empty dict
return (
yaml.load(content, Loader=lambda stream: loader(stream, secrets))
or OrderedDict()
)
@overload @overload
def _add_reference( def _add_reference(
obj: list | NodeListClass, loader: SafeLineLoader, node: yaml.nodes.Node obj: list | NodeListClass,
loader: LoaderType,
node: yaml.nodes.Node,
) -> NodeListClass: ) -> NodeListClass:
... ...
@overload @overload
def _add_reference( def _add_reference(
obj: str | NodeStrClass, loader: SafeLineLoader, node: yaml.nodes.Node obj: str | NodeStrClass,
loader: LoaderType,
node: yaml.nodes.Node,
) -> NodeStrClass: ) -> NodeStrClass:
... ...
@overload @overload
def _add_reference( def _add_reference(obj: _DictT, loader: LoaderType, node: yaml.nodes.Node) -> _DictT:
obj: _DictT, loader: SafeLineLoader, node: yaml.nodes.Node
) -> _DictT:
... ...
def _add_reference(obj, loader: SafeLineLoader, node: yaml.nodes.Node): # type: ignore[no-untyped-def] def _add_reference(obj, loader: LoaderType, node: yaml.nodes.Node): # type: ignore[no-untyped-def]
"""Add file reference information to an object.""" """Add file reference information to an object."""
if isinstance(obj, list): if isinstance(obj, list):
obj = NodeListClass(obj) obj = NodeListClass(obj)
if isinstance(obj, str): if isinstance(obj, str):
obj = NodeStrClass(obj) obj = NodeStrClass(obj)
setattr(obj, "__config_file__", loader.name) setattr(obj, "__config_file__", loader.get_name())
setattr(obj, "__line__", node.start_mark.line) setattr(obj, "__line__", node.start_mark.line)
return obj return obj
def _include_yaml(loader: SafeLineLoader, node: yaml.nodes.Node) -> JSON_TYPE: def _include_yaml(loader: LoaderType, node: yaml.nodes.Node) -> JSON_TYPE:
"""Load another YAML file and embeds it using the !include tag. """Load another YAML file and embeds it using the !include tag.
Example: Example:
device_tracker: !include device_tracker.yaml device_tracker: !include device_tracker.yaml
""" """
fname = os.path.join(os.path.dirname(loader.name), node.value) fname = os.path.join(os.path.dirname(loader.get_name()), node.value)
try: try:
return _add_reference(load_yaml(fname, loader.secrets), loader, node) return _add_reference(load_yaml(fname, loader.secrets), loader, node)
except FileNotFoundError as exc: except FileNotFoundError as exc:
@ -191,12 +265,10 @@ def _find_files(directory: str, pattern: str) -> Iterator[str]:
yield filename yield filename
def _include_dir_named_yaml( def _include_dir_named_yaml(loader: LoaderType, node: yaml.nodes.Node) -> OrderedDict:
loader: SafeLineLoader, node: yaml.nodes.Node
) -> OrderedDict:
"""Load multiple files from directory as a dictionary.""" """Load multiple files from directory as a dictionary."""
mapping: OrderedDict = OrderedDict() mapping: OrderedDict = OrderedDict()
loc = os.path.join(os.path.dirname(loader.name), node.value) loc = os.path.join(os.path.dirname(loader.get_name()), node.value)
for fname in _find_files(loc, "*.yaml"): for fname in _find_files(loc, "*.yaml"):
filename = os.path.splitext(os.path.basename(fname))[0] filename = os.path.splitext(os.path.basename(fname))[0]
if os.path.basename(fname) == SECRET_YAML: if os.path.basename(fname) == SECRET_YAML:
@ -206,11 +278,11 @@ def _include_dir_named_yaml(
def _include_dir_merge_named_yaml( def _include_dir_merge_named_yaml(
loader: SafeLineLoader, node: yaml.nodes.Node loader: LoaderType, node: yaml.nodes.Node
) -> OrderedDict: ) -> OrderedDict:
"""Load multiple files from directory as a merged dictionary.""" """Load multiple files from directory as a merged dictionary."""
mapping: OrderedDict = OrderedDict() mapping: OrderedDict = OrderedDict()
loc = os.path.join(os.path.dirname(loader.name), node.value) loc = os.path.join(os.path.dirname(loader.get_name()), node.value)
for fname in _find_files(loc, "*.yaml"): for fname in _find_files(loc, "*.yaml"):
if os.path.basename(fname) == SECRET_YAML: if os.path.basename(fname) == SECRET_YAML:
continue continue
@ -221,10 +293,10 @@ def _include_dir_merge_named_yaml(
def _include_dir_list_yaml( def _include_dir_list_yaml(
loader: SafeLineLoader, node: yaml.nodes.Node loader: LoaderType, node: yaml.nodes.Node
) -> list[JSON_TYPE]: ) -> list[JSON_TYPE]:
"""Load multiple files from directory as a list.""" """Load multiple files from directory as a list."""
loc = os.path.join(os.path.dirname(loader.name), node.value) loc = os.path.join(os.path.dirname(loader.get_name()), node.value)
return [ return [
load_yaml(f, loader.secrets) load_yaml(f, loader.secrets)
for f in _find_files(loc, "*.yaml") for f in _find_files(loc, "*.yaml")
@ -233,10 +305,10 @@ def _include_dir_list_yaml(
def _include_dir_merge_list_yaml( def _include_dir_merge_list_yaml(
loader: SafeLineLoader, node: yaml.nodes.Node loader: LoaderType, node: yaml.nodes.Node
) -> JSON_TYPE: ) -> JSON_TYPE:
"""Load multiple files from directory as a merged list.""" """Load multiple files from directory as a merged list."""
loc: str = os.path.join(os.path.dirname(loader.name), node.value) loc: str = os.path.join(os.path.dirname(loader.get_name()), node.value)
merged_list: list[JSON_TYPE] = [] merged_list: list[JSON_TYPE] = []
for fname in _find_files(loc, "*.yaml"): for fname in _find_files(loc, "*.yaml"):
if os.path.basename(fname) == SECRET_YAML: if os.path.basename(fname) == SECRET_YAML:
@ -247,7 +319,7 @@ def _include_dir_merge_list_yaml(
return _add_reference(merged_list, loader, node) return _add_reference(merged_list, loader, node)
def _ordered_dict(loader: SafeLineLoader, node: yaml.nodes.MappingNode) -> OrderedDict: def _ordered_dict(loader: LoaderType, node: yaml.nodes.MappingNode) -> OrderedDict:
"""Load YAML mappings into an ordered dictionary to preserve key order.""" """Load YAML mappings into an ordered dictionary to preserve key order."""
loader.flatten_mapping(node) loader.flatten_mapping(node)
nodes = loader.construct_pairs(node) nodes = loader.construct_pairs(node)
@ -259,14 +331,14 @@ def _ordered_dict(loader: SafeLineLoader, node: yaml.nodes.MappingNode) -> Order
try: try:
hash(key) hash(key)
except TypeError as exc: except TypeError as exc:
fname = getattr(loader.stream, "name", "") fname = loader.get_stream_name()
raise yaml.MarkedYAMLError( raise yaml.MarkedYAMLError(
context=f'invalid key: "{key}"', context=f'invalid key: "{key}"',
context_mark=yaml.Mark(fname, 0, line, -1, None, None), # type: ignore[arg-type] context_mark=yaml.Mark(fname, 0, line, -1, None, None), # type: ignore[arg-type]
) from exc ) from exc
if key in seen: if key in seen:
fname = getattr(loader.stream, "name", "") fname = loader.get_stream_name()
_LOGGER.warning( _LOGGER.warning(
'YAML file %s contains duplicate key "%s". Check lines %d and %d', 'YAML file %s contains duplicate key "%s". Check lines %d and %d',
fname, fname,
@ -279,13 +351,13 @@ def _ordered_dict(loader: SafeLineLoader, node: yaml.nodes.MappingNode) -> Order
return _add_reference(OrderedDict(nodes), loader, node) return _add_reference(OrderedDict(nodes), loader, node)
def _construct_seq(loader: SafeLineLoader, node: yaml.nodes.Node) -> JSON_TYPE: def _construct_seq(loader: LoaderType, node: yaml.nodes.Node) -> JSON_TYPE:
"""Add line number and file name to Load YAML sequence.""" """Add line number and file name to Load YAML sequence."""
(obj,) = loader.construct_yaml_seq(node) (obj,) = loader.construct_yaml_seq(node)
return _add_reference(obj, loader, node) return _add_reference(obj, loader, node)
def _env_var_yaml(loader: SafeLineLoader, node: yaml.nodes.Node) -> str: def _env_var_yaml(loader: LoaderType, node: yaml.nodes.Node) -> str:
"""Load environment variables and embed it into the configuration YAML.""" """Load environment variables and embed it into the configuration YAML."""
args = node.value.split() args = node.value.split()
@ -298,27 +370,27 @@ def _env_var_yaml(loader: SafeLineLoader, node: yaml.nodes.Node) -> str:
raise HomeAssistantError(node.value) raise HomeAssistantError(node.value)
def secret_yaml(loader: SafeLineLoader, node: yaml.nodes.Node) -> JSON_TYPE: def secret_yaml(loader: LoaderType, node: yaml.nodes.Node) -> JSON_TYPE:
"""Load secrets and embed it into the configuration YAML.""" """Load secrets and embed it into the configuration YAML."""
if loader.secrets is None: if loader.secrets is None:
raise HomeAssistantError("Secrets not supported in this YAML file") raise HomeAssistantError("Secrets not supported in this YAML file")
return loader.secrets.get(loader.name, node.value) return loader.secrets.get(loader.get_name(), node.value)
SafeLineLoader.add_constructor("!include", _include_yaml) def add_constructor(tag: Any, constructor: Any) -> None:
SafeLineLoader.add_constructor( """Add to constructor to all loaders."""
yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _ordered_dict for yaml_loader in (SafeLoader, SafeLineLoader):
) yaml_loader.add_constructor(tag, constructor)
SafeLineLoader.add_constructor(
yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG, _construct_seq
) add_constructor("!include", _include_yaml)
SafeLineLoader.add_constructor("!env_var", _env_var_yaml) add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _ordered_dict)
SafeLineLoader.add_constructor("!secret", secret_yaml) add_constructor(yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG, _construct_seq)
SafeLineLoader.add_constructor("!include_dir_list", _include_dir_list_yaml) add_constructor("!env_var", _env_var_yaml)
SafeLineLoader.add_constructor("!include_dir_merge_list", _include_dir_merge_list_yaml) add_constructor("!secret", secret_yaml)
SafeLineLoader.add_constructor("!include_dir_named", _include_dir_named_yaml) add_constructor("!include_dir_list", _include_dir_list_yaml)
SafeLineLoader.add_constructor( add_constructor("!include_dir_merge_list", _include_dir_merge_list_yaml)
"!include_dir_merge_named", _include_dir_merge_named_yaml add_constructor("!include_dir_named", _include_dir_named_yaml)
) add_constructor("!include_dir_merge_named", _include_dir_merge_named_yaml)
SafeLineLoader.add_constructor("!input", Input.from_node) add_constructor("!input", Input.from_node)

View File

@ -1,2 +1,2 @@
*!* NOT YAML -*!*- NOT YAML

View File

@ -1,6 +1,7 @@
"""Test config utils.""" """Test config utils."""
# pylint: disable=protected-access # pylint: disable=protected-access
from collections import OrderedDict from collections import OrderedDict
import contextlib
import copy import copy
import os import os
from unittest import mock from unittest import mock
@ -147,7 +148,7 @@ def test_load_yaml_config_raises_error_if_not_dict():
def test_load_yaml_config_raises_error_if_malformed_yaml(): def test_load_yaml_config_raises_error_if_malformed_yaml():
"""Test error raised if invalid YAML.""" """Test error raised if invalid YAML."""
with open(YAML_PATH, "w") as fp: with open(YAML_PATH, "w") as fp:
fp.write(":") fp.write(":-")
with pytest.raises(HomeAssistantError): with pytest.raises(HomeAssistantError):
config_util.load_yaml_config_file(YAML_PATH) config_util.load_yaml_config_file(YAML_PATH)
@ -156,11 +157,22 @@ def test_load_yaml_config_raises_error_if_malformed_yaml():
def test_load_yaml_config_raises_error_if_unsafe_yaml(): def test_load_yaml_config_raises_error_if_unsafe_yaml():
"""Test error raised if unsafe YAML.""" """Test error raised if unsafe YAML."""
with open(YAML_PATH, "w") as fp: with open(YAML_PATH, "w") as fp:
fp.write("hello: !!python/object/apply:os.system") fp.write("- !!python/object/apply:os.system []")
with pytest.raises(HomeAssistantError): with patch.object(os, "system") as system_mock, contextlib.suppress(
HomeAssistantError
):
config_util.load_yaml_config_file(YAML_PATH) config_util.load_yaml_config_file(YAML_PATH)
assert len(system_mock.mock_calls) == 0
# Here we validate that the test above is a good test
# since previously the syntax was not valid
with open(YAML_PATH) as fp, patch.object(os, "system") as system_mock:
list(yaml.unsafe_load_all(fp))
assert len(system_mock.mock_calls) == 1
def test_load_yaml_config_preserves_key_order(): def test_load_yaml_config_preserves_key_order():
"""Test removal of library.""" """Test removal of library."""

View File

@ -1,10 +1,12 @@
"""Test Home Assistant yaml loader.""" """Test Home Assistant yaml loader."""
import importlib
import io import io
import os import os
import unittest import unittest
from unittest.mock import patch from unittest.mock import patch
import pytest import pytest
import yaml as pyyaml
from homeassistant.config import YAML_CONFIG_FILE, load_yaml_config_file from homeassistant.config import YAML_CONFIG_FILE, load_yaml_config_file
from homeassistant.exceptions import HomeAssistantError from homeassistant.exceptions import HomeAssistantError
@ -14,7 +16,24 @@ from homeassistant.util.yaml import loader as yaml_loader
from tests.common import get_test_config_dir, patch_yaml_files from tests.common import get_test_config_dir, patch_yaml_files
def test_simple_list(): @pytest.fixture(params=["enable_c_loader", "disable_c_loader"])
def try_both_loaders(request):
"""Disable the yaml c loader."""
if not request.param == "disable_c_loader":
yield
return
try:
cloader = pyyaml.CSafeLoader
except ImportError:
return
del pyyaml.CSafeLoader
importlib.reload(yaml_loader)
yield
pyyaml.CSafeLoader = cloader
importlib.reload(yaml_loader)
def test_simple_list(try_both_loaders):
"""Test simple list.""" """Test simple list."""
conf = "config:\n - simple\n - list" conf = "config:\n - simple\n - list"
with io.StringIO(conf) as file: with io.StringIO(conf) as file:
@ -22,7 +41,7 @@ def test_simple_list():
assert doc["config"] == ["simple", "list"] assert doc["config"] == ["simple", "list"]
def test_simple_dict(): def test_simple_dict(try_both_loaders):
"""Test simple dict.""" """Test simple dict."""
conf = "key: value" conf = "key: value"
with io.StringIO(conf) as file: with io.StringIO(conf) as file:
@ -37,14 +56,14 @@ def test_unhashable_key():
load_yaml_config_file(YAML_CONFIG_FILE) load_yaml_config_file(YAML_CONFIG_FILE)
def test_no_key(): def test_no_key(try_both_loaders):
"""Test item without a key.""" """Test item without a key."""
files = {YAML_CONFIG_FILE: "a: a\nnokeyhere"} files = {YAML_CONFIG_FILE: "a: a\nnokeyhere"}
with pytest.raises(HomeAssistantError), patch_yaml_files(files): with pytest.raises(HomeAssistantError), patch_yaml_files(files):
yaml.load_yaml(YAML_CONFIG_FILE) yaml.load_yaml(YAML_CONFIG_FILE)
def test_environment_variable(): def test_environment_variable(try_both_loaders):
"""Test config file with environment variable.""" """Test config file with environment variable."""
os.environ["PASSWORD"] = "secret_password" os.environ["PASSWORD"] = "secret_password"
conf = "password: !env_var PASSWORD" conf = "password: !env_var PASSWORD"
@ -54,7 +73,7 @@ def test_environment_variable():
del os.environ["PASSWORD"] del os.environ["PASSWORD"]
def test_environment_variable_default(): def test_environment_variable_default(try_both_loaders):
"""Test config file with default value for environment variable.""" """Test config file with default value for environment variable."""
conf = "password: !env_var PASSWORD secret_password" conf = "password: !env_var PASSWORD secret_password"
with io.StringIO(conf) as file: with io.StringIO(conf) as file:
@ -62,14 +81,14 @@ def test_environment_variable_default():
assert doc["password"] == "secret_password" assert doc["password"] == "secret_password"
def test_invalid_environment_variable(): def test_invalid_environment_variable(try_both_loaders):
"""Test config file with no environment variable sat.""" """Test config file with no environment variable sat."""
conf = "password: !env_var PASSWORD" conf = "password: !env_var PASSWORD"
with pytest.raises(HomeAssistantError), io.StringIO(conf) as file: with pytest.raises(HomeAssistantError), io.StringIO(conf) as file:
yaml_loader.yaml.load(file, Loader=yaml_loader.SafeLineLoader) yaml_loader.yaml.load(file, Loader=yaml_loader.SafeLineLoader)
def test_include_yaml(): def test_include_yaml(try_both_loaders):
"""Test include yaml.""" """Test include yaml."""
with patch_yaml_files({"test.yaml": "value"}): with patch_yaml_files({"test.yaml": "value"}):
conf = "key: !include test.yaml" conf = "key: !include test.yaml"
@ -85,7 +104,7 @@ def test_include_yaml():
@patch("homeassistant.util.yaml.loader.os.walk") @patch("homeassistant.util.yaml.loader.os.walk")
def test_include_dir_list(mock_walk): def test_include_dir_list(mock_walk, try_both_loaders):
"""Test include dir list yaml.""" """Test include dir list yaml."""
mock_walk.return_value = [["/test", [], ["two.yaml", "one.yaml"]]] mock_walk.return_value = [["/test", [], ["two.yaml", "one.yaml"]]]
@ -97,7 +116,7 @@ def test_include_dir_list(mock_walk):
@patch("homeassistant.util.yaml.loader.os.walk") @patch("homeassistant.util.yaml.loader.os.walk")
def test_include_dir_list_recursive(mock_walk): def test_include_dir_list_recursive(mock_walk, try_both_loaders):
"""Test include dir recursive list yaml.""" """Test include dir recursive list yaml."""
mock_walk.return_value = [ mock_walk.return_value = [
["/test", ["tmp2", ".ignore", "ignore"], ["zero.yaml"]], ["/test", ["tmp2", ".ignore", "ignore"], ["zero.yaml"]],
@ -124,7 +143,7 @@ def test_include_dir_list_recursive(mock_walk):
@patch("homeassistant.util.yaml.loader.os.walk") @patch("homeassistant.util.yaml.loader.os.walk")
def test_include_dir_named(mock_walk): def test_include_dir_named(mock_walk, try_both_loaders):
"""Test include dir named yaml.""" """Test include dir named yaml."""
mock_walk.return_value = [ mock_walk.return_value = [
["/test", [], ["first.yaml", "second.yaml", "secrets.yaml"]] ["/test", [], ["first.yaml", "second.yaml", "secrets.yaml"]]
@ -139,7 +158,7 @@ def test_include_dir_named(mock_walk):
@patch("homeassistant.util.yaml.loader.os.walk") @patch("homeassistant.util.yaml.loader.os.walk")
def test_include_dir_named_recursive(mock_walk): def test_include_dir_named_recursive(mock_walk, try_both_loaders):
"""Test include dir named yaml.""" """Test include dir named yaml."""
mock_walk.return_value = [ mock_walk.return_value = [
["/test", ["tmp2", ".ignore", "ignore"], ["first.yaml"]], ["/test", ["tmp2", ".ignore", "ignore"], ["first.yaml"]],
@ -167,7 +186,7 @@ def test_include_dir_named_recursive(mock_walk):
@patch("homeassistant.util.yaml.loader.os.walk") @patch("homeassistant.util.yaml.loader.os.walk")
def test_include_dir_merge_list(mock_walk): def test_include_dir_merge_list(mock_walk, try_both_loaders):
"""Test include dir merge list yaml.""" """Test include dir merge list yaml."""
mock_walk.return_value = [["/test", [], ["first.yaml", "second.yaml"]]] mock_walk.return_value = [["/test", [], ["first.yaml", "second.yaml"]]]
@ -181,7 +200,7 @@ def test_include_dir_merge_list(mock_walk):
@patch("homeassistant.util.yaml.loader.os.walk") @patch("homeassistant.util.yaml.loader.os.walk")
def test_include_dir_merge_list_recursive(mock_walk): def test_include_dir_merge_list_recursive(mock_walk, try_both_loaders):
"""Test include dir merge list yaml.""" """Test include dir merge list yaml."""
mock_walk.return_value = [ mock_walk.return_value = [
["/test", ["tmp2", ".ignore", "ignore"], ["first.yaml"]], ["/test", ["tmp2", ".ignore", "ignore"], ["first.yaml"]],
@ -208,7 +227,7 @@ def test_include_dir_merge_list_recursive(mock_walk):
@patch("homeassistant.util.yaml.loader.os.walk") @patch("homeassistant.util.yaml.loader.os.walk")
def test_include_dir_merge_named(mock_walk): def test_include_dir_merge_named(mock_walk, try_both_loaders):
"""Test include dir merge named yaml.""" """Test include dir merge named yaml."""
mock_walk.return_value = [["/test", [], ["first.yaml", "second.yaml"]]] mock_walk.return_value = [["/test", [], ["first.yaml", "second.yaml"]]]
@ -225,7 +244,7 @@ def test_include_dir_merge_named(mock_walk):
@patch("homeassistant.util.yaml.loader.os.walk") @patch("homeassistant.util.yaml.loader.os.walk")
def test_include_dir_merge_named_recursive(mock_walk): def test_include_dir_merge_named_recursive(mock_walk, try_both_loaders):
"""Test include dir merge named yaml.""" """Test include dir merge named yaml."""
mock_walk.return_value = [ mock_walk.return_value = [
["/test", ["tmp2", ".ignore", "ignore"], ["first.yaml"]], ["/test", ["tmp2", ".ignore", "ignore"], ["first.yaml"]],
@ -257,7 +276,7 @@ def test_include_dir_merge_named_recursive(mock_walk):
@patch("homeassistant.util.yaml.loader.open", create=True) @patch("homeassistant.util.yaml.loader.open", create=True)
def test_load_yaml_encoding_error(mock_open): def test_load_yaml_encoding_error(mock_open, try_both_loaders):
"""Test raising a UnicodeDecodeError.""" """Test raising a UnicodeDecodeError."""
mock_open.side_effect = UnicodeDecodeError("", b"", 1, 0, "") mock_open.side_effect = UnicodeDecodeError("", b"", 1, 0, "")
with pytest.raises(HomeAssistantError): with pytest.raises(HomeAssistantError):
@ -413,7 +432,7 @@ def test_representing_yaml_loaded_data():
assert yaml.dump(data) == "key:\n- 1\n- '2'\n- 3\n" assert yaml.dump(data) == "key:\n- 1\n- '2'\n- 3\n"
def test_duplicate_key(caplog): def test_duplicate_key(caplog, try_both_loaders):
"""Test duplicate dict keys.""" """Test duplicate dict keys."""
files = {YAML_CONFIG_FILE: "key: thing1\nkey: thing2"} files = {YAML_CONFIG_FILE: "key: thing1\nkey: thing2"}
with patch_yaml_files(files): with patch_yaml_files(files):
@ -421,7 +440,7 @@ def test_duplicate_key(caplog):
assert "contains duplicate key" in caplog.text assert "contains duplicate key" in caplog.text
def test_no_recursive_secrets(caplog): def test_no_recursive_secrets(caplog, try_both_loaders):
"""Test that loading of secrets from the secrets file fails correctly.""" """Test that loading of secrets from the secrets file fails correctly."""
files = {YAML_CONFIG_FILE: "key: !secret a", yaml.SECRET_YAML: "a: 1\nb: !secret a"} files = {YAML_CONFIG_FILE: "key: !secret a", yaml.SECRET_YAML: "a: 1\nb: !secret a"}
with patch_yaml_files(files), pytest.raises(HomeAssistantError) as e: with patch_yaml_files(files), pytest.raises(HomeAssistantError) as e:
@ -441,7 +460,16 @@ def test_input_class():
assert len({input, input2}) == 1 assert len({input, input2}) == 1
def test_input(): def test_input(try_both_loaders):
"""Test loading inputs.""" """Test loading inputs."""
data = {"hello": yaml.Input("test_name")} data = {"hello": yaml.Input("test_name")}
assert yaml.parse_yaml(yaml.dump(data)) == data assert yaml.parse_yaml(yaml.dump(data)) == data
@pytest.mark.skipif(
not os.environ.get("HASS_CI"),
reason="This test validates that the CI has the C loader available",
)
def test_c_loader_is_available_in_ci():
"""Verify we are testing the C loader in the CI."""
assert yaml.loader.HAS_C_LOADER is True