Make typing checks more strict (#14429)

## Description:

Make typing checks more strict: add `--strict-optional` flag that forbids implicit None return type. This flag will become default in the next version of mypy (0.600)

Add `homeassistant/util/` to checked dirs.

## Checklist:
  - [x] The code change is tested and works locally.
  - [x] Local tests pass with `tox`. **Your PR cannot be merged unless tests pass**

homeassistant/__main__.py

@@ -241,7 +241,7 @@ def cmdline() -> List[str]:
 
 
 def setup_and_run_hass(config_dir: str,
-                       args: argparse.Namespace) -> Optional[int]:
+                       args: argparse.Namespace) -> int:
     """Set up HASS and run."""
     from homeassistant import bootstrap
 
@@ -274,7 +274,7 @@ def setup_and_run_hass(config_dir: str,
             log_no_color=args.log_no_color)
 
     if hass is None:
-        return None
+        return -1
 
     if args.open_ui:
         # Imported here to avoid importing asyncio before monkey patch

homeassistant/bootstrap.py

@@ -28,9 +28,8 @@ ERROR_LOG_FILENAME = 'home-assistant.log'
 # hass.data key for logging information.
 DATA_LOGGING = 'logging'
 
-FIRST_INIT_COMPONENT = set((
-    'system_log', 'recorder', 'mqtt', 'mqtt_eventstream', 'logger',
-    'introduction', 'frontend', 'history'))
+FIRST_INIT_COMPONENT = {'system_log', 'recorder', 'mqtt', 'mqtt_eventstream',
+                        'logger', 'introduction', 'frontend', 'history'}
 
 
 def from_config_dict(config: Dict[str, Any],
@@ -95,7 +94,8 @@ async def async_from_config_dict(config: Dict[str, Any],
         conf_util.async_log_exception(ex, 'homeassistant', core_config, hass)
         return None
 
-    await hass.async_add_job(conf_util.process_ha_config_upgrade, hass)
+    await hass.async_add_executor_job(
+        conf_util.process_ha_config_upgrade, hass)
 
     hass.config.skip_pip = skip_pip
     if skip_pip:
@@ -137,7 +137,7 @@ async def async_from_config_dict(config: Dict[str, Any],
     for component in components:
         if component not in FIRST_INIT_COMPONENT:
             continue
-        hass.async_add_job(async_setup_component(hass, component, config))
+        hass.async_create_task(async_setup_component(hass, component, config))
 
     await hass.async_block_till_done()
 
@@ -145,7 +145,7 @@ async def async_from_config_dict(config: Dict[str, Any],
     for component in components:
         if component in FIRST_INIT_COMPONENT:
             continue
-        hass.async_add_job(async_setup_component(hass, component, config))
+        hass.async_create_task(async_setup_component(hass, component, config))
 
     await hass.async_block_till_done()
 
@@ -162,7 +162,8 @@ def from_config_file(config_path: str,
                      skip_pip: bool = True,
                      log_rotate_days: Any = None,
                      log_file: Any = None,
-                     log_no_color: bool = False):
+                     log_no_color: bool = False)\
+        -> Optional[core.HomeAssistant]:
     """Read the configuration file and try to start all the functionality.
 
     Will add functionality to 'hass' parameter if given,
@@ -187,7 +188,8 @@ async def async_from_config_file(config_path: str,
                                  skip_pip: bool = True,
                                  log_rotate_days: Any = None,
                                  log_file: Any = None,
-                                 log_no_color: bool = False):
+                                 log_no_color: bool = False)\
+        -> Optional[core.HomeAssistant]:
     """Read the configuration file and try to start all the functionality.
 
     Will add functionality to 'hass' parameter.
@@ -204,7 +206,7 @@ async def async_from_config_file(config_path: str,
                          log_no_color)
 
     try:
-        config_dict = await hass.async_add_job(
+        config_dict = await hass.async_add_executor_job(
             conf_util.load_yaml_config_file, config_path)
     except HomeAssistantError as err:
         _LOGGER.error("Error loading %s: %s", config_path, err)

homeassistant/components/logbook.py

@@ -83,7 +83,7 @@ def async_log_entry(hass, name, message, domain=None, entity_id=None):
     hass.bus.async_fire(EVENT_LOGBOOK_ENTRY, data)
 
 
-async def setup(hass, config):
+async def async_setup(hass, config):
     """Listen for download events to download files."""
     @callback
     def log_message(service):

homeassistant/components/panel_iframe.py

@@ -4,8 +4,6 @@ Register an iFrame front end panel.
 For more details about this component, please refer to the documentation at
 https://home-assistant.io/components/panel_iframe/
 """
-import asyncio
-
 import voluptuous as vol
 
 from homeassistant.const import (CONF_ICON, CONF_URL)
@@ -34,11 +32,10 @@ CONFIG_SCHEMA = vol.Schema({
         }})}, extra=vol.ALLOW_EXTRA)
 
 
-@asyncio.coroutine
-def setup(hass, config):
+async def async_setup(hass, config):
     """Set up the iFrame frontend panels."""
     for url_path, info in config[DOMAIN].items():
-        yield from hass.components.frontend.async_register_built_in_panel(
+        await hass.components.frontend.async_register_built_in_panel(
             'iframe', info.get(CONF_TITLE), info.get(CONF_ICON),
             url_path, {'url': info[CONF_URL]})
 

homeassistant/core.py

@@ -17,7 +17,8 @@ import threading
 from time import monotonic
 
 from types import MappingProxyType
-from typing import Optional, Any, Callable, List, TypeVar, Dict  # NOQA
+from typing import (  # NOQA
+    Optional, Any, Callable, List, TypeVar, Dict, Coroutine)
 
 from async_timeout import timeout
 import voluptuous as vol
@@ -205,8 +206,8 @@ class HomeAssistant(object):
     def async_add_job(
             self,
             target: Callable[..., Any],
-            *args: Any) -> Optional[asyncio.tasks.Task]:
-        """Add a job from within the eventloop.
+            *args: Any) -> Optional[asyncio.Future]:
+        """Add a job from within the event loop.
 
         This method must be run in the event loop.
 
@@ -230,11 +231,26 @@ class HomeAssistant(object):
 
         return task
 
+    @callback
+    def async_create_task(self, target: Coroutine) -> asyncio.tasks.Task:
+        """Create a task from within the eventloop.
+
+        This method must be run in the event loop.
+
+        target: target to call.
+        """
+        task = self.loop.create_task(target)
+
+        if self._track_task:
+            self._pending_tasks.append(task)
+
+        return task
+
     @callback
     def async_add_executor_job(
             self,
             target: Callable[..., Any],
-            *args: Any) -> asyncio.tasks.Task:
+            *args: Any) -> asyncio.Future:
         """Add an executor job from within the event loop."""
         task = self.loop.run_in_executor(None, target, *args)
 

homeassistant/helpers/storage.py

@@ -80,11 +80,10 @@ class Store:
             data = self._data
         else:
             data = await self.hass.async_add_executor_job(
-                json.load_json, self.path, None)
+                json.load_json, self.path)
 
-            if data is None:
+            if data == {}:
                 return None
-
         if data['version'] == self.version:
             stored = data['data']
         else:

homeassistant/loader.py

@@ -16,14 +16,20 @@ import logging
 import sys
 from types import ModuleType
 
-from typing import Optional, Set
+# pylint: disable=unused-import
+from typing import Dict, List, Optional, Sequence, Set, TYPE_CHECKING  # NOQA
 
 from homeassistant.const import PLATFORM_FORMAT
 from homeassistant.util import OrderedSet
 
+# Typing imports that create a circular dependency
+# pylint: disable=using-constant-test,unused-import
+if TYPE_CHECKING:
+    from homeassistant.core import HomeAssistant  # NOQA
+
 PREPARED = False
 
-DEPENDENCY_BLACKLIST = set(('config',))
+DEPENDENCY_BLACKLIST = {'config'}
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -33,7 +39,8 @@ PATH_CUSTOM_COMPONENTS = 'custom_components'
 PACKAGE_COMPONENTS = 'homeassistant.components'
 
 
-def set_component(hass, comp_name: str, component: ModuleType) -> None:
+def set_component(hass,  # type: HomeAssistant
+                  comp_name: str, component: Optional[ModuleType]) -> None:
     """Set a component in the cache.
 
     Async friendly.

homeassistant/setup.py

@@ -50,7 +50,7 @@ async def async_setup_component(hass: core.HomeAssistant, domain: str,
     if setup_tasks is None:
         setup_tasks = hass.data[DATA_SETUP] = {}
 
-    task = setup_tasks[domain] = hass.async_add_job(
+    task = setup_tasks[domain] = hass.async_create_task(
         _async_setup_component(hass, domain, config))
 
     return await task
@@ -142,7 +142,7 @@ async def _async_setup_component(hass: core.HomeAssistant,
             result = await component.async_setup(  # type: ignore
                 hass, processed_config)
         else:
-            result = await hass.async_add_job(
+            result = await hass.async_add_executor_job(
                 component.setup, hass, processed_config)  # type: ignore
     except Exception:  # pylint: disable=broad-except
         _LOGGER.exception("Error during setup of component %s", domain)

homeassistant/util/color.py

@@ -267,8 +267,8 @@ def color_xy_brightness_to_RGB(vX: float, vY: float,
 def color_hsb_to_RGB(fH: float, fS: float, fB: float) -> Tuple[int, int, int]:
     """Convert a hsb into its rgb representation."""
     if fS == 0:
-        fV = fB * 255
-        return (fV, fV, fV)
+        fV = int(fB * 255)
+        return fV, fV, fV
 
     r = g = b = 0
     h = fH / 60

homeassistant/util/dt.py

@@ -6,9 +6,11 @@ import re
 from typing import Any, Dict, Union, Optional, Tuple  # NOQA
 
 import pytz
+import pytz.exceptions as pytzexceptions
 
 DATE_STR_FORMAT = "%Y-%m-%d"
-UTC = DEFAULT_TIME_ZONE = pytz.utc  # type: dt.tzinfo
+UTC = pytz.utc
+DEFAULT_TIME_ZONE = pytz.utc  # type: dt.tzinfo
 
 
 # Copyright (c) Django Software Foundation and individual contributors.
@@ -42,7 +44,7 @@ def get_time_zone(time_zone_str: str) -> Optional[dt.tzinfo]:
     """
     try:
         return pytz.timezone(time_zone_str)
-    except pytz.exceptions.UnknownTimeZoneError:
+    except pytzexceptions.UnknownTimeZoneError:
         return None
 
 
@@ -64,7 +66,7 @@ def as_utc(dattim: dt.datetime) -> dt.datetime:
     if dattim.tzinfo == UTC:
         return dattim
     elif dattim.tzinfo is None:
-        dattim = DEFAULT_TIME_ZONE.localize(dattim)
+        dattim = DEFAULT_TIME_ZONE.localize(dattim)  # type: ignore
 
     return dattim.astimezone(UTC)
 
@@ -92,7 +94,7 @@ def as_local(dattim: dt.datetime) -> dt.datetime:
 
 def utc_from_timestamp(timestamp: float) -> dt.datetime:
     """Return a UTC time from a timestamp."""
-    return dt.datetime.utcfromtimestamp(timestamp).replace(tzinfo=UTC)
+    return UTC.localize(dt.datetime.utcfromtimestamp(timestamp))
 
 
 def start_of_local_day(dt_or_d:
@@ -102,13 +104,14 @@ def start_of_local_day(dt_or_d:
         date = now().date()  # type: dt.date
     elif isinstance(dt_or_d, dt.datetime):
         date = dt_or_d.date()
-    return DEFAULT_TIME_ZONE.localize(dt.datetime.combine(date, dt.time()))
+    return DEFAULT_TIME_ZONE.localize(dt.datetime.combine(  # type: ignore
+        date, dt.time()))
 
 
 # Copyright (c) Django Software Foundation and individual contributors.
 # All rights reserved.
 # https://github.com/django/django/blob/master/LICENSE
-def parse_datetime(dt_str: str) -> dt.datetime:
+def parse_datetime(dt_str: str) -> Optional[dt.datetime]:
     """Parse a string and return a datetime.datetime.
 
     This function supports time zone offsets. When the input contains one,
@@ -134,14 +137,12 @@ def parse_datetime(dt_str: str) -> dt.datetime:
         if tzinfo_str[0] == '-':
             offset = -offset
         tzinfo = dt.timezone(offset)
-    else:
-        tzinfo = None
     kws = {k: int(v) for k, v in kws.items() if v is not None}
     kws['tzinfo'] = tzinfo
     return dt.datetime(**kws)
 
 
-def parse_date(dt_str: str) -> dt.date:
+def parse_date(dt_str: str) -> Optional[dt.date]:
     """Convert a date string to a date object."""
     try:
         return dt.datetime.strptime(dt_str, DATE_STR_FORMAT).date()
@@ -180,9 +181,8 @@ def get_age(date: dt.datetime) -> str:
     def formatn(number: int, unit: str) -> str:
         """Add "unit" if it's plural."""
         if number == 1:
-            return "1 %s" % unit
-        elif number > 1:
-            return "%d %ss" % (number, unit)
+            return '1 {}'.format(unit)
+        return '{:d} {}s'.format(number, unit)
 
     def q_n_r(first: int, second: int) -> Tuple[int, int]:
         """Return quotient and remaining."""
@@ -210,4 +210,4 @@ def get_age(date: dt.datetime) -> str:
     if minute > 0:
         return formatn(minute, 'minute')
 
-    return formatn(second, 'second') if second > 0 else "0 seconds"
+    return formatn(second, 'second')

homeassistant/util/json.py

@@ -8,8 +8,6 @@ from homeassistant.exceptions import HomeAssistantError
 
 _LOGGER = logging.getLogger(__name__)
 
-_UNDEFINED = object()
-
 
 class SerializationError(HomeAssistantError):
     """Error serializing the data to JSON."""
@@ -19,7 +17,7 @@ class WriteError(HomeAssistantError):
     """Error writing the data."""
 
 
-def load_json(filename: str, default: Union[List, Dict] = _UNDEFINED) \
+def load_json(filename: str, default: Union[List, Dict, None] = None) \
         -> Union[List, Dict]:
     """Load JSON data from a file and return as dict or list.
 
@@ -37,7 +35,7 @@ def load_json(filename: str, default: Union[List, Dict] = _UNDEFINED) \
     except OSError as error:
         _LOGGER.exception('JSON file reading failed: %s', filename)
         raise HomeAssistantError(error)
-    return {} if default is _UNDEFINED else default
+    return {} if default is None else default
 
 
 def save_json(filename: str, data: Union[List, Dict]):
@@ -46,9 +44,9 @@ def save_json(filename: str, data: Union[List, Dict]):
     Returns True on success.
     """
     try:
-        data = json.dumps(data, sort_keys=True, indent=4)
+        json_data = json.dumps(data, sort_keys=True, indent=4)
         with open(filename, 'w', encoding='utf-8') as fdesc:
-            fdesc.write(data)
+            fdesc.write(json_data)
     except TypeError as error:
         _LOGGER.exception('Failed to serialize to JSON: %s',
                           filename)

homeassistant/util/unit_system.py

@@ -86,11 +86,11 @@ class UnitSystem(object):
         self.volume_unit = volume
 
     @property
-    def is_metric(self: object) -> bool:
+    def is_metric(self) -> bool:
         """Determine if this is the metric unit system."""
         return self.name == CONF_UNIT_SYSTEM_METRIC
 
-    def temperature(self: object, temperature: float, from_unit: str) -> float:
+    def temperature(self, temperature: float, from_unit: str) -> float:
         """Convert the given temperature to this unit system."""
         if not isinstance(temperature, Number):
             raise TypeError(
@@ -99,7 +99,7 @@ class UnitSystem(object):
         return temperature_util.convert(temperature,
                                         from_unit, self.temperature_unit)
 
-    def length(self: object, length: float, from_unit: str) -> float:
+    def length(self, length: float, from_unit: str) -> float:
         """Convert the given length to this unit system."""
         if not isinstance(length, Number):
             raise TypeError('{} is not a numeric value.'.format(str(length)))

homeassistant/util/yaml.py

@@ -57,7 +57,7 @@ class SafeLineLoader(yaml.SafeLoader):
         last_line = self.line  # type: int
         node = super(SafeLineLoader,
                      self).compose_node(parent, index)  # type: yaml.nodes.Node
-        node.__line__ = last_line + 1
+        node.__line__ = last_line + 1  # type: ignore
         return node
 
 
@@ -69,7 +69,7 @@ def load_yaml(fname: str) -> Union[List, Dict]:
             # We convert that to an empty dict
             return yaml.load(conf_file, Loader=SafeLineLoader) or OrderedDict()
     except yaml.YAMLError as exc:
-        _LOGGER.error(exc)
+        _LOGGER.error(str(exc))
         raise HomeAssistantError(exc)
     except UnicodeDecodeError as exc:
         _LOGGER.error("Unable to read file %s: %s", fname, exc)
@@ -232,6 +232,8 @@ def _load_secret_yaml(secret_path: str) -> Dict:
     _LOGGER.debug('Loading %s', secret_path)
     try:
         secrets = load_yaml(secret_path)
+        if not isinstance(secrets, dict):
+            raise HomeAssistantError('Secrets is not a dictionary')
         if 'logger' in secrets:
             logger = str(secrets['logger']).lower()
             if logger == 'debug':

tests/test_bootstrap.py

@@ -81,7 +81,8 @@ def test_from_config_dict_not_mount_deps_folder(loop):
 
 async def test_async_from_config_file_not_mount_deps_folder(loop):
     """Test that we not mount the deps folder inside async_from_config_file."""
-    hass = Mock(async_add_job=Mock(side_effect=lambda *args: mock_coro()))
+    hass = Mock(
+        async_add_executor_job=Mock(side_effect=lambda *args: mock_coro()))
 
     with patch('homeassistant.bootstrap.is_virtual_env', return_value=False), \
         patch('homeassistant.bootstrap.async_enable_logging',

tests/test_core.py

@@ -67,6 +67,18 @@ def test_async_add_job_add_threaded_job_to_pool(mock_iscoro):
     assert len(hass.loop.run_in_executor.mock_calls) == 1
 
 
+@patch('asyncio.iscoroutine', return_value=True)
+def test_async_create_task_schedule_coroutine(mock_iscoro):
+    """Test that we schedule coroutines and add jobs to the job pool."""
+    hass = MagicMock()
+    job = MagicMock()
+
+    ha.HomeAssistant.async_create_task(hass, job)
+    assert len(hass.loop.call_soon.mock_calls) == 0
+    assert len(hass.loop.create_task.mock_calls) == 1
+    assert len(hass.add_job.mock_calls) == 0
+
+
 def test_async_run_job_calls_callback():
     """Test that the callback annotation is respected."""
     hass = MagicMock()

tests/util/test_yaml.py

@@ -411,6 +411,22 @@ class TestSecrets(unittest.TestCase):
         assert mock_error.call_count == 1, \
             "Expected an error about logger: value"
 
+    def test_secrets_are_not_dict(self):
+        """Did secrets handle non-dict file."""
+        FILES[self._secret_path] = (
+                  '- http_pw: pwhttp\n'
+                  '  comp1_un: un1\n'
+                  '  comp1_pw: pw1\n')
+        yaml.clear_secret_cache()
+        with self.assertRaises(HomeAssistantError):
+            load_yaml(self._yaml_path,
+                      'http:\n'
+                      '  api_password: !secret http_pw\n'
+                      'component:\n'
+                      '  username: !secret comp1_un\n'
+                      '  password: !secret comp1_pw\n'
+                      '')
+
 
 def test_representing_yaml_loaded_data():
     """Test we can represent YAML loaded data."""

tox.ini

@@ -42,4 +42,4 @@ whitelist_externals=/bin/bash
 deps =
      -r{toxinidir}/requirements_test.txt
 commands =
-         /bin/bash -c 'mypy --ignore-missing-imports --follow-imports=silent homeassistant/*.py'
+         /bin/bash -c 'mypy --ignore-missing-imports --follow-imports=silent --strict-optional --warn-unused-ignores homeassistant/*.py homeassistant/util/'