From aa8622f8e8471890b8b9daeeff2f98c12581ca81 Mon Sep 17 00:00:00 2001 From: wokar Date: Thu, 13 Oct 2016 17:54:45 +0200 Subject: [PATCH] Added include and exclude functionality to history component (#3674) * added include and exclude functionality to history component * fixed summary lines in test method doc. * cleanup of query filter creation * o improved config validation o move move IGNORE_DOMAINS to Filter.apply() o removed config from Last5StatesView o Filters instance is now created on setup o config values are processed in setup and set to the Filters instance o function _set_filters_in_query() moved to Filters class and renamed to apply() * added unittests for more include/exclude filter combinations * make pylint happy --- homeassistant/components/history.py | 153 ++++++++++++++--- tests/components/test_history.py | 250 +++++++++++++++++++++++++++- 2 files changed, 371 insertions(+), 32 deletions(-) diff --git a/homeassistant/components/history.py b/homeassistant/components/history.py index 9a09f56c474..4cebf637c16 100644 --- a/homeassistant/components/history.py +++ b/homeassistant/components/history.py @@ -7,16 +7,39 @@ https://home-assistant.io/components/history/ from collections import defaultdict from datetime import timedelta from itertools import groupby +import voluptuous as vol +import homeassistant.helpers.config_validation as cv import homeassistant.util.dt as dt_util from homeassistant.components import recorder, script from homeassistant.components.frontend import register_built_in_panel from homeassistant.components.http import HomeAssistantView +from homeassistant.const import ATTR_HIDDEN DOMAIN = 'history' DEPENDENCIES = ['recorder', 'http'] -SIGNIFICANT_DOMAINS = ('thermostat',) +CONF_EXCLUDE = 'exclude' +CONF_INCLUDE = 'include' +CONF_ENTITIES = 'entities' +CONF_DOMAINS = 'domains' + +CONFIG_SCHEMA = vol.Schema({ + DOMAIN: vol.Schema({ + CONF_EXCLUDE: vol.Schema({ + vol.Optional(CONF_ENTITIES, default=[]): cv.entity_ids, + vol.Optional(CONF_DOMAINS, default=[]): vol.All(cv.ensure_list, + [cv.string]) + }), + CONF_INCLUDE: vol.Schema({ + vol.Optional(CONF_ENTITIES, default=[]): cv.entity_ids, + vol.Optional(CONF_DOMAINS, default=[]): vol.All(cv.ensure_list, + [cv.string]) + }) + }), +}, extra=vol.ALLOW_EXTRA) + +SIGNIFICANT_DOMAINS = ('thermostat', 'climate') IGNORE_DOMAINS = ('zone', 'scene',) @@ -32,7 +55,8 @@ def last_5_states(entity_id): ).order_by(states.state_id.desc()).limit(5)) -def get_significant_states(start_time, end_time=None, entity_id=None): +def get_significant_states(start_time, end_time=None, entity_id=None, + filters=None): """ Return states changes during UTC period start_time - end_time. @@ -40,25 +64,25 @@ def get_significant_states(start_time, end_time=None, entity_id=None): as well as all states from certain domains (for instance thermostat so that we get current temperature in our graphs). """ + entity_ids = (entity_id.lower(), ) if entity_id is not None else None states = recorder.get_model('States') query = recorder.query('States').filter( (states.domain.in_(SIGNIFICANT_DOMAINS) | (states.last_changed == states.last_updated)) & - ((~states.domain.in_(IGNORE_DOMAINS)) & - (states.last_updated > start_time))) + (states.last_updated > start_time)) + if filters: + query = filters.apply(query, entity_ids) if end_time is not None: query = query.filter(states.last_updated < end_time) - if entity_id is not None: - query = query.filter_by(entity_id=entity_id.lower()) - states = ( state for state in recorder.execute( query.order_by(states.entity_id, states.last_updated)) - if _is_significant(state)) + if (_is_significant(state) and + not state.attributes.get(ATTR_HIDDEN, False))) - return states_to_json(states, start_time, entity_id) + return states_to_json(states, start_time, entity_id, filters) def state_changes_during_period(start_time, end_time=None, entity_id=None): @@ -80,7 +104,7 @@ def state_changes_during_period(start_time, end_time=None, entity_id=None): return states_to_json(states, start_time, entity_id) -def get_states(utc_point_in_time, entity_ids=None, run=None): +def get_states(utc_point_in_time, entity_ids=None, run=None, filters=None): """Return the states at a specific point in time.""" if run is None: run = recorder.run_information(utc_point_in_time) @@ -96,12 +120,11 @@ def get_states(utc_point_in_time, entity_ids=None, run=None): func.max(states.state_id).label('max_state_id') ).filter( (states.created >= run.start) & - (states.created < utc_point_in_time) - ) - - if entity_ids is not None: - most_recent_state_ids = most_recent_state_ids.filter( - states.entity_id.in_(entity_ids)) + (states.created < utc_point_in_time) & + (~states.domain.in_(IGNORE_DOMAINS))) + if filters: + most_recent_state_ids = filters.apply(most_recent_state_ids, + entity_ids) most_recent_state_ids = most_recent_state_ids.group_by( states.entity_id).subquery() @@ -109,10 +132,12 @@ def get_states(utc_point_in_time, entity_ids=None, run=None): query = recorder.query('States').join(most_recent_state_ids, and_( states.state_id == most_recent_state_ids.c.max_state_id)) - return recorder.execute(query) + for state in recorder.execute(query): + if not state.attributes.get(ATTR_HIDDEN, False): + yield state -def states_to_json(states, start_time, entity_id): +def states_to_json(states, start_time, entity_id, filters=None): """Convert SQL results into JSON friendly data structure. This takes our state list and turns it into a JSON friendly data @@ -127,7 +152,7 @@ def states_to_json(states, start_time, entity_id): entity_ids = [entity_id] if entity_id is not None else None # Get the states at the start time - for state in get_states(start_time, entity_ids): + for state in get_states(start_time, entity_ids, filters=filters): state.last_changed = start_time state.last_updated = start_time result[state.entity_id].append(state) @@ -140,16 +165,25 @@ def states_to_json(states, start_time, entity_id): def get_state(utc_point_in_time, entity_id, run=None): """Return a state at a specific point in time.""" - states = get_states(utc_point_in_time, (entity_id,), run) - + states = list(get_states(utc_point_in_time, (entity_id,), run)) return states[0] if states else None # pylint: disable=unused-argument def setup(hass, config): """Setup the history hooks.""" - hass.wsgi.register_view(Last5StatesView) - hass.wsgi.register_view(HistoryPeriodView) + filters = Filters() + exclude = config[DOMAIN].get(CONF_EXCLUDE) + if exclude: + filters.excluded_entities = exclude[CONF_ENTITIES] + filters.excluded_domains = exclude[CONF_DOMAINS] + include = config[DOMAIN].get(CONF_INCLUDE) + if include: + filters.included_entities = include[CONF_ENTITIES] + filters.included_domains = include[CONF_DOMAINS] + + hass.wsgi.register_view(Last5StatesView(hass)) + hass.wsgi.register_view(HistoryPeriodView(hass, filters)) register_built_in_panel(hass, 'history', 'History', 'mdi:poll-box') return True @@ -161,6 +195,10 @@ class Last5StatesView(HomeAssistantView): url = '/api/history/entity//recent_states' name = 'api:history:entity-recent-states' + def __init__(self, hass): + """Initilalize the history last 5 states view.""" + super().__init__(hass) + def get(self, request, entity_id): """Retrieve last 5 states of entity.""" return self.json(last_5_states(entity_id)) @@ -173,6 +211,11 @@ class HistoryPeriodView(HomeAssistantView): name = 'api:history:view-period' extra_urls = ['/api/history/period/'] + def __init__(self, hass, filters): + """Initilalize the history period view.""" + super().__init__(hass) + self.filters = filters + def get(self, request, datetime=None): """Return history over a period of time.""" one_day = timedelta(days=1) @@ -185,8 +228,68 @@ class HistoryPeriodView(HomeAssistantView): end_time = start_time + one_day entity_id = request.args.get('filter_entity_id') - return self.json( - get_significant_states(start_time, end_time, entity_id).values()) + return self.json(get_significant_states( + start_time, end_time, entity_id, self.filters).values()) + + +# pylint: disable=too-few-public-methods +class Filters(object): + """Container for the configured include and exclude filters.""" + + def __init__(self): + """Initialise the include and exclude filters.""" + self.excluded_entities = [] + self.excluded_domains = [] + self.included_entities = [] + self.included_domains = [] + + def apply(self, query, entity_ids=None): + """Apply the Include/exclude filter on domains and entities on query. + + Following rules apply: + * only the include section is configured - just query the specified + entities or domains. + * only the exclude section is configured - filter the specified + entities and domains from all the entities in the system. + * if include and exclude is defined - select the entities specified in + the include and filter out the ones from the exclude list. + """ + states = recorder.get_model('States') + # specific entities requested - do not in/exclude anything + if entity_ids is not None: + return query.filter(states.entity_id.in_(entity_ids)) + query = query.filter(~states.domain.in_(IGNORE_DOMAINS)) + + filter_query = None + # filter if only excluded domain is configured + if self.excluded_domains and not self.included_domains: + filter_query = ~states.domain.in_(self.excluded_domains) + if self.included_entities: + filter_query &= states.entity_id.in_(self.included_entities) + # filter if only included domain is configured + elif not self.excluded_domains and self.included_domains: + filter_query = states.domain.in_(self.included_domains) + if self.included_entities: + filter_query |= states.entity_id.in_(self.included_entities) + # filter if included and excluded domain is configured + elif self.excluded_domains and self.included_domains: + filter_query = ~states.domain.in_(self.excluded_domains) + if self.included_entities: + filter_query &= (states.domain.in_(self.included_domains) | + states.entity_id.in_(self.included_entities)) + else: + filter_query &= (states.domain.in_(self.included_domains) & + ~states.domain.in_(self.excluded_domains)) + # no domain filter just included entities + elif not self.excluded_domains and not self.included_domains and \ + self.included_entities: + filter_query = states.entity_id.in_(self.included_entities) + if filter_query is not None: + query = query.filter(filter_query) + # finally apply excluded entities filter if configured + if self.excluded_entities: + query = query.filter(~states.entity_id.in_(self.excluded_entities)) + return query def _is_significant(state): diff --git a/tests/components/test_history.py b/tests/components/test_history.py index 80d0b1e9f9d..520afed81d9 100644 --- a/tests/components/test_history.py +++ b/tests/components/test_history.py @@ -43,7 +43,15 @@ class TestComponentHistory(unittest.TestCase): def test_setup(self): """Test setup method of history.""" mock_http_component(self.hass) - self.assertTrue(setup_component(self.hass, history.DOMAIN, {})) + config = history.CONFIG_SCHEMA({ + ha.DOMAIN: {}, + history.DOMAIN: {history.CONF_INCLUDE: { + history.CONF_DOMAINS: ['media_player'], + history.CONF_ENTITIES: ['thermostat.test']}, + history.CONF_EXCLUDE: { + history.CONF_DOMAINS: ['thermostat'], + history.CONF_ENTITIES: ['media_player.test']}}}) + self.assertTrue(setup_component(self.hass, history.DOMAIN, config)) def test_last_5_states(self): """Test retrieving the last 5 states.""" @@ -145,14 +153,236 @@ class TestComponentHistory(unittest.TestCase): def test_get_significant_states(self): """Test that only significant states are returned. - We inject a bunch of state updates from media player, zone and - thermostat. We should get back every thermostat change that + We should get back every thermostat change that includes an attribute change, but only the state updates for media player (attribute changes are not significant and not returned). """ + zero, four, states = self.record_states() + hist = history.get_significant_states( + zero, four, filters=history.Filters()) + assert states == hist + + def test_get_significant_states_entity_id(self): + """Test that only significant states are returned for one entity.""" + zero, four, states = self.record_states() + del states['media_player.test2'] + del states['thermostat.test'] + del states['thermostat.test2'] + del states['script.can_cancel_this_one'] + + hist = history.get_significant_states( + zero, four, 'media_player.test', + filters=history.Filters()) + assert states == hist + + def test_get_significant_states_exclude_domain(self): + """Test if significant states are returned when excluding domains. + + We should get back every thermostat change that includes an attribute + change, but no media player changes. + """ + zero, four, states = self.record_states() + del states['media_player.test'] + del states['media_player.test2'] + + config = history.CONFIG_SCHEMA({ + ha.DOMAIN: {}, + history.DOMAIN: {history.CONF_EXCLUDE: { + history.CONF_DOMAINS: ['media_player', ]}}}) + self.check_significant_states(zero, four, states, config) + + def test_get_significant_states_exclude_entity(self): + """Test if significant states are returned when excluding entities. + + We should get back every thermostat and script changes, but no media + player changes. + """ + zero, four, states = self.record_states() + del states['media_player.test'] + + config = history.CONFIG_SCHEMA({ + ha.DOMAIN: {}, + history.DOMAIN: {history.CONF_EXCLUDE: { + history.CONF_ENTITIES: ['media_player.test', ]}}}) + self.check_significant_states(zero, four, states, config) + + def test_get_significant_states_exclude(self): + """Test significant states when excluding entities and domains. + + We should not get back every thermostat and media player test changes. + """ + zero, four, states = self.record_states() + del states['media_player.test'] + del states['thermostat.test'] + del states['thermostat.test2'] + + config = history.CONFIG_SCHEMA({ + ha.DOMAIN: {}, + history.DOMAIN: {history.CONF_EXCLUDE: { + history.CONF_DOMAINS: ['thermostat', ], + history.CONF_ENTITIES: ['media_player.test', ]}}}) + self.check_significant_states(zero, four, states, config) + + def test_get_significant_states_exclude_include_entity(self): + """Test significant states when excluding domains and include entities. + + We should not get back every thermostat and media player test changes. + """ + zero, four, states = self.record_states() + del states['media_player.test2'] + del states['thermostat.test'] + del states['thermostat.test2'] + del states['script.can_cancel_this_one'] + + config = history.CONFIG_SCHEMA({ + ha.DOMAIN: {}, + history.DOMAIN: { + history.CONF_INCLUDE: { + history.CONF_ENTITIES: ['media_player.test', + 'thermostat.test']}, + history.CONF_EXCLUDE: { + history.CONF_DOMAINS: ['thermostat']}}}) + self.check_significant_states(zero, four, states, config) + + def test_get_significant_states_include_domain(self): + """Test if significant states are returned when including domains. + + We should get back every thermostat and script changes, but no media + player changes. + """ + zero, four, states = self.record_states() + del states['media_player.test'] + del states['media_player.test2'] + + config = history.CONFIG_SCHEMA({ + ha.DOMAIN: {}, + history.DOMAIN: {history.CONF_INCLUDE: { + history.CONF_DOMAINS: ['thermostat', 'script']}}}) + self.check_significant_states(zero, four, states, config) + + def test_get_significant_states_include_entity(self): + """Test if significant states are returned when including entities. + + We should only get back changes of the media_player.test entity. + """ + zero, four, states = self.record_states() + del states['media_player.test2'] + del states['thermostat.test'] + del states['thermostat.test2'] + del states['script.can_cancel_this_one'] + + config = history.CONFIG_SCHEMA({ + ha.DOMAIN: {}, + history.DOMAIN: {history.CONF_INCLUDE: { + history.CONF_ENTITIES: ['media_player.test']}}}) + self.check_significant_states(zero, four, states, config) + + def test_get_significant_states_include(self): + """Test significant states when including domains and entities. + + We should only get back changes of the media_player.test entity and the + thermostat domain. + """ + zero, four, states = self.record_states() + del states['media_player.test2'] + del states['script.can_cancel_this_one'] + + config = history.CONFIG_SCHEMA({ + ha.DOMAIN: {}, + history.DOMAIN: {history.CONF_INCLUDE: { + history.CONF_DOMAINS: ['thermostat'], + history.CONF_ENTITIES: ['media_player.test']}}}) + self.check_significant_states(zero, four, states, config) + + def test_get_significant_states_include_exclude_domain(self): + """Test if significant states when excluding and including domains. + + We should not get back any changes since we include only the + media_player domain but also exclude it. + """ + zero, four, states = self.record_states() + del states['media_player.test'] + del states['media_player.test2'] + del states['thermostat.test'] + del states['thermostat.test2'] + del states['script.can_cancel_this_one'] + + config = history.CONFIG_SCHEMA({ + ha.DOMAIN: {}, + history.DOMAIN: {history.CONF_INCLUDE: { + history.CONF_DOMAINS: ['media_player']}, + history.CONF_EXCLUDE: { + history.CONF_DOMAINS: ['media_player']}}}) + self.check_significant_states(zero, four, states, config) + + def test_get_significant_states_include_exclude_entity(self): + """Test if significant states when excluding and including domains. + + We should not get back any changes since we include only + media_player.test but also exclude it. + """ + zero, four, states = self.record_states() + del states['media_player.test'] + del states['media_player.test2'] + del states['thermostat.test'] + del states['thermostat.test2'] + del states['script.can_cancel_this_one'] + + config = history.CONFIG_SCHEMA({ + ha.DOMAIN: {}, + history.DOMAIN: {history.CONF_INCLUDE: { + history.CONF_ENTITIES: ['media_player.test']}, + history.CONF_EXCLUDE: { + history.CONF_ENTITIES: ['media_player.test']}}}) + self.check_significant_states(zero, four, states, config) + + def test_get_significant_states_include_exclude(self): + """Test if significant states when in/excluding domains and entities. + + We should only get back changes of the media_player.test2 entity. + """ + zero, four, states = self.record_states() + del states['media_player.test'] + del states['thermostat.test'] + del states['thermostat.test2'] + del states['script.can_cancel_this_one'] + + config = history.CONFIG_SCHEMA({ + ha.DOMAIN: {}, + history.DOMAIN: {history.CONF_INCLUDE: { + history.CONF_DOMAINS: ['media_player'], + history.CONF_ENTITIES: ['thermostat.test']}, + history.CONF_EXCLUDE: { + history.CONF_DOMAINS: ['thermostat'], + history.CONF_ENTITIES: ['media_player.test']}}}) + self.check_significant_states(zero, four, states, config) + + def check_significant_states(self, zero, four, states, config): + """Check if significant states are retrieved.""" + filters = history.Filters() + exclude = config[history.DOMAIN].get(history.CONF_EXCLUDE) + if exclude: + filters.excluded_entities = exclude[history.CONF_ENTITIES] + filters.excluded_domains = exclude[history.CONF_DOMAINS] + include = config[history.DOMAIN].get(history.CONF_INCLUDE) + if include: + filters.included_entities = include[history.CONF_ENTITIES] + filters.included_domains = include[history.CONF_DOMAINS] + + hist = history.get_significant_states(zero, four, filters=filters) + assert states == hist + + def record_states(self): + """Record some test states. + + We inject a bunch of state updates from media player, zone and + thermostat. + """ self.init_recorder() mp = 'media_player.test' + mp2 = 'media_player.test2' therm = 'thermostat.test' + therm2 = 'thermostat.test2' zone = 'zone.home' script_nc = 'script.cannot_cancel_this_one' script_c = 'script.can_cancel_this_one' @@ -168,7 +398,7 @@ class TestComponentHistory(unittest.TestCase): three = two + timedelta(seconds=1) four = three + timedelta(seconds=1) - states = {therm: [], mp: [], script_c: []} + states = {therm: [], therm2: [], mp: [], mp2: [], script_c: []} with patch('homeassistant.components.recorder.dt_util.utcnow', return_value=one): states[mp].append( @@ -177,6 +407,9 @@ class TestComponentHistory(unittest.TestCase): states[mp].append( set_state(mp, 'YouTube', attributes={'media_title': str(sentinel.mt2)})) + states[mp2].append( + set_state(mp2, 'YouTube', + attributes={'media_title': str(sentinel.mt2)})) states[therm].append( set_state(therm, 20, attributes={'current_temperature': 19.5})) @@ -192,6 +425,8 @@ class TestComponentHistory(unittest.TestCase): set_state(script_c, 'off', attributes={'can_cancel': True})) states[therm].append( set_state(therm, 21, attributes={'current_temperature': 19.8})) + states[therm2].append( + set_state(therm2, 20, attributes={'current_temperature': 19})) with patch('homeassistant.components.recorder.dt_util.utcnow', return_value=three): @@ -201,6 +436,7 @@ class TestComponentHistory(unittest.TestCase): # Attributes changed even though state is the same states[therm].append( set_state(therm, 21, attributes={'current_temperature': 20})) - - hist = history.get_significant_states(zero, four) - assert states == hist + # state will be skipped since entity is hidden + set_state(therm, 22, attributes={'current_temperature': 21, + 'hidden': True}) + return zero, four, states