From 0c8a31b8ec11d123cf74664af33c5ca651ac6e17 Mon Sep 17 00:00:00 2001 From: Anders Melchiorsen Date: Sat, 2 Mar 2019 08:23:45 +0100 Subject: [PATCH] Memory optimization for logbook (#21549) --- homeassistant/components/logbook/__init__.py | 111 +++++++++---------- homeassistant/scripts/benchmark/__init__.py | 11 +- tests/components/logbook/test_init.py | 106 +++++++++++------- 3 files changed, 125 insertions(+), 103 deletions(-) diff --git a/homeassistant/components/logbook/__init__.py b/homeassistant/components/logbook/__init__.py index dbedc8c6d70..7a0fb5e2654 100644 --- a/homeassistant/components/logbook/__init__.py +++ b/homeassistant/components/logbook/__init__.py @@ -146,8 +146,8 @@ class LogbookView(HomeAssistantView): def json_events(): """Fetch events and generate JSON.""" - return self.json(list( - _get_events(hass, self.config, start_day, end_day, entity_id))) + return self.json( + _get_events(hass, self.config, start_day, end_day, entity_id)) return await hass.async_add_job(json_events) @@ -393,11 +393,17 @@ def _generate_filter_from_config(config): def _get_events(hass, config, start_day, end_day, entity_id=None): """Get events for a period of time.""" from homeassistant.components.recorder.models import Events, States - from homeassistant.components.recorder.util import ( - execute, session_scope) + from homeassistant.components.recorder.util import session_scope entities_filter = _generate_filter_from_config(config) + def yield_events(query): + """Yield Events that are not filtered away.""" + for row in query.yield_per(500): + event = row.to_native() + if _keep_event(event, entities_filter): + yield event + with session_scope(hass=hass) as session: if entity_id is not None: entity_ids = [entity_id.lower()] @@ -413,77 +419,70 @@ def _get_events(hass, config, start_day, end_day, entity_id=None): States.entity_id.in_(entity_ids)) | (States.state_id.is_(None))) - events = execute(query) - - return humanify(hass, _exclude_events(events, entities_filter)) + return list(humanify(hass, yield_events(query))) -def _exclude_events(events, entities_filter): - filtered_events = [] - for event in events: - domain, entity_id = None, None +def _keep_event(event, entities_filter): + domain, entity_id = None, None - if event.event_type == EVENT_STATE_CHANGED: - entity_id = event.data.get('entity_id') + if event.event_type == EVENT_STATE_CHANGED: + entity_id = event.data.get('entity_id') - if entity_id is None: - continue + if entity_id is None: + return False - # Do not report on new entities - if event.data.get('old_state') is None: - continue + # Do not report on new entities + if event.data.get('old_state') is None: + return False - new_state = event.data.get('new_state') + new_state = event.data.get('new_state') - # Do not report on entity removal - if not new_state: - continue + # Do not report on entity removal + if not new_state: + return False - attributes = new_state.get('attributes', {}) + attributes = new_state.get('attributes', {}) - # If last_changed != last_updated only attributes have changed - # we do not report on that yet. - last_changed = new_state.get('last_changed') - last_updated = new_state.get('last_updated') - if last_changed != last_updated: - continue + # If last_changed != last_updated only attributes have changed + # we do not report on that yet. + last_changed = new_state.get('last_changed') + last_updated = new_state.get('last_updated') + if last_changed != last_updated: + return False - domain = split_entity_id(entity_id)[0] + domain = split_entity_id(entity_id)[0] - # Also filter auto groups. - if domain == 'group' and attributes.get('auto', False): - continue + # Also filter auto groups. + if domain == 'group' and attributes.get('auto', False): + return False - # exclude entities which are customized hidden - hidden = attributes.get(ATTR_HIDDEN, False) - if hidden: - continue + # exclude entities which are customized hidden + hidden = attributes.get(ATTR_HIDDEN, False) + if hidden: + return False - elif event.event_type == EVENT_LOGBOOK_ENTRY: - domain = event.data.get(ATTR_DOMAIN) - entity_id = event.data.get(ATTR_ENTITY_ID) + elif event.event_type == EVENT_LOGBOOK_ENTRY: + domain = event.data.get(ATTR_DOMAIN) + entity_id = event.data.get(ATTR_ENTITY_ID) - elif event.event_type == EVENT_AUTOMATION_TRIGGERED: - domain = 'automation' - entity_id = event.data.get(ATTR_ENTITY_ID) + elif event.event_type == EVENT_AUTOMATION_TRIGGERED: + domain = 'automation' + entity_id = event.data.get(ATTR_ENTITY_ID) - elif event.event_type == EVENT_SCRIPT_STARTED: - domain = 'script' - entity_id = event.data.get(ATTR_ENTITY_ID) + elif event.event_type == EVENT_SCRIPT_STARTED: + domain = 'script' + entity_id = event.data.get(ATTR_ENTITY_ID) - elif event.event_type == EVENT_ALEXA_SMART_HOME: - domain = 'alexa' + elif event.event_type == EVENT_ALEXA_SMART_HOME: + domain = 'alexa' - elif event.event_type == EVENT_HOMEKIT_CHANGED: - domain = DOMAIN_HOMEKIT + elif event.event_type == EVENT_HOMEKIT_CHANGED: + domain = DOMAIN_HOMEKIT - if not entity_id and domain: - entity_id = "%s." % (domain, ) + if not entity_id and domain: + entity_id = "%s." % (domain, ) - if not entity_id or entities_filter(entity_id): - filtered_events.append(event) - - return filtered_events + return not entity_id or entities_filter(entity_id) def _entry_message_from_state(domain, state): diff --git a/homeassistant/scripts/benchmark/__init__.py b/homeassistant/scripts/benchmark/__init__.py index f0df58a51f4..e231d7602cd 100644 --- a/homeassistant/scripts/benchmark/__init__.py +++ b/homeassistant/scripts/benchmark/__init__.py @@ -180,12 +180,15 @@ def _logbook_filtering(hass, last_changed, last_updated): 'new_state': new_state }) - events = [event] * 10**5 + def yield_events(event): + # pylint: disable=protected-access + entities_filter = logbook._generate_filter_from_config({}) + for _ in range(10**5): + if logbook._keep_event(event, entities_filter): + yield event start = timer() - # pylint: disable=protected-access - events = logbook._exclude_events(events, {}) - list(logbook.humanify(None, events)) + list(logbook.humanify(None, yield_events(event))) return timer() - start diff --git a/tests/components/logbook/test_init.py b/tests/components/logbook/test_init.py index c8ade907dd3..9d69affae4a 100644 --- a/tests/components/logbook/test_init.py +++ b/tests/components/logbook/test_init.py @@ -148,10 +148,11 @@ class TestComponentLogbook(unittest.TestCase): eventB = self.create_state_changed_event(pointB, entity_id2, 20) eventA.data['old_state'] = None - events = logbook._exclude_events( - (ha.Event(EVENT_HOMEASSISTANT_STOP), - eventA, eventB), - logbook._generate_filter_from_config({})) + entities_filter = logbook._generate_filter_from_config({}) + events = [e for e in + (ha.Event(EVENT_HOMEASSISTANT_STOP), + eventA, eventB) + if logbook._keep_event(e, entities_filter)] entries = list(logbook.humanify(self.hass, events)) assert 2 == len(entries) @@ -172,10 +173,11 @@ class TestComponentLogbook(unittest.TestCase): eventB = self.create_state_changed_event(pointB, entity_id2, 20) eventA.data['new_state'] = None - events = logbook._exclude_events( - (ha.Event(EVENT_HOMEASSISTANT_STOP), - eventA, eventB), - logbook._generate_filter_from_config({})) + entities_filter = logbook._generate_filter_from_config({}) + events = [e for e in + (ha.Event(EVENT_HOMEASSISTANT_STOP), + eventA, eventB) + if logbook._keep_event(e, entities_filter)] entries = list(logbook.humanify(self.hass, events)) assert 2 == len(entries) @@ -196,10 +198,11 @@ class TestComponentLogbook(unittest.TestCase): {ATTR_HIDDEN: 'true'}) eventB = self.create_state_changed_event(pointB, entity_id2, 20) - events = logbook._exclude_events( - (ha.Event(EVENT_HOMEASSISTANT_STOP), - eventA, eventB), - logbook._generate_filter_from_config({})) + entities_filter = logbook._generate_filter_from_config({}) + events = [e for e in + (ha.Event(EVENT_HOMEASSISTANT_STOP), + eventA, eventB) + if logbook._keep_event(e, entities_filter)] entries = list(logbook.humanify(self.hass, events)) assert 2 == len(entries) @@ -223,9 +226,11 @@ class TestComponentLogbook(unittest.TestCase): ha.DOMAIN: {}, logbook.DOMAIN: {logbook.CONF_EXCLUDE: { logbook.CONF_ENTITIES: [entity_id, ]}}}) - events = logbook._exclude_events( - (ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB), - logbook._generate_filter_from_config(config[logbook.DOMAIN])) + entities_filter = logbook._generate_filter_from_config( + config[logbook.DOMAIN]) + events = [e for e in + (ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB) + if logbook._keep_event(e, entities_filter)] entries = list(logbook.humanify(self.hass, events)) assert 2 == len(entries) @@ -249,11 +254,13 @@ class TestComponentLogbook(unittest.TestCase): ha.DOMAIN: {}, logbook.DOMAIN: {logbook.CONF_EXCLUDE: { logbook.CONF_DOMAINS: ['switch', 'alexa', DOMAIN_HOMEKIT]}}}) - events = logbook._exclude_events( - (ha.Event(EVENT_HOMEASSISTANT_START), - ha.Event(EVENT_ALEXA_SMART_HOME), - ha.Event(EVENT_HOMEKIT_CHANGED), eventA, eventB), - logbook._generate_filter_from_config(config[logbook.DOMAIN])) + entities_filter = logbook._generate_filter_from_config( + config[logbook.DOMAIN]) + events = [e for e in + (ha.Event(EVENT_HOMEASSISTANT_START), + ha.Event(EVENT_ALEXA_SMART_HOME), + ha.Event(EVENT_HOMEKIT_CHANGED), eventA, eventB) + if logbook._keep_event(e, entities_filter)] entries = list(logbook.humanify(self.hass, events)) assert 2 == len(entries) @@ -283,9 +290,11 @@ class TestComponentLogbook(unittest.TestCase): ha.DOMAIN: {}, logbook.DOMAIN: {logbook.CONF_EXCLUDE: { logbook.CONF_ENTITIES: [entity_id, ]}}}) - events = logbook._exclude_events( - (ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB), - logbook._generate_filter_from_config(config[logbook.DOMAIN])) + entities_filter = logbook._generate_filter_from_config( + config[logbook.DOMAIN]) + events = [e for e in + (ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB) + if logbook._keep_event(e, entities_filter)] entries = list(logbook.humanify(self.hass, events)) assert 2 == len(entries) @@ -316,9 +325,11 @@ class TestComponentLogbook(unittest.TestCase): ha.DOMAIN: {}, logbook.DOMAIN: {logbook.CONF_EXCLUDE: { logbook.CONF_ENTITIES: [entity_id, ]}}}) - events = logbook._exclude_events( - (ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB), - logbook._generate_filter_from_config(config[logbook.DOMAIN])) + entities_filter = logbook._generate_filter_from_config( + config[logbook.DOMAIN]) + events = [e for e in + (ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB) + if logbook._keep_event(e, entities_filter)] entries = list(logbook.humanify(self.hass, events)) assert 2 == len(entries) @@ -342,9 +353,11 @@ class TestComponentLogbook(unittest.TestCase): ha.DOMAIN: {}, logbook.DOMAIN: {logbook.CONF_INCLUDE: { logbook.CONF_ENTITIES: [entity_id2, ]}}}) - events = logbook._exclude_events( - (ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB), - logbook._generate_filter_from_config(config[logbook.DOMAIN])) + entities_filter = logbook._generate_filter_from_config( + config[logbook.DOMAIN]) + events = [e for e in + (ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB) + if logbook._keep_event(e, entities_filter)] entries = list(logbook.humanify(self.hass, events)) assert 2 == len(entries) @@ -378,10 +391,12 @@ class TestComponentLogbook(unittest.TestCase): ha.DOMAIN: {}, logbook.DOMAIN: {logbook.CONF_INCLUDE: { logbook.CONF_DOMAINS: ['sensor', 'alexa', DOMAIN_HOMEKIT]}}}) - events = logbook._exclude_events( - (ha.Event(EVENT_HOMEASSISTANT_START), - event_alexa, event_homekit, eventA, eventB), - logbook._generate_filter_from_config(config[logbook.DOMAIN])) + entities_filter = logbook._generate_filter_from_config( + config[logbook.DOMAIN]) + events = [e for e in + (ha.Event(EVENT_HOMEASSISTANT_START), + event_alexa, event_homekit, eventA, eventB) + if logbook._keep_event(e, entities_filter)] entries = list(logbook.humanify(self.hass, events)) assert 4 == len(entries) @@ -415,10 +430,13 @@ class TestComponentLogbook(unittest.TestCase): logbook.CONF_EXCLUDE: { logbook.CONF_DOMAINS: ['switch', ], logbook.CONF_ENTITIES: ['sensor.bli', ]}}}) - events = logbook._exclude_events( - (ha.Event(EVENT_HOMEASSISTANT_START), eventA1, eventA2, eventA3, - eventB1, eventB2), - logbook._generate_filter_from_config(config[logbook.DOMAIN])) + entities_filter = logbook._generate_filter_from_config( + config[logbook.DOMAIN]) + events = [e for e in + (ha.Event(EVENT_HOMEASSISTANT_START), + eventA1, eventA2, eventA3, + eventB1, eventB2) + if logbook._keep_event(e, entities_filter)] entries = list(logbook.humanify(self.hass, events)) assert 5 == len(entries) @@ -443,9 +461,10 @@ class TestComponentLogbook(unittest.TestCase): eventB = self.create_state_changed_event(pointA, entity_id2, 20, {'auto': True}) - events = logbook._exclude_events( - (eventA, eventB), - logbook._generate_filter_from_config({})) + entities_filter = logbook._generate_filter_from_config({}) + events = [e for e in + (eventA, eventB) + if logbook._keep_event(e, entities_filter)] entries = list(logbook.humanify(self.hass, events)) assert 1 == len(entries) @@ -463,9 +482,10 @@ class TestComponentLogbook(unittest.TestCase): eventB = self.create_state_changed_event( pointA, entity_id2, 20, last_changed=pointA, last_updated=pointB) - events = logbook._exclude_events( - (eventA, eventB), - logbook._generate_filter_from_config({})) + entities_filter = logbook._generate_filter_from_config({}) + events = [e for e in + (eventA, eventB) + if logbook._keep_event(e, entities_filter)] entries = list(logbook.humanify(self.hass, events)) assert 1 == len(entries)