Home-Assistant
/
core
mirror of https://github.com/home-assistant/core.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Memory optimization for logbook (#21549)

pull/21582/head
Anders Melchiorsen 2019-03-02 08:23:45 +01:00 committed by Paulus Schoutsen
parent f1b867dccb
commit 0c8a31b8ec
3 changed files with 125 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
homeassistant/components/logbook/__init__.py
View File

@ -146,8 +146,8 @@ class LogbookView(HomeAssistantView):
def json_events():
"""Fetch events and generate JSON."""
return self.json(list(
_get_events(hass, self.config, start_day, end_day, entity_id)))
return self.json(
_get_events(hass, self.config, start_day, end_day, entity_id))
return await hass.async_add_job(json_events)
@ -393,11 +393,17 @@ def _generate_filter_from_config(config):
def _get_events(hass, config, start_day, end_day, entity_id=None):
"""Get events for a period of time."""
from homeassistant.components.recorder.models import Events, States
from homeassistant.components.recorder.util import (
execute, session_scope)
from homeassistant.components.recorder.util import session_scope
entities_filter = _generate_filter_from_config(config)
def yield_events(query):
"""Yield Events that are not filtered away."""
for row in query.yield_per(500):
event = row.to_native()
if _keep_event(event, entities_filter):
yield event
with session_scope(hass=hass) as session:
if entity_id is not None:
entity_ids = [entity_id.lower()]
@ -413,77 +419,70 @@ def _get_events(hass, config, start_day, end_day, entity_id=None):
States.entity_id.in_(entity_ids))
| (States.state_id.is_(None)))
events = execute(query)
return humanify(hass, _exclude_events(events, entities_filter))
return list(humanify(hass, yield_events(query)))
def _exclude_events(events, entities_filter):
filtered_events = []
for event in events:
domain, entity_id = None, None
def _keep_event(event, entities_filter):
domain, entity_id = None, None
if event.event_type == EVENT_STATE_CHANGED:
entity_id = event.data.get('entity_id')
if event.event_type == EVENT_STATE_CHANGED:
entity_id = event.data.get('entity_id')
if entity_id is None:
continue
if entity_id is None:
return False
# Do not report on new entities
if event.data.get('old_state') is None:
continue
# Do not report on new entities
if event.data.get('old_state') is None:
return False
new_state = event.data.get('new_state')
new_state = event.data.get('new_state')
# Do not report on entity removal
if not new_state:
continue
# Do not report on entity removal
if not new_state:
return False
attributes = new_state.get('attributes', {})
attributes = new_state.get('attributes', {})
# If last_changed != last_updated only attributes have changed
# we do not report on that yet.
last_changed = new_state.get('last_changed')
last_updated = new_state.get('last_updated')
if last_changed != last_updated:
continue
# If last_changed != last_updated only attributes have changed
# we do not report on that yet.
last_changed = new_state.get('last_changed')
last_updated = new_state.get('last_updated')
if last_changed != last_updated:
return False
domain = split_entity_id(entity_id)[0]
domain = split_entity_id(entity_id)[0]
# Also filter auto groups.
if domain == 'group' and attributes.get('auto', False):
continue
# Also filter auto groups.
if domain == 'group' and attributes.get('auto', False):
return False
# exclude entities which are customized hidden
hidden = attributes.get(ATTR_HIDDEN, False)
if hidden:
continue
# exclude entities which are customized hidden
hidden = attributes.get(ATTR_HIDDEN, False)
if hidden:
return False
elif event.event_type == EVENT_LOGBOOK_ENTRY:
domain = event.data.get(ATTR_DOMAIN)
entity_id = event.data.get(ATTR_ENTITY_ID)
elif event.event_type == EVENT_LOGBOOK_ENTRY:
domain = event.data.get(ATTR_DOMAIN)
entity_id = event.data.get(ATTR_ENTITY_ID)
elif event.event_type == EVENT_AUTOMATION_TRIGGERED:
domain = 'automation'
entity_id = event.data.get(ATTR_ENTITY_ID)
elif event.event_type == EVENT_AUTOMATION_TRIGGERED:
domain = 'automation'
entity_id = event.data.get(ATTR_ENTITY_ID)
elif event.event_type == EVENT_SCRIPT_STARTED:
domain = 'script'
entity_id = event.data.get(ATTR_ENTITY_ID)
elif event.event_type == EVENT_SCRIPT_STARTED:
domain = 'script'
entity_id = event.data.get(ATTR_ENTITY_ID)
elif event.event_type == EVENT_ALEXA_SMART_HOME:
domain = 'alexa'
elif event.event_type == EVENT_ALEXA_SMART_HOME:
domain = 'alexa'
elif event.event_type == EVENT_HOMEKIT_CHANGED:
domain = DOMAIN_HOMEKIT
elif event.event_type == EVENT_HOMEKIT_CHANGED:
domain = DOMAIN_HOMEKIT
if not entity_id and domain:
entity_id = "%s." % (domain, )
if not entity_id and domain:
entity_id = "%s." % (domain, )
if not entity_id or entities_filter(entity_id):
filtered_events.append(event)
return filtered_events
return not entity_id or entities_filter(entity_id)
def _entry_message_from_state(domain, state):

11
homeassistant/scripts/benchmark/__init__.py
View File

@ -180,12 +180,15 @@ def _logbook_filtering(hass, last_changed, last_updated):
'new_state': new_state
})
events = [event] * 10**5
def yield_events(event):
# pylint: disable=protected-access
entities_filter = logbook._generate_filter_from_config({})
for _ in range(10**5):
if logbook._keep_event(event, entities_filter):
yield event
start = timer()
# pylint: disable=protected-access
events = logbook._exclude_events(events, {})
list(logbook.humanify(None, events))
list(logbook.humanify(None, yield_events(event)))
return timer() - start

106
tests/components/logbook/test_init.py
View File

@ -148,10 +148,11 @@ class TestComponentLogbook(unittest.TestCase):
eventB = self.create_state_changed_event(pointB, entity_id2, 20)
eventA.data['old_state'] = None
events = logbook._exclude_events(
(ha.Event(EVENT_HOMEASSISTANT_STOP),
eventA, eventB),
logbook._generate_filter_from_config({}))
entities_filter = logbook._generate_filter_from_config({})
events = [e for e in
(ha.Event(EVENT_HOMEASSISTANT_STOP),
eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events))
assert 2 == len(entries)
@ -172,10 +173,11 @@ class TestComponentLogbook(unittest.TestCase):
eventB = self.create_state_changed_event(pointB, entity_id2, 20)
eventA.data['new_state'] = None
events = logbook._exclude_events(
(ha.Event(EVENT_HOMEASSISTANT_STOP),
eventA, eventB),
logbook._generate_filter_from_config({}))
entities_filter = logbook._generate_filter_from_config({})
events = [e for e in
(ha.Event(EVENT_HOMEASSISTANT_STOP),
eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events))
assert 2 == len(entries)
@ -196,10 +198,11 @@ class TestComponentLogbook(unittest.TestCase):
{ATTR_HIDDEN: 'true'})
eventB = self.create_state_changed_event(pointB, entity_id2, 20)
events = logbook._exclude_events(
(ha.Event(EVENT_HOMEASSISTANT_STOP),
eventA, eventB),
logbook._generate_filter_from_config({}))
entities_filter = logbook._generate_filter_from_config({})
events = [e for e in
(ha.Event(EVENT_HOMEASSISTANT_STOP),
eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events))
assert 2 == len(entries)
@ -223,9 +226,11 @@ class TestComponentLogbook(unittest.TestCase):
ha.DOMAIN: {},
logbook.DOMAIN: {logbook.CONF_EXCLUDE: {
logbook.CONF_ENTITIES: [entity_id, ]}}})
events = logbook._exclude_events(
(ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB),
logbook._generate_filter_from_config(config[logbook.DOMAIN]))
entities_filter = logbook._generate_filter_from_config(
config[logbook.DOMAIN])
events = [e for e in
(ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events))
assert 2 == len(entries)
@ -249,11 +254,13 @@ class TestComponentLogbook(unittest.TestCase):
ha.DOMAIN: {},
logbook.DOMAIN: {logbook.CONF_EXCLUDE: {
logbook.CONF_DOMAINS: ['switch', 'alexa', DOMAIN_HOMEKIT]}}})
events = logbook._exclude_events(
(ha.Event(EVENT_HOMEASSISTANT_START),
ha.Event(EVENT_ALEXA_SMART_HOME),
ha.Event(EVENT_HOMEKIT_CHANGED), eventA, eventB),
logbook._generate_filter_from_config(config[logbook.DOMAIN]))
entities_filter = logbook._generate_filter_from_config(
config[logbook.DOMAIN])
events = [e for e in
(ha.Event(EVENT_HOMEASSISTANT_START),
ha.Event(EVENT_ALEXA_SMART_HOME),
ha.Event(EVENT_HOMEKIT_CHANGED), eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events))
assert 2 == len(entries)
@ -283,9 +290,11 @@ class TestComponentLogbook(unittest.TestCase):
ha.DOMAIN: {},
logbook.DOMAIN: {logbook.CONF_EXCLUDE: {
logbook.CONF_ENTITIES: [entity_id, ]}}})
events = logbook._exclude_events(
(ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB),
logbook._generate_filter_from_config(config[logbook.DOMAIN]))
entities_filter = logbook._generate_filter_from_config(
config[logbook.DOMAIN])
events = [e for e in
(ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events))
assert 2 == len(entries)
@ -316,9 +325,11 @@ class TestComponentLogbook(unittest.TestCase):
ha.DOMAIN: {},
logbook.DOMAIN: {logbook.CONF_EXCLUDE: {
logbook.CONF_ENTITIES: [entity_id, ]}}})
events = logbook._exclude_events(
(ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB),
logbook._generate_filter_from_config(config[logbook.DOMAIN]))
entities_filter = logbook._generate_filter_from_config(
config[logbook.DOMAIN])
events = [e for e in
(ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events))
assert 2 == len(entries)
@ -342,9 +353,11 @@ class TestComponentLogbook(unittest.TestCase):
ha.DOMAIN: {},
logbook.DOMAIN: {logbook.CONF_INCLUDE: {
logbook.CONF_ENTITIES: [entity_id2, ]}}})
events = logbook._exclude_events(
(ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB),
logbook._generate_filter_from_config(config[logbook.DOMAIN]))
entities_filter = logbook._generate_filter_from_config(
config[logbook.DOMAIN])
events = [e for e in
(ha.Event(EVENT_HOMEASSISTANT_STOP), eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events))
assert 2 == len(entries)
@ -378,10 +391,12 @@ class TestComponentLogbook(unittest.TestCase):
ha.DOMAIN: {},
logbook.DOMAIN: {logbook.CONF_INCLUDE: {
logbook.CONF_DOMAINS: ['sensor', 'alexa', DOMAIN_HOMEKIT]}}})
events = logbook._exclude_events(
(ha.Event(EVENT_HOMEASSISTANT_START),
event_alexa, event_homekit, eventA, eventB),
logbook._generate_filter_from_config(config[logbook.DOMAIN]))
entities_filter = logbook._generate_filter_from_config(
config[logbook.DOMAIN])
events = [e for e in
(ha.Event(EVENT_HOMEASSISTANT_START),
event_alexa, event_homekit, eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events))
assert 4 == len(entries)
@ -415,10 +430,13 @@ class TestComponentLogbook(unittest.TestCase):
logbook.CONF_EXCLUDE: {
logbook.CONF_DOMAINS: ['switch', ],
logbook.CONF_ENTITIES: ['sensor.bli', ]}}})
events = logbook._exclude_events(
(ha.Event(EVENT_HOMEASSISTANT_START), eventA1, eventA2, eventA3,
eventB1, eventB2),
logbook._generate_filter_from_config(config[logbook.DOMAIN]))
entities_filter = logbook._generate_filter_from_config(
config[logbook.DOMAIN])
events = [e for e in
(ha.Event(EVENT_HOMEASSISTANT_START),
eventA1, eventA2, eventA3,
eventB1, eventB2)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events))
assert 5 == len(entries)
@ -443,9 +461,10 @@ class TestComponentLogbook(unittest.TestCase):
eventB = self.create_state_changed_event(pointA, entity_id2, 20,
{'auto': True})
events = logbook._exclude_events(
(eventA, eventB),
logbook._generate_filter_from_config({}))
entities_filter = logbook._generate_filter_from_config({})
events = [e for e in
(eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events))
assert 1 == len(entries)
@ -463,9 +482,10 @@ class TestComponentLogbook(unittest.TestCase):
eventB = self.create_state_changed_event(
pointA, entity_id2, 20, last_changed=pointA, last_updated=pointB)
events = logbook._exclude_events(
(eventA, eventB),
logbook._generate_filter_from_config({}))
entities_filter = logbook._generate_filter_from_config({})
events = [e for e in
(eventA, eventB)
if logbook._keep_event(e, entities_filter)]
entries = list(logbook.humanify(self.hass, events))
assert 1 == len(entries)