From 692d08226dc467973dbbc158a6153b32d99a0d1d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 14 Sep 2020 12:03:01 -0500 Subject: [PATCH 01/13] Add support for selecting multiple entity ids from logbook --- homeassistant/components/logbook/__init__.py | 55 ++++++------ tests/components/logbook/test_init.py | 88 ++++++++++++++++++++ 2 files changed, 116 insertions(+), 27 deletions(-) diff --git a/homeassistant/components/logbook/__init__.py b/homeassistant/components/logbook/__init__.py index 476aa62501a58..061d4e1512e54 100644 --- a/homeassistant/components/logbook/__init__.py +++ b/homeassistant/components/logbook/__init__.py @@ -39,12 +39,7 @@ STATE_OFF, STATE_ON, ) -from homeassistant.core import ( - DOMAIN as HA_DOMAIN, - callback, - split_entity_id, - valid_entity_id, -) +from homeassistant.core import DOMAIN as HA_DOMAIN, callback, split_entity_id from homeassistant.exceptions import InvalidEntityFormatError import homeassistant.helpers.config_validation as cv from homeassistant.helpers.entityfilter import ( @@ -208,7 +203,15 @@ async def get(self, request, datetime=None): else: period = int(period) - entity_id = request.query.get("entity") + entity_ids = request.query.get("entity") + if entity_ids: + try: + entity_ids = cv.entity_ids(entity_ids) + except vol.Invalid: + raise InvalidEntityFormatError( + f"Invalid entity id(s) encountered: {entity_ids}. " + "Format should be ." + ) from vol.Invalid end_time = request.query.get("end_time") if end_time is None: @@ -231,7 +234,7 @@ def json_events(): hass, start_day, end_day, - entity_id, + entity_ids, self.filters, self.entities_filter, entity_matches_only, @@ -409,7 +412,7 @@ def _get_events( hass, start_day, end_day, - entity_id=None, + entity_ids=None, filters=None, entities_filter=None, entity_matches_only=False, @@ -417,7 +420,6 @@ def _get_events( """Get events for a period of time.""" entity_attr_cache = EntityAttributeCache(hass) context_lookup = {None: None} - entity_id_lower = None apply_sql_entities_filter = True def yield_events(query): @@ -428,14 +430,8 @@ def yield_events(query): if _keep_event(hass, event, entities_filter): yield event - if entity_id is not None: - entity_id_lower = entity_id.lower() - if not valid_entity_id(entity_id_lower): - raise InvalidEntityFormatError( - f"Invalid entity id encountered: {entity_id_lower}. " - "Format should be ." - ) - entities_filter = generate_filter([], [entity_id_lower], [], []) + if entity_ids is not None: + entities_filter = generate_filter([], entity_ids, [], []) apply_sql_entities_filter = False with session_scope(hass=hass) as session: @@ -484,26 +480,31 @@ def yield_events(query): .filter((Events.time_fired > start_day) & (Events.time_fired < end_day)) ) - if entity_id_lower is not None: + if entity_ids is not None: if entity_matches_only: # When entity_matches_only is provided, contexts and events that do not - # contain the entity_id are not included in the logbook response. - entity_id_json = ENTITY_ID_JSON_TEMPLATE.format(entity_id_lower) + # contain the entity_ids are not included in the logbook response. + states_matchers = Events.event_data.contains( + ENTITY_ID_JSON_TEMPLATE.format(entity_ids[0]) + ) + + for entity_id in entity_ids[1:]: + states_matchers |= Events.event_data.contains( + ENTITY_ID_JSON_TEMPLATE.format(entity_id) + ) + query = query.filter( ( (States.last_updated == States.last_changed) - & (States.entity_id == entity_id_lower) - ) - | ( - States.state_id.is_(None) - & Events.event_data.contains(entity_id_json) + & States.entity_id.in_(entity_ids) ) + | (States.state_id.is_(None) & states_matchers) ) else: query = query.filter( ( (States.last_updated == States.last_changed) - & (States.entity_id == entity_id_lower) + & States.entity_id.in_(entity_ids) ) | (States.state_id.is_(None)) ) diff --git a/tests/components/logbook/test_init.py b/tests/components/logbook/test_init.py index 745d809ca7f86..d805eb40ec158 100644 --- a/tests/components/logbook/test_init.py +++ b/tests/components/logbook/test_init.py @@ -2157,6 +2157,94 @@ async def test_logbook_entity_matches_only(hass, hass_client): assert json_dict[1]["message"] == "turned on" +async def test_logbook_entity_matches_only_multiple(hass, hass_client): + """Test the logbook view with a multiple entities and entity_matches_only.""" + await hass.async_add_executor_job(init_recorder_component, hass) + await async_setup_component(hass, "logbook", {}) + assert await async_setup_component( + hass, + "switch", + { + "switch": { + "platform": "template", + "switches": { + "test_template_switch": { + "value_template": "{{ states.switch.test_state.state }}", + "turn_on": { + "service": "switch.turn_on", + "entity_id": "switch.test_state", + }, + "turn_off": { + "service": "switch.turn_off", + "entity_id": "switch.test_state", + }, + } + }, + } + }, + ) + await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done) + await hass.async_block_till_done() + await hass.async_start() + await hass.async_block_till_done() + + # Entity added (should not be logged) + hass.states.async_set("switch.test_state", STATE_ON) + hass.states.async_set("light.test_state", STATE_ON) + + await hass.async_block_till_done() + + # First state change (should be logged) + hass.states.async_set("switch.test_state", STATE_OFF) + hass.states.async_set("light.test_state", STATE_OFF) + + await hass.async_block_till_done() + + switch_turn_off_context = ha.Context( + id="9c5bd62de45711eaaeb351041eec8dd9", + user_id="9400facee45711eaa9308bfd3d19e474", + ) + hass.states.async_set( + "switch.test_state", STATE_ON, context=switch_turn_off_context + ) + hass.states.async_set("light.test_state", STATE_ON, context=switch_turn_off_context) + await hass.async_block_till_done() + + await hass.async_add_executor_job(trigger_db_commit, hass) + await hass.async_block_till_done() + await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done) + + client = await hass_client() + + # Today time 00:00:00 + start = dt_util.utcnow().date() + start_date = datetime(start.year, start.month, start.day) + + # Test today entries with filter by end_time + end_time = start + timedelta(hours=24) + response = await client.get( + f"/api/logbook/{start_date.isoformat()}?end_time={end_time}&entity=switch.test_state,light.test_state&entity_matches_only" + ) + assert response.status == 200 + json_dict = await response.json() + + assert len(json_dict) == 4 + + assert json_dict[0]["entity_id"] == "switch.test_state" + assert json_dict[0]["message"] == "turned off" + + assert json_dict[1]["entity_id"] == "light.test_state" + assert json_dict[1]["message"] == "turned off" + + assert json_dict[2]["entity_id"] == "switch.test_state" + assert json_dict[2]["context_user_id"] == "9400facee45711eaa9308bfd3d19e474" + assert json_dict[2]["message"] == "turned on" + + assert json_dict[3]["entity_id"] == "light.test_state" + assert json_dict[3]["context_user_id"] == "9400facee45711eaa9308bfd3d19e474" + assert json_dict[3]["message"] == "turned on" + + async def test_logbook_invalid_entity(hass, hass_client): """Test the logbook view with requesting an invalid entity.""" await hass.async_add_executor_job(init_recorder_component, hass) From 05a03cb1a116465c4b95c482970290cbecb99243 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 20 Sep 2020 13:37:46 -0500 Subject: [PATCH 02/13] Optimize states lookup --- homeassistant/components/logbook/__init__.py | 211 ++++++++++++------- 1 file changed, 140 insertions(+), 71 deletions(-) diff --git a/homeassistant/components/logbook/__init__.py b/homeassistant/components/logbook/__init__.py index 061d4e1512e54..bd56442e165c7 100644 --- a/homeassistant/components/logbook/__init__.py +++ b/homeassistant/components/logbook/__init__.py @@ -7,6 +7,7 @@ import sqlalchemy from sqlalchemy.orm import aliased +from sqlalchemy.sql.expression import literal import voluptuous as vol from homeassistant.components import sun @@ -82,13 +83,17 @@ EVENT_HOMEASSISTANT_STOP, ] -ALL_EVENT_TYPES = [ - EVENT_STATE_CHANGED, +ALL_EVENT_TYPES_EXCEPT_STATE_CHANGED = [ EVENT_LOGBOOK_ENTRY, EVENT_CALL_SERVICE, *HOMEASSISTANT_EVENTS, ] +ALL_EVENT_TYPES = [ + EVENT_STATE_CHANGED, + *ALL_EVENT_TYPES_EXCEPT_STATE_CHANGED, +] + SCRIPT_AUTOMATION_EVENTS = [EVENT_AUTOMATION_TRIGGERED, EVENT_SCRIPT_STARTED] LOG_MESSAGE_SCHEMA = vol.Schema( @@ -436,96 +441,160 @@ def yield_events(query): with session_scope(hass=hass) as session: old_state = aliased(States, name="old_state") + entity_filter = None + if apply_sql_entities_filter and filters: + entity_filter = filters.entity_filter() - query = ( - session.query( - Events.event_type, - Events.event_data, - Events.time_fired, - Events.context_id, - Events.context_user_id, - States.state, - States.entity_id, - States.domain, - States.attributes, - ) - .order_by(Events.time_fired) - .outerjoin(States, (Events.event_id == States.event_id)) - .outerjoin(old_state, (States.old_state_id == old_state.state_id)) - # The below filter, removes state change events that do not have - # and old_state, new_state, or the old and - # new state. - # - .filter( - (Events.event_type != EVENT_STATE_CHANGED) - | ( - (States.state_id.isnot(None)) - & (old_state.state_id.isnot(None)) - & (States.state.isnot(None)) - & (States.state != old_state.state) - ) - ) - # - # Prefilter out continuous domains that have - # ATTR_UNIT_OF_MEASUREMENT as its much faster in sql. - # - .filter( - (Events.event_type != EVENT_STATE_CHANGED) - | sqlalchemy.not_(States.domain.in_(CONTINUOUS_DOMAINS)) - | sqlalchemy.not_(States.attributes.contains(UNIT_OF_MEASUREMENT_JSON)) + if entity_ids is not None: + events_query = _generate_events_query_without_states(session) + events_query = _apply_event_time_filter(events_query, start_day, end_day) + events_query = _apply_event_types_filter( + hass, events_query, ALL_EVENT_TYPES_EXCEPT_STATE_CHANGED ) - .filter( - Events.event_type.in_(ALL_EVENT_TYPES + list(hass.data.get(DOMAIN, {}))) + + states_query = _generate_states_query( + session, start_day, end_day, old_state + ).filter( + (States.last_updated == States.last_changed) + & States.entity_id.in_(entity_ids) ) - .filter((Events.time_fired > start_day) & (Events.time_fired < end_day)) - ) - if entity_ids is not None: if entity_matches_only: # When entity_matches_only is provided, contexts and events that do not # contain the entity_ids are not included in the logbook response. - states_matchers = Events.event_data.contains( - ENTITY_ID_JSON_TEMPLATE.format(entity_ids[0]) - ) + events_query = _apply_state_matchers(events_query, entity_ids) - for entity_id in entity_ids[1:]: - states_matchers |= Events.event_data.contains( - ENTITY_ID_JSON_TEMPLATE.format(entity_id) - ) + if entity_filter is not None: + states_query = states_query.filter(entity_filter) - query = query.filter( - ( - (States.last_updated == States.last_changed) - & States.entity_id.in_(entity_ids) - ) - | (States.state_id.is_(None) & states_matchers) - ) - else: - query = query.filter( - ( - (States.last_updated == States.last_changed) - & States.entity_id.in_(entity_ids) - ) - | (States.state_id.is_(None)) - ) + query = events_query.union_all(states_query) else: - query = query.filter( + events_query = _generate_events_query(session) + events_query = _apply_event_time_filter(events_query, start_day, end_day) + events_query = _apply_events_states_filter( + hass, events_query, old_state + ).filter( (States.last_updated == States.last_changed) - | (States.state_id.is_(None)) + | (Events.event_type != EVENT_STATE_CHANGED) ) - - if apply_sql_entities_filter and filters: - entity_filter = filters.entity_filter() if entity_filter is not None: - query = query.filter( + events_query = events_query.filter( entity_filter | (Events.event_type != EVENT_STATE_CHANGED) ) + query = events_query + + query = query.order_by(Events.time_fired) return list( humanify(hass, yield_events(query), entity_attr_cache, context_lookup) ) +def _generate_events_query(session): + return session.query( + Events.event_type, + Events.event_data, + Events.time_fired, + Events.context_id, + Events.context_user_id, + States.state, + States.entity_id, + States.domain, + States.attributes, + ) + + +def _generate_events_query_without_states(session): + return session.query( + Events.event_type, + Events.event_data, + Events.time_fired, + Events.context_id, + Events.context_user_id, + literal(None).label("state"), + literal(None).label("entity_id"), + literal(None).label("domain"), + literal(None).label("attributes"), + ) + + +def _generate_states_query(session, start_day, end_day, old_state): + + return ( + _generate_events_query(session) + .outerjoin(Events, (States.event_id == Events.event_id)) + .outerjoin(old_state, (States.old_state_id == old_state.state_id)) + # The below filter, removes state change events that do not have + # and old_state, new_state, or the old and + # new state. + # + .filter((old_state.state_id.isnot(None)) & (States.state != old_state.state)) + # + # Prefilter out continuous domains that have + # ATTR_UNIT_OF_MEASUREMENT as its much faster in sql. + # + .filter( + sqlalchemy.not_(States.domain.in_(CONTINUOUS_DOMAINS)) + | sqlalchemy.not_(States.attributes.contains(UNIT_OF_MEASUREMENT_JSON)) + ) + .filter((States.last_updated > start_day) & (States.last_updated < end_day)) + ) + + +def _apply_events_states_filter(hass, query, old_state): + events_query = ( + # The below filter, removes state change events that do not have + # and old_state, new_state, or the old and + # new state. + # + query.outerjoin(States, (Events.event_id == States.event_id)) + .outerjoin(old_state, (States.old_state_id == old_state.state_id)) + .filter( + (Events.event_type != EVENT_STATE_CHANGED) + | ( + (States.state_id.isnot(None)) + & (old_state.state_id.isnot(None)) + & (States.state.isnot(None)) + & (States.state != old_state.state) + ) + ) + # + # Prefilter out continuous domains that have + # ATTR_UNIT_OF_MEASUREMENT as its much faster in sql. + # + .filter( + (Events.event_type != EVENT_STATE_CHANGED) + | sqlalchemy.not_(States.domain.in_(CONTINUOUS_DOMAINS)) + | sqlalchemy.not_(States.attributes.contains(UNIT_OF_MEASUREMENT_JSON)) + ) + ) + return _apply_event_types_filter(hass, events_query, ALL_EVENT_TYPES) + + +def _apply_event_time_filter(events_query, start_day, end_day): + return events_query.filter( + (Events.time_fired > start_day) & (Events.time_fired < end_day) + ) + + +def _apply_event_types_filter(hass, query, event_types): + return query.filter( + Events.event_type.in_(event_types + list(hass.data.get(DOMAIN, {}))) + ) + + +def _apply_state_matchers(events_query, entity_ids): + states_matchers = Events.event_data.contains( + ENTITY_ID_JSON_TEMPLATE.format(entity_ids[0]) + ) + for entity_id in entity_ids[1:]: + states_matchers |= Events.event_data.contains( + ENTITY_ID_JSON_TEMPLATE.format(entity_id) + ) + + return events_query.filter(states_matchers) + + def _keep_event(hass, event, entities_filter): if event.event_type == EVENT_STATE_CHANGED: entity_id = event.entity_id From 009f27f83a2f623cb0fc1561c1fc3ddda13bbeb8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 20 Sep 2020 17:00:16 -0500 Subject: [PATCH 03/13] tweaks --- homeassistant/components/logbook/__init__.py | 36 +++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/homeassistant/components/logbook/__init__.py b/homeassistant/components/logbook/__init__.py index bd56442e165c7..219ba7c512dd0 100644 --- a/homeassistant/components/logbook/__init__.py +++ b/homeassistant/components/logbook/__init__.py @@ -74,6 +74,8 @@ EMPTY_JSON_OBJECT = "{}" UNIT_OF_MEASUREMENT_JSON = '"unit_of_measurement":' +HA_DOMAIN_ENTITY_ID = f"{HA_DOMAIN}." + CONFIG_SCHEMA = vol.Schema( {DOMAIN: INCLUDE_EXCLUDE_BASE_FILTER_SCHEMA}, extra=vol.ALLOW_EXTRA ) @@ -597,25 +599,27 @@ def _apply_state_matchers(events_query, entity_ids): def _keep_event(hass, event, entities_filter): if event.event_type == EVENT_STATE_CHANGED: - entity_id = event.entity_id - elif event.event_type in HOMEASSISTANT_EVENTS: - entity_id = f"{HA_DOMAIN}." - elif event.event_type == EVENT_CALL_SERVICE: + return entities_filter is None or entities_filter(event.entity_id) + if event.event_type in HOMEASSISTANT_EVENTS: + return entities_filter is None or entities_filter(HA_DOMAIN_ENTITY_ID) + if event.event_type == EVENT_CALL_SERVICE: return False + + entity_id = event.data_entity_id + if entity_id: + return entities_filter is None or entities_filter(entity_id) + + if event.event_type in hass.data[DOMAIN]: + # If the entity_id isn't described, use the domain that describes + # the event for filtering. + domain = hass.data[DOMAIN][event.event_type][0] else: - entity_id = event.data_entity_id - if not entity_id: - if event.event_type in hass.data[DOMAIN]: - # If the entity_id isn't described, use the domain that describes - # the event for filtering. - domain = hass.data[DOMAIN][event.event_type][0] - else: - domain = event.data_domain - if domain is None: - return False - entity_id = f"{domain}." + domain = event.data_domain + + if domain is None: + return False - return entities_filter is None or entities_filter(entity_id) + return entities_filter is None or entities_filter(f"{domain}.") def _entry_message_from_event(entity_id, domain, event, entity_attr_cache): From 7920b579d4fab3a7c180409458be42a3b67b58bd Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 20 Sep 2020 17:10:20 -0500 Subject: [PATCH 04/13] simplify --- homeassistant/components/logbook/__init__.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/homeassistant/components/logbook/__init__.py b/homeassistant/components/logbook/__init__.py index 219ba7c512dd0..9e942ff95d722 100644 --- a/homeassistant/components/logbook/__init__.py +++ b/homeassistant/components/logbook/__init__.py @@ -427,7 +427,6 @@ def _get_events( """Get events for a period of time.""" entity_attr_cache = EntityAttributeCache(hass) context_lookup = {None: None} - apply_sql_entities_filter = True def yield_events(query): """Yield Events that are not filtered away.""" @@ -439,13 +438,9 @@ def yield_events(query): if entity_ids is not None: entities_filter = generate_filter([], entity_ids, [], []) - apply_sql_entities_filter = False with session_scope(hass=hass) as session: old_state = aliased(States, name="old_state") - entity_filter = None - if apply_sql_entities_filter and filters: - entity_filter = filters.entity_filter() if entity_ids is not None: events_query = _generate_events_query_without_states(session) @@ -466,9 +461,6 @@ def yield_events(query): # contain the entity_ids are not included in the logbook response. events_query = _apply_state_matchers(events_query, entity_ids) - if entity_filter is not None: - states_query = states_query.filter(entity_filter) - query = events_query.union_all(states_query) else: events_query = _generate_events_query(session) @@ -479,9 +471,9 @@ def yield_events(query): (States.last_updated == States.last_changed) | (Events.event_type != EVENT_STATE_CHANGED) ) - if entity_filter is not None: + if filters: events_query = events_query.filter( - entity_filter | (Events.event_type != EVENT_STATE_CHANGED) + filters.entity_filter() | (Events.event_type != EVENT_STATE_CHANGED) ) query = events_query From b53c06dc2b4fe9a6c7fcdd7d92843168f3685a20 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 20 Sep 2020 17:11:37 -0500 Subject: [PATCH 05/13] simplify --- homeassistant/components/logbook/__init__.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/homeassistant/components/logbook/__init__.py b/homeassistant/components/logbook/__init__.py index 9e942ff95d722..f8bd4caecae29 100644 --- a/homeassistant/components/logbook/__init__.py +++ b/homeassistant/components/logbook/__init__.py @@ -463,19 +463,16 @@ def yield_events(query): query = events_query.union_all(states_query) else: - events_query = _generate_events_query(session) - events_query = _apply_event_time_filter(events_query, start_day, end_day) - events_query = _apply_events_states_filter( - hass, events_query, old_state - ).filter( + query = _generate_events_query(session) + query = _apply_event_time_filter(query, start_day, end_day) + query = _apply_events_states_filter(hass, query, old_state).filter( (States.last_updated == States.last_changed) | (Events.event_type != EVENT_STATE_CHANGED) ) if filters: - events_query = events_query.filter( + query = query.filter( filters.entity_filter() | (Events.event_type != EVENT_STATE_CHANGED) ) - query = events_query query = query.order_by(Events.time_fired) From 3bde072017d9c4d3b56f22018e4120601a948b7c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 20 Sep 2020 17:14:26 -0500 Subject: [PATCH 06/13] state changes are always filtered in sql --- homeassistant/components/logbook/__init__.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/homeassistant/components/logbook/__init__.py b/homeassistant/components/logbook/__init__.py index f8bd4caecae29..b5a3f7f4e61d1 100644 --- a/homeassistant/components/logbook/__init__.py +++ b/homeassistant/components/logbook/__init__.py @@ -433,7 +433,9 @@ def yield_events(query): for row in query.yield_per(1000): event = LazyEventPartialState(row) context_lookup.setdefault(event.context_id, event) - if _keep_event(hass, event, entities_filter): + if event.event_type == EVENT_STATE_CHANGED or _keep_event( + hass, event, entities_filter + ): yield event if entity_ids is not None: @@ -587,13 +589,15 @@ def _apply_state_matchers(events_query, entity_ids): def _keep_event(hass, event, entities_filter): - if event.event_type == EVENT_STATE_CHANGED: - return entities_filter is None or entities_filter(event.entity_id) - if event.event_type in HOMEASSISTANT_EVENTS: - return entities_filter is None or entities_filter(HA_DOMAIN_ENTITY_ID) if event.event_type == EVENT_CALL_SERVICE: return False + if event.event_type in HOMEASSISTANT_EVENTS: + return entities_filter is None or entities_filter(HA_DOMAIN_ENTITY_ID) + + if event.event_type == EVENT_STATE_CHANGED: + return entities_filter is None or entities_filter(event.entity_id) + entity_id = event.data_entity_id if entity_id: return entities_filter is None or entities_filter(entity_id) From 99299070a5c9be8f6db24fe8b043b2497bb596e6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 20 Sep 2020 17:38:39 -0500 Subject: [PATCH 07/13] rewrite --- homeassistant/components/logbook/__init__.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/homeassistant/components/logbook/__init__.py b/homeassistant/components/logbook/__init__.py index b5a3f7f4e61d1..09272f1cfe723 100644 --- a/homeassistant/components/logbook/__init__.py +++ b/homeassistant/components/logbook/__init__.py @@ -577,15 +577,14 @@ def _apply_event_types_filter(hass, query, event_types): def _apply_state_matchers(events_query, entity_ids): - states_matchers = Events.event_data.contains( - ENTITY_ID_JSON_TEMPLATE.format(entity_ids[0]) - ) - for entity_id in entity_ids[1:]: - states_matchers |= Events.event_data.contains( - ENTITY_ID_JSON_TEMPLATE.format(entity_id) + return events_query.filter( + sqlalchemy.or_( + *[ + Events.event_data.contains(ENTITY_ID_JSON_TEMPLATE.format(entity_id)) + for entity_id in entity_ids + ] ) - - return events_query.filter(states_matchers) + ) def _keep_event(hass, event, entities_filter): From 9dd7d9b8bdb3534acad633552c9c563a910f8fd7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 20 Sep 2020 17:43:43 -0500 Subject: [PATCH 08/13] tweaks --- homeassistant/components/logbook/__init__.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/homeassistant/components/logbook/__init__.py b/homeassistant/components/logbook/__init__.py index 09272f1cfe723..8c30039e60cea 100644 --- a/homeassistant/components/logbook/__init__.py +++ b/homeassistant/components/logbook/__init__.py @@ -450,20 +450,16 @@ def yield_events(query): events_query = _apply_event_types_filter( hass, events_query, ALL_EVENT_TYPES_EXCEPT_STATE_CHANGED ) - - states_query = _generate_states_query( - session, start_day, end_day, old_state - ).filter( - (States.last_updated == States.last_changed) - & States.entity_id.in_(entity_ids) - ) - if entity_matches_only: # When entity_matches_only is provided, contexts and events that do not # contain the entity_ids are not included in the logbook response. events_query = _apply_state_matchers(events_query, entity_ids) - query = events_query.union_all(states_query) + query = events_query.union_all( + _generate_states_query( + session, start_day, end_day, old_state, entity_ids + ) + ) else: query = _generate_events_query(session) query = _apply_event_time_filter(query, start_day, end_day) @@ -511,7 +507,7 @@ def _generate_events_query_without_states(session): ) -def _generate_states_query(session, start_day, end_day, old_state): +def _generate_states_query(session, start_day, end_day, old_state, entity_ids): return ( _generate_events_query(session) @@ -531,6 +527,10 @@ def _generate_states_query(session, start_day, end_day, old_state): | sqlalchemy.not_(States.attributes.contains(UNIT_OF_MEASUREMENT_JSON)) ) .filter((States.last_updated > start_day) & (States.last_updated < end_day)) + .filter( + (States.last_updated == States.last_changed) + & States.entity_id.in_(entity_ids) + ) ) From fc7be4f390ec2b95ace9c04f5430596093bc5766 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 20 Sep 2020 17:49:21 -0500 Subject: [PATCH 09/13] tweaks --- homeassistant/components/logbook/__init__.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/homeassistant/components/logbook/__init__.py b/homeassistant/components/logbook/__init__.py index 8c30039e60cea..8b69b1f5fa9bf 100644 --- a/homeassistant/components/logbook/__init__.py +++ b/homeassistant/components/logbook/__init__.py @@ -425,6 +425,7 @@ def _get_events( entity_matches_only=False, ): """Get events for a period of time.""" + entity_attr_cache = EntityAttributeCache(hass) context_lookup = {None: None} @@ -445,17 +446,17 @@ def yield_events(query): old_state = aliased(States, name="old_state") if entity_ids is not None: - events_query = _generate_events_query_without_states(session) - events_query = _apply_event_time_filter(events_query, start_day, end_day) - events_query = _apply_event_types_filter( - hass, events_query, ALL_EVENT_TYPES_EXCEPT_STATE_CHANGED + query = _generate_events_query_without_states(session) + query = _apply_event_time_filter(query, start_day, end_day) + query = _apply_event_types_filter( + hass, query, ALL_EVENT_TYPES_EXCEPT_STATE_CHANGED ) if entity_matches_only: # When entity_matches_only is provided, contexts and events that do not # contain the entity_ids are not included in the logbook response. - events_query = _apply_state_matchers(events_query, entity_ids) + query = _apply_state_matchers(query, entity_ids) - query = events_query.union_all( + query = query.union_all( _generate_states_query( session, start_day, end_day, old_state, entity_ids ) @@ -463,7 +464,9 @@ def yield_events(query): else: query = _generate_events_query(session) query = _apply_event_time_filter(query, start_day, end_day) - query = _apply_events_states_filter(hass, query, old_state).filter( + query = _apply_events_types_and_states_filter( + hass, query, old_state + ).filter( (States.last_updated == States.last_changed) | (Events.event_type != EVENT_STATE_CHANGED) ) @@ -534,7 +537,7 @@ def _generate_states_query(session, start_day, end_day, old_state, entity_ids): ) -def _apply_events_states_filter(hass, query, old_state): +def _apply_events_types_and_states_filter(hass, query, old_state): events_query = ( # The below filter, removes state change events that do not have # and old_state, new_state, or the old and From 36d9b03f2ceeb9b3185d48cf3c1df244cede00c8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 20 Sep 2020 17:51:19 -0500 Subject: [PATCH 10/13] naming --- homeassistant/components/logbook/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/homeassistant/components/logbook/__init__.py b/homeassistant/components/logbook/__init__.py index 8b69b1f5fa9bf..e3dcd66ee0234 100644 --- a/homeassistant/components/logbook/__init__.py +++ b/homeassistant/components/logbook/__init__.py @@ -454,7 +454,7 @@ def yield_events(query): if entity_matches_only: # When entity_matches_only is provided, contexts and events that do not # contain the entity_ids are not included in the logbook response. - query = _apply_state_matchers(query, entity_ids) + query = _apply_event_entity_id_matchers(query, entity_ids) query = query.union_all( _generate_states_query( @@ -579,7 +579,7 @@ def _apply_event_types_filter(hass, query, event_types): ) -def _apply_state_matchers(events_query, entity_ids): +def _apply_event_entity_id_matchers(events_query, entity_ids): return events_query.filter( sqlalchemy.or_( *[ From 8b9d3226d703433cfc0551d162d455d8af15edbc Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 20 Sep 2020 23:33:05 -0500 Subject: [PATCH 11/13] tweaks --- homeassistant/components/logbook/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/logbook/__init__.py b/homeassistant/components/logbook/__init__.py index e3dcd66ee0234..c47ca84fd8604 100644 --- a/homeassistant/components/logbook/__init__.py +++ b/homeassistant/components/logbook/__init__.py @@ -434,6 +434,8 @@ def yield_events(query): for row in query.yield_per(1000): event = LazyEventPartialState(row) context_lookup.setdefault(event.context_id, event) + if event.event_type == EVENT_CALL_SERVICE: + continue if event.event_type == EVENT_STATE_CHANGED or _keep_event( hass, event, entities_filter ): @@ -591,9 +593,6 @@ def _apply_event_entity_id_matchers(events_query, entity_ids): def _keep_event(hass, event, entities_filter): - if event.event_type == EVENT_CALL_SERVICE: - return False - if event.event_type in HOMEASSISTANT_EVENTS: return entities_filter is None or entities_filter(HA_DOMAIN_ENTITY_ID) From 024320c087a4e7f2d3e1cee46e69a604c8c3d1ad Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 21 Sep 2020 08:30:37 -0500 Subject: [PATCH 12/13] DRY --- homeassistant/components/logbook/__init__.py | 47 +++++++++++--------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/homeassistant/components/logbook/__init__.py b/homeassistant/components/logbook/__init__.py index c47ca84fd8604..21e2a260328cb 100644 --- a/homeassistant/components/logbook/__init__.py +++ b/homeassistant/components/logbook/__init__.py @@ -96,6 +96,14 @@ *ALL_EVENT_TYPES_EXCEPT_STATE_CHANGED, ] +EVENT_COLUMNS = [ + Events.event_type, + Events.event_data, + Events.time_fired, + Events.context_id, + Events.context_user_id, +] + SCRIPT_AUTOMATION_EVENTS = [EVENT_AUTOMATION_TRIGGERED, EVENT_SCRIPT_STARTED] LOG_MESSAGE_SCHEMA = vol.Schema( @@ -486,11 +494,7 @@ def yield_events(query): def _generate_events_query(session): return session.query( - Events.event_type, - Events.event_data, - Events.time_fired, - Events.context_id, - Events.context_user_id, + *EVENT_COLUMNS, States.state, States.entity_id, States.domain, @@ -500,11 +504,7 @@ def _generate_events_query(session): def _generate_events_query_without_states(session): return session.query( - Events.event_type, - Events.event_data, - Events.time_fired, - Events.context_id, - Events.context_user_id, + *EVENT_COLUMNS, literal(None).label("state"), literal(None).label("entity_id"), literal(None).label("domain"), @@ -522,15 +522,12 @@ def _generate_states_query(session, start_day, end_day, old_state, entity_ids): # and old_state, new_state, or the old and # new state. # - .filter((old_state.state_id.isnot(None)) & (States.state != old_state.state)) + .filter(_missing_old_state_matcher(old_state)) # # Prefilter out continuous domains that have # ATTR_UNIT_OF_MEASUREMENT as its much faster in sql. # - .filter( - sqlalchemy.not_(States.domain.in_(CONTINUOUS_DOMAINS)) - | sqlalchemy.not_(States.attributes.contains(UNIT_OF_MEASUREMENT_JSON)) - ) + .filter(_continuous_entity_matcher()) .filter((States.last_updated > start_day) & (States.last_updated < end_day)) .filter( (States.last_updated == States.last_changed) @@ -551,9 +548,8 @@ def _apply_events_types_and_states_filter(hass, query, old_state): (Events.event_type != EVENT_STATE_CHANGED) | ( (States.state_id.isnot(None)) - & (old_state.state_id.isnot(None)) & (States.state.isnot(None)) - & (States.state != old_state.state) + & _missing_old_state_matcher(old_state) ) ) # @@ -561,14 +557,25 @@ def _apply_events_types_and_states_filter(hass, query, old_state): # ATTR_UNIT_OF_MEASUREMENT as its much faster in sql. # .filter( - (Events.event_type != EVENT_STATE_CHANGED) - | sqlalchemy.not_(States.domain.in_(CONTINUOUS_DOMAINS)) - | sqlalchemy.not_(States.attributes.contains(UNIT_OF_MEASUREMENT_JSON)) + (Events.event_type != EVENT_STATE_CHANGED) | _continuous_entity_matcher() ) ) return _apply_event_types_filter(hass, events_query, ALL_EVENT_TYPES) +def _missing_old_state_matcher(old_state): + return sqlalchemy.and_( + old_state.state_id.isnot(None), (States.state != old_state.state) + ) + + +def _continuous_entity_matcher(): + return sqlalchemy.or_( + sqlalchemy.not_(States.domain.in_(CONTINUOUS_DOMAINS)), + sqlalchemy.not_(States.attributes.contains(UNIT_OF_MEASUREMENT_JSON)), + ) + + def _apply_event_time_filter(events_query, start_day, end_day): return events_query.filter( (Events.time_fired > start_day) & (Events.time_fired < end_day) From 8a49922975e0f5f399abe90e246273b7177eadca Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 21 Sep 2020 08:34:56 -0500 Subject: [PATCH 13/13] DRY --- homeassistant/components/logbook/__init__.py | 38 +++++++------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/homeassistant/components/logbook/__init__.py b/homeassistant/components/logbook/__init__.py index 21e2a260328cb..29068c1e261e2 100644 --- a/homeassistant/components/logbook/__init__.py +++ b/homeassistant/components/logbook/__init__.py @@ -513,20 +513,11 @@ def _generate_events_query_without_states(session): def _generate_states_query(session, start_day, end_day, old_state, entity_ids): - return ( _generate_events_query(session) .outerjoin(Events, (States.event_id == Events.event_id)) .outerjoin(old_state, (States.old_state_id == old_state.state_id)) - # The below filter, removes state change events that do not have - # and old_state, new_state, or the old and - # new state. - # - .filter(_missing_old_state_matcher(old_state)) - # - # Prefilter out continuous domains that have - # ATTR_UNIT_OF_MEASUREMENT as its much faster in sql. - # + .filter(_missing_state_matcher(old_state)) .filter(_continuous_entity_matcher()) .filter((States.last_updated > start_day) & (States.last_updated < end_day)) .filter( @@ -538,24 +529,12 @@ def _generate_states_query(session, start_day, end_day, old_state, entity_ids): def _apply_events_types_and_states_filter(hass, query, old_state): events_query = ( - # The below filter, removes state change events that do not have - # and old_state, new_state, or the old and - # new state. - # query.outerjoin(States, (Events.event_id == States.event_id)) .outerjoin(old_state, (States.old_state_id == old_state.state_id)) .filter( (Events.event_type != EVENT_STATE_CHANGED) - | ( - (States.state_id.isnot(None)) - & (States.state.isnot(None)) - & _missing_old_state_matcher(old_state) - ) + | _missing_state_matcher(old_state) ) - # - # Prefilter out continuous domains that have - # ATTR_UNIT_OF_MEASUREMENT as its much faster in sql. - # .filter( (Events.event_type != EVENT_STATE_CHANGED) | _continuous_entity_matcher() ) @@ -563,13 +542,22 @@ def _apply_events_types_and_states_filter(hass, query, old_state): return _apply_event_types_filter(hass, events_query, ALL_EVENT_TYPES) -def _missing_old_state_matcher(old_state): +def _missing_state_matcher(old_state): + # The below removes state change events that do not have + # and old_state or the old_state is missing (newly added entities) + # or the new_state is missing (removed entities) return sqlalchemy.and_( - old_state.state_id.isnot(None), (States.state != old_state.state) + old_state.state_id.isnot(None), + (States.state != old_state.state), + States.state.isnot(None), ) def _continuous_entity_matcher(): + # + # Prefilter out continuous domains that have + # ATTR_UNIT_OF_MEASUREMENT as its much faster in sql. + # return sqlalchemy.or_( sqlalchemy.not_(States.domain.in_(CONTINUOUS_DOMAINS)), sqlalchemy.not_(States.attributes.contains(UNIT_OF_MEASUREMENT_JSON)),