From b83de90b327e25f6e467b415dae10d7e852a7988 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 15 May 2022 07:33:27 -0500 Subject: [PATCH 1/9] Reduce logbook websocket payload size Breaking: - The entity name is now always shown with the current name instead of the old name if it was renamed. If the entity no longer exists we now show the original entity_id instead Requires https://github.com/home-assistant/frontend/pull/12667 - The name of the entity is no longer sent, and the current name in the state machine is now used - We no longer parse attributes in python. This task is offloaded to the database since we only need it for the icon. --- homeassistant/components/logbook/__init__.py | 45 +++++++++++--------- homeassistant/components/logbook/queries.py | 23 +++++++--- tests/components/logbook/test_init.py | 2 + 3 files changed, 44 insertions(+), 26 deletions(-) diff --git a/homeassistant/components/logbook/__init__.py b/homeassistant/components/logbook/__init__.py index 4bef1f1a23d78b..92c1898b61c914 100644 --- a/homeassistant/components/logbook/__init__.py +++ b/homeassistant/components/logbook/__init__.py @@ -40,7 +40,6 @@ ATTR_SERVICE, EVENT_CALL_SERVICE, EVENT_LOGBOOK_ENTRY, - EVENT_STATE_CHANGED, ) from homeassistant.core import ( Context, @@ -65,14 +64,12 @@ from homeassistant.loader import bind_hass import homeassistant.util.dt as dt_util -from .queries import statement_for_request +from .queries import PSUEDO_EVENT_STATE_CHANGED, statement_for_request _LOGGER = logging.getLogger(__name__) -FRIENDLY_NAME_JSON_EXTRACT = re.compile('"friendly_name": ?"([^"]+)"') ENTITY_ID_JSON_EXTRACT = re.compile('"entity_id": ?"([^"]+)"') DOMAIN_JSON_EXTRACT = re.compile('"domain": ?"([^"]+)"') -ICON_JSON_EXTRACT = re.compile('"icon": ?"([^"]+)"') ATTR_MESSAGE = "message" DOMAIN = "logbook" @@ -235,6 +232,7 @@ def _ws_formatted_get_events( entities_filter, context_id, True, + False, ), ) ) @@ -368,6 +366,7 @@ def json_events() -> web.Response: self.entities_filter, context_id, False, + True, ) ) @@ -385,6 +384,7 @@ def _humanify( ], entity_name_cache: EntityNameCache, format_time: Callable[[Row], Any], + include_entity_name: bool = True, ) -> Generator[dict[str, Any], None, None]: """Generate a converted list of events into entries.""" # Continuous sensors, will be excluded from the logbook @@ -419,13 +419,13 @@ def _keep_row(row: Row, event_type: str) -> bool: continue event_type = row.event_type if event_type == EVENT_CALL_SERVICE or ( - event_type != EVENT_STATE_CHANGED + event_type is not PSUEDO_EVENT_STATE_CHANGED and entities_filter is not None and not _keep_row(row, event_type) ): continue - if event_type == EVENT_STATE_CHANGED: + if event_type == PSUEDO_EVENT_STATE_CHANGED: entity_id = row.entity_id assert entity_id is not None # Skip continuous sensors @@ -439,14 +439,15 @@ def _keep_row(row: Row, event_type: str) -> bool: data = { LOGBOOK_ENTRY_WHEN: format_time(row), - LOGBOOK_ENTRY_NAME: entity_name_cache.get(entity_id, row), LOGBOOK_ENTRY_STATE: row.state, LOGBOOK_ENTRY_ENTITY_ID: entity_id, } - if icon := _row_attributes_extract(row, ICON_JSON_EXTRACT): + if include_entity_name: + data[LOGBOOK_ENTRY_NAME] = entity_name_cache.get(entity_id, row) + if icon := row.icon: data[LOGBOOK_ENTRY_ICON] = icon - context_augmenter.augment(data, row, context_id) + context_augmenter.augment(data, row, context_id, include_entity_name) yield data elif event_type in external_events: @@ -454,7 +455,7 @@ def _keep_row(row: Row, event_type: str) -> bool: data = describe_event(event_cache.get(row)) data[LOGBOOK_ENTRY_WHEN] = format_time(row) data[LOGBOOK_ENTRY_DOMAIN] = domain - context_augmenter.augment(data, row, context_id) + context_augmenter.augment(data, row, context_id, include_entity_name) yield data elif event_type == EVENT_LOGBOOK_ENTRY: @@ -474,7 +475,7 @@ def _keep_row(row: Row, event_type: str) -> bool: LOGBOOK_ENTRY_DOMAIN: entry_domain, LOGBOOK_ENTRY_ENTITY_ID: entry_entity_id, } - context_augmenter.augment(data, row, context_id) + context_augmenter.augment(data, row, context_id, include_entity_name) yield data @@ -487,6 +488,7 @@ def _get_events( entities_filter: EntityFilter | Callable[[str], bool] | None = None, context_id: str | None = None, timestamp: bool = False, + include_entity_name: bool = True, ) -> list[dict[str, Any]]: """Get events for a period of time.""" assert not ( @@ -540,6 +542,7 @@ def yield_rows(query: Query) -> Generator[Row, None, None]: external_events, entity_name_cache, format_time, + include_entity_name, ) ) @@ -562,7 +565,9 @@ def __init__( self.external_events = external_events self.event_cache = event_cache - def augment(self, data: dict[str, Any], row: Row, context_id: str) -> None: + def augment( + self, data: dict[str, Any], row: Row, context_id: str, include_entity_name: bool + ) -> None: """Augment data from the row and cache.""" if context_user_id := row.context_user_id: data[CONTEXT_USER_ID] = context_user_id @@ -589,9 +594,10 @@ def augment(self, data: dict[str, Any], row: Row, context_id: str) -> None: # State change if context_entity_id := context_row.entity_id: data[CONTEXT_ENTITY_ID] = context_entity_id - data[CONTEXT_ENTITY_ID_NAME] = self.entity_name_cache.get( - context_entity_id, context_row - ) + if include_entity_name: + data[CONTEXT_ENTITY_ID_NAME] = self.entity_name_cache.get( + context_entity_id, context_row + ) data[CONTEXT_EVENT_TYPE] = event_type return @@ -619,9 +625,10 @@ def augment(self, data: dict[str, Any], row: Row, context_id: str) -> None: if not (attr_entity_id := described.get(ATTR_ENTITY_ID)): return data[CONTEXT_ENTITY_ID] = attr_entity_id - data[CONTEXT_ENTITY_ID_NAME] = self.entity_name_cache.get( - attr_entity_id, context_row - ) + if include_entity_name: + data[CONTEXT_ENTITY_ID_NAME] = self.entity_name_cache.get( + attr_entity_id, context_row + ) def _is_sensor_continuous(ent_reg: er.EntityRegistry, entity_id: str) -> bool: @@ -735,7 +742,7 @@ def get(self, entity_id: str, row: Row) -> str: friendly_name := current_state.attributes.get(ATTR_FRIENDLY_NAME) ): self._names[entity_id] = friendly_name - elif extracted_name := _row_attributes_extract(row, FRIENDLY_NAME_JSON_EXTRACT): + elif extracted_name := row.friendly_name: self._names[entity_id] = extracted_name else: return split_entity_id(entity_id)[1].replace("_", " ") diff --git a/homeassistant/components/logbook/queries.py b/homeassistant/components/logbook/queries.py index 89c530aec43d81..2b2471ec7d5eec 100644 --- a/homeassistant/components/logbook/queries.py +++ b/homeassistant/components/logbook/queries.py @@ -5,7 +5,7 @@ from datetime import datetime as dt import sqlalchemy -from sqlalchemy import lambda_stmt, select, union_all +from sqlalchemy import JSON, lambda_stmt, select, type_coerce, union_all from sqlalchemy.orm import Query, aliased from sqlalchemy.sql.elements import ClauseList from sqlalchemy.sql.expression import literal @@ -23,7 +23,6 @@ States, ) from homeassistant.components.sensor import DOMAIN as SENSOR_DOMAIN -from homeassistant.const import EVENT_STATE_CHANGED ENTITY_ID_JSON_TEMPLATE = '%"entity_id":"{}"%' @@ -34,7 +33,15 @@ UNIT_OF_MEASUREMENT_JSON_LIKE = f"%{UNIT_OF_MEASUREMENT_JSON}%" OLD_STATE = aliased(States, name="old_state") +SHARED_ATTRS_JSON = type_coerce(StateAttributes.shared_attrs, JSON(none_as_null=True)) +PSUEDO_EVENT_STATE_CHANGED = None +# Since we don't store event_types and None +# and we don't store state_changed in events +# we use a NULL for state_changed events +# when we synthesize them from the states table +# since it avoids another column being sent +# in the payload EVENT_COLUMNS = ( Events.event_id.label("event_id"), @@ -50,18 +57,18 @@ States.state_id.label("state_id"), States.state.label("state"), States.entity_id.label("entity_id"), - States.attributes.label("attributes"), - StateAttributes.shared_attrs.label("shared_attrs"), + SHARED_ATTRS_JSON["icon"].as_string().label("icon"), ) + EMPTY_STATE_COLUMNS = ( literal(value=None, type_=sqlalchemy.String).label("state_id"), literal(value=None, type_=sqlalchemy.String).label("state"), literal(value=None, type_=sqlalchemy.String).label("entity_id"), - literal(value=None, type_=sqlalchemy.Text).label("attributes"), - literal(value=None, type_=sqlalchemy.Text).label("shared_attrs"), + literal(value=None, type_=sqlalchemy.String).label("icon"), ) + EVENT_ROWS_NO_STATES = ( *EVENT_COLUMNS, EventData.shared_data.label("shared_data"), @@ -326,7 +333,9 @@ def _select_states() -> Select: """Generate a states select that formats the states table as event rows.""" return select( literal(value=None, type_=sqlalchemy.Text).label("event_id"), - literal(value=EVENT_STATE_CHANGED, type_=sqlalchemy.String).label("event_type"), + literal(value=PSUEDO_EVENT_STATE_CHANGED, type_=sqlalchemy.String).label( + "event_type" + ), literal(value=None, type_=sqlalchemy.Text).label("event_data"), States.last_updated.label("time_fired"), States.context_id.label("context_id"), diff --git a/tests/components/logbook/test_init.py b/tests/components/logbook/test_init.py index a515afdf16e327..35d0f5fb5d00ac 100644 --- a/tests/components/logbook/test_init.py +++ b/tests/components/logbook/test_init.py @@ -338,6 +338,8 @@ def create_state_changed_event_from_old_new( row.domain = entity_id and ha.split_entity_id(entity_id)[0] row.context_only = False row.context_id = None + row.friendly_name = None + row.icon = None row.context_user_id = None row.context_parent_id = None row.old_state_id = old_state and 1 From d03aad3c1cb66f4adac1a0eba8b0b16efdf02666 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 15 May 2022 07:36:46 -0500 Subject: [PATCH 2/9] get old format as well --- homeassistant/components/logbook/__init__.py | 2 +- homeassistant/components/logbook/queries.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/homeassistant/components/logbook/__init__.py b/homeassistant/components/logbook/__init__.py index 92c1898b61c914..d9287415d08df8 100644 --- a/homeassistant/components/logbook/__init__.py +++ b/homeassistant/components/logbook/__init__.py @@ -444,7 +444,7 @@ def _keep_row(row: Row, event_type: str) -> bool: } if include_entity_name: data[LOGBOOK_ENTRY_NAME] = entity_name_cache.get(entity_id, row) - if icon := row.icon: + if icon := row.icon or row.old_format_icon: data[LOGBOOK_ENTRY_ICON] = icon context_augmenter.augment(data, row, context_id, include_entity_name) diff --git a/homeassistant/components/logbook/queries.py b/homeassistant/components/logbook/queries.py index 2b2471ec7d5eec..1d9c9eab3d2772 100644 --- a/homeassistant/components/logbook/queries.py +++ b/homeassistant/components/logbook/queries.py @@ -34,6 +34,7 @@ OLD_STATE = aliased(States, name="old_state") SHARED_ATTRS_JSON = type_coerce(StateAttributes.shared_attrs, JSON(none_as_null=True)) +OLD_FORMAT_ATTRS_JSON = type_coerce(States.attributes, JSON(none_as_null=True)) PSUEDO_EVENT_STATE_CHANGED = None # Since we don't store event_types and None @@ -58,6 +59,7 @@ States.state.label("state"), States.entity_id.label("entity_id"), SHARED_ATTRS_JSON["icon"].as_string().label("icon"), + OLD_FORMAT_ATTRS_JSON["icon"].as_string().label("old_format_icon"), ) @@ -66,6 +68,7 @@ literal(value=None, type_=sqlalchemy.String).label("state"), literal(value=None, type_=sqlalchemy.String).label("entity_id"), literal(value=None, type_=sqlalchemy.String).label("icon"), + literal(value=None, type_=sqlalchemy.String).label("old_format_icon"), ) From cfd35a4d6403cdb68f9e63f52855f4351bdf81f8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 15 May 2022 07:51:06 -0500 Subject: [PATCH 3/9] tweak --- homeassistant/components/logbook/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/homeassistant/components/logbook/__init__.py b/homeassistant/components/logbook/__init__.py index d9287415d08df8..98543f62bde025 100644 --- a/homeassistant/components/logbook/__init__.py +++ b/homeassistant/components/logbook/__init__.py @@ -742,8 +742,6 @@ def get(self, entity_id: str, row: Row) -> str: friendly_name := current_state.attributes.get(ATTR_FRIENDLY_NAME) ): self._names[entity_id] = friendly_name - elif extracted_name := row.friendly_name: - self._names[entity_id] = extracted_name else: return split_entity_id(entity_id)[1].replace("_", " ") From 817b4b3a61d25fbdd5038bc9722f13cf8ae9785b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 15 May 2022 07:53:07 -0500 Subject: [PATCH 4/9] fix --- tests/components/logbook/test_init.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/components/logbook/test_init.py b/tests/components/logbook/test_init.py index 35d0f5fb5d00ac..1f3cbae93ca5c4 100644 --- a/tests/components/logbook/test_init.py +++ b/tests/components/logbook/test_init.py @@ -340,6 +340,7 @@ def create_state_changed_event_from_old_new( row.context_id = None row.friendly_name = None row.icon = None + row.old_format_icon = None row.context_user_id = None row.context_parent_id = None row.old_state_id = old_state and 1 @@ -721,7 +722,7 @@ async def test_logbook_entity_no_longer_in_state_machine( ) assert response.status == HTTPStatus.OK json_dict = await response.json() - assert json_dict[0]["name"] == "Alarm Control Panel" + assert json_dict[0]["name"] == "area 001" async def test_filter_continuous_sensor_values( From 6c8289637721710774f399e3e6b4c6671733f74b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 15 May 2022 07:55:49 -0500 Subject: [PATCH 5/9] fix test --- tests/components/logbook/test_init.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/components/logbook/test_init.py b/tests/components/logbook/test_init.py index 1f3cbae93ca5c4..ed95b4d10bcdd2 100644 --- a/tests/components/logbook/test_init.py +++ b/tests/components/logbook/test_init.py @@ -30,7 +30,6 @@ EVENT_HOMEASSISTANT_START, EVENT_HOMEASSISTANT_STARTED, EVENT_HOMEASSISTANT_STOP, - EVENT_STATE_CHANGED, STATE_OFF, STATE_ON, ) @@ -327,7 +326,7 @@ def create_state_changed_event_from_old_new( ], ) - row.event_type = EVENT_STATE_CHANGED + row.event_type = logbook.PSUEDO_EVENT_STATE_CHANGED row.event_data = "{}" row.shared_data = "{}" row.attributes = attributes_json From 20dcc9446420121aca7af1bedc1cb093517eccce Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 15 May 2022 08:17:56 -0500 Subject: [PATCH 6/9] cleanups --- homeassistant/components/logbook/__init__.py | 2 +- homeassistant/components/logbook/queries.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/homeassistant/components/logbook/__init__.py b/homeassistant/components/logbook/__init__.py index 98543f62bde025..35fa17bb3a4f26 100644 --- a/homeassistant/components/logbook/__init__.py +++ b/homeassistant/components/logbook/__init__.py @@ -425,7 +425,7 @@ def _keep_row(row: Row, event_type: str) -> bool: ): continue - if event_type == PSUEDO_EVENT_STATE_CHANGED: + if event_type is PSUEDO_EVENT_STATE_CHANGED: entity_id = row.entity_id assert entity_id is not None # Skip continuous sensors diff --git a/homeassistant/components/logbook/queries.py b/homeassistant/components/logbook/queries.py index 1d9c9eab3d2772..c7c21074e0eab7 100644 --- a/homeassistant/components/logbook/queries.py +++ b/homeassistant/components/logbook/queries.py @@ -336,6 +336,10 @@ def _select_states() -> Select: """Generate a states select that formats the states table as event rows.""" return select( literal(value=None, type_=sqlalchemy.Text).label("event_id"), + # We use PSUEDO_EVENT_STATE_CHANGED aka None for + # state_changed events since it takes up less + # space in the response and every row has to be + # marked with the event_type literal(value=PSUEDO_EVENT_STATE_CHANGED, type_=sqlalchemy.String).label( "event_type" ), From b3668d284ac1de42e1ebd9293e213a6e639abdf1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 15 May 2022 09:27:06 -0500 Subject: [PATCH 7/9] postgresql is special --- homeassistant/components/logbook/queries.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/logbook/queries.py b/homeassistant/components/logbook/queries.py index c7c21074e0eab7..1e43023a32ed18 100644 --- a/homeassistant/components/logbook/queries.py +++ b/homeassistant/components/logbook/queries.py @@ -5,7 +5,7 @@ from datetime import datetime as dt import sqlalchemy -from sqlalchemy import JSON, lambda_stmt, select, type_coerce, union_all +from sqlalchemy import JSON, Text, lambda_stmt, select, type_coerce, union_all from sqlalchemy.orm import Query, aliased from sqlalchemy.sql.elements import ClauseList from sqlalchemy.sql.expression import literal @@ -33,8 +33,18 @@ UNIT_OF_MEASUREMENT_JSON_LIKE = f"%{UNIT_OF_MEASUREMENT_JSON}%" OLD_STATE = aliased(States, name="old_state") -SHARED_ATTRS_JSON = type_coerce(StateAttributes.shared_attrs, JSON(none_as_null=True)) -OLD_FORMAT_ATTRS_JSON = type_coerce(States.attributes, JSON(none_as_null=True)) + +JSON_VARIENT_CAST = Text().with_variant( + type_=JSON(none_as_null=True), dialect_name="postgresql" +) + +SHARED_ATTRS_JSON = type_coerce( + StateAttributes.shared_attrs.cast(JSON_VARIENT_CAST), JSON(none_as_null=True) +) +OLD_FORMAT_ATTRS_JSON = type_coerce( + States.attributes.cast(JSON_VARIENT_CAST), JSON(none_as_null=True) +) + PSUEDO_EVENT_STATE_CHANGED = None # Since we don't store event_types and None From 71a44e74a70067fff19d69052e8eaa0562d13e73 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 15 May 2022 09:29:36 -0500 Subject: [PATCH 8/9] tweak --- homeassistant/components/logbook/queries.py | 6 ++---- homeassistant/components/recorder/models.py | 5 +++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/logbook/queries.py b/homeassistant/components/logbook/queries.py index 1e43023a32ed18..6fe20bfc5618b6 100644 --- a/homeassistant/components/logbook/queries.py +++ b/homeassistant/components/logbook/queries.py @@ -5,7 +5,7 @@ from datetime import datetime as dt import sqlalchemy -from sqlalchemy import JSON, Text, lambda_stmt, select, type_coerce, union_all +from sqlalchemy import JSON, lambda_stmt, select, type_coerce, union_all from sqlalchemy.orm import Query, aliased from sqlalchemy.sql.elements import ClauseList from sqlalchemy.sql.expression import literal @@ -16,6 +16,7 @@ from homeassistant.components.recorder.filters import Filters from homeassistant.components.recorder.models import ( ENTITY_ID_LAST_UPDATED_INDEX, + JSON_VARIENT_CAST, LAST_UPDATED_INDEX, EventData, Events, @@ -34,9 +35,6 @@ OLD_STATE = aliased(States, name="old_state") -JSON_VARIENT_CAST = Text().with_variant( - type_=JSON(none_as_null=True), dialect_name="postgresql" -) SHARED_ATTRS_JSON = type_coerce( StateAttributes.shared_attrs.cast(JSON_VARIENT_CAST), JSON(none_as_null=True) diff --git a/homeassistant/components/recorder/models.py b/homeassistant/components/recorder/models.py index d64d85f3ce4959..0e15f3fd1c2d96 100644 --- a/homeassistant/components/recorder/models.py +++ b/homeassistant/components/recorder/models.py @@ -102,6 +102,11 @@ def result_processor(self, dialect, coltype): # type: ignore[no-untyped-def] return lambda value: None if value is None else ciso8601.parse_datetime(value) +JSON_VARIENT_CAST = ( + Text() + .with_variant(postgresql.JSON(none_as_null=True), "postgresql") + .with_variant(mysql.LONGTEXT, "mysql") +) DATETIME_TYPE = ( DateTime(timezone=True) .with_variant(mysql.DATETIME(timezone=True, fsp=6), "mysql") From 3ac395298fe5b3a02665f984a98568211efe1dbf Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 15 May 2022 09:31:46 -0500 Subject: [PATCH 9/9] tweak --- homeassistant/components/recorder/models.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/recorder/models.py b/homeassistant/components/recorder/models.py index 0e15f3fd1c2d96..f5498e941d3eb9 100644 --- a/homeassistant/components/recorder/models.py +++ b/homeassistant/components/recorder/models.py @@ -102,10 +102,8 @@ def result_processor(self, dialect, coltype): # type: ignore[no-untyped-def] return lambda value: None if value is None else ciso8601.parse_datetime(value) -JSON_VARIENT_CAST = ( - Text() - .with_variant(postgresql.JSON(none_as_null=True), "postgresql") - .with_variant(mysql.LONGTEXT, "mysql") +JSON_VARIENT_CAST = Text().with_variant( + postgresql.JSON(none_as_null=True), "postgresql" ) DATETIME_TYPE = ( DateTime(timezone=True)