Improve performance of fetching and storing history and events with the database

homeassistant/components/logbook/models.py

@@ -2,14 +2,14 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from datetime import datetime as dt
 import json
 from typing import Any, cast
 
 from sqlalchemy.engine.row import Row
 
 from homeassistant.const import ATTR_ICON, EVENT_STATE_CHANGED
 from homeassistant.core import Context, Event, State, callback
+import homeassistant.util.dt as dt_util
 
 
 class LazyEventPartialState:
@@ -66,7 +66,7 @@ class EventAsRow:
     data: dict[str, Any]
     context: Context
     context_id: str
-    time_fired: dt
+    time_fired_ts: float
     state_id: int
     event_data: str | None = None
     old_format_icon: None = None
@@ -92,7 +92,7 @@ def async_event_to_row(event: Event) -> EventAsRow | None:
             context_id=event.context.id,
             context_user_id=event.context.user_id,
             context_parent_id=event.context.parent_id,
-            time_fired=event.time_fired,
+            time_fired_ts=dt_util.utc_to_timestamp(event.time_fired),
             state_id=hash(event),
         )
     # States are prefiltered so we never get states
@@ -107,7 +107,7 @@ def async_event_to_row(event: Event) -> EventAsRow | None:
         context_id=new_state.context.id,
         context_user_id=new_state.context.user_id,
         context_parent_id=new_state.context.parent_id,
-        time_fired=new_state.last_updated,
+        time_fired_ts=dt_util.utc_to_timestamp(new_state.last_updated),
         state_id=hash(event),
         icon=new_state.attributes.get(ATTR_ICON),
     )

homeassistant/components/logbook/processor.py

@@ -388,12 +388,14 @@ def _rows_match(row: Row | EventAsRow, other_row: Row | EventAsRow) -> bool:
 
 def _row_time_fired_isoformat(row: Row | EventAsRow) -> str:
     """Convert the row timed_fired to isoformat."""
-    return process_timestamp_to_utc_isoformat(row.time_fired or dt_util.utcnow())
+    return process_timestamp_to_utc_isoformat(
+        dt_util.utc_from_timestamp(row.time_fired_ts) or dt_util.utcnow()
+    )
 
 
 def _row_time_fired_timestamp(row: Row | EventAsRow) -> float:
     """Convert the row timed_fired to timestamp."""
-    return process_datetime_to_timestamp(row.time_fired or dt_util.utcnow())
+    return row.time_fired_ts or process_datetime_to_timestamp(dt_util.utcnow())
 
 
 class EntityNameCache:

homeassistant/components/logbook/queries/__init__.py

@@ -7,6 +7,7 @@
 
 from homeassistant.components.recorder.filters import Filters
 from homeassistant.helpers.json import json_dumps
+from homeassistant.util import dt as dt_util
 
 from .all import all_stmt
 from .devices import devices_stmt
@@ -15,16 +16,17 @@
 
 
 def statement_for_request(
-    start_day: dt,
-    end_day: dt,
+    start_day_dt: dt,
+    end_day_dt: dt,
     event_types: tuple[str, ...],
     entity_ids: list[str] | None = None,
     device_ids: list[str] | None = None,
     filters: Filters | None = None,
     context_id: str | None = None,
 ) -> StatementLambdaElement:
     """Generate the logbook statement for a logbook request."""
-
+    start_day = dt_util.utc_to_timestamp(start_day_dt)
+    end_day = dt_util.utc_to_timestamp(end_day_dt)
     # No entities: logbook sends everything for the timeframe
     # limited by the context_id and the yaml configured filter
     if not entity_ids and not device_ids:

homeassistant/components/logbook/queries/all.py

@@ -1,15 +1,13 @@
 """All queries for logbook."""
 from __future__ import annotations
 
-from datetime import datetime as dt
-
 from sqlalchemy import lambda_stmt
 from sqlalchemy.orm import Query
 from sqlalchemy.sql.elements import ClauseList
 from sqlalchemy.sql.lambdas import StatementLambdaElement
 
 from homeassistant.components.recorder.db_schema import (
-    LAST_UPDATED_INDEX,
+    LAST_UPDATED_INDEX_TS,
     Events,
     States,
 )
@@ -23,8 +21,8 @@
 
 
 def all_stmt(
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
     states_entity_filter: ClauseList | None = None,
     events_entity_filter: ClauseList | None = None,
@@ -53,22 +51,24 @@ def all_stmt(
         else:
             stmt += lambda s: s.union_all(_states_query_for_all(start_day, end_day))
 
-    stmt += lambda s: s.order_by(Events.time_fired)
+    stmt += lambda s: s.order_by(Events.time_fired_ts)
     return stmt
 
 
-def _states_query_for_all(start_day: dt, end_day: dt) -> Query:
+def _states_query_for_all(start_day: float, end_day: float) -> Query:
     return apply_states_filters(_apply_all_hints(select_states()), start_day, end_day)
 
 
 def _apply_all_hints(query: Query) -> Query:
     """Force mysql to use the right index on large selects."""
     return query.with_hint(
-        States, f"FORCE INDEX ({LAST_UPDATED_INDEX})", dialect_name="mysql"
+        States, f"FORCE INDEX ({LAST_UPDATED_INDEX_TS})", dialect_name="mysql"
     )
 
 
-def _states_query_for_context_id(start_day: dt, end_day: dt, context_id: str) -> Query:
+def _states_query_for_context_id(
+    start_day: float, end_day: float, context_id: str
+) -> Query:
     return apply_states_filters(select_states(), start_day, end_day).where(
         States.context_id == context_id
     )

homeassistant/components/logbook/queries/common.py

@@ -1,8 +1,6 @@
 """Queries for logbook."""
 from __future__ import annotations
 
-from datetime import datetime as dt
-
 import sqlalchemy
 from sqlalchemy import select
 from sqlalchemy.orm import Query
@@ -47,7 +45,7 @@
     Events.event_id.label("event_id"),
     Events.event_type.label("event_type"),
     Events.event_data.label("event_data"),
-    Events.time_fired.label("time_fired"),
+    Events.time_fired_ts.label("time_fired_ts"),
     Events.context_id.label("context_id"),
     Events.context_user_id.label("context_user_id"),
     Events.context_parent_id.label("context_parent_id"),
@@ -79,7 +77,7 @@
         "event_type"
     ),
     literal(value=None, type_=sqlalchemy.Text).label("event_data"),
-    States.last_updated.label("time_fired"),
+    States.last_updated_ts.label("time_fired_ts"),
     States.context_id.label("context_id"),
     States.context_user_id.label("context_user_id"),
     States.context_parent_id.label("context_parent_id"),
@@ -108,14 +106,14 @@
 
 
 def select_events_context_id_subquery(
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
 ) -> Select:
     """Generate the select for a context_id subquery."""
     return (
         select(Events.context_id)
-        .where((Events.time_fired > start_day) & (Events.time_fired < end_day))
+        .where((Events.time_fired_ts > start_day) & (Events.time_fired_ts < end_day))
         .where(Events.event_type.in_(event_types))
         .outerjoin(EventData, (Events.data_id == EventData.data_id))
     )
@@ -142,12 +140,12 @@ def select_states_context_only() -> Select:
 
 
 def select_events_without_states(
-    start_day: dt, end_day: dt, event_types: tuple[str, ...]
+    start_day: float, end_day: float, event_types: tuple[str, ...]
 ) -> Select:
     """Generate an events select that does not join states."""
     return (
         select(*EVENT_ROWS_NO_STATES, NOT_CONTEXT_ONLY)
-        .where((Events.time_fired > start_day) & (Events.time_fired < end_day))
+        .where((Events.time_fired_ts > start_day) & (Events.time_fired_ts < end_day))
         .where(Events.event_type.in_(event_types))
         .outerjoin(EventData, (Events.data_id == EventData.data_id))
     )
@@ -163,7 +161,7 @@ def select_states() -> Select:
 
 
 def legacy_select_events_context_id(
-    start_day: dt, end_day: dt, context_id: str
+    start_day: float, end_day: float, context_id: str
 ) -> Select:
     """Generate a legacy events context id select that also joins states."""
     # This can be removed once we no longer have event_ids in the states table
@@ -176,33 +174,35 @@ def legacy_select_events_context_id(
         )
         .outerjoin(States, (Events.event_id == States.event_id))
         .where(
-            (States.last_updated == States.last_changed) | States.last_changed.is_(None)
+            (States.last_updated_ts == States.last_changed_ts)
+            | States.last_changed_ts.is_(None)
         )
         .where(_not_continuous_entity_matcher())
         .outerjoin(
             StateAttributes, (States.attributes_id == StateAttributes.attributes_id)
         )
-        .where((Events.time_fired > start_day) & (Events.time_fired < end_day))
+        .where((Events.time_fired_ts > start_day) & (Events.time_fired_ts < end_day))
         .where(Events.context_id == context_id)
     )
 
 
-def apply_states_filters(query: Query, start_day: dt, end_day: dt) -> Query:
+def apply_states_filters(query: Query, start_day: float, end_day: float) -> Query:
     """Filter states by time range.
 
     Filters states that do not have an old state or new state (added / removed)
     Filters states that are in a continuous domain with a UOM.
-    Filters states that do not have matching last_updated and last_changed.
+    Filters states that do not have matching last_updated_ts and last_changed_ts.
     """
     return (
         query.filter(
-            (States.last_updated > start_day) & (States.last_updated < end_day)
+            (States.last_updated_ts > start_day) & (States.last_updated_ts < end_day)
         )
         .outerjoin(OLD_STATE, (States.old_state_id == OLD_STATE.state_id))
         .where(_missing_state_matcher())
         .where(_not_continuous_entity_matcher())
         .where(
-            (States.last_updated == States.last_changed) | States.last_changed.is_(None)
+            (States.last_updated_ts == States.last_changed_ts)
+            | States.last_changed_ts.is_(None)
         )
         .outerjoin(
             StateAttributes, (States.attributes_id == StateAttributes.attributes_id)

homeassistant/components/logbook/queries/devices.py

@@ -2,7 +2,6 @@
 from __future__ import annotations
 
 from collections.abc import Iterable
-from datetime import datetime as dt
 
 import sqlalchemy
 from sqlalchemy import lambda_stmt, select
@@ -29,8 +28,8 @@
 
 
 def _select_device_id_context_ids_sub_query(
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
     json_quotable_device_ids: list[str],
 ) -> CompoundSelect:
@@ -43,8 +42,8 @@ def _select_device_id_context_ids_sub_query(
 
 def _apply_devices_context_union(
     query: Query,
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
     json_quotable_device_ids: list[str],
 ) -> CompoundSelect:
@@ -70,8 +69,8 @@ def _apply_devices_context_union(
 
 
 def devices_stmt(
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
     json_quotable_device_ids: list[str],
 ) -> StatementLambdaElement:
@@ -85,7 +84,7 @@ def devices_stmt(
             end_day,
             event_types,
             json_quotable_device_ids,
-        ).order_by(Events.time_fired)
+        ).order_by(Events.time_fired_ts)
     )
     return stmt
 

homeassistant/components/logbook/queries/entities.py

@@ -2,7 +2,6 @@
 from __future__ import annotations
 
 from collections.abc import Iterable
-from datetime import datetime as dt
 
 import sqlalchemy
 from sqlalchemy import lambda_stmt, select, union_all
@@ -12,7 +11,7 @@
 
 from homeassistant.components.recorder.db_schema import (
     ENTITY_ID_IN_EVENT,
-    ENTITY_ID_LAST_UPDATED_INDEX,
+    ENTITY_ID_LAST_UPDATED_INDEX_TS,
     OLD_ENTITY_ID_IN_EVENT,
     EventData,
     Events,
@@ -32,8 +31,8 @@
 
 
 def _select_entities_context_ids_sub_query(
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
     entity_ids: list[str],
     json_quoted_entity_ids: list[str],
@@ -44,16 +43,18 @@ def _select_entities_context_ids_sub_query(
             apply_event_entity_id_matchers(json_quoted_entity_ids)
         ),
         apply_entities_hints(select(States.context_id))
-        .filter((States.last_updated > start_day) & (States.last_updated < end_day))
+        .filter(
+            (States.last_updated_ts > start_day) & (States.last_updated_ts < end_day)
+        )
         .where(States.entity_id.in_(entity_ids)),
     )
     return select(union.c.context_id).group_by(union.c.context_id)
 
 
 def _apply_entities_context_union(
     query: Query,
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
     entity_ids: list[str],
     json_quoted_entity_ids: list[str],
@@ -87,8 +88,8 @@ def _apply_entities_context_union(
 
 
 def entities_stmt(
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
     entity_ids: list[str],
     json_quoted_entity_ids: list[str],
@@ -104,12 +105,12 @@ def entities_stmt(
             event_types,
             entity_ids,
             json_quoted_entity_ids,
-        ).order_by(Events.time_fired)
+        ).order_by(Events.time_fired_ts)
     )
 
 
 def states_query_for_entity_ids(
-    start_day: dt, end_day: dt, entity_ids: list[str]
+    start_day: float, end_day: float, entity_ids: list[str]
 ) -> Query:
     """Generate a select for states from the States table for specific entities."""
     return apply_states_filters(
@@ -136,5 +137,5 @@ def apply_event_entity_id_matchers(
 def apply_entities_hints(query: Query) -> Query:
     """Force mysql to use the right index on large selects."""
     return query.with_hint(
-        States, f"FORCE INDEX ({ENTITY_ID_LAST_UPDATED_INDEX})", dialect_name="mysql"
+        States, f"FORCE INDEX ({ENTITY_ID_LAST_UPDATED_INDEX_TS})", dialect_name="mysql"
     )