From b9bb35c39b5757e91c8f6e5fbb0847ad66504c3e Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Fri, 30 Dec 2022 09:39:32 -1000
Subject: [PATCH 01/38] Avoid the overhead of storing database timestamps as
 strings

Reduces database size by ~30%
---
 homeassistant/components/recorder/db_schema.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/homeassistant/components/recorder/db_schema.py b/homeassistant/components/recorder/db_schema.py
index 2c8c1ad2fffb5c..6d3a7c9aead758 100644
--- a/homeassistant/components/recorder/db_schema.py
+++ b/homeassistant/components/recorder/db_schema.py
@@ -4,6 +4,7 @@
 from collections.abc import Callable
 from datetime import datetime, timedelta
 import logging
+import time
 from typing import Any, TypeVar, cast
 
 import ciso8601
@@ -121,6 +122,7 @@ def result_processor(self, dialect, coltype):  # type: ignore[no-untyped-def]
     .with_variant(oracle.DOUBLE_PRECISION(), "oracle")
     .with_variant(postgresql.DOUBLE_PRECISION(), "postgresql")
 )
+TIMESTAMP_TYPE = Float(7)
 
 
 class JSONLiteral(JSON):  # type: ignore[misc]
@@ -155,7 +157,8 @@ class Events(Base):  # type: ignore[misc,valid-type]
     event_data = Column(Text().with_variant(mysql.LONGTEXT, "mysql"))
     origin = Column(String(MAX_LENGTH_EVENT_ORIGIN))  # no longer used for new rows
     origin_idx = Column(SmallInteger)
-    time_fired = Column(DATETIME_TYPE, index=True)
+    time_fired = Column(DATETIME_TYPE, index=True)  # no longer used for new rows
+    timed_fired_ts = Column(TIMESTAMP_TYPE, index=True)
     context_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID), index=True)
     context_user_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID))
     context_parent_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID))
@@ -179,6 +182,7 @@ def from_event(event: Event) -> Events:
             event_data=None,
             origin_idx=EVENT_ORIGIN_TO_IDX.get(event.origin),
             time_fired=event.time_fired,
+            time_fired_ts=event.time_fired.timestamp(),
             context_id=event.context.id,
             context_user_id=event.context.user_id,
             context_parent_id=event.context.parent_id,
@@ -274,8 +278,12 @@ class States(Base):  # type: ignore[misc,valid-type]
     event_id = Column(  # no longer used for new rows
         Integer, ForeignKey("events.event_id", ondelete="CASCADE"), index=True
     )
-    last_changed = Column(DATETIME_TYPE)
-    last_updated = Column(DATETIME_TYPE, default=dt_util.utcnow, index=True)
+    last_changed = Column(DATETIME_TYPE)  # no longer used for new rows
+    last_changed_ts = Column(TIMESTAMP_TYPE)
+    last_updated = Column(
+        DATETIME_TYPE, default=dt_util.utcnow, index=True
+    )  # no longer used for new rows
+    last_updated_ts = Column(TIMESTAMP_TYPE, default=time.time, index=True)
     old_state_id = Column(Integer, ForeignKey("states.state_id"), index=True)
     attributes_id = Column(
         Integer, ForeignKey("state_attributes.attributes_id"), index=True
@@ -314,6 +322,7 @@ def from_event(event: Event) -> States:
         if state is None:
             dbstate.state = ""
             dbstate.last_updated = event.time_fired
+            dbstate.last_changed_ts = event.time_fired.timestamp()
             dbstate.last_changed = None
             return dbstate
 
@@ -321,8 +330,10 @@ def from_event(event: Event) -> States:
         dbstate.last_updated = state.last_updated
         if state.last_updated == state.last_changed:
             dbstate.last_changed = None
+            dbstate.last_changed_ts = None
         else:
             dbstate.last_changed = state.last_changed
+            dbstate.last_changed_ts = state.last_changed.timestamp()
 
         return dbstate
 

From bad8f1748c446d34e63d5b82f7be30e8e4360417 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Fri, 30 Dec 2022 10:03:43 -1000
Subject: [PATCH 02/38] Avoid the overhead of storing database timestamps as
 strings

Reduces database size by ~30%
---
 .../components/recorder/db_schema.py          |   2 +-
 homeassistant/components/recorder/history.py  | 413 ++++++++++++++----
 2 files changed, 324 insertions(+), 91 deletions(-)

diff --git a/homeassistant/components/recorder/db_schema.py b/homeassistant/components/recorder/db_schema.py
index 6d3a7c9aead758..a349b5c92dcf81 100644
--- a/homeassistant/components/recorder/db_schema.py
+++ b/homeassistant/components/recorder/db_schema.py
@@ -54,7 +54,7 @@
 # pylint: disable=invalid-name
 Base = declarative_base()
 
-SCHEMA_VERSION = 30
+SCHEMA_VERSION = 31
 
 _StatisticsBaseSelfT = TypeVar("_StatisticsBaseSelfT", bound="StatisticsBase")
 
diff --git a/homeassistant/components/recorder/history.py b/homeassistant/components/recorder/history.py
index 5c3f47c02ed955..dc1fe25fbfb53f 100644
--- a/homeassistant/components/recorder/history.py
+++ b/homeassistant/components/recorder/history.py
@@ -25,7 +25,7 @@
 import homeassistant.util.dt as dt_util
 
 from .. import recorder
-from .db_schema import RecorderRuns, StateAttributes, States
+from .db_schema import TIMESTAMP_TYPE, RecorderRuns, StateAttributes, States
 from .filters import Filters
 from .models import (
     LazyState,
@@ -59,49 +59,84 @@
     "water_heater",
 }
 
-BASE_STATES = [
+
+_BASE_STATES = [
+    States.entity_id,
+    States.state,
+    States.last_changed_ts,
+    States.last_updated_ts,
+]
+_BASE_STATES_NO_LAST_CHANGED = [
+    States.entity_id,
+    States.state,
+    literal(value=None, type_=TIMESTAMP_TYPE).label("last_changed_ts"),
+    States.last_updated_ts,
+]
+_QUERY_STATE_NO_ATTR = [
+    *_BASE_STATES,
+    literal(value=None, type_=Text).label("attributes"),
+    literal(value=None, type_=Text).label("shared_attrs"),
+]
+_QUERY_STATE_NO_ATTR_NO_LAST_CHANGED = [
+    *_BASE_STATES_NO_LAST_CHANGED,
+    literal(value=None, type_=Text).label("attributes"),
+    literal(value=None, type_=Text).label("shared_attrs"),
+]
+_BASE_STATES_PRE_SCHEMA_31 = [
     States.entity_id,
     States.state,
     States.last_changed,
     States.last_updated,
 ]
-BASE_STATES_NO_LAST_CHANGED = [
+_BASE_STATES_NO_LAST_CHANGED_PRE_SCHEMA_31 = [
     States.entity_id,
     States.state,
     literal(value=None, type_=Text).label("last_changed"),
     States.last_updated,
 ]
-QUERY_STATE_NO_ATTR = [
-    *BASE_STATES,
+_QUERY_STATE_NO_ATTR_PRE_SCHEMA_31 = [
+    *_BASE_STATES_PRE_SCHEMA_31,
     literal(value=None, type_=Text).label("attributes"),
     literal(value=None, type_=Text).label("shared_attrs"),
 ]
-QUERY_STATE_NO_ATTR_NO_LAST_CHANGED = [
-    *BASE_STATES_NO_LAST_CHANGED,
+_QUERY_STATE_NO_ATTR_NO_LAST_CHANGED_PRE_SCHEMA_31 = [
+    *_BASE_STATES_NO_LAST_CHANGED_PRE_SCHEMA_31,
     literal(value=None, type_=Text).label("attributes"),
     literal(value=None, type_=Text).label("shared_attrs"),
 ]
 # Remove QUERY_STATES_PRE_SCHEMA_25
 # and the migration_in_progress check
 # once schema 26 is created
-QUERY_STATES_PRE_SCHEMA_25 = [
-    *BASE_STATES,
+_QUERY_STATES_PRE_SCHEMA_25 = [
+    *_BASE_STATES_PRE_SCHEMA_31,
     States.attributes,
     literal(value=None, type_=Text).label("shared_attrs"),
 ]
-QUERY_STATES_PRE_SCHEMA_25_NO_LAST_CHANGED = [
-    *BASE_STATES_NO_LAST_CHANGED,
+_QUERY_STATES_PRE_SCHEMA_25_NO_LAST_CHANGED = [
+    *_BASE_STATES_NO_LAST_CHANGED_PRE_SCHEMA_31,
     States.attributes,
     literal(value=None, type_=Text).label("shared_attrs"),
 ]
-QUERY_STATES = [
-    *BASE_STATES,
+_QUERY_STATES_PRE_SCHEMA_31 = [
+    *_BASE_STATES_PRE_SCHEMA_31,
+    # Remove States.attributes once all attributes are in StateAttributes.shared_attrs
+    States.attributes,
+    StateAttributes.shared_attrs,
+]
+_QUERY_STATES_NO_LAST_CHANGED_PRE_SCHEMA_31 = [
+    *_BASE_STATES_NO_LAST_CHANGED_PRE_SCHEMA_31,
     # Remove States.attributes once all attributes are in StateAttributes.shared_attrs
     States.attributes,
     StateAttributes.shared_attrs,
 ]
-QUERY_STATES_NO_LAST_CHANGED = [
-    *BASE_STATES_NO_LAST_CHANGED,
+_QUERY_STATES = [
+    *_BASE_STATES,
+    # Remove States.attributes once all attributes are in StateAttributes.shared_attrs
+    States.attributes,
+    StateAttributes.shared_attrs,
+]
+_QUERY_STATES_NO_LAST_CHANGED = [
+    *_BASE_STATES_NO_LAST_CHANGED,
     # Remove States.attributes once all attributes are in StateAttributes.shared_attrs
     States.attributes,
     StateAttributes.shared_attrs,
@@ -124,10 +159,25 @@ def lambda_stmt_and_join_attributes(
     # without the attributes fields and do not join the
     # state_attributes table
     if no_attributes:
+        if schema_version >= 31:
+            if include_last_changed:
+                return (
+                    lambda_stmt(lambda: select(*_QUERY_STATE_NO_ATTR)),
+                    False,
+                )
+            return (
+                lambda_stmt(lambda: select(*_QUERY_STATE_NO_ATTR_NO_LAST_CHANGED)),
+                False,
+            )
         if include_last_changed:
-            return lambda_stmt(lambda: select(*QUERY_STATE_NO_ATTR)), False
+            return (
+                lambda_stmt(lambda: select(*_QUERY_STATE_NO_ATTR_PRE_SCHEMA_31)),
+                False,
+            )
         return (
-            lambda_stmt(lambda: select(*QUERY_STATE_NO_ATTR_NO_LAST_CHANGED)),
+            lambda_stmt(
+                lambda: select(*_QUERY_STATE_NO_ATTR_NO_LAST_CHANGED_PRE_SCHEMA_31)
+            ),
             False,
         )
     # If we in the process of migrating schema we do
@@ -136,19 +186,27 @@ def lambda_stmt_and_join_attributes(
     if schema_version < 25:
         if include_last_changed:
             return (
-                lambda_stmt(lambda: select(*QUERY_STATES_PRE_SCHEMA_25)),
+                lambda_stmt(lambda: select(*_QUERY_STATES_PRE_SCHEMA_25)),
                 False,
             )
         return (
-            lambda_stmt(lambda: select(*QUERY_STATES_PRE_SCHEMA_25_NO_LAST_CHANGED)),
+            lambda_stmt(lambda: select(*_QUERY_STATES_PRE_SCHEMA_25_NO_LAST_CHANGED)),
             False,
         )
+
+    if schema_version >= 31:
+        if include_last_changed:
+            return lambda_stmt(lambda: select(*_QUERY_STATES)), True
+        return lambda_stmt(lambda: select(*_QUERY_STATES_NO_LAST_CHANGED)), True
     # Finally if no migration is in progress and no_attributes
     # was not requested, we query both attributes columns and
     # join state_attributes
     if include_last_changed:
-        return lambda_stmt(lambda: select(*QUERY_STATES)), True
-    return lambda_stmt(lambda: select(*QUERY_STATES_NO_LAST_CHANGED)), True
+        return lambda_stmt(lambda: select(*_QUERY_STATES_PRE_SCHEMA_31)), True
+    return (
+        lambda_stmt(lambda: select(*_QUERY_STATES_NO_LAST_CHANGED_PRE_SCHEMA_31)),
+        True,
+    )
 
 
 def get_significant_states(
@@ -211,22 +269,41 @@ def _significant_states_stmt(
         and significant_changes_only
         and split_entity_id(entity_ids[0])[0] not in SIGNIFICANT_DOMAINS
     ):
+        if schema_version >= 31:
+            stmt += lambda q: q.filter(
+                (States.last_changed_ts == States.last_updated_ts)
+                | States.last_changed_ts.is_(None)
+            )
         stmt += lambda q: q.filter(
             (States.last_changed == States.last_updated) | States.last_changed.is_(None)
         )
     elif significant_changes_only:
-        stmt += lambda q: q.filter(
-            or_(
-                *[
-                    States.entity_id.like(entity_domain)
-                    for entity_domain in SIGNIFICANT_DOMAINS_ENTITY_ID_LIKE
-                ],
-                (
-                    (States.last_changed == States.last_updated)
-                    | States.last_changed.is_(None)
-                ),
+        if schema_version >= 31:
+            stmt += lambda q: q.filter(
+                or_(
+                    *[
+                        States.entity_id.like(entity_domain)
+                        for entity_domain in SIGNIFICANT_DOMAINS_ENTITY_ID_LIKE
+                    ],
+                    (
+                        (States.last_changed_ts == States.last_updated_ts)
+                        | States.last_changed_ts.is_(None)
+                    ),
+                )
+            )
+        else:
+            stmt += lambda q: q.filter(
+                or_(
+                    *[
+                        States.entity_id.like(entity_domain)
+                        for entity_domain in SIGNIFICANT_DOMAINS_ENTITY_ID_LIKE
+                    ],
+                    (
+                        (States.last_changed == States.last_updated)
+                        | States.last_changed.is_(None)
+                    ),
+                )
             )
-        )
 
     if entity_ids:
         stmt += lambda q: q.filter(States.entity_id.in_(entity_ids))
@@ -238,15 +315,25 @@ def _significant_states_stmt(
                 lambda q: q.filter(entity_filter), track_on=[filters]
             )
 
-    stmt += lambda q: q.filter(States.last_updated > start_time)
-    if end_time:
-        stmt += lambda q: q.filter(States.last_updated < end_time)
+    if schema_version >= 31:
+        start_time_ts = start_time.timestamp()
+        stmt += lambda q: q.filter(States.last_updated_ts > start_time_ts)
+        if end_time:
+            end_time_ts = end_time.timestamp()
+            stmt += lambda q: q.filter(States.last_updated_ts < end_time_ts)
+    else:
+        stmt += lambda q: q.filter(States.last_updated > start_time)
+        if end_time:
+            stmt += lambda q: q.filter(States.last_updated < end_time)
 
     if join_attributes:
         stmt += lambda q: q.outerjoin(
             StateAttributes, States.attributes_id == StateAttributes.attributes_id
         )
-    stmt += lambda q: q.order_by(States.entity_id, States.last_updated)
+    if schema_version >= 31:
+        stmt += lambda q: q.order_by(States.entity_id, States.last_updated_ts)
+    else:
+        stmt += lambda q: q.order_by(States.entity_id, States.last_updated)
     return stmt
 
 
@@ -342,12 +429,29 @@ def _state_changed_during_period_stmt(
     stmt, join_attributes = lambda_stmt_and_join_attributes(
         schema_version, no_attributes, include_last_changed=False
     )
-    stmt += lambda q: q.filter(
-        ((States.last_changed == States.last_updated) | States.last_changed.is_(None))
-        & (States.last_updated > start_time)
-    )
+    if schema_version >= 31:
+        start_time_ts = start_time.timestamp()
+        stmt += lambda q: q.filter(
+            (
+                (States.last_changed_ts == States.last_updated_ts)
+                | States.last_changed_ts.is_(None)
+            )
+            & (States.last_updated_ts > start_time_ts)
+        )
+    else:
+        stmt += lambda q: q.filter(
+            (
+                (States.last_changed == States.last_updated)
+                | States.last_changed.is_(None)
+            )
+            & (States.last_updated > start_time)
+        )
     if end_time:
-        stmt += lambda q: q.filter(States.last_updated < end_time)
+        if schema_version >= 31:
+            end_time_ts = end_time.timestamp()
+            stmt += lambda q: q.filter(States.last_updated_ts < end_time_ts)
+        else:
+            stmt += lambda q: q.filter(States.last_updated < end_time)
     if entity_id:
         stmt += lambda q: q.filter(States.entity_id == entity_id)
     if join_attributes:
@@ -355,9 +459,17 @@ def _state_changed_during_period_stmt(
             StateAttributes, States.attributes_id == StateAttributes.attributes_id
         )
     if descending:
-        stmt += lambda q: q.order_by(States.entity_id, States.last_updated.desc())
+        if schema_version >= 31:
+            stmt += lambda q: q.order_by(States.entity_id, States.last_updated.desc())
+        else:
+            stmt += lambda q: q.order_by(
+                States.entity_id, States.last_updated_ts.desc()
+            )
     else:
-        stmt += lambda q: q.order_by(States.entity_id, States.last_updated)
+        if schema_version >= 31:
+            stmt += lambda q: q.order_by(States.entity_id, States.last_updated_ts)
+        else:
+            stmt += lambda q: q.order_by(States.entity_id, States.last_updated)
     if limit:
         stmt += lambda q: q.limit(limit)
     return stmt
@@ -409,18 +521,29 @@ def _get_last_state_changes_stmt(
     stmt, join_attributes = lambda_stmt_and_join_attributes(
         schema_version, False, include_last_changed=False
     )
-    stmt += lambda q: q.filter(
-        (States.last_changed == States.last_updated) | States.last_changed.is_(None)
-    )
+    if schema_version >= 31:
+        stmt += lambda q: q.filter(
+            (States.last_changed_ts == States.last_updated_ts)
+            | States.last_changed_ts.is_(None)
+        )
+    else:
+        stmt += lambda q: q.filter(
+            (States.last_changed == States.last_updated) | States.last_changed.is_(None)
+        )
     if entity_id:
         stmt += lambda q: q.filter(States.entity_id == entity_id)
     if join_attributes:
         stmt += lambda q: q.outerjoin(
             StateAttributes, States.attributes_id == StateAttributes.attributes_id
         )
-    stmt += lambda q: q.order_by(States.entity_id, States.last_updated.desc()).limit(
-        number_of_states
-    )
+    if schema_version >= 31:
+        stmt += lambda q: q.order_by(
+            States.entity_id, States.last_updated_ts.desc()
+        ).limit(number_of_states)
+    else:
+        stmt += lambda q: q.order_by(
+            States.entity_id, States.last_updated.desc()
+        ).limit(number_of_states)
     return stmt
 
 
@@ -463,19 +586,36 @@ def _get_states_for_entites_stmt(
     )
     # We got an include-list of entities, accelerate the query by filtering already
     # in the inner query.
-    stmt += lambda q: q.where(
-        States.state_id
-        == (
-            select(func.max(States.state_id).label("max_state_id"))
-            .filter(
-                (States.last_updated >= run_start)
-                & (States.last_updated < utc_point_in_time)
-            )
-            .filter(States.entity_id.in_(entity_ids))
-            .group_by(States.entity_id)
-            .subquery()
-        ).c.max_state_id
-    )
+    if schema_version >= 31:
+        run_start_ts = run_start.timestamp()
+        utc_point_in_time_ts = utc_point_in_time.timestamp()
+        stmt += lambda q: q.where(
+            States.state_id
+            == (
+                select(func.max(States.state_id).label("max_state_id"))
+                .filter(
+                    (States.last_updated_ts >= run_start_ts)
+                    & (States.last_updated_ts < utc_point_in_time_ts)
+                )
+                .filter(States.entity_id.in_(entity_ids))
+                .group_by(States.entity_id)
+                .subquery()
+            ).c.max_state_id
+        )
+    else:
+        stmt += lambda q: q.where(
+            States.state_id
+            == (
+                select(func.max(States.state_id).label("max_state_id"))
+                .filter(
+                    (States.last_updated >= run_start)
+                    & (States.last_updated < utc_point_in_time)
+                )
+                .filter(States.entity_id.in_(entity_ids))
+                .group_by(States.entity_id)
+                .subquery()
+            ).c.max_state_id
+        )
     if join_attributes:
         stmt += lambda q: q.outerjoin(
             StateAttributes, (States.attributes_id == StateAttributes.attributes_id)
@@ -484,10 +624,26 @@ def _get_states_for_entites_stmt(
 
 
 def _generate_most_recent_states_by_date(
+    schema_version: int,
     run_start: datetime,
     utc_point_in_time: datetime,
 ) -> Subquery:
     """Generate the sub query for the most recent states by data."""
+    if schema_version >= 31:
+        run_start_ts = run_start.timestamp()
+        utc_point_in_time_ts = utc_point_in_time.timestamp()
+        return (
+            select(
+                States.entity_id.label("max_entity_id"),
+                func.max(States.last_updated_ts).label("max_last_updated"),
+            )
+            .filter(
+                (States.last_updated_ts >= run_start_ts)
+                & (States.last_updated_ts < utc_point_in_time_ts)
+            )
+            .group_by(States.entity_id)
+            .subquery()
+        )
     return (
         select(
             States.entity_id.label("max_entity_id"),
@@ -518,24 +674,42 @@ def _get_states_for_all_stmt(
     # This filtering can't be done in the inner query because the domain column is
     # not indexed and we can't control what's in the custom filter.
     most_recent_states_by_date = _generate_most_recent_states_by_date(
-        run_start, utc_point_in_time
-    )
-    stmt += lambda q: q.where(
-        States.state_id
-        == (
-            select(func.max(States.state_id).label("max_state_id"))
-            .join(
-                most_recent_states_by_date,
-                and_(
-                    States.entity_id == most_recent_states_by_date.c.max_entity_id,
-                    States.last_updated
-                    == most_recent_states_by_date.c.max_last_updated,
-                ),
-            )
-            .group_by(States.entity_id)
-            .subquery()
-        ).c.max_state_id,
+        schema_version, run_start, utc_point_in_time
     )
+    if schema_version >= 31:
+        stmt += lambda q: q.where(
+            States.state_id
+            == (
+                select(func.max(States.state_id).label("max_state_id"))
+                .join(
+                    most_recent_states_by_date,
+                    and_(
+                        States.entity_id == most_recent_states_by_date.c.max_entity_id,
+                        States.last_updated_ts
+                        == most_recent_states_by_date.c.max_last_updated,
+                    ),
+                )
+                .group_by(States.entity_id)
+                .subquery()
+            ).c.max_state_id,
+        )
+    else:
+        stmt += lambda q: q.where(
+            States.state_id
+            == (
+                select(func.max(States.state_id).label("max_state_id"))
+                .join(
+                    most_recent_states_by_date,
+                    and_(
+                        States.entity_id == most_recent_states_by_date.c.max_entity_id,
+                        States.last_updated
+                        == most_recent_states_by_date.c.max_last_updated,
+                    ),
+                )
+                .group_by(States.entity_id)
+                .subquery()
+            ).c.max_state_id,
+        )
     stmt += _ignore_domains_filter
     if filters and filters.has_config:
         entity_filter = filters.states_entity_filter()
@@ -598,14 +772,25 @@ def _get_single_entity_states_stmt(
     stmt, join_attributes = lambda_stmt_and_join_attributes(
         schema_version, no_attributes, include_last_changed=True
     )
-    stmt += (
-        lambda q: q.filter(
-            States.last_updated < utc_point_in_time,
-            States.entity_id == entity_id,
+    if schema_version >= 31:
+        utc_point_in_time_ts = utc_point_in_time.timestamp()
+        stmt += (
+            lambda q: q.filter(
+                States.last_updated_ts < utc_point_in_time_ts,
+                States.entity_id == entity_id,
+            )
+            .order_by(States.last_updated_ts.desc())
+            .limit(1)
+        )
+    else:
+        stmt += (
+            lambda q: q.filter(
+                States.last_updated < utc_point_in_time,
+                States.entity_id == entity_id,
+            )
+            .order_by(States.last_updated.desc())
+            .limit(1)
         )
-        .order_by(States.last_updated.desc())
-        .limit(1)
-    )
     if join_attributes:
         stmt += lambda q: q.outerjoin(
             StateAttributes, States.attributes_id == StateAttributes.attributes_id
@@ -636,19 +821,23 @@ def _sorted_states_to_dict(
     each list of states, otherwise our graphs won't start on the Y
     axis correctly.
     """
+    schema_version = _schema_version(hass)
+    _process_timestamp: Callable[[datetime], float | str]
     if compressed_state_format:
         state_class = row_to_compressed_state
-        _process_timestamp: Callable[
-            [datetime], float | str
-        ] = process_datetime_to_timestamp
         attr_time = COMPRESSED_STATE_LAST_UPDATED
         attr_state = COMPRESSED_STATE_STATE
     else:
         state_class = LazyState  # type: ignore[assignment]
-        _process_timestamp = process_timestamp_to_utc_isoformat
         attr_time = LAST_CHANGED_KEY
         attr_state = STATE_KEY
 
+    if schema_version < 31:
+        if compressed_state_format:
+            _process_timestamp = process_datetime_to_timestamp
+        else:
+            _process_timestamp = process_timestamp_to_utc_isoformat
+
     result: dict[str, list[State | dict[str, Any]]] = defaultdict(list)
     # Set all entity IDs to empty lists in result set to maintain the order
     if entity_ids is not None:
@@ -705,6 +894,50 @@ def _sorted_states_to_dict(
             prev_state = first_state.state
             ent_results.append(state_class(first_state, attr_cache))
 
+        if schema_version >= 31:
+            if compressed_state_format:
+                for row in group:
+                    # With minimal response we do not care about attribute
+                    # changes so we can filter out duplicate states
+                    if (state := row.state) == prev_state:
+                        continue
+
+                    ent_results.append(
+                        {
+                            attr_state: state,
+                            #
+                            # minimal_response only makes sense with last_updated == last_updated
+                            #
+                            # We use last_updated for for last_changed since its the same
+                            #
+                            attr_time: row.last_updated_ts,
+                        }
+                    )
+                    prev_state = state
+            else:
+                for row in group:
+                    # With minimal response we do not care about attribute
+                    # changes so we can filter out duplicate states
+                    if (state := row.state) == prev_state:
+                        continue
+
+                    ent_results.append(
+                        {
+                            attr_state: state,
+                            #
+                            # minimal_response only makes sense with last_updated == last_updated
+                            #
+                            # We use last_updated for for last_changed since its the same
+                            #
+                            attr_time: process_timestamp_to_utc_isoformat(
+                                datetime.fromtimestamp(row.last_updated_ts)
+                            ),
+                        }
+                    )
+                    prev_state = state
+
+            continue
+
         for row in group:
             # With minimal response we do not care about attribute
             # changes so we can filter out duplicate states

From 14b9e9cabf172a05d04d6856a427489aaf839d0b Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Fri, 30 Dec 2022 10:04:28 -1000
Subject: [PATCH 03/38] Avoid the overhead of storing database timestamps as
 strings

Reduces database size by ~30%
---
 homeassistant/components/recorder/db_schema.py | 2 +-
 homeassistant/components/recorder/history.py   | 8 ++------
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/homeassistant/components/recorder/db_schema.py b/homeassistant/components/recorder/db_schema.py
index a349b5c92dcf81..e777a15c6855d1 100644
--- a/homeassistant/components/recorder/db_schema.py
+++ b/homeassistant/components/recorder/db_schema.py
@@ -158,7 +158,7 @@ class Events(Base):  # type: ignore[misc,valid-type]
     origin = Column(String(MAX_LENGTH_EVENT_ORIGIN))  # no longer used for new rows
     origin_idx = Column(SmallInteger)
     time_fired = Column(DATETIME_TYPE, index=True)  # no longer used for new rows
-    timed_fired_ts = Column(TIMESTAMP_TYPE, index=True)
+    time_fired_ts = Column(TIMESTAMP_TYPE, index=True)
     context_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID), index=True)
     context_user_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID))
     context_parent_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID))
diff --git a/homeassistant/components/recorder/history.py b/homeassistant/components/recorder/history.py
index dc1fe25fbfb53f..6610d3fcfb0f2d 100644
--- a/homeassistant/components/recorder/history.py
+++ b/homeassistant/components/recorder/history.py
@@ -825,19 +825,15 @@ def _sorted_states_to_dict(
     _process_timestamp: Callable[[datetime], float | str]
     if compressed_state_format:
         state_class = row_to_compressed_state
+        _process_timestamp = process_datetime_to_timestamp
         attr_time = COMPRESSED_STATE_LAST_UPDATED
         attr_state = COMPRESSED_STATE_STATE
     else:
         state_class = LazyState  # type: ignore[assignment]
+        _process_timestamp = process_timestamp_to_utc_isoformat
         attr_time = LAST_CHANGED_KEY
         attr_state = STATE_KEY
 
-    if schema_version < 31:
-        if compressed_state_format:
-            _process_timestamp = process_datetime_to_timestamp
-        else:
-            _process_timestamp = process_timestamp_to_utc_isoformat
-
     result: dict[str, list[State | dict[str, Any]]] = defaultdict(list)
     # Set all entity IDs to empty lists in result set to maintain the order
     if entity_ids is not None:

From 8e77224c98b263b14a6cf5cabee8244b1a46ac2a Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Fri, 30 Dec 2022 10:07:21 -1000
Subject: [PATCH 04/38] Avoid the overhead of storing database timestamps as
 strings

Reduces database size by ~30%
---
 homeassistant/components/recorder/db_schema.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/homeassistant/components/recorder/db_schema.py b/homeassistant/components/recorder/db_schema.py
index e777a15c6855d1..410f3a05e39c9d 100644
--- a/homeassistant/components/recorder/db_schema.py
+++ b/homeassistant/components/recorder/db_schema.py
@@ -93,6 +93,7 @@
 
 LAST_UPDATED_INDEX = "ix_states_last_updated"
 ENTITY_ID_LAST_UPDATED_INDEX = "ix_states_entity_id_last_updated"
+ENTITY_ID_LAST_UPDATED_INDEX_TS = "ix_states_entity_id_last_updated_ts"
 EVENTS_CONTEXT_ID_INDEX = "ix_events_context_id"
 STATES_CONTEXT_ID_INDEX = "ix_states_context_id"
 
@@ -149,6 +150,7 @@ class Events(Base):  # type: ignore[misc,valid-type]
         # Used for fetching events at a specific time
         # see logbook
         Index("ix_events_event_type_time_fired", "event_type", "time_fired"),
+        Index("ix_events_event_type_time_fired_ts", "event_type", "time_fired_ts"),
         {"mysql_default_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"},
     )
     __tablename__ = TABLE_EVENTS
@@ -266,6 +268,7 @@ class States(Base):  # type: ignore[misc,valid-type]
         # Used for fetching the state of entities at a specific time
         # (get_states in history.py)
         Index(ENTITY_ID_LAST_UPDATED_INDEX, "entity_id", "last_updated"),
+        Index(ENTITY_ID_LAST_UPDATED_INDEX_TS, "entity_id", "last_updated_ts"),
         {"mysql_default_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"},
     )
     __tablename__ = TABLE_STATES

From 830025b2c75db5f5a8c023c9b1d885e0ab5d3af3 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Fri, 30 Dec 2022 10:37:25 -1000
Subject: [PATCH 05/38] Avoid the overhead of storing database timestamps as
 strings

Reduces database size by ~30%
---
 homeassistant/components/recorder/history.py |  19 ++-
 homeassistant/components/recorder/models.py  | 134 ++++++++++++++++++-
 2 files changed, 146 insertions(+), 7 deletions(-)

diff --git a/homeassistant/components/recorder/history.py b/homeassistant/components/recorder/history.py
index 6610d3fcfb0f2d..04a875bc2e8448 100644
--- a/homeassistant/components/recorder/history.py
+++ b/homeassistant/components/recorder/history.py
@@ -29,10 +29,12 @@
 from .filters import Filters
 from .models import (
     LazyState,
+    LazyStatePreSchema31,
     process_datetime_to_timestamp,
     process_timestamp,
     process_timestamp_to_utc_isoformat,
     row_to_compressed_state,
+    row_to_compressed_state_pre_schema_31,
 )
 from .util import execute_stmt_lambda_element, session_scope
 
@@ -823,13 +825,20 @@ def _sorted_states_to_dict(
     """
     schema_version = _schema_version(hass)
     _process_timestamp: Callable[[datetime], float | str]
+    state_class: Callable[[Row, dict[str, dict[str, Any]], Any], Any]
     if compressed_state_format:
-        state_class = row_to_compressed_state
+        if schema_version >= 31:
+            state_class = row_to_compressed_state
+        else:
+            state_class = row_to_compressed_state_pre_schema_31
         _process_timestamp = process_datetime_to_timestamp
         attr_time = COMPRESSED_STATE_LAST_UPDATED
         attr_state = COMPRESSED_STATE_STATE
     else:
-        state_class = LazyState  # type: ignore[assignment]
+        if schema_version >= 31:
+            state_class = LazyState
+        else:
+            state_class = LazyStatePreSchema31
         _process_timestamp = process_timestamp_to_utc_isoformat
         attr_time = LAST_CHANGED_KEY
         attr_state = STATE_KEY
@@ -877,7 +886,9 @@ def _sorted_states_to_dict(
             ent_results.append(state_class(row, attr_cache, start_time))
 
         if not minimal_response or split_entity_id(ent_id)[0] in NEED_ATTRIBUTE_DOMAINS:
-            ent_results.extend(state_class(db_state, attr_cache) for db_state in group)
+            ent_results.extend(
+                state_class(db_state, attr_cache, None) for db_state in group
+            )
             continue
 
         # With minimal response we only provide a native
@@ -888,7 +899,7 @@ def _sorted_states_to_dict(
             if (first_state := next(group, None)) is None:
                 continue
             prev_state = first_state.state
-            ent_results.append(state_class(first_state, attr_cache))
+            ent_results.append(state_class(first_state, attr_cache, None))
 
         if schema_version >= 31:
             if compressed_state_format:
diff --git a/homeassistant/components/recorder/models.py b/homeassistant/components/recorder/models.py
index 48b45b4da2e678..351fce56aadf35 100644
--- a/homeassistant/components/recorder/models.py
+++ b/homeassistant/components/recorder/models.py
@@ -120,8 +120,8 @@ def process_datetime_to_timestamp(ts: datetime) -> float:
     return ts.timestamp()
 
 
-class LazyState(State):
-    """A lazy version of core State."""
+class LazyStatePreSchema31(State):
+    """A lazy version of core State before schema 31."""
 
     __slots__ = [
         "_row",
@@ -243,6 +243,112 @@ def __eq__(self, other: Any) -> bool:
         )
 
 
+class LazyState(State):
+    """A lazy version of core State after schema 31."""
+
+    __slots__ = [
+        "_row",
+        "_attributes",
+        "_last_changed_ts",
+        "_last_updated_ts",
+        "_context",
+        "attr_cache",
+    ]
+
+    def __init__(  # pylint: disable=super-init-not-called
+        self,
+        row: Row,
+        attr_cache: dict[str, dict[str, Any]],
+        start_time: float | None = None,
+    ) -> None:
+        """Init the lazy state."""
+        self._row = row
+        self.entity_id: str = self._row.entity_id
+        self.state = self._row.state or ""
+        self._attributes: dict[str, Any] | None = None
+        self._last_updated_ts: float | None = self._row.last_updated_ts or start_time
+        self._last_changed_ts: float | None = (
+            self._row.last_changed_ts or self._last_updated_ts or start_time
+        )
+        self._context: Context | None = None
+        self.attr_cache = attr_cache
+
+    @property  # type: ignore[override]
+    def attributes(self) -> dict[str, Any]:
+        """State attributes."""
+        if self._attributes is None:
+            self._attributes = decode_attributes_from_row(self._row, self.attr_cache)
+        return self._attributes
+
+    @attributes.setter
+    def attributes(self, value: dict[str, Any]) -> None:
+        """Set attributes."""
+        self._attributes = value
+
+    @property
+    def context(self) -> Context:
+        """State context."""
+        if self._context is None:
+            self._context = Context(id=None)
+        return self._context
+
+    @context.setter
+    def context(self, value: Context) -> None:
+        """Set context."""
+        self._context = value
+
+    @property
+    def last_changed(self) -> datetime:
+        """Last changed datetime."""
+        assert self._last_changed_ts is not None
+        return datetime.fromtimestamp(self._last_changed_ts)
+
+    @last_changed.setter
+    def last_changed(self, value: datetime) -> None:
+        """Set last changed datetime."""
+        self._last_changed_ts = value.timestamp()
+
+    @property
+    def last_updated(self) -> datetime:
+        """Last updated datetime."""
+        assert self._last_updated_ts is not None
+        return datetime.fromtimestamp(self._last_updated_ts)
+
+    @last_updated.setter
+    def last_updated(self, value: datetime) -> None:
+        """Set last updated datetime."""
+        self._last_updated_ts = value.timestamp()
+
+    def as_dict(self) -> dict[str, Any]:  # type: ignore[override]
+        """Return a dict representation of the LazyState.
+
+        Async friendly.
+
+        To be used for JSON serialization.
+        """
+        last_updated_isoformat = self.last_updated.isoformat()
+        if self._last_changed_ts == self._last_updated_ts:
+            last_changed_isoformat = last_updated_isoformat
+        else:
+            last_changed_isoformat = self.last_changed.isoformat()
+        return {
+            "entity_id": self.entity_id,
+            "state": self.state,
+            "attributes": self._attributes or self.attributes,
+            "last_changed": last_changed_isoformat,
+            "last_updated": last_updated_isoformat,
+        }
+
+    def __eq__(self, other: Any) -> bool:
+        """Return the comparison."""
+        return (
+            other.__class__ in [self.__class__, State]
+            and self.entity_id == other.entity_id
+            and self.state == other.state
+            and self.attributes == other.attributes
+        )
+
+
 def decode_attributes_from_row(
     row: Row, attr_cache: dict[str, dict[str, Any]]
 ) -> dict[str, Any]:
@@ -261,11 +367,33 @@ def decode_attributes_from_row(
 
 
 def row_to_compressed_state(
+    row: Row,
+    attr_cache: dict[str, dict[str, Any]],
+    start_time: float | None = None,
+) -> dict[str, Any]:
+    """Convert a database row to a compressed state after schema 31."""
+    comp_state = {
+        COMPRESSED_STATE_STATE: row.state,
+        COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_row(row, attr_cache),
+    }
+    if start_time:
+        comp_state[COMPRESSED_STATE_LAST_UPDATED] = start_time
+    else:
+        row_last_updated_ts: float = row.last_updated_ts
+        comp_state[COMPRESSED_STATE_LAST_UPDATED] = row_last_updated_ts
+        if (
+            row_changed_changed_ts := row.last_changed_ts
+        ) and row_last_updated_ts != row_changed_changed_ts:
+            comp_state[COMPRESSED_STATE_LAST_CHANGED] = row_changed_changed_ts
+    return comp_state
+
+
+def row_to_compressed_state_pre_schema_31(
     row: Row,
     attr_cache: dict[str, dict[str, Any]],
     start_time: datetime | None = None,
 ) -> dict[str, Any]:
-    """Convert a database row to a compressed state."""
+    """Convert a database row to a compressed state before schema 31."""
     comp_state = {
         COMPRESSED_STATE_STATE: row.state,
         COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_row(row, attr_cache),

From 224c0ee948c491a116f51b8156e4546346f2119d Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Fri, 30 Dec 2022 11:17:22 -1000
Subject: [PATCH 06/38] fixes

---
 .../components/recorder/db_schema.py          | 20 +++++++++++--------
 homeassistant/components/recorder/history.py  |  2 +-
 homeassistant/components/recorder/models.py   | 19 ++++++++++++++----
 tests/components/recorder/test_history.py     | 14 ++++++++++---
 4 files changed, 39 insertions(+), 16 deletions(-)

diff --git a/homeassistant/components/recorder/db_schema.py b/homeassistant/components/recorder/db_schema.py
index 410f3a05e39c9d..56df4561d9faee 100644
--- a/homeassistant/components/recorder/db_schema.py
+++ b/homeassistant/components/recorder/db_schema.py
@@ -48,7 +48,7 @@
 import homeassistant.util.dt as dt_util
 
 from .const import ALL_DOMAIN_EXCLUDE_ATTRS
-from .models import StatisticData, StatisticMetaData, process_timestamp
+from .models import StatisticData, StatisticMetaData
 
 # SQLAlchemy Schema
 # pylint: disable=invalid-name
@@ -184,7 +184,7 @@ def from_event(event: Event) -> Events:
             event_data=None,
             origin_idx=EVENT_ORIGIN_TO_IDX.get(event.origin),
             time_fired=event.time_fired,
-            time_fired_ts=event.time_fired.timestamp(),
+            time_fired_ts=dt_util.utc_to_timestamp(event.time_fired),
             context_id=event.context.id,
             context_user_id=event.context.user_id,
             context_parent_id=event.context.parent_id,
@@ -204,7 +204,7 @@ def to_native(self, validate_entity_id: bool = True) -> Event | None:
                 EventOrigin(self.origin)
                 if self.origin
                 else EVENT_ORIGIN_ORDER[self.origin_idx],
-                process_timestamp(self.time_fired),
+                dt_util.utc_from_timestamp(self.time_fired_ts),
                 context=context,
             )
         except JSON_DECODE_EXCEPTIONS:
@@ -325,18 +325,20 @@ def from_event(event: Event) -> States:
         if state is None:
             dbstate.state = ""
             dbstate.last_updated = event.time_fired
-            dbstate.last_changed_ts = event.time_fired.timestamp()
+            dbstate.last_updated_ts = dt_util.utc_to_timestamp(event.time_fired)
             dbstate.last_changed = None
+            dbstate.last_changed_ts = None
             return dbstate
 
         dbstate.state = state.state
         dbstate.last_updated = state.last_updated
+        dbstate.last_updated_ts = dt_util.utc_to_timestamp(state.last_updated)
         if state.last_updated == state.last_changed:
             dbstate.last_changed = None
             dbstate.last_changed_ts = None
         else:
             dbstate.last_changed = state.last_changed
-            dbstate.last_changed_ts = state.last_changed.timestamp()
+            dbstate.last_changed_ts = dt_util.utc_to_timestamp(state.last_changed)
 
         return dbstate
 
@@ -354,10 +356,12 @@ def to_native(self, validate_entity_id: bool = True) -> State | None:
             _LOGGER.exception("Error converting row to state: %s", self)
             return None
         if self.last_changed is None or self.last_changed == self.last_updated:
-            last_changed = last_updated = process_timestamp(self.last_updated)
+            last_changed = last_updated = dt_util.utc_from_timestamp(
+                self.last_updated_ts
+            )
         else:
-            last_updated = process_timestamp(self.last_updated)
-            last_changed = process_timestamp(self.last_changed)
+            last_updated = dt_util.utc_from_timestamp(self.last_updated_ts)
+            last_changed = dt_util.utc_from_timestamp(self.last_changed_ts)
         return State(
             self.entity_id,
             self.state,
diff --git a/homeassistant/components/recorder/history.py b/homeassistant/components/recorder/history.py
index 04a875bc2e8448..b8d7a5568fd889 100644
--- a/homeassistant/components/recorder/history.py
+++ b/homeassistant/components/recorder/history.py
@@ -937,7 +937,7 @@ def _sorted_states_to_dict(
                             # We use last_updated for for last_changed since its the same
                             #
                             attr_time: process_timestamp_to_utc_isoformat(
-                                datetime.fromtimestamp(row.last_updated_ts)
+                                dt_util.utc_from_timestamp(row.last_updated_ts)
                             ),
                         }
                     )
diff --git a/homeassistant/components/recorder/models.py b/homeassistant/components/recorder/models.py
index 351fce56aadf35..59fc40df0e5a6b 100644
--- a/homeassistant/components/recorder/models.py
+++ b/homeassistant/components/recorder/models.py
@@ -187,6 +187,11 @@ def last_changed(self, value: datetime) -> None:
         """Set last changed datetime."""
         self._last_changed = value
 
+    @property
+    def last_updated_ts(self) -> float:
+        """Last updated timestamp."""
+        return self.last_updated.timestamp()
+
     @property
     def last_updated(self) -> datetime:
         """Last updated datetime."""
@@ -301,23 +306,29 @@ def context(self, value: Context) -> None:
     def last_changed(self) -> datetime:
         """Last changed datetime."""
         assert self._last_changed_ts is not None
-        return datetime.fromtimestamp(self._last_changed_ts)
+        return dt_util.utc_from_timestamp(self._last_changed_ts)
 
     @last_changed.setter
     def last_changed(self, value: datetime) -> None:
         """Set last changed datetime."""
-        self._last_changed_ts = value.timestamp()
+        self._last_changed_ts = process_timestamp(value).timestamp()
+
+    @property
+    def last_updated_ts(self) -> float:
+        """Last updated timestamp."""
+        assert self._last_updated_ts is not None
+        return self._last_updated_ts
 
     @property
     def last_updated(self) -> datetime:
         """Last updated datetime."""
         assert self._last_updated_ts is not None
-        return datetime.fromtimestamp(self._last_updated_ts)
+        return dt_util.utc_from_timestamp(self._last_updated_ts)
 
     @last_updated.setter
     def last_updated(self, value: datetime) -> None:
         """Set last updated datetime."""
-        self._last_updated_ts = value.timestamp()
+        self._last_updated_ts = process_timestamp(value).timestamp()
 
     def as_dict(self) -> dict[str, Any]:  # type: ignore[override]
         """Return a dict representation of the LazyState.
diff --git a/tests/components/recorder/test_history.py b/tests/components/recorder/test_history.py
index 6362b83f78a328..be6ed317eaa904 100644
--- a/tests/components/recorder/test_history.py
+++ b/tests/components/recorder/test_history.py
@@ -11,14 +11,18 @@
 from sqlalchemy import text
 
 from homeassistant.components import recorder
-from homeassistant.components.recorder import history
+from homeassistant.components.recorder import get_instance, history
 from homeassistant.components.recorder.db_schema import (
     Events,
     RecorderRuns,
     StateAttributes,
     States,
 )
-from homeassistant.components.recorder.models import LazyState, process_timestamp
+from homeassistant.components.recorder.models import (
+    LazyState,
+    LazyStatePreSchema31,
+    process_timestamp,
+)
 from homeassistant.components.recorder.util import session_scope
 import homeassistant.core as ha
 from homeassistant.core import HomeAssistant, State
@@ -40,10 +44,14 @@ async def _async_get_states(
     """Get states from the database."""
 
     def _get_states_with_session():
+        if get_instance(hass).schema_version < 31:
+            klass = LazyStatePreSchema31
+        else:
+            klass = LazyState
         with session_scope(hass=hass) as session:
             attr_cache = {}
             return [
-                LazyState(row, attr_cache)
+                klass(row, attr_cache)
                 for row in history._get_rows_with_session(
                     hass,
                     session,

From 1caa0208966575077493f235726dcef160298701 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Fri, 30 Dec 2022 11:29:17 -1000
Subject: [PATCH 07/38] cleanup

---
 homeassistant/components/recorder/db_schema.py | 11 ++++-------
 tests/components/recorder/test_history.py      | 16 +++++++++++++---
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/homeassistant/components/recorder/db_schema.py b/homeassistant/components/recorder/db_schema.py
index 56df4561d9faee..7328bfc206b930 100644
--- a/homeassistant/components/recorder/db_schema.py
+++ b/homeassistant/components/recorder/db_schema.py
@@ -183,7 +183,7 @@ def from_event(event: Event) -> Events:
             event_type=event.event_type,
             event_data=None,
             origin_idx=EVENT_ORIGIN_TO_IDX.get(event.origin),
-            time_fired=event.time_fired,
+            time_fired=None,
             time_fired_ts=dt_util.utc_to_timestamp(event.time_fired),
             context_id=event.context.id,
             context_user_id=event.context.user_id,
@@ -319,25 +319,22 @@ def from_event(event: Event) -> States:
             context_user_id=event.context.user_id,
             context_parent_id=event.context.parent_id,
             origin_idx=EVENT_ORIGIN_TO_IDX.get(event.origin),
+            last_updated=None,
+            last_changed=None,
         )
 
         # None state means the state was removed from the state machine
         if state is None:
             dbstate.state = ""
-            dbstate.last_updated = event.time_fired
             dbstate.last_updated_ts = dt_util.utc_to_timestamp(event.time_fired)
-            dbstate.last_changed = None
             dbstate.last_changed_ts = None
             return dbstate
 
         dbstate.state = state.state
-        dbstate.last_updated = state.last_updated
         dbstate.last_updated_ts = dt_util.utc_to_timestamp(state.last_updated)
         if state.last_updated == state.last_changed:
-            dbstate.last_changed = None
             dbstate.last_changed_ts = None
         else:
-            dbstate.last_changed = state.last_changed
             dbstate.last_changed_ts = dt_util.utc_to_timestamp(state.last_changed)
 
         return dbstate
@@ -355,7 +352,7 @@ def to_native(self, validate_entity_id: bool = True) -> State | None:
             # When json_loads fails
             _LOGGER.exception("Error converting row to state: %s", self)
             return None
-        if self.last_changed is None or self.last_changed == self.last_updated:
+        if self.last_changed_ts is None or self.last_changed_ts == self.last_updated_ts:
             last_changed = last_updated = dt_util.utc_from_timestamp(
                 self.last_updated_ts
             )
diff --git a/tests/components/recorder/test_history.py b/tests/components/recorder/test_history.py
index be6ed317eaa904..cf9015ff7ca1c6 100644
--- a/tests/components/recorder/test_history.py
+++ b/tests/components/recorder/test_history.py
@@ -892,7 +892,7 @@ def _fetch_native_states() -> list[State]:
         != native_sensor_one_states[1].last_updated
     )
 
-    def _fetch_db_states() -> list[State]:
+    def _fetch_db_states() -> list[States]:
         with session_scope(hass=hass) as session:
             states = list(session.query(States))
             session.expunge_all()
@@ -902,12 +902,22 @@ def _fetch_db_states() -> list[State]:
         _fetch_db_states
     )
     assert db_sensor_one_states[0].last_changed is None
+    assert db_sensor_one_states[0].last_changed_ts is None
+
     assert (
-        process_timestamp(db_sensor_one_states[1].last_changed) == state0.last_changed
+        process_timestamp(
+            dt_util.utc_from_timestamp(db_sensor_one_states[1].last_changed_ts)
+        )
+        == state0.last_changed
     )
     assert db_sensor_one_states[0].last_updated is not None
+    assert db_sensor_one_states[0].last_updated_ts is not None
     assert db_sensor_one_states[1].last_updated is not None
-    assert db_sensor_one_states[0].last_updated != db_sensor_one_states[1].last_updated
+    assert db_sensor_one_states[1].last_updated_ts is not None
+    assert (
+        db_sensor_one_states[0].last_updated_ts
+        != db_sensor_one_states[1].last_updated_ts
+    )
 
 
 def test_state_changes_during_period_multiple_entities_single_test(hass_recorder):

From 2ca2414241292fd86028d6611135bbaf18541c69 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Fri, 30 Dec 2022 11:44:50 -1000
Subject: [PATCH 08/38] more fixes

---
 .../components/recorder/db_schema.py          |  2 -
 .../components/recorder/migration.py          | 12 +++
 homeassistant/components/recorder/purge.py    | 13 +++-
 homeassistant/components/recorder/queries.py  | 12 +--
 tests/components/recorder/test_purge.py       | 78 +++++++++----------
 5 files changed, 67 insertions(+), 50 deletions(-)

diff --git a/homeassistant/components/recorder/db_schema.py b/homeassistant/components/recorder/db_schema.py
index 7328bfc206b930..cd7da874381884 100644
--- a/homeassistant/components/recorder/db_schema.py
+++ b/homeassistant/components/recorder/db_schema.py
@@ -149,7 +149,6 @@ class Events(Base):  # type: ignore[misc,valid-type]
     __table_args__ = (
         # Used for fetching events at a specific time
         # see logbook
-        Index("ix_events_event_type_time_fired", "event_type", "time_fired"),
         Index("ix_events_event_type_time_fired_ts", "event_type", "time_fired_ts"),
         {"mysql_default_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"},
     )
@@ -267,7 +266,6 @@ class States(Base):  # type: ignore[misc,valid-type]
     __table_args__ = (
         # Used for fetching the state of entities at a specific time
         # (get_states in history.py)
-        Index(ENTITY_ID_LAST_UPDATED_INDEX, "entity_id", "last_updated"),
         Index(ENTITY_ID_LAST_UPDATED_INDEX_TS, "entity_id", "last_updated_ts"),
         {"mysql_default_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"},
     )
diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py
index 5b4b3afb3d9e65..a5f86462a409ce 100644
--- a/homeassistant/components/recorder/migration.py
+++ b/homeassistant/components/recorder/migration.py
@@ -821,6 +821,18 @@ def _apply_update(  # noqa: C901
         # Once we require SQLite >= 3.35.5, we should drop the column:
         # ALTER TABLE statistics_meta DROP COLUMN state_unit_of_measurement
         pass
+    elif new_version == 31:
+        # Migrate all data in Events.time_fired to Events.time_fired_ts and
+        # wipe Events.time_fired
+        # Migrate all data in States.last_updated to States.last_updated_ts and
+        # wipe States.last_updated
+        # Migrate all data in States.last_changed to States.last_changed_ts and
+        # wipe States.last_changed
+        # TODO: implement
+        _create_index(session_maker, "events", "ix_events_event_type_time_fired_ts")
+        _create_index(session_maker, "states", "ix_states_entity_id_last_updated_ts")
+        _drop_index(session_maker, "states", "ix_states_entity_id_last_updated")
+        _drop_index(session_maker, "events", "ix_events_event_type_time_fired")
     else:
         raise ValueError(f"No schema migration defined for version {new_version}")
 
diff --git a/homeassistant/components/recorder/purge.py b/homeassistant/components/recorder/purge.py
index 30f4d34e331ace..00673d86cf6fa8 100644
--- a/homeassistant/components/recorder/purge.py
+++ b/homeassistant/components/recorder/purge.py
@@ -12,6 +12,7 @@
 from sqlalchemy.sql.expression import distinct
 
 from homeassistant.const import EVENT_STATE_CHANGED
+import homeassistant.util.dt as dt_util
 
 from .const import MAX_ROWS_TO_PURGE, SupportedDialect
 from .db_schema import Events, StateAttributes, States
@@ -233,7 +234,9 @@ def _select_state_attributes_ids_to_purge(
     """Return sets of state and attribute ids to purge."""
     state_ids = set()
     attributes_ids = set()
-    for state in session.execute(find_states_to_purge(purge_before)).all():
+    for state in session.execute(
+        find_states_to_purge(dt_util.utc_to_timestamp(purge_before))
+    ).all():
         state_ids.add(state.state_id)
         if state.attributes_id:
             attributes_ids.add(state.attributes_id)
@@ -251,7 +254,9 @@ def _select_event_data_ids_to_purge(
     """Return sets of event and data ids to purge."""
     event_ids = set()
     data_ids = set()
-    for event in session.execute(find_events_to_purge(purge_before)).all():
+    for event in session.execute(
+        find_events_to_purge(dt_util.utc_to_timestamp(purge_before))
+    ).all():
         event_ids.add(event.event_id)
         if event.data_id:
             data_ids.add(event.data_id)
@@ -420,7 +425,9 @@ def _select_legacy_event_state_and_attributes_and_data_ids_to_purge(
     still need to be able to purge them.
     """
     events = session.execute(
-        find_legacy_event_state_and_attributes_and_data_ids_to_purge(purge_before)
+        find_legacy_event_state_and_attributes_and_data_ids_to_purge(
+            dt_util.utc_to_timestamp(purge_before)
+        )
     ).all()
     _LOGGER.debug("Selected %s event ids to remove", len(events))
     event_ids = set()
diff --git a/homeassistant/components/recorder/queries.py b/homeassistant/components/recorder/queries.py
index 4b4488d4dad7c7..0591fda4713fb7 100644
--- a/homeassistant/components/recorder/queries.py
+++ b/homeassistant/components/recorder/queries.py
@@ -578,20 +578,20 @@ def delete_recorder_runs_rows(
     )
 
 
-def find_events_to_purge(purge_before: datetime) -> StatementLambdaElement:
+def find_events_to_purge(purge_before: float) -> StatementLambdaElement:
     """Find events to purge."""
     return lambda_stmt(
         lambda: select(Events.event_id, Events.data_id)
-        .filter(Events.time_fired < purge_before)
+        .filter(Events.time_fired_ts < purge_before)
         .limit(MAX_ROWS_TO_PURGE)
     )
 
 
-def find_states_to_purge(purge_before: datetime) -> StatementLambdaElement:
+def find_states_to_purge(purge_before: float) -> StatementLambdaElement:
     """Find states to purge."""
     return lambda_stmt(
         lambda: select(States.state_id, States.attributes_id)
-        .filter(States.last_updated < purge_before)
+        .filter(States.last_updated_ts < purge_before)
         .limit(MAX_ROWS_TO_PURGE)
     )
 
@@ -624,7 +624,7 @@ def find_latest_statistics_runs_run_id() -> StatementLambdaElement:
 
 
 def find_legacy_event_state_and_attributes_and_data_ids_to_purge(
-    purge_before: datetime,
+    purge_before: float,
 ) -> StatementLambdaElement:
     """Find the latest row in the legacy format to purge."""
     return lambda_stmt(
@@ -632,7 +632,7 @@ def find_legacy_event_state_and_attributes_and_data_ids_to_purge(
             Events.event_id, Events.data_id, States.state_id, States.attributes_id
         )
         .outerjoin(States, Events.event_id == States.event_id)
-        .filter(Events.time_fired < purge_before)
+        .filter(Events.time_fired_ts < purge_before)
         .limit(MAX_ROWS_TO_PURGE)
     )
 
diff --git a/tests/components/recorder/test_purge.py b/tests/components/recorder/test_purge.py
index f135ae8af435ae..a3b32fc7e373b4 100644
--- a/tests/components/recorder/test_purge.py
+++ b/tests/components/recorder/test_purge.py
@@ -460,7 +460,7 @@ async def _add_db_entries(hass: HomeAssistant, timestamp: datetime) -> None:
                     event_type="EVENT_TEST_PURGE",
                     event_data="{}",
                     origin="LOCAL",
-                    time_fired=timestamp,
+                    time_fired_ts=dt_util.utc_to_timestamp(timestamp),
                 )
             )
             session.add(
@@ -468,8 +468,8 @@ async def _add_db_entries(hass: HomeAssistant, timestamp: datetime) -> None:
                     entity_id="test.recorder2",
                     state="purgeme",
                     attributes="{}",
-                    last_changed=timestamp,
-                    last_updated=timestamp,
+                    last_changed_ts=dt_util.utc_to_timestamp(timestamp),
+                    last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                     event_id=1001,
                     attributes_id=1002,
                 )
@@ -529,7 +529,7 @@ async def _add_db_entries(hass: HomeAssistant, cutoff: datetime, rows: int) -> N
                     event_type="KEEP",
                     event_data="{}",
                     origin="LOCAL",
-                    time_fired=timestamp_keep,
+                    time_fired_ts=dt_util.utc_to_timestamp(timestamp_keep),
                 )
             )
             session.add(
@@ -537,8 +537,8 @@ async def _add_db_entries(hass: HomeAssistant, cutoff: datetime, rows: int) -> N
                     entity_id="test.cutoff",
                     state="keep",
                     attributes="{}",
-                    last_changed=timestamp_keep,
-                    last_updated=timestamp_keep,
+                    last_changed_ts=dt_util.utc_to_timestamp(timestamp_keep),
+                    last_updated_ts=dt_util.utc_to_timestamp(timestamp_keep),
                     event_id=1000,
                     attributes_id=1000,
                 )
@@ -557,7 +557,7 @@ async def _add_db_entries(hass: HomeAssistant, cutoff: datetime, rows: int) -> N
                         event_type="PURGE",
                         event_data="{}",
                         origin="LOCAL",
-                        time_fired=timestamp_purge,
+                        time_fired_ts=dt_util.utc_to_timestamp(timestamp_purge),
                     )
                 )
                 session.add(
@@ -565,8 +565,8 @@ async def _add_db_entries(hass: HomeAssistant, cutoff: datetime, rows: int) -> N
                         entity_id="test.cutoff",
                         state="purge",
                         attributes="{}",
-                        last_changed=timestamp_purge,
-                        last_updated=timestamp_purge,
+                        last_changed_ts=dt_util.utc_to_timestamp(timestamp_purge),
+                        last_updated_ts=dt_util.utc_to_timestamp(timestamp_purge),
                         event_id=1000 + row,
                         attributes_id=1000 + row,
                     )
@@ -690,8 +690,8 @@ def _add_db_entries(hass: HomeAssistant) -> None:
                     entity_id="sensor.excluded",
                     state="purgeme",
                     attributes="{}",
-                    last_changed=timestamp,
-                    last_updated=timestamp,
+                    last_changed_ts=dt_util.utc_to_timestamp(timestamp),
+                    last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                 )
             )
             # Add states and state_changed events that should be keeped
@@ -716,8 +716,8 @@ def _add_db_entries(hass: HomeAssistant) -> None:
                 entity_id="sensor.linked_old_state_id",
                 state="keep",
                 attributes="{}",
-                last_changed=timestamp,
-                last_updated=timestamp,
+                last_changed_ts=dt_util.utc_to_timestamp(timestamp),
+                last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                 old_state_id=1,
                 state_attributes=state_attrs,
             )
@@ -726,8 +726,8 @@ def _add_db_entries(hass: HomeAssistant) -> None:
                 entity_id="sensor.linked_old_state_id",
                 state="keep",
                 attributes="{}",
-                last_changed=timestamp,
-                last_updated=timestamp,
+                last_changed_ts=dt_util.utc_to_timestamp(timestamp),
+                last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                 old_state_id=2,
                 state_attributes=state_attrs,
             )
@@ -735,8 +735,8 @@ def _add_db_entries(hass: HomeAssistant) -> None:
                 entity_id="sensor.linked_old_state_id",
                 state="keep",
                 attributes="{}",
-                last_changed=timestamp,
-                last_updated=timestamp,
+                last_changed_ts=dt_util.utc_to_timestamp(timestamp),
+                last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                 old_state_id=62,  # keep
                 state_attributes=state_attrs,
             )
@@ -748,7 +748,7 @@ def _add_db_entries(hass: HomeAssistant) -> None:
                     event_type="EVENT_KEEP",
                     event_data="{}",
                     origin="LOCAL",
-                    time_fired=timestamp,
+                    time_fired_ts=dt_util.utc_to_timestamp(timestamp),
                 )
             )
 
@@ -920,8 +920,8 @@ def _add_db_entries(hass: HomeAssistant) -> None:
                     entity_id="sensor.old_format",
                     state=STATE_ON,
                     attributes=json.dumps({"old": "not_using_state_attributes"}),
-                    last_changed=timestamp,
-                    last_updated=timestamp,
+                    last_changed_ts=dt_util.utc_to_timestamp(timestamp),
+                    last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                     event_id=event_id,
                     state_attributes=None,
                 )
@@ -932,7 +932,7 @@ def _add_db_entries(hass: HomeAssistant) -> None:
                     event_type=EVENT_STATE_CHANGED,
                     event_data="{}",
                     origin="LOCAL",
-                    time_fired=timestamp,
+                    time_fired_ts=dt_util.utc_to_timestamp(timestamp),
                 )
             )
             session.add(
@@ -941,7 +941,7 @@ def _add_db_entries(hass: HomeAssistant) -> None:
                     event_type=EVENT_THEMES_UPDATED,
                     event_data="{}",
                     origin="LOCAL",
-                    time_fired=timestamp,
+                    time_fired_ts=dt_util.utc_to_timestamp(timestamp),
                 )
             )
 
@@ -993,7 +993,7 @@ def _add_db_entries(hass: HomeAssistant) -> None:
                             event_type="EVENT_PURGE",
                             event_data="{}",
                             origin="LOCAL",
-                            time_fired=timestamp,
+                            time_fired_ts=dt_util.utc_to_timestamp(timestamp),
                         )
                     )
 
@@ -1093,7 +1093,7 @@ def _add_db_entries(hass: HomeAssistant) -> None:
                         event_type="EVENT_KEEP",
                         event_data="{}",
                         origin="LOCAL",
-                        time_fired=timestamp,
+                        time_fired_ts=dt_util.utc_to_timestamp(timestamp),
                     )
                 )
             # Add states with linked old_state_ids that need to be handled
@@ -1102,8 +1102,8 @@ def _add_db_entries(hass: HomeAssistant) -> None:
                 entity_id="sensor.linked_old_state_id",
                 state="keep",
                 attributes="{}",
-                last_changed=timestamp,
-                last_updated=timestamp,
+                last_changed_ts=dt_util.utc_to_timestamp(timestamp),
+                last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                 old_state_id=1,
             )
             timestamp = dt_util.utcnow() - timedelta(days=4)
@@ -1111,16 +1111,16 @@ def _add_db_entries(hass: HomeAssistant) -> None:
                 entity_id="sensor.linked_old_state_id",
                 state="keep",
                 attributes="{}",
-                last_changed=timestamp,
-                last_updated=timestamp,
+                last_changed_ts=dt_util.utc_to_timestamp(timestamp),
+                last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                 old_state_id=2,
             )
             state_3 = States(
                 entity_id="sensor.linked_old_state_id",
                 state="keep",
                 attributes="{}",
-                last_changed=timestamp,
-                last_updated=timestamp,
+                last_changed_ts=dt_util.utc_to_timestamp(timestamp),
+                last_updated_ts=dt_util.utc_to_timestamp(timestamp),
                 old_state_id=62,  # keep
             )
             session.add_all((state_1, state_2, state_3))
@@ -1355,7 +1355,7 @@ async def _add_test_events(hass: HomeAssistant, iterations: int = 1):
                         event_type=event_type,
                         event_data=json.dumps(event_data),
                         origin="LOCAL",
-                        time_fired=timestamp,
+                        time_fired_ts=dt_util.utc_to_timestamp(timestamp),
                     )
                 )
 
@@ -1392,7 +1392,7 @@ async def _add_events_with_event_data(hass: HomeAssistant, iterations: int = 1):
                     Events(
                         event_type=event_type,
                         origin="LOCAL",
-                        time_fired=timestamp,
+                        time_fired_ts=dt_util.utc_to_timestamp(timestamp),
                         event_data_rel=event_data,
                     )
                 )
@@ -1494,8 +1494,8 @@ def _add_state_without_event_linkage(
             entity_id=entity_id,
             state=state,
             attributes=None,
-            last_changed=timestamp,
-            last_updated=timestamp,
+            last_changed_ts=dt_util.utc_to_timestamp(timestamp),
+            last_updated_ts=dt_util.utc_to_timestamp(timestamp),
             event_id=None,
             state_attributes=state_attrs,
         )
@@ -1519,8 +1519,8 @@ def _add_state_and_state_changed_event(
             entity_id=entity_id,
             state=state,
             attributes=None,
-            last_changed=timestamp,
-            last_updated=timestamp,
+            last_changed_ts=dt_util.utc_to_timestamp(timestamp),
+            last_updated_ts=dt_util.utc_to_timestamp(timestamp),
             event_id=event_id,
             state_attributes=state_attrs,
         )
@@ -1531,7 +1531,7 @@ def _add_state_and_state_changed_event(
             event_type=EVENT_STATE_CHANGED,
             event_data="{}",
             origin="LOCAL",
-            time_fired=timestamp,
+            time_fired_ts=dt_util.utc_to_timestamp(timestamp),
         )
     )
 
@@ -1600,8 +1600,8 @@ async def test_purge_can_mix_legacy_and_new_format(
         broken_state_no_time = States(
             event_id=None,
             entity_id="orphened.state",
-            last_updated=None,
-            last_changed=None,
+            last_updated_ts=None,
+            last_changed_ts=None,
         )
         session.add(broken_state_no_time)
         start_id = 50000

From 611000fcc992664fd57d095ee50de334d7a8b56a Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Fri, 30 Dec 2022 11:49:41 -1000
Subject: [PATCH 09/38] updated

---
 homeassistant/components/recorder/db_schema.py | 7 ++-----
 homeassistant/components/recorder/migration.py | 4 ++++
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/homeassistant/components/recorder/db_schema.py b/homeassistant/components/recorder/db_schema.py
index cd7da874381884..f91da8ba03fbeb 100644
--- a/homeassistant/components/recorder/db_schema.py
+++ b/homeassistant/components/recorder/db_schema.py
@@ -158,7 +158,7 @@ class Events(Base):  # type: ignore[misc,valid-type]
     event_data = Column(Text().with_variant(mysql.LONGTEXT, "mysql"))
     origin = Column(String(MAX_LENGTH_EVENT_ORIGIN))  # no longer used for new rows
     origin_idx = Column(SmallInteger)
-    time_fired = Column(DATETIME_TYPE, index=True)  # no longer used for new rows
+    time_fired = Column(DATETIME_TYPE)  # no longer used for new rows
     time_fired_ts = Column(TIMESTAMP_TYPE, index=True)
     context_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID), index=True)
     context_user_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID))
@@ -281,9 +281,7 @@ class States(Base):  # type: ignore[misc,valid-type]
     )
     last_changed = Column(DATETIME_TYPE)  # no longer used for new rows
     last_changed_ts = Column(TIMESTAMP_TYPE)
-    last_updated = Column(
-        DATETIME_TYPE, default=dt_util.utcnow, index=True
-    )  # no longer used for new rows
+    last_updated = Column(DATETIME_TYPE)  # no longer used for new rows
     last_updated_ts = Column(TIMESTAMP_TYPE, default=time.time, index=True)
     old_state_id = Column(Integer, ForeignKey("states.state_id"), index=True)
     attributes_id = Column(
@@ -320,7 +318,6 @@ def from_event(event: Event) -> States:
             last_updated=None,
             last_changed=None,
         )
-
         # None state means the state was removed from the state machine
         if state is None:
             dbstate.state = ""
diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py
index a5f86462a409ce..03761c3dcafbf3 100644
--- a/homeassistant/components/recorder/migration.py
+++ b/homeassistant/components/recorder/migration.py
@@ -829,10 +829,14 @@ def _apply_update(  # noqa: C901
         # Migrate all data in States.last_changed to States.last_changed_ts and
         # wipe States.last_changed
         # TODO: implement
+        _create_index(session_maker, "events", "ix_events_time_fired_ts")
         _create_index(session_maker, "events", "ix_events_event_type_time_fired_ts")
         _create_index(session_maker, "states", "ix_states_entity_id_last_updated_ts")
+        _create_index(session_maker, "states", "ix_states_last_updated_ts")
         _drop_index(session_maker, "states", "ix_states_entity_id_last_updated")
         _drop_index(session_maker, "events", "ix_events_event_type_time_fired")
+        _drop_index(session_maker, "states", "ix_states_last_updated")
+        _drop_index(session_maker, "events", "ix_events_time_fired")
     else:
         raise ValueError(f"No schema migration defined for version {new_version}")
 

From a8ed817360201b585ddfcf1b3dca7475460c94cd Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Fri, 30 Dec 2022 12:15:37 -1000
Subject: [PATCH 10/38] fix a few more

---
 homeassistant/components/logbook/models.py    |  8 ++---
 homeassistant/components/logbook/processor.py |  6 ++--
 .../components/logbook/queries/__init__.py    |  8 +++--
 .../components/logbook/queries/all.py         | 18 +++++------
 .../components/logbook/queries/common.py      | 32 +++++++++----------
 .../components/logbook/queries/devices.py     | 15 ++++-----
 .../components/logbook/queries/entities.py    | 25 ++++++++-------
 .../logbook/queries/entities_and_devices.py   | 19 +++++------
 .../components/recorder/db_schema.py          |  3 +-
 tests/components/logbook/common.py            |  1 +
 tests/components/logbook/test_init.py         |  5 ++-
 11 files changed, 72 insertions(+), 68 deletions(-)

diff --git a/homeassistant/components/logbook/models.py b/homeassistant/components/logbook/models.py
index 591781745f708e..0b0a9aeb414c62 100644
--- a/homeassistant/components/logbook/models.py
+++ b/homeassistant/components/logbook/models.py
@@ -2,7 +2,6 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from datetime import datetime as dt
 import json
 from typing import Any, cast
 
@@ -10,6 +9,7 @@
 
 from homeassistant.const import ATTR_ICON, EVENT_STATE_CHANGED
 from homeassistant.core import Context, Event, State, callback
+import homeassistant.util.dt as dt_util
 
 
 class LazyEventPartialState:
@@ -66,7 +66,7 @@ class EventAsRow:
     data: dict[str, Any]
     context: Context
     context_id: str
-    time_fired: dt
+    time_fired_ts: float
     state_id: int
     event_data: str | None = None
     old_format_icon: None = None
@@ -92,7 +92,7 @@ def async_event_to_row(event: Event) -> EventAsRow | None:
             context_id=event.context.id,
             context_user_id=event.context.user_id,
             context_parent_id=event.context.parent_id,
-            time_fired=event.time_fired,
+            time_fired_ts=dt_util.utc_to_timestamp(event.time_fired),
             state_id=hash(event),
         )
     # States are prefiltered so we never get states
@@ -107,7 +107,7 @@ def async_event_to_row(event: Event) -> EventAsRow | None:
         context_id=new_state.context.id,
         context_user_id=new_state.context.user_id,
         context_parent_id=new_state.context.parent_id,
-        time_fired=new_state.last_updated,
+        time_fired_ts=dt_util.utc_to_timestamp(new_state.last_updated),
         state_id=hash(event),
         icon=new_state.attributes.get(ATTR_ICON),
     )
diff --git a/homeassistant/components/logbook/processor.py b/homeassistant/components/logbook/processor.py
index 6d491ec2892c70..1a0dd478c0324f 100644
--- a/homeassistant/components/logbook/processor.py
+++ b/homeassistant/components/logbook/processor.py
@@ -388,12 +388,14 @@ def _rows_match(row: Row | EventAsRow, other_row: Row | EventAsRow) -> bool:
 
 def _row_time_fired_isoformat(row: Row | EventAsRow) -> str:
     """Convert the row timed_fired to isoformat."""
-    return process_timestamp_to_utc_isoformat(row.time_fired or dt_util.utcnow())
+    return process_timestamp_to_utc_isoformat(
+        dt_util.utc_from_timestamp(row.time_fired_ts) or dt_util.utcnow()
+    )
 
 
 def _row_time_fired_timestamp(row: Row | EventAsRow) -> float:
     """Convert the row timed_fired to timestamp."""
-    return process_datetime_to_timestamp(row.time_fired or dt_util.utcnow())
+    return row.time_fired_ts or process_datetime_to_timestamp(dt_util.utcnow())
 
 
 class EntityNameCache:
diff --git a/homeassistant/components/logbook/queries/__init__.py b/homeassistant/components/logbook/queries/__init__.py
index 0c3a63f990edd8..8a2ee40de4f295 100644
--- a/homeassistant/components/logbook/queries/__init__.py
+++ b/homeassistant/components/logbook/queries/__init__.py
@@ -7,6 +7,7 @@
 
 from homeassistant.components.recorder.filters import Filters
 from homeassistant.helpers.json import json_dumps
+from homeassistant.util import dt as dt_util
 
 from .all import all_stmt
 from .devices import devices_stmt
@@ -15,8 +16,8 @@
 
 
 def statement_for_request(
-    start_day: dt,
-    end_day: dt,
+    start_day_dt: dt,
+    end_day_dt: dt,
     event_types: tuple[str, ...],
     entity_ids: list[str] | None = None,
     device_ids: list[str] | None = None,
@@ -24,7 +25,8 @@ def statement_for_request(
     context_id: str | None = None,
 ) -> StatementLambdaElement:
     """Generate the logbook statement for a logbook request."""
-
+    start_day = dt_util.utc_to_timestamp(start_day_dt)
+    end_day = dt_util.utc_to_timestamp(end_day_dt)
     # No entities: logbook sends everything for the timeframe
     # limited by the context_id and the yaml configured filter
     if not entity_ids and not device_ids:
diff --git a/homeassistant/components/logbook/queries/all.py b/homeassistant/components/logbook/queries/all.py
index da05aa02fff1d4..21624181a3b3d0 100644
--- a/homeassistant/components/logbook/queries/all.py
+++ b/homeassistant/components/logbook/queries/all.py
@@ -1,15 +1,13 @@
 """All queries for logbook."""
 from __future__ import annotations
 
-from datetime import datetime as dt
-
 from sqlalchemy import lambda_stmt
 from sqlalchemy.orm import Query
 from sqlalchemy.sql.elements import ClauseList
 from sqlalchemy.sql.lambdas import StatementLambdaElement
 
 from homeassistant.components.recorder.db_schema import (
-    LAST_UPDATED_INDEX,
+    LAST_UPDATED_INDEX_TS,
     Events,
     States,
 )
@@ -23,8 +21,8 @@
 
 
 def all_stmt(
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
     states_entity_filter: ClauseList | None = None,
     events_entity_filter: ClauseList | None = None,
@@ -53,22 +51,24 @@ def all_stmt(
         else:
             stmt += lambda s: s.union_all(_states_query_for_all(start_day, end_day))
 
-    stmt += lambda s: s.order_by(Events.time_fired)
+    stmt += lambda s: s.order_by(Events.time_fired_ts)
     return stmt
 
 
-def _states_query_for_all(start_day: dt, end_day: dt) -> Query:
+def _states_query_for_all(start_day: float, end_day: float) -> Query:
     return apply_states_filters(_apply_all_hints(select_states()), start_day, end_day)
 
 
 def _apply_all_hints(query: Query) -> Query:
     """Force mysql to use the right index on large selects."""
     return query.with_hint(
-        States, f"FORCE INDEX ({LAST_UPDATED_INDEX})", dialect_name="mysql"
+        States, f"FORCE INDEX ({LAST_UPDATED_INDEX_TS})", dialect_name="mysql"
     )
 
 
-def _states_query_for_context_id(start_day: dt, end_day: dt, context_id: str) -> Query:
+def _states_query_for_context_id(
+    start_day: float, end_day: float, context_id: str
+) -> Query:
     return apply_states_filters(select_states(), start_day, end_day).where(
         States.context_id == context_id
     )
diff --git a/homeassistant/components/logbook/queries/common.py b/homeassistant/components/logbook/queries/common.py
index 466df668da8b54..424a174b7afae1 100644
--- a/homeassistant/components/logbook/queries/common.py
+++ b/homeassistant/components/logbook/queries/common.py
@@ -1,8 +1,6 @@
 """Queries for logbook."""
 from __future__ import annotations
 
-from datetime import datetime as dt
-
 import sqlalchemy
 from sqlalchemy import select
 from sqlalchemy.orm import Query
@@ -47,7 +45,7 @@
     Events.event_id.label("event_id"),
     Events.event_type.label("event_type"),
     Events.event_data.label("event_data"),
-    Events.time_fired.label("time_fired"),
+    Events.time_fired_ts.label("time_fired_ts"),
     Events.context_id.label("context_id"),
     Events.context_user_id.label("context_user_id"),
     Events.context_parent_id.label("context_parent_id"),
@@ -79,7 +77,7 @@
         "event_type"
     ),
     literal(value=None, type_=sqlalchemy.Text).label("event_data"),
-    States.last_updated.label("time_fired"),
+    States.last_updated_ts.label("time_fired_ts"),
     States.context_id.label("context_id"),
     States.context_user_id.label("context_user_id"),
     States.context_parent_id.label("context_parent_id"),
@@ -108,14 +106,14 @@
 
 
 def select_events_context_id_subquery(
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
 ) -> Select:
     """Generate the select for a context_id subquery."""
     return (
         select(Events.context_id)
-        .where((Events.time_fired > start_day) & (Events.time_fired < end_day))
+        .where((Events.time_fired_ts > start_day) & (Events.time_fired_ts < end_day))
         .where(Events.event_type.in_(event_types))
         .outerjoin(EventData, (Events.data_id == EventData.data_id))
     )
@@ -142,12 +140,12 @@ def select_states_context_only() -> Select:
 
 
 def select_events_without_states(
-    start_day: dt, end_day: dt, event_types: tuple[str, ...]
+    start_day: float, end_day: float, event_types: tuple[str, ...]
 ) -> Select:
     """Generate an events select that does not join states."""
     return (
         select(*EVENT_ROWS_NO_STATES, NOT_CONTEXT_ONLY)
-        .where((Events.time_fired > start_day) & (Events.time_fired < end_day))
+        .where((Events.time_fired_ts > start_day) & (Events.time_fired_ts < end_day))
         .where(Events.event_type.in_(event_types))
         .outerjoin(EventData, (Events.data_id == EventData.data_id))
     )
@@ -163,7 +161,7 @@ def select_states() -> Select:
 
 
 def legacy_select_events_context_id(
-    start_day: dt, end_day: dt, context_id: str
+    start_day: float, end_day: float, context_id: str
 ) -> Select:
     """Generate a legacy events context id select that also joins states."""
     # This can be removed once we no longer have event_ids in the states table
@@ -176,33 +174,35 @@ def legacy_select_events_context_id(
         )
         .outerjoin(States, (Events.event_id == States.event_id))
         .where(
-            (States.last_updated == States.last_changed) | States.last_changed.is_(None)
+            (States.last_updated_ts == States.last_changed_ts)
+            | States.last_changed_ts.is_(None)
         )
         .where(_not_continuous_entity_matcher())
         .outerjoin(
             StateAttributes, (States.attributes_id == StateAttributes.attributes_id)
         )
-        .where((Events.time_fired > start_day) & (Events.time_fired < end_day))
+        .where((Events.time_fired_ts > start_day) & (Events.time_fired_ts < end_day))
         .where(Events.context_id == context_id)
     )
 
 
-def apply_states_filters(query: Query, start_day: dt, end_day: dt) -> Query:
+def apply_states_filters(query: Query, start_day: float, end_day: float) -> Query:
     """Filter states by time range.
 
     Filters states that do not have an old state or new state (added / removed)
     Filters states that are in a continuous domain with a UOM.
-    Filters states that do not have matching last_updated and last_changed.
+    Filters states that do not have matching last_updated_ts and last_changed_ts.
     """
     return (
         query.filter(
-            (States.last_updated > start_day) & (States.last_updated < end_day)
+            (States.last_updated_ts > start_day) & (States.last_updated_ts < end_day)
         )
         .outerjoin(OLD_STATE, (States.old_state_id == OLD_STATE.state_id))
         .where(_missing_state_matcher())
         .where(_not_continuous_entity_matcher())
         .where(
-            (States.last_updated == States.last_changed) | States.last_changed.is_(None)
+            (States.last_updated_ts == States.last_changed_ts)
+            | States.last_changed_ts.is_(None)
         )
         .outerjoin(
             StateAttributes, (States.attributes_id == StateAttributes.attributes_id)
diff --git a/homeassistant/components/logbook/queries/devices.py b/homeassistant/components/logbook/queries/devices.py
index e268c2d3ac3242..a270f1996cea05 100644
--- a/homeassistant/components/logbook/queries/devices.py
+++ b/homeassistant/components/logbook/queries/devices.py
@@ -2,7 +2,6 @@
 from __future__ import annotations
 
 from collections.abc import Iterable
-from datetime import datetime as dt
 
 import sqlalchemy
 from sqlalchemy import lambda_stmt, select
@@ -29,8 +28,8 @@
 
 
 def _select_device_id_context_ids_sub_query(
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
     json_quotable_device_ids: list[str],
 ) -> CompoundSelect:
@@ -43,8 +42,8 @@ def _select_device_id_context_ids_sub_query(
 
 def _apply_devices_context_union(
     query: Query,
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
     json_quotable_device_ids: list[str],
 ) -> CompoundSelect:
@@ -70,8 +69,8 @@ def _apply_devices_context_union(
 
 
 def devices_stmt(
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
     json_quotable_device_ids: list[str],
 ) -> StatementLambdaElement:
@@ -85,7 +84,7 @@ def devices_stmt(
             end_day,
             event_types,
             json_quotable_device_ids,
-        ).order_by(Events.time_fired)
+        ).order_by(Events.time_fired_ts)
     )
     return stmt
 
diff --git a/homeassistant/components/logbook/queries/entities.py b/homeassistant/components/logbook/queries/entities.py
index 3803da6f4e8bd5..afe7c7c7c2e4dc 100644
--- a/homeassistant/components/logbook/queries/entities.py
+++ b/homeassistant/components/logbook/queries/entities.py
@@ -2,7 +2,6 @@
 from __future__ import annotations
 
 from collections.abc import Iterable
-from datetime import datetime as dt
 
 import sqlalchemy
 from sqlalchemy import lambda_stmt, select, union_all
@@ -12,7 +11,7 @@
 
 from homeassistant.components.recorder.db_schema import (
     ENTITY_ID_IN_EVENT,
-    ENTITY_ID_LAST_UPDATED_INDEX,
+    ENTITY_ID_LAST_UPDATED_INDEX_TS,
     OLD_ENTITY_ID_IN_EVENT,
     EventData,
     Events,
@@ -32,8 +31,8 @@
 
 
 def _select_entities_context_ids_sub_query(
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
     entity_ids: list[str],
     json_quoted_entity_ids: list[str],
@@ -44,7 +43,9 @@ def _select_entities_context_ids_sub_query(
             apply_event_entity_id_matchers(json_quoted_entity_ids)
         ),
         apply_entities_hints(select(States.context_id))
-        .filter((States.last_updated > start_day) & (States.last_updated < end_day))
+        .filter(
+            (States.last_updated_ts > start_day) & (States.last_updated_ts < end_day)
+        )
         .where(States.entity_id.in_(entity_ids)),
     )
     return select(union.c.context_id).group_by(union.c.context_id)
@@ -52,8 +53,8 @@ def _select_entities_context_ids_sub_query(
 
 def _apply_entities_context_union(
     query: Query,
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
     entity_ids: list[str],
     json_quoted_entity_ids: list[str],
@@ -87,8 +88,8 @@ def _apply_entities_context_union(
 
 
 def entities_stmt(
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
     entity_ids: list[str],
     json_quoted_entity_ids: list[str],
@@ -104,12 +105,12 @@ def entities_stmt(
             event_types,
             entity_ids,
             json_quoted_entity_ids,
-        ).order_by(Events.time_fired)
+        ).order_by(Events.time_fired_ts)
     )
 
 
 def states_query_for_entity_ids(
-    start_day: dt, end_day: dt, entity_ids: list[str]
+    start_day: float, end_day: float, entity_ids: list[str]
 ) -> Query:
     """Generate a select for states from the States table for specific entities."""
     return apply_states_filters(
@@ -136,5 +137,5 @@ def apply_event_entity_id_matchers(
 def apply_entities_hints(query: Query) -> Query:
     """Force mysql to use the right index on large selects."""
     return query.with_hint(
-        States, f"FORCE INDEX ({ENTITY_ID_LAST_UPDATED_INDEX})", dialect_name="mysql"
+        States, f"FORCE INDEX ({ENTITY_ID_LAST_UPDATED_INDEX_TS})", dialect_name="mysql"
     )
diff --git a/homeassistant/components/logbook/queries/entities_and_devices.py b/homeassistant/components/logbook/queries/entities_and_devices.py
index f22a8392e19221..94e9afc551d32f 100644
--- a/homeassistant/components/logbook/queries/entities_and_devices.py
+++ b/homeassistant/components/logbook/queries/entities_and_devices.py
@@ -2,7 +2,6 @@
 from __future__ import annotations
 
 from collections.abc import Iterable
-from datetime import datetime as dt
 
 import sqlalchemy
 from sqlalchemy import lambda_stmt, select, union_all
@@ -29,8 +28,8 @@
 
 
 def _select_entities_device_id_context_ids_sub_query(
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
     entity_ids: list[str],
     json_quoted_entity_ids: list[str],
@@ -44,7 +43,9 @@ def _select_entities_device_id_context_ids_sub_query(
             )
         ),
         apply_entities_hints(select(States.context_id))
-        .filter((States.last_updated > start_day) & (States.last_updated < end_day))
+        .filter(
+            (States.last_updated_ts > start_day) & (States.last_updated_ts < end_day)
+        )
         .where(States.entity_id.in_(entity_ids)),
     )
     return select(union.c.context_id).group_by(union.c.context_id)
@@ -52,8 +53,8 @@ def _select_entities_device_id_context_ids_sub_query(
 
 def _apply_entities_devices_context_union(
     query: Query,
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
     entity_ids: list[str],
     json_quoted_entity_ids: list[str],
@@ -88,8 +89,8 @@ def _apply_entities_devices_context_union(
 
 
 def entities_devices_stmt(
-    start_day: dt,
-    end_day: dt,
+    start_day: float,
+    end_day: float,
     event_types: tuple[str, ...],
     entity_ids: list[str],
     json_quoted_entity_ids: list[str],
@@ -109,7 +110,7 @@ def entities_devices_stmt(
             entity_ids,
             json_quoted_entity_ids,
             json_quoted_device_ids,
-        ).order_by(Events.time_fired)
+        ).order_by(Events.time_fired_ts)
     )
     return stmt
 
diff --git a/homeassistant/components/recorder/db_schema.py b/homeassistant/components/recorder/db_schema.py
index f91da8ba03fbeb..ecf6ed8c202457 100644
--- a/homeassistant/components/recorder/db_schema.py
+++ b/homeassistant/components/recorder/db_schema.py
@@ -91,8 +91,7 @@
     TABLE_SCHEMA_CHANGES,
 ]
 
-LAST_UPDATED_INDEX = "ix_states_last_updated"
-ENTITY_ID_LAST_UPDATED_INDEX = "ix_states_entity_id_last_updated"
+LAST_UPDATED_INDEX_TS = "ix_states_last_updated_ts"
 ENTITY_ID_LAST_UPDATED_INDEX_TS = "ix_states_entity_id_last_updated_ts"
 EVENTS_CONTEXT_ID_INDEX = "ix_events_context_id"
 STATES_CONTEXT_ID_INDEX = "ix_states_context_id"
diff --git a/tests/components/logbook/common.py b/tests/components/logbook/common.py
index a41f983bfedf36..e6bce9e6fbc0b8 100644
--- a/tests/components/logbook/common.py
+++ b/tests/components/logbook/common.py
@@ -27,6 +27,7 @@ def __init__(
         self.shared_data = json.dumps(data, cls=JSONEncoder)
         self.data = data
         self.time_fired = dt_util.utcnow()
+        self.time_fired_ts = dt_util.utc_to_timestamp(self.time_fired)
         self.context_parent_id = context.parent_id if context else None
         self.context_user_id = context.user_id if context else None
         self.context_id = context.id if context else None
diff --git a/tests/components/logbook/test_init.py b/tests/components/logbook/test_init.py
index 366b4b30ed5431..889e8f97106933 100644
--- a/tests/components/logbook/test_init.py
+++ b/tests/components/logbook/test_init.py
@@ -313,9 +313,7 @@ def create_state_changed_event_from_old_new(
     row = collections.namedtuple(
         "Row",
         [
-            "event_type"
-            "event_data"
-            "time_fired"
+            "event_type" "event_data" "time_fired" "time_fired_ts",
             "context_id"
             "context_user_id"
             "context_parent_id"
@@ -337,6 +335,7 @@ def create_state_changed_event_from_old_new(
     row.attributes = attributes_json
     row.shared_attrs = attributes_json
     row.time_fired = event_time_fired
+    row.time_fired_ts = dt_util.utc_to_timestamp(event_time_fired)
     row.state = new_state and new_state.get("state")
     row.entity_id = entity_id
     row.domain = entity_id and ha.split_entity_id(entity_id)[0]

From 378e05fdafe311adbf7a4b90f45c376b66002f8e Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 31 Dec 2022 09:40:25 -1000
Subject: [PATCH 11/38] tweak

---
 homeassistant/components/recorder/migration.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py
index 03761c3dcafbf3..fedf5d741ee924 100644
--- a/homeassistant/components/recorder/migration.py
+++ b/homeassistant/components/recorder/migration.py
@@ -829,6 +829,10 @@ def _apply_update(  # noqa: C901
         # Migrate all data in States.last_changed to States.last_changed_ts and
         # wipe States.last_changed
         # TODO: implement
+        # Once we require SQLite >= 3.35.5, we should drop the column:
+        # ALTER TABLE events DROP COLUMN time_fired
+        # ALTER TABLE states DROP COLUMN last_updated
+        # ALTER TABLE states DROP COLUMN last_changed
         _create_index(session_maker, "events", "ix_events_time_fired_ts")
         _create_index(session_maker, "events", "ix_events_event_type_time_fired_ts")
         _create_index(session_maker, "states", "ix_states_entity_id_last_updated_ts")

From 8a65d42f1c1bd785f44f191bdd53debf9678e446 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 31 Dec 2022 09:52:32 -1000
Subject: [PATCH 12/38] fix schema

---
 .../components/recorder/db_schema.py          | 22 ++++++++++++++++---
 homeassistant/components/recorder/history.py  |  4 ++--
 .../components/recorder/migration.py          |  2 +-
 tests/components/recorder/test_history.py     |  2 --
 4 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/homeassistant/components/recorder/db_schema.py b/homeassistant/components/recorder/db_schema.py
index ecf6ed8c202457..0cbbbe90b2aef8 100644
--- a/homeassistant/components/recorder/db_schema.py
+++ b/homeassistant/components/recorder/db_schema.py
@@ -48,7 +48,7 @@
 import homeassistant.util.dt as dt_util
 
 from .const import ALL_DOMAIN_EXCLUDE_ATTRS
-from .models import StatisticData, StatisticMetaData
+from .models import StatisticData, StatisticMetaData, process_timestamp
 
 # SQLAlchemy Schema
 # pylint: disable=invalid-name
@@ -170,10 +170,18 @@ def __repr__(self) -> str:
         return (
             "<recorder.Events("
             f"id={self.event_id}, type='{self.event_type}', "
-            f"origin_idx='{self.origin_idx}', time_fired='{self.time_fired}'"
+            f"origin_idx='{self.origin_idx}', time_fired='{self.time_fired_isotime}'"
             f", data_id={self.data_id})>"
         )
 
+    @property
+    def time_fired_isotime(self) -> str:
+        """Return time_fired as an isotime string."""
+        date_time = dt_util.utc_from_timestamp(self.time_fired_ts) or process_timestamp(
+            self.time_fired
+        )
+        return date_time.isoformat(sep=" ", timespec="seconds")
+
     @staticmethod
     def from_event(event: Event) -> Events:
         """Create an event database object from a native event."""
@@ -298,10 +306,18 @@ def __repr__(self) -> str:
         return (
             f"<recorder.States(id={self.state_id}, entity_id='{self.entity_id}',"
             f" state='{self.state}', event_id='{self.event_id}',"
-            f" last_updated='{self.last_updated.isoformat(sep=' ', timespec='seconds')}',"
+            f" last_updated='{self.last_updated_isotime}',"
             f" old_state_id={self.old_state_id}, attributes_id={self.attributes_id})>"
         )
 
+    @property
+    def last_updated_isotime(self) -> str:
+        """Return last_updated as an isotime string."""
+        date_time = dt_util.utc_from_timestamp(
+            self.last_updated_ts
+        ) or process_timestamp(self.last_updated)
+        return date_time.isoformat(sep=" ", timespec="seconds")
+
     @staticmethod
     def from_event(event: Event) -> States:
         """Create object from a state_changed event."""
diff --git a/homeassistant/components/recorder/history.py b/homeassistant/components/recorder/history.py
index b8d7a5568fd889..265ab68dce8d1e 100644
--- a/homeassistant/components/recorder/history.py
+++ b/homeassistant/components/recorder/history.py
@@ -462,11 +462,11 @@ def _state_changed_during_period_stmt(
         )
     if descending:
         if schema_version >= 31:
-            stmt += lambda q: q.order_by(States.entity_id, States.last_updated.desc())
-        else:
             stmt += lambda q: q.order_by(
                 States.entity_id, States.last_updated_ts.desc()
             )
+        else:
+            stmt += lambda q: q.order_by(States.entity_id, States.last_updated.desc())
     else:
         if schema_version >= 31:
             stmt += lambda q: q.order_by(States.entity_id, States.last_updated_ts)
diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py
index fedf5d741ee924..1b8aaf9555818c 100644
--- a/homeassistant/components/recorder/migration.py
+++ b/homeassistant/components/recorder/migration.py
@@ -860,7 +860,7 @@ def _initialize_database(session: Session) -> bool:
     indexes = inspector.get_indexes("events")
 
     for index in indexes:
-        if index["column_names"] == ["time_fired"]:
+        if index["column_names"] == ["time_fired_ts"]:
             # Schema addition from version 1 detected. New DB.
             session.add(StatisticsRuns(start=get_start_time()))
             session.add(SchemaChanges(schema_version=SCHEMA_VERSION))
diff --git a/tests/components/recorder/test_history.py b/tests/components/recorder/test_history.py
index cf9015ff7ca1c6..48b5cd7243666c 100644
--- a/tests/components/recorder/test_history.py
+++ b/tests/components/recorder/test_history.py
@@ -910,9 +910,7 @@ def _fetch_db_states() -> list[States]:
         )
         == state0.last_changed
     )
-    assert db_sensor_one_states[0].last_updated is not None
     assert db_sensor_one_states[0].last_updated_ts is not None
-    assert db_sensor_one_states[1].last_updated is not None
     assert db_sensor_one_states[1].last_updated_ts is not None
     assert (
         db_sensor_one_states[0].last_updated_ts

From cecc80b141dad04315921dac41a68000fdabd889 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 31 Dec 2022 09:59:37 -1000
Subject: [PATCH 13/38] fixes

---
 homeassistant/components/recorder/migration.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py
index 1b8aaf9555818c..c6f2c9d9fef80b 100644
--- a/homeassistant/components/recorder/migration.py
+++ b/homeassistant/components/recorder/migration.py
@@ -860,7 +860,7 @@ def _initialize_database(session: Session) -> bool:
     indexes = inspector.get_indexes("events")
 
     for index in indexes:
-        if index["column_names"] == ["time_fired_ts"]:
+        if index["column_names"] in (["time_fired"], ["time_fired_ts"]):
             # Schema addition from version 1 detected. New DB.
             session.add(StatisticsRuns(start=get_start_time()))
             session.add(SchemaChanges(schema_version=SCHEMA_VERSION))

From 2a73ca48f1700d18dabedbbbee5e0bc3b178617a Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 31 Dec 2022 10:32:25 -1000
Subject: [PATCH 14/38] tweak

---
 homeassistant/components/recorder/db_schema.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/homeassistant/components/recorder/db_schema.py b/homeassistant/components/recorder/db_schema.py
index 0cbbbe90b2aef8..a35a1ad996ba52 100644
--- a/homeassistant/components/recorder/db_schema.py
+++ b/homeassistant/components/recorder/db_schema.py
@@ -122,7 +122,8 @@ def result_processor(self, dialect, coltype):  # type: ignore[no-untyped-def]
     .with_variant(oracle.DOUBLE_PRECISION(), "oracle")
     .with_variant(postgresql.DOUBLE_PRECISION(), "postgresql")
 )
-TIMESTAMP_TYPE = Float(7)
+
+TIMESTAMP_TYPE = Float(16)  # Covers up to at least 2250 AD
 
 
 class JSONLiteral(JSON):  # type: ignore[misc]

From fd7b1fd6306b9cf25fd0705ddf462495de160036 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 31 Dec 2022 10:45:36 -1000
Subject: [PATCH 15/38] fix migration

---
 .../components/recorder/migration.py          | 20 ++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py
index c6f2c9d9fef80b..8fd8c65b7094b1 100644
--- a/homeassistant/components/recorder/migration.py
+++ b/homeassistant/components/recorder/migration.py
@@ -822,21 +822,27 @@ def _apply_update(  # noqa: C901
         # ALTER TABLE statistics_meta DROP COLUMN state_unit_of_measurement
         pass
     elif new_version == 31:
-        # Migrate all data in Events.time_fired to Events.time_fired_ts and
-        # wipe Events.time_fired
-        # Migrate all data in States.last_updated to States.last_updated_ts and
-        # wipe States.last_updated
-        # Migrate all data in States.last_changed to States.last_changed_ts and
-        # wipe States.last_changed
-        # TODO: implement
         # Once we require SQLite >= 3.35.5, we should drop the column:
         # ALTER TABLE events DROP COLUMN time_fired
         # ALTER TABLE states DROP COLUMN last_updated
         # ALTER TABLE states DROP COLUMN last_changed
+        _add_columns(session_maker, "events", ["time_fired_ts FLOAT"])
+        _add_columns(
+            session_maker, "states", ["last_updated_ts FLOAT", "last_changed_ts FLOAT"]
+        )
+
         _create_index(session_maker, "events", "ix_events_time_fired_ts")
         _create_index(session_maker, "events", "ix_events_event_type_time_fired_ts")
         _create_index(session_maker, "states", "ix_states_entity_id_last_updated_ts")
         _create_index(session_maker, "states", "ix_states_last_updated_ts")
+
+        # Migrate all data in Events.time_fired to Events.time_fired_ts and
+        # wipe Events.time_fired
+        # Migrate all data in States.last_updated to States.last_updated_ts and
+        # wipe States.last_updated
+        # Migrate all data in States.last_changed to States.last_changed_ts and
+        # wipe States.last_changed
+        # TODO: implement
         _drop_index(session_maker, "states", "ix_states_entity_id_last_updated")
         _drop_index(session_maker, "events", "ix_events_event_type_time_fired")
         _drop_index(session_maker, "states", "ix_states_last_updated")

From 5c12054f3bdc5f50914e4226c0a82a68dc2c20b6 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 31 Dec 2022 10:51:51 -1000
Subject: [PATCH 16/38] fix model tests

---
 .../components/recorder/db_schema.py          |  6 +++---
 tests/components/recorder/test_models.py      | 20 +++++++++----------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/homeassistant/components/recorder/db_schema.py b/homeassistant/components/recorder/db_schema.py
index a35a1ad996ba52..bd7c1a94e09242 100644
--- a/homeassistant/components/recorder/db_schema.py
+++ b/homeassistant/components/recorder/db_schema.py
@@ -365,11 +365,11 @@ def to_native(self, validate_entity_id: bool = True) -> State | None:
             return None
         if self.last_changed_ts is None or self.last_changed_ts == self.last_updated_ts:
             last_changed = last_updated = dt_util.utc_from_timestamp(
-                self.last_updated_ts
+                self.last_updated_ts or 0
             )
         else:
-            last_updated = dt_util.utc_from_timestamp(self.last_updated_ts)
-            last_changed = dt_util.utc_from_timestamp(self.last_changed_ts)
+            last_updated = dt_util.utc_from_timestamp(self.last_updated_ts or 0)
+            last_changed = dt_util.utc_from_timestamp(self.last_changed_ts or 0)
         return State(
             self.entity_id,
             self.state,
diff --git a/tests/components/recorder/test_models.py b/tests/components/recorder/test_models.py
index 81469ab1dabbb7..22484e0489ed49 100644
--- a/tests/components/recorder/test_models.py
+++ b/tests/components/recorder/test_models.py
@@ -81,8 +81,8 @@ def test_from_event_to_delete_state():
 
     assert db_state.entity_id == "sensor.temperature"
     assert db_state.state == ""
-    assert db_state.last_changed is None
-    assert db_state.last_updated == event.time_fired
+    assert db_state.last_changed_ts is None
+    assert db_state.last_updated_ts == event.time_fired.timestamp()
 
 
 def test_entity_ids():
@@ -272,8 +272,8 @@ async def test_lazy_state_handles_different_last_updated_and_last_changed(caplog
         entity_id="sensor.valid",
         state="off",
         shared_attrs='{"shared":true}',
-        last_updated=now,
-        last_changed=now - timedelta(seconds=60),
+        last_updated_ts=now.timestamp(),
+        last_changed_ts=(now - timedelta(seconds=60)).timestamp(),
     )
     lstate = LazyState(row, {})
     assert lstate.as_dict() == {
@@ -283,8 +283,8 @@ async def test_lazy_state_handles_different_last_updated_and_last_changed(caplog
         "last_updated": "2021-06-12T03:04:01.000323+00:00",
         "state": "off",
     }
-    assert lstate.last_updated == row.last_updated
-    assert lstate.last_changed == row.last_changed
+    assert lstate.last_updated.timestamp() == row.last_updated_ts
+    assert lstate.last_changed.timestamp() == row.last_changed_ts
     assert lstate.as_dict() == {
         "attributes": {"shared": True},
         "entity_id": "sensor.valid",
@@ -301,8 +301,8 @@ async def test_lazy_state_handles_same_last_updated_and_last_changed(caplog):
         entity_id="sensor.valid",
         state="off",
         shared_attrs='{"shared":true}',
-        last_updated=now,
-        last_changed=now,
+        last_updated_ts=now.timestamp(),
+        last_changed_ts=now.timestamp(),
     )
     lstate = LazyState(row, {})
     assert lstate.as_dict() == {
@@ -312,8 +312,8 @@ async def test_lazy_state_handles_same_last_updated_and_last_changed(caplog):
         "last_updated": "2021-06-12T03:04:01.000323+00:00",
         "state": "off",
     }
-    assert lstate.last_updated == row.last_updated
-    assert lstate.last_changed == row.last_changed
+    assert lstate.last_updated.timestamp() == row.last_updated_ts
+    assert lstate.last_changed.timestamp() == row.last_changed_ts
     assert lstate.as_dict() == {
         "attributes": {"shared": True},
         "entity_id": "sensor.valid",

From d9a1f5f1b0ad975c31d183011b5f66dbaf2bd1f8 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 31 Dec 2022 11:13:20 -1000
Subject: [PATCH 17/38] use the same as stats

---
 homeassistant/components/recorder/db_schema.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/homeassistant/components/recorder/db_schema.py b/homeassistant/components/recorder/db_schema.py
index bd7c1a94e09242..7c3bc830f327c2 100644
--- a/homeassistant/components/recorder/db_schema.py
+++ b/homeassistant/components/recorder/db_schema.py
@@ -123,7 +123,7 @@ def result_processor(self, dialect, coltype):  # type: ignore[no-untyped-def]
     .with_variant(postgresql.DOUBLE_PRECISION(), "postgresql")
 )
 
-TIMESTAMP_TYPE = Float(16)  # Covers up to at least 2250 AD
+TIMESTAMP_TYPE = DOUBLE_TYPE
 
 
 class JSONLiteral(JSON):  # type: ignore[misc]

From 919263c4303e2a3735cdaf2cd4eeea693e709d9c Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 31 Dec 2022 14:01:39 -1000
Subject: [PATCH 18/38] fix

---
 homeassistant/components/recorder/history.py |  4 +++-
 homeassistant/components/recorder/models.py  | 18 ++++++++++--------
 tests/components/recorder/test_history.py    |  2 +-
 3 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/homeassistant/components/recorder/history.py b/homeassistant/components/recorder/history.py
index 265ab68dce8d1e..044f7bfb6b5237 100644
--- a/homeassistant/components/recorder/history.py
+++ b/homeassistant/components/recorder/history.py
@@ -825,7 +825,9 @@ def _sorted_states_to_dict(
     """
     schema_version = _schema_version(hass)
     _process_timestamp: Callable[[datetime], float | str]
-    state_class: Callable[[Row, dict[str, dict[str, Any]], Any], Any]
+    state_class: Callable[
+        [Row, dict[str, dict[str, Any]], datetime | None], State | dict[str, Any]
+    ]
     if compressed_state_format:
         if schema_version >= 31:
             state_class = row_to_compressed_state
diff --git a/homeassistant/components/recorder/models.py b/homeassistant/components/recorder/models.py
index 59fc40df0e5a6b..c3000c183955ab 100644
--- a/homeassistant/components/recorder/models.py
+++ b/homeassistant/components/recorder/models.py
@@ -136,7 +136,7 @@ def __init__(  # pylint: disable=super-init-not-called
         self,
         row: Row,
         attr_cache: dict[str, dict[str, Any]],
-        start_time: datetime | None = None,
+        start_time: datetime | None,
     ) -> None:
         """Init the lazy state."""
         self._row = row
@@ -264,16 +264,18 @@ def __init__(  # pylint: disable=super-init-not-called
         self,
         row: Row,
         attr_cache: dict[str, dict[str, Any]],
-        start_time: float | None = None,
+        start_time: datetime | None,
     ) -> None:
         """Init the lazy state."""
         self._row = row
         self.entity_id: str = self._row.entity_id
         self.state = self._row.state or ""
         self._attributes: dict[str, Any] | None = None
-        self._last_updated_ts: float | None = self._row.last_updated_ts or start_time
+        self._last_updated_ts: float | None = self._row.last_updated_ts or (
+            dt_util.utc_to_timestamp(start_time) if start_time else None
+        )
         self._last_changed_ts: float | None = (
-            self._row.last_changed_ts or self._last_updated_ts or start_time
+            self._row.last_changed_ts or self._last_updated_ts
         )
         self._context: Context | None = None
         self.attr_cache = attr_cache
@@ -380,15 +382,15 @@ def decode_attributes_from_row(
 def row_to_compressed_state(
     row: Row,
     attr_cache: dict[str, dict[str, Any]],
-    start_time: float | None = None,
+    start_time: datetime | None,
 ) -> dict[str, Any]:
-    """Convert a database row to a compressed state after schema 31."""
+    """Convert a database row to a compressed state schema 31 and later."""
     comp_state = {
         COMPRESSED_STATE_STATE: row.state,
         COMPRESSED_STATE_ATTRIBUTES: decode_attributes_from_row(row, attr_cache),
     }
     if start_time:
-        comp_state[COMPRESSED_STATE_LAST_UPDATED] = start_time
+        comp_state[COMPRESSED_STATE_LAST_UPDATED] = dt_util.utc_to_timestamp(start_time)
     else:
         row_last_updated_ts: float = row.last_updated_ts
         comp_state[COMPRESSED_STATE_LAST_UPDATED] = row_last_updated_ts
@@ -402,7 +404,7 @@ def row_to_compressed_state(
 def row_to_compressed_state_pre_schema_31(
     row: Row,
     attr_cache: dict[str, dict[str, Any]],
-    start_time: datetime | None = None,
+    start_time: datetime | None,
 ) -> dict[str, Any]:
     """Convert a database row to a compressed state before schema 31."""
     comp_state = {
diff --git a/tests/components/recorder/test_history.py b/tests/components/recorder/test_history.py
index 48b5cd7243666c..adf70d87945ff1 100644
--- a/tests/components/recorder/test_history.py
+++ b/tests/components/recorder/test_history.py
@@ -51,7 +51,7 @@ def _get_states_with_session():
         with session_scope(hass=hass) as session:
             attr_cache = {}
             return [
-                klass(row, attr_cache)
+                klass(row, attr_cache, None)
                 for row in history._get_rows_with_session(
                     hass,
                     session,

From e4d4f8eaed042840086bd601cd0f708dc278b93e Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 31 Dec 2022 14:09:08 -1000
Subject: [PATCH 19/38] reduce

---
 homeassistant/components/recorder/history.py | 83 ++++++++------------
 1 file changed, 33 insertions(+), 50 deletions(-)

diff --git a/homeassistant/components/recorder/history.py b/homeassistant/components/recorder/history.py
index 044f7bfb6b5237..3ae84a0b0237e2 100644
--- a/homeassistant/components/recorder/history.py
+++ b/homeassistant/components/recorder/history.py
@@ -903,65 +903,48 @@ def _sorted_states_to_dict(
             prev_state = first_state.state
             ent_results.append(state_class(first_state, attr_cache, None))
 
-        if schema_version >= 31:
-            if compressed_state_format:
-                for row in group:
-                    # With minimal response we do not care about attribute
-                    # changes so we can filter out duplicate states
-                    if (state := row.state) == prev_state:
-                        continue
-
-                    ent_results.append(
-                        {
-                            attr_state: state,
-                            #
-                            # minimal_response only makes sense with last_updated == last_updated
-                            #
-                            # We use last_updated for for last_changed since its the same
-                            #
-                            attr_time: row.last_updated_ts,
-                        }
-                    )
-                    prev_state = state
-            else:
-                for row in group:
-                    # With minimal response we do not care about attribute
-                    # changes so we can filter out duplicate states
-                    if (state := row.state) == prev_state:
-                        continue
-
-                    ent_results.append(
-                        {
-                            attr_state: state,
-                            #
-                            # minimal_response only makes sense with last_updated == last_updated
-                            #
-                            # We use last_updated for for last_changed since its the same
-                            #
-                            attr_time: process_timestamp_to_utc_isoformat(
-                                dt_util.utc_from_timestamp(row.last_updated_ts)
-                            ),
-                        }
-                    )
-                    prev_state = state
+        #
+        # minimal_response only makes sense with last_updated == last_updated
+        #
+        # We use last_updated for for last_changed since its the same
+        #
+        # With minimal response we do not care about attribute
+        # changes so we can filter out duplicate states
+        if schema_version < 31:
+            for row in group:
+                if (state := row.state) == prev_state:
+                    continue
+                ent_results.append(
+                    {
+                        attr_state: state,
+                        attr_time: _process_timestamp(row.last_updated),
+                    }
+                )
+                prev_state = state
+            continue
 
+        if compressed_state_format:
+            for row in group:
+                if (state := row.state) == prev_state:
+                    continue
+                ent_results.append(
+                    {
+                        attr_state: state,
+                        attr_time: row.last_updated_ts,
+                    }
+                )
+                prev_state = state
             continue
 
         for row in group:
-            # With minimal response we do not care about attribute
-            # changes so we can filter out duplicate states
             if (state := row.state) == prev_state:
                 continue
-
             ent_results.append(
                 {
                     attr_state: state,
-                    #
-                    # minimal_response only makes sense with last_updated == last_updated
-                    #
-                    # We use last_updated for for last_changed since its the same
-                    #
-                    attr_time: _process_timestamp(row.last_updated),
+                    attr_time: process_timestamp_to_utc_isoformat(
+                        dt_util.utc_from_timestamp(row.last_updated_ts)
+                    ),
                 }
             )
             prev_state = state

From d127cfc332f97a2d043feca6804400ad6be79dcd Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 31 Dec 2022 14:22:50 -1000
Subject: [PATCH 20/38] add year 2038 test

---
 tests/components/recorder/test_history.py | 36 +++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/tests/components/recorder/test_history.py b/tests/components/recorder/test_history.py
index adf70d87945ff1..d679a9a934dce8 100644
--- a/tests/components/recorder/test_history.py
+++ b/tests/components/recorder/test_history.py
@@ -7,6 +7,7 @@
 import json
 from unittest.mock import patch, sentinel
 
+from freezegun import freeze_time
 import pytest
 from sqlalchemy import text
 
@@ -945,3 +946,38 @@ def test_state_changes_during_period_multiple_entities_single_test(hass_recorder
     hist = history.state_changes_during_period(hass, start, end, None)
     for entity_id, value in test_entites.items():
         hist[entity_id][0].state == value
+
+
+async def test_get_full_significant_states_past_year_2038(
+    async_setup_recorder_instance: SetupRecorderInstanceT,
+    hass: ha.HomeAssistant,
+):
+    """Test we can store times past year 2038."""
+    await async_setup_recorder_instance(hass, {})
+    past_2038_time = dt_util.parse_datetime("2039-01-19 03:14:07.555000-00:00")
+
+    with freeze_time(past_2038_time):
+        hass.states.async_set("sensor.one", "on", {"attr": "original"})
+        state0 = hass.states.get("sensor.one")
+        await hass.async_block_till_done()
+        hass.states.async_set("sensor.one", "on", {"attr": "new"})
+        state1 = hass.states.get("sensor.one")
+        await async_wait_recording_done(hass)
+
+        def _get_entries():
+            with session_scope(hass=hass) as session:
+                return history.get_full_significant_states_with_session(
+                    hass,
+                    session,
+                    past_2038_time - timedelta(days=365),
+                    past_2038_time + timedelta(days=365),
+                    entity_ids=["sensor.one"],
+                    significant_changes_only=False,
+                )
+
+        states = await recorder.get_instance(hass).async_add_executor_job(_get_entries)
+        sensor_one_states: list[State] = states["sensor.one"]
+        assert sensor_one_states[0] == state0
+        assert sensor_one_states[1] == state1
+        assert sensor_one_states[0].last_changed == past_2038_time
+        assert sensor_one_states[0].last_updated == past_2038_time

From bf1ee08437719db1c2c3fd7dd2c7115748f5872b Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 31 Dec 2022 14:23:06 -1000
Subject: [PATCH 21/38] add year 2038 test

---
 tests/components/recorder/test_history.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/components/recorder/test_history.py b/tests/components/recorder/test_history.py
index d679a9a934dce8..014ef1bd5905f5 100644
--- a/tests/components/recorder/test_history.py
+++ b/tests/components/recorder/test_history.py
@@ -954,7 +954,7 @@ async def test_get_full_significant_states_past_year_2038(
 ):
     """Test we can store times past year 2038."""
     await async_setup_recorder_instance(hass, {})
-    past_2038_time = dt_util.parse_datetime("2039-01-19 03:14:07.555000-00:00")
+    past_2038_time = dt_util.parse_datetime("2039-01-19 03:14:07.555555-00:00")
 
     with freeze_time(past_2038_time):
         hass.states.async_set("sensor.one", "on", {"attr": "original"})

From d5925ae031f401da224da231c9d2ad253b5b8570 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 31 Dec 2022 15:10:50 -1000
Subject: [PATCH 22/38] implement migration

---
 .../components/recorder/db_schema.py          |  2 +-
 .../components/recorder/migration.py          | 70 ++++++++++++++++---
 tests/components/recorder/test_models.py      |  8 +--
 3 files changed, 66 insertions(+), 14 deletions(-)

diff --git a/homeassistant/components/recorder/db_schema.py b/homeassistant/components/recorder/db_schema.py
index 7c3bc830f327c2..0aa1f163d3d265 100644
--- a/homeassistant/components/recorder/db_schema.py
+++ b/homeassistant/components/recorder/db_schema.py
@@ -54,7 +54,7 @@
 # pylint: disable=invalid-name
 Base = declarative_base()
 
-SCHEMA_VERSION = 31
+SCHEMA_VERSION = 32
 
 _StatisticsBaseSelfT = TypeVar("_StatisticsBaseSelfT", bound="StatisticsBase")
 
diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py
index 8fd8c65b7094b1..5cd02784ac992e 100644
--- a/homeassistant/components/recorder/migration.py
+++ b/homeassistant/components/recorder/migration.py
@@ -830,19 +830,17 @@ def _apply_update(  # noqa: C901
         _add_columns(
             session_maker, "states", ["last_updated_ts FLOAT", "last_changed_ts FLOAT"]
         )
-
         _create_index(session_maker, "events", "ix_events_time_fired_ts")
         _create_index(session_maker, "events", "ix_events_event_type_time_fired_ts")
         _create_index(session_maker, "states", "ix_states_entity_id_last_updated_ts")
         _create_index(session_maker, "states", "ix_states_last_updated_ts")
-
-        # Migrate all data in Events.time_fired to Events.time_fired_ts and
-        # wipe Events.time_fired
-        # Migrate all data in States.last_updated to States.last_updated_ts and
-        # wipe States.last_updated
-        # Migrate all data in States.last_changed to States.last_changed_ts and
-        # wipe States.last_changed
-        # TODO: implement
+        with session_scope(session=session_maker()) as session:
+            _migrate_columns_to_timestamp(hass, session, engine)
+    elif new_version == 32:
+        # Migration is done in two steps to ensure we can start using
+        # the new columns before we wipe the old ones.
+        with session_scope(session=session_maker()) as session:
+            _wipe_old_string_time_columns(hass, session, engine)
         _drop_index(session_maker, "states", "ix_states_entity_id_last_updated")
         _drop_index(session_maker, "events", "ix_events_event_type_time_fired")
         _drop_index(session_maker, "states", "ix_states_last_updated")
@@ -851,6 +849,60 @@ def _apply_update(  # noqa: C901
         raise ValueError(f"No schema migration defined for version {new_version}")
 
 
+def _wipe_old_string_time_columns(
+    hass: HomeAssistant, session: Session, engine: Engine
+) -> None:
+    """Wipe old string time columns."""
+    # Wipe Events.time_fired
+    # Wipe States.last_updated
+    # Wipe States.last_changed
+    connection = session.connection()
+    connection.execute(text("UPDATE events set time_fired=NULL;"))
+    connection.execute(text("UPDATE states set last_updated=NULL, last_changed=NULL;"))
+
+
+def _migrate_columns_to_timestamp(
+    hass: HomeAssistant, session: Session, engine: Engine
+) -> None:
+    """Migrate columns to use timestamp."""
+    # Migrate all data in Events.time_fired to Events.time_fired_ts and wipe Events.time_fired
+    # Migrate all data in States.last_updated to States.last_updated_ts and wipe States.last_updated
+    # Migrate all data in States.last_changed to States.last_changed_ts and wipe States.last_changed
+    connection = session.connection()
+    if engine.dialect.name == SupportedDialect.SQLITE:
+        connection.execute(
+            text(
+                'UPDATE events set time_fired_ts=strftime("%s",time_fired) + cast(substr(time_fired,-7) AS FLOAT);'
+            )
+        )
+        connection.execute(
+            text(
+                'UPDATE states set last_updated_ts=strftime("%s",last_updated) + cast(substr(last_updated,-7) AS FLOAT), '
+                'last_changed_ts=strftime("%s",last_changed) + cast(substr(last_changed,-7) AS FLOAT);'
+            )
+        )
+    elif engine.dialect.name == SupportedDialect.MYSQL:
+        connection.execute(
+            text("UPDATE events set time_fired_ts=UNIX_TIMESTAMP(time_fired);")
+        )
+        connection.execute(
+            text(
+                "UPDATE states set last_updated_ts=UNIX_TIMESTAMP(last_updated), "
+                "last_changed_ts=UNIX_TIMESTAMP(last_changed);"
+            )
+        )
+    elif engine.dialect.name == SupportedDialect.POSTGRESQL:
+        connection.execute(
+            text("UPDATE events set time_fired_ts=EXTRACT(EPOCH FROM time_fired);")
+        )
+        connection.execute(
+            text(
+                "UPDATE states set last_updated_ts=EXTRACT(EPOCH FROM last_updated), "
+                "last_changed_ts=EXTRACT(EPOCH FROM last_changed);"
+            )
+        )
+
+
 def _initialize_database(session: Session) -> bool:
     """Initialize a new database, or a database created before introducing schema changes.
 
diff --git a/tests/components/recorder/test_models.py b/tests/components/recorder/test_models.py
index 22484e0489ed49..c8e1cc3a25b477 100644
--- a/tests/components/recorder/test_models.py
+++ b/tests/components/recorder/test_models.py
@@ -251,7 +251,7 @@ async def test_lazy_state_handles_include_json(caplog):
         entity_id="sensor.invalid",
         shared_attrs="{INVALID_JSON}",
     )
-    assert LazyState(row, {}).attributes == {}
+    assert LazyState(row, {}, None).attributes == {}
     assert "Error converting row to state attributes" in caplog.text
 
 
@@ -262,7 +262,7 @@ async def test_lazy_state_prefers_shared_attrs_over_attrs(caplog):
         shared_attrs='{"shared":true}',
         attributes='{"shared":false}',
     )
-    assert LazyState(row, {}).attributes == {"shared": True}
+    assert LazyState(row, {}, None).attributes == {"shared": True}
 
 
 async def test_lazy_state_handles_different_last_updated_and_last_changed(caplog):
@@ -275,7 +275,7 @@ async def test_lazy_state_handles_different_last_updated_and_last_changed(caplog
         last_updated_ts=now.timestamp(),
         last_changed_ts=(now - timedelta(seconds=60)).timestamp(),
     )
-    lstate = LazyState(row, {})
+    lstate = LazyState(row, {}, None)
     assert lstate.as_dict() == {
         "attributes": {"shared": True},
         "entity_id": "sensor.valid",
@@ -304,7 +304,7 @@ async def test_lazy_state_handles_same_last_updated_and_last_changed(caplog):
         last_updated_ts=now.timestamp(),
         last_changed_ts=now.timestamp(),
     )
-    lstate = LazyState(row, {})
+    lstate = LazyState(row, {}, None)
     assert lstate.as_dict() == {
         "attributes": {"shared": True},
         "entity_id": "sensor.valid",

From 31d9cc533a08361c112ddb2c981803b6c2d7ccca Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 31 Dec 2022 17:03:16 -1000
Subject: [PATCH 23/38] add time migration test

---
 tests/components/recorder/db_schema_30.py     | 674 ++++++++++++++++++
 .../components/recorder/test_v32_migration.py | 135 ++++
 2 files changed, 809 insertions(+)
 create mode 100644 tests/components/recorder/db_schema_30.py
 create mode 100644 tests/components/recorder/test_v32_migration.py

diff --git a/tests/components/recorder/db_schema_30.py b/tests/components/recorder/db_schema_30.py
new file mode 100644
index 00000000000000..8854cd33a619ff
--- /dev/null
+++ b/tests/components/recorder/db_schema_30.py
@@ -0,0 +1,674 @@
+"""Models for SQLAlchemy.
+
+This file contains the model definitions for schema version 30.
+It is used to test the schema migration logic.
+"""
+from __future__ import annotations
+
+from collections.abc import Callable
+from datetime import datetime, timedelta
+import logging
+from typing import Any, TypedDict, TypeVar, cast, overload
+
+import ciso8601
+from fnvhash import fnv1a_32
+from sqlalchemy import (
+    JSON,
+    BigInteger,
+    Boolean,
+    Column,
+    DateTime,
+    Float,
+    ForeignKey,
+    Identity,
+    Index,
+    Integer,
+    SmallInteger,
+    String,
+    Text,
+    distinct,
+    type_coerce,
+)
+from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import aliased, declarative_base, relationship
+from sqlalchemy.orm.session import Session
+
+from homeassistant.const import (
+    ATTR_ATTRIBUTION,
+    ATTR_RESTORED,
+    ATTR_SUPPORTED_FEATURES,
+    MAX_LENGTH_EVENT_CONTEXT_ID,
+    MAX_LENGTH_EVENT_EVENT_TYPE,
+    MAX_LENGTH_EVENT_ORIGIN,
+    MAX_LENGTH_STATE_ENTITY_ID,
+    MAX_LENGTH_STATE_STATE,
+)
+from homeassistant.core import Context, Event, EventOrigin, State, split_entity_id
+from homeassistant.helpers.json import (
+    JSON_DECODE_EXCEPTIONS,
+    JSON_DUMP,
+    json_bytes,
+    json_loads,
+)
+import homeassistant.util.dt as dt_util
+
+ALL_DOMAIN_EXCLUDE_ATTRS = {ATTR_ATTRIBUTION, ATTR_RESTORED, ATTR_SUPPORTED_FEATURES}
+
+# SQLAlchemy Schema
+# pylint: disable=invalid-name
+Base = declarative_base()
+
+SCHEMA_VERSION = 30
+
+_StatisticsBaseSelfT = TypeVar("_StatisticsBaseSelfT", bound="StatisticsBase")
+
+_LOGGER = logging.getLogger(__name__)
+
+TABLE_EVENTS = "events"
+TABLE_EVENT_DATA = "event_data"
+TABLE_STATES = "states"
+TABLE_STATE_ATTRIBUTES = "state_attributes"
+TABLE_RECORDER_RUNS = "recorder_runs"
+TABLE_SCHEMA_CHANGES = "schema_changes"
+TABLE_STATISTICS = "statistics"
+TABLE_STATISTICS_META = "statistics_meta"
+TABLE_STATISTICS_RUNS = "statistics_runs"
+TABLE_STATISTICS_SHORT_TERM = "statistics_short_term"
+
+ALL_TABLES = [
+    TABLE_STATES,
+    TABLE_STATE_ATTRIBUTES,
+    TABLE_EVENTS,
+    TABLE_EVENT_DATA,
+    TABLE_RECORDER_RUNS,
+    TABLE_SCHEMA_CHANGES,
+    TABLE_STATISTICS,
+    TABLE_STATISTICS_META,
+    TABLE_STATISTICS_RUNS,
+    TABLE_STATISTICS_SHORT_TERM,
+]
+
+TABLES_TO_CHECK = [
+    TABLE_STATES,
+    TABLE_EVENTS,
+    TABLE_RECORDER_RUNS,
+    TABLE_SCHEMA_CHANGES,
+]
+
+LAST_UPDATED_INDEX = "ix_states_last_updated"
+ENTITY_ID_LAST_UPDATED_INDEX = "ix_states_entity_id_last_updated"
+EVENTS_CONTEXT_ID_INDEX = "ix_events_context_id"
+STATES_CONTEXT_ID_INDEX = "ix_states_context_id"
+
+
+class FAST_PYSQLITE_DATETIME(sqlite.DATETIME):  # type: ignore[misc]
+    """Use ciso8601 to parse datetimes instead of sqlalchemy built-in regex."""
+
+    def result_processor(self, dialect, coltype):  # type: ignore[no-untyped-def]
+        """Offload the datetime parsing to ciso8601."""
+        return lambda value: None if value is None else ciso8601.parse_datetime(value)
+
+
+JSON_VARIANT_CAST = Text().with_variant(
+    postgresql.JSON(none_as_null=True), "postgresql"
+)
+JSONB_VARIANT_CAST = Text().with_variant(
+    postgresql.JSONB(none_as_null=True), "postgresql"
+)
+DATETIME_TYPE = (
+    DateTime(timezone=True)
+    .with_variant(mysql.DATETIME(timezone=True, fsp=6), "mysql")
+    .with_variant(FAST_PYSQLITE_DATETIME(), "sqlite")
+)
+DOUBLE_TYPE = (
+    Float()
+    .with_variant(mysql.DOUBLE(asdecimal=False), "mysql")
+    .with_variant(oracle.DOUBLE_PRECISION(), "oracle")
+    .with_variant(postgresql.DOUBLE_PRECISION(), "postgresql")
+)
+
+TIMESTAMP_TYPE = DOUBLE_TYPE
+
+
+class UnsupportedDialect(Exception):
+    """The dialect or its version is not supported."""
+
+
+class StatisticResult(TypedDict):
+    """Statistic result data class.
+
+    Allows multiple datapoints for the same statistic_id.
+    """
+
+    meta: StatisticMetaData
+    stat: StatisticData
+
+
+class StatisticDataBase(TypedDict):
+    """Mandatory fields for statistic data class."""
+
+    start: datetime
+
+
+class StatisticData(StatisticDataBase, total=False):
+    """Statistic data class."""
+
+    mean: float
+    min: float
+    max: float
+    last_reset: datetime | None
+    state: float
+    sum: float
+
+
+class StatisticMetaData(TypedDict):
+    """Statistic meta data class."""
+
+    has_mean: bool
+    has_sum: bool
+    name: str | None
+    source: str
+    statistic_id: str
+    unit_of_measurement: str | None
+
+
+class JSONLiteral(JSON):  # type: ignore[misc]
+    """Teach SA how to literalize json."""
+
+    def literal_processor(self, dialect: str) -> Callable[[Any], str]:
+        """Processor to convert a value to JSON."""
+
+        def process(value: Any) -> str:
+            """Dump json."""
+            return JSON_DUMP(value)
+
+        return process
+
+
+EVENT_ORIGIN_ORDER = [EventOrigin.local, EventOrigin.remote]
+EVENT_ORIGIN_TO_IDX = {origin: idx for idx, origin in enumerate(EVENT_ORIGIN_ORDER)}
+
+
+class Events(Base):  # type: ignore[misc,valid-type]
+    """Event history data."""
+
+    __table_args__ = (
+        # Used for fetching events at a specific time
+        # see logbook
+        Index("ix_events_event_type_time_fired", "event_type", "time_fired"),
+        {"mysql_default_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"},
+    )
+    __tablename__ = TABLE_EVENTS
+    event_id = Column(Integer, Identity(), primary_key=True)
+    event_type = Column(String(MAX_LENGTH_EVENT_EVENT_TYPE))
+    event_data = Column(Text().with_variant(mysql.LONGTEXT, "mysql"))
+    origin = Column(String(MAX_LENGTH_EVENT_ORIGIN))  # no longer used for new rows
+    origin_idx = Column(SmallInteger)
+    time_fired = Column(DATETIME_TYPE, index=True)
+    context_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID), index=True)
+    context_user_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID))
+    context_parent_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID))
+    data_id = Column(Integer, ForeignKey("event_data.data_id"), index=True)
+    event_data_rel = relationship("EventData")
+
+    def __repr__(self) -> str:
+        """Return string representation of instance for debugging."""
+        return (
+            "<recorder.Events("
+            f"id={self.event_id}, type='{self.event_type}', "
+            f"origin_idx='{self.origin_idx}', time_fired='{self.time_fired}'"
+            f", data_id={self.data_id})>"
+        )
+
+    @staticmethod
+    def from_event(event: Event) -> Events:
+        """Create an event database object from a native event."""
+        return Events(
+            event_type=event.event_type,
+            event_data=None,
+            origin_idx=EVENT_ORIGIN_TO_IDX.get(event.origin),
+            time_fired=event.time_fired,
+            context_id=event.context.id,
+            context_user_id=event.context.user_id,
+            context_parent_id=event.context.parent_id,
+        )
+
+    def to_native(self, validate_entity_id: bool = True) -> Event | None:
+        """Convert to a native HA Event."""
+        context = Context(
+            id=self.context_id,
+            user_id=self.context_user_id,
+            parent_id=self.context_parent_id,
+        )
+        try:
+            return Event(
+                self.event_type,
+                json_loads(self.event_data) if self.event_data else {},
+                EventOrigin(self.origin)
+                if self.origin
+                else EVENT_ORIGIN_ORDER[self.origin_idx],
+                process_timestamp(self.time_fired),
+                context=context,
+            )
+        except JSON_DECODE_EXCEPTIONS:
+            # When json_loads fails
+            _LOGGER.exception("Error converting to event: %s", self)
+            return None
+
+
+class EventData(Base):  # type: ignore[misc,valid-type]
+    """Event data history."""
+
+    __table_args__ = (
+        {"mysql_default_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"},
+    )
+    __tablename__ = TABLE_EVENT_DATA
+    data_id = Column(Integer, Identity(), primary_key=True)
+    hash = Column(BigInteger, index=True)
+    # Note that this is not named attributes to avoid confusion with the states table
+    shared_data = Column(Text().with_variant(mysql.LONGTEXT, "mysql"))
+
+    def __repr__(self) -> str:
+        """Return string representation of instance for debugging."""
+        return (
+            "<recorder.EventData("
+            f"id={self.data_id}, hash='{self.hash}', data='{self.shared_data}'"
+            ")>"
+        )
+
+    @staticmethod
+    def from_event(event: Event) -> EventData:
+        """Create object from an event."""
+        shared_data = json_bytes(event.data)
+        return EventData(
+            shared_data=shared_data.decode("utf-8"),
+            hash=EventData.hash_shared_data_bytes(shared_data),
+        )
+
+    @staticmethod
+    def shared_data_bytes_from_event(event: Event) -> bytes:
+        """Create shared_data from an event."""
+        return json_bytes(event.data)
+
+    @staticmethod
+    def hash_shared_data_bytes(shared_data_bytes: bytes) -> int:
+        """Return the hash of json encoded shared data."""
+        return cast(int, fnv1a_32(shared_data_bytes))
+
+    def to_native(self) -> dict[str, Any]:
+        """Convert to an HA state object."""
+        try:
+            return cast(dict[str, Any], json_loads(self.shared_data))
+        except JSON_DECODE_EXCEPTIONS:
+            _LOGGER.exception("Error converting row to event data: %s", self)
+            return {}
+
+
+class States(Base):  # type: ignore[misc,valid-type]
+    """State change history."""
+
+    __table_args__ = (
+        # Used for fetching the state of entities at a specific time
+        # (get_states in history.py)
+        Index(ENTITY_ID_LAST_UPDATED_INDEX, "entity_id", "last_updated"),
+        {"mysql_default_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"},
+    )
+    __tablename__ = TABLE_STATES
+    state_id = Column(Integer, Identity(), primary_key=True)
+    entity_id = Column(String(MAX_LENGTH_STATE_ENTITY_ID))
+    state = Column(String(MAX_LENGTH_STATE_STATE))
+    attributes = Column(
+        Text().with_variant(mysql.LONGTEXT, "mysql")
+    )  # no longer used for new rows
+    event_id = Column(  # no longer used for new rows
+        Integer, ForeignKey("events.event_id", ondelete="CASCADE"), index=True
+    )
+    last_changed = Column(DATETIME_TYPE)
+    last_updated = Column(DATETIME_TYPE, default=dt_util.utcnow, index=True)
+    old_state_id = Column(Integer, ForeignKey("states.state_id"), index=True)
+    attributes_id = Column(
+        Integer, ForeignKey("state_attributes.attributes_id"), index=True
+    )
+    context_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID), index=True)
+    context_user_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID))
+    context_parent_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID))
+    origin_idx = Column(SmallInteger)  # 0 is local, 1 is remote
+    old_state = relationship("States", remote_side=[state_id])
+    state_attributes = relationship("StateAttributes")
+
+    def __repr__(self) -> str:
+        """Return string representation of instance for debugging."""
+        return (
+            f"<recorder.States(id={self.state_id}, entity_id='{self.entity_id}',"
+            f" state='{self.state}', event_id='{self.event_id}',"
+            f" last_updated='{self.last_updated.isoformat(sep=' ', timespec='seconds')}',"
+            f" old_state_id={self.old_state_id}, attributes_id={self.attributes_id})>"
+        )
+
+    @staticmethod
+    def from_event(event: Event) -> States:
+        """Create object from a state_changed event."""
+        entity_id = event.data["entity_id"]
+        state: State | None = event.data.get("new_state")
+        dbstate = States(
+            entity_id=entity_id,
+            attributes=None,
+            context_id=event.context.id,
+            context_user_id=event.context.user_id,
+            context_parent_id=event.context.parent_id,
+            origin_idx=EVENT_ORIGIN_TO_IDX.get(event.origin),
+        )
+
+        # None state means the state was removed from the state machine
+        if state is None:
+            dbstate.state = ""
+            dbstate.last_updated = event.time_fired
+            dbstate.last_changed = None
+            return dbstate
+
+        dbstate.state = state.state
+        dbstate.last_updated = state.last_updated
+        if state.last_updated == state.last_changed:
+            dbstate.last_changed = None
+        else:
+            dbstate.last_changed = state.last_changed
+
+        return dbstate
+
+    def to_native(self, validate_entity_id: bool = True) -> State | None:
+        """Convert to an HA state object."""
+        context = Context(
+            id=self.context_id,
+            user_id=self.context_user_id,
+            parent_id=self.context_parent_id,
+        )
+        try:
+            attrs = json_loads(self.attributes) if self.attributes else {}
+        except JSON_DECODE_EXCEPTIONS:
+            # When json_loads fails
+            _LOGGER.exception("Error converting row to state: %s", self)
+            return None
+        if self.last_changed is None or self.last_changed == self.last_updated:
+            last_changed = last_updated = process_timestamp(self.last_updated)
+        else:
+            last_updated = process_timestamp(self.last_updated)
+            last_changed = process_timestamp(self.last_changed)
+        return State(
+            self.entity_id,
+            self.state,
+            # Join the state_attributes table on attributes_id to get the attributes
+            # for newer states
+            attrs,
+            last_changed,
+            last_updated,
+            context=context,
+            validate_entity_id=validate_entity_id,
+        )
+
+
+class StateAttributes(Base):  # type: ignore[misc,valid-type]
+    """State attribute change history."""
+
+    __table_args__ = (
+        {"mysql_default_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"},
+    )
+    __tablename__ = TABLE_STATE_ATTRIBUTES
+    attributes_id = Column(Integer, Identity(), primary_key=True)
+    hash = Column(BigInteger, index=True)
+    # Note that this is not named attributes to avoid confusion with the states table
+    shared_attrs = Column(Text().with_variant(mysql.LONGTEXT, "mysql"))
+
+    def __repr__(self) -> str:
+        """Return string representation of instance for debugging."""
+        return (
+            f"<recorder.StateAttributes(id={self.attributes_id}, hash='{self.hash}',"
+            f" attributes='{self.shared_attrs}')>"
+        )
+
+    @staticmethod
+    def from_event(event: Event) -> StateAttributes:
+        """Create object from a state_changed event."""
+        state: State | None = event.data.get("new_state")
+        # None state means the state was removed from the state machine
+        attr_bytes = b"{}" if state is None else json_bytes(state.attributes)
+        dbstate = StateAttributes(shared_attrs=attr_bytes.decode("utf-8"))
+        dbstate.hash = StateAttributes.hash_shared_attrs_bytes(attr_bytes)
+        return dbstate
+
+    @staticmethod
+    def shared_attrs_bytes_from_event(
+        event: Event, exclude_attrs_by_domain: dict[str, set[str]]
+    ) -> bytes:
+        """Create shared_attrs from a state_changed event."""
+        state: State | None = event.data.get("new_state")
+        # None state means the state was removed from the state machine
+        if state is None:
+            return b"{}"
+        domain = split_entity_id(state.entity_id)[0]
+        exclude_attrs = (
+            exclude_attrs_by_domain.get(domain, set()) | ALL_DOMAIN_EXCLUDE_ATTRS
+        )
+        return json_bytes(
+            {k: v for k, v in state.attributes.items() if k not in exclude_attrs}
+        )
+
+    @staticmethod
+    def hash_shared_attrs_bytes(shared_attrs_bytes: bytes) -> int:
+        """Return the hash of json encoded shared attributes."""
+        return cast(int, fnv1a_32(shared_attrs_bytes))
+
+    def to_native(self) -> dict[str, Any]:
+        """Convert to an HA state object."""
+        try:
+            return cast(dict[str, Any], json_loads(self.shared_attrs))
+        except JSON_DECODE_EXCEPTIONS:
+            # When json_loads fails
+            _LOGGER.exception("Error converting row to state attributes: %s", self)
+            return {}
+
+
+class StatisticsBase:
+    """Statistics base class."""
+
+    id = Column(Integer, Identity(), primary_key=True)
+    created = Column(DATETIME_TYPE, default=dt_util.utcnow)
+
+    @declared_attr  # type: ignore[misc]
+    def metadata_id(self) -> Column:
+        """Define the metadata_id column for sub classes."""
+        return Column(
+            Integer,
+            ForeignKey(f"{TABLE_STATISTICS_META}.id", ondelete="CASCADE"),
+            index=True,
+        )
+
+    start = Column(DATETIME_TYPE, index=True)
+    mean = Column(DOUBLE_TYPE)
+    min = Column(DOUBLE_TYPE)
+    max = Column(DOUBLE_TYPE)
+    last_reset = Column(DATETIME_TYPE)
+    state = Column(DOUBLE_TYPE)
+    sum = Column(DOUBLE_TYPE)
+
+    @classmethod
+    def from_stats(
+        cls: type[_StatisticsBaseSelfT], metadata_id: int, stats: StatisticData
+    ) -> _StatisticsBaseSelfT:
+        """Create object from a statistics."""
+        return cls(  # type: ignore[call-arg,misc]
+            metadata_id=metadata_id,
+            **stats,
+        )
+
+
+class Statistics(Base, StatisticsBase):  # type: ignore[misc,valid-type]
+    """Long term statistics."""
+
+    duration = timedelta(hours=1)
+
+    __table_args__ = (
+        # Used for fetching statistics for a certain entity at a specific time
+        Index("ix_statistics_statistic_id_start", "metadata_id", "start", unique=True),
+    )
+    __tablename__ = TABLE_STATISTICS
+
+
+class StatisticsShortTerm(Base, StatisticsBase):  # type: ignore[misc,valid-type]
+    """Short term statistics."""
+
+    duration = timedelta(minutes=5)
+
+    __table_args__ = (
+        # Used for fetching statistics for a certain entity at a specific time
+        Index(
+            "ix_statistics_short_term_statistic_id_start",
+            "metadata_id",
+            "start",
+            unique=True,
+        ),
+    )
+    __tablename__ = TABLE_STATISTICS_SHORT_TERM
+
+
+class StatisticsMeta(Base):  # type: ignore[misc,valid-type]
+    """Statistics meta data."""
+
+    __table_args__ = (
+        {"mysql_default_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"},
+    )
+    __tablename__ = TABLE_STATISTICS_META
+    id = Column(Integer, Identity(), primary_key=True)
+    statistic_id = Column(String(255), index=True, unique=True)
+    source = Column(String(32))
+    unit_of_measurement = Column(String(255))
+    has_mean = Column(Boolean)
+    has_sum = Column(Boolean)
+    name = Column(String(255))
+
+    @staticmethod
+    def from_meta(meta: StatisticMetaData) -> StatisticsMeta:
+        """Create object from meta data."""
+        return StatisticsMeta(**meta)
+
+
+class RecorderRuns(Base):  # type: ignore[misc,valid-type]
+    """Representation of recorder run."""
+
+    __table_args__ = (Index("ix_recorder_runs_start_end", "start", "end"),)
+    __tablename__ = TABLE_RECORDER_RUNS
+    run_id = Column(Integer, Identity(), primary_key=True)
+    start = Column(DATETIME_TYPE, default=dt_util.utcnow)
+    end = Column(DATETIME_TYPE)
+    closed_incorrect = Column(Boolean, default=False)
+    created = Column(DATETIME_TYPE, default=dt_util.utcnow)
+
+    def __repr__(self) -> str:
+        """Return string representation of instance for debugging."""
+        end = (
+            f"'{self.end.isoformat(sep=' ', timespec='seconds')}'" if self.end else None
+        )
+        return (
+            f"<recorder.RecorderRuns(id={self.run_id},"
+            f" start='{self.start.isoformat(sep=' ', timespec='seconds')}', end={end},"
+            f" closed_incorrect={self.closed_incorrect},"
+            f" created='{self.created.isoformat(sep=' ', timespec='seconds')}')>"
+        )
+
+    def entity_ids(self, point_in_time: datetime | None = None) -> list[str]:
+        """Return the entity ids that existed in this run.
+
+        Specify point_in_time if you want to know which existed at that point
+        in time inside the run.
+        """
+        session = Session.object_session(self)
+
+        assert session is not None, "RecorderRuns need to be persisted"
+
+        query = session.query(distinct(States.entity_id)).filter(
+            States.last_updated >= self.start
+        )
+
+        if point_in_time is not None:
+            query = query.filter(States.last_updated < point_in_time)
+        elif self.end is not None:
+            query = query.filter(States.last_updated < self.end)
+
+        return [row[0] for row in query]
+
+    def to_native(self, validate_entity_id: bool = True) -> RecorderRuns:
+        """Return self, native format is this model."""
+        return self
+
+
+class SchemaChanges(Base):  # type: ignore[misc,valid-type]
+    """Representation of schema version changes."""
+
+    __tablename__ = TABLE_SCHEMA_CHANGES
+    change_id = Column(Integer, Identity(), primary_key=True)
+    schema_version = Column(Integer)
+    changed = Column(DATETIME_TYPE, default=dt_util.utcnow)
+
+    def __repr__(self) -> str:
+        """Return string representation of instance for debugging."""
+        return (
+            "<recorder.SchemaChanges("
+            f"id={self.change_id}, schema_version={self.schema_version}, "
+            f"changed='{self.changed.isoformat(sep=' ', timespec='seconds')}'"
+            ")>"
+        )
+
+
+class StatisticsRuns(Base):  # type: ignore[misc,valid-type]
+    """Representation of statistics run."""
+
+    __tablename__ = TABLE_STATISTICS_RUNS
+    run_id = Column(Integer, Identity(), primary_key=True)
+    start = Column(DATETIME_TYPE, index=True)
+
+    def __repr__(self) -> str:
+        """Return string representation of instance for debugging."""
+        return (
+            f"<recorder.StatisticsRuns(id={self.run_id},"
+            f" start='{self.start.isoformat(sep=' ', timespec='seconds')}', )>"
+        )
+
+
+EVENT_DATA_JSON = type_coerce(
+    EventData.shared_data.cast(JSONB_VARIANT_CAST), JSONLiteral(none_as_null=True)
+)
+OLD_FORMAT_EVENT_DATA_JSON = type_coerce(
+    Events.event_data.cast(JSONB_VARIANT_CAST), JSONLiteral(none_as_null=True)
+)
+
+SHARED_ATTRS_JSON = type_coerce(
+    StateAttributes.shared_attrs.cast(JSON_VARIANT_CAST), JSON(none_as_null=True)
+)
+OLD_FORMAT_ATTRS_JSON = type_coerce(
+    States.attributes.cast(JSON_VARIANT_CAST), JSON(none_as_null=True)
+)
+
+ENTITY_ID_IN_EVENT: Column = EVENT_DATA_JSON["entity_id"]
+OLD_ENTITY_ID_IN_EVENT: Column = OLD_FORMAT_EVENT_DATA_JSON["entity_id"]
+DEVICE_ID_IN_EVENT: Column = EVENT_DATA_JSON["device_id"]
+OLD_STATE = aliased(States, name="old_state")
+
+
+@overload
+def process_timestamp(ts: None) -> None:
+    ...
+
+
+@overload
+def process_timestamp(ts: datetime) -> datetime:
+    ...
+
+
+def process_timestamp(ts: datetime | None) -> datetime | None:
+    """Process a timestamp into datetime object."""
+    if ts is None:
+        return None
+    if ts.tzinfo is None:
+        return ts.replace(tzinfo=dt_util.UTC)
+
+    return dt_util.as_utc(ts)
diff --git a/tests/components/recorder/test_v32_migration.py b/tests/components/recorder/test_v32_migration.py
new file mode 100644
index 00000000000000..c7333c6b7ca5fa
--- /dev/null
+++ b/tests/components/recorder/test_v32_migration.py
@@ -0,0 +1,135 @@
+"""The tests for recorder platform migrating data from v30."""
+# pylint: disable=protected-access,invalid-name
+from datetime import timedelta
+import importlib
+import sys
+from unittest.mock import patch
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from homeassistant.components import recorder
+from homeassistant.components.recorder import SQLITE_URL_PREFIX, core, statistics
+from homeassistant.components.recorder.util import session_scope
+from homeassistant.core import EVENT_STATE_CHANGED, Event, EventOrigin, State
+from homeassistant.helpers import recorder as recorder_helper
+from homeassistant.setup import setup_component
+import homeassistant.util.dt as dt_util
+
+from .common import wait_recording_done
+
+from tests.common import get_test_home_assistant
+
+ORIG_TZ = dt_util.DEFAULT_TIME_ZONE
+
+CREATE_ENGINE_TARGET = "homeassistant.components.recorder.core.create_engine"
+SCHEMA_MODULE = "tests.components.recorder.db_schema_30"
+
+
+def _create_engine_test(*args, **kwargs):
+    """Test version of create_engine that initializes with old schema.
+
+    This simulates an existing db with the old schema.
+    """
+    importlib.import_module(SCHEMA_MODULE)
+    old_db_schema = sys.modules[SCHEMA_MODULE]
+    engine = create_engine(*args, **kwargs)
+    old_db_schema.Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        session.add(
+            recorder.db_schema.StatisticsRuns(start=statistics.get_start_time())
+        )
+        session.add(
+            recorder.db_schema.SchemaChanges(
+                schema_version=old_db_schema.SCHEMA_VERSION
+            )
+        )
+        session.commit()
+    return engine
+
+
+def test_migrate_times(caplog, tmpdir):
+    """Test we can migrate times."""
+    test_db_file = tmpdir.mkdir("sqlite").join("test_run_info.db")
+    dburl = f"{SQLITE_URL_PREFIX}//{test_db_file}"
+
+    importlib.import_module(SCHEMA_MODULE)
+    old_db_schema = sys.modules[SCHEMA_MODULE]
+    now = dt_util.utcnow()
+    one_second_past = now - timedelta(seconds=1)
+    now_timestamp = now.timestamp()
+    one_second_past_timestamp = one_second_past.timestamp()
+
+    mock_state = State(
+        "sensor.test",
+        "old",
+        {"last_reset": now.isoformat()},
+        last_changed=one_second_past,
+        last_updated=now,
+    )
+    state_changed_event = Event(
+        EVENT_STATE_CHANGED,
+        {
+            "entity_id": "sensor.test",
+            "old_state": None,
+            "new_state": mock_state,
+        },
+        EventOrigin.local,
+        time_fired=now,
+    )
+    custom_event = Event(
+        "custom_event",
+        {"entity_id": "sensor.custom"},
+        EventOrigin.local,
+        time_fired=now,
+    )
+
+    with patch.object(recorder, "db_schema", old_db_schema), patch.object(
+        recorder.migration, "SCHEMA_VERSION", old_db_schema.SCHEMA_VERSION
+    ), patch.object(core, "EventData", old_db_schema.EventData), patch.object(
+        core, "States", old_db_schema.States
+    ), patch.object(
+        core, "Events", old_db_schema.Events
+    ), patch(
+        CREATE_ENGINE_TARGET, new=_create_engine_test
+    ):
+        hass = get_test_home_assistant()
+        recorder_helper.async_initialize_recorder(hass)
+        setup_component(hass, "recorder", {"recorder": {"db_url": dburl}})
+        wait_recording_done(hass)
+        wait_recording_done(hass)
+
+        with session_scope(hass=hass) as session:
+            session.add(old_db_schema.Events.from_event(custom_event))
+            session.add(old_db_schema.States.from_event(state_changed_event))
+
+        hass.stop()
+
+        dt_util.DEFAULT_TIME_ZONE = ORIG_TZ
+
+    # Test that the duplicates are removed during migration from schema 23
+    hass = get_test_home_assistant()
+    recorder_helper.async_initialize_recorder(hass)
+    setup_component(hass, "recorder", {"recorder": {"db_url": dburl}})
+    hass.start()
+    wait_recording_done(hass)
+    wait_recording_done(hass)
+    with session_scope(hass=hass) as session:
+        result = list(
+            session.query(recorder.db_schema.Events).where(
+                recorder.db_schema.Events.event_type == "custom_event"
+            )
+        )
+        assert len(result) == 1
+        assert result[0].time_fired_ts == now_timestamp
+        result = list(
+            session.query(recorder.db_schema.States).where(
+                recorder.db_schema.States.entity_id == "sensor.test"
+            )
+        )
+        assert len(result) == 1
+        assert result[0].last_changed_ts == one_second_past_timestamp
+        assert result[0].last_updated_ts == now_timestamp
+
+    hass.stop()
+    dt_util.DEFAULT_TIME_ZONE = ORIG_TZ

From 75c9c3a411d92fa4798d64fd662ff853c5259840 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 31 Dec 2022 17:10:52 -1000
Subject: [PATCH 24/38] Update homeassistant/components/recorder/migration.py

---
 homeassistant/components/recorder/migration.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py
index 5cd02784ac992e..bb8e7d15f3b0ae 100644
--- a/homeassistant/components/recorder/migration.py
+++ b/homeassistant/components/recorder/migration.py
@@ -865,9 +865,9 @@ def _migrate_columns_to_timestamp(
     hass: HomeAssistant, session: Session, engine: Engine
 ) -> None:
     """Migrate columns to use timestamp."""
-    # Migrate all data in Events.time_fired to Events.time_fired_ts and wipe Events.time_fired
-    # Migrate all data in States.last_updated to States.last_updated_ts and wipe States.last_updated
-    # Migrate all data in States.last_changed to States.last_changed_ts and wipe States.last_changed
+    # Migrate all data in Events.time_fired to Events.time_fired_ts
+    # Migrate all data in States.last_updated to States.last_updated_ts
+    # Migrate all data in States.last_changed to States.last_changed_ts
     connection = session.connection()
     if engine.dialect.name == SupportedDialect.SQLITE:
         connection.execute(

From a8a7e9b22217f393055e36d625abaf6910d971e3 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 31 Dec 2022 17:11:27 -1000
Subject: [PATCH 25/38] Update homeassistant/components/recorder/migration.py

---
 homeassistant/components/recorder/migration.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py
index bb8e7d15f3b0ae..3b4495957435bc 100644
--- a/homeassistant/components/recorder/migration.py
+++ b/homeassistant/components/recorder/migration.py
@@ -853,9 +853,9 @@ def _wipe_old_string_time_columns(
     hass: HomeAssistant, session: Session, engine: Engine
 ) -> None:
     """Wipe old string time columns."""
-    # Wipe Events.time_fired
-    # Wipe States.last_updated
-    # Wipe States.last_changed
+    # Wipe Events.time_fired since its been replaced by Events.time_fired_ts
+    # Wipe States.last_updated since its been replaced by States.last_updated_ts
+    # Wipe States.last_changed since its been replaced by States.last_changed_ts
     connection = session.connection()
     connection.execute(text("UPDATE events set time_fired=NULL;"))
     connection.execute(text("UPDATE states set last_updated=NULL, last_changed=NULL;"))

From ea6602dd1fe1e2cbdfa2fd51442d0abaddbac122 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 31 Dec 2022 17:21:10 -1000
Subject: [PATCH 26/38] Fix failing HomeKit Controller diagnostics tests

These tests were checking for a specific timestamp which
is no longer the same after the new year. As we do not
need to know the timestamp here we can remove it from
the test case with ANY
---
 .../homekit_controller/test_diagnostics.py     | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/tests/components/homekit_controller/test_diagnostics.py b/tests/components/homekit_controller/test_diagnostics.py
index eecb7ff51f8305..218347c4ad6022 100644
--- a/tests/components/homekit_controller/test_diagnostics.py
+++ b/tests/components/homekit_controller/test_diagnostics.py
@@ -1,4 +1,6 @@
 """Test homekit_controller diagnostics."""
+from unittest.mock import ANY
+
 from aiohttp import ClientSession
 
 from homeassistant.components.homekit_controller.const import KNOWN_DEVICES
@@ -247,8 +249,8 @@ async def test_config_entry(hass: HomeAssistant, hass_client: ClientSession, utc
                                 "friendly_name": "Koogeek-LS1-20833F Identify"
                             },
                             "entity_id": "button.koogeek_ls1_20833f_identify",
-                            "last_changed": "2023-01-01T00:00:00+00:00",
-                            "last_updated": "2023-01-01T00:00:00+00:00",
+                            "last_changed": ANY,
+                            "last_updated": ANY,
                             "state": "unknown",
                         },
                         "unit_of_measurement": None,
@@ -269,8 +271,8 @@ async def test_config_entry(hass: HomeAssistant, hass_client: ClientSession, utc
                                 "supported_features": 0,
                             },
                             "entity_id": "light.koogeek_ls1_20833f_light_strip",
-                            "last_changed": "2023-01-01T00:00:00+00:00",
-                            "last_updated": "2023-01-01T00:00:00+00:00",
+                            "last_changed": ANY,
+                            "last_updated": ANY,
                             "state": "off",
                         },
                         "unit_of_measurement": None,
@@ -518,8 +520,8 @@ async def test_device(hass: HomeAssistant, hass_client: ClientSession, utcnow):
                             "friendly_name": "Koogeek-LS1-20833F " "Identify"
                         },
                         "entity_id": "button.koogeek_ls1_20833f_identify",
-                        "last_changed": "2023-01-01T00:00:00+00:00",
-                        "last_updated": "2023-01-01T00:00:00+00:00",
+                        "last_changed": ANY,
+                        "last_updated": ANY,
                         "state": "unknown",
                     },
                     "unit_of_measurement": None,
@@ -540,8 +542,8 @@ async def test_device(hass: HomeAssistant, hass_client: ClientSession, utcnow):
                             "supported_features": 0,
                         },
                         "entity_id": "light.koogeek_ls1_20833f_light_strip",
-                        "last_changed": "2023-01-01T00:00:00+00:00",
-                        "last_updated": "2023-01-01T00:00:00+00:00",
+                        "last_changed": ANY,
+                        "last_updated": ANY,
                         "state": "off",
                     },
                     "unit_of_measurement": None,

From c60ab5c3f8176ed11ef508633961837add852ca1 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sat, 31 Dec 2022 21:50:28 -1000
Subject: [PATCH 27/38] fix v23 migration tests

---
 .../recorder/db_schema_23_with_newer_columns.py      | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tests/components/recorder/db_schema_23_with_newer_columns.py b/tests/components/recorder/db_schema_23_with_newer_columns.py
index a086aa588d4c92..d63e8d59d25738 100644
--- a/tests/components/recorder/db_schema_23_with_newer_columns.py
+++ b/tests/components/recorder/db_schema_23_with_newer_columns.py
@@ -14,6 +14,7 @@
 from datetime import datetime, timedelta
 import json
 import logging
+import time
 from typing import TypedDict, overload
 
 from sqlalchemy import (
@@ -89,6 +90,8 @@
     .with_variant(postgresql.DOUBLE_PRECISION(), "postgresql")
 )
 
+TIMESTAMP_TYPE = DOUBLE_TYPE
+
 
 class Events(Base):  # type: ignore
     """Event history data."""
@@ -108,6 +111,9 @@ class Events(Base):  # type: ignore
         SmallInteger
     )  # *** Not originally in v23, only added for recorder to startup ok
     time_fired = Column(DATETIME_TYPE, index=True)
+    time_fired_ts = Column(
+        TIMESTAMP_TYPE, index=True
+    )  # *** Not originally in v23, only added for recorder to startup ok
     created = Column(DATETIME_TYPE, default=dt_util.utcnow)
     context_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID), index=True)
     context_user_id = Column(String(MAX_LENGTH_EVENT_CONTEXT_ID), index=True)
@@ -197,7 +203,13 @@ class States(Base):  # type: ignore
         Integer, ForeignKey("events.event_id", ondelete="CASCADE"), index=True
     )
     last_changed = Column(DATETIME_TYPE, default=dt_util.utcnow)
+    last_updated_ts = Column(
+        TIMESTAMP_TYPE, default=time.time
+    )  # *** Not originally in v23, only added for recorder to startup ok
     last_updated = Column(DATETIME_TYPE, default=dt_util.utcnow, index=True)
+    last_updated_ts = Column(
+        TIMESTAMP_TYPE, default=time.time, index=True
+    )  # *** Not originally in v23, only added for recorder to startup ok
     created = Column(DATETIME_TYPE, default=dt_util.utcnow)
     old_state_id = Column(Integer, ForeignKey("states.state_id"), index=True)
     event = relationship("Events", uselist=False)

From 7d5313aa334efbc07ff1ee4cb8ae6379e859b2a2 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 1 Jan 2023 10:55:34 -1000
Subject: [PATCH 28/38] tests for legacy schema

---
 .../recorder/test_history_db_schema_30.py     | 624 ++++++++++++++++++
 1 file changed, 624 insertions(+)
 create mode 100644 tests/components/recorder/test_history_db_schema_30.py

diff --git a/tests/components/recorder/test_history_db_schema_30.py b/tests/components/recorder/test_history_db_schema_30.py
new file mode 100644
index 00000000000000..4c5ae693702b55
--- /dev/null
+++ b/tests/components/recorder/test_history_db_schema_30.py
@@ -0,0 +1,624 @@
+"""The tests the History component."""
+from __future__ import annotations
+
+# pylint: disable=protected-access,invalid-name
+from copy import copy
+from datetime import datetime, timedelta
+import importlib
+import json
+import sys
+from unittest.mock import patch, sentinel
+
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from homeassistant.components import recorder
+from homeassistant.components.recorder import core, history, statistics
+from homeassistant.components.recorder.models import process_timestamp
+from homeassistant.components.recorder.util import session_scope
+from homeassistant.core import State
+from homeassistant.helpers.json import JSONEncoder
+import homeassistant.util.dt as dt_util
+
+from .common import wait_recording_done
+
+CREATE_ENGINE_TARGET = "homeassistant.components.recorder.core.create_engine"
+SCHEMA_MODULE = "tests.components.recorder.db_schema_30"
+
+
+def _create_engine_test(*args, **kwargs):
+    """Test version of create_engine that initializes with old schema.
+
+    This simulates an existing db with the old schema.
+    """
+    importlib.import_module(SCHEMA_MODULE)
+    old_db_schema = sys.modules[SCHEMA_MODULE]
+    engine = create_engine(*args, **kwargs)
+    old_db_schema.Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        session.add(
+            recorder.db_schema.StatisticsRuns(start=statistics.get_start_time())
+        )
+        session.add(
+            recorder.db_schema.SchemaChanges(
+                schema_version=old_db_schema.SCHEMA_VERSION
+            )
+        )
+        session.commit()
+    return engine
+
+
+@pytest.fixture(autouse=True)
+def db_schema_30():
+    """Fixture to initialize the db with the old schema."""
+    importlib.import_module(SCHEMA_MODULE)
+    old_db_schema = sys.modules[SCHEMA_MODULE]
+
+    with patch.object(recorder, "db_schema", old_db_schema), patch.object(
+        recorder.migration, "SCHEMA_VERSION", old_db_schema.SCHEMA_VERSION
+    ), patch.object(core, "EventData", old_db_schema.EventData), patch.object(
+        core, "States", old_db_schema.States
+    ), patch.object(
+        core, "Events", old_db_schema.Events
+    ), patch.object(
+        core, "StateAttributes", old_db_schema.StateAttributes
+    ), patch(
+        CREATE_ENGINE_TARGET, new=_create_engine_test
+    ):
+        yield
+
+
+def test_get_full_significant_states_with_session_entity_no_matches(hass_recorder):
+    """Test getting states at a specific point in time for entities that never have been recorded."""
+    hass = hass_recorder()
+    now = dt_util.utcnow()
+    time_before_recorder_ran = now - timedelta(days=1000)
+    with session_scope(hass=hass) as session:
+        assert (
+            history.get_full_significant_states_with_session(
+                hass, session, time_before_recorder_ran, now, entity_ids=["demo.id"]
+            )
+            == {}
+        )
+        assert (
+            history.get_full_significant_states_with_session(
+                hass,
+                session,
+                time_before_recorder_ran,
+                now,
+                entity_ids=["demo.id", "demo.id2"],
+            )
+            == {}
+        )
+
+
+def test_significant_states_with_session_entity_minimal_response_no_matches(
+    hass_recorder,
+):
+    """Test getting states at a specific point in time for entities that never have been recorded."""
+    hass = hass_recorder()
+    now = dt_util.utcnow()
+    time_before_recorder_ran = now - timedelta(days=1000)
+    with session_scope(hass=hass) as session:
+        assert (
+            history.get_significant_states_with_session(
+                hass,
+                session,
+                time_before_recorder_ran,
+                now,
+                entity_ids=["demo.id"],
+                minimal_response=True,
+            )
+            == {}
+        )
+        assert (
+            history.get_significant_states_with_session(
+                hass,
+                session,
+                time_before_recorder_ran,
+                now,
+                entity_ids=["demo.id", "demo.id2"],
+                minimal_response=True,
+            )
+            == {}
+        )
+
+
+@pytest.mark.parametrize(
+    "attributes, no_attributes, limit",
+    [
+        ({"attr": True}, False, 5000),
+        ({}, True, 5000),
+        ({"attr": True}, False, 3),
+        ({}, True, 3),
+    ],
+)
+def test_state_changes_during_period(hass_recorder, attributes, no_attributes, limit):
+    """Test state change during period."""
+    hass = hass_recorder()
+    entity_id = "media_player.test"
+
+    def set_state(state):
+        """Set the state."""
+        hass.states.set(entity_id, state, attributes)
+        wait_recording_done(hass)
+        return hass.states.get(entity_id)
+
+    start = dt_util.utcnow()
+    point = start + timedelta(seconds=1)
+    end = point + timedelta(seconds=1)
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=start
+    ):
+        set_state("idle")
+        set_state("YouTube")
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=point
+    ):
+        states = [
+            set_state("idle"),
+            set_state("Netflix"),
+            set_state("Plex"),
+            set_state("YouTube"),
+        ]
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=end
+    ):
+        set_state("Netflix")
+        set_state("Plex")
+
+    hist = history.state_changes_during_period(
+        hass, start, end, entity_id, no_attributes, limit=limit
+    )
+
+    assert states[:limit] == hist[entity_id]
+
+
+def test_state_changes_during_period_descending(hass_recorder):
+    """Test state change during period descending."""
+    hass = hass_recorder()
+    entity_id = "media_player.test"
+
+    def set_state(state):
+        """Set the state."""
+        hass.states.set(entity_id, state, {"any": 1})
+        wait_recording_done(hass)
+        return hass.states.get(entity_id)
+
+    start = dt_util.utcnow()
+    point = start + timedelta(seconds=1)
+    point2 = start + timedelta(seconds=1, microseconds=2)
+    point3 = start + timedelta(seconds=1, microseconds=3)
+    point4 = start + timedelta(seconds=1, microseconds=4)
+    end = point + timedelta(seconds=1)
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=start
+    ):
+        set_state("idle")
+        set_state("YouTube")
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=point
+    ):
+        states = [set_state("idle")]
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=point2
+    ):
+        states.append(set_state("Netflix"))
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=point3
+    ):
+        states.append(set_state("Plex"))
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=point4
+    ):
+        states.append(set_state("YouTube"))
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=end
+    ):
+        set_state("Netflix")
+        set_state("Plex")
+
+    hist = history.state_changes_during_period(
+        hass, start, end, entity_id, no_attributes=False, descending=False
+    )
+    assert states == hist[entity_id]
+
+    hist = history.state_changes_during_period(
+        hass, start, end, entity_id, no_attributes=False, descending=True
+    )
+    assert states == list(reversed(list(hist[entity_id])))
+
+
+def test_get_last_state_changes(hass_recorder):
+    """Test number of state changes."""
+    hass = hass_recorder()
+    entity_id = "sensor.test"
+
+    def set_state(state):
+        """Set the state."""
+        hass.states.set(entity_id, state)
+        wait_recording_done(hass)
+        return hass.states.get(entity_id)
+
+    start = dt_util.utcnow() - timedelta(minutes=2)
+    point = start + timedelta(minutes=1)
+    point2 = point + timedelta(minutes=1)
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=start
+    ):
+        set_state("1")
+
+    states = []
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=point
+    ):
+        states.append(set_state("2"))
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=point2
+    ):
+        states.append(set_state("3"))
+
+    hist = history.get_last_state_changes(hass, 2, entity_id)
+
+    assert states == hist[entity_id]
+
+
+def test_ensure_state_can_be_copied(hass_recorder):
+    """Ensure a state can pass though copy().
+
+    The filter integration uses copy() on states
+    from history.
+    """
+    hass = hass_recorder()
+    entity_id = "sensor.test"
+
+    def set_state(state):
+        """Set the state."""
+        hass.states.set(entity_id, state)
+        wait_recording_done(hass)
+        return hass.states.get(entity_id)
+
+    start = dt_util.utcnow() - timedelta(minutes=2)
+    point = start + timedelta(minutes=1)
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=start
+    ):
+        set_state("1")
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=point
+    ):
+        set_state("2")
+
+    hist = history.get_last_state_changes(hass, 2, entity_id)
+
+    assert copy(hist[entity_id][0]) == hist[entity_id][0]
+    assert copy(hist[entity_id][1]) == hist[entity_id][1]
+
+
+def test_get_significant_states(hass_recorder):
+    """Test that only significant states are returned.
+
+    We should get back every thermostat change that
+    includes an attribute change, but only the state updates for
+    media player (attribute changes are not significant and not returned).
+    """
+    hass = hass_recorder()
+    zero, four, states = record_states(hass)
+    hist = history.get_significant_states(hass, zero, four)
+    assert states == hist
+
+
+def test_get_significant_states_minimal_response(hass_recorder):
+    """Test that only significant states are returned.
+
+    When minimal responses is set only the first and
+    last states return a complete state.
+    We should get back every thermostat change that
+    includes an attribute change, but only the state updates for
+    media player (attribute changes are not significant and not returned).
+    """
+    hass = hass_recorder()
+    zero, four, states = record_states(hass)
+    hist = history.get_significant_states(hass, zero, four, minimal_response=True)
+    entites_with_reducable_states = [
+        "media_player.test",
+        "media_player.test3",
+    ]
+
+    # All states for media_player.test state are reduced
+    # down to last_changed and state when minimal_response
+    # is set except for the first state.
+    # is set.  We use JSONEncoder to make sure that are
+    # pre-encoded last_changed is always the same as what
+    # will happen with encoding a native state
+    for entity_id in entites_with_reducable_states:
+        entity_states = states[entity_id]
+        for state_idx in range(1, len(entity_states)):
+            input_state = entity_states[state_idx]
+            orig_last_changed = orig_last_changed = json.dumps(
+                process_timestamp(input_state.last_changed),
+                cls=JSONEncoder,
+            ).replace('"', "")
+            orig_state = input_state.state
+            entity_states[state_idx] = {
+                "last_changed": orig_last_changed,
+                "state": orig_state,
+            }
+    assert states == hist
+
+
+def test_get_significant_states_with_initial(hass_recorder):
+    """Test that only significant states are returned.
+
+    We should get back every thermostat change that
+    includes an attribute change, but only the state updates for
+    media player (attribute changes are not significant and not returned).
+    """
+    hass = hass_recorder()
+    zero, four, states = record_states(hass)
+    one = zero + timedelta(seconds=1)
+    one_and_half = zero + timedelta(seconds=1.5)
+    for entity_id in states:
+        if entity_id == "media_player.test":
+            states[entity_id] = states[entity_id][1:]
+        for state in states[entity_id]:
+            if state.last_changed == one:
+                state.last_changed = one_and_half
+
+    hist = history.get_significant_states(
+        hass,
+        one_and_half,
+        four,
+        include_start_time_state=True,
+    )
+    assert states == hist
+
+
+def test_get_significant_states_without_initial(hass_recorder):
+    """Test that only significant states are returned.
+
+    We should get back every thermostat change that
+    includes an attribute change, but only the state updates for
+    media player (attribute changes are not significant and not returned).
+    """
+    hass = hass_recorder()
+    zero, four, states = record_states(hass)
+    one = zero + timedelta(seconds=1)
+    one_and_half = zero + timedelta(seconds=1.5)
+    for entity_id in states:
+        states[entity_id] = list(
+            filter(lambda s: s.last_changed != one, states[entity_id])
+        )
+    del states["media_player.test2"]
+
+    hist = history.get_significant_states(
+        hass,
+        one_and_half,
+        four,
+        include_start_time_state=False,
+    )
+    assert states == hist
+
+
+def test_get_significant_states_entity_id(hass_recorder):
+    """Test that only significant states are returned for one entity."""
+    hass = hass_recorder()
+    zero, four, states = record_states(hass)
+    del states["media_player.test2"]
+    del states["media_player.test3"]
+    del states["thermostat.test"]
+    del states["thermostat.test2"]
+    del states["script.can_cancel_this_one"]
+
+    hist = history.get_significant_states(hass, zero, four, ["media_player.test"])
+    assert states == hist
+
+
+def test_get_significant_states_multiple_entity_ids(hass_recorder):
+    """Test that only significant states are returned for one entity."""
+    hass = hass_recorder()
+    zero, four, states = record_states(hass)
+    del states["media_player.test2"]
+    del states["media_player.test3"]
+    del states["thermostat.test2"]
+    del states["script.can_cancel_this_one"]
+
+    hist = history.get_significant_states(
+        hass,
+        zero,
+        four,
+        ["media_player.test", "thermostat.test"],
+    )
+    assert states == hist
+
+
+def test_get_significant_states_are_ordered(hass_recorder):
+    """Test order of results from get_significant_states.
+
+    When entity ids are given, the results should be returned with the data
+    in the same order.
+    """
+    hass = hass_recorder()
+    zero, four, _states = record_states(hass)
+    entity_ids = ["media_player.test", "media_player.test2"]
+    hist = history.get_significant_states(hass, zero, four, entity_ids)
+    assert list(hist.keys()) == entity_ids
+    entity_ids = ["media_player.test2", "media_player.test"]
+    hist = history.get_significant_states(hass, zero, four, entity_ids)
+    assert list(hist.keys()) == entity_ids
+
+
+def test_get_significant_states_only(hass_recorder):
+    """Test significant states when significant_states_only is set."""
+    hass = hass_recorder()
+    entity_id = "sensor.test"
+
+    def set_state(state, **kwargs):
+        """Set the state."""
+        hass.states.set(entity_id, state, **kwargs)
+        wait_recording_done(hass)
+        return hass.states.get(entity_id)
+
+    start = dt_util.utcnow() - timedelta(minutes=4)
+    points = []
+    for i in range(1, 4):
+        points.append(start + timedelta(minutes=i))
+
+    states = []
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=start
+    ):
+        set_state("123", attributes={"attribute": 10.64})
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow",
+        return_value=points[0],
+    ):
+        # Attributes are different, state not
+        states.append(set_state("123", attributes={"attribute": 21.42}))
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow",
+        return_value=points[1],
+    ):
+        # state is different, attributes not
+        states.append(set_state("32", attributes={"attribute": 21.42}))
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow",
+        return_value=points[2],
+    ):
+        # everything is different
+        states.append(set_state("412", attributes={"attribute": 54.23}))
+
+    hist = history.get_significant_states(hass, start, significant_changes_only=True)
+
+    assert len(hist[entity_id]) == 2
+    assert states[0] not in hist[entity_id]
+    assert states[1] in hist[entity_id]
+    assert states[2] in hist[entity_id]
+
+    hist = history.get_significant_states(hass, start, significant_changes_only=False)
+
+    assert len(hist[entity_id]) == 3
+    assert states == hist[entity_id]
+
+
+def record_states(hass) -> tuple[datetime, datetime, dict[str, list[State]]]:
+    """Record some test states.
+
+    We inject a bunch of state updates from media player, zone and
+    thermostat.
+    """
+    mp = "media_player.test"
+    mp2 = "media_player.test2"
+    mp3 = "media_player.test3"
+    therm = "thermostat.test"
+    therm2 = "thermostat.test2"
+    zone = "zone.home"
+    script_c = "script.can_cancel_this_one"
+
+    def set_state(entity_id, state, **kwargs):
+        """Set the state."""
+        hass.states.set(entity_id, state, **kwargs)
+        wait_recording_done(hass)
+        return hass.states.get(entity_id)
+
+    zero = dt_util.utcnow()
+    one = zero + timedelta(seconds=1)
+    two = one + timedelta(seconds=1)
+    three = two + timedelta(seconds=1)
+    four = three + timedelta(seconds=1)
+
+    states = {therm: [], therm2: [], mp: [], mp2: [], mp3: [], script_c: []}
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=one
+    ):
+        states[mp].append(
+            set_state(mp, "idle", attributes={"media_title": str(sentinel.mt1)})
+        )
+        states[mp].append(
+            set_state(mp, "YouTube", attributes={"media_title": str(sentinel.mt2)})
+        )
+        states[mp2].append(
+            set_state(mp2, "YouTube", attributes={"media_title": str(sentinel.mt2)})
+        )
+        states[mp3].append(
+            set_state(mp3, "idle", attributes={"media_title": str(sentinel.mt1)})
+        )
+        states[therm].append(
+            set_state(therm, 20, attributes={"current_temperature": 19.5})
+        )
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=two
+    ):
+        # This state will be skipped only different in time
+        set_state(mp, "YouTube", attributes={"media_title": str(sentinel.mt3)})
+        # This state will be skipped because domain is excluded
+        set_state(zone, "zoning")
+        states[script_c].append(
+            set_state(script_c, "off", attributes={"can_cancel": True})
+        )
+        states[therm].append(
+            set_state(therm, 21, attributes={"current_temperature": 19.8})
+        )
+        states[therm2].append(
+            set_state(therm2, 20, attributes={"current_temperature": 19})
+        )
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=three
+    ):
+        states[mp].append(
+            set_state(mp, "Netflix", attributes={"media_title": str(sentinel.mt4)})
+        )
+        states[mp3].append(
+            set_state(mp3, "Netflix", attributes={"media_title": str(sentinel.mt3)})
+        )
+        # Attributes changed even though state is the same
+        states[therm].append(
+            set_state(therm, 21, attributes={"current_temperature": 20})
+        )
+
+    return zero, four, states
+
+
+def test_state_changes_during_period_multiple_entities_single_test(hass_recorder):
+    """Test state change during period with multiple entities in the same test.
+
+    This test ensures the sqlalchemy query cache does not
+    generate incorrect results.
+    """
+    hass = hass_recorder()
+    start = dt_util.utcnow()
+    test_entites = {f"sensor.{i}": str(i) for i in range(30)}
+    for entity_id, value in test_entites.items():
+        hass.states.set(entity_id, value)
+
+    wait_recording_done(hass)
+    end = dt_util.utcnow()
+
+    hist = history.state_changes_during_period(hass, start, end, None)
+    for entity_id, value in test_entites.items():
+        hist[entity_id][0].state == value
+
+    for entity_id, value in test_entites.items():
+        hist = history.state_changes_during_period(hass, start, end, entity_id)
+        assert len(hist) == 1
+        hist[entity_id][0].state == value
+
+    hist = history.state_changes_during_period(hass, start, end, None)
+    for entity_id, value in test_entites.items():
+        hist[entity_id][0].state == value

From ff088a3acf89c7e0171bf99938218766afa1246f Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 1 Jan 2023 10:59:18 -1000
Subject: [PATCH 29/38] Apply suggestions from code review

---
 homeassistant/components/recorder/migration.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py
index 3b4495957435bc..dd046d973b2b52 100644
--- a/homeassistant/components/recorder/migration.py
+++ b/homeassistant/components/recorder/migration.py
@@ -872,13 +872,16 @@ def _migrate_columns_to_timestamp(
     if engine.dialect.name == SupportedDialect.SQLITE:
         connection.execute(
             text(
-                'UPDATE events set time_fired_ts=strftime("%s",time_fired) + cast(substr(time_fired,-7) AS FLOAT);'
+                'UPDATE events set time_fired_ts=strftime("%s",time_fired) + '
+                'cast(substr(time_fired,-7) AS FLOAT);'
             )
         )
         connection.execute(
             text(
-                'UPDATE states set last_updated_ts=strftime("%s",last_updated) + cast(substr(last_updated,-7) AS FLOAT), '
-                'last_changed_ts=strftime("%s",last_changed) + cast(substr(last_changed,-7) AS FLOAT);'
+                'UPDATE states set last_updated_ts=strftime("%s",last_updated) + '
+                'cast(substr(last_updated,-7) AS FLOAT), '
+                'last_changed_ts=strftime("%s",last_changed) + '
+                'cast(substr(last_changed,-7) AS FLOAT);'
             )
         )
     elif engine.dialect.name == SupportedDialect.MYSQL:

From 25b1289e774501a317773e4ee4772a9f5cba7edd Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 1 Jan 2023 10:59:40 -1000
Subject: [PATCH 30/38] lint

---
 homeassistant/components/recorder/migration.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py
index dd046d973b2b52..b94763995c17b8 100644
--- a/homeassistant/components/recorder/migration.py
+++ b/homeassistant/components/recorder/migration.py
@@ -873,15 +873,15 @@ def _migrate_columns_to_timestamp(
         connection.execute(
             text(
                 'UPDATE events set time_fired_ts=strftime("%s",time_fired) + '
-                'cast(substr(time_fired,-7) AS FLOAT);'
+                "cast(substr(time_fired,-7) AS FLOAT);"
             )
         )
         connection.execute(
             text(
                 'UPDATE states set last_updated_ts=strftime("%s",last_updated) + '
-                'cast(substr(last_updated,-7) AS FLOAT), '
+                "cast(substr(last_updated,-7) AS FLOAT), "
                 'last_changed_ts=strftime("%s",last_changed) + '
-                'cast(substr(last_changed,-7) AS FLOAT);'
+                "cast(substr(last_changed,-7) AS FLOAT);"
             )
         )
     elif engine.dialect.name == SupportedDialect.MYSQL:

From a1e972ff8b2f4e5dee3f22b2d09e53fef4eb0745 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 1 Jan 2023 11:01:51 -1000
Subject: [PATCH 31/38] fix missing ,s

---
 tests/components/logbook/test_init.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/tests/components/logbook/test_init.py b/tests/components/logbook/test_init.py
index 889e8f97106933..601eed0dc717a5 100644
--- a/tests/components/logbook/test_init.py
+++ b/tests/components/logbook/test_init.py
@@ -313,14 +313,17 @@ def create_state_changed_event_from_old_new(
     row = collections.namedtuple(
         "Row",
         [
-            "event_type" "event_data" "time_fired" "time_fired_ts",
-            "context_id"
-            "context_user_id"
-            "context_parent_id"
-            "state"
-            "entity_id"
-            "domain"
-            "attributes"
+            "event_type",
+            "event_data",
+            "time_fired",
+            "time_fired_ts",
+            "context_id",
+            "context_user_id",
+            "context_parent_id",
+            "state",
+            "entity_id",
+            "domain",
+            "attributes",
             "state_id",
             "old_state_id",
             "shared_attrs",

From 221c5bf568eafd055f047af45d0c4a3fd4575087 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 1 Jan 2023 11:22:30 -1000
Subject: [PATCH 32/38] postgres fixes

---
 homeassistant/components/recorder/history.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/homeassistant/components/recorder/history.py b/homeassistant/components/recorder/history.py
index 3ae84a0b0237e2..a78fa468d9417d 100644
--- a/homeassistant/components/recorder/history.py
+++ b/homeassistant/components/recorder/history.py
@@ -25,7 +25,7 @@
 import homeassistant.util.dt as dt_util
 
 from .. import recorder
-from .db_schema import TIMESTAMP_TYPE, RecorderRuns, StateAttributes, States
+from .db_schema import RecorderRuns, StateAttributes, States
 from .filters import Filters
 from .models import (
     LazyState,
@@ -71,7 +71,7 @@
 _BASE_STATES_NO_LAST_CHANGED = [
     States.entity_id,
     States.state,
-    literal(value=None, type_=TIMESTAMP_TYPE).label("last_changed_ts"),
+    literal(value=None).label("last_changed_ts"),
     States.last_updated_ts,
 ]
 _QUERY_STATE_NO_ATTR = [

From b792769bf93a40329d2bfd4cfd1e8491bf26df56 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 1 Jan 2023 12:05:41 -1000
Subject: [PATCH 33/38] reduce memory pressure and ensure we switch to the new
 history path as soon as its ready

---
 homeassistant/components/recorder/core.py     |  4 ++
 .../components/recorder/migration.py          | 38 +++++++++++++++----
 homeassistant/components/recorder/tasks.py    | 14 +++++++
 3 files changed, 48 insertions(+), 8 deletions(-)

diff --git a/homeassistant/components/recorder/core.py b/homeassistant/components/recorder/core.py
index 66e85eac2b30d3..f9d1c2966766f1 100644
--- a/homeassistant/components/recorder/core.py
+++ b/homeassistant/components/recorder/core.py
@@ -1021,6 +1021,10 @@ def _open_event_session(self) -> None:
         self.event_session = self.get_session()
         self.event_session.expire_on_commit = False
 
+    def _post_schema_migration(self, old_version: int, new_version: int) -> None:
+        """Run post schema migration tasks."""
+        migration.post_schema_migration(self.event_session, old_version, new_version)
+
     def _send_keep_alive(self) -> None:
         """Send a keep alive to keep the db connection open."""
         assert self.event_session is not None
diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py
index b94763995c17b8..0257ba291b396c 100644
--- a/homeassistant/components/recorder/migration.py
+++ b/homeassistant/components/recorder/migration.py
@@ -43,6 +43,7 @@
     get_start_time,
     validate_db_schema as statistics_validate_db_schema,
 )
+from .tasks import PostSchemaMigrationTask
 from .util import session_scope
 
 if TYPE_CHECKING:
@@ -163,6 +164,9 @@ def migrate_schema(
         )
         statistics_correct_db_schema(instance, engine, session_maker, schema_errors)
 
+    if current_version != SCHEMA_VERSION:
+        instance.queue_task(PostSchemaMigrationTask(current_version, SCHEMA_VERSION))
+
 
 def _create_index(
     session_maker: Callable[[], Session], table_name: str, index_name: str
@@ -839,8 +843,6 @@ def _apply_update(  # noqa: C901
     elif new_version == 32:
         # Migration is done in two steps to ensure we can start using
         # the new columns before we wipe the old ones.
-        with session_scope(session=session_maker()) as session:
-            _wipe_old_string_time_columns(hass, session, engine)
         _drop_index(session_maker, "states", "ix_states_entity_id_last_updated")
         _drop_index(session_maker, "events", "ix_events_event_type_time_fired")
         _drop_index(session_maker, "states", "ix_states_last_updated")
@@ -849,16 +851,36 @@ def _apply_update(  # noqa: C901
         raise ValueError(f"No schema migration defined for version {new_version}")
 
 
-def _wipe_old_string_time_columns(
-    hass: HomeAssistant, session: Session, engine: Engine
+def post_schema_migration(
+    session: Session,
+    old_version: int,
+    new_version: int,
 ) -> None:
-    """Wipe old string time columns."""
+    """Post schema migration.
+
+    Run any housekeeping tasks after the schema migration has completed.
+
+    Post schema migration is run after the schema migration has completed
+    and the queue has been processed to ensure that we reduce the memory
+    pressure since events are held in memory until the queue is processed
+    which is blocked from being processed until the schema migration is
+    complete.
+    """
+    if old_version < 32 <= new_version:
+        # In version 31 we migrated all the time_fired, last_updated, and last_changed
+        # columns to be timestamps. In version 32 we need to wipe the old columns
+        # since they are no longer used and take up a significant amount of space.
+        _wipe_old_string_time_columns(session)
+
+
+def _wipe_old_string_time_columns(session: Session) -> None:
+    """Wipe old string time columns to save space."""
     # Wipe Events.time_fired since its been replaced by Events.time_fired_ts
     # Wipe States.last_updated since its been replaced by States.last_updated_ts
     # Wipe States.last_changed since its been replaced by States.last_changed_ts
-    connection = session.connection()
-    connection.execute(text("UPDATE events set time_fired=NULL;"))
-    connection.execute(text("UPDATE states set last_updated=NULL, last_changed=NULL;"))
+    session.execute(text("UPDATE events set time_fired=NULL;"))
+    session.execute(text("UPDATE states set last_updated=NULL, last_changed=NULL;"))
+    session.commit()
 
 
 def _migrate_columns_to_timestamp(
diff --git a/homeassistant/components/recorder/tasks.py b/homeassistant/components/recorder/tasks.py
index 01723a509609a3..ba6c8dd04276a8 100644
--- a/homeassistant/components/recorder/tasks.py
+++ b/homeassistant/components/recorder/tasks.py
@@ -297,3 +297,17 @@ def run(self, instance: Recorder) -> None:
         # Does not use a tracked task to avoid
         # blocking shutdown if the recorder is broken
         instance.hass.loop.call_soon_threadsafe(self.event.set)
+
+
+@dataclass
+class PostSchemaMigrationTask(RecorderTask):
+    """Post migration task to update schema."""
+
+    old_version: int
+    new_version: int
+
+    def run(self, instance: Recorder) -> None:
+        """Handle the task."""
+        instance._post_schema_migration(  # pylint: disable=[protected-access]
+            self.old_version, self.new_version
+        )

From 4079504b3b55a61713e3bf21dda8a2811d00b418 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 1 Jan 2023 12:19:38 -1000
Subject: [PATCH 34/38] fix mysql

---
 homeassistant/components/recorder/migration.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py
index 0257ba291b396c..b8a303104b978c 100644
--- a/homeassistant/components/recorder/migration.py
+++ b/homeassistant/components/recorder/migration.py
@@ -496,6 +496,10 @@ def _apply_update(  # noqa: C901
     """Perform operations to bring schema up to date."""
     dialect = engine.dialect.name
     big_int = "INTEGER(20)" if dialect == SupportedDialect.MYSQL else "INTEGER"
+    if dialect in (SupportedDialect.MYSQL, SupportedDialect.POSTGRESQL):
+        timestamp_type = "DOUBLE PRECISION"
+    else:
+        timestamp_type = "FLOAT"
 
     if new_version == 1:
         _create_index(session_maker, "events", "ix_events_time_fired")
@@ -830,9 +834,11 @@ def _apply_update(  # noqa: C901
         # ALTER TABLE events DROP COLUMN time_fired
         # ALTER TABLE states DROP COLUMN last_updated
         # ALTER TABLE states DROP COLUMN last_changed
-        _add_columns(session_maker, "events", ["time_fired_ts FLOAT"])
+        _add_columns(session_maker, "events", [f"time_fired_ts {timestamp_type}"])
         _add_columns(
-            session_maker, "states", ["last_updated_ts FLOAT", "last_changed_ts FLOAT"]
+            session_maker,
+            "states",
+            [f"last_updated_ts {timestamp_type}", f"last_changed_ts {timestamp_type}"],
         )
         _create_index(session_maker, "events", "ix_events_time_fired_ts")
         _create_index(session_maker, "events", "ix_events_event_type_time_fired_ts")

From 7c8e9914f87f21baa89fa40ee14cb0f232f534d7 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 1 Jan 2023 13:25:04 -1000
Subject: [PATCH 35/38] preen

---
 homeassistant/components/recorder/models.py | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/homeassistant/components/recorder/models.py b/homeassistant/components/recorder/models.py
index c3000c183955ab..9972b1f4efc1b2 100644
--- a/homeassistant/components/recorder/models.py
+++ b/homeassistant/components/recorder/models.py
@@ -187,11 +187,6 @@ def last_changed(self, value: datetime) -> None:
         """Set last changed datetime."""
         self._last_changed = value
 
-    @property
-    def last_updated_ts(self) -> float:
-        """Last updated timestamp."""
-        return self.last_updated.timestamp()
-
     @property
     def last_updated(self) -> datetime:
         """Last updated datetime."""
@@ -315,12 +310,6 @@ def last_changed(self, value: datetime) -> None:
         """Set last changed datetime."""
         self._last_changed_ts = process_timestamp(value).timestamp()
 
-    @property
-    def last_updated_ts(self) -> float:
-        """Last updated timestamp."""
-        assert self._last_updated_ts is not None
-        return self._last_updated_ts
-
     @property
     def last_updated(self) -> datetime:
         """Last updated datetime."""

From c4bc74e0eeee5bf51e519a5fdc5180a392bd93ea Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 1 Jan 2023 13:28:29 -1000
Subject: [PATCH 36/38] add tests for history apis still work during migration

---
 .../history/test_init_db_schema_30.py         | 1353 +++++++++++++++++
 1 file changed, 1353 insertions(+)
 create mode 100644 tests/components/history/test_init_db_schema_30.py

diff --git a/tests/components/history/test_init_db_schema_30.py b/tests/components/history/test_init_db_schema_30.py
new file mode 100644
index 00000000000000..f96a9d030d3de9
--- /dev/null
+++ b/tests/components/history/test_init_db_schema_30.py
@@ -0,0 +1,1353 @@
+"""The tests the History component."""
+from __future__ import annotations
+
+# pylint: disable=protected-access,invalid-name
+from datetime import timedelta
+from http import HTTPStatus
+import importlib
+import json
+import sys
+from unittest.mock import patch, sentinel
+
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from homeassistant.components import history, recorder
+from homeassistant.components.recorder import core, statistics
+from homeassistant.components.recorder.history import get_significant_states
+from homeassistant.components.recorder.models import process_timestamp
+from homeassistant.const import CONF_DOMAINS, CONF_ENTITIES, CONF_EXCLUDE, CONF_INCLUDE
+import homeassistant.core as ha
+from homeassistant.helpers.json import JSONEncoder
+from homeassistant.setup import async_setup_component
+import homeassistant.util.dt as dt_util
+
+from tests.components.recorder.common import (
+    async_recorder_block_till_done,
+    async_wait_recording_done,
+    wait_recording_done,
+)
+
+CREATE_ENGINE_TARGET = "homeassistant.components.recorder.core.create_engine"
+SCHEMA_MODULE = "tests.components.recorder.db_schema_30"
+
+
+def _create_engine_test(*args, **kwargs):
+    """Test version of create_engine that initializes with old schema.
+
+    This simulates an existing db with the old schema.
+    """
+    importlib.import_module(SCHEMA_MODULE)
+    old_db_schema = sys.modules[SCHEMA_MODULE]
+    engine = create_engine(*args, **kwargs)
+    old_db_schema.Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        session.add(
+            recorder.db_schema.StatisticsRuns(start=statistics.get_start_time())
+        )
+        session.add(
+            recorder.db_schema.SchemaChanges(
+                schema_version=old_db_schema.SCHEMA_VERSION
+            )
+        )
+        session.commit()
+    return engine
+
+
+@pytest.fixture(autouse=True)
+def db_schema_30():
+    """Fixture to initialize the db with the old schema."""
+    importlib.import_module(SCHEMA_MODULE)
+    old_db_schema = sys.modules[SCHEMA_MODULE]
+
+    with patch.object(recorder, "db_schema", old_db_schema), patch.object(
+        recorder.migration, "SCHEMA_VERSION", old_db_schema.SCHEMA_VERSION
+    ), patch.object(core, "EventData", old_db_schema.EventData), patch.object(
+        core, "States", old_db_schema.States
+    ), patch.object(
+        core, "Events", old_db_schema.Events
+    ), patch.object(
+        core, "StateAttributes", old_db_schema.StateAttributes
+    ), patch(
+        CREATE_ENGINE_TARGET, new=_create_engine_test
+    ):
+        yield
+
+
+@pytest.mark.usefixtures("hass_history")
+def test_setup():
+    """Test setup method of history."""
+    # Verification occurs in the fixture
+
+
+def test_get_significant_states(hass_history):
+    """Test that only significant states are returned.
+
+    We should get back every thermostat change that
+    includes an attribute change, but only the state updates for
+    media player (attribute changes are not significant and not returned).
+    """
+    hass = hass_history
+    zero, four, states = record_states(hass)
+    hist = get_significant_states(hass, zero, four, filters=history.Filters())
+    assert states == hist
+
+
+def test_get_significant_states_minimal_response(hass_history):
+    """Test that only significant states are returned.
+
+    When minimal responses is set only the first and
+    last states return a complete state.
+
+    We should get back every thermostat change that
+    includes an attribute change, but only the state updates for
+    media player (attribute changes are not significant and not returned).
+    """
+    hass = hass_history
+    zero, four, states = record_states(hass)
+    hist = get_significant_states(
+        hass, zero, four, filters=history.Filters(), minimal_response=True
+    )
+    entites_with_reducable_states = [
+        "media_player.test",
+        "media_player.test3",
+    ]
+
+    # All states for media_player.test state are reduced
+    # down to last_changed and state when minimal_response
+    # is set except for the first state.
+    # is set.  We use JSONEncoder to make sure that are
+    # pre-encoded last_changed is always the same as what
+    # will happen with encoding a native state
+    for entity_id in entites_with_reducable_states:
+        entity_states = states[entity_id]
+        for state_idx in range(1, len(entity_states)):
+            input_state = entity_states[state_idx]
+            orig_last_changed = orig_last_changed = json.dumps(
+                process_timestamp(input_state.last_changed),
+                cls=JSONEncoder,
+            ).replace('"', "")
+            orig_state = input_state.state
+            entity_states[state_idx] = {
+                "last_changed": orig_last_changed,
+                "state": orig_state,
+            }
+    assert states == hist
+
+
+def test_get_significant_states_with_initial(hass_history):
+    """Test that only significant states are returned.
+
+    We should get back every thermostat change that
+    includes an attribute change, but only the state updates for
+    media player (attribute changes are not significant and not returned).
+    """
+    hass = hass_history
+    zero, four, states = record_states(hass)
+    one = zero + timedelta(seconds=1)
+    one_and_half = zero + timedelta(seconds=1.5)
+    for entity_id in states:
+        if entity_id == "media_player.test":
+            states[entity_id] = states[entity_id][1:]
+        for state in states[entity_id]:
+            if state.last_changed == one:
+                state.last_changed = one_and_half
+
+    hist = get_significant_states(
+        hass,
+        one_and_half,
+        four,
+        filters=history.Filters(),
+        include_start_time_state=True,
+    )
+    assert states == hist
+
+
+def test_get_significant_states_without_initial(hass_history):
+    """Test that only significant states are returned.
+
+    We should get back every thermostat change that
+    includes an attribute change, but only the state updates for
+    media player (attribute changes are not significant and not returned).
+    """
+    hass = hass_history
+    zero, four, states = record_states(hass)
+    one = zero + timedelta(seconds=1)
+    one_and_half = zero + timedelta(seconds=1.5)
+    for entity_id in states:
+        states[entity_id] = list(
+            filter(lambda s: s.last_changed != one, states[entity_id])
+        )
+    del states["media_player.test2"]
+
+    hist = get_significant_states(
+        hass,
+        one_and_half,
+        four,
+        filters=history.Filters(),
+        include_start_time_state=False,
+    )
+    assert states == hist
+
+
+def test_get_significant_states_entity_id(hass_history):
+    """Test that only significant states are returned for one entity."""
+    hass = hass_history
+    zero, four, states = record_states(hass)
+    del states["media_player.test2"]
+    del states["media_player.test3"]
+    del states["thermostat.test"]
+    del states["thermostat.test2"]
+    del states["script.can_cancel_this_one"]
+
+    hist = get_significant_states(
+        hass, zero, four, ["media_player.test"], filters=history.Filters()
+    )
+    assert states == hist
+
+
+def test_get_significant_states_multiple_entity_ids(hass_history):
+    """Test that only significant states are returned for one entity."""
+    hass = hass_history
+    zero, four, states = record_states(hass)
+    del states["media_player.test2"]
+    del states["media_player.test3"]
+    del states["thermostat.test2"]
+    del states["script.can_cancel_this_one"]
+
+    hist = get_significant_states(
+        hass,
+        zero,
+        four,
+        ["media_player.test", "thermostat.test"],
+        filters=history.Filters(),
+    )
+    assert states == hist
+
+
+def test_get_significant_states_exclude_domain(hass_history):
+    """Test if significant states are returned when excluding domains.
+
+    We should get back every thermostat change that includes an attribute
+    change, but no media player changes.
+    """
+    hass = hass_history
+    zero, four, states = record_states(hass)
+    del states["media_player.test"]
+    del states["media_player.test2"]
+    del states["media_player.test3"]
+
+    config = history.CONFIG_SCHEMA(
+        {
+            ha.DOMAIN: {},
+            history.DOMAIN: {CONF_EXCLUDE: {CONF_DOMAINS: ["media_player"]}},
+        }
+    )
+    check_significant_states(hass, zero, four, states, config)
+
+
+def test_get_significant_states_exclude_entity(hass_history):
+    """Test if significant states are returned when excluding entities.
+
+    We should get back every thermostat and script changes, but no media
+    player changes.
+    """
+    hass = hass_history
+    zero, four, states = record_states(hass)
+    del states["media_player.test"]
+
+    config = history.CONFIG_SCHEMA(
+        {
+            ha.DOMAIN: {},
+            history.DOMAIN: {CONF_EXCLUDE: {CONF_ENTITIES: ["media_player.test"]}},
+        }
+    )
+    check_significant_states(hass, zero, four, states, config)
+
+
+def test_get_significant_states_exclude(hass_history):
+    """Test significant states when excluding entities and domains.
+
+    We should not get back every thermostat and media player test changes.
+    """
+    hass = hass_history
+    zero, four, states = record_states(hass)
+    del states["media_player.test"]
+    del states["thermostat.test"]
+    del states["thermostat.test2"]
+
+    config = history.CONFIG_SCHEMA(
+        {
+            ha.DOMAIN: {},
+            history.DOMAIN: {
+                CONF_EXCLUDE: {
+                    CONF_DOMAINS: ["thermostat"],
+                    CONF_ENTITIES: ["media_player.test"],
+                }
+            },
+        }
+    )
+    check_significant_states(hass, zero, four, states, config)
+
+
+def test_get_significant_states_exclude_include_entity(hass_history):
+    """Test significant states when excluding domains and include entities.
+
+    We should not get back every thermostat change unless its specifically included
+    """
+    hass = hass_history
+    zero, four, states = record_states(hass)
+    del states["thermostat.test2"]
+
+    config = history.CONFIG_SCHEMA(
+        {
+            ha.DOMAIN: {},
+            history.DOMAIN: {
+                CONF_INCLUDE: {CONF_ENTITIES: ["media_player.test", "thermostat.test"]},
+                CONF_EXCLUDE: {CONF_DOMAINS: ["thermostat"]},
+            },
+        }
+    )
+    check_significant_states(hass, zero, four, states, config)
+
+
+def test_get_significant_states_include_domain(hass_history):
+    """Test if significant states are returned when including domains.
+
+    We should get back every thermostat and script changes, but no media
+    player changes.
+    """
+    hass = hass_history
+    zero, four, states = record_states(hass)
+    del states["media_player.test"]
+    del states["media_player.test2"]
+    del states["media_player.test3"]
+
+    config = history.CONFIG_SCHEMA(
+        {
+            ha.DOMAIN: {},
+            history.DOMAIN: {CONF_INCLUDE: {CONF_DOMAINS: ["thermostat", "script"]}},
+        }
+    )
+    check_significant_states(hass, zero, four, states, config)
+
+
+def test_get_significant_states_include_entity(hass_history):
+    """Test if significant states are returned when including entities.
+
+    We should only get back changes of the media_player.test entity.
+    """
+    hass = hass_history
+    zero, four, states = record_states(hass)
+    del states["media_player.test2"]
+    del states["media_player.test3"]
+    del states["thermostat.test"]
+    del states["thermostat.test2"]
+    del states["script.can_cancel_this_one"]
+
+    config = history.CONFIG_SCHEMA(
+        {
+            ha.DOMAIN: {},
+            history.DOMAIN: {CONF_INCLUDE: {CONF_ENTITIES: ["media_player.test"]}},
+        }
+    )
+    check_significant_states(hass, zero, four, states, config)
+
+
+def test_get_significant_states_include(hass_history):
+    """Test significant states when including domains and entities.
+
+    We should only get back changes of the media_player.test entity and the
+    thermostat domain.
+    """
+    hass = hass_history
+    zero, four, states = record_states(hass)
+    del states["media_player.test2"]
+    del states["media_player.test3"]
+    del states["script.can_cancel_this_one"]
+
+    config = history.CONFIG_SCHEMA(
+        {
+            ha.DOMAIN: {},
+            history.DOMAIN: {
+                CONF_INCLUDE: {
+                    CONF_DOMAINS: ["thermostat"],
+                    CONF_ENTITIES: ["media_player.test"],
+                }
+            },
+        }
+    )
+    check_significant_states(hass, zero, four, states, config)
+
+
+def test_get_significant_states_include_exclude_domain(hass_history):
+    """Test if significant states when excluding and including domains.
+
+    We should get back all the media_player domain changes
+    only since the include wins over the exclude but will
+    exclude everything else.
+    """
+    hass = hass_history
+    zero, four, states = record_states(hass)
+    del states["thermostat.test"]
+    del states["thermostat.test2"]
+    del states["script.can_cancel_this_one"]
+
+    config = history.CONFIG_SCHEMA(
+        {
+            ha.DOMAIN: {},
+            history.DOMAIN: {
+                CONF_INCLUDE: {CONF_DOMAINS: ["media_player"]},
+                CONF_EXCLUDE: {CONF_DOMAINS: ["media_player"]},
+            },
+        }
+    )
+    check_significant_states(hass, zero, four, states, config)
+
+
+def test_get_significant_states_include_exclude_entity(hass_history):
+    """Test if significant states when excluding and including domains.
+
+    We should not get back any changes since we include only
+    media_player.test but also exclude it.
+    """
+    hass = hass_history
+    zero, four, states = record_states(hass)
+    del states["media_player.test2"]
+    del states["media_player.test3"]
+    del states["thermostat.test"]
+    del states["thermostat.test2"]
+    del states["script.can_cancel_this_one"]
+
+    config = history.CONFIG_SCHEMA(
+        {
+            ha.DOMAIN: {},
+            history.DOMAIN: {
+                CONF_INCLUDE: {CONF_ENTITIES: ["media_player.test"]},
+                CONF_EXCLUDE: {CONF_ENTITIES: ["media_player.test"]},
+            },
+        }
+    )
+    check_significant_states(hass, zero, four, states, config)
+
+
+def test_get_significant_states_include_exclude(hass_history):
+    """Test if significant states when in/excluding domains and entities.
+
+    We should get back changes of the media_player.test2, media_player.test3,
+    and thermostat.test.
+    """
+    hass = hass_history
+    zero, four, states = record_states(hass)
+    del states["media_player.test"]
+    del states["thermostat.test2"]
+    del states["script.can_cancel_this_one"]
+
+    config = history.CONFIG_SCHEMA(
+        {
+            ha.DOMAIN: {},
+            history.DOMAIN: {
+                CONF_INCLUDE: {
+                    CONF_DOMAINS: ["media_player"],
+                    CONF_ENTITIES: ["thermostat.test"],
+                },
+                CONF_EXCLUDE: {
+                    CONF_DOMAINS: ["thermostat"],
+                    CONF_ENTITIES: ["media_player.test"],
+                },
+            },
+        }
+    )
+    check_significant_states(hass, zero, four, states, config)
+
+
+def test_get_significant_states_are_ordered(hass_history):
+    """Test order of results from get_significant_states.
+
+    When entity ids are given, the results should be returned with the data
+    in the same order.
+    """
+    hass = hass_history
+    zero, four, _states = record_states(hass)
+    entity_ids = ["media_player.test", "media_player.test2"]
+    hist = get_significant_states(
+        hass, zero, four, entity_ids, filters=history.Filters()
+    )
+    assert list(hist.keys()) == entity_ids
+    entity_ids = ["media_player.test2", "media_player.test"]
+    hist = get_significant_states(
+        hass, zero, four, entity_ids, filters=history.Filters()
+    )
+    assert list(hist.keys()) == entity_ids
+
+
+def test_get_significant_states_only(hass_history):
+    """Test significant states when significant_states_only is set."""
+    hass = hass_history
+    entity_id = "sensor.test"
+
+    def set_state(state, **kwargs):
+        """Set the state."""
+        hass.states.set(entity_id, state, **kwargs)
+        wait_recording_done(hass)
+        return hass.states.get(entity_id)
+
+    start = dt_util.utcnow() - timedelta(minutes=4)
+    points = []
+    for i in range(1, 4):
+        points.append(start + timedelta(minutes=i))
+
+    states = []
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=start
+    ):
+        set_state("123", attributes={"attribute": 10.64})
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow",
+        return_value=points[0],
+    ):
+        # Attributes are different, state not
+        states.append(set_state("123", attributes={"attribute": 21.42}))
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow",
+        return_value=points[1],
+    ):
+        # state is different, attributes not
+        states.append(set_state("32", attributes={"attribute": 21.42}))
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow",
+        return_value=points[2],
+    ):
+        # everything is different
+        states.append(set_state("412", attributes={"attribute": 54.23}))
+
+    hist = get_significant_states(hass, start, significant_changes_only=True)
+
+    assert len(hist[entity_id]) == 2
+    assert states[0] not in hist[entity_id]
+    assert states[1] in hist[entity_id]
+    assert states[2] in hist[entity_id]
+
+    hist = get_significant_states(hass, start, significant_changes_only=False)
+
+    assert len(hist[entity_id]) == 3
+    assert states == hist[entity_id]
+
+
+def check_significant_states(hass, zero, four, states, config):
+    """Check if significant states are retrieved."""
+    filters = history.Filters()
+    exclude = config[history.DOMAIN].get(CONF_EXCLUDE)
+    if exclude:
+        filters.excluded_entities = exclude.get(CONF_ENTITIES, [])
+        filters.excluded_domains = exclude.get(CONF_DOMAINS, [])
+    include = config[history.DOMAIN].get(CONF_INCLUDE)
+    if include:
+        filters.included_entities = include.get(CONF_ENTITIES, [])
+        filters.included_domains = include.get(CONF_DOMAINS, [])
+
+    hist = get_significant_states(hass, zero, four, filters=filters)
+    assert states == hist
+
+
+def record_states(hass):
+    """Record some test states.
+
+    We inject a bunch of state updates from media player, zone and
+    thermostat.
+    """
+    mp = "media_player.test"
+    mp2 = "media_player.test2"
+    mp3 = "media_player.test3"
+    therm = "thermostat.test"
+    therm2 = "thermostat.test2"
+    zone = "zone.home"
+    script_c = "script.can_cancel_this_one"
+
+    def set_state(entity_id, state, **kwargs):
+        """Set the state."""
+        hass.states.set(entity_id, state, **kwargs)
+        wait_recording_done(hass)
+        return hass.states.get(entity_id)
+
+    zero = dt_util.utcnow()
+    one = zero + timedelta(seconds=1)
+    two = one + timedelta(seconds=1)
+    three = two + timedelta(seconds=1)
+    four = three + timedelta(seconds=1)
+
+    states = {therm: [], therm2: [], mp: [], mp2: [], mp3: [], script_c: []}
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=one
+    ):
+        states[mp].append(
+            set_state(mp, "idle", attributes={"media_title": str(sentinel.mt1)})
+        )
+        states[mp].append(
+            set_state(mp, "YouTube", attributes={"media_title": str(sentinel.mt2)})
+        )
+        states[mp2].append(
+            set_state(mp2, "YouTube", attributes={"media_title": str(sentinel.mt2)})
+        )
+        states[mp3].append(
+            set_state(mp3, "idle", attributes={"media_title": str(sentinel.mt1)})
+        )
+        states[therm].append(
+            set_state(therm, 20, attributes={"current_temperature": 19.5})
+        )
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=two
+    ):
+        # This state will be skipped only different in time
+        set_state(mp, "YouTube", attributes={"media_title": str(sentinel.mt3)})
+        # This state will be skipped because domain is excluded
+        set_state(zone, "zoning")
+        states[script_c].append(
+            set_state(script_c, "off", attributes={"can_cancel": True})
+        )
+        states[therm].append(
+            set_state(therm, 21, attributes={"current_temperature": 19.8})
+        )
+        states[therm2].append(
+            set_state(therm2, 20, attributes={"current_temperature": 19})
+        )
+
+    with patch(
+        "homeassistant.components.recorder.core.dt_util.utcnow", return_value=three
+    ):
+        states[mp].append(
+            set_state(mp, "Netflix", attributes={"media_title": str(sentinel.mt4)})
+        )
+        states[mp3].append(
+            set_state(mp3, "Netflix", attributes={"media_title": str(sentinel.mt3)})
+        )
+        # Attributes changed even though state is the same
+        states[therm].append(
+            set_state(therm, 21, attributes={"current_temperature": 20})
+        )
+
+    return zero, four, states
+
+
+async def test_fetch_period_api(recorder_mock, hass, hass_client):
+    """Test the fetch period view for history."""
+    await async_setup_component(hass, "history", {})
+    client = await hass_client()
+    response = await client.get(f"/api/history/period/{dt_util.utcnow().isoformat()}")
+    assert response.status == HTTPStatus.OK
+
+
+async def test_fetch_period_api_with_use_include_order(
+    recorder_mock, hass, hass_client
+):
+    """Test the fetch period view for history with include order."""
+    await async_setup_component(
+        hass, "history", {history.DOMAIN: {history.CONF_ORDER: True}}
+    )
+    client = await hass_client()
+    response = await client.get(f"/api/history/period/{dt_util.utcnow().isoformat()}")
+    assert response.status == HTTPStatus.OK
+
+
+async def test_fetch_period_api_with_minimal_response(recorder_mock, hass, hass_client):
+    """Test the fetch period view for history with minimal_response."""
+    now = dt_util.utcnow()
+    await async_setup_component(hass, "history", {})
+
+    hass.states.async_set("sensor.power", 0, {"attr": "any"})
+    await async_wait_recording_done(hass)
+    hass.states.async_set("sensor.power", 50, {"attr": "any"})
+    await async_wait_recording_done(hass)
+    hass.states.async_set("sensor.power", 23, {"attr": "any"})
+    last_changed = hass.states.get("sensor.power").last_changed
+    await async_wait_recording_done(hass)
+    hass.states.async_set("sensor.power", 23, {"attr": "any"})
+    await async_wait_recording_done(hass)
+    client = await hass_client()
+    response = await client.get(
+        f"/api/history/period/{now.isoformat()}?filter_entity_id=sensor.power&minimal_response&no_attributes"
+    )
+    assert response.status == HTTPStatus.OK
+    response_json = await response.json()
+    assert len(response_json[0]) == 3
+    state_list = response_json[0]
+
+    assert state_list[0]["entity_id"] == "sensor.power"
+    assert state_list[0]["attributes"] == {}
+    assert state_list[0]["state"] == "0"
+
+    assert "attributes" not in state_list[1]
+    assert "entity_id" not in state_list[1]
+    assert state_list[1]["state"] == "50"
+
+    assert "attributes" not in state_list[2]
+    assert "entity_id" not in state_list[2]
+    assert state_list[2]["state"] == "23"
+    assert state_list[2]["last_changed"] == json.dumps(
+        process_timestamp(last_changed),
+        cls=JSONEncoder,
+    ).replace('"', "")
+
+
+async def test_fetch_period_api_with_no_timestamp(recorder_mock, hass, hass_client):
+    """Test the fetch period view for history with no timestamp."""
+    await async_setup_component(hass, "history", {})
+    client = await hass_client()
+    response = await client.get("/api/history/period")
+    assert response.status == HTTPStatus.OK
+
+
+async def test_fetch_period_api_with_include_order(recorder_mock, hass, hass_client):
+    """Test the fetch period view for history."""
+    await async_setup_component(
+        hass,
+        "history",
+        {
+            "history": {
+                "use_include_order": True,
+                "include": {"entities": ["light.kitchen"]},
+            }
+        },
+    )
+    client = await hass_client()
+    response = await client.get(
+        f"/api/history/period/{dt_util.utcnow().isoformat()}",
+        params={"filter_entity_id": "non.existing,something.else"},
+    )
+    assert response.status == HTTPStatus.OK
+
+
+async def test_fetch_period_api_with_entity_glob_include(
+    recorder_mock, hass, hass_client
+):
+    """Test the fetch period view for history."""
+    await async_setup_component(
+        hass,
+        "history",
+        {
+            "history": {
+                "include": {"entity_globs": ["light.k*"]},
+            }
+        },
+    )
+    hass.states.async_set("light.kitchen", "on")
+    hass.states.async_set("light.cow", "on")
+    hass.states.async_set("light.nomatch", "on")
+
+    await async_wait_recording_done(hass)
+
+    client = await hass_client()
+    response = await client.get(
+        f"/api/history/period/{dt_util.utcnow().isoformat()}",
+    )
+    assert response.status == HTTPStatus.OK
+    response_json = await response.json()
+    assert response_json[0][0]["entity_id"] == "light.kitchen"
+
+
+async def test_fetch_period_api_with_entity_glob_exclude(
+    recorder_mock, hass, hass_client
+):
+    """Test the fetch period view for history."""
+    await async_setup_component(
+        hass,
+        "history",
+        {
+            "history": {
+                "exclude": {
+                    "entity_globs": ["light.k*", "binary_sensor.*_?"],
+                    "domains": "switch",
+                    "entities": "media_player.test",
+                },
+            }
+        },
+    )
+    hass.states.async_set("light.kitchen", "on")
+    hass.states.async_set("light.cow", "on")
+    hass.states.async_set("light.match", "on")
+    hass.states.async_set("switch.match", "on")
+    hass.states.async_set("media_player.test", "on")
+    hass.states.async_set("binary_sensor.sensor_l", "on")
+    hass.states.async_set("binary_sensor.sensor_r", "on")
+    hass.states.async_set("binary_sensor.sensor", "on")
+
+    await async_wait_recording_done(hass)
+
+    client = await hass_client()
+    response = await client.get(
+        f"/api/history/period/{dt_util.utcnow().isoformat()}",
+    )
+    assert response.status == HTTPStatus.OK
+    response_json = await response.json()
+    assert len(response_json) == 3
+    assert response_json[0][0]["entity_id"] == "binary_sensor.sensor"
+    assert response_json[1][0]["entity_id"] == "light.cow"
+    assert response_json[2][0]["entity_id"] == "light.match"
+
+
+async def test_fetch_period_api_with_entity_glob_include_and_exclude(
+    recorder_mock, hass, hass_client
+):
+    """Test the fetch period view for history."""
+    await async_setup_component(
+        hass,
+        "history",
+        {
+            "history": {
+                "exclude": {
+                    "entity_globs": ["light.many*", "binary_sensor.*"],
+                },
+                "include": {
+                    "entity_globs": ["light.m*"],
+                    "domains": "switch",
+                    "entities": "media_player.test",
+                },
+            }
+        },
+    )
+    hass.states.async_set("light.kitchen", "on")
+    hass.states.async_set("light.cow", "on")
+    hass.states.async_set("light.match", "on")
+    hass.states.async_set("light.many_state_changes", "on")
+    hass.states.async_set("switch.match", "on")
+    hass.states.async_set("media_player.test", "on")
+    hass.states.async_set("binary_sensor.exclude", "on")
+
+    await async_wait_recording_done(hass)
+
+    client = await hass_client()
+    response = await client.get(
+        f"/api/history/period/{dt_util.utcnow().isoformat()}",
+    )
+    assert response.status == HTTPStatus.OK
+    response_json = await response.json()
+    assert len(response_json) == 4
+    assert response_json[0][0]["entity_id"] == "light.many_state_changes"
+    assert response_json[1][0]["entity_id"] == "light.match"
+    assert response_json[2][0]["entity_id"] == "media_player.test"
+    assert response_json[3][0]["entity_id"] == "switch.match"
+
+
+async def test_entity_ids_limit_via_api(recorder_mock, hass, hass_client):
+    """Test limiting history to entity_ids."""
+    await async_setup_component(
+        hass,
+        "history",
+        {"history": {}},
+    )
+    hass.states.async_set("light.kitchen", "on")
+    hass.states.async_set("light.cow", "on")
+    hass.states.async_set("light.nomatch", "on")
+
+    await async_wait_recording_done(hass)
+
+    client = await hass_client()
+    response = await client.get(
+        f"/api/history/period/{dt_util.utcnow().isoformat()}?filter_entity_id=light.kitchen,light.cow",
+    )
+    assert response.status == HTTPStatus.OK
+    response_json = await response.json()
+    assert len(response_json) == 2
+    assert response_json[0][0]["entity_id"] == "light.kitchen"
+    assert response_json[1][0]["entity_id"] == "light.cow"
+
+
+async def test_entity_ids_limit_via_api_with_skip_initial_state(
+    recorder_mock, hass, hass_client
+):
+    """Test limiting history to entity_ids with skip_initial_state."""
+    await async_setup_component(
+        hass,
+        "history",
+        {"history": {}},
+    )
+    hass.states.async_set("light.kitchen", "on")
+    hass.states.async_set("light.cow", "on")
+    hass.states.async_set("light.nomatch", "on")
+
+    await async_wait_recording_done(hass)
+
+    client = await hass_client()
+    response = await client.get(
+        f"/api/history/period/{dt_util.utcnow().isoformat()}?filter_entity_id=light.kitchen,light.cow&skip_initial_state",
+    )
+    assert response.status == HTTPStatus.OK
+    response_json = await response.json()
+    assert len(response_json) == 0
+
+    when = dt_util.utcnow() - timedelta(minutes=1)
+    response = await client.get(
+        f"/api/history/period/{when.isoformat()}?filter_entity_id=light.kitchen,light.cow&skip_initial_state",
+    )
+    assert response.status == HTTPStatus.OK
+    response_json = await response.json()
+    assert len(response_json) == 2
+    assert response_json[0][0]["entity_id"] == "light.kitchen"
+    assert response_json[1][0]["entity_id"] == "light.cow"
+
+
+async def test_history_during_period(recorder_mock, hass, hass_ws_client):
+    """Test history_during_period."""
+    now = dt_util.utcnow()
+
+    await async_setup_component(hass, "history", {})
+    await async_setup_component(hass, "sensor", {})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.test", "on", attributes={"any": "attr"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.test", "off", attributes={"any": "attr"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.test", "off", attributes={"any": "changed"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.test", "off", attributes={"any": "again"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.test", "on", attributes={"any": "attr"})
+    await async_wait_recording_done(hass)
+
+    await async_wait_recording_done(hass)
+
+    client = await hass_ws_client()
+    await client.send_json(
+        {
+            "id": 1,
+            "type": "history/history_during_period",
+            "start_time": now.isoformat(),
+            "end_time": now.isoformat(),
+            "entity_ids": ["sensor.test"],
+            "include_start_time_state": True,
+            "significant_changes_only": False,
+            "no_attributes": True,
+        }
+    )
+    response = await client.receive_json()
+    assert response["success"]
+    assert response["result"] == {}
+
+    await client.send_json(
+        {
+            "id": 2,
+            "type": "history/history_during_period",
+            "start_time": now.isoformat(),
+            "entity_ids": ["sensor.test"],
+            "include_start_time_state": True,
+            "significant_changes_only": False,
+            "no_attributes": True,
+            "minimal_response": True,
+        }
+    )
+    response = await client.receive_json()
+    assert response["success"]
+    assert response["id"] == 2
+
+    sensor_test_history = response["result"]["sensor.test"]
+    assert len(sensor_test_history) == 3
+
+    assert sensor_test_history[0]["s"] == "on"
+    assert sensor_test_history[0]["a"] == {}
+    assert isinstance(sensor_test_history[0]["lu"], float)
+    assert "lc" not in sensor_test_history[0]  # skipped if the same a last_updated (lu)
+
+    assert "a" not in sensor_test_history[1]
+    assert sensor_test_history[1]["s"] == "off"
+    assert isinstance(sensor_test_history[1]["lu"], float)
+    assert "lc" not in sensor_test_history[1]  # skipped if the same a last_updated (lu)
+
+    assert sensor_test_history[2]["s"] == "on"
+    assert "a" not in sensor_test_history[2]
+
+    await client.send_json(
+        {
+            "id": 3,
+            "type": "history/history_during_period",
+            "start_time": now.isoformat(),
+            "entity_ids": ["sensor.test"],
+            "include_start_time_state": True,
+            "significant_changes_only": False,
+            "no_attributes": False,
+        }
+    )
+    response = await client.receive_json()
+    assert response["success"]
+    assert response["id"] == 3
+    sensor_test_history = response["result"]["sensor.test"]
+
+    assert len(sensor_test_history) == 5
+
+    assert sensor_test_history[0]["s"] == "on"
+    assert sensor_test_history[0]["a"] == {"any": "attr"}
+    assert isinstance(sensor_test_history[0]["lu"], float)
+    assert "lc" not in sensor_test_history[0]  # skipped if the same a last_updated (lu)
+
+    assert sensor_test_history[1]["s"] == "off"
+    assert isinstance(sensor_test_history[1]["lu"], float)
+    assert "lc" not in sensor_test_history[1]  # skipped if the same a last_updated (lu)
+    assert sensor_test_history[1]["a"] == {"any": "attr"}
+
+    assert sensor_test_history[4]["s"] == "on"
+    assert sensor_test_history[4]["a"] == {"any": "attr"}
+
+    await client.send_json(
+        {
+            "id": 4,
+            "type": "history/history_during_period",
+            "start_time": now.isoformat(),
+            "entity_ids": ["sensor.test"],
+            "include_start_time_state": True,
+            "significant_changes_only": True,
+            "no_attributes": False,
+        }
+    )
+    response = await client.receive_json()
+    assert response["success"]
+    assert response["id"] == 4
+    sensor_test_history = response["result"]["sensor.test"]
+
+    assert len(sensor_test_history) == 3
+
+    assert sensor_test_history[0]["s"] == "on"
+    assert sensor_test_history[0]["a"] == {"any": "attr"}
+    assert isinstance(sensor_test_history[0]["lu"], float)
+    assert "lc" not in sensor_test_history[0]  # skipped if the same a last_updated (lu)
+
+    assert sensor_test_history[1]["s"] == "off"
+    assert isinstance(sensor_test_history[1]["lu"], float)
+    assert "lc" not in sensor_test_history[1]  # skipped if the same a last_updated (lu)
+    assert sensor_test_history[1]["a"] == {"any": "attr"}
+
+    assert sensor_test_history[2]["s"] == "on"
+    assert sensor_test_history[2]["a"] == {"any": "attr"}
+
+
+async def test_history_during_period_impossible_conditions(
+    recorder_mock, hass, hass_ws_client
+):
+    """Test history_during_period returns when condition cannot be true."""
+    await async_setup_component(hass, "history", {})
+    await async_setup_component(hass, "sensor", {})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.test", "on", attributes={"any": "attr"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.test", "off", attributes={"any": "attr"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.test", "off", attributes={"any": "changed"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.test", "off", attributes={"any": "again"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.test", "on", attributes={"any": "attr"})
+    await async_wait_recording_done(hass)
+
+    await async_wait_recording_done(hass)
+
+    after = dt_util.utcnow()
+
+    client = await hass_ws_client()
+    await client.send_json(
+        {
+            "id": 1,
+            "type": "history/history_during_period",
+            "start_time": after.isoformat(),
+            "end_time": after.isoformat(),
+            "entity_ids": ["sensor.test"],
+            "include_start_time_state": False,
+            "significant_changes_only": False,
+            "no_attributes": True,
+        }
+    )
+    response = await client.receive_json()
+    assert response["success"]
+    assert response["id"] == 1
+    assert response["result"] == {}
+
+    future = dt_util.utcnow() + timedelta(hours=10)
+
+    await client.send_json(
+        {
+            "id": 2,
+            "type": "history/history_during_period",
+            "start_time": future.isoformat(),
+            "entity_ids": ["sensor.test"],
+            "include_start_time_state": True,
+            "significant_changes_only": True,
+            "no_attributes": True,
+        }
+    )
+    response = await client.receive_json()
+    assert response["success"]
+    assert response["id"] == 2
+    assert response["result"] == {}
+
+
+@pytest.mark.parametrize(
+    "time_zone", ["UTC", "Europe/Berlin", "America/Chicago", "US/Hawaii"]
+)
+async def test_history_during_period_significant_domain(
+    time_zone, recorder_mock, hass, hass_ws_client
+):
+    """Test history_during_period with climate domain."""
+    hass.config.set_time_zone(time_zone)
+    now = dt_util.utcnow()
+
+    await async_setup_component(hass, "history", {})
+    await async_setup_component(hass, "sensor", {})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("climate.test", "on", attributes={"temperature": "1"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("climate.test", "off", attributes={"temperature": "2"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("climate.test", "off", attributes={"temperature": "3"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("climate.test", "off", attributes={"temperature": "4"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("climate.test", "on", attributes={"temperature": "5"})
+    await async_wait_recording_done(hass)
+
+    await async_wait_recording_done(hass)
+
+    client = await hass_ws_client()
+    await client.send_json(
+        {
+            "id": 1,
+            "type": "history/history_during_period",
+            "start_time": now.isoformat(),
+            "end_time": now.isoformat(),
+            "entity_ids": ["climate.test"],
+            "include_start_time_state": True,
+            "significant_changes_only": False,
+            "no_attributes": True,
+        }
+    )
+    response = await client.receive_json()
+    assert response["success"]
+    assert response["result"] == {}
+
+    await client.send_json(
+        {
+            "id": 2,
+            "type": "history/history_during_period",
+            "start_time": now.isoformat(),
+            "entity_ids": ["climate.test"],
+            "include_start_time_state": True,
+            "significant_changes_only": False,
+            "no_attributes": True,
+            "minimal_response": True,
+        }
+    )
+    response = await client.receive_json()
+    assert response["success"]
+    assert response["id"] == 2
+
+    sensor_test_history = response["result"]["climate.test"]
+    assert len(sensor_test_history) == 5
+
+    assert sensor_test_history[0]["s"] == "on"
+    assert sensor_test_history[0]["a"] == {}
+    assert isinstance(sensor_test_history[0]["lu"], float)
+    assert "lc" not in sensor_test_history[0]  # skipped if the same a last_updated (lu)
+
+    assert "a" in sensor_test_history[1]
+    assert sensor_test_history[1]["s"] == "off"
+    assert "lc" not in sensor_test_history[1]  # skipped if the same a last_updated (lu)
+
+    assert sensor_test_history[4]["s"] == "on"
+    assert sensor_test_history[4]["a"] == {}
+
+    await client.send_json(
+        {
+            "id": 3,
+            "type": "history/history_during_period",
+            "start_time": now.isoformat(),
+            "entity_ids": ["climate.test"],
+            "include_start_time_state": True,
+            "significant_changes_only": False,
+            "no_attributes": False,
+        }
+    )
+    response = await client.receive_json()
+    assert response["success"]
+    assert response["id"] == 3
+    sensor_test_history = response["result"]["climate.test"]
+
+    assert len(sensor_test_history) == 5
+
+    assert sensor_test_history[0]["s"] == "on"
+    assert sensor_test_history[0]["a"] == {"temperature": "1"}
+    assert isinstance(sensor_test_history[0]["lu"], float)
+    assert "lc" not in sensor_test_history[0]  # skipped if the same a last_updated (lu)
+
+    assert sensor_test_history[1]["s"] == "off"
+    assert isinstance(sensor_test_history[1]["lu"], float)
+    assert "lc" not in sensor_test_history[1]  # skipped if the same a last_updated (lu)
+    assert sensor_test_history[1]["a"] == {"temperature": "2"}
+
+    assert sensor_test_history[4]["s"] == "on"
+    assert sensor_test_history[4]["a"] == {"temperature": "5"}
+
+    await client.send_json(
+        {
+            "id": 4,
+            "type": "history/history_during_period",
+            "start_time": now.isoformat(),
+            "entity_ids": ["climate.test"],
+            "include_start_time_state": True,
+            "significant_changes_only": True,
+            "no_attributes": False,
+        }
+    )
+    response = await client.receive_json()
+    assert response["success"]
+    assert response["id"] == 4
+    sensor_test_history = response["result"]["climate.test"]
+
+    assert len(sensor_test_history) == 5
+
+    assert sensor_test_history[0]["s"] == "on"
+    assert sensor_test_history[0]["a"] == {"temperature": "1"}
+    assert isinstance(sensor_test_history[0]["lu"], float)
+    assert "lc" not in sensor_test_history[0]  # skipped if the same a last_updated (lu)
+
+    assert sensor_test_history[1]["s"] == "off"
+    assert isinstance(sensor_test_history[1]["lu"], float)
+    assert "lc" not in sensor_test_history[1]  # skipped if the same a last_updated (lu)
+    assert sensor_test_history[1]["a"] == {"temperature": "2"}
+
+    assert sensor_test_history[2]["s"] == "off"
+    assert sensor_test_history[2]["a"] == {"temperature": "3"}
+
+    assert sensor_test_history[3]["s"] == "off"
+    assert sensor_test_history[3]["a"] == {"temperature": "4"}
+
+    assert sensor_test_history[4]["s"] == "on"
+    assert sensor_test_history[4]["a"] == {"temperature": "5"}
+
+    # Test we impute the state time state
+    later = dt_util.utcnow()
+    await client.send_json(
+        {
+            "id": 5,
+            "type": "history/history_during_period",
+            "start_time": later.isoformat(),
+            "entity_ids": ["climate.test"],
+            "include_start_time_state": True,
+            "significant_changes_only": True,
+            "no_attributes": False,
+        }
+    )
+    response = await client.receive_json()
+    assert response["success"]
+    assert response["id"] == 5
+    sensor_test_history = response["result"]["climate.test"]
+
+    assert len(sensor_test_history) == 1
+
+    assert sensor_test_history[0]["s"] == "on"
+    assert sensor_test_history[0]["a"] == {"temperature": "5"}
+    assert sensor_test_history[0]["lu"] == later.timestamp()
+    assert "lc" not in sensor_test_history[0]  # skipped if the same a last_updated (lu)
+
+
+async def test_history_during_period_bad_start_time(
+    recorder_mock, hass, hass_ws_client
+):
+    """Test history_during_period bad state time."""
+    await async_setup_component(
+        hass,
+        "history",
+        {"history": {}},
+    )
+
+    client = await hass_ws_client()
+    await client.send_json(
+        {
+            "id": 1,
+            "type": "history/history_during_period",
+            "start_time": "cats",
+        }
+    )
+    response = await client.receive_json()
+    assert not response["success"]
+    assert response["error"]["code"] == "invalid_start_time"
+
+
+async def test_history_during_period_bad_end_time(recorder_mock, hass, hass_ws_client):
+    """Test history_during_period bad end time."""
+    now = dt_util.utcnow()
+
+    await async_setup_component(
+        hass,
+        "history",
+        {"history": {}},
+    )
+
+    client = await hass_ws_client()
+    await client.send_json(
+        {
+            "id": 1,
+            "type": "history/history_during_period",
+            "start_time": now.isoformat(),
+            "end_time": "dogs",
+        }
+    )
+    response = await client.receive_json()
+    assert not response["success"]
+    assert response["error"]["code"] == "invalid_end_time"
+
+
+async def test_history_during_period_with_use_include_order(
+    recorder_mock, hass, hass_ws_client
+):
+    """Test history_during_period."""
+    now = dt_util.utcnow()
+    sort_order = ["sensor.two", "sensor.four", "sensor.one"]
+    await async_setup_component(
+        hass,
+        "history",
+        {
+            history.DOMAIN: {
+                history.CONF_ORDER: True,
+                CONF_INCLUDE: {
+                    CONF_ENTITIES: sort_order,
+                    CONF_DOMAINS: ["sensor"],
+                },
+            }
+        },
+    )
+    await async_setup_component(hass, "sensor", {})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.one", "on", attributes={"any": "attr"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.two", "off", attributes={"any": "attr"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.three", "off", attributes={"any": "changed"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.four", "off", attributes={"any": "again"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("switch.excluded", "off", attributes={"any": "again"})
+    await async_wait_recording_done(hass)
+
+    await async_wait_recording_done(hass)
+
+    client = await hass_ws_client()
+    await client.send_json(
+        {
+            "id": 1,
+            "type": "history/history_during_period",
+            "start_time": now.isoformat(),
+            "include_start_time_state": True,
+            "significant_changes_only": False,
+            "no_attributes": True,
+            "minimal_response": True,
+        }
+    )
+    response = await client.receive_json()
+    assert response["success"]
+    assert response["id"] == 1
+
+    assert list(response["result"]) == [
+        *sort_order,
+        "sensor.three",
+    ]

From acebe899e827e79391e6f02849d9201650a1ce74 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Sun, 1 Jan 2023 19:19:10 -1000
Subject: [PATCH 37/38] small cleanups

---
 homeassistant/components/recorder/history.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/homeassistant/components/recorder/history.py b/homeassistant/components/recorder/history.py
index a78fa468d9417d..e91e7d6ab9bebe 100644
--- a/homeassistant/components/recorder/history.py
+++ b/homeassistant/components/recorder/history.py
@@ -590,7 +590,7 @@ def _get_states_for_entites_stmt(
     # in the inner query.
     if schema_version >= 31:
         run_start_ts = run_start.timestamp()
-        utc_point_in_time_ts = utc_point_in_time.timestamp()
+        utc_point_in_time_ts = dt_util.utc_to_timestamp(utc_point_in_time)
         stmt += lambda q: q.where(
             States.state_id
             == (
@@ -633,7 +633,7 @@ def _generate_most_recent_states_by_date(
     """Generate the sub query for the most recent states by data."""
     if schema_version >= 31:
         run_start_ts = run_start.timestamp()
-        utc_point_in_time_ts = utc_point_in_time.timestamp()
+        utc_point_in_time_ts = dt_util.utc_to_timestamp(utc_point_in_time)
         return (
             select(
                 States.entity_id.label("max_entity_id"),
@@ -775,7 +775,7 @@ def _get_single_entity_states_stmt(
         schema_version, no_attributes, include_last_changed=True
     )
     if schema_version >= 31:
-        utc_point_in_time_ts = utc_point_in_time.timestamp()
+        utc_point_in_time_ts = dt_util.utc_to_timestamp(utc_point_in_time)
         stmt += (
             lambda q: q.filter(
                 States.last_updated_ts < utc_point_in_time_ts,

From 294f4312fca0dc1b9672f20b551e9e4519ddfa49 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@koston.org>
Date: Mon, 2 Jan 2023 09:28:07 -1000
Subject: [PATCH 38/38] more cover

---
 homeassistant/components/recorder/history.py | 48 +++++++++-----------
 tests/components/recorder/test_history.py    | 27 ++++++++++-
 tests/components/recorder/test_models.py     | 21 +++++++++
 3 files changed, 69 insertions(+), 27 deletions(-)

diff --git a/homeassistant/components/recorder/history.py b/homeassistant/components/recorder/history.py
index e91e7d6ab9bebe..095a4e55b703f9 100644
--- a/homeassistant/components/recorder/history.py
+++ b/homeassistant/components/recorder/history.py
@@ -912,42 +912,38 @@ def _sorted_states_to_dict(
         # changes so we can filter out duplicate states
         if schema_version < 31:
             for row in group:
-                if (state := row.state) == prev_state:
-                    continue
-                ent_results.append(
-                    {
-                        attr_state: state,
-                        attr_time: _process_timestamp(row.last_updated),
-                    }
-                )
-                prev_state = state
+                if (state := row.state) != prev_state:
+                    ent_results.append(
+                        {
+                            attr_state: state,
+                            attr_time: _process_timestamp(row.last_updated),
+                        }
+                    )
+                    prev_state = state
             continue
 
         if compressed_state_format:
             for row in group:
-                if (state := row.state) == prev_state:
-                    continue
+                if (state := row.state) != prev_state:
+                    ent_results.append(
+                        {
+                            attr_state: state,
+                            attr_time: row.last_updated_ts,
+                        }
+                    )
+                    prev_state = state
+
+        for row in group:
+            if (state := row.state) != prev_state:
                 ent_results.append(
                     {
                         attr_state: state,
-                        attr_time: row.last_updated_ts,
+                        attr_time: process_timestamp_to_utc_isoformat(
+                            dt_util.utc_from_timestamp(row.last_updated_ts)
+                        ),
                     }
                 )
                 prev_state = state
-            continue
-
-        for row in group:
-            if (state := row.state) == prev_state:
-                continue
-            ent_results.append(
-                {
-                    attr_state: state,
-                    attr_time: process_timestamp_to_utc_isoformat(
-                        dt_util.utc_from_timestamp(row.last_updated_ts)
-                    ),
-                }
-            )
-            prev_state = state
 
     # If there are no states beyond the initial state,
     # the state a was never popped from initial_states
diff --git a/tests/components/recorder/test_history.py b/tests/components/recorder/test_history.py
index 014ef1bd5905f5..913ae3d8bf603c 100644
--- a/tests/components/recorder/test_history.py
+++ b/tests/components/recorder/test_history.py
@@ -30,7 +30,11 @@
 from homeassistant.helpers.json import JSONEncoder
 import homeassistant.util.dt as dt_util
 
-from .common import async_wait_recording_done, wait_recording_done
+from .common import (
+    async_recorder_block_till_done,
+    async_wait_recording_done,
+    wait_recording_done,
+)
 
 from tests.common import SetupRecorderInstanceT, mock_state_change_event
 
@@ -588,6 +592,27 @@ def set_state(state, **kwargs):
     assert states == hist[entity_id]
 
 
+async def test_get_significant_states_only_minimal_response(recorder_mock, hass):
+    """Test significant states when significant_states_only is True."""
+    now = dt_util.utcnow()
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.test", "on", attributes={"any": "attr"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.test", "off", attributes={"any": "attr"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.test", "off", attributes={"any": "changed"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.test", "off", attributes={"any": "again"})
+    await async_recorder_block_till_done(hass)
+    hass.states.async_set("sensor.test", "on", attributes={"any": "attr"})
+    await async_wait_recording_done(hass)
+
+    hist = history.get_significant_states(
+        hass, now, minimal_response=True, significant_changes_only=False
+    )
+    assert len(hist["sensor.test"]) == 3
+
+
 def record_states(hass) -> tuple[datetime, datetime, dict[str, list[State]]]:
     """Record some test states.
 
diff --git a/tests/components/recorder/test_models.py b/tests/components/recorder/test_models.py
index c8e1cc3a25b477..2e823bc19e9ebf 100644
--- a/tests/components/recorder/test_models.py
+++ b/tests/components/recorder/test_models.py
@@ -58,6 +58,27 @@ def test_from_event_to_db_state_attributes():
     assert StateAttributes.from_event(event).to_native() == attrs
 
 
+def test_repr():
+    """Test converting event to db state repr."""
+    attrs = {"this_attr": True}
+    fixed_time = datetime(2016, 7, 9, 11, 0, 0, tzinfo=dt.UTC, microsecond=432432)
+    state = ha.State(
+        "sensor.temperature",
+        "18",
+        attrs,
+        last_changed=fixed_time,
+        last_updated=fixed_time,
+    )
+    event = ha.Event(
+        EVENT_STATE_CHANGED,
+        {"entity_id": "sensor.temperature", "old_state": None, "new_state": state},
+        context=state.context,
+        time_fired=fixed_time,
+    )
+    assert "2016-07-09 11:00:00+00:00" in repr(States.from_event(event))
+    assert "2016-07-09 11:00:00+00:00" in repr(Events.from_event(event))
+
+
 def test_handling_broken_json_state_attributes(caplog):
     """Test we handle broken json in state attributes."""
     state_attributes = StateAttributes(