From 3ee4470a52c8d29cddb3c3ceed0fbcbdc2522033 Mon Sep 17 00:00:00 2001 From: moinmoin-sh <53935853+moinmoin-sh@users.noreply.github.com> Date: Tue, 24 Nov 2020 19:21:10 +0100 Subject: [PATCH 1/6] MariaDB doesn't purge #42402 This addresses home-assistant#42402 Relationships within table "states" and between tables "states" and "events " home-assistant#40467 prevent the purge from working correctly. The database increases w/o any purge. This proposal sets related indices to NULL and permits deleting of rows. Further explanations can be found here home-assistant#42402 This proposal also allows to purge the tables "events" and "states" in any order. --- homeassistant/components/recorder/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/homeassistant/components/recorder/models.py b/homeassistant/components/recorder/models.py index 4756ac13ce300..36af120129c82 100644 --- a/homeassistant/components/recorder/models.py +++ b/homeassistant/components/recorder/models.py @@ -102,11 +102,11 @@ class States(Base): # type: ignore entity_id = Column(String(255)) state = Column(String(255)) attributes = Column(Text) - event_id = Column(Integer, ForeignKey("events.event_id"), index=True) + event_id = Column(Integer, ForeignKey("events.event_id", ondelete="SET NULL"), index=True) last_changed = Column(DateTime(timezone=True), default=dt_util.utcnow) last_updated = Column(DateTime(timezone=True), default=dt_util.utcnow, index=True) created = Column(DateTime(timezone=True), default=dt_util.utcnow) - old_state_id = Column(Integer, ForeignKey("states.state_id")) + old_state_id = Column(Integer, ForeignKey("states.state_id", ondelete="SET NULL"), index=True) event = relationship("Events", uselist=False) old_state = relationship("States", remote_side=[state_id]) From 31778047252ba7812be26ad09cd48c897d7d1cb2 Mon Sep 17 00:00:00 2001 From: moinmoin-sh <53935853+moinmoin-sh@users.noreply.github.com> Date: Wed, 25 Nov 2020 18:09:01 +0100 Subject: [PATCH 2/6] Update models.py Corrected for Black style requirements --- homeassistant/components/recorder/models.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/homeassistant/components/recorder/models.py b/homeassistant/components/recorder/models.py index 36af120129c82..ab50337cbf353 100644 --- a/homeassistant/components/recorder/models.py +++ b/homeassistant/components/recorder/models.py @@ -102,11 +102,15 @@ class States(Base): # type: ignore entity_id = Column(String(255)) state = Column(String(255)) attributes = Column(Text) - event_id = Column(Integer, ForeignKey("events.event_id", ondelete="SET NULL"), index=True) + event_id = Column( + Integer, ForeignKey("events.event_id", ondelete="SET NULL"), index=True + ) last_changed = Column(DateTime(timezone=True), default=dt_util.utcnow) last_updated = Column(DateTime(timezone=True), default=dt_util.utcnow, index=True) created = Column(DateTime(timezone=True), default=dt_util.utcnow) - old_state_id = Column(Integer, ForeignKey("states.state_id", ondelete="SET NULL"), index=True) + old_state_id = Column( + Integer, ForeignKey("states.state_id", ondelete="SET NULL"), index=True + ) event = relationship("Events", uselist=False) old_state = relationship("States", remote_side=[state_id]) From 9f95f15129054368dd10a0dead8c0133a79cb29a Mon Sep 17 00:00:00 2001 From: moinmoin-sh <53935853+moinmoin-sh@users.noreply.github.com> Date: Thu, 26 Nov 2020 20:27:21 +0100 Subject: [PATCH 3/6] Update homeassistant/components/recorder/models.py Co-authored-by: J. Nick Koston --- homeassistant/components/recorder/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/recorder/models.py b/homeassistant/components/recorder/models.py index ab50337cbf353..247c2c57705c9 100644 --- a/homeassistant/components/recorder/models.py +++ b/homeassistant/components/recorder/models.py @@ -103,7 +103,7 @@ class States(Base): # type: ignore state = Column(String(255)) attributes = Column(Text) event_id = Column( - Integer, ForeignKey("events.event_id", ondelete="SET NULL"), index=True + Integer, ForeignKey("events.event_id", ondelete="CASCADE"), index=True ) last_changed = Column(DateTime(timezone=True), default=dt_util.utcnow) last_updated = Column(DateTime(timezone=True), default=dt_util.utcnow, index=True) From 3b55e7bf724ec9d5d68f840d64546dbe4fac6786 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 26 Nov 2020 11:59:56 -1000 Subject: [PATCH 4/6] Add the options to foreign key constraints --- .../components/recorder/migration.py | 40 ++++++++++++++++++- homeassistant/components/recorder/models.py | 4 +- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py index e88852e4a5a0e..bd2fd04840367 100644 --- a/homeassistant/components/recorder/migration.py +++ b/homeassistant/components/recorder/migration.py @@ -1,12 +1,13 @@ """Schema migration helpers.""" import logging -from sqlalchemy import Table, text +from sqlalchemy import ForeignKeyConstraint, MetaData, Table, text from sqlalchemy.engine import reflection from sqlalchemy.exc import InternalError, OperationalError, SQLAlchemyError +from sqlalchemy.schema import AddConstraint, DropConstraint from .const import DOMAIN -from .models import SCHEMA_VERSION, Base, SchemaChanges +from .models import SCHEMA_VERSION, TABLE_STATES, Base, SchemaChanges from .util import session_scope _LOGGER = logging.getLogger(__name__) @@ -205,6 +206,39 @@ def _add_columns(engine, table_name, columns_def): ) +def _update_states_table_with_foreign_key_options(engine): + """Add the options to foreign key constraints.""" + inspector = reflection.Inspector.from_engine(engine) + alters = [] + for fk in inspector.get_foreign_keys(TABLE_STATES): + if fk["name"] and not fk["options"]: + alters.append( + { + "old_fk": ForeignKeyConstraint((), (), name=fk["name"]), + "columns": fk["constrained_columns"], + } + ) + + if not alters: + return + + states_key_constraints = Base.metadata.tables[TABLE_STATES].foreign_key_constraints + old_states_table = Table( # noqa: F841 + TABLE_STATES, MetaData(), *[alter["old_fk"] for alter in alters] + ) + + for alter in alters: + try: + engine.execute(DropConstraint(alter["old_fk"])) + for fkc in states_key_constraints: + if fkc.column_keys == alter["columns"]: + engine.execute(AddConstraint(fkc)) + except (InternalError, OperationalError): + _LOGGER.exception( + "Could not update foreign options in % table", TABLE_STATES + ) + + def _apply_update(engine, new_version, old_version): """Perform operations to bring schema up to date.""" if new_version == 1: @@ -277,6 +311,8 @@ def _apply_update(engine, new_version, old_version): _drop_index(engine, "states", "ix_states_entity_id") _create_index(engine, "events", "ix_events_event_type_time_fired") _drop_index(engine, "events", "ix_events_event_type") + elif new_version == 10: + _update_states_table_with_foreign_key_options(engine) else: raise ValueError(f"No schema migration defined for version {new_version}") diff --git a/homeassistant/components/recorder/models.py b/homeassistant/components/recorder/models.py index 247c2c57705c9..6c8b6050a9a5b 100644 --- a/homeassistant/components/recorder/models.py +++ b/homeassistant/components/recorder/models.py @@ -25,7 +25,7 @@ # pylint: disable=invalid-name Base = declarative_base() -SCHEMA_VERSION = 9 +SCHEMA_VERSION = 10 _LOGGER = logging.getLogger(__name__) @@ -36,7 +36,7 @@ TABLE_RECORDER_RUNS = "recorder_runs" TABLE_SCHEMA_CHANGES = "schema_changes" -ALL_TABLES = [TABLE_EVENTS, TABLE_STATES, TABLE_RECORDER_RUNS, TABLE_SCHEMA_CHANGES] +ALL_TABLES = [TABLE_STATES, TABLE_EVENTS, TABLE_RECORDER_RUNS, TABLE_SCHEMA_CHANGES] class Events(Base): # type: ignore From 106711a1c4533e429c60aaef81e51eda2f9d2903 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 26 Nov 2020 02:00:49 -1000 Subject: [PATCH 5/6] purge old states when database gets deleted out from under us --- homeassistant/components/recorder/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/homeassistant/components/recorder/__init__.py b/homeassistant/components/recorder/__init__.py index 18d364315b734..0f8a5ae7f8f91 100644 --- a/homeassistant/components/recorder/__init__.py +++ b/homeassistant/components/recorder/__init__.py @@ -514,6 +514,14 @@ def _commit_event_session(self): self.event_session.expunge(dbstate) self._pending_expunge = [] self.event_session.commit() + except exc.IntegrityError as err: + _LOGGER.error( + "Integrity error executing query (database likely deleted out from under us): %s", + err, + ) + self.event_session.rollback() + self._old_states = {} + raise except Exception as err: _LOGGER.error("Error executing query: %s", err) self.event_session.rollback() From 16432e7c0860c7cd8baea0076847d7cf8e65f9bd Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 26 Nov 2020 14:32:48 -1000 Subject: [PATCH 6/6] pylint --- homeassistant/components/recorder/migration.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/homeassistant/components/recorder/migration.py b/homeassistant/components/recorder/migration.py index bd2fd04840367..c633c114b4614 100644 --- a/homeassistant/components/recorder/migration.py +++ b/homeassistant/components/recorder/migration.py @@ -210,12 +210,12 @@ def _update_states_table_with_foreign_key_options(engine): """Add the options to foreign key constraints.""" inspector = reflection.Inspector.from_engine(engine) alters = [] - for fk in inspector.get_foreign_keys(TABLE_STATES): - if fk["name"] and not fk["options"]: + for foreign_key in inspector.get_foreign_keys(TABLE_STATES): + if foreign_key["name"] and not foreign_key["options"]: alters.append( { - "old_fk": ForeignKeyConstraint((), (), name=fk["name"]), - "columns": fk["constrained_columns"], + "old_fk": ForeignKeyConstraint((), (), name=foreign_key["name"]), + "columns": foreign_key["constrained_columns"], } ) @@ -223,7 +223,7 @@ def _update_states_table_with_foreign_key_options(engine): return states_key_constraints = Base.metadata.tables[TABLE_STATES].foreign_key_constraints - old_states_table = Table( # noqa: F841 + old_states_table = Table( # noqa: F841 pylint: disable=unused-variable TABLE_STATES, MetaData(), *[alter["old_fk"] for alter in alters] ) @@ -235,7 +235,7 @@ def _update_states_table_with_foreign_key_options(engine): engine.execute(AddConstraint(fkc)) except (InternalError, OperationalError): _LOGGER.exception( - "Could not update foreign options in % table", TABLE_STATES + "Could not update foreign options in %s table", TABLE_STATES )