Skip to content

Setup recorder model relationships to avoid calling flush #40467

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
balloob merged 2 commits into from
Sep 30, 2020
Merged

Setup recorder model relationships to avoid calling flush #40467

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6ec00db
Setup recorder model relationships to avoid calling flush
bdraco Sep 22, 2020
23b98d7
convert legacy
bdraco Sep 28, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 5 additions & 8 deletions homeassistant/components/recorder/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ def __init__(

self._timechanges_seen = 0
self._keepalive_count = 0
self._old_state_ids = {}
self._old_states = {}
self.event_session = None
self.get_session = None
self._completed_database_setup = False
Expand Down Expand Up @@ -385,7 +385,6 @@ def async_purge(now):
if event.event_type == EVENT_STATE_CHANGED:
dbevent.event_data = "{}"
self.event_session.add(dbevent)
self.event_session.flush()
except (TypeError, ValueError):
_LOGGER.warning("Event is not JSON serializable: %s", event)
except Exception as err: # pylint: disable=broad-except
Expand All @@ -396,16 +395,14 @@ def async_purge(now):
try:
dbstate = States.from_event(event)
has_new_state = event.data.get("new_state")
dbstate.old_state_id = self._old_state_ids.get(dbstate.entity_id)
if dbstate.entity_id in self._old_states:
dbstate.old_state = self._old_states.pop(dbstate.entity_id)
if not has_new_state:
dbstate.state = None
dbstate.event_id = dbevent.event_id
dbstate.event = dbevent
self.event_session.add(dbstate)
self.event_session.flush()
if has_new_state:
self._old_state_ids[dbstate.entity_id] = dbstate.state_id
elif dbstate.entity_id in self._old_state_ids:
del self._old_state_ids[dbstate.entity_id]
self._old_states[dbstate.entity_id] = dbstate
except (TypeError, ValueError):
_LOGGER.warning(
"State is not JSON serializable: %s",
Expand Down
5 changes: 4 additions & 1 deletion homeassistant/components/recorder/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
distinct,
)
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy.orm.session import Session

from homeassistant.core import Context, Event, EventOrigin, State, split_entity_id
Expand Down Expand Up @@ -105,7 +106,9 @@ class States(Base): # type: ignore
last_changed = Column(DateTime(timezone=True), default=dt_util.utcnow)
last_updated = Column(DateTime(timezone=True), default=dt_util.utcnow, index=True)
created = Column(DateTime(timezone=True), default=dt_util.utcnow)
old_state_id = Column(Integer)
old_state_id = Column(Integer, ForeignKey("states.state_id"))
event = relationship("Events", uselist=False)
old_state = relationship("States", remote_side=[state_id])

__table_args__ = (
# Used for fetching the state of entities at a specific time
Expand Down
77 changes: 41 additions & 36 deletions tests/components/logbook/test_init.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ def event_listener(event):
# Logbook entry service call results in firing an event.
# Our service call will unblock when the event listeners have been
# scheduled. This means that they may not have been processed yet.
trigger_db_commit(self.hass)
self.hass.block_till_done()
self.hass.data[recorder.DATA_INSTANCE].block_till_done()

Expand Down Expand Up @@ -935,7 +936,7 @@ async def test_logbook_view(hass, hass_client):
"""Test the logbook view."""
await hass.async_add_executor_job(init_recorder_component, hass)
await async_setup_component(hass, "logbook", {})
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
client = await hass_client()
response = await client.get(f"/api/logbook/{dt_util.utcnow().isoformat()}")
assert response.status == 200
Expand All @@ -945,17 +946,17 @@ async def test_logbook_view_period_entity(hass, hass_client):
"""Test the logbook view with period and entity."""
await hass.async_add_executor_job(init_recorder_component, hass)
await async_setup_component(hass, "logbook", {})
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

entity_id_test = "switch.test"
hass.states.async_set(entity_id_test, STATE_OFF)
hass.states.async_set(entity_id_test, STATE_ON)
entity_id_second = "switch.second"
hass.states.async_set(entity_id_second, STATE_OFF)
hass.states.async_set(entity_id_second, STATE_ON)
await hass.async_add_job(trigger_db_commit, hass)
await hass.async_add_executor_job(trigger_db_commit, hass)
await hass.async_block_till_done()
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

client = await hass_client()

Expand Down Expand Up @@ -1051,6 +1052,8 @@ def _describe(event):
):
hass.bus.async_fire("some_event")
await hass.async_block_till_done()
await hass.async_add_executor_job(trigger_db_commit, hass)
await hass.async_block_till_done()
await hass.async_add_executor_job(
hass.data[recorder.DATA_INSTANCE].block_till_done
)
Expand Down Expand Up @@ -1119,6 +1122,8 @@ def async_describe_events(hass, async_describe_event):
"some_event", {logbook.ATTR_NAME: name, logbook.ATTR_ENTITY_ID: entity_id3}
)
await hass.async_block_till_done()
await hass.async_add_executor_job(trigger_db_commit, hass)
await hass.async_block_till_done()
await hass.async_add_executor_job(
hass.data[recorder.DATA_INSTANCE].block_till_done
)
Expand All @@ -1138,17 +1143,17 @@ async def test_logbook_view_end_time_entity(hass, hass_client):
"""Test the logbook view with end_time and entity."""
await hass.async_add_executor_job(init_recorder_component, hass)
await async_setup_component(hass, "logbook", {})
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

entity_id_test = "switch.test"
hass.states.async_set(entity_id_test, STATE_OFF)
hass.states.async_set(entity_id_test, STATE_ON)
entity_id_second = "switch.second"
hass.states.async_set(entity_id_second, STATE_OFF)
hass.states.async_set(entity_id_second, STATE_ON)
await hass.async_add_job(trigger_db_commit, hass)
await hass.async_add_executor_job(trigger_db_commit, hass)
await hass.async_block_till_done()
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

client = await hass_client()

Expand Down Expand Up @@ -1199,7 +1204,7 @@ async def test_logbook_entity_filter_with_automations(hass, hass_client):
await async_setup_component(hass, "automation", {})
await async_setup_component(hass, "script", {})

await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

entity_id_test = "alarm_control_panel.area_001"
hass.states.async_set(entity_id_test, STATE_OFF)
Expand All @@ -1218,9 +1223,9 @@ async def test_logbook_entity_filter_with_automations(hass, hass_client):
)
hass.bus.async_fire(EVENT_HOMEASSISTANT_START)

await hass.async_add_job(trigger_db_commit, hass)
await hass.async_add_executor_job(trigger_db_commit, hass)
await hass.async_block_till_done()
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

client = await hass_client()

Expand Down Expand Up @@ -1271,7 +1276,7 @@ async def test_filter_continuous_sensor_values(hass, hass_client):
"""Test remove continuous sensor events from logbook."""
await hass.async_add_executor_job(init_recorder_component, hass)
await async_setup_component(hass, "logbook", {})
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

entity_id_test = "switch.test"
hass.states.async_set(entity_id_test, STATE_OFF)
Expand All @@ -1283,9 +1288,9 @@ async def test_filter_continuous_sensor_values(hass, hass_client):
hass.states.async_set(entity_id_third, STATE_OFF, {"unit_of_measurement": "foo"})
hass.states.async_set(entity_id_third, STATE_ON, {"unit_of_measurement": "foo"})

await hass.async_add_job(trigger_db_commit, hass)
await hass.async_add_executor_job(trigger_db_commit, hass)
await hass.async_block_till_done()
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

client = await hass_client()

Expand All @@ -1307,7 +1312,7 @@ async def test_exclude_new_entities(hass, hass_client):
"""Test if events are excluded on first update."""
await hass.async_add_executor_job(init_recorder_component, hass)
await async_setup_component(hass, "logbook", {})
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

entity_id = "climate.bla"
entity_id2 = "climate.blu"
Expand All @@ -1317,9 +1322,9 @@ async def test_exclude_new_entities(hass, hass_client):
hass.states.async_set(entity_id2, STATE_OFF)
hass.bus.async_fire(EVENT_HOMEASSISTANT_START)

await hass.async_add_job(trigger_db_commit, hass)
await hass.async_add_executor_job(trigger_db_commit, hass)
await hass.async_block_till_done()
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

client = await hass_client()

Expand All @@ -1342,7 +1347,7 @@ async def test_exclude_removed_entities(hass, hass_client):
"""Test if events are excluded on last update."""
await hass.async_add_executor_job(init_recorder_component, hass)
await async_setup_component(hass, "logbook", {})
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

entity_id = "climate.bla"
entity_id2 = "climate.blu"
Expand All @@ -1358,9 +1363,9 @@ async def test_exclude_removed_entities(hass, hass_client):
hass.states.async_remove(entity_id)
hass.states.async_remove(entity_id2)

await hass.async_add_job(trigger_db_commit, hass)
await hass.async_add_executor_job(trigger_db_commit, hass)
await hass.async_block_till_done()
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

client = await hass_client()

Expand All @@ -1384,7 +1389,7 @@ async def test_exclude_attribute_changes(hass, hass_client):
"""Test if events of attribute changes are filtered."""
await hass.async_add_executor_job(init_recorder_component, hass)
await async_setup_component(hass, "logbook", {})
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

hass.bus.async_fire(EVENT_HOMEASSISTANT_START)

Expand All @@ -1397,9 +1402,9 @@ async def test_exclude_attribute_changes(hass, hass_client):

await hass.async_block_till_done()

await hass.async_add_job(trigger_db_commit, hass)
await hass.async_add_executor_job(trigger_db_commit, hass)
await hass.async_block_till_done()
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

client = await hass_client()

Expand Down Expand Up @@ -1427,7 +1432,7 @@ async def test_logbook_entity_context_id(hass, hass_client):
await async_setup_component(hass, "automation", {})
await async_setup_component(hass, "script", {})

await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

context = ha.Context(
id="ac5bd62de45711eaaeb351041eec8dd9",
Expand Down Expand Up @@ -1467,7 +1472,7 @@ async def test_logbook_entity_context_id(hass, hass_client):
hass.bus.async_fire(EVENT_HOMEASSISTANT_START)
await hass.async_block_till_done()

await hass.async_add_job(
await hass.async_add_executor_job(
logbook.log_entry,
hass,
"mock_name",
Expand All @@ -1478,7 +1483,7 @@ async def test_logbook_entity_context_id(hass, hass_client):
)
await hass.async_block_till_done()

await hass.async_add_job(
await hass.async_add_executor_job(
logbook.log_entry,
hass,
"mock_name",
Expand Down Expand Up @@ -1513,9 +1518,9 @@ async def test_logbook_entity_context_id(hass, hass_client):
)
await hass.async_block_till_done()

await hass.async_add_job(trigger_db_commit, hass)
await hass.async_add_executor_job(trigger_db_commit, hass)
await hass.async_block_till_done()
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

client = await hass_client()

Expand Down Expand Up @@ -1846,7 +1851,7 @@ async def test_icon_and_state(hass, hass_client):
"""Test to ensure state and custom icons are returned."""
await hass.async_add_executor_job(init_recorder_component, hass)
await async_setup_component(hass, "logbook", {})
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

hass.bus.async_fire(EVENT_HOMEASSISTANT_START)

Expand Down Expand Up @@ -1895,7 +1900,7 @@ async def test_exclude_events_domain(hass, hass_client):
)
await hass.async_add_executor_job(init_recorder_component, hass)
await async_setup_component(hass, "logbook", config)
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

hass.bus.async_fire(EVENT_HOMEASSISTANT_START)
hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED)
Expand Down Expand Up @@ -1935,7 +1940,7 @@ async def test_exclude_events_domain_glob(hass, hass_client):
)
await hass.async_add_executor_job(init_recorder_component, hass)
await async_setup_component(hass, "logbook", config)
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

hass.bus.async_fire(EVENT_HOMEASSISTANT_START)
hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED)
Expand Down Expand Up @@ -1975,7 +1980,7 @@ async def test_include_events_entity(hass, hass_client):
)
await hass.async_add_executor_job(init_recorder_component, hass)
await async_setup_component(hass, "logbook", config)
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

hass.bus.async_fire(EVENT_HOMEASSISTANT_START)
hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED)
Expand Down Expand Up @@ -2008,7 +2013,7 @@ async def test_exclude_events_entity(hass, hass_client):
)
await hass.async_add_executor_job(init_recorder_component, hass)
await async_setup_component(hass, "logbook", config)
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

hass.bus.async_fire(EVENT_HOMEASSISTANT_START)
hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED)
Expand Down Expand Up @@ -2042,7 +2047,7 @@ async def test_include_events_domain(hass, hass_client):
)
await hass.async_add_executor_job(init_recorder_component, hass)
await async_setup_component(hass, "logbook", config)
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

hass.bus.async_fire(EVENT_HOMEASSISTANT_START)
hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED)
Expand Down Expand Up @@ -2086,7 +2091,7 @@ async def test_include_events_domain_glob(hass, hass_client):
)
await hass.async_add_executor_job(init_recorder_component, hass)
await async_setup_component(hass, "logbook", config)
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

hass.bus.async_fire(EVENT_HOMEASSISTANT_START)
hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED)
Expand Down Expand Up @@ -2138,7 +2143,7 @@ async def test_include_exclude_events(hass, hass_client):
)
await hass.async_add_executor_job(init_recorder_component, hass)
await async_setup_component(hass, "logbook", config)
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

hass.bus.async_fire(EVENT_HOMEASSISTANT_START)
hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED)
Expand Down Expand Up @@ -2192,7 +2197,7 @@ async def test_include_exclude_events_with_glob_filters(hass, hass_client):
)
await hass.async_add_executor_job(init_recorder_component, hass)
await async_setup_component(hass, "logbook", config)
await hass.async_add_job(hass.data[recorder.DATA_INSTANCE].block_till_done)
await hass.async_add_executor_job(hass.data[recorder.DATA_INSTANCE].block_till_done)

hass.bus.async_fire(EVENT_HOMEASSISTANT_START)
hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED)
Expand Down
15 changes: 14 additions & 1 deletion tests/components/recorder/test_init.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,20 @@ def test_saving_state_incl_entities(hass_recorder):

def test_saving_event_exclude_event_type(hass_recorder):
"""Test saving and restoring an event."""
hass = hass_recorder({"exclude": {"event_types": "test"}})
hass = hass_recorder(
{
"exclude": {
"event_types": [
"service_registered",
"homeassistant_start",
"component_loaded",
"core_config_updated",
"homeassistant_started",
"test",
]
}
}
)
events = _add_events(hass, ["test", "test2"])
assert len(events) == 1
assert events[0].event_type == "test2"
Expand Down