Skip to content

WIP Purge old data from sqlite db on hass start-up #1681

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
infamy merged 4 commits into from
May 15, 2016
Merged

WIP Purge old data from sqlite db on hass start-up #1681

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c89cd6a
Add 'purge_days' option to the history/recorder component
justyns Apr 2, 2016
fd48fc5
Add CONFIG_SCHEMA to verify config. Move purge_days key name to
justyns Apr 14, 2016
d5ca97b
Add tests for purging old states and events
justyns Apr 16, 2016
bf3b77e
Change sqlite queries to work with older versions of sqlite
justyns Apr 16, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 42 additions & 5 deletions homeassistant/components/recorder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@
import queue
import sqlite3
import threading
from datetime import date, datetime
from datetime import date, datetime, timedelta
import voluptuous as vol

import homeassistant.util.dt as dt_util
from homeassistant.const import (
Expand All @@ -30,6 +31,14 @@
RETURN_LASTROWID = "lastrowid"
RETURN_ONE_ROW = "one_row"

CONF_PURGE_DAYS = "purge_days"
CONFIG_SCHEMA = vol.Schema({
DOMAIN: vol.All(dict, {
CONF_PURGE_DAYS: int
Copy link
Copy Markdown
Member

@balloob balloob Apr 17, 2016 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it's optional, please mark it so. It also should be at least 1:

        vol.Optional(CONF_PURGE_DAYS): vol.All(vol.Coerce(int), vol.Range(min=1)),

})
}, extra=vol.ALLOW_EXTRA)


_INSTANCE = None
_LOGGER = logging.getLogger(__name__)

Expand Down Expand Up @@ -102,14 +111,14 @@ def setup(hass, config):
"""Setup the recorder."""
# pylint: disable=global-statement
global _INSTANCE

_INSTANCE = Recorder(hass)
purge_days = config.get(DOMAIN, {}).get(CONF_PURGE_DAYS)
_INSTANCE = Recorder(hass, purge_days=purge_days)

return True


class RecorderRun(object):
"""Representation of arecorder run."""
"""Representation of a recorder run."""

def __init__(self, row=None):
"""Initialize the recorder run."""
Expand Down Expand Up @@ -169,11 +178,12 @@ class Recorder(threading.Thread):
"""A threaded recorder class."""

# pylint: disable=too-many-instance-attributes
def __init__(self, hass):
def __init__(self, hass, purge_days):
"""Initialize the recorder."""
threading.Thread.__init__(self)

self.hass = hass
self.purge_days = purge_days
self.conn = None
self.queue = queue.Queue()
self.quit_object = object()
Expand All @@ -194,6 +204,7 @@ def run(self):
"""Start processing events to save."""
self._setup_connection()
self._setup_run()
self._purge_old_data()
Copy link
Copy Markdown
Member

@balloob balloob Apr 17, 2016 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it would make sense to schedule this some time after launch so it won't slow down the launch of Home Assistant.

from homeassistant.helpers.event import track_point_in_utc_time

if self.purge_days is not None:
    track_point_in_utc_time(self.hass, lambda now: self._purge_old_data(),
                            dt_util.utcnow() + timedelta(minutes=5))

Copy link
Copy Markdown
Member

@balloob balloob Apr 17, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As this will break your tests, you can just call recorder._purge_old_data() in your tests


while True:
event = self.queue.get()
Expand Down Expand Up @@ -475,6 +486,32 @@ def _close_run(self):
"UPDATE recorder_runs SET end=? WHERE start=?",
(dt_util.utcnow(), self.recording_start))

def _purge_old_data(self):
"""Purge events and states older than purge_days ago."""
if not self.purge_days or self.purge_days < 1:
_LOGGER.debug("purge_days set to %s, will not purge any old data.",
Copy link
Copy Markdown
Member

@balloob balloob Apr 17, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No need for this if you never call it when purge_days is not None, like i suggested above.

self.purge_days)
return

purge_before = dt_util.utcnow() - timedelta(days=self.purge_days)

_LOGGER.info("Purging events created before %s", purge_before)
Copy link
Copy Markdown
Member

@balloob balloob Apr 17, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would merge this with the query below and only print it to info if there were actually any rows deleted. Purged X events created before X

deleted_rows = self.query(
sql_query="DELETE FROM events WHERE created < ?;",
data=(int(purge_before.timestamp()),),
Copy link
Copy Markdown
Member

@balloob balloob Apr 17, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You should be able to just put in purge_before and it will be translated.

return_value=RETURN_ROWCOUNT)
_LOGGER.debug("Deleted %s events", deleted_rows)

_LOGGER.info("Purging states created before %s", purge_before)
Copy link
Copy Markdown
Member

@balloob balloob Apr 17, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same

deleted_rows = self.query(
sql_query="DELETE FROM states WHERE created < ?;",
data=(int(purge_before.timestamp()),),
return_value=RETURN_ROWCOUNT)
_LOGGER.debug("Deleted %s states", deleted_rows)

# Execute sqlite vacuum command to free up space on disk
self.query("VACUUM;")


def _adapt_datetime(datetimestamp):
"""Turn a datetime into an integer for in the DB."""
Expand Down
104 changes: 103 additions & 1 deletion tests/components/test_recorder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
"""The tests for the Recorder component."""
# pylint: disable=too-many-public-methods,protected-access
import unittest
import time
import json
from unittest.mock import patch

from homeassistant.const import MATCH_ALL
Expand All @@ -10,7 +12,7 @@


class TestRecorder(unittest.TestCase):
"""Test the chromecast module."""
"""Test the recorder module."""

def setUp(self): # pylint: disable=invalid-name
"""Setup things to be run when tests are started."""
Expand All @@ -25,6 +27,51 @@ def tearDown(self): # pylint: disable=invalid-name
self.hass.stop()
recorder._INSTANCE.block_till_done()

def _add_test_states(self):
"""Adds multiple states to the db for testing."""
now = int(time.time())
five_days_ago = now - (60*60*24*5)
attributes = {'test_attr': 5, 'test_attr_10': 'nice'}

self.hass.pool.block_till_done()
recorder._INSTANCE.block_till_done()
for event_id in range(5):
if event_id < 3:
timestamp = five_days_ago
state = 'purgeme'
else:
timestamp = now
state = 'dontpurgeme'
recorder.query("INSERT INTO states ("
"entity_id, domain, state, attributes, last_changed,"
"last_updated, created, utc_offset, event_id)"
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
('test.recorder2', 'sensor', state,
json.dumps(attributes), timestamp, timestamp,
timestamp, -18000, event_id + 1000))

def _add_test_events(self):
"""Adds a few events for testing."""
now = int(time.time())
five_days_ago = now - (60*60*24*5)
event_data = {'test_attr': 5, 'test_attr_10': 'nice'}

self.hass.pool.block_till_done()
recorder._INSTANCE.block_till_done()
for event_id in range(5):
if event_id < 2:
timestamp = five_days_ago
event_type = 'EVENT_TEST_PURGE'
else:
timestamp = now
event_type = 'EVENT_TEST'
recorder.query("INSERT INTO events"
"(event_type, event_data, origin, created,"
"time_fired, utc_offset)"
"VALUES (?, ?, ?, ?, ?, ?)",
(event_type, json.dumps(event_data), 'LOCAL',
timestamp, timestamp, -18000))

def test_saving_state(self):
"""Test saving and restoring a state."""
entity_id = 'test.recorder'
Expand Down Expand Up @@ -64,3 +111,58 @@ def event_listener(event):
'SELECT * FROM events WHERE event_type = ?', (event_type, ))

self.assertEqual(events, db_events)

def test_purge_old_states(self):
"""Tests deleting old states."""
self._add_test_states()
# make sure we start with 5 states
states = recorder.query_states('SELECT * FROM states')
self.assertEqual(len(states), 5)

# run purge_old_data()
recorder._INSTANCE.purge_days = 4
recorder._INSTANCE._purge_old_data()

# we should only have 2 states left after purging
states = recorder.query_states('SELECT * FROM states')
self.assertEqual(len(states), 2)

def test_purge_old_events(self):
"""Tests deleting old events."""
self._add_test_events()
events = recorder.query_events('SELECT * FROM events WHERE '
'event_type LIKE "EVENT_TEST%"')
self.assertEqual(len(events), 5)

# run purge_old_data()
recorder._INSTANCE.purge_days = 4
recorder._INSTANCE._purge_old_data()

# now we should only have 3 events left
events = recorder.query_events('SELECT * FROM events WHERE '
'event_type LIKE "EVENT_TEST%"')
self.assertEqual(len(events), 3)


def test_purge_disabled(self):
"""Tests leaving purge_days disabled."""
self._add_test_states()
self._add_test_events()
# make sure we start with 5 states and events
states = recorder.query_states('SELECT * FROM states')
events = recorder.query_events('SELECT * FROM events WHERE '
'event_type LIKE "EVENT_TEST%"')
self.assertEqual(len(states), 5)
self.assertEqual(len(events), 5)


# run purge_old_data()
recorder._INSTANCE.purge_days = None
recorder._INSTANCE._purge_old_data()

# we should have all of our states still
states = recorder.query_states('SELECT * FROM states')
events = recorder.query_events('SELECT * FROM events WHERE '
'event_type LIKE "EVENT_TEST%"')
self.assertEqual(len(states), 5)
self.assertEqual(len(events), 5)