From e1280b7d11291a52d537925b25ddd0df255307d7 Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Sat, 10 Mar 2018 23:51:54 +0000 Subject: [PATCH 01/27] Initialise filter with historical values Added get_last_state_changes() --- homeassistant/components/history.py | 24 ++++++++++++ homeassistant/components/sensor/filter.py | 31 +++++++++++++-- tests/components/sensor/test_filter.py | 46 ++++++++++++++++++----- tests/components/test_history.py | 33 ++++++++++++++++ 4 files changed, 121 insertions(+), 13 deletions(-) diff --git a/homeassistant/components/history.py b/homeassistant/components/history.py index dd14bbf68111c..d937ae0c020a8 100644 --- a/homeassistant/components/history.py +++ b/homeassistant/components/history.py @@ -118,6 +118,30 @@ def state_changes_during_period(hass, start_time, end_time=None, return states_to_json(hass, states, start_time, entity_ids) +def get_last_state_changes(hass, number_of_states, entity_id=None): + """Return the last number_of_states.""" + from homeassistant.components.recorder.models import States + + start_time = dt_util.utcnow() + + with session_scope(hass=hass) as session: + query = session.query(States).filter( + (States.last_changed == States.last_updated)) + + if entity_id is not None: + query = query.filter_by(entity_id=entity_id.lower()) + + entity_ids = [entity_id] if entity_id is not None else None + + states = execute( + query.order_by(States.last_updated.desc()).limit(number_of_states)) + + return states_to_json(hass, states[::-1], + start_time, + entity_ids, + include_start_time_state=False) + + def get_states(hass, utc_point_in_time, entity_ids=None, run=None, filters=None): """Return the states at a specific point in time.""" diff --git a/homeassistant/components/sensor/filter.py b/homeassistant/components/sensor/filter.py index cde50699b29c8..c4bb797544e02 100644 --- a/homeassistant/components/sensor/filter.py +++ b/homeassistant/components/sensor/filter.py @@ -8,6 +8,7 @@ import statistics from collections import deque, Counter from numbers import Number +from functools import partial import voluptuous as vol @@ -20,6 +21,8 @@ from homeassistant.util.decorator import Registry from homeassistant.helpers.entity import Entity from homeassistant.helpers.event import async_track_state_change +import homeassistant.components.history as history +import homeassistant.util.dt as dt_util _LOGGER = logging.getLogger(__name__) @@ -30,6 +33,7 @@ CONF_FILTERS = 'filters' CONF_FILTER_NAME = 'filter' +CONF_HISTORY_PERIOD = 'history_period' CONF_FILTER_WINDOW_SIZE = 'window_size' CONF_FILTER_PRECISION = 'precision' CONF_FILTER_RADIUS = 'radius' @@ -69,6 +73,8 @@ PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend({ vol.Required(CONF_ENTITY_ID): cv.entity_id, vol.Optional(CONF_NAME): cv.string, + vol.Optional(CONF_HISTORY_PERIOD): vol.All(cv.time_period, + cv.positive_timedelta), vol.Required(CONF_FILTERS): vol.All(cv.ensure_list, [vol.Any(FILTER_OUTLIER_SCHEMA, FILTER_LOWPASS_SCHEMA, @@ -81,18 +87,19 @@ async def async_setup_platform(hass, config, async_add_devices, """Set up the template sensors.""" name = config.get(CONF_NAME) entity_id = config.get(CONF_ENTITY_ID) + history_period = config.get(CONF_HISTORY_PERIOD) filters = [FILTERS[_filter.pop(CONF_FILTER_NAME)]( entity=entity_id, **_filter) for _filter in config[CONF_FILTERS]] - async_add_devices([SensorFilter(name, entity_id, filters)]) + async_add_devices([SensorFilter(name, entity_id, history_period, filters)]) class SensorFilter(Entity): """Representation of a Filter Sensor.""" - def __init__(self, name, entity_id, filters): + def __init__(self, name, entity_id, history_period, filters): """Initialize the sensor.""" self._name = name self._entity = entity_id @@ -100,11 +107,13 @@ def __init__(self, name, entity_id, filters): self._state = None self._filters = filters self._icon = None + self._history_period = history_period async def async_added_to_hass(self): """Register callbacks.""" @callback - def filter_sensor_state_listener(entity, old_state, new_state): + def filter_sensor_state_listener(entity, old_state, new_state, + update_ha=True): """Handle device state changes.""" if new_state.state in [STATE_UNKNOWN, STATE_UNAVAILABLE]: return @@ -137,7 +146,21 @@ def filter_sensor_state_listener(entity, old_state, new_state): self._unit_of_measurement = new_state.attributes.get( ATTR_UNIT_OF_MEASUREMENT) - self.async_schedule_update_ha_state() + if update_ha: + self.async_schedule_update_ha_state() + + if self._history_period and 'recorder' in self.hass.config.components: + start = dt_util.utcnow() - self._history_period + + history_list = await self.hass.async_add_job(partial( + history.state_changes_during_period, self.hass, + start, entity_id=self._entity)) + + prev_state = None + for state in history_list[self._entity]: + filter_sensor_state_listener( + self._entity, prev_state, state, False) + prev_state = state.state async_track_state_change( self.hass, self._entity, filter_sensor_state_listener) diff --git a/tests/components/sensor/test_filter.py b/tests/components/sensor/test_filter.py index dd1112d65f883..ab6b9ba9a7a64 100644 --- a/tests/components/sensor/test_filter.py +++ b/tests/components/sensor/test_filter.py @@ -1,10 +1,15 @@ """The test for the data filter sensor platform.""" +from datetime import timedelta import unittest +from unittest.mock import patch from homeassistant.components.sensor.filter import ( LowPassFilter, OutlierFilter, ThrottleFilter) from homeassistant.setup import setup_component -from tests.common import get_test_home_assistant, assert_setup_component +import homeassistant.core as ha +import homeassistant.util.dt as dt_util +from tests.common import (get_test_home_assistant, assert_setup_component, + init_recorder_component) class TestFilterSensor(unittest.TestCase): @@ -19,6 +24,11 @@ def teardown_method(self, method): """Stop everything that was started.""" self.hass.stop() + def init_recorder(self): + """Initialize the recorder.""" + init_recorder_component(self.hass) + self.hass.start() + def test_setup_fail(self): """Test if filter doesn't exist.""" config = { @@ -33,31 +43,49 @@ def test_setup_fail(self): def test_chain(self): """Test if filter chaining works.""" + self.init_recorder() config = { + 'history': { + }, 'sensor': { 'platform': 'filter', 'name': 'test', 'entity_id': 'sensor.test_monitored', + 'history_period': '00:05', 'filters': [{ 'filter': 'outlier', + 'window_size': 10, 'radius': 4.0 }, { 'filter': 'lowpass', - 'window_size': 4, 'time_constant': 10, 'precision': 2 }] } } - with assert_setup_component(1): - assert setup_component(self.hass, 'sensor', config) + t_0 = dt_util.utcnow() - timedelta(minutes=1) + t_1 = dt_util.utcnow() - timedelta(minutes=2) + t_2 = dt_util.utcnow() - timedelta(minutes=3) + + fake_states = { + 'sensor.test_monitored': [ + ha.State('sensor.test_monitored', 18.0, last_changed=t_0), + ha.State('sensor.test_monitored', 19.0, last_changed=t_1), + ha.State('sensor.test_monitored', 18.2, last_changed=t_2), + ] + } + + with patch('homeassistant.components.history.' + 'state_changes_during_period', return_value=fake_states): + with assert_setup_component(1, 'sensor'): + assert setup_component(self.hass, 'sensor', config) - for value in self.values: - self.hass.states.set(config['sensor']['entity_id'], value) - self.hass.block_till_done() + for value in self.values: + self.hass.states.set(config['sensor']['entity_id'], value) + self.hass.block_till_done() - state = self.hass.states.get('sensor.test') - self.assertEqual('20.25', state.state) + state = self.hass.states.get('sensor.test') + self.assertEqual('19.25', state.state) def test_outlier(self): """Test if outlier filter works.""" diff --git a/tests/components/test_history.py b/tests/components/test_history.py index 4a759e7e0acbd..64d908bd28aa0 100644 --- a/tests/components/test_history.py +++ b/tests/components/test_history.py @@ -131,6 +131,39 @@ def set_state(state): self.assertEqual(states, hist[entity_id]) + def test_get_last_state_changes(self): + """Test number of state changes.""" + self.init_recorder() + entity_id = 'sensor.test' + + def set_state(state): + """Set the state.""" + self.hass.states.set(entity_id, state) + self.wait_recording_done() + return self.hass.states.get(entity_id) + + start = dt_util.utcnow() - timedelta(minutes=2) + point = start + timedelta(minutes=1) + point2 = point + timedelta(minutes=1) + + with patch('homeassistant.components.recorder.dt_util.utcnow', + return_value=start): + set_state('1') + + states = [] + with patch('homeassistant.components.recorder.dt_util.utcnow', + return_value=point): + states.append(set_state('2')) + + with patch('homeassistant.components.recorder.dt_util.utcnow', + return_value=point2): + states.append(set_state('3')) + + hist = history.get_last_state_changes( + self.hass, 2, entity_id) + + self.assertEqual(states, hist[entity_id]) + def test_get_significant_states(self): """Test that only significant states are returned. From d374f906554648d044e64532b99950448c32dd97 Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Mon, 19 Mar 2018 00:29:14 +0000 Subject: [PATCH 02/27] fix test --- tests/components/test_history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/components/test_history.py b/tests/components/test_history.py index 64d908bd28aa0..4ebe1d11241d9 100644 --- a/tests/components/test_history.py +++ b/tests/components/test_history.py @@ -141,7 +141,7 @@ def set_state(state): self.hass.states.set(entity_id, state) self.wait_recording_done() return self.hass.states.get(entity_id) - + start = dt_util.utcnow() - timedelta(minutes=2) point = start + timedelta(minutes=1) point2 = point + timedelta(minutes=1) From 71ae96239c4c080c4de734450a7385ce4041ceb6 Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Wed, 21 Mar 2018 00:14:17 +0000 Subject: [PATCH 03/27] Major changes to accommodate history + time_SMA # Conflicts: # homeassistant/components/sensor/filter.py --- homeassistant/components/sensor/filter.py | 131 ++++++++++++++-------- tests/components/sensor/test_filter.py | 37 +++--- 2 files changed, 107 insertions(+), 61 deletions(-) diff --git a/homeassistant/components/sensor/filter.py b/homeassistant/components/sensor/filter.py index 947f2e21f5272..cf40c65de7c58 100644 --- a/homeassistant/components/sensor/filter.py +++ b/homeassistant/components/sensor/filter.py @@ -9,6 +9,8 @@ from collections import deque, Counter from numbers import Number from functools import partial +from copy import copy +import datetime import voluptuous as vol @@ -34,7 +36,6 @@ CONF_FILTERS = 'filters' CONF_FILTER_NAME = 'filter' -CONF_HISTORY_PERIOD = 'history_period' CONF_FILTER_WINDOW_SIZE = 'window_size' CONF_FILTER_PRECISION = 'precision' CONF_FILTER_RADIUS = 'radius' @@ -90,8 +91,6 @@ PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend({ vol.Required(CONF_ENTITY_ID): cv.entity_id, vol.Optional(CONF_NAME): cv.string, - vol.Optional(CONF_HISTORY_PERIOD): vol.All(cv.time_period, - cv.positive_timedelta), vol.Required(CONF_FILTERS): vol.All(cv.ensure_list, [vol.Any(FILTER_OUTLIER_SCHEMA, FILTER_LOWPASS_SCHEMA, @@ -105,19 +104,18 @@ async def async_setup_platform(hass, config, async_add_devices, """Set up the template sensors.""" name = config.get(CONF_NAME) entity_id = config.get(CONF_ENTITY_ID) - history_period = config.get(CONF_HISTORY_PERIOD) filters = [FILTERS[_filter.pop(CONF_FILTER_NAME)]( entity=entity_id, **_filter) for _filter in config[CONF_FILTERS]] - async_add_devices([SensorFilter(name, entity_id, history_period, filters)]) + async_add_devices([SensorFilter(name, entity_id, filters)]) class SensorFilter(Entity): """Representation of a Filter Sensor.""" - def __init__(self, name, entity_id, history_period, filters): + def __init__(self, name, entity_id, filters): """Initialize the sensor.""" self._name = name self._entity = entity_id @@ -125,7 +123,6 @@ def __init__(self, name, entity_id, history_period, filters): self._state = None self._filters = filters self._icon = None - self._history_period = history_period async def async_added_to_hass(self): """Register callbacks.""" @@ -136,7 +133,7 @@ def filter_sensor_state_listener(entity, old_state, new_state, if new_state.state in [STATE_UNKNOWN, STATE_UNAVAILABLE]: return - temp_state = new_state.state + temp_state = new_state try: for filt in self._filters: @@ -167,15 +164,26 @@ def filter_sensor_state_listener(entity, old_state, new_state, if update_ha: self.async_schedule_update_ha_state() - if self._history_period and 'recorder' in self.hass.config.components: - start = dt_util.utcnow() - self._history_period - - history_list = await self.hass.async_add_job(partial( - history.state_changes_during_period, self.hass, - start, entity_id=self._entity)) - + if 'recorder' in self.hass.config.components: + history_list = [] + for filt in self._filters: + if isinstance(filt.window_size, int): + filter_history = await self.hass.async_add_job(partial( + history.get_last_state_changes, self.hass, + filt.window_size, entity_id=self._entity)) + elif isinstance(filt.window_size, datetime.timedelta): + start = dt_util.utcnow() - filt.window_size + filter_history = await self.hass.async_add_job(partial( + history.state_changes_during_period, self.hass, + start, entity_id=self._entity)) + for _, states in filter_history.items(): + history_list.extend( + [st for st in states if st not in history_list]) + history_list = sorted(history_list, key=lambda s: s.last_updated) + + # Replay history through the filter chain prev_state = None - for state in history_list[self._entity]: + for state in history_list: filter_sensor_state_listener( self._entity, prev_state, state, False) prev_state = state.state @@ -217,6 +225,31 @@ def device_state_attributes(self): return state_attr +class FilterState(object): + """State abstraction for filter usage.""" + + def __init__(self, state): + """Initialize with HA State object.""" + self.timestamp = state.last_updated + try: + self.state = float(state.state) + except ValueError: + self.state = state.state + + def set_precision(self, precision): + """Set precision of Number based states.""" + if isinstance(self.state, Number): + self.state = round(float(self.state), precision) + + def __str__(self): + """Return state as the string representation of FilterState.""" + return str(self.state) + + def __repr__(self): + """Return timestamp and state as the representation of FilterState.""" + return "{} : {}".format(self.timestamp, self.state) + + class Filter(object): """Filter skeleton. @@ -229,11 +262,20 @@ class Filter(object): def __init__(self, name, window_size=1, precision=None, entity=None): """Initialize common attributes.""" - self.states = deque(maxlen=window_size) + if isinstance(window_size, int): + self.states = deque(maxlen=window_size) + else: + self.states = deque(maxlen=0) self.precision = precision self._name = name self._entity = entity self._skip_processing = False + self._window_size = window_size + + @property + def window_size(self): + """Return window size.""" + return self._window_size @property def name(self): @@ -251,11 +293,11 @@ def _filter_state(self, new_state): def filter_state(self, new_state): """Implement a common interface for filters.""" - filtered = self._filter_state(new_state) - if isinstance(filtered, Number): - filtered = round(float(filtered), self.precision) - self.states.append(filtered) - return filtered + filtered = self._filter_state(FilterState(new_state)) + filtered.set_precision(self.precision) + self.states.append(copy(filtered)) + new_state.state = filtered.state + return new_state @FILTERS.register(FILTER_NAME_OUTLIER) @@ -276,11 +318,10 @@ def __init__(self, window_size, precision, entity, radius): def _filter_state(self, new_state): """Implement the outlier filter.""" - new_state = float(new_state) - if (self.states and - abs(new_state - statistics.median(self.states)) - > self._radius): + abs(new_state.state - + statistics.median([s.state for s in self.states])) > + self._radius): self._stats_internal['erasures'] += 1 @@ -306,16 +347,15 @@ def __init__(self, window_size, precision, entity, time_constant): def _filter_state(self, new_state): """Implement the low pass filter.""" - new_state = float(new_state) - if not self.states: return new_state new_weight = 1.0 / self._time_constant prev_weight = 1.0 - new_weight - filtered = prev_weight * self.states[-1] + new_weight * new_state + new_state.state = prev_weight * self.states[-1].state +\ + new_weight * new_state.state - return filtered + return new_state @FILTERS.register(FILTER_NAME_TIME_SMA) @@ -330,35 +370,36 @@ class TimeSMAFilter(Filter): def __init__(self, window_size, precision, entity, type): """Initialize Filter.""" - super().__init__(FILTER_NAME_TIME_SMA, 0, precision, entity) - self._time_window = int(window_size.total_seconds()) + super().__init__(FILTER_NAME_TIME_SMA, window_size, precision, entity) + self._time_window = window_size self.last_leak = None self.queue = deque() - def _leak(self, now): + def _leak(self, left_boundary): """Remove timeouted elements.""" while self.queue: - timestamp, _ = self.queue[0] - if timestamp + self._time_window <= now: + if self.queue[0].timestamp + self._time_window <= left_boundary: self.last_leak = self.queue.popleft() else: return def _filter_state(self, new_state): - now = int(dt_util.utcnow().timestamp()) + """Implement the Simple Moving Average filter.""" + self._leak(new_state.timestamp) + self.queue.append(copy(new_state)) - self._leak(now) - self.queue.append((now, float(new_state))) moving_sum = 0 - start = now - self._time_window - _, prev_val = self.last_leak or (0, float(new_state)) + start = new_state.timestamp - self._time_window + prev_state = self.last_leak or self.queue[0] + for state in self.queue: + moving_sum += (state.timestamp-start).total_seconds()\ + * prev_state.state + start = state.timestamp + prev_state = state - for timestamp, val in self.queue: - moving_sum += (timestamp-start)*prev_val - start, prev_val = timestamp, val - moving_sum += (now-start)*prev_val + new_state.state = moving_sum / self._time_window.total_seconds() - return moving_sum/self._time_window + return new_state @FILTERS.register(FILTER_NAME_THROTTLE) diff --git a/tests/components/sensor/test_filter.py b/tests/components/sensor/test_filter.py index 716227d18538f..257be3845463c 100644 --- a/tests/components/sensor/test_filter.py +++ b/tests/components/sensor/test_filter.py @@ -19,7 +19,13 @@ class TestFilterSensor(unittest.TestCase): def setup_method(self, method): """Setup things to be run when tests are started.""" self.hass = get_test_home_assistant() - self.values = [20, 19, 18, 21, 22, 0] + raw_values = [20, 19, 18, 21, 22, 0] + self.values = [] + + ts = dt_util.utcnow() + for val in raw_values: + self.values.append(ha.State('sensor.test_monitored', val, last_updated=ts)) + ts = ts + timedelta(minutes=1) def teardown_method(self, method): """Stop everything that was started.""" @@ -78,15 +84,17 @@ def test_chain(self): with patch('homeassistant.components.history.' 'state_changes_during_period', return_value=fake_states): - with assert_setup_component(1, 'sensor'): - assert setup_component(self.hass, 'sensor', config) + with patch('homeassistant.components.history.' + 'get_last_state_changes', return_value=fake_states): + with assert_setup_component(1, 'sensor'): + assert setup_component(self.hass, 'sensor', config) - for value in self.values: - self.hass.states.set(config['sensor']['entity_id'], value) - self.hass.block_till_done() + for value in self.values: + self.hass.states.set(config['sensor']['entity_id'], value.state) + self.hass.block_till_done() - state = self.hass.states.get('sensor.test') - self.assertEqual('19.25', state.state) + state = self.hass.states.get('sensor.test_monitored') + self.assertEqual(19.25, state.state) def test_outlier(self): """Test if outlier filter works.""" @@ -96,7 +104,7 @@ def test_outlier(self): radius=4.0) for state in self.values: filtered = filt.filter_state(state) - self.assertEqual(22, filtered) + self.assertEqual(22, filtered.state) def test_lowpass(self): """Test if lowpass filter works.""" @@ -106,7 +114,7 @@ def test_lowpass(self): time_constant=10) for state in self.values: filtered = filt.filter_state(state) - self.assertEqual(18.05, filtered) + self.assertEqual(18.05, filtered.state) def test_throttle(self): """Test if lowpass filter works.""" @@ -118,7 +126,7 @@ def test_throttle(self): new_state = filt.filter_state(state) if not filt.skip_processing: filtered.append(new_state) - self.assertEqual([20, 21], filtered) + self.assertEqual([20, 21], [f.state for f in filtered]) def test_time_sma(self): """Test if time_sma filter works.""" @@ -126,9 +134,6 @@ def test_time_sma(self): precision=2, entity=None, type='last') - past = dt_util.utcnow() - timedelta(minutes=5) for state in self.values: - with patch('homeassistant.util.dt.utcnow', return_value=past): - filtered = filt.filter_state(state) - past += timedelta(minutes=1) - self.assertEqual(21.5, filtered) + filtered = filt.filter_state(state) + self.assertEqual(21.5, filtered.state) From 511342de19923c7f3a9f0af76741e71b9ea3112c Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Wed, 21 Mar 2018 00:16:52 +0000 Subject: [PATCH 04/27] hail the hound! --- tests/components/sensor/test_filter.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/components/sensor/test_filter.py b/tests/components/sensor/test_filter.py index 257be3845463c..9a9a1c792c425 100644 --- a/tests/components/sensor/test_filter.py +++ b/tests/components/sensor/test_filter.py @@ -24,7 +24,8 @@ def setup_method(self, method): ts = dt_util.utcnow() for val in raw_values: - self.values.append(ha.State('sensor.test_monitored', val, last_updated=ts)) + self.values.append(ha.State('sensor.test_monitored', + val, last_updated=ts)) ts = ts + timedelta(minutes=1) def teardown_method(self, method): @@ -90,7 +91,8 @@ def test_chain(self): assert setup_component(self.hass, 'sensor', config) for value in self.values: - self.hass.states.set(config['sensor']['entity_id'], value.state) + self.hass.states.set( + config['sensor']['entity_id'], value.state) self.hass.block_till_done() state = self.hass.states.get('sensor.test_monitored') From 84c2424b25fd31663cce4adb53988355960cde47 Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Wed, 21 Mar 2018 00:20:44 +0000 Subject: [PATCH 05/27] lint fixed --- tests/components/sensor/test_filter.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/components/sensor/test_filter.py b/tests/components/sensor/test_filter.py index 9a9a1c792c425..1fe8767ab7f32 100644 --- a/tests/components/sensor/test_filter.py +++ b/tests/components/sensor/test_filter.py @@ -8,7 +8,6 @@ import homeassistant.util.dt as dt_util from homeassistant.setup import setup_component import homeassistant.core as ha -import homeassistant.util.dt as dt_util from tests.common import (get_test_home_assistant, assert_setup_component, init_recorder_component) @@ -22,11 +21,11 @@ def setup_method(self, method): raw_values = [20, 19, 18, 21, 22, 0] self.values = [] - ts = dt_util.utcnow() + timestamp = dt_util.utcnow() for val in raw_values: self.values.append(ha.State('sensor.test_monitored', - val, last_updated=ts)) - ts = ts + timedelta(minutes=1) + val, last_updated=timestamp)) + timestamp += timedelta(minutes=1) def teardown_method(self, method): """Stop everything that was started.""" From d5cdd5ced39718e74c29c4f97e5a391f05b85f1d Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Wed, 21 Mar 2018 00:26:34 +0000 Subject: [PATCH 06/27] less debug --- homeassistant/components/sensor/filter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/sensor/filter.py b/homeassistant/components/sensor/filter.py index cf40c65de7c58..8255a31105454 100644 --- a/homeassistant/components/sensor/filter.py +++ b/homeassistant/components/sensor/filter.py @@ -137,12 +137,12 @@ def filter_sensor_state_listener(entity, old_state, new_state, try: for filt in self._filters: - filtered_state = filt.filter_state(temp_state) + filtered_state = filt.filter_state(copy(temp_state)) _LOGGER.debug("%s(%s=%s) -> %s", filt.name, self._entity, - temp_state, + temp_state.state, "skip" if filt.skip_processing else - filtered_state) + filtered_state.state) if filt.skip_processing: return temp_state = filtered_state From a6fcf180bc4309ceba63c9b3eb556ca4fa7f5584 Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Wed, 21 Mar 2018 00:46:12 +0000 Subject: [PATCH 07/27] ups --- homeassistant/components/sensor/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/sensor/filter.py b/homeassistant/components/sensor/filter.py index 8255a31105454..3688378299d02 100644 --- a/homeassistant/components/sensor/filter.py +++ b/homeassistant/components/sensor/filter.py @@ -151,7 +151,7 @@ def filter_sensor_state_listener(entity, old_state, new_state, self._state) return - self._state = temp_state + self._state = temp_state.state if self._icon is None: self._icon = new_state.attributes.get( From 2d2edaada0f2f64d88d83171d3276f08185aed1b Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Wed, 21 Mar 2018 17:00:58 +0000 Subject: [PATCH 08/27] get state from the proper entity --- tests/components/sensor/test_filter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/components/sensor/test_filter.py b/tests/components/sensor/test_filter.py index 1fe8767ab7f32..8b8e7607b0776 100644 --- a/tests/components/sensor/test_filter.py +++ b/tests/components/sensor/test_filter.py @@ -94,8 +94,8 @@ def test_chain(self): config['sensor']['entity_id'], value.state) self.hass.block_till_done() - state = self.hass.states.get('sensor.test_monitored') - self.assertEqual(19.25, state.state) + state = self.hass.states.get('sensor.test') + self.assertEqual('19.25', state.state) def test_outlier(self): """Test if outlier filter works.""" From 7421475c80d56d1b7a96095654ada0cd45c30f7f Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Wed, 21 Mar 2018 17:03:58 +0000 Subject: [PATCH 09/27] sensible default --- homeassistant/components/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/history.py b/homeassistant/components/history.py index 9bd0eaf96e653..52589ce2c896f 100644 --- a/homeassistant/components/history.py +++ b/homeassistant/components/history.py @@ -118,7 +118,7 @@ def state_changes_during_period(hass, start_time, end_time=None, return states_to_json(hass, states, start_time, entity_ids) -def get_last_state_changes(hass, number_of_states, entity_id=None): +def get_last_state_changes(hass, number_of_states=10, entity_id=None): """Return the last number_of_states.""" from homeassistant.components.recorder.models import States From 6e8243643f65a4e12a52cfbc4afb0a7a7def662a Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Thu, 22 Mar 2018 11:52:22 +0000 Subject: [PATCH 10/27] No defaults in get_last_state_changes --- homeassistant/components/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/history.py b/homeassistant/components/history.py index 52589ce2c896f..0a7f9c8f2c24c 100644 --- a/homeassistant/components/history.py +++ b/homeassistant/components/history.py @@ -118,7 +118,7 @@ def state_changes_during_period(hass, start_time, end_time=None, return states_to_json(hass, states, start_time, entity_ids) -def get_last_state_changes(hass, number_of_states=10, entity_id=None): +def get_last_state_changes(hass, number_of_states, entity_id): """Return the last number_of_states.""" from homeassistant.components.recorder.models import States From 0bfee87777391048ee664d22b5c6b4876b891e45 Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Thu, 22 Mar 2018 11:55:16 +0000 Subject: [PATCH 11/27] list_reverseiterator instead of list --- homeassistant/components/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/history.py b/homeassistant/components/history.py index 0a7f9c8f2c24c..b5ac37b1451b2 100644 --- a/homeassistant/components/history.py +++ b/homeassistant/components/history.py @@ -136,7 +136,7 @@ def get_last_state_changes(hass, number_of_states, entity_id): states = execute( query.order_by(States.last_updated.desc()).limit(number_of_states)) - return states_to_json(hass, states[::-1], + return states_to_json(hass, reversed(states), start_time, entity_ids, include_start_time_state=False) From 58337911884b3ba98d26ffb25e52079d337f7b5c Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Thu, 22 Mar 2018 11:56:44 +0000 Subject: [PATCH 12/27] prev_state to state --- homeassistant/components/sensor/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/sensor/filter.py b/homeassistant/components/sensor/filter.py index 3688378299d02..b573f58bf0045 100644 --- a/homeassistant/components/sensor/filter.py +++ b/homeassistant/components/sensor/filter.py @@ -186,7 +186,7 @@ def filter_sensor_state_listener(entity, old_state, new_state, for state in history_list: filter_sensor_state_listener( self._entity, prev_state, state, False) - prev_state = state.state + prev_state = state async_track_state_change( self.hass, self._entity, filter_sensor_state_listener) From f5b7bc4592ba5091683bf0b9319c3661d3510a74 Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Sat, 10 Mar 2018 23:51:54 +0000 Subject: [PATCH 13/27] Initialise filter with historical values Added get_last_state_changes() --- homeassistant/components/history.py | 24 +++++++++++++ homeassistant/components/sensor/filter.py | 30 +++++++++++++--- tests/components/sensor/test_filter.py | 44 ++++++++++++++++++----- tests/components/test_history.py | 33 +++++++++++++++++ 4 files changed, 118 insertions(+), 13 deletions(-) diff --git a/homeassistant/components/history.py b/homeassistant/components/history.py index 8ab91b08a3dcd..9bd0eaf96e653 100644 --- a/homeassistant/components/history.py +++ b/homeassistant/components/history.py @@ -118,6 +118,30 @@ def state_changes_during_period(hass, start_time, end_time=None, return states_to_json(hass, states, start_time, entity_ids) +def get_last_state_changes(hass, number_of_states, entity_id=None): + """Return the last number_of_states.""" + from homeassistant.components.recorder.models import States + + start_time = dt_util.utcnow() + + with session_scope(hass=hass) as session: + query = session.query(States).filter( + (States.last_changed == States.last_updated)) + + if entity_id is not None: + query = query.filter_by(entity_id=entity_id.lower()) + + entity_ids = [entity_id] if entity_id is not None else None + + states = execute( + query.order_by(States.last_updated.desc()).limit(number_of_states)) + + return states_to_json(hass, states[::-1], + start_time, + entity_ids, + include_start_time_state=False) + + def get_states(hass, utc_point_in_time, entity_ids=None, run=None, filters=None): """Return the states at a specific point in time.""" diff --git a/homeassistant/components/sensor/filter.py b/homeassistant/components/sensor/filter.py index 3faf51a5f47ee..93d2f34dbbfa0 100644 --- a/homeassistant/components/sensor/filter.py +++ b/homeassistant/components/sensor/filter.py @@ -8,6 +8,7 @@ import statistics from collections import deque, Counter from numbers import Number +from functools import partial import voluptuous as vol @@ -20,6 +21,7 @@ from homeassistant.util.decorator import Registry from homeassistant.helpers.entity import Entity from homeassistant.helpers.event import async_track_state_change +import homeassistant.components.history as history import homeassistant.util.dt as dt_util _LOGGER = logging.getLogger(__name__) @@ -32,6 +34,7 @@ CONF_FILTERS = 'filters' CONF_FILTER_NAME = 'filter' +CONF_HISTORY_PERIOD = 'history_period' CONF_FILTER_WINDOW_SIZE = 'window_size' CONF_FILTER_PRECISION = 'precision' CONF_FILTER_RADIUS = 'radius' @@ -87,6 +90,8 @@ PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend({ vol.Required(CONF_ENTITY_ID): cv.entity_id, vol.Optional(CONF_NAME): cv.string, + vol.Optional(CONF_HISTORY_PERIOD): vol.All(cv.time_period, + cv.positive_timedelta), vol.Required(CONF_FILTERS): vol.All(cv.ensure_list, [vol.Any(FILTER_OUTLIER_SCHEMA, FILTER_LOWPASS_SCHEMA, @@ -100,18 +105,19 @@ async def async_setup_platform(hass, config, async_add_devices, """Set up the template sensors.""" name = config.get(CONF_NAME) entity_id = config.get(CONF_ENTITY_ID) + history_period = config.get(CONF_HISTORY_PERIOD) filters = [FILTERS[_filter.pop(CONF_FILTER_NAME)]( entity=entity_id, **_filter) for _filter in config[CONF_FILTERS]] - async_add_devices([SensorFilter(name, entity_id, filters)]) + async_add_devices([SensorFilter(name, entity_id, history_period, filters)]) class SensorFilter(Entity): """Representation of a Filter Sensor.""" - def __init__(self, name, entity_id, filters): + def __init__(self, name, entity_id, history_period, filters): """Initialize the sensor.""" self._name = name self._entity = entity_id @@ -119,11 +125,13 @@ def __init__(self, name, entity_id, filters): self._state = None self._filters = filters self._icon = None + self._history_period = history_period async def async_added_to_hass(self): """Register callbacks.""" @callback - def filter_sensor_state_listener(entity, old_state, new_state): + def filter_sensor_state_listener(entity, old_state, new_state, + update_ha=True): """Handle device state changes.""" if new_state.state in [STATE_UNKNOWN, STATE_UNAVAILABLE]: return @@ -156,7 +164,21 @@ def filter_sensor_state_listener(entity, old_state, new_state): self._unit_of_measurement = new_state.attributes.get( ATTR_UNIT_OF_MEASUREMENT) - self.async_schedule_update_ha_state() + if update_ha: + self.async_schedule_update_ha_state() + + if self._history_period and 'recorder' in self.hass.config.components: + start = dt_util.utcnow() - self._history_period + + history_list = await self.hass.async_add_job(partial( + history.state_changes_during_period, self.hass, + start, entity_id=self._entity)) + + prev_state = None + for state in history_list[self._entity]: + filter_sensor_state_listener( + self._entity, prev_state, state, False) + prev_state = state.state async_track_state_change( self.hass, self._entity, filter_sensor_state_listener) diff --git a/tests/components/sensor/test_filter.py b/tests/components/sensor/test_filter.py index 0d4082731ab38..716227d18538f 100644 --- a/tests/components/sensor/test_filter.py +++ b/tests/components/sensor/test_filter.py @@ -7,7 +7,10 @@ LowPassFilter, OutlierFilter, ThrottleFilter, TimeSMAFilter) import homeassistant.util.dt as dt_util from homeassistant.setup import setup_component -from tests.common import get_test_home_assistant, assert_setup_component +import homeassistant.core as ha +import homeassistant.util.dt as dt_util +from tests.common import (get_test_home_assistant, assert_setup_component, + init_recorder_component) class TestFilterSensor(unittest.TestCase): @@ -22,6 +25,11 @@ def teardown_method(self, method): """Stop everything that was started.""" self.hass.stop() + def init_recorder(self): + """Initialize the recorder.""" + init_recorder_component(self.hass) + self.hass.start() + def test_setup_fail(self): """Test if filter doesn't exist.""" config = { @@ -36,31 +44,49 @@ def test_setup_fail(self): def test_chain(self): """Test if filter chaining works.""" + self.init_recorder() config = { + 'history': { + }, 'sensor': { 'platform': 'filter', 'name': 'test', 'entity_id': 'sensor.test_monitored', + 'history_period': '00:05', 'filters': [{ 'filter': 'outlier', + 'window_size': 10, 'radius': 4.0 }, { 'filter': 'lowpass', - 'window_size': 4, 'time_constant': 10, 'precision': 2 }] } } - with assert_setup_component(1): - assert setup_component(self.hass, 'sensor', config) + t_0 = dt_util.utcnow() - timedelta(minutes=1) + t_1 = dt_util.utcnow() - timedelta(minutes=2) + t_2 = dt_util.utcnow() - timedelta(minutes=3) + + fake_states = { + 'sensor.test_monitored': [ + ha.State('sensor.test_monitored', 18.0, last_changed=t_0), + ha.State('sensor.test_monitored', 19.0, last_changed=t_1), + ha.State('sensor.test_monitored', 18.2, last_changed=t_2), + ] + } + + with patch('homeassistant.components.history.' + 'state_changes_during_period', return_value=fake_states): + with assert_setup_component(1, 'sensor'): + assert setup_component(self.hass, 'sensor', config) - for value in self.values: - self.hass.states.set(config['sensor']['entity_id'], value) - self.hass.block_till_done() + for value in self.values: + self.hass.states.set(config['sensor']['entity_id'], value) + self.hass.block_till_done() - state = self.hass.states.get('sensor.test') - self.assertEqual('20.25', state.state) + state = self.hass.states.get('sensor.test') + self.assertEqual('19.25', state.state) def test_outlier(self): """Test if outlier filter works.""" diff --git a/tests/components/test_history.py b/tests/components/test_history.py index bea2af396cbc5..5074b7ed6b293 100644 --- a/tests/components/test_history.py +++ b/tests/components/test_history.py @@ -131,6 +131,39 @@ def set_state(state): self.assertEqual(states, hist[entity_id]) + def test_get_last_state_changes(self): + """Test number of state changes.""" + self.init_recorder() + entity_id = 'sensor.test' + + def set_state(state): + """Set the state.""" + self.hass.states.set(entity_id, state) + self.wait_recording_done() + return self.hass.states.get(entity_id) + + start = dt_util.utcnow() - timedelta(minutes=2) + point = start + timedelta(minutes=1) + point2 = point + timedelta(minutes=1) + + with patch('homeassistant.components.recorder.dt_util.utcnow', + return_value=start): + set_state('1') + + states = [] + with patch('homeassistant.components.recorder.dt_util.utcnow', + return_value=point): + states.append(set_state('2')) + + with patch('homeassistant.components.recorder.dt_util.utcnow', + return_value=point2): + states.append(set_state('3')) + + hist = history.get_last_state_changes( + self.hass, 2, entity_id) + + self.assertEqual(states, hist[entity_id]) + def test_get_significant_states(self): """Test that only significant states are returned. From 6556e8ebb13cab8542d4b22b121a6f00ec62a77d Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Mon, 19 Mar 2018 00:29:14 +0000 Subject: [PATCH 14/27] fix test --- tests/components/test_history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/components/test_history.py b/tests/components/test_history.py index 5074b7ed6b293..5d909492380c1 100644 --- a/tests/components/test_history.py +++ b/tests/components/test_history.py @@ -141,7 +141,7 @@ def set_state(state): self.hass.states.set(entity_id, state) self.wait_recording_done() return self.hass.states.get(entity_id) - + start = dt_util.utcnow() - timedelta(minutes=2) point = start + timedelta(minutes=1) point2 = point + timedelta(minutes=1) From 8e055c227741554e5e4541303780e88ab67d2b4c Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Wed, 21 Mar 2018 00:14:17 +0000 Subject: [PATCH 15/27] Major changes to accommodate history + time_SMA # Conflicts: # homeassistant/components/sensor/filter.py --- homeassistant/components/sensor/filter.py | 131 ++++++++++++++-------- tests/components/sensor/test_filter.py | 37 +++--- 2 files changed, 107 insertions(+), 61 deletions(-) diff --git a/homeassistant/components/sensor/filter.py b/homeassistant/components/sensor/filter.py index 93d2f34dbbfa0..7e1cf5a867c90 100644 --- a/homeassistant/components/sensor/filter.py +++ b/homeassistant/components/sensor/filter.py @@ -9,6 +9,8 @@ from collections import deque, Counter from numbers import Number from functools import partial +from copy import copy +import datetime import voluptuous as vol @@ -34,7 +36,6 @@ CONF_FILTERS = 'filters' CONF_FILTER_NAME = 'filter' -CONF_HISTORY_PERIOD = 'history_period' CONF_FILTER_WINDOW_SIZE = 'window_size' CONF_FILTER_PRECISION = 'precision' CONF_FILTER_RADIUS = 'radius' @@ -90,8 +91,6 @@ PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend({ vol.Required(CONF_ENTITY_ID): cv.entity_id, vol.Optional(CONF_NAME): cv.string, - vol.Optional(CONF_HISTORY_PERIOD): vol.All(cv.time_period, - cv.positive_timedelta), vol.Required(CONF_FILTERS): vol.All(cv.ensure_list, [vol.Any(FILTER_OUTLIER_SCHEMA, FILTER_LOWPASS_SCHEMA, @@ -105,19 +104,18 @@ async def async_setup_platform(hass, config, async_add_devices, """Set up the template sensors.""" name = config.get(CONF_NAME) entity_id = config.get(CONF_ENTITY_ID) - history_period = config.get(CONF_HISTORY_PERIOD) filters = [FILTERS[_filter.pop(CONF_FILTER_NAME)]( entity=entity_id, **_filter) for _filter in config[CONF_FILTERS]] - async_add_devices([SensorFilter(name, entity_id, history_period, filters)]) + async_add_devices([SensorFilter(name, entity_id, filters)]) class SensorFilter(Entity): """Representation of a Filter Sensor.""" - def __init__(self, name, entity_id, history_period, filters): + def __init__(self, name, entity_id, filters): """Initialize the sensor.""" self._name = name self._entity = entity_id @@ -125,7 +123,6 @@ def __init__(self, name, entity_id, history_period, filters): self._state = None self._filters = filters self._icon = None - self._history_period = history_period async def async_added_to_hass(self): """Register callbacks.""" @@ -136,7 +133,7 @@ def filter_sensor_state_listener(entity, old_state, new_state, if new_state.state in [STATE_UNKNOWN, STATE_UNAVAILABLE]: return - temp_state = new_state.state + temp_state = new_state try: for filt in self._filters: @@ -167,15 +164,26 @@ def filter_sensor_state_listener(entity, old_state, new_state, if update_ha: self.async_schedule_update_ha_state() - if self._history_period and 'recorder' in self.hass.config.components: - start = dt_util.utcnow() - self._history_period - - history_list = await self.hass.async_add_job(partial( - history.state_changes_during_period, self.hass, - start, entity_id=self._entity)) - + if 'recorder' in self.hass.config.components: + history_list = [] + for filt in self._filters: + if isinstance(filt.window_size, int): + filter_history = await self.hass.async_add_job(partial( + history.get_last_state_changes, self.hass, + filt.window_size, entity_id=self._entity)) + elif isinstance(filt.window_size, datetime.timedelta): + start = dt_util.utcnow() - filt.window_size + filter_history = await self.hass.async_add_job(partial( + history.state_changes_during_period, self.hass, + start, entity_id=self._entity)) + for _, states in filter_history.items(): + history_list.extend( + [st for st in states if st not in history_list]) + history_list = sorted(history_list, key=lambda s: s.last_updated) + + # Replay history through the filter chain prev_state = None - for state in history_list[self._entity]: + for state in history_list: filter_sensor_state_listener( self._entity, prev_state, state, False) prev_state = state.state @@ -217,6 +225,31 @@ def device_state_attributes(self): return state_attr +class FilterState(object): + """State abstraction for filter usage.""" + + def __init__(self, state): + """Initialize with HA State object.""" + self.timestamp = state.last_updated + try: + self.state = float(state.state) + except ValueError: + self.state = state.state + + def set_precision(self, precision): + """Set precision of Number based states.""" + if isinstance(self.state, Number): + self.state = round(float(self.state), precision) + + def __str__(self): + """Return state as the string representation of FilterState.""" + return str(self.state) + + def __repr__(self): + """Return timestamp and state as the representation of FilterState.""" + return "{} : {}".format(self.timestamp, self.state) + + class Filter(object): """Filter skeleton. @@ -229,11 +262,20 @@ class Filter(object): def __init__(self, name, window_size=1, precision=None, entity=None): """Initialize common attributes.""" - self.states = deque(maxlen=window_size) + if isinstance(window_size, int): + self.states = deque(maxlen=window_size) + else: + self.states = deque(maxlen=0) self.precision = precision self._name = name self._entity = entity self._skip_processing = False + self._window_size = window_size + + @property + def window_size(self): + """Return window size.""" + return self._window_size @property def name(self): @@ -251,11 +293,11 @@ def _filter_state(self, new_state): def filter_state(self, new_state): """Implement a common interface for filters.""" - filtered = self._filter_state(new_state) - if isinstance(filtered, Number): - filtered = round(float(filtered), self.precision) - self.states.append(filtered) - return filtered + filtered = self._filter_state(FilterState(new_state)) + filtered.set_precision(self.precision) + self.states.append(copy(filtered)) + new_state.state = filtered.state + return new_state @FILTERS.register(FILTER_NAME_OUTLIER) @@ -276,11 +318,10 @@ def __init__(self, window_size, precision, entity, radius): def _filter_state(self, new_state): """Implement the outlier filter.""" - new_state = float(new_state) - if (self.states and - abs(new_state - statistics.median(self.states)) - > self._radius): + abs(new_state.state - + statistics.median([s.state for s in self.states])) > + self._radius): self._stats_internal['erasures'] += 1 @@ -306,16 +347,15 @@ def __init__(self, window_size, precision, entity, time_constant): def _filter_state(self, new_state): """Implement the low pass filter.""" - new_state = float(new_state) - if not self.states: return new_state new_weight = 1.0 / self._time_constant prev_weight = 1.0 - new_weight - filtered = prev_weight * self.states[-1] + new_weight * new_state + new_state.state = prev_weight * self.states[-1].state +\ + new_weight * new_state.state - return filtered + return new_state @FILTERS.register(FILTER_NAME_TIME_SMA) @@ -330,35 +370,36 @@ class TimeSMAFilter(Filter): def __init__(self, window_size, precision, entity, type): """Initialize Filter.""" - super().__init__(FILTER_NAME_TIME_SMA, 0, precision, entity) - self._time_window = int(window_size.total_seconds()) + super().__init__(FILTER_NAME_TIME_SMA, window_size, precision, entity) + self._time_window = window_size self.last_leak = None self.queue = deque() - def _leak(self, now): + def _leak(self, left_boundary): """Remove timeouted elements.""" while self.queue: - timestamp, _ = self.queue[0] - if timestamp + self._time_window <= now: + if self.queue[0].timestamp + self._time_window <= left_boundary: self.last_leak = self.queue.popleft() else: return def _filter_state(self, new_state): - now = int(dt_util.utcnow().timestamp()) + """Implement the Simple Moving Average filter.""" + self._leak(new_state.timestamp) + self.queue.append(copy(new_state)) - self._leak(now) - self.queue.append((now, float(new_state))) moving_sum = 0 - start = now - self._time_window - _, prev_val = self.last_leak or (0, float(new_state)) + start = new_state.timestamp - self._time_window + prev_state = self.last_leak or self.queue[0] + for state in self.queue: + moving_sum += (state.timestamp-start).total_seconds()\ + * prev_state.state + start = state.timestamp + prev_state = state - for timestamp, val in self.queue: - moving_sum += (timestamp-start)*prev_val - start, prev_val = timestamp, val - moving_sum += (now-start)*prev_val + new_state.state = moving_sum / self._time_window.total_seconds() - return moving_sum/self._time_window + return new_state @FILTERS.register(FILTER_NAME_THROTTLE) diff --git a/tests/components/sensor/test_filter.py b/tests/components/sensor/test_filter.py index 716227d18538f..257be3845463c 100644 --- a/tests/components/sensor/test_filter.py +++ b/tests/components/sensor/test_filter.py @@ -19,7 +19,13 @@ class TestFilterSensor(unittest.TestCase): def setup_method(self, method): """Setup things to be run when tests are started.""" self.hass = get_test_home_assistant() - self.values = [20, 19, 18, 21, 22, 0] + raw_values = [20, 19, 18, 21, 22, 0] + self.values = [] + + ts = dt_util.utcnow() + for val in raw_values: + self.values.append(ha.State('sensor.test_monitored', val, last_updated=ts)) + ts = ts + timedelta(minutes=1) def teardown_method(self, method): """Stop everything that was started.""" @@ -78,15 +84,17 @@ def test_chain(self): with patch('homeassistant.components.history.' 'state_changes_during_period', return_value=fake_states): - with assert_setup_component(1, 'sensor'): - assert setup_component(self.hass, 'sensor', config) + with patch('homeassistant.components.history.' + 'get_last_state_changes', return_value=fake_states): + with assert_setup_component(1, 'sensor'): + assert setup_component(self.hass, 'sensor', config) - for value in self.values: - self.hass.states.set(config['sensor']['entity_id'], value) - self.hass.block_till_done() + for value in self.values: + self.hass.states.set(config['sensor']['entity_id'], value.state) + self.hass.block_till_done() - state = self.hass.states.get('sensor.test') - self.assertEqual('19.25', state.state) + state = self.hass.states.get('sensor.test_monitored') + self.assertEqual(19.25, state.state) def test_outlier(self): """Test if outlier filter works.""" @@ -96,7 +104,7 @@ def test_outlier(self): radius=4.0) for state in self.values: filtered = filt.filter_state(state) - self.assertEqual(22, filtered) + self.assertEqual(22, filtered.state) def test_lowpass(self): """Test if lowpass filter works.""" @@ -106,7 +114,7 @@ def test_lowpass(self): time_constant=10) for state in self.values: filtered = filt.filter_state(state) - self.assertEqual(18.05, filtered) + self.assertEqual(18.05, filtered.state) def test_throttle(self): """Test if lowpass filter works.""" @@ -118,7 +126,7 @@ def test_throttle(self): new_state = filt.filter_state(state) if not filt.skip_processing: filtered.append(new_state) - self.assertEqual([20, 21], filtered) + self.assertEqual([20, 21], [f.state for f in filtered]) def test_time_sma(self): """Test if time_sma filter works.""" @@ -126,9 +134,6 @@ def test_time_sma(self): precision=2, entity=None, type='last') - past = dt_util.utcnow() - timedelta(minutes=5) for state in self.values: - with patch('homeassistant.util.dt.utcnow', return_value=past): - filtered = filt.filter_state(state) - past += timedelta(minutes=1) - self.assertEqual(21.5, filtered) + filtered = filt.filter_state(state) + self.assertEqual(21.5, filtered.state) From 7a6b0d0a4dd03b836745f3f3a8fad2576585dd4e Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Wed, 21 Mar 2018 00:16:52 +0000 Subject: [PATCH 16/27] hail the hound! --- tests/components/sensor/test_filter.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/components/sensor/test_filter.py b/tests/components/sensor/test_filter.py index 257be3845463c..9a9a1c792c425 100644 --- a/tests/components/sensor/test_filter.py +++ b/tests/components/sensor/test_filter.py @@ -24,7 +24,8 @@ def setup_method(self, method): ts = dt_util.utcnow() for val in raw_values: - self.values.append(ha.State('sensor.test_monitored', val, last_updated=ts)) + self.values.append(ha.State('sensor.test_monitored', + val, last_updated=ts)) ts = ts + timedelta(minutes=1) def teardown_method(self, method): @@ -90,7 +91,8 @@ def test_chain(self): assert setup_component(self.hass, 'sensor', config) for value in self.values: - self.hass.states.set(config['sensor']['entity_id'], value.state) + self.hass.states.set( + config['sensor']['entity_id'], value.state) self.hass.block_till_done() state = self.hass.states.get('sensor.test_monitored') From b55581f43211a861e942bcdfb01e189306d1965e Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Wed, 21 Mar 2018 00:20:44 +0000 Subject: [PATCH 17/27] lint fixed --- tests/components/sensor/test_filter.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/components/sensor/test_filter.py b/tests/components/sensor/test_filter.py index 9a9a1c792c425..1fe8767ab7f32 100644 --- a/tests/components/sensor/test_filter.py +++ b/tests/components/sensor/test_filter.py @@ -8,7 +8,6 @@ import homeassistant.util.dt as dt_util from homeassistant.setup import setup_component import homeassistant.core as ha -import homeassistant.util.dt as dt_util from tests.common import (get_test_home_assistant, assert_setup_component, init_recorder_component) @@ -22,11 +21,11 @@ def setup_method(self, method): raw_values = [20, 19, 18, 21, 22, 0] self.values = [] - ts = dt_util.utcnow() + timestamp = dt_util.utcnow() for val in raw_values: self.values.append(ha.State('sensor.test_monitored', - val, last_updated=ts)) - ts = ts + timedelta(minutes=1) + val, last_updated=timestamp)) + timestamp += timedelta(minutes=1) def teardown_method(self, method): """Stop everything that was started.""" From 39083b63129478c3ecd9b3f3fad9bdaf8df872f7 Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Wed, 21 Mar 2018 00:26:34 +0000 Subject: [PATCH 18/27] less debug --- homeassistant/components/sensor/filter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/sensor/filter.py b/homeassistant/components/sensor/filter.py index 7e1cf5a867c90..6bb6ee626548c 100644 --- a/homeassistant/components/sensor/filter.py +++ b/homeassistant/components/sensor/filter.py @@ -137,12 +137,12 @@ def filter_sensor_state_listener(entity, old_state, new_state, try: for filt in self._filters: - filtered_state = filt.filter_state(temp_state) + filtered_state = filt.filter_state(copy(temp_state)) _LOGGER.debug("%s(%s=%s) -> %s", filt.name, self._entity, - temp_state, + temp_state.state, "skip" if filt.skip_processing else - filtered_state) + filtered_state.state) if filt.skip_processing: return temp_state = filtered_state From cffec75854b7df8c27e020131939580a47029a36 Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Wed, 21 Mar 2018 00:46:12 +0000 Subject: [PATCH 19/27] ups --- homeassistant/components/sensor/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/sensor/filter.py b/homeassistant/components/sensor/filter.py index 6bb6ee626548c..59bcf32ab5f50 100644 --- a/homeassistant/components/sensor/filter.py +++ b/homeassistant/components/sensor/filter.py @@ -151,7 +151,7 @@ def filter_sensor_state_listener(entity, old_state, new_state, self._state) return - self._state = temp_state + self._state = temp_state.state if self._icon is None: self._icon = new_state.attributes.get( From 03e0b8fd5e09394f4938640d67bf966c50b92fc8 Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Wed, 21 Mar 2018 17:00:58 +0000 Subject: [PATCH 20/27] get state from the proper entity --- tests/components/sensor/test_filter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/components/sensor/test_filter.py b/tests/components/sensor/test_filter.py index 1fe8767ab7f32..8b8e7607b0776 100644 --- a/tests/components/sensor/test_filter.py +++ b/tests/components/sensor/test_filter.py @@ -94,8 +94,8 @@ def test_chain(self): config['sensor']['entity_id'], value.state) self.hass.block_till_done() - state = self.hass.states.get('sensor.test_monitored') - self.assertEqual(19.25, state.state) + state = self.hass.states.get('sensor.test') + self.assertEqual('19.25', state.state) def test_outlier(self): """Test if outlier filter works.""" From 333647cc21438766bb392317591a905661086ca9 Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Wed, 21 Mar 2018 17:03:58 +0000 Subject: [PATCH 21/27] sensible default --- homeassistant/components/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/history.py b/homeassistant/components/history.py index 9bd0eaf96e653..52589ce2c896f 100644 --- a/homeassistant/components/history.py +++ b/homeassistant/components/history.py @@ -118,7 +118,7 @@ def state_changes_during_period(hass, start_time, end_time=None, return states_to_json(hass, states, start_time, entity_ids) -def get_last_state_changes(hass, number_of_states, entity_id=None): +def get_last_state_changes(hass, number_of_states=10, entity_id=None): """Return the last number_of_states.""" from homeassistant.components.recorder.models import States From c6cf0993ffc8d602968d884a3dace7815d8fb139 Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Thu, 22 Mar 2018 11:52:22 +0000 Subject: [PATCH 22/27] No defaults in get_last_state_changes --- homeassistant/components/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/history.py b/homeassistant/components/history.py index 52589ce2c896f..0a7f9c8f2c24c 100644 --- a/homeassistant/components/history.py +++ b/homeassistant/components/history.py @@ -118,7 +118,7 @@ def state_changes_during_period(hass, start_time, end_time=None, return states_to_json(hass, states, start_time, entity_ids) -def get_last_state_changes(hass, number_of_states=10, entity_id=None): +def get_last_state_changes(hass, number_of_states, entity_id): """Return the last number_of_states.""" from homeassistant.components.recorder.models import States From d9a83cbafc0b4648d02f16b90d223da49167fa40 Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Thu, 22 Mar 2018 11:55:16 +0000 Subject: [PATCH 23/27] list_reverseiterator instead of list --- homeassistant/components/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/history.py b/homeassistant/components/history.py index 0a7f9c8f2c24c..b5ac37b1451b2 100644 --- a/homeassistant/components/history.py +++ b/homeassistant/components/history.py @@ -136,7 +136,7 @@ def get_last_state_changes(hass, number_of_states, entity_id): states = execute( query.order_by(States.last_updated.desc()).limit(number_of_states)) - return states_to_json(hass, states[::-1], + return states_to_json(hass, reversed(states), start_time, entity_ids, include_start_time_state=False) From 24bfa0e57da6bd9e3c4ba8ea17ca2f6950fbb0b7 Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Thu, 22 Mar 2018 11:56:44 +0000 Subject: [PATCH 24/27] prev_state to state --- homeassistant/components/sensor/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/homeassistant/components/sensor/filter.py b/homeassistant/components/sensor/filter.py index 59bcf32ab5f50..62eefd63e1ca0 100644 --- a/homeassistant/components/sensor/filter.py +++ b/homeassistant/components/sensor/filter.py @@ -186,7 +186,7 @@ def filter_sensor_state_listener(entity, old_state, new_state, for state in history_list: filter_sensor_state_listener( self._entity, prev_state, state, False) - prev_state = state.state + prev_state = state async_track_state_change( self.hass, self._entity, filter_sensor_state_listener) From bbd792f4ad07c5215f37fa6159b15dbeefbeb66b Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Fri, 23 Mar 2018 00:02:50 +0000 Subject: [PATCH 25/27] update --- homeassistant/components/sensor/filter.py | 44 ++++++++++++++++------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/homeassistant/components/sensor/filter.py b/homeassistant/components/sensor/filter.py index 62eefd63e1ca0..66fec21dd1439 100644 --- a/homeassistant/components/sensor/filter.py +++ b/homeassistant/components/sensor/filter.py @@ -10,7 +10,7 @@ from numbers import Number from functools import partial from copy import copy -import datetime +from datetime import timedelta import voluptuous as vol @@ -166,20 +166,38 @@ def filter_sensor_state_listener(entity, old_state, new_state, if 'recorder' in self.hass.config.components: history_list = [] + largest_window_items = 0 + largest_window_time = timedelta(0) + + # Determine the largest window_size by type for filt in self._filters: - if isinstance(filt.window_size, int): - filter_history = await self.hass.async_add_job(partial( - history.get_last_state_changes, self.hass, - filt.window_size, entity_id=self._entity)) - elif isinstance(filt.window_size, datetime.timedelta): - start = dt_util.utcnow() - filt.window_size - filter_history = await self.hass.async_add_job(partial( - history.state_changes_during_period, self.hass, - start, entity_id=self._entity)) - for _, states in filter_history.items(): - history_list.extend( - [st for st in states if st not in history_list]) + if isinstance(filt.window_size, int)\ + and largest_window_items < filt.window_size: + largest_window_items = filt.window_size + elif isinstance(filt.window_size, timedelta)\ + and largest_window_time < filt.window_size: + largest_window_time = filt.window_size + + # Retrieve the largest window_size of each type + if largest_window_items > 0: + filter_history = await self.hass.async_add_job(partial( + history.get_last_state_changes, self.hass, + largest_window_items, entity_id=self._entity)) + history_list.extend( + [state for state in filter_history[self._entity]]) + if largest_window_time > timedelta(seconds=0): + start = dt_util.utcnow() - largest_window_time + filter_history = await self.hass.async_add_job(partial( + history.state_changes_during_period, self.hass, + start, entity_id=self._entity)) + history_list.extend( + [state for state in filter_history[self._entity] + if state not in history_list]) + + # Sort the window states history_list = sorted(history_list, key=lambda s: s.last_updated) + _LOGGER.debug("Loading from history: %s", + [(s.state, s.last_updated) for s in history_list]) # Replay history through the filter chain prev_state = None From 7f05c48cf9fcf92df0dd5f8209e171e40c0c58b3 Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Fri, 6 Apr 2018 22:02:22 +0100 Subject: [PATCH 26/27] added window_unit --- homeassistant/components/sensor/filter.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/homeassistant/components/sensor/filter.py b/homeassistant/components/sensor/filter.py index 66fec21dd1439..4f70d8442aee7 100644 --- a/homeassistant/components/sensor/filter.py +++ b/homeassistant/components/sensor/filter.py @@ -44,6 +44,9 @@ TIME_SMA_LAST = 'last' +WINDOW_SIZE_UNIT_NUMBER_EVENTS = 1 +WINDOW_SIZE_UNIT_TIME = 2 + DEFAULT_WINDOW_SIZE = 1 DEFAULT_PRECISION = 2 DEFAULT_FILTER_RADIUS = 2.0 @@ -282,8 +285,10 @@ def __init__(self, name, window_size=1, precision=None, entity=None): """Initialize common attributes.""" if isinstance(window_size, int): self.states = deque(maxlen=window_size) + self.window_unit = WINDOW_SIZE_UNIT_NUMBER_EVENTS else: self.states = deque(maxlen=0) + self.window_unit = WINDOW_SIZE_UNIT_TIME self.precision = precision self._name = name self._entity = entity From e668f26e0a44986847b8f701f7c7ee3bf0a11cff Mon Sep 17 00:00:00 2001 From: Diogo Gomes Date: Fri, 6 Apr 2018 22:03:51 +0100 Subject: [PATCH 27/27] replace isinstance with window_unit --- homeassistant/components/sensor/filter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/homeassistant/components/sensor/filter.py b/homeassistant/components/sensor/filter.py index 4f70d8442aee7..27730a8f63e4e 100644 --- a/homeassistant/components/sensor/filter.py +++ b/homeassistant/components/sensor/filter.py @@ -174,10 +174,10 @@ def filter_sensor_state_listener(entity, old_state, new_state, # Determine the largest window_size by type for filt in self._filters: - if isinstance(filt.window_size, int)\ + if filt.window_unit == WINDOW_SIZE_UNIT_NUMBER_EVENTS\ and largest_window_items < filt.window_size: largest_window_items = filt.window_size - elif isinstance(filt.window_size, timedelta)\ + elif filt.window_unit == WINDOW_SIZE_UNIT_TIME\ and largest_window_time < filt.window_size: largest_window_time = filt.window_size