Avoid returning statistics columns that the metadata knows are impossible

homeassistant/components/recorder/statistics.py

@@ -1518,6 +1518,28 @@ def _generate_select_columns_for_types_stmt(
     return lambda_stmt(lambda: columns, track_on=track_on)
 
 
+def _extract_metadata_and_discard_impossible_columns(
+    metadata: dict[str, tuple[int, StatisticMetaData]],
+    types: set[Literal["last_reset", "max", "mean", "min", "state", "sum"]],
+) -> list[int]:
+    """Extract metadata ids from metadata and discard impossible columns."""
+    metadata_ids = []
+    has_mean = False
+    has_sum = False
+    for metadata_id, stats_metadata in metadata.values():
+        metadata_ids.append(metadata_id)
+        has_mean |= stats_metadata["has_mean"]
+        has_sum |= stats_metadata["has_sum"]
+    if not has_mean:
+        types.discard("mean")
+        types.discard("min")
+        types.discard("max")
+    if not has_sum:
+        types.discard("sum")
+        types.discard("state")
+    return metadata_ids
+
+
 def _statistics_during_period_with_session(
     hass: HomeAssistant,
     session: Session,
@@ -1547,7 +1569,7 @@ def _statistics_during_period_with_session(
 
     metadata_ids = None
     if statistic_ids is not None:
-        metadata_ids = [metadata_id for metadata_id, _ in metadata.values()]
+        metadata_ids = _extract_metadata_and_discard_impossible_columns(metadata, types)
 
     table: type[Statistics | StatisticsShortTerm] = (
         Statistics if period != "5minute" else StatisticsShortTerm
@@ -1661,7 +1683,8 @@ def _get_last_statistics(
         )
         if not metadata:
             return {}
-        metadata_id = metadata[statistic_id][0]
+        metadata_ids = _extract_metadata_and_discard_impossible_columns(metadata, types)
+        metadata_id = metadata_ids[0]
         if table == Statistics:
             stmt = _get_last_statistics_stmt(metadata_id, number_of_stats)
         else:
@@ -1753,11 +1776,7 @@ def get_latest_short_term_statistics(
             )
         if not metadata:
             return {}
-        metadata_ids = [
-            metadata[statistic_id][0]
-            for statistic_id in statistic_ids
-            if statistic_id in metadata
-        ]
+        metadata_ids = _extract_metadata_and_discard_impossible_columns(metadata, types)
         stmt = _latest_short_term_statistics_stmt(metadata_ids)
         stats = cast(
             Sequence[Row], execute_stmt_lambda_element(session, stmt, orm_rows=False)

homeassistant/components/sensor/recorder.py

@@ -557,8 +557,11 @@ def _compile_statistics(  # noqa: C901
                 last_stat = last_stats[entity_id][0]
                 last_reset = _timestamp_to_isoformat_or_none(last_stat["last_reset"])
                 old_last_reset = last_reset
-                new_state = old_state = last_stat["state"]
-                _sum = last_stat["sum"] or 0.0
+                # If there are no previous values and has_sum
+                # was previously false there will be no last_stat
+                # for state or sum
+                new_state = old_state = last_stat.get("state")
+                _sum = last_stat.get("sum") or 0.0
 
             for fstate, state in valid_float_states:
                 reset = False

tests/components/recorder/test_statistics.py

@@ -92,15 +92,19 @@ def test_compile_hourly_statistics(hass_recorder: Callable[..., HomeAssistant])
     do_adhoc_statistics(hass, start=zero)
     do_adhoc_statistics(hass, start=four)
     wait_recording_done(hass)
+
+    metadata = get_metadata(hass, statistic_ids={"sensor.test1", "sensor.test2"})
+    assert metadata["sensor.test1"][1]["has_mean"] is True
+    assert metadata["sensor.test1"][1]["has_sum"] is False
+    assert metadata["sensor.test2"][1]["has_mean"] is True
+    assert metadata["sensor.test2"][1]["has_sum"] is False
     expected_1 = {
         "start": process_timestamp(zero).timestamp(),
         "end": process_timestamp(zero + timedelta(minutes=5)).timestamp(),
         "mean": pytest.approx(14.915254237288135),
         "min": pytest.approx(10.0),
         "max": pytest.approx(20.0),
         "last_reset": None,
-        "state": None,
-        "sum": None,
     }
     expected_2 = {
         "start": process_timestamp(four).timestamp(),
@@ -109,32 +113,44 @@ def test_compile_hourly_statistics(hass_recorder: Callable[..., HomeAssistant])
         "min": pytest.approx(20.0),
         "max": pytest.approx(20.0),
         "last_reset": None,
-        "state": None,
-        "sum": None,
     }
     expected_stats1 = [expected_1, expected_2]
     expected_stats2 = [expected_1, expected_2]
 
     # Test statistics_during_period
-    stats = statistics_during_period(hass, zero, period="5minute")
+    stats = statistics_during_period(
+        hass, zero, period="5minute", statistic_ids={"sensor.test1", "sensor.test2"}
+    )
     assert stats == {"sensor.test1": expected_stats1, "sensor.test2": expected_stats2}
 
     # Test statistics_during_period with a far future start and end date
     future = dt_util.as_utc(dt_util.parse_datetime("2221-11-01 00:00:00"))
-    stats = statistics_during_period(hass, future, end_time=future, period="5minute")
+    stats = statistics_during_period(
+        hass,
+        future,
+        end_time=future,
+        period="5minute",
+        statistic_ids={"sensor.test1", "sensor.test2"},
+    )
     assert stats == {}
 
     # Test statistics_during_period with a far future end date
-    stats = statistics_during_period(hass, zero, end_time=future, period="5minute")
+    stats = statistics_during_period(
+        hass,
+        zero,
+        end_time=future,
+        period="5minute",
+        statistic_ids={"sensor.test1", "sensor.test2"},
+    )
     assert stats == {"sensor.test1": expected_stats1, "sensor.test2": expected_stats2}
 
     stats = statistics_during_period(
-        hass, zero, statistic_ids=["sensor.test2"], period="5minute"
+        hass, zero, statistic_ids={"sensor.test2"}, period="5minute"
     )
     assert stats == {"sensor.test2": expected_stats2}
 
     stats = statistics_during_period(
-        hass, zero, statistic_ids=["sensor.test3"], period="5minute"
+        hass, zero, statistic_ids={"sensor.test3"}, period="5minute"
     )
     assert stats == {}
 
@@ -567,25 +583,21 @@ async def test_import_statistics(
 
     import_fn(hass, external_metadata, (external_statistics1, external_statistics2))
     await async_wait_recording_done(hass)
-    stats = statistics_during_period(hass, zero, period="hour")
+    stats = statistics_during_period(
+        hass, zero, period="hour", statistic_ids={statistic_id}
+    )
     assert stats == {
         statistic_id: [
             {
                 "start": process_timestamp(period1).timestamp(),
                 "end": process_timestamp(period1 + timedelta(hours=1)).timestamp(),
-                "max": None,
-                "mean": None,
-                "min": None,
                 "last_reset": datetime_to_timestamp_or_none(last_reset_utc),
                 "state": pytest.approx(0.0),
                 "sum": pytest.approx(2.0),
             },
             {
                 "start": process_timestamp(period2).timestamp(),
                 "end": process_timestamp(period2 + timedelta(hours=1)).timestamp(),
-                "max": None,
-                "mean": None,
-                "min": None,
                 "last_reset": datetime_to_timestamp_or_none(last_reset_utc),
                 "state": pytest.approx(1.0),
                 "sum": pytest.approx(3.0),
@@ -631,9 +643,6 @@ async def test_import_statistics(
             {
                 "start": process_timestamp(period2).timestamp(),
                 "end": process_timestamp(period2 + timedelta(hours=1)).timestamp(),
-                "max": None,
-                "mean": None,
-                "min": None,
                 "last_reset": datetime_to_timestamp_or_none(last_reset_utc),
                 "state": pytest.approx(1.0),
                 "sum": pytest.approx(3.0),
@@ -650,25 +659,21 @@ async def test_import_statistics(
     }
     import_fn(hass, external_metadata, (external_statistics,))
     await async_wait_recording_done(hass)
-    stats = statistics_during_period(hass, zero, period="hour")
+    stats = statistics_during_period(
+        hass, zero, period="hour", statistic_ids={statistic_id}
+    )
     assert stats == {
         statistic_id: [
             {
                 "start": process_timestamp(period1).timestamp(),
                 "end": process_timestamp(period1 + timedelta(hours=1)).timestamp(),
-                "max": None,
-                "mean": None,
-                "min": None,
                 "last_reset": None,
                 "state": pytest.approx(5.0),
                 "sum": pytest.approx(6.0),
             },
             {
                 "start": process_timestamp(period2).timestamp(),
                 "end": process_timestamp(period2 + timedelta(hours=1)).timestamp(),
-                "max": None,
-                "mean": None,
-                "min": None,
                 "last_reset": datetime_to_timestamp_or_none(last_reset_utc),
                 "state": pytest.approx(1.0),
                 "sum": pytest.approx(3.0),
@@ -716,25 +721,21 @@ async def test_import_statistics(
             },
         )
     }
-    stats = statistics_during_period(hass, zero, period="hour")
+    stats = statistics_during_period(
+        hass, zero, period="hour", statistic_ids={statistic_id}
+    )
     assert stats == {
         statistic_id: [
             {
                 "start": process_timestamp(period1).timestamp(),
                 "end": process_timestamp(period1 + timedelta(hours=1)).timestamp(),
-                "max": pytest.approx(1.0),
-                "mean": pytest.approx(2.0),
-                "min": pytest.approx(3.0),
                 "last_reset": datetime_to_timestamp_or_none(last_reset_utc),
                 "state": pytest.approx(4.0),
                 "sum": pytest.approx(5.0),
             },
             {
                 "start": process_timestamp(period2).timestamp(),
                 "end": process_timestamp(period2 + timedelta(hours=1)).timestamp(),
-                "max": None,
-                "mean": None,
-                "min": None,
                 "last_reset": datetime_to_timestamp_or_none(last_reset_utc),
                 "state": pytest.approx(1.0),
                 "sum": pytest.approx(3.0),
@@ -757,25 +758,21 @@ async def test_import_statistics(
     assert response["success"]
 
     await async_wait_recording_done(hass)
-    stats = statistics_during_period(hass, zero, period="hour")
+    stats = statistics_during_period(
+        hass, zero, period="hour", statistic_ids={statistic_id}
+    )
     assert stats == {
         statistic_id: [
             {
                 "start": process_timestamp(period1).timestamp(),
                 "end": process_timestamp(period1 + timedelta(hours=1)).timestamp(),
-                "max": pytest.approx(1.0),
-                "mean": pytest.approx(2.0),
-                "min": pytest.approx(3.0),
                 "last_reset": datetime_to_timestamp_or_none(last_reset_utc),
                 "state": pytest.approx(4.0),
                 "sum": pytest.approx(5.0),
             },
             {
                 "start": process_timestamp(period2).timestamp(),
                 "end": process_timestamp(period2 + timedelta(hours=1)).timestamp(),
-                "max": None,
-                "mean": None,
-                "min": None,
                 "last_reset": datetime_to_timestamp_or_none(last_reset_utc),
                 "state": pytest.approx(1.0),
                 "sum": pytest.approx(1000 * 1000 + 3.0),
@@ -1020,7 +1017,9 @@ def test_weekly_statistics(
 
     async_add_external_statistics(hass, external_metadata, external_statistics)
     wait_recording_done(hass)
-    stats = statistics_during_period(hass, zero, period="week")
+    stats = statistics_during_period(
+        hass, zero, period="week", statistic_ids={"test:total_energy_import"}
+    )
     week1_start = dt_util.as_utc(dt_util.parse_datetime("2022-10-03 00:00:00"))
     week1_end = dt_util.as_utc(dt_util.parse_datetime("2022-10-10 00:00:00"))
     week2_start = dt_util.as_utc(dt_util.parse_datetime("2022-10-10 00:00:00"))
@@ -1030,19 +1029,13 @@ def test_weekly_statistics(
             {
                 "start": week1_start.timestamp(),
                 "end": week1_end.timestamp(),
-                "max": None,
-                "mean": None,
-                "min": None,
                 "last_reset": None,
                 "state": 1.0,
                 "sum": 3.0,
             },
             {
                 "start": week2_start.timestamp(),
                 "end": week2_end.timestamp(),
-                "max": None,
-                "mean": None,
-                "min": None,
                 "last_reset": None,
                 "state": 3.0,
                 "sum": 5.0,
@@ -1061,19 +1054,13 @@ def test_weekly_statistics(
             {
                 "start": week1_start.timestamp(),
                 "end": week1_end.timestamp(),
-                "max": None,
-                "mean": None,
-                "min": None,
                 "last_reset": None,
                 "state": 1.0,
                 "sum": 3.0,
             },
             {
                 "start": week2_start.timestamp(),
                 "end": week2_end.timestamp(),
-                "max": None,
-                "mean": None,
-                "min": None,
                 "last_reset": None,
                 "state": 3.0,
                 "sum": 5.0,
@@ -1158,7 +1145,9 @@ def test_monthly_statistics(
 
     async_add_external_statistics(hass, external_metadata, external_statistics)
     wait_recording_done(hass)
-    stats = statistics_during_period(hass, zero, period="month")
+    stats = statistics_during_period(
+        hass, zero, period="month", statistic_ids={"test:total_energy_import"}
+    )
     sep_start = dt_util.as_utc(dt_util.parse_datetime("2021-09-01 00:00:00"))
     sep_end = dt_util.as_utc(dt_util.parse_datetime("2021-10-01 00:00:00"))
     oct_start = dt_util.as_utc(dt_util.parse_datetime("2021-10-01 00:00:00"))
@@ -1168,19 +1157,13 @@ def test_monthly_statistics(
             {
                 "start": sep_start.timestamp(),
                 "end": sep_end.timestamp(),
-                "max": None,
-                "mean": None,
-                "min": None,
                 "last_reset": None,
                 "state": pytest.approx(1.0),
                 "sum": pytest.approx(3.0),
             },
             {
                 "start": oct_start.timestamp(),
                 "end": oct_end.timestamp(),
-                "max": None,
-                "mean": None,
-                "min": None,
                 "last_reset": None,
                 "state": pytest.approx(3.0),
                 "sum": pytest.approx(5.0),
@@ -1203,19 +1186,13 @@ def test_monthly_statistics(
             {
                 "start": sep_start.timestamp(),
                 "end": sep_end.timestamp(),
-                "max": None,
-                "mean": None,
-                "min": None,
                 "last_reset": None,
                 "state": pytest.approx(1.0),
                 "sum": pytest.approx(3.0),
             },
             {
                 "start": oct_start.timestamp(),
                 "end": oct_end.timestamp(),
-                "max": None,
-                "mean": None,
-                "min": None,
                 "last_reset": None,
                 "state": pytest.approx(3.0),
                 "sum": pytest.approx(5.0),