From 5ea6684aefd2aaeea7ec3f08add2e998e7120adc Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 14 Feb 2023 15:24:49 -0600 Subject: [PATCH 1/4] Remove profiler.memory service guppy3 is not python3.11 compat https://github.com/zhuyifei1999/guppy3/issues/41 This service will return if and when guppy3 becomes python3.11 compat --- homeassistant/components/profiler/__init__.py | 51 ------------------- .../components/profiler/manifest.json | 2 +- .../components/profiler/services.yaml | 13 ----- requirements_all.txt | 3 -- requirements_test_all.txt | 3 -- tests/components/profiler/test_init.py | 31 ----------- 6 files changed, 1 insertion(+), 102 deletions(-) diff --git a/homeassistant/components/profiler/__init__.py b/homeassistant/components/profiler/__init__.py index 9b4164c9f4144e..d2a94dfae05c45 100644 --- a/homeassistant/components/profiler/__init__.py +++ b/homeassistant/components/profiler/__init__.py @@ -22,7 +22,6 @@ from .const import DOMAIN SERVICE_START = "start" -SERVICE_MEMORY = "memory" SERVICE_START_LOG_OBJECTS = "start_log_objects" SERVICE_STOP_LOG_OBJECTS = "stop_log_objects" SERVICE_DUMP_LOG_OBJECTS = "dump_log_objects" @@ -32,7 +31,6 @@ SERVICES = ( SERVICE_START, - SERVICE_MEMORY, SERVICE_START_LOG_OBJECTS, SERVICE_STOP_LOG_OBJECTS, SERVICE_DUMP_LOG_OBJECTS, @@ -58,10 +56,6 @@ async def _async_run_profile(call: ServiceCall) -> None: async with lock: await _async_generate_profile(hass, call) - async def _async_run_memory_profile(call: ServiceCall) -> None: - async with lock: - await _async_generate_memory_profile(hass, call) - async def _async_start_log_objects(call: ServiceCall) -> None: if LOG_INTERVAL_SUB in domain_data: domain_data[LOG_INTERVAL_SUB]() @@ -162,16 +156,6 @@ async def _async_dump_scheduled(call: ServiceCall) -> None: ), ) - async_register_admin_service( - hass, - DOMAIN, - SERVICE_MEMORY, - _async_run_memory_profile, - schema=vol.Schema( - {vol.Optional(CONF_SECONDS, default=60.0): vol.Coerce(float)} - ), - ) - async_register_admin_service( hass, DOMAIN, @@ -265,37 +249,6 @@ async def _async_generate_profile(hass: HomeAssistant, call: ServiceCall): ) -async def _async_generate_memory_profile(hass: HomeAssistant, call: ServiceCall): - # Imports deferred to avoid loading modules - # in memory since usually only one part of this - # integration is used at a time - from guppy import hpy # pylint: disable=import-outside-toplevel - - start_time = int(time.time() * 1000000) - persistent_notification.async_create( - hass, - ( - "The memory profile has started. This notification will be updated when it" - " is complete." - ), - title="Profile Started", - notification_id=f"memory_profiler_{start_time}", - ) - heap_profiler = hpy() - heap_profiler.setref() - await asyncio.sleep(float(call.data[CONF_SECONDS])) - heap = heap_profiler.heap() - - heap_path = hass.config.path(f"heap_profile.{start_time}.hpy") - await hass.async_add_executor_job(_write_memory_profile, heap, heap_path) - persistent_notification.async_create( - hass, - f"Wrote heapy memory profile to {heap_path}", - title="Profile Complete", - notification_id=f"memory_profiler_{start_time}", - ) - - def _write_profile(profiler, cprofile_path, callgrind_path): # Imports deferred to avoid loading modules # in memory since usually only one part of this @@ -307,10 +260,6 @@ def _write_profile(profiler, cprofile_path, callgrind_path): convert(profiler.getstats(), callgrind_path) -def _write_memory_profile(heap, heap_path): - heap.byrcs.dump(heap_path) - - def _log_objects(*_): # Imports deferred to avoid loading modules # in memory since usually only one part of this diff --git a/homeassistant/components/profiler/manifest.json b/homeassistant/components/profiler/manifest.json index 11896c02fdd3b6..ce4a5b2bf3e1b3 100644 --- a/homeassistant/components/profiler/manifest.json +++ b/homeassistant/components/profiler/manifest.json @@ -5,5 +5,5 @@ "config_flow": true, "documentation": "https://www.home-assistant.io/integrations/profiler", "quality_scale": "internal", - "requirements": ["pyprof2calltree==1.4.5", "guppy3==3.1.2", "objgraph==3.5.0"] + "requirements": ["pyprof2calltree==1.4.5", "objgraph==3.5.0"] } diff --git a/homeassistant/components/profiler/services.yaml b/homeassistant/components/profiler/services.yaml index 8d9ae35ed1074e..da3128ca60abe0 100644 --- a/homeassistant/components/profiler/services.yaml +++ b/homeassistant/components/profiler/services.yaml @@ -11,19 +11,6 @@ start: min: 1 max: 3600 unit_of_measurement: seconds -memory: - name: Memory - description: Start the Memory Profiler - fields: - seconds: - name: Seconds - description: The number of seconds to run the memory profiler. - default: 60.0 - selector: - number: - min: 1 - max: 3600 - unit_of_measurement: seconds start_log_objects: name: Start log objects description: Start logging growth of objects in memory diff --git a/requirements_all.txt b/requirements_all.txt index 9e452e9d5df5fb..0e361981a359a7 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -848,9 +848,6 @@ gspread==5.5.0 # homeassistant.components.gstreamer gstreamer-player==1.1.2 -# homeassistant.components.profiler -guppy3==3.1.2 - # homeassistant.components.iaqualink h2==4.1.0 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 7e127811fc382c..06e7e624de85d4 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -643,9 +643,6 @@ growattServer==1.3.0 # homeassistant.components.google_sheets gspread==5.5.0 -# homeassistant.components.profiler -guppy3==3.1.2 - # homeassistant.components.iaqualink h2==4.1.0 diff --git a/tests/components/profiler/test_init.py b/tests/components/profiler/test_init.py index 0fc70d3ce0d891..ade27f1ad16033 100644 --- a/tests/components/profiler/test_init.py +++ b/tests/components/profiler/test_init.py @@ -10,7 +10,6 @@ SERVICE_DUMP_LOG_OBJECTS, SERVICE_LOG_EVENT_LOOP_SCHEDULED, SERVICE_LOG_THREAD_FRAMES, - SERVICE_MEMORY, SERVICE_START, SERVICE_START_LOG_OBJECTS, SERVICE_STOP_LOG_OBJECTS, @@ -53,36 +52,6 @@ def _mock_path(filename): await hass.async_block_till_done() -async def test_memory_usage(hass, tmpdir): - """Test we can setup and the service is registered.""" - test_dir = tmpdir.mkdir("profiles") - - entry = MockConfigEntry(domain=DOMAIN) - entry.add_to_hass(hass) - - assert await hass.config_entries.async_setup(entry.entry_id) - await hass.async_block_till_done() - - assert hass.services.has_service(DOMAIN, SERVICE_MEMORY) - - last_filename = None - - def _mock_path(filename): - nonlocal last_filename - last_filename = f"{test_dir}/{filename}" - return last_filename - - with patch("guppy.hpy") as mock_hpy, patch.object(hass.config, "path", _mock_path): - await hass.services.async_call( - DOMAIN, SERVICE_MEMORY, {CONF_SECONDS: 0.000001}, blocking=True - ) - - mock_hpy.assert_called_once() - - assert await hass.config_entries.async_unload(entry.entry_id) - await hass.async_block_till_done() - - async def test_object_growth_logging( hass: HomeAssistant, caplog: pytest.LogCaptureFixture ) -> None: From 789b0b42e50d6c21e77d9126a8c0003dc11de2d4 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Feb 2023 08:13:53 -0600 Subject: [PATCH 2/4] conditional --- homeassistant/components/profiler/__init__.py | 57 +++++++++++++++++++ .../components/profiler/manifest.json | 6 +- .../components/profiler/services.yaml | 13 +++++ requirements_all.txt | 3 + requirements_test_all.txt | 3 + script/hassfest/requirements.py | 2 +- tests/components/profiler/test_init.py | 33 ++++++++++- tests/hassfest/test_requirements.py | 1 + 8 files changed, 115 insertions(+), 3 deletions(-) diff --git a/homeassistant/components/profiler/__init__.py b/homeassistant/components/profiler/__init__.py index d2a94dfae05c45..fab6932edd2049 100644 --- a/homeassistant/components/profiler/__init__.py +++ b/homeassistant/components/profiler/__init__.py @@ -15,6 +15,7 @@ from homeassistant.config_entries import ConfigEntry from homeassistant.const import CONF_SCAN_INTERVAL, CONF_TYPE from homeassistant.core import HomeAssistant, ServiceCall +from homeassistant.exceptions import HomeAssistantError import homeassistant.helpers.config_validation as cv from homeassistant.helpers.event import async_track_time_interval from homeassistant.helpers.service import async_register_admin_service @@ -22,6 +23,7 @@ from .const import DOMAIN SERVICE_START = "start" +SERVICE_MEMORY = "memory" SERVICE_START_LOG_OBJECTS = "start_log_objects" SERVICE_STOP_LOG_OBJECTS = "stop_log_objects" SERVICE_DUMP_LOG_OBJECTS = "dump_log_objects" @@ -31,6 +33,7 @@ SERVICES = ( SERVICE_START, + SERVICE_MEMORY, SERVICE_START_LOG_OBJECTS, SERVICE_STOP_LOG_OBJECTS, SERVICE_DUMP_LOG_OBJECTS, @@ -56,6 +59,10 @@ async def _async_run_profile(call: ServiceCall) -> None: async with lock: await _async_generate_profile(hass, call) + async def _async_run_memory_profile(call: ServiceCall) -> None: + async with lock: + await _async_generate_memory_profile(hass, call) + async def _async_start_log_objects(call: ServiceCall) -> None: if LOG_INTERVAL_SUB in domain_data: domain_data[LOG_INTERVAL_SUB]() @@ -156,6 +163,16 @@ async def _async_dump_scheduled(call: ServiceCall) -> None: ), ) + async_register_admin_service( + hass, + DOMAIN, + SERVICE_MEMORY, + _async_run_memory_profile, + schema=vol.Schema( + {vol.Optional(CONF_SECONDS, default=60.0): vol.Coerce(float)} + ), + ) + async_register_admin_service( hass, DOMAIN, @@ -249,6 +266,42 @@ async def _async_generate_profile(hass: HomeAssistant, call: ServiceCall): ) +async def _async_generate_memory_profile(hass: HomeAssistant, call: ServiceCall): + # Imports deferred to avoid loading modules + # in memory since usually only one part of this + # integration is used at a time + if sys.version_info >= (3, 11): + raise HomeAssistantError( + "Memory profiling is not supported on Python 3.11. Please use Python 3.10." + ) + + from guppy import hpy # pylint: disable=import-outside-toplevel + + start_time = int(time.time() * 1000000) + persistent_notification.async_create( + hass, + ( + "The memory profile has started. This notification will be updated when it" + " is complete." + ), + title="Profile Started", + notification_id=f"memory_profiler_{start_time}", + ) + heap_profiler = hpy() + heap_profiler.setref() + await asyncio.sleep(float(call.data[CONF_SECONDS])) + heap = heap_profiler.heap() + + heap_path = hass.config.path(f"heap_profile.{start_time}.hpy") + await hass.async_add_executor_job(_write_memory_profile, heap, heap_path) + persistent_notification.async_create( + hass, + f"Wrote heapy memory profile to {heap_path}", + title="Profile Complete", + notification_id=f"memory_profiler_{start_time}", + ) + + def _write_profile(profiler, cprofile_path, callgrind_path): # Imports deferred to avoid loading modules # in memory since usually only one part of this @@ -260,6 +313,10 @@ def _write_profile(profiler, cprofile_path, callgrind_path): convert(profiler.getstats(), callgrind_path) +def _write_memory_profile(heap, heap_path): + heap.byrcs.dump(heap_path) + + def _log_objects(*_): # Imports deferred to avoid loading modules # in memory since usually only one part of this diff --git a/homeassistant/components/profiler/manifest.json b/homeassistant/components/profiler/manifest.json index ce4a5b2bf3e1b3..81eb77537fb7a6 100644 --- a/homeassistant/components/profiler/manifest.json +++ b/homeassistant/components/profiler/manifest.json @@ -5,5 +5,9 @@ "config_flow": true, "documentation": "https://www.home-assistant.io/integrations/profiler", "quality_scale": "internal", - "requirements": ["pyprof2calltree==1.4.5", "objgraph==3.5.0"] + "requirements": [ + "pyprof2calltree==1.4.5", + "guppy3==3.1.2;python_version<'3.11'", + "objgraph==3.5.0" + ] } diff --git a/homeassistant/components/profiler/services.yaml b/homeassistant/components/profiler/services.yaml index da3128ca60abe0..8d9ae35ed1074e 100644 --- a/homeassistant/components/profiler/services.yaml +++ b/homeassistant/components/profiler/services.yaml @@ -11,6 +11,19 @@ start: min: 1 max: 3600 unit_of_measurement: seconds +memory: + name: Memory + description: Start the Memory Profiler + fields: + seconds: + name: Seconds + description: The number of seconds to run the memory profiler. + default: 60.0 + selector: + number: + min: 1 + max: 3600 + unit_of_measurement: seconds start_log_objects: name: Start log objects description: Start logging growth of objects in memory diff --git a/requirements_all.txt b/requirements_all.txt index 0e361981a359a7..b79f3f1d6c56d9 100644 --- a/requirements_all.txt +++ b/requirements_all.txt @@ -848,6 +848,9 @@ gspread==5.5.0 # homeassistant.components.gstreamer gstreamer-player==1.1.2 +# homeassistant.components.profiler +guppy3==3.1.2;python_version<'3.11' + # homeassistant.components.iaqualink h2==4.1.0 diff --git a/requirements_test_all.txt b/requirements_test_all.txt index 06e7e624de85d4..c68e002e9436af 100644 --- a/requirements_test_all.txt +++ b/requirements_test_all.txt @@ -643,6 +643,9 @@ growattServer==1.3.0 # homeassistant.components.google_sheets gspread==5.5.0 +# homeassistant.components.profiler +guppy3==3.1.2;python_version<'3.11' + # homeassistant.components.iaqualink h2==4.1.0 diff --git a/script/hassfest/requirements.py b/script/hassfest/requirements.py index 8e4ac524b2f760..8b9f73336fe484 100644 --- a/script/hassfest/requirements.py +++ b/script/hassfest/requirements.py @@ -90,7 +90,7 @@ def validate_requirements_format(integration: Integration) -> bool: if not version: continue - for part in version.split(","): + for part in version.split(";", 1)[0].split(","): version_part = PIP_VERSION_RANGE_SEPARATOR.match(part) if ( version_part diff --git a/tests/components/profiler/test_init.py b/tests/components/profiler/test_init.py index ade27f1ad16033..a83bd3b2b1bfd7 100644 --- a/tests/components/profiler/test_init.py +++ b/tests/components/profiler/test_init.py @@ -10,6 +10,7 @@ SERVICE_DUMP_LOG_OBJECTS, SERVICE_LOG_EVENT_LOOP_SCHEDULED, SERVICE_LOG_THREAD_FRAMES, + SERVICE_MEMORY, SERVICE_START, SERVICE_START_LOG_OBJECTS, SERVICE_STOP_LOG_OBJECTS, @@ -22,7 +23,7 @@ from tests.common import MockConfigEntry, async_fire_time_changed -async def test_basic_usage(hass, tmpdir): +async def test_basic_usage(hass: HomeAssistant, tmpdir) -> None: """Test we can setup and the service is registered.""" test_dir = tmpdir.mkdir("profiles") @@ -52,6 +53,36 @@ def _mock_path(filename): await hass.async_block_till_done() +async def test_memory_usage(hass: HomeAssistant, tmpdir) -> None: + """Test we can setup and the service is registered.""" + test_dir = tmpdir.mkdir("profiles") + + entry = MockConfigEntry(domain=DOMAIN) + entry.add_to_hass(hass) + + assert await hass.config_entries.async_setup(entry.entry_id) + await hass.async_block_till_done() + + assert hass.services.has_service(DOMAIN, SERVICE_MEMORY) + + last_filename = None + + def _mock_path(filename): + nonlocal last_filename + last_filename = f"{test_dir}/{filename}" + return last_filename + + with patch("guppy.hpy") as mock_hpy, patch.object(hass.config, "path", _mock_path): + await hass.services.async_call( + DOMAIN, SERVICE_MEMORY, {CONF_SECONDS: 0.000001}, blocking=True + ) + + mock_hpy.assert_called_once() + + assert await hass.config_entries.async_unload(entry.entry_id) + await hass.async_block_till_done() + + async def test_object_growth_logging( hass: HomeAssistant, caplog: pytest.LogCaptureFixture ) -> None: diff --git a/tests/hassfest/test_requirements.py b/tests/hassfest/test_requirements.py index cb73e5b9603f88..174bf84178839c 100644 --- a/tests/hassfest/test_requirements.py +++ b/tests/hassfest/test_requirements.py @@ -52,6 +52,7 @@ def test_validate_requirements_format_ignore_pin_for_custom(integration: Integra "test_package~=0.5.0", "test_package>=1.4.2,<1.4.99,>=1.7,<1.8.99", "test_package>=1.4.2,<1.9,!=1.5", + "test_package>=1.4.2;python_version<'3.11'", ] integration.path = Path("") assert validate_requirements_format(integration) From c923fe179aee7fe5d5c2d79b2d93eacc8821279d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Feb 2023 08:17:17 -0600 Subject: [PATCH 3/4] adjust est --- tests/components/profiler/test_init.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/components/profiler/test_init.py b/tests/components/profiler/test_init.py index a83bd3b2b1bfd7..aac49c89b7eab5 100644 --- a/tests/components/profiler/test_init.py +++ b/tests/components/profiler/test_init.py @@ -1,6 +1,7 @@ """Test the Profiler config flow.""" from datetime import timedelta import os +import sys from unittest.mock import patch import pytest @@ -53,6 +54,9 @@ def _mock_path(filename): await hass.async_block_till_done() +@pytest.mark.skipif( + sys.version_info >= (3, 11), reason="not yet available on python 3.11" +) async def test_memory_usage(hass: HomeAssistant, tmpdir) -> None: """Test we can setup and the service is registered.""" test_dir = tmpdir.mkdir("profiles") @@ -83,6 +87,16 @@ def _mock_path(filename): await hass.async_block_till_done() +@pytest.mark.skipif(sys.version_info < (3, 11), reason="still works on python 3.10") +async def test_memory_usage_py311(hass: HomeAssistant, tmpdir) -> None: + """Test raise an error on python3.11.""" + assert hass.services.has_service(DOMAIN, SERVICE_MEMORY) + with pytest.raises(HomeAssistant, match="not yet available on python 3.11"): + await hass.services.async_call( + DOMAIN, SERVICE_MEMORY, {CONF_SECONDS: 0.000001}, blocking=True + ) + + async def test_object_growth_logging( hass: HomeAssistant, caplog: pytest.LogCaptureFixture ) -> None: From 9b6fdc86a8a3137e4073b8670c38b6ddd62e5150 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Feb 2023 08:19:39 -0600 Subject: [PATCH 4/4] test --- tests/components/profiler/test_init.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/components/profiler/test_init.py b/tests/components/profiler/test_init.py index aac49c89b7eab5..636067341e11cc 100644 --- a/tests/components/profiler/test_init.py +++ b/tests/components/profiler/test_init.py @@ -19,6 +19,7 @@ from homeassistant.components.profiler.const import DOMAIN from homeassistant.const import CONF_SCAN_INTERVAL, CONF_TYPE from homeassistant.core import HomeAssistant +from homeassistant.exceptions import HomeAssistantError import homeassistant.util.dt as dt_util from tests.common import MockConfigEntry, async_fire_time_changed @@ -90,8 +91,16 @@ def _mock_path(filename): @pytest.mark.skipif(sys.version_info < (3, 11), reason="still works on python 3.10") async def test_memory_usage_py311(hass: HomeAssistant, tmpdir) -> None: """Test raise an error on python3.11.""" + entry = MockConfigEntry(domain=DOMAIN) + entry.add_to_hass(hass) + + assert await hass.config_entries.async_setup(entry.entry_id) + await hass.async_block_till_done() assert hass.services.has_service(DOMAIN, SERVICE_MEMORY) - with pytest.raises(HomeAssistant, match="not yet available on python 3.11"): + with pytest.raises( + HomeAssistantError, + match="Memory profiling is not supported on Python 3.11. Please use Python 3.10.", + ): await hass.services.async_call( DOMAIN, SERVICE_MEMORY, {CONF_SECONDS: 0.000001}, blocking=True )