Skip to content

Fail tests with uncaught exceptions #33312

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
balloob merged 2 commits into from
Mar 28, 2020
Merged

Fail tests with uncaught exceptions #33312

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d58df0c
initial implementation and ignore file
ziv1234 Mar 27, 2020
53321d4
updated ignore list for the tests in the most recent dev branch
ziv1234 Mar 27, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,12 @@
TYPE_AUTH_REQUIRED,
)
from homeassistant.components.websocket_api.http import URL
from homeassistant.exceptions import ServiceNotFound
from homeassistant.setup import async_setup_component
from homeassistant.util import location

from tests.ignore_uncaught_exceptions import IGNORE_UNCAUGHT_EXCEPTIONS

pytest.register_assert_rewrite("tests.common")

from tests.common import ( # noqa: E402, isort:skip
Expand Down Expand Up @@ -77,13 +80,31 @@ def hass_storage():


@pytest.fixture
def hass(loop, hass_storage):
def hass(loop, hass_storage, request):
"""Fixture to provide a test instance of Home Assistant."""

def exc_handle(loop, context):
"""Handle exceptions by rethrowing them, which will fail the test."""
exceptions.append(context["exception"])
orig_exception_handler(loop, context)

exceptions = []
hass = loop.run_until_complete(async_test_home_assistant(loop))
orig_exception_handler = loop.get_exception_handler()
loop.set_exception_handler(exc_handle)

yield hass

loop.run_until_complete(hass.async_stop(force=True))
for ex in exceptions:
if (
request.module.__name__,
request.function.__name__,
) in IGNORE_UNCAUGHT_EXCEPTIONS:
continue
if isinstance(ex, ServiceNotFound):
continue
Comment on lines +105 to +106
Copy link
Copy Markdown
Member

@balloob balloob Mar 28, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if we can reduce this by setting up persistent notification, which seems to be causing most errors. But that is for future us, this is already a great improvement.

Copy link
Copy Markdown
Contributor Author

@ziv1234 ziv1234 Mar 28, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the problem is that it happens for components that don't use persistent notifications directly (like mine, Dynalite...). I plan to take a look at this as part of the exception reduction, but i have a bit less than 200 exceptions today in the file. If i remove this line it becomes 1600.

Copy link
Copy Markdown
Member

@balloob balloob Mar 28, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could consider to set persistent_notification up as part of the hass fixture.

Copy link
Copy Markdown
Member

@balloob balloob Mar 28, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right now we ignore all service not found errors, I bet that we're missing some real ones too. But let's keep this for the future.

Copy link
Copy Markdown
Contributor Author

@ziv1234 ziv1234 Mar 28, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i believe that setting it for all the tests will increase the test time significantly. We can add mock service calls for the persistent notifications in the hass fixture and will likely catch the same errors since persistent notifications are always supposed to be set up, right?

raise ex


@pytest.fixture
Expand Down
235 changes: 235 additions & 0 deletions tests/ignore_uncaught_exceptions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,235 @@
"""List of modules that have uncaught exceptions today. Will be shrunk over time."""
IGNORE_UNCAUGHT_EXCEPTIONS = [
("tests.components.abode.test_alarm_control_panel", "test_entity_registry"),
("tests.components.abode.test_alarm_control_panel", "test_attributes"),
("tests.components.abode.test_alarm_control_panel", "test_set_alarm_away"),
("tests.components.abode.test_alarm_control_panel", "test_set_alarm_home"),
("tests.components.abode.test_alarm_control_panel", "test_set_alarm_standby"),
("tests.components.abode.test_alarm_control_panel", "test_state_unknown"),
("tests.components.abode.test_binary_sensor", "test_entity_registry"),
("tests.components.abode.test_binary_sensor", "test_attributes"),
("tests.components.abode.test_camera", "test_entity_registry"),
("tests.components.abode.test_camera", "test_attributes"),
("tests.components.abode.test_camera", "test_capture_image"),
("tests.components.abode.test_cover", "test_entity_registry"),
("tests.components.abode.test_cover", "test_attributes"),
("tests.components.abode.test_cover", "test_open"),
("tests.components.abode.test_cover", "test_close"),
("tests.components.abode.test_init", "test_change_settings"),
("tests.components.abode.test_light", "test_entity_registry"),
("tests.components.abode.test_light", "test_attributes"),
("tests.components.abode.test_light", "test_switch_off"),
("tests.components.abode.test_light", "test_switch_on"),
("tests.components.abode.test_light", "test_set_brightness"),
("tests.components.abode.test_light", "test_set_color"),
("tests.components.abode.test_light", "test_set_color_temp"),
("tests.components.abode.test_lock", "test_entity_registry"),
("tests.components.abode.test_lock", "test_attributes"),
("tests.components.abode.test_lock", "test_lock"),
("tests.components.abode.test_lock", "test_unlock"),
("tests.components.abode.test_sensor", "test_entity_registry"),
("tests.components.abode.test_sensor", "test_attributes"),
("tests.components.abode.test_switch", "test_entity_registry"),
("tests.components.abode.test_switch", "test_attributes"),
("tests.components.abode.test_switch", "test_switch_on"),
("tests.components.abode.test_switch", "test_switch_off"),
("tests.components.abode.test_switch", "test_automation_attributes"),
("tests.components.abode.test_switch", "test_turn_automation_off"),
("tests.components.abode.test_switch", "test_turn_automation_on"),
("tests.components.abode.test_switch", "test_trigger_automation"),
("tests.components.cast.test_media_player", "test_start_discovery_called_once"),
("tests.components.cast.test_media_player", "test_entry_setup_single_config"),
("tests.components.cast.test_media_player", "test_entry_setup_list_config"),
("tests.components.cast.test_media_player", "test_entry_setup_platform_not_ready"),
("tests.components.config.test_automation", "test_delete_automation"),
("tests.components.config.test_group", "test_update_device_config"),
("tests.components.deconz.test_binary_sensor", "test_allow_clip_sensor"),
("tests.components.deconz.test_climate", "test_clip_climate_device"),
("tests.components.deconz.test_init", "test_unload_entry_multiple_gateways"),
("tests.components.deconz.test_light", "test_disable_light_groups"),
("tests.components.deconz.test_sensor", "test_allow_clip_sensors"),
("tests.components.default_config.test_init", "test_setup"),
("tests.components.demo.test_init", "test_setting_up_demo"),
("tests.components.discovery.test_init", "test_discover_config_flow"),
("tests.components.dsmr.test_sensor", "test_default_setup"),
("tests.components.dsmr.test_sensor", "test_v4_meter"),
("tests.components.dsmr.test_sensor", "test_v5_meter"),
("tests.components.dsmr.test_sensor", "test_belgian_meter"),
("tests.components.dsmr.test_sensor", "test_belgian_meter_low"),
("tests.components.dsmr.test_sensor", "test_tcp"),
("tests.components.dsmr.test_sensor", "test_connection_errors_retry"),
("tests.components.dynalite.test_bridge", "test_add_devices_then_register"),
("tests.components.dynalite.test_bridge", "test_register_then_add_devices"),
("tests.components.dynalite.test_config_flow", "test_existing_update"),
("tests.components.dyson.test_air_quality", "test_purecool_aiq_attributes"),
("tests.components.dyson.test_air_quality", "test_purecool_aiq_update_state"),
(
"tests.components.dyson.test_air_quality",
"test_purecool_component_setup_only_once",
),
("tests.components.dyson.test_air_quality", "test_purecool_aiq_without_discovery"),
(
"tests.components.dyson.test_air_quality",
"test_purecool_aiq_empty_environment_state",
),
(
"tests.components.dyson.test_climate",
"test_setup_component_with_parent_discovery",
),
("tests.components.dyson.test_fan", "test_purecoollink_attributes"),
("tests.components.dyson.test_fan", "test_purecool_turn_on"),
("tests.components.dyson.test_fan", "test_purecool_set_speed"),
("tests.components.dyson.test_fan", "test_purecool_turn_off"),
("tests.components.dyson.test_fan", "test_purecool_set_dyson_speed"),
("tests.components.dyson.test_fan", "test_purecool_oscillate"),
("tests.components.dyson.test_fan", "test_purecool_set_night_mode"),
("tests.components.dyson.test_fan", "test_purecool_set_auto_mode"),
("tests.components.dyson.test_fan", "test_purecool_set_angle"),
("tests.components.dyson.test_fan", "test_purecool_set_flow_direction_front"),
("tests.components.dyson.test_fan", "test_purecool_set_timer"),
("tests.components.dyson.test_fan", "test_purecool_update_state"),
("tests.components.dyson.test_fan", "test_purecool_update_state_filter_inv"),
("tests.components.dyson.test_fan", "test_purecool_component_setup_only_once"),
("tests.components.dyson.test_sensor", "test_purecool_component_setup_only_once"),
("tests.components.gdacs.test_geo_location", "test_setup"),
("tests.components.gdacs.test_sensor", "test_setup"),
("tests.components.geonetnz_quakes.test_geo_location", "test_setup"),
("tests.components.geonetnz_quakes.test_sensor", "test_setup"),
("test_homeassistant_bridge", "test_homeassistant_bridge_fan_setup"),
("tests.components.homematicip_cloud.test_config_flow", "test_flow_works"),
("tests.components.homematicip_cloud.test_config_flow", "test_import_config"),
("tests.components.homematicip_cloud.test_device", "test_hmip_remove_group"),
(
"tests.components.homematicip_cloud.test_init",
"test_config_with_accesspoint_passed_to_config_entry",
),
(
"tests.components.homematicip_cloud.test_init",
"test_config_already_registered_not_passed_to_config_entry",
),
(
"tests.components.homematicip_cloud.test_init",
"test_load_entry_fails_due_to_generic_exception",
),
("tests.components.hue.test_bridge", "test_handle_unauthorized"),
("tests.components.hue.test_init", "test_security_vuln_check"),
("tests.components.hue.test_light", "test_group_features"),
("tests.components.ios.test_init", "test_creating_entry_sets_up_sensor"),
("tests.components.ios.test_init", "test_not_configuring_ios_not_creates_entry"),
("tests.components.local_file.test_camera", "test_file_not_readable"),
("tests.components.meteo_france.test_config_flow", "test_user"),
("tests.components.meteo_france.test_config_flow", "test_import"),
("tests.components.mikrotik.test_device_tracker", "test_restoring_devices"),
("tests.components.mikrotik.test_hub", "test_arp_ping"),
("tests.components.mqtt.test_alarm_control_panel", "test_unique_id"),
("tests.components.mqtt.test_binary_sensor", "test_unique_id"),
("tests.components.mqtt.test_camera", "test_unique_id"),
("tests.components.mqtt.test_climate", "test_unique_id"),
("tests.components.mqtt.test_cover", "test_unique_id"),
("tests.components.mqtt.test_fan", "test_unique_id"),
(
"tests.components.mqtt.test_init",
"test_setup_uses_certificate_on_certificate_set_to_auto",
),
(
"tests.components.mqtt.test_init",
"test_setup_does_not_use_certificate_on_mqtts_port",
),
(
"tests.components.mqtt.test_init",
"test_setup_without_tls_config_uses_tlsv1_under_python36",
),
(
"tests.components.mqtt.test_init",
"test_setup_with_tls_config_uses_tls_version1_2",
),
(
"tests.components.mqtt.test_init",
"test_setup_with_tls_config_of_v1_under_python36_only_uses_v1",
),
("tests.components.mqtt.test_legacy_vacuum", "test_unique_id"),
("tests.components.mqtt.test_light", "test_unique_id"),
("tests.components.mqtt.test_light", "test_entity_device_info_remove"),
("tests.components.mqtt.test_light_json", "test_unique_id"),
("tests.components.mqtt.test_light_json", "test_entity_device_info_remove"),
("tests.components.mqtt.test_light_template", "test_entity_device_info_remove"),
("tests.components.mqtt.test_lock", "test_unique_id"),
("tests.components.mqtt.test_sensor", "test_unique_id"),
("tests.components.mqtt.test_state_vacuum", "test_unique_id"),
("tests.components.mqtt.test_switch", "test_unique_id"),
("tests.components.mqtt.test_switch", "test_entity_device_info_remove"),
("tests.components.plex.test_config_flow", "test_import_success"),
("tests.components.plex.test_config_flow", "test_single_available_server"),
("tests.components.plex.test_config_flow", "test_multiple_servers_with_selection"),
("tests.components.plex.test_config_flow", "test_adding_last_unconfigured_server"),
("tests.components.plex.test_config_flow", "test_option_flow"),
("tests.components.plex.test_config_flow", "test_option_flow_new_users_available"),
("tests.components.plex.test_init", "test_setup_with_config"),
("tests.components.plex.test_init", "test_setup_with_config_entry"),
("tests.components.plex.test_init", "test_set_config_entry_unique_id"),
("tests.components.plex.test_init", "test_setup_with_insecure_config_entry"),
("tests.components.plex.test_init", "test_setup_with_photo_session"),
("tests.components.plex.test_server", "test_new_users_available"),
("tests.components.plex.test_server", "test_new_ignored_users_available"),
("tests.components.plex.test_server", "test_mark_sessions_idle"),
("tests.components.qwikswitch.test_init", "test_binary_sensor_device"),
("tests.components.qwikswitch.test_init", "test_sensor_device"),
("tests.components.rflink.test_init", "test_send_command_invalid_arguments"),
("tests.components.samsungtv.test_media_player", "test_update_connection_failure"),
("tests.components.tplink.test_init", "test_configuring_device_types"),
(
"tests.components.tplink.test_init",
"test_configuring_devices_from_multiple_sources",
),
("tests.components.tradfri.test_light", "test_light"),
("tests.components.tradfri.test_light", "test_light_observed"),
("tests.components.tradfri.test_light", "test_light_available"),
("tests.components.tradfri.test_light", "test_turn_on"),
("tests.components.tradfri.test_light", "test_turn_off"),
("tests.components.unifi_direct.test_device_tracker", "test_get_scanner"),
("tests.components.upnp.test_init", "test_async_setup_entry_default"),
("tests.components.upnp.test_init", "test_async_setup_entry_port_mapping"),
("tests.components.vera.test_init", "test_init"),
("tests.components.wunderground.test_sensor", "test_fails_because_of_unique_id"),
("tests.components.yr.test_sensor", "test_default_setup"),
("tests.components.yr.test_sensor", "test_custom_setup"),
("tests.components.yr.test_sensor", "test_forecast_setup"),
("tests.components.zha.test_api", "test_device_clusters"),
("tests.components.zha.test_api", "test_device_cluster_attributes"),
("tests.components.zha.test_api", "test_device_cluster_commands"),
("tests.components.zha.test_api", "test_list_devices"),
("tests.components.zha.test_api", "test_device_not_found"),
("tests.components.zha.test_api", "test_list_groups"),
("tests.components.zha.test_api", "test_get_group"),
("tests.components.zha.test_api", "test_get_group_not_found"),
("tests.components.zha.test_api", "test_list_groupable_devices"),
("tests.components.zha.test_api", "test_add_group"),
("tests.components.zha.test_api", "test_remove_group"),
("tests.components.zha.test_binary_sensor", "test_binary_sensor"),
("tests.components.zha.test_cover", "test_cover"),
("tests.components.zha.test_device_action", "test_get_actions"),
("tests.components.zha.test_device_action", "test_action"),
("tests.components.zha.test_device_tracker", "test_device_tracker"),
("tests.components.zha.test_device_trigger", "test_triggers"),
("tests.components.zha.test_device_trigger", "test_no_triggers"),
("tests.components.zha.test_device_trigger", "test_if_fires_on_event"),
("tests.components.zha.test_device_trigger", "test_exception_no_triggers"),
("tests.components.zha.test_device_trigger", "test_exception_bad_trigger"),
("tests.components.zha.test_discover", "test_devices"),
("tests.components.zha.test_discover", "test_device_override"),
("tests.components.zha.test_fan", "test_fan"),
("tests.components.zha.test_gateway", "test_gateway_group_methods"),
("tests.components.zha.test_light", "test_light"),
("tests.components.zha.test_lock", "test_lock"),
("tests.components.zha.test_sensor", "test_sensor"),
("tests.components.zha.test_switch", "test_switch"),
("tests.components.zwave.test_init", "test_power_schemes"),
(
"tests.helpers.test_entity_platform",
"test_adding_entities_with_generator_and_thread_callback",
),
(
"tests.helpers.test_entity_platform",
"test_not_adding_duplicate_entities_with_unique_id",
),
]