diff --git a/homeassistant/helpers/script.py b/homeassistant/helpers/script.py
index ac53a3e32a2de..5e660ba7b7f79 100644
--- a/homeassistant/helpers/script.py
+++ b/homeassistant/helpers/script.py
@@ -1,6 +1,7 @@
 """Helpers to execute scripts."""
 
 import logging
+from contextlib import suppress
 from itertools import islice
 from typing import Optional, Sequence
 
@@ -95,7 +96,9 @@ async def async_run(self, variables: Optional[Sequence] = None,
                 def async_script_delay(now):
                     """Handle delay."""
                     # pylint: disable=cell-var-from-loop
-                    self._async_remove_listener()
+                    with suppress(ValueError):
+                        self._async_listener.remove(unsub)
+
                     self.hass.async_create_task(
                         self.async_run(variables, context))
 
@@ -240,7 +243,8 @@ def _async_set_timeout(self, action, variables, context,
         @callback
         def async_script_timeout(now):
             """Call after timeout is retrieve."""
-            self._async_remove_listener()
+            with suppress(ValueError):
+                self._async_listener.remove(unsub)
 
             # Check if we want to continue to execute
             # the script after the timeout