chore: 🔖 add retries

datonic · May 20, 2024 · af83dae · af83dae
1 parent aaff537
commit af83dae
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 5 deletions.
diff --git a/datadex/assets/others.py b/datadex/assets/others.py
@@ -2,7 +2,7 @@
 
 import httpx
 import polars as pl
-from dagster import AssetExecutionContext, asset
+from dagster import AssetExecutionContext, Backoff, RetryPolicy, asset
 
 from ..resources import IUCNRedListAPI
 
@@ -31,7 +31,9 @@ def threatened_animal_species(
     return pl.DataFrame(all_results, infer_schema_length=None)
 
 
-@asset()
+@asset(
+    retry_policy=RetryPolicy(max_retries=5, delay=1, backoff=Backoff.EXPONENTIAL),
+)
 def wikidata_asteroids() -> pl.DataFrame:
     """
     Wikidata asteroids data.

diff --git a/datadex/assets/spain.py b/datadex/assets/spain.py
@@ -1,4 +1,5 @@
 from datetime import datetime, timedelta
+import time
 
 import httpx
 import polars as pl
@@ -8,7 +9,9 @@
 from ..resources import AEMETAPI, MITECOArcGisAPI
 
 
-@asset()
+@asset(
+    retry_policy=RetryPolicy(max_retries=3, delay=10, backoff=Backoff.EXPONENTIAL),
+)
 def spain_energy_demand(context: AssetExecutionContext) -> pl.DataFrame:
     """
     Spain energy demand data.
@@ -26,7 +29,7 @@ def spain_energy_demand(context: AssetExecutionContext) -> pl.DataFrame:
 
     while start_date < yesterday:
         url = f"{ENDPOINT}?start_date={start_date_str}T00:00&end_date={end_date_str}T00:00&time_trunc=hour"
-        response = httpx.get(url)
+        response = httpx.get(url, timeout=60)
 
         context.log.info(
             f"Start date: {start_date_str} status code: {response.status_code}"
@@ -48,7 +51,7 @@ def spain_energy_demand(context: AssetExecutionContext) -> pl.DataFrame:
 
 
 @asset(
-    retry_policy=RetryPolicy(max_retries=5, delay=0.2, backoff=Backoff.EXPONENTIAL),
+    retry_policy=RetryPolicy(max_retries=5, delay=1, backoff=Backoff.EXPONENTIAL),
 )
 def spain_ipc() -> pl.DataFrame:
     """