Skip to content

Commit

Permalink
Temporarily change BigQueryTablePartitionExistenceSensors to run in…
Browse files Browse the repository at this point in the history 
… reschedule mode.

There's a bug in those sensors when they run in deferrable mode where the secondary queries they run to detect partitions fail, which is being fixed in apache/airflow#44225.
  • Loading branch information
@sean-rose
sean-rose committed Nov 20, 2024
1 parent 7d252ab commit 606b7da
Showing 1 changed file with 3 additions and 2 deletions.
5 changes: 3 additions & 2 deletions bigquery_etl/query_scheduling/templates/airflow_dag.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,11 +149,12 @@ with DAG('{{ name }}', default_args=default_args{%+ if schedule_interval != None
table_id={{ table_name | format_repr }},
partition_id={{ table_partition_sensor_task.partition_id | format_repr }},
gcp_conn_id='google_cloud_shared_prod',
deferrable=True,
deferrable=False,
mode='reschedule',
{% if table_partition_sensor_task.poke_interval != None -%}
poke_interval={{ table_partition_sensor_task.poke_interval | format_timedelta | format_repr }},
{% else -%}
poke_interval=datetime.timedelta(minutes=5),
poke_interval=datetime.timedelta(minutes=15),
{% endif -%}
{% if table_partition_sensor_task.timeout != None -%}
timeout={{ table_partition_sensor_task.timeout | format_timedelta | format_repr }},
Expand Down

1 comment on commit 606b7da

@dataops-ci-bot
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Integration report for "Temporarily change BigQueryTablePartitionExistenceSensors to run in reschedule mode."

sql.diff

Click to expand! 
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_google_search_console.py /tmp/workspace/generated-sql/dags/bqetl_google_search_console.py
--- /tmp/workspace/main-generated-sql/dags/bqetl_google_search_console.py	2024-11-20 20:48:18.000000000 +0000
+++ /tmp/workspace/generated-sql/dags/bqetl_google_search_console.py	2024-11-20 20:50:28.000000000 +0000
@@ -67,8 +67,9 @@
             table_id="searchdata_url_impression",
             partition_id="{{ data_interval_start.subtract(days=1) | ds_nodash }}",
             gcp_conn_id="google_cloud_shared_prod",
-            deferrable=True,
-            poke_interval=datetime.timedelta(minutes=5),
+            deferrable=False,
+            mode="reschedule",
+            poke_interval=datetime.timedelta(minutes=15),
             timeout=datetime.timedelta(hours=8),
         )
     )
@@ -81,8 +82,9 @@
             table_id="searchdata_url_impression",
             partition_id="{{ data_interval_start.subtract(days=1) | ds_nodash }}",
             gcp_conn_id="google_cloud_shared_prod",
-            deferrable=True,
-            poke_interval=datetime.timedelta(minutes=5),
+            deferrable=False,
+            mode="reschedule",
+            poke_interval=datetime.timedelta(minutes=15),
             timeout=datetime.timedelta(hours=8),
         )
     )
@@ -95,8 +97,9 @@
             table_id="searchdata_url_impression",
             partition_id="{{ data_interval_start.subtract(days=1) | ds_nodash }}",
             gcp_conn_id="google_cloud_shared_prod",
-            deferrable=True,
-            poke_interval=datetime.timedelta(minutes=5),
+            deferrable=False,
+            mode="reschedule",
+            poke_interval=datetime.timedelta(minutes=15),
             timeout=datetime.timedelta(hours=8),
         )
     )
@@ -109,8 +112,9 @@
             table_id="searchdata_url_impression",
             partition_id="{{ data_interval_start.subtract(days=1) | ds_nodash }}",
             gcp_conn_id="google_cloud_shared_prod",
-            deferrable=True,
-            poke_interval=datetime.timedelta(minutes=5),
+            deferrable=False,
+            mode="reschedule",
+            poke_interval=datetime.timedelta(minutes=15),
             timeout=datetime.timedelta(hours=8),
         )
     )
@@ -123,8 +127,9 @@
             table_id="searchdata_url_impression",
             partition_id="{{ data_interval_start.subtract(days=1) | ds_nodash }}",
             gcp_conn_id="google_cloud_shared_prod",
-            deferrable=True,
-            poke_interval=datetime.timedelta(minutes=5),
+            deferrable=False,
+            mode="reschedule",
+            poke_interval=datetime.timedelta(minutes=15),
             timeout=datetime.timedelta(hours=8),
         )
     )
@@ -137,8 +142,9 @@
             table_id="searchdata_url_impression",
             partition_id="{{ data_interval_start.subtract(days=1) | ds_nodash }}",
             gcp_conn_id="google_cloud_shared_prod",
-            deferrable=True,
-            poke_interval=datetime.timedelta(minutes=5),
+            deferrable=False,
+            mode="reschedule",
+            poke_interval=datetime.timedelta(minutes=15),
             timeout=datetime.timedelta(hours=8),
         )
     )
@@ -151,8 +157,9 @@
             table_id="searchdata_site_impression",
             partition_id="{{ data_interval_start.subtract(days=1) | ds_nodash }}",
             gcp_conn_id="google_cloud_shared_prod",
-            deferrable=True,
-            poke_interval=datetime.timedelta(minutes=5),
+            deferrable=False,
+            mode="reschedule",
+            poke_interval=datetime.timedelta(minutes=15),
             timeout=datetime.timedelta(hours=8),
         )
     )
@@ -165,8 +172,9 @@
             table_id="searchdata_site_impression",
             partition_id="{{ data_interval_start.subtract(days=1) | ds_nodash }}",
             gcp_conn_id="google_cloud_shared_prod",
-            deferrable=True,
-            poke_interval=datetime.timedelta(minutes=5),
+            deferrable=False,
+            mode="reschedule",
+            poke_interval=datetime.timedelta(minutes=15),
             timeout=datetime.timedelta(hours=8),
         )
     )
@@ -179,8 +187,9 @@
             table_id="searchdata_site_impression",
             partition_id="{{ data_interval_start.subtract(days=1) | ds_nodash }}",
             gcp_conn_id="google_cloud_shared_prod",
-            deferrable=True,
-            poke_interval=datetime.timedelta(minutes=5),
+            deferrable=False,
+            mode="reschedule",
+            poke_interval=datetime.timedelta(minutes=15),
             timeout=datetime.timedelta(hours=8),
         )
     )
@@ -193,8 +202,9 @@
             table_id="searchdata_site_impression",
             partition_id="{{ data_interval_start.subtract(days=1) | ds_nodash }}",
             gcp_conn_id="google_cloud_shared_prod",
-            deferrable=True,
-            poke_interval=datetime.timedelta(minutes=5),
+            deferrable=False,
+            mode="reschedule",
+            poke_interval=datetime.timedelta(minutes=15),
             timeout=datetime.timedelta(hours=8),
         )
     )
@@ -207,8 +217,9 @@
             table_id="searchdata_site_impression",
             partition_id="{{ data_interval_start.subtract(days=1) | ds_nodash }}",
             gcp_conn_id="google_cloud_shared_prod",
-            deferrable=True,
-            poke_interval=datetime.timedelta(minutes=5),
+            deferrable=False,
+            mode="reschedule",
+            poke_interval=datetime.timedelta(minutes=15),
             timeout=datetime.timedelta(hours=8),
         )
     )
@@ -221,8 +232,9 @@
             table_id="searchdata_site_impression",
             partition_id="{{ data_interval_start.subtract(days=1) | ds_nodash }}",
             gcp_conn_id="google_cloud_shared_prod",
-            deferrable=True,
-            poke_interval=datetime.timedelta(minutes=5),
+            deferrable=False,
+            mode="reschedule",
+            poke_interval=datetime.timedelta(minutes=15),
             timeout=datetime.timedelta(hours=8),
         )
     )

Link to full diff

Please sign in to comment.