diff --git a/synapse_data_warehouse/synapse/dynamic_tables/V2.26.4__node_latest.sql b/synapse_data_warehouse/synapse/dynamic_tables/V2.26.4__node_latest.sql new file mode 100644 index 00000000..50040313 --- /dev/null +++ b/synapse_data_warehouse/synapse/dynamic_tables/V2.26.4__node_latest.sql @@ -0,0 +1,26 @@ +USE SCHEMA {{database_name}}.synapse; --noqa: JJ01,PRS,TMP + +CREATE DYNAMIC TABLE IF NOT EXISTS NODE_LATEST + TARGET_LAG = '1 day' + WAREHOUSE = compute_xsmall + AS + WITH latest_unique_rows AS ( + SELECT + * + FROM + {{database_name}}.synapse_raw.nodesnapshots --noqa: TMP + WHERE + SNAPSHOT_TIMESTAMP >= CURRENT_TIMESTAMP - INTERVAL '14 DAYS' + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY id + ORDER BY change_timestamp DESC, snapshot_timestamp DESC + ) = 1 + ) + SELECT + * + FROM + latest_unique_rows + WHERE + CHANGE_TYPE != 'DELETE' + ORDER BY + latest_unique_rows.id ASC; diff --git a/synapse_data_warehouse/synapse/tables/V2.26.0__add_node_latest_backup.sql b/synapse_data_warehouse/synapse/tables/V2.26.0__add_node_latest_backup.sql new file mode 100644 index 00000000..cb2fa2e1 --- /dev/null +++ b/synapse_data_warehouse/synapse/tables/V2.26.0__add_node_latest_backup.sql @@ -0,0 +1,5 @@ +USE SCHEMA {{database_name}}.synapse; --noqa: JJ01,PRS,TMP + +-- Clone the NODE_LATEST table to ``NODE_LATEST_BACKUP`` +-- This will begin the process of converting ``NODE_LATEST`` table into a dynamic table +CREATE OR REPLACE TABLE NODE_LATEST_BACKUP CLONE NODE_LATEST; diff --git a/synapse_data_warehouse/synapse/tables/V2.26.3__drop_node_latest.sql b/synapse_data_warehouse/synapse/tables/V2.26.3__drop_node_latest.sql new file mode 100644 index 00000000..16d7d546 --- /dev/null +++ b/synapse_data_warehouse/synapse/tables/V2.26.3__drop_node_latest.sql @@ -0,0 +1,4 @@ +USE SCHEMA {{database_name}}.synapse; --noqa: JJ01,PRS,TMP + +-- Drop the ``NODE_LATEST`` table so that a new dynamic table with the same name can be created in its place +DROP TABLE NODE_LATEST; diff --git a/synapse_data_warehouse/synapse_raw/streams/V2.26.2__delete_nodesnapshots_stream.sql b/synapse_data_warehouse/synapse_raw/streams/V2.26.2__delete_nodesnapshots_stream.sql new file mode 100644 index 00000000..0015c246 --- /dev/null +++ b/synapse_data_warehouse/synapse_raw/streams/V2.26.2__delete_nodesnapshots_stream.sql @@ -0,0 +1,3 @@ +USE SCHEMA {{database_name}}.synapse_raw; --noqa: JJ01,PRS,TMP + +DROP STREAM IF EXISTS NODESNAPSHOTS_STREAM; diff --git a/synapse_data_warehouse/synapse_raw/tasks/V2.26.1__delete_node_latest_tasks.sql b/synapse_data_warehouse/synapse_raw/tasks/V2.26.1__delete_node_latest_tasks.sql new file mode 100644 index 00000000..ce06c097 --- /dev/null +++ b/synapse_data_warehouse/synapse_raw/tasks/V2.26.1__delete_node_latest_tasks.sql @@ -0,0 +1,11 @@ +USE SCHEMA {{database_name}}.synapse_raw; --noqa: JJ01,PRS,TMP + +-- Momentarily suspend the ROOT task so we can remove the child tasks +ALTER TASK IF EXISTS REFRESH_SYNAPSE_WAREHOUSE_S3_STAGE_TASK SUSPEND; + +-- Remove the tasks in question +DROP TASK IF EXISTS UPSERT_TO_NODE_LATEST_TASK; +DROP TASK IF EXISTS REMOVE_DELETE_NODES_TASK; + +-- Resume the ROOT task and its child tasks +SELECT SYSTEM$TASK_DEPENDENTS_ENABLE( 'REFRESH_SYNAPSE_WAREHOUSE_S3_STAGE_TASK' ); \ No newline at end of file