From 7a575b08ebc89d7bf9f228db2a27fb5737b0388a Mon Sep 17 00:00:00 2001 From: Roman Mitasov Date: Fri, 31 Jul 2026 16:40:01 +0800 Subject: [PATCH 1/2] fix(dbt): align staging contributor types with their silver union targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The staging -> silver field-parity audit (#2080) fails on 13 type divergences across 5 root causes. A silver class table is a positional UNION ALL of its staging contributors, so each divergence either widens the published silver type depending on which connectors are enabled, or coerces values at the insert boundary. All fixes cast the outlier branch toward the type the data-carrying branch already publishes; silver schemas on warm clusters do not change (every write stays compatible), so no full-refresh is required for correctness — only for the deployed staging tables' declared types to converge, which can happen in any convenient window: dbt run --full-refresh --select github__pull_requests_commits \ gitlab__pull_requests_commits salesforce__crm_accounts \ salesforce__crm_activities salesforce__crm_contacts \ salesforce__crm_deals salesforce__crm_users \ m365__collab_document_activity_sharepoint * commit_order (github, gitlab): the models emit a literal `0` (the APIs provide no ordering), which ClickHouse types as UInt8; bitbucket emits Int64 from real data. toInt64(0) pins the branch to the contract type. * custom_fields (5 salesforce models, 6 branches): passed through from bronze as Nullable(String) while heal_crm_table ALTERs the silver tables to `String DEFAULT '{}'` — NULLs were being coerced to the default at insert time via insert_null_as_default. coalesce makes the '{}' fallback explicit in the model, matching hubspot's literal. * visited_page_count (m365 sharepoint): bronze carries the JSON `number` as Nullable(Decimal(38, 9)) and the model passed it through; the onedrive branch emits Nullable(Int64). A page count is integral — cast to Int64. Nothing downstream reads the column (checked migrations, gold, silver), so the silver type change on fresh clusters is safe. * close_date (hubspot deals): toDate() narrowed to Date while salesforce bronze is natively Nullable(Date32). toDate32 matches the wider type. * hire_date/termination_date (ms-entra, active-directory): constant-NULL placeholders typed Nullable(Date) against the Nullable(DateTime) the data-carrying branches (bamboohr, workday) emit. Retype the NULLs; zero rows are affected by construction. The regenerated snapshot carries the visited_page_count type change and also normalizes silver.contract_version to dump-ddl.sh's statement format — that entry was appended by hand with the semicolon on the statement line, a shape the dumper never produces, so the next convergence check would have flagged it as drift. Verified on a from-scratch bootstrap (all connectors green, dbt PASS=218/ERROR=0): the field-parity audit reports 0 failures, down from 13, across 39 union targets and 273 relations. Co-Authored-By: Claude Opus 5 Signed-off-by: Roman Mitasov --- .../m365/dbt/m365__collab_document_activity_sharepoint.sql | 2 +- .../connectors/crm/hubspot/dbt/hubspot__crm_deals.sql | 2 +- .../crm/salesforce/dbt/salesforce__crm_accounts.sql | 2 +- .../crm/salesforce/dbt/salesforce__crm_activities.sql | 4 ++-- .../crm/salesforce/dbt/salesforce__crm_contacts.sql | 2 +- .../connectors/crm/salesforce/dbt/salesforce__crm_deals.sql | 2 +- .../connectors/crm/salesforce/dbt/salesforce__crm_users.sql | 2 +- .../git/github-v2/dbt/github__pull_requests_commits.sql | 2 +- .../git/gitlab/dbt/gitlab__pull_requests_commits.sql | 2 +- .../dbt/active_directory__to_class_people.sql | 4 ++-- .../hr-directory/ms-entra/dbt/ms_entra__to_class_people.sql | 4 ++-- src/ingestion/scripts/connectors-ddl/silver.sql | 5 +++-- 12 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/ingestion/connectors/collaboration/m365/dbt/m365__collab_document_activity_sharepoint.sql b/src/ingestion/connectors/collaboration/m365/dbt/m365__collab_document_activity_sharepoint.sql index f8405fb1c..f747c8afa 100644 --- a/src/ingestion/connectors/collaboration/m365/dbt/m365__collab_document_activity_sharepoint.sql +++ b/src/ingestion/connectors/collaboration/m365/dbt/m365__collab_document_activity_sharepoint.sql @@ -28,7 +28,7 @@ SELECT syncedFileCount AS synced_count, sharedInternallyFileCount AS shared_internally_count, sharedExternallyFileCount AS shared_externally_count, - visitedPageCount AS visited_page_count, + CAST(visitedPageCount AS Nullable(Int64)) AS visited_page_count, reportPeriod AS report_period, now() AS collected_at, 'insight_m365' AS data_source, diff --git a/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_deals.sql b/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_deals.sql index 401116382..25be6c671 100644 --- a/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_deals.sql +++ b/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_deals.sql @@ -49,7 +49,7 @@ WITH src AS ( -- "2025-10-23T08:49:39Z"). `toDateOrNull` only handles -- YYYY-MM-DD; we parse via `parseDateTime64BestEffortOrNull` -- first then truncate to Date. - toDate(parseDateTime64BestEffortOrNull(properties_closedate)) AS close_date, + toDate32(parseDateTime64BestEffortOrNull(properties_closedate)) AS close_date, properties_hubspot_owner_id AS owner_id, -- Rep who logged / created the deal — distinct from owner_id, which -- can be the contact owner. Resolves to silver.class_crm_users diff --git a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_accounts.sql b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_accounts.sql index a20e6d59d..72eac4119 100644 --- a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_accounts.sql +++ b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_accounts.sql @@ -29,7 +29,7 @@ WITH src AS ( 'AnnualRevenue', coalesce(toString(AnnualRevenue), ''), 'IsDeleted', toString(coalesce(IsDeleted, false)) )) AS metadata, - custom_fields, + coalesce(custom_fields, '{}') AS custom_fields, CreatedDate AS created_at, LastModifiedDate AS updated_at, data_source, diff --git a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_activities.sql b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_activities.sql index e90eddbac..6ab316e92 100644 --- a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_activities.sql +++ b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_activities.sql @@ -48,7 +48,7 @@ WITH tasks AS ( 'CallType', coalesce(toString(CallType), ''), 'IsDeleted', toString(coalesce(IsDeleted, false)) )) AS metadata, - custom_fields, + coalesce(custom_fields, '{}') AS custom_fields, CreatedDate AS created_at, data_source, coalesce(toUnixTimestamp64Milli(SystemModstamp), 0) AS _version @@ -93,7 +93,7 @@ events AS ( 'EventSubtype', coalesce(toString(EventSubtype), ''), 'IsDeleted', toString(coalesce(IsDeleted, false)) )) AS metadata, - custom_fields, + coalesce(custom_fields, '{}') AS custom_fields, CreatedDate AS created_at, data_source, coalesce(toUnixTimestamp64Milli(SystemModstamp), 0) AS _version diff --git a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_contacts.sql b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_contacts.sql index 6a3163128..4c6738802 100644 --- a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_contacts.sql +++ b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_contacts.sql @@ -27,7 +27,7 @@ WITH src AS ( 'LeadSource', coalesce(toString(LeadSource), ''), 'IsDeleted', toString(coalesce(IsDeleted, false)) )) AS metadata, - custom_fields, + coalesce(custom_fields, '{}') AS custom_fields, CreatedDate AS created_at, LastModifiedDate AS updated_at, data_source, diff --git a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_deals.sql b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_deals.sql index 98aa20607..d484ed91e 100644 --- a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_deals.sql +++ b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_deals.sql @@ -55,7 +55,7 @@ WITH src AS ( 'Type', coalesce(toString(Type), ''), 'IsDeleted', if(coalesce(IsDeleted, false), 'true', 'false') )) AS metadata, - custom_fields, + coalesce(custom_fields, '{}') AS custom_fields, CreatedDate AS created_at, LastModifiedDate AS updated_at, data_source, diff --git a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_users.sql b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_users.sql index 3c24c28eb..8b40b157a 100644 --- a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_users.sql +++ b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_users.sql @@ -29,7 +29,7 @@ WITH src AS ( 'Username', coalesce(toString(Username), ''), 'UserRoleId', coalesce(toString(UserRoleId), '') )) AS metadata, - custom_fields, + coalesce(custom_fields, '{}') AS custom_fields, collected_at, data_source, coalesce(toUnixTimestamp64Milli(SystemModstamp), 0) AS _version diff --git a/src/ingestion/connectors/git/github-v2/dbt/github__pull_requests_commits.sql b/src/ingestion/connectors/git/github-v2/dbt/github__pull_requests_commits.sql index 4056f2298..767255d18 100644 --- a/src/ingestion/connectors/git/github-v2/dbt/github__pull_requests_commits.sql +++ b/src/ingestion/connectors/git/github-v2/dbt/github__pull_requests_commits.sql @@ -15,7 +15,7 @@ SELECT COALESCE(repo_name, '') AS repo_slug, COALESCE(pull_request_id, 0) AS pr_id, COALESCE(sha, '') AS commit_hash, - 0 AS commit_order, + toInt64(0) AS commit_order, 'insight_github' AS data_source, toUnixTimestamp64Milli(now64()) AS _version, _airbyte_extracted_at diff --git a/src/ingestion/connectors/git/gitlab/dbt/gitlab__pull_requests_commits.sql b/src/ingestion/connectors/git/gitlab/dbt/gitlab__pull_requests_commits.sql index d8f7b53e7..2b941b056 100644 --- a/src/ingestion/connectors/git/gitlab/dbt/gitlab__pull_requests_commits.sql +++ b/src/ingestion/connectors/git/gitlab/dbt/gitlab__pull_requests_commits.sql @@ -28,7 +28,7 @@ SELECT COALESCE(p.repo_slug, '') AS repo_slug, COALESCE(mc.mr_iid, 0) AS pr_id, COALESCE(mc.id, '') AS commit_hash, - 0 AS commit_order, + toInt64(0) AS commit_order, 'insight_gitlab' AS data_source, toUnixTimestamp64Milli(now64()) AS _version, mc._airbyte_extracted_at diff --git a/src/ingestion/connectors/hr-directory/active-directory/dbt/active_directory__to_class_people.sql b/src/ingestion/connectors/hr-directory/active-directory/dbt/active_directory__to_class_people.sql index 2e7a916b9..d8b4f378d 100644 --- a/src/ingestion/connectors/hr-directory/active-directory/dbt/active_directory__to_class_people.sql +++ b/src/ingestion/connectors/hr-directory/active-directory/dbt/active_directory__to_class_people.sql @@ -52,8 +52,8 @@ SELECT -- AD has no employment-type field; default until the BambooHR join in -- Silver Step 2 (Identity Manager) supplies the real value. 'full_time' AS employment_type, - CAST(NULL AS Nullable(Date)) AS hire_date, - CAST(NULL AS Nullable(Date)) AS termination_date, + CAST(NULL AS Nullable(DateTime)) AS hire_date, + CAST(NULL AS Nullable(DateTime)) AS termination_date, CAST(NULL AS Nullable(String)) AS location, CAST(NULL AS Nullable(String)) AS country, CAST(NULL AS Nullable(Float64)) AS fte, diff --git a/src/ingestion/connectors/hr-directory/ms-entra/dbt/ms_entra__to_class_people.sql b/src/ingestion/connectors/hr-directory/ms-entra/dbt/ms_entra__to_class_people.sql index fea40f7b6..00ec1522b 100644 --- a/src/ingestion/connectors/hr-directory/ms-entra/dbt/ms_entra__to_class_people.sql +++ b/src/ingestion/connectors/hr-directory/ms-entra/dbt/ms_entra__to_class_people.sql @@ -46,8 +46,8 @@ SELECT -- Entra has no employment-type field; default until the BambooHR join -- in Silver Step 2 (Identity Manager) supplies the real value. 'full_time' AS employment_type, - CAST(NULL AS Nullable(Date)) AS hire_date, - CAST(NULL AS Nullable(Date)) AS termination_date, + CAST(NULL AS Nullable(DateTime)) AS hire_date, + CAST(NULL AS Nullable(DateTime)) AS termination_date, CAST(NULL AS Nullable(String)) AS location, CAST(NULL AS Nullable(String)) AS country, CAST(NULL AS Nullable(Float64)) AS fte, diff --git a/src/ingestion/scripts/connectors-ddl/silver.sql b/src/ingestion/scripts/connectors-ddl/silver.sql index 1a4a66d92..0c5099d13 100644 --- a/src/ingestion/scripts/connectors-ddl/silver.sql +++ b/src/ingestion/scripts/connectors-ddl/silver.sql @@ -165,7 +165,7 @@ CREATE TABLE IF NOT EXISTS silver.class_collab_document_activity `synced_count` Nullable(Decimal(38, 9)), `shared_internally_count` Nullable(Decimal(38, 9)), `shared_externally_count` Nullable(Decimal(38, 9)), - `visited_page_count` Nullable(Decimal(38, 9)), + `visited_page_count` Nullable(Int64), `report_period` Nullable(String), `collected_at` DateTime, `data_source` String, @@ -1194,5 +1194,6 @@ CREATE OR REPLACE VIEW silver.contract_version ( `version` UInt32 ) -AS SELECT toUInt32(1) AS version; +AS SELECT toUInt32(1) AS version +; From 5e09aebc42c2b42c6b80575759e7cbbb5f5fffda Mon Sep 17 00:00:00 2001 From: Roman Mitasov Date: Fri, 31 Jul 2026 18:45:16 +0800 Subject: [PATCH 2/2] refactor(dbt): declare the Nullable side of every widened union branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The field-parity audit reports 61 nullable-widening warnings: a branch declares `T` where its silver union target publishes `Nullable(T)`, because some other branch of the same target is nullable. The published silver type is correct — NULL there means "this source cannot measure this" (bitbucket carries no diff stats, zoom cannot split meeting kinds) and must stay distinct from a measured zero — but it was a supertype accident, dependent on which connectors happen to be enabled. Pin it: every such branch now emits Nullable explicitly via toNullable(), so the silver type no longer depends on the connector set. 47 casts across 17 models (ai counters, m365 meeting counters, git files_changed/lines_added/lines_removed, chat/meeting user_name + email, crm and task boolean flags). Values are untouched — same numbers, same zeros; only the declared type widens. Two of these were previously masked in CI: heal_collab_chat_table ALTERs slack/zulip/m365 staging to the contract types after dbt runs, so slack.direct_and_group_messages only surfaced once the table was rebuilt from the model. The model now owns the type instead of relying on the heal. NOT aligned — left as warnings on purpose: data_source, day, insight_source_id/field_* (jira metadata), status_id/status_name, collected_at, timestamp (15 findings). Those are mandatory-by-meaning columns where the nullable branch is the defect and the contract should narrow to NOT NULL — the opposite direction, tracked for the silver→gold contract work rather than blanket-widened here. Verified against the same from-scratch warehouse: 0 failures, warnings 61 -> 15, and the re-dumped connectors-ddl snapshot is byte-identical — silver already published these Nullable types, so nothing downstream moves. Co-Authored-By: Claude Opus 5 Signed-off-by: Roman Mitasov --- .../dbt/claude_enterprise__ai_dev_usage.sql | 12 +++++----- .../ai/cursor/dbt/cursor__ai_dev_usage.sql | 8 +++---- .../dbt/copilot__ai_dev_usage.sql | 10 ++++----- .../dbt/m365__collab_meeting_activity.sql | 22 +++++++++---------- .../slack/dbt/slack__collab_chat_activity.sql | 8 +++---- .../dbt/zoom__collab_meeting_activity.sql | 2 +- .../dbt/zulip_proxy__collab_chat_activity.sql | 2 +- .../crm/hubspot/dbt/hubspot__crm_deals.sql | 4 ++-- .../crm/hubspot/dbt/hubspot__crm_users.sql | 2 +- .../git/github-v2/dbt/github__commits.sql | 6 ++--- .../github-v2/dbt/github__file_changes.sql | 4 ++-- .../github-v2/dbt/github__pull_requests.sql | 6 ++--- .../git/gitlab/dbt/gitlab__commits.sql | 6 ++--- .../git/gitlab/dbt/gitlab__file_changes.sql | 4 ++-- .../git/gitlab/dbt/gitlab__pull_requests.sql | 6 ++--- .../jira/dbt/jira__task_comments.sql | 2 +- 16 files changed, 52 insertions(+), 52 deletions(-) diff --git a/src/ingestion/connectors/ai/claude-enterprise/dbt/claude_enterprise__ai_dev_usage.sql b/src/ingestion/connectors/ai/claude-enterprise/dbt/claude_enterprise__ai_dev_usage.sql index ea3e5880f..263b40963 100644 --- a/src/ingestion/connectors/ai/claude-enterprise/dbt/claude_enterprise__ai_dev_usage.sql +++ b/src/ingestion/connectors/ai/claude-enterprise/dbt/claude_enterprise__ai_dev_usage.sql @@ -56,14 +56,14 @@ SELECT toUInt32(coalesce(code_session_count, 0)) AS session_count, toUInt32OrNull(toString(code_session_count)) AS conversation_count, toUInt32(coalesce(code_lines_added, 0)) AS lines_added, - toUInt32(coalesce(code_lines_removed, 0)) AS lines_removed, + toNullable(toUInt32(coalesce(code_lines_removed, 0))) AS lines_removed, -- Enterprise reports AI-accepted lines only — no view of total user keystrokes. -- ai_loc_share_pct downstream filters to tool='cursor' precisely because of this gap. CAST(NULL AS Nullable(UInt32)) AS total_lines_added, CAST(NULL AS Nullable(UInt32)) AS total_lines_removed, - toUInt32(coalesce(code_tool_accepted_count, 0) - + coalesce(code_tool_rejected_count, 0)) AS tool_use_offered, - toUInt32(coalesce(code_tool_accepted_count, 0)) AS tool_use_accepted, + toNullable(toUInt32(coalesce(code_tool_accepted_count, 0) + + coalesce(code_tool_rejected_count, 0))) AS tool_use_offered, + toNullable(toUInt32(coalesce(code_tool_accepted_count, 0))) AS tool_use_accepted, -- #262: `completions_count` dropped from class_ai_dev_usage — it was -- numerically identical to tool_use_accepted (both = code_tool_accepted_count). -- agent_sessions := no direct Enterprise equivalent. Enterprise's cowork_dispatch_turn_count @@ -78,8 +78,8 @@ SELECT -- Cost is not surfaced per-user in Enterprise; tied to org subscription, not consumption. CAST(NULL AS Nullable(UInt32)) AS cost_cents, -- Enterprise exposes commit and PR counts per user per day via core_metrics. - toUInt32(coalesce(code_commit_count, 0)) AS commits_count, - toUInt32(coalesce(code_pull_request_count, 0)) AS pull_requests_count, + toNullable(toUInt32(coalesce(code_commit_count, 0))) AS commits_count, + toNullable(toUInt32(coalesce(code_pull_request_count, 0))) AS pull_requests_count, -- prs_with_cc_count / prs_total_count: Claude Team-only (Anthropic GitHub-app attribution). -- Enterprise exposes code_pull_request_count (above, → pull_requests_count) but not the -- GitHub-app split between "PRs with CC active" and "total PRs in window". diff --git a/src/ingestion/connectors/ai/cursor/dbt/cursor__ai_dev_usage.sql b/src/ingestion/connectors/ai/cursor/dbt/cursor__ai_dev_usage.sql index 56400af94..4505a39e8 100644 --- a/src/ingestion/connectors/ai/cursor/dbt/cursor__ai_dev_usage.sql +++ b/src/ingestion/connectors/ai/cursor/dbt/cursor__ai_dev_usage.sql @@ -87,18 +87,18 @@ SELECT toUInt32(1) AS session_count, CAST(NULL AS Nullable(UInt32)) AS conversation_count, toUInt32(coalesce(d.acceptedLinesAdded, 0)) AS lines_added, - toUInt32(coalesce(d.acceptedLinesDeleted, 0)) AS lines_removed, + toNullable(toUInt32(coalesce(d.acceptedLinesDeleted, 0))) AS lines_removed, -- total_lines_added/removed = ALL lines the user wrote/deleted that day -- (not just AI-accepted ones). Needed by gold metrics like -- ai_loc_share = accepted/total to express AI contribution percentage. - toUInt32(coalesce(d.totalLinesAdded, 0)) AS total_lines_added, - toUInt32(coalesce(d.totalLinesDeleted, 0)) AS total_lines_removed, + toNullable(toUInt32(coalesce(d.totalLinesAdded, 0))) AS total_lines_added, + toNullable(toUInt32(coalesce(d.totalLinesDeleted, 0))) AS total_lines_removed, toUInt32OrNull(toString(d.totalTabsShown)) AS tool_use_offered, toUInt32OrNull(toString(d.totalTabsAccepted)) AS tool_use_accepted, -- #262: `completions_count` was numerically identical to tool_use_accepted -- (both = totalTabsAccepted) and dropped from class_ai_dev_usage. toUInt32OrNull(toString(d.agentRequests)) AS agent_sessions, - toUInt32(coalesce(d.chatRequests, 0) + coalesce(d.composerRequests, 0)) + toNullable(toUInt32(coalesce(d.chatRequests, 0) + coalesce(d.composerRequests, 0))) AS chat_requests, -- Rank guard: cost is aggregated per (tenant, source, email, day) because -- Cursor's events carry only userEmail, while a row is identified by userId. diff --git a/src/ingestion/connectors/ai/github-copilot/dbt/copilot__ai_dev_usage.sql b/src/ingestion/connectors/ai/github-copilot/dbt/copilot__ai_dev_usage.sql index ddb3fec78..640ac5237 100644 --- a/src/ingestion/connectors/ai/github-copilot/dbt/copilot__ai_dev_usage.sql +++ b/src/ingestion/connectors/ai/github-copilot/dbt/copilot__ai_dev_usage.sql @@ -110,15 +110,15 @@ SELECT toUInt32(1) AS session_count, CAST(NULL AS Nullable(UInt32)) AS conversation_count, toUInt32(coalesce(m.loc_added_sum, 0)) AS lines_added, - toUInt32(coalesce(m.loc_deleted_sum, 0)) AS lines_removed, + toNullable(toUInt32(coalesce(m.loc_deleted_sum, 0))) AS lines_removed, -- See header comment — Copilot reports AI-accepted lines only. CAST(NULL AS Nullable(UInt32)) AS total_lines_added, CAST(NULL AS Nullable(UInt32)) AS total_lines_removed, -- code_generation_activity_count = number of code generation events -- (a proxy for "suggestions offered"). The API does not split offered -- from rejected, so this is the closest mappable signal. - toUInt32(coalesce(m.code_generation_activity_count, 0)) AS tool_use_offered, - toUInt32(coalesce(m.code_acceptance_activity_count, 0)) AS tool_use_accepted, + toNullable(toUInt32(coalesce(m.code_generation_activity_count, 0))) AS tool_use_offered, + toNullable(toUInt32(coalesce(m.code_acceptance_activity_count, 0))) AS tool_use_accepted, -- #262: `completions_count` dropped from class_ai_dev_usage — it was numerically identical -- to tool_use_accepted (code_acceptance_activity_count). Same drop applied to cursor and -- claude_enterprise in PR #262; copilot now aligned. @@ -144,12 +144,12 @@ SELECT -- column slot) and re-surfaces the chat/agent flags for downstream -- consumers that want raw boolean state rather than the marker -- counts above. - concat( + toNullable(concat( '{"used_chat":', if(coalesce(m.used_chat, false), 'true', 'false'), ',"used_agent":', if(coalesce(m.used_agent, false), 'true', 'false'), ',"used_cli":', if(coalesce(m.used_cli, false), 'true', 'false'), '}' - ) AS tool_action_breakdown_json, + )) AS tool_action_breakdown_json, 'copilot' AS source, 'insight_github_copilot' AS data_source, parseDateTime64BestEffortOrNull(coalesce(m.collected_at, ''), 3) AS collected_at, diff --git a/src/ingestion/connectors/collaboration/m365/dbt/m365__collab_meeting_activity.sql b/src/ingestion/connectors/collaboration/m365/dbt/m365__collab_meeting_activity.sql index a8b8d7aa4..d30c515fa 100644 --- a/src/ingestion/connectors/collaboration/m365/dbt/m365__collab_meeting_activity.sql +++ b/src/ingestion/connectors/collaboration/m365/dbt/m365__collab_meeting_activity.sql @@ -23,18 +23,18 @@ SELECT -- when both feeders UNION ALL into silver.class_collab_meeting_activity. -- Bronze stores them as Decimal(38, 9) / Float64; CH 25.3 refuses -- Int64 ∪ Decimal/Float with NO_COMMON_TYPE. - toInt64(coalesce(callCount, 0)) AS calls_count, - toInt64(coalesce(meetingsOrganizedCount, 0)) AS meetings_organized, + toNullable(toInt64(coalesce(callCount, 0))) AS calls_count, + toNullable(toInt64(coalesce(meetingsOrganizedCount, 0))) AS meetings_organized, toInt64(coalesce(meetingsAttendedCount, 0)) AS meetings_attended, - toInt64(coalesce(adHocMeetingsOrganizedCount, 0)) AS adhoc_meetings_organized, - toInt64(coalesce(adHocMeetingsAttendedCount, 0)) AS adhoc_meetings_attended, - toInt64(COALESCE(scheduledOneTimeMeetingsOrganizedCount, 0) - + COALESCE(scheduledRecurringMeetingsOrganizedCount, 0)) AS scheduled_meetings_organized, - toInt64(COALESCE(scheduledOneTimeMeetingsAttendedCount, 0) - + COALESCE(scheduledRecurringMeetingsAttendedCount, 0)) AS scheduled_meetings_attended, - toInt64({{ iso8601_duration_seconds("ifNull(audioDuration, 'PT0S')") }}) AS audio_duration_seconds, - toInt64({{ iso8601_duration_seconds("ifNull(videoDuration, 'PT0S')") }}) AS video_duration_seconds, - toInt64({{ iso8601_duration_seconds("ifNull(screenShareDuration, 'PT0S')") }}) AS screen_share_duration_seconds, + toNullable(toInt64(coalesce(adHocMeetingsOrganizedCount, 0))) AS adhoc_meetings_organized, + toNullable(toInt64(coalesce(adHocMeetingsAttendedCount, 0))) AS adhoc_meetings_attended, + toNullable(toInt64(COALESCE(scheduledOneTimeMeetingsOrganizedCount, 0) + + COALESCE(scheduledRecurringMeetingsOrganizedCount, 0))) AS scheduled_meetings_organized, + toNullable(toInt64(COALESCE(scheduledOneTimeMeetingsAttendedCount, 0) + + COALESCE(scheduledRecurringMeetingsAttendedCount, 0))) AS scheduled_meetings_attended, + toNullable(toInt64({{ iso8601_duration_seconds("ifNull(audioDuration, 'PT0S')") }})) AS audio_duration_seconds, + toNullable(toInt64({{ iso8601_duration_seconds("ifNull(videoDuration, 'PT0S')") }})) AS video_duration_seconds, + toNullable(toInt64({{ iso8601_duration_seconds("ifNull(screenShareDuration, 'PT0S')") }})) AS screen_share_duration_seconds, reportPeriod AS report_period, now() AS collected_at, 'insight_m365' AS data_source, diff --git a/src/ingestion/connectors/collaboration/slack/dbt/slack__collab_chat_activity.sql b/src/ingestion/connectors/collaboration/slack/dbt/slack__collab_chat_activity.sql index bf60e7a48..f8cd3b9b2 100644 --- a/src/ingestion/connectors/collaboration/slack/dbt/slack__collab_chat_activity.sql +++ b/src/ingestion/connectors/collaboration/slack/dbt/slack__collab_chat_activity.sql @@ -38,8 +38,8 @@ SELECT toString(toDate(parseDateTimeBestEffortOrNull(u.date))) )) AS unique_key, u.user_id, - coalesce(u.email_address, '') AS user_name, - coalesce(u.email_address, '') AS email, + toNullable(coalesce(u.email_address, '')) AS user_name, + toNullable(coalesce(u.email_address, '')) AS email, if(coalesce(u.email_address, '') != '', lower(u.email_address), lower(u.user_id)) AS person_key, @@ -47,10 +47,10 @@ SELECT CAST(NULL AS Nullable(Int64)) AS direct_messages, CAST(NULL AS Nullable(Int64)) AS group_chat_messages, -- #266: total - channel = DMs + MPIMs (everything posted outside channels). - toInt64(greatest( + toNullable(toInt64(greatest( coalesce(u.messages_posted_count, 0) - coalesce(u.channel_messages_posted_count, 0), 0 - )) AS direct_and_group_messages, + ))) AS direct_and_group_messages, coalesce(u.messages_posted_count, 0) AS total_chat_messages, u.channel_messages_posted_count AS channel_posts, CAST(NULL AS Nullable(Int64)) AS channel_replies, diff --git a/src/ingestion/connectors/collaboration/zoom/dbt/zoom__collab_meeting_activity.sql b/src/ingestion/connectors/collaboration/zoom/dbt/zoom__collab_meeting_activity.sql index 63ef8e4e1..cad910310 100644 --- a/src/ingestion/connectors/collaboration/zoom/dbt/zoom__collab_meeting_activity.sql +++ b/src/ingestion/connectors/collaboration/zoom/dbt/zoom__collab_meeting_activity.sql @@ -87,7 +87,7 @@ SELECT -- this, GROUP BY would split them and produce two rows with identical -- unique_key — the staging model's `unique_key` is keyed on -- (tenant, source, lower(email), date), so user_name is non-keying. - coalesce(any(p.user_name), '') AS user_name, + toNullable(coalesce(any(p.user_name), '')) AS user_name, p.email AS email, lower(p.email) AS person_key, toDate(parseDateTimeBestEffortOrNull(p.join_time)) AS date, diff --git a/src/ingestion/connectors/collaboration/zulip-proxy/dbt/zulip_proxy__collab_chat_activity.sql b/src/ingestion/connectors/collaboration/zulip-proxy/dbt/zulip_proxy__collab_chat_activity.sql index cc7f53ec1..3c7678f49 100644 --- a/src/ingestion/connectors/collaboration/zulip-proxy/dbt/zulip_proxy__collab_chat_activity.sql +++ b/src/ingestion/connectors/collaboration/zulip-proxy/dbt/zulip_proxy__collab_chat_activity.sql @@ -40,7 +40,7 @@ SELECT toString(toDate(parseDateTimeBestEffortOrNull(m.created_at))) )) AS unique_key, lower(u.email) AS user_id, - coalesce(any(u.full_name), '') AS user_name, + toNullable(coalesce(any(u.full_name), '')) AS user_name, lower(u.email) AS email, lower(u.email) AS person_key, toDate(parseDateTimeBestEffortOrNull(m.created_at)) AS date, diff --git a/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_deals.sql b/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_deals.sql index 25be6c671..927d18d86 100644 --- a/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_deals.sql +++ b/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_deals.sql @@ -58,8 +58,8 @@ WITH src AS ( nullIf(arrayElement( JSONExtract(coalesce(associations_companies, '[]'), 'Array(String)'), 1 ), '') AS account_id, - toInt64(coalesce(properties_hs_is_closed, 'false') = 'true') AS is_closed, - toInt64(coalesce(properties_hs_is_closed_won, 'false') = 'true') AS is_won, + toNullable(toInt64(coalesce(properties_hs_is_closed, 'false') = 'true')) AS is_closed, + toNullable(toInt64(coalesce(properties_hs_is_closed_won, 'false') = 'true')) AS is_won, properties_hs_analytics_source AS lead_source, toFloat64OrNull(properties_hs_deal_stage_probability) AS probability, properties_dealtype AS deal_type, diff --git a/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_users.sql b/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_users.sql index 22818943b..705c94fc2 100644 --- a/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_users.sql +++ b/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_users.sql @@ -49,7 +49,7 @@ WITH src AS ( -- the columns to exist so emit explicit NULLs. CAST(NULL AS Nullable(String)) AS title, CAST(NULL AS Nullable(String)) AS department, - toInt64(NOT coalesce(archived, false)) AS is_active, + toNullable(toInt64(NOT coalesce(archived, false))) AS is_active, toJSONString(map( 'userId', coalesce(toString(userId), ''), 'archived', toString(coalesce(archived, false)) diff --git a/src/ingestion/connectors/git/github-v2/dbt/github__commits.sql b/src/ingestion/connectors/git/github-v2/dbt/github__commits.sql index 2374e5031..2b6fe97fa 100644 --- a/src/ingestion/connectors/git/github-v2/dbt/github__commits.sql +++ b/src/ingestion/connectors/git/github-v2/dbt/github__commits.sql @@ -21,9 +21,9 @@ SELECT COALESCE(committer_email, '') AS committer_email, COALESCE(message, '') AS message, parseDateTimeBestEffortOrNull(committed_date) AS date, - COALESCE(changed_files, 0) AS files_changed, - COALESCE(additions, 0) AS lines_added, - COALESCE(deletions, 0) AS lines_removed, + toNullable(COALESCE(changed_files, 0)) AS files_changed, + toNullable(COALESCE(additions, 0)) AS lines_added, + toNullable(COALESCE(deletions, 0)) AS lines_removed, -- parent_hashes arrives as a JSON-array string (Airbyte serializes the -- connector's array field into the Nullable(String) bronze column), so -- count elements with JSONLength — plain length() would count characters diff --git a/src/ingestion/connectors/git/github-v2/dbt/github__file_changes.sql b/src/ingestion/connectors/git/github-v2/dbt/github__file_changes.sql index 004ba2af0..db3d40bcc 100644 --- a/src/ingestion/connectors/git/github-v2/dbt/github__file_changes.sql +++ b/src/ingestion/connectors/git/github-v2/dbt/github__file_changes.sql @@ -31,8 +31,8 @@ SELECT '' ) AS file_extension, COALESCE(status, '') AS change_type, - COALESCE(additions, 0) AS lines_added, - COALESCE(deletions, 0) AS lines_removed, + toNullable(COALESCE(additions, 0)) AS lines_added, + toNullable(COALESCE(deletions, 0)) AS lines_removed, COALESCE(source_type, '') AS source_type, 'insight_github' AS data_source, toUnixTimestamp64Milli(now64()) AS _version, diff --git a/src/ingestion/connectors/git/github-v2/dbt/github__pull_requests.sql b/src/ingestion/connectors/git/github-v2/dbt/github__pull_requests.sql index 6ecdaeae9..f25374fe6 100644 --- a/src/ingestion/connectors/git/github-v2/dbt/github__pull_requests.sql +++ b/src/ingestion/connectors/git/github-v2/dbt/github__pull_requests.sql @@ -26,9 +26,9 @@ SELECT parseDateTimeBestEffortOrNull(updated_at) AS updated_on, parseDateTimeBestEffortOrNull(closed_at) AS closed_on, COALESCE(merge_commit_sha, '') AS merge_commit_hash, - COALESCE(changed_files, 0) AS files_changed, - COALESCE(additions, 0) AS lines_added, - COALESCE(deletions, 0) AS lines_removed, + toNullable(COALESCE(changed_files, 0)) AS files_changed, + toNullable(COALESCE(additions, 0)) AS lines_added, + toNullable(COALESCE(deletions, 0)) AS lines_removed, 'insight_github' AS data_source, toUnixTimestamp64Milli(now64()) AS _version, _airbyte_extracted_at diff --git a/src/ingestion/connectors/git/gitlab/dbt/gitlab__commits.sql b/src/ingestion/connectors/git/gitlab/dbt/gitlab__commits.sql index 8e470547e..3dec91078 100644 --- a/src/ingestion/connectors/git/gitlab/dbt/gitlab__commits.sql +++ b/src/ingestion/connectors/git/gitlab/dbt/gitlab__commits.sql @@ -47,9 +47,9 @@ SELECT COALESCE(c.committer_email, '') AS committer_email, COALESCE(c.message, '') AS message, parseDateTimeBestEffortOrNull(c.committed_date) AS date, - toInt64(COALESCE(f.files_changed, 0)) AS files_changed, - COALESCE(c.stats_additions, 0) AS lines_added, - COALESCE(c.stats_deletions, 0) AS lines_removed, + toNullable(toInt64(COALESCE(f.files_changed, 0))) AS files_changed, + toNullable(COALESCE(c.stats_additions, 0)) AS lines_added, + toNullable(COALESCE(c.stats_deletions, 0)) AS lines_removed, if(COALESCE(c.parent_count, 0) > 1, 1, 0) AS is_merge_commit, 'insight_gitlab' AS data_source, toUnixTimestamp64Milli(now64()) AS _version, diff --git a/src/ingestion/connectors/git/gitlab/dbt/gitlab__file_changes.sql b/src/ingestion/connectors/git/gitlab/dbt/gitlab__file_changes.sql index 1cef74a9d..64610ad72 100644 --- a/src/ingestion/connectors/git/gitlab/dbt/gitlab__file_changes.sql +++ b/src/ingestion/connectors/git/gitlab/dbt/gitlab__file_changes.sql @@ -39,8 +39,8 @@ SELECT fc.renamed_file = true, 'renamed', 'modified' ) AS change_type, - COALESCE(fc.lines_added, 0) AS lines_added, - COALESCE(fc.lines_removed, 0) AS lines_removed, + toNullable(COALESCE(fc.lines_added, 0)) AS lines_added, + toNullable(COALESCE(fc.lines_removed, 0)) AS lines_removed, '' AS source_type, 'insight_gitlab' AS data_source, toUnixTimestamp64Milli(now64()) AS _version, diff --git a/src/ingestion/connectors/git/gitlab/dbt/gitlab__pull_requests.sql b/src/ingestion/connectors/git/gitlab/dbt/gitlab__pull_requests.sql index 8f5db0ddc..33bfb1abb 100644 --- a/src/ingestion/connectors/git/gitlab/dbt/gitlab__pull_requests.sql +++ b/src/ingestion/connectors/git/gitlab/dbt/gitlab__pull_requests.sql @@ -56,9 +56,9 @@ SELECT parseDateTimeBestEffortOrNull(mr.updated_at) AS updated_on, parseDateTimeBestEffortOrNull(COALESCE(mr.closed_at, mr.merged_at)) AS closed_on, COALESCE(mr.merge_commit_sha, '') AS merge_commit_hash, - toInt64(0) AS files_changed, - toInt64(0) AS lines_added, - toInt64(0) AS lines_removed, + toNullable(toInt64(0)) AS files_changed, + toNullable(toInt64(0)) AS lines_added, + toNullable(toInt64(0)) AS lines_removed, 'insight_gitlab' AS data_source, toUnixTimestamp64Milli(now64()) AS _version, mr._airbyte_extracted_at diff --git a/src/ingestion/connectors/task-tracking/jira/dbt/jira__task_comments.sql b/src/ingestion/connectors/task-tracking/jira/dbt/jira__task_comments.sql index 732ec9734..603254bcb 100644 --- a/src/ingestion/connectors/task-tracking/jira/dbt/jira__task_comments.sql +++ b/src/ingestion/connectors/task-tracking/jira/dbt/jira__task_comments.sql @@ -22,7 +22,7 @@ SELECT parseDateTime64BestEffortOrNull(c.created, 3) AS created_at, parseDateTime64BestEffortOrNull(c.updated, 3) AS updated_at, c.body AS body, - toUInt8(0) AS is_deleted, + toNullable(toUInt8(0)) AS is_deleted, toUnixTimestamp64Milli(now64(3)) AS _version FROM ( SELECT * FROM {{ source('bronze_jira', 'jira_comments') }}