Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 27 additions & 7 deletions src/ingestion/connectors/ai/chatgpt-team/dbt/schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,39 @@ version: 2

sources:
- name: bronze_chatgpt_team
# Source freshness (TAF #1322): the newest Airbyte row must be recent,
# else the sync silently stopped and dashboards go stale with no error.
# Source freshness (TAF #1322). Source-level default = _airbyte_extracted_at
# (sync-liveness) for the roster table; the WINDOWED daily/snapshot tables
# override below to anchor on their business date. ChatGPT Team re-emits a
# fixed window every sync, so extracted_at is false-green for them — VERIFIED
# on live data (extracted_at ~8.6h fresh while `date` 60–85h stale). Report
# tier absorbs the vendor publish lag.
freshness:
warn_after: {count: 36, period: hour}
error_after: {count: 72, period: hour}
loaded_at_field: _airbyte_extracted_at
schema: bronze_chatgpt_team
tables:
- name: chatgpt_team_seats
- name: chatgpt_team_chat_activity
- name: chatgpt_team_codex_user_daily
- name: chatgpt_team_subscription_usage
- name: chatgpt_team_subscription_balance
- name: chatgpt_team_seats # roster — sync-liveness default is correct
- name: chatgpt_team_chat_activity # VERIFIED windowed
loaded_at_field: parseDateTimeBestEffortOrNull(date)
freshness:
warn_after: {count: 72, period: hour}
error_after: {count: 120, period: hour}
- name: chatgpt_team_codex_user_daily # VERIFIED windowed
loaded_at_field: parseDateTimeBestEffortOrNull(date)
freshness:
warn_after: {count: 72, period: hour}
error_after: {count: 120, period: hour}
- name: chatgpt_team_subscription_usage # inferred windowed (snapshot_date)
loaded_at_field: parseDateTimeBestEffortOrNull(snapshot_date)
freshness:
warn_after: {count: 72, period: hour}
error_after: {count: 120, period: hour}
- name: chatgpt_team_subscription_balance # inferred windowed (snapshot_date)
loaded_at_field: parseDateTimeBestEffortOrNull(snapshot_date)
freshness:
warn_after: {count: 72, period: hour}
error_after: {count: 120, period: hour}

models:
- name: chatgpt_team__ai_dev_usage
Expand Down
26 changes: 21 additions & 5 deletions src/ingestion/connectors/ai/claude-admin/dbt/schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,34 @@ version: 2

sources:
- name: bronze_claude_admin
# Source freshness (TAF #1322): the newest Airbyte row must be recent,
# else the sync silently stopped and dashboards go stale with no error.
# Source freshness (TAF #1322). INFERRED, NOT live-verified — bronze_claude_admin
# has no rows in any reachable install (config is not a reliable signal, cf.
# M365). Source-level default = _airbyte_extracted_at (sync-liveness) for the
# roster/lookup tables (full-refresh → extracted_at stays fresh, so it still
# catches a stopped sync). The daily usage tables are WINDOWED (incremental on
# `date`) and override below. Confirm against live data once ingested.
freshness:
warn_after: {count: 36, period: hour}
error_after: {count: 72, period: hour}
loaded_at_field: _airbyte_extracted_at
schema: bronze_claude_admin
tables:
- name: claude_admin_users
- name: claude_admin_messages_usage
- name: claude_admin_cost_report
- name: claude_admin_code_usage
- name: claude_admin_messages_usage # inferred windowed (date)
loaded_at_field: parseDateTimeBestEffortOrNull(date)
freshness:
warn_after: {count: 72, period: hour}
error_after: {count: 120, period: hour}
- name: claude_admin_cost_report # inferred windowed (date)
loaded_at_field: parseDateTimeBestEffortOrNull(date)
freshness:
warn_after: {count: 72, period: hour}
error_after: {count: 120, period: hour}
- name: claude_admin_code_usage # inferred windowed (date)
loaded_at_field: parseDateTimeBestEffortOrNull(date)
freshness:
warn_after: {count: 72, period: hour}
error_after: {count: 120, period: hour}
- name: claude_admin_api_keys
- name: claude_admin_workspaces
- name: claude_admin_workspace_members
Expand Down
14 changes: 11 additions & 3 deletions src/ingestion/connectors/ai/claude-enterprise/dbt/schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,23 @@ version: 2

sources:
- name: bronze_claude_enterprise
# Source freshness (TAF #1322): the newest Airbyte row must be recent,
# else the sync silently stopped and dashboards go stale with no error.
# Source freshness (TAF #1322). INFERRED, NOT live-verified — bronze_claude_enterprise
# has no rows in any reachable install (config is not a reliable signal, cf.
# M365). Source-level default = _airbyte_extracted_at (sync-liveness) for the
# roster/lookup tables. The daily usage summaries table is WINDOWED
# (incremental on `date`) and overrides below. Confirm against live data
# once Claude Enterprise is ingested.
freshness:
warn_after: {count: 36, period: hour}
error_after: {count: 72, period: hour}
loaded_at_field: _airbyte_extracted_at
schema: bronze_claude_enterprise
tables:
- name: claude_enterprise_summaries
- name: claude_enterprise_summaries # inferred windowed (date)
loaded_at_field: parseDateTimeBestEffortOrNull(date)
freshness:
warn_after: {count: 72, period: hour}
error_after: {count: 120, period: hour}
- name: claude_enterprise_users
- name: claude_enterprise_chat_projects
- name: claude_enterprise_skills
Expand Down
16 changes: 12 additions & 4 deletions src/ingestion/connectors/ai/claude-team/dbt/schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,24 @@ version: 2

sources:
- name: bronze_claude_team
# Source freshness (TAF #1322): the newest Airbyte row must be recent,
# else the sync silently stopped and dashboards go stale with no error.
# Source freshness (TAF #1322). Source-level default = _airbyte_extracted_at
# (sync-liveness) for the roster table; the WINDOWED daily metrics table
# overrides below. Claude Team re-emits a fixed window every sync, so
# extracted_at is false-green for code_metrics — VERIFIED on live data
# (extracted_at ~8.6h fresh while metric_date 60.7h stale). Report tier
# absorbs the vendor publish lag.
freshness:
warn_after: {count: 36, period: hour}
error_after: {count: 72, period: hour}
loaded_at_field: _airbyte_extracted_at
schema: bronze_claude_team
tables:
- name: claude_team_members
- name: claude_team_code_metrics
- name: claude_team_members # roster — sync-liveness default is correct
- name: claude_team_code_metrics # VERIFIED windowed
loaded_at_field: parseDateTimeBestEffortOrNull(metric_date)
freshness:
warn_after: {count: 72, period: hour}
error_after: {count: 120, period: hour}

models:
- name: claude_team__ai_dev_usage
Expand Down
15 changes: 12 additions & 3 deletions src/ingestion/connectors/ai/cursor/dbt/schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,13 @@ version: 2

sources:
- name: bronze_cursor
# Source freshness (TAF #1322): the newest Airbyte row must be recent,
# else the sync silently stopped and dashboards go stale with no error.
# Source freshness (TAF #1322). Source-level default = _airbyte_extracted_at
# (sync-liveness) for the roster + event-stream tables (members, audit_logs,
# usage_events — incremental on an event cursor, where extracted_at tracks
# reality). The aggregated daily table is WINDOWED (the connector re-fetches
# yesterday on every run — `cursor_usage_events_daily_resync: true`), so its
# extracted_at is false-green; it overrides below to anchor on `day` (column
# name verified in ClickHouse; behavior inferred — no live rows in Eternal).
freshness:
warn_after: {count: 36, period: hour}
error_after: {count: 72, period: hour}
Expand All @@ -14,7 +19,11 @@ sources:
- name: cursor_audit_logs
- name: cursor_usage_events
- name: cursor_usage_events_daily_resync
- name: cursor_daily_usage
- name: cursor_daily_usage # inferred windowed (daily resync re-fetch)
loaded_at_field: parseDateTimeBestEffortOrNull(day)
freshness:
warn_after: {count: 72, period: hour}
error_after: {count: 120, period: hour}

models:
- name: cursor__ai_dev_usage
Expand Down
22 changes: 17 additions & 5 deletions src/ingestion/connectors/ai/github-copilot/dbt/schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,29 @@ version: 2

sources:
- name: bronze_github_copilot
# Source freshness (TAF #1322): the newest Airbyte row must be recent,
# else the sync silently stopped and dashboards go stale with no error.
# Source freshness (TAF #1322). INFERRED, NOT live-verified — bronze_github_copilot
# has no rows in any reachable install. The metrics streams are per-day
# reports with a documented ~24h+ publish lag (cursor `day`, confirmed in
# README + bronze schema); anchoring on `day` with the report tier is correct
# if the API re-emits a window and harmless if it is strictly incremental.
# `copilot_seats` is a roster → sync-liveness default. Confirm against live data.
freshness:
warn_after: {count: 36, period: hour}
error_after: {count: 72, period: hour}
loaded_at_field: _airbyte_extracted_at
schema: bronze_github_copilot
tables:
- name: copilot_seats
- name: copilot_user_metrics
- name: copilot_org_metrics
- name: copilot_seats # roster — sync-liveness default
- name: copilot_user_metrics # inferred windowed (day, ~24h+ report lag)
loaded_at_field: parseDateTimeBestEffortOrNull(day)
freshness:
warn_after: {count: 72, period: hour}
error_after: {count: 120, period: hour}
- name: copilot_org_metrics # inferred windowed (day, ~24h+ report lag)
loaded_at_field: parseDateTimeBestEffortOrNull(day)
freshness:
warn_after: {count: 72, period: hour}
error_after: {count: 120, period: hour}

models:
- name: copilot__ai_dev_usage
Expand Down
22 changes: 16 additions & 6 deletions src/ingestion/connectors/ai/openai/dbt/schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,25 @@ version: 2

sources:
- name: bronze_openai
# Source freshness (TAF #1322): the newest Airbyte row must be recent,
# else the sync silently stopped and dashboards go stale with no error.
# Source freshness (TAF #1322). INFERRED, NOT live-verified — bronze_openai
# has no rows in any reachable install, and connector config is not a
# reliable windowed/incremental signal (cf. M365, which declared
# lookback_window: P0D yet re-fetches a 24-day window). The usage/cost
# streams are incremental on `bucket_start_time` (the OpenAI usage API's
# business bucket), so anchor source-level on it with the report tier;
# `users` is a full-refresh roster with no bucket column, overridden to the
# sync-liveness default. Confirm against live data once OpenAI is ingested.
freshness:
warn_after: {count: 36, period: hour}
error_after: {count: 72, period: hour}
loaded_at_field: _airbyte_extracted_at
warn_after: {count: 72, period: hour}
error_after: {count: 120, period: hour}
loaded_at_field: parseDateTimeBestEffortOrNull(bucket_start_time)
schema: bronze_openai
tables:
- name: users
- name: users # roster — no bucket column; sync-liveness default
loaded_at_field: _airbyte_extracted_at
freshness:
warn_after: {count: 36, period: hour}
error_after: {count: 72, period: hour}
- name: usage_completions
- name: usage_embeddings
- name: usage_moderations
Expand Down
17 changes: 12 additions & 5 deletions src/ingestion/connectors/collaboration/m365/dbt/schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,19 @@ version: 2

sources:
- name: bronze_m365
# Source freshness (TAF #1322): the newest Airbyte row must be recent,
# else the sync silently stopped and dashboards go stale with no error.
# Source freshness (TAF #1322). WINDOWED connector — verified on live data:
# the Graph getReport endpoint re-emits the whole `reportPeriod` window
# every sync, so `_airbyte_extracted_at` is re-stamped each run and tracks
# sync liveness, NOT data freshness (measured: extracted_at 10.6h fresh
# while reportRefreshDate 108.7h / 4.5d stale; 24 business dates re-emitted
# in the last extraction batch). Anchor on the report's own business date
# and use the report tier to absorb Graph's 24–48h publish lag.
freshness:
warn_after: {count: 36, period: hour}
error_after: {count: 72, period: hour}
loaded_at_field: _airbyte_extracted_at
warn_after: {count: 72, period: hour}
error_after: {count: 120, period: hour}
# reportRefreshDate is Nullable(String) 'YYYY-MM-DD'; dbt freshness needs a
# timestamp expression, so parse it (verified against live ClickHouse).
loaded_at_field: parseDateTimeBestEffortOrNull(reportRefreshDate)
schema: bronze_m365
tables:
- name: email_activity
Expand Down
15 changes: 10 additions & 5 deletions src/ingestion/connectors/collaboration/slack/dbt/schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,17 @@ version: 2

sources:
- name: bronze_slack
# Source freshness (TAF #1322): the newest Airbyte row must be recent,
# else the sync silently stopped and dashboards go stale with no error.
# Source freshness (TAF #1322). WINDOWED connector — Slack
# admin.analytics.getFile re-emits a fixed window each sync, so
# _airbyte_extracted_at tracks sync liveness, not data freshness. Anchor on
# the report's `date` (column verified in ClickHouse; windowed behavior
# inferred — no live rows in Eternal). report_extended tier: Slack analytics
# has a documented ~3-day publish lag (up to 5 during Slack maintenance).
freshness:
warn_after: {count: 36, period: hour}
error_after: {count: 72, period: hour}
loaded_at_field: _airbyte_extracted_at
warn_after: {count: 120, period: hour}
error_after: {count: 168, period: hour}
# `date` is String 'YYYY-MM-DD'; parse for the freshness timestamp compare.
loaded_at_field: parseDateTimeBestEffortOrNull(date)
schema: bronze_slack
tables:
- name: users_details
Expand Down
23 changes: 18 additions & 5 deletions src/ingestion/connectors/collaboration/zoom/dbt/schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,30 @@ version: 2

sources:
- name: bronze_zoom
# Source freshness (TAF #1322): the newest Airbyte row must be recent,
# else the sync silently stopped and dashboards go stale with no error.
# Source freshness (TAF #1322). Source-level default = _airbyte_extracted_at
# (sync-liveness) for the roster table. The event tables are WINDOWED — the
# connector re-fetches a 30-day window (lookback_window: P7D, step: P30D), so
# extracted_at is false-green; they override below to anchor on the event
# time. event tier (96h/168h) absorbs legitimately quiet weekends/holidays
# where zero meetings is normal. (columns from connector.yaml / participants
# verified in ClickHouse; windowed behavior inferred — meetings empty in Eternal.)
freshness:
warn_after: {count: 36, period: hour}
error_after: {count: 72, period: hour}
loaded_at_field: _airbyte_extracted_at
schema: bronze_zoom
tables:
- name: users
- name: meetings
- name: participants
- name: users # roster — sync-liveness default is correct
- name: meetings # inferred windowed (event)
loaded_at_field: parseDateTimeBestEffortOrNull(end_time)
freshness:
warn_after: {count: 96, period: hour}
error_after: {count: 168, period: hour}
- name: participants # inferred windowed (event)
loaded_at_field: parseDateTimeBestEffortOrNull(join_time)
freshness:
warn_after: {count: 96, period: hour}
error_after: {count: 168, period: hour}

models:
- name: zoom__meeting_sessions
Expand Down
15 changes: 12 additions & 3 deletions src/ingestion/connectors/wiki/confluence/dbt/schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,25 @@ version: 2

sources:
- name: bronze_confluence
# Source freshness (TAF #1322): the newest Airbyte row must be recent,
# else the sync silently stopped and dashboards go stale with no error.
# Source freshness (TAF #1322). INFERRED, NOT live-verified — bronze_confluence
# has no rows in any reachable install. Edit activity is event-style: a quiet
# weekend/holiday with zero page edits is normal, so flat _airbyte_extracted_at
# would false-alarm and the event tier (96h/168h) is the right band. The
# content table anchors on `updated_at` (cursor-confirmed); the remaining
# version/comment/space tables stay on the sync-liveness default rather than
# guess per-table edit-time columns without data. Confirm once ingested.
freshness:
warn_after: {count: 36, period: hour}
error_after: {count: 72, period: hour}
loaded_at_field: _airbyte_extracted_at
schema: bronze_confluence
tables:
- name: wiki_spaces
- name: wiki_pages
- name: wiki_pages # inferred windowed/event (updated_at)
loaded_at_field: parseDateTimeBestEffortOrNull(updated_at)
freshness:
warn_after: {count: 96, period: hour}
error_after: {count: 168, period: hour}
- name: wiki_page_versions
- name: wiki_footer_comments
- name: wiki_footer_comment_replies
Expand Down
13 changes: 10 additions & 3 deletions src/ingestion/connectors/wiki/outline/dbt/schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,23 @@ version: 2

sources:
- name: bronze_outline
# Source freshness (TAF #1322): the newest Airbyte row must be recent,
# else the sync silently stopped and dashboards go stale with no error.
# Source freshness (TAF #1322). INFERRED, NOT live-verified — bronze_outline
# has no rows in any reachable install. Edit activity is event-style (quiet
# periods are normal), so the content table anchors on `updated_at`
# (cursor-confirmed) with the event tier (96h/168h); the rest stay on the
# sync-liveness default. Confirm once ingested.
freshness:
warn_after: {count: 36, period: hour}
error_after: {count: 72, period: hour}
loaded_at_field: _airbyte_extracted_at
schema: bronze_outline
tables:
- name: wiki_spaces
- name: wiki_pages
- name: wiki_pages # inferred windowed/event (updated_at)
loaded_at_field: parseDateTimeBestEffortOrNull(updated_at)
freshness:
warn_after: {count: 96, period: hour}
error_after: {count: 168, period: hour}
- name: wiki_page_versions
- name: wiki_comments
- name: wiki_users
Expand Down