From e2d5e3672c0761aa6a9e38e25b14f03a705dfc03 Mon Sep 17 00:00:00 2001 From: Sergei Mozhaev Date: Sat, 27 Jun 2026 07:59:17 +0300 Subject: [PATCH] test(e2e): cover cc_tool_accept + cc_tool_acceptance (Claude Enterprise) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last two AI-bullet metrics that claude_team can't supply (it sets tool_use_accepted/offered = NULL): both come from the Claude Enterprise admin Analytics API, where tool_use_accepted ← code_tool_accepted_count and tool_use_offered ← code_tool_accepted_count + code_tool_rejected_count. - cc_tool_accept — sum metric; alice 50 · bob 20 · carol 10 → median 20, range [10, 50] (dedup holds). - cc_tool_acceptance — reconstructed ratio 100 × Σ accepted ÷ Σ offered; alice 50 of 80 → 62.5 (value-only). Pipeline: bronze_claude_enterprise.claude_enterprise_users → claude_enterprise__ai_dev_usage (tool='claude_code') → silver class_ai_dev_usage → gold insight.ai_bullet_rows → IC Bullet AI (…0013). Supporting: - bronze placeholder for claude_enterprise_users (+ schema + template). Note: user_id MUST be distinct per seat — the staging model dedups with `LIMIT 1 BY tenant, source, user_id, date`, so a shared/NULL user_id collapses the cohort to one row and the team median is wrong. - claude_enterprise__ai_dev_usage: same empty-table watermark guard as the other AI staging models (INTERVAL 3 DAY underflows over an empty `this`). - conftest: reset staging.claude_enterprise__ai_dev_usage at session start. Verified locally: 2 passed. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Sergei Mozhaev --- .../dbt/claude_enterprise__ai_dev_usage.sql | 14 +++- .../scripts/create-bronze-placeholders.sh | 56 +++++++++++++++ src/ingestion/tests/e2e/conftest.py | 5 ++ .../e2e/metrics/ai_cc_tool_accept.test.yaml | 54 +++++++++++++++ .../metrics/ai_cc_tool_acceptance.test.yaml | 58 ++++++++++++++++ ...de_enterprise.claude_enterprise_users.yaml | 53 ++++++++++++++ .../templates/claude_enterprise_users.yaml | 69 +++++++++++++++++++ 7 files changed, 306 insertions(+), 3 deletions(-) create mode 100644 src/ingestion/tests/e2e/metrics/ai_cc_tool_accept.test.yaml create mode 100644 src/ingestion/tests/e2e/metrics/ai_cc_tool_acceptance.test.yaml create mode 100644 src/ingestion/tests/e2e/metrics/schemas/bronze_claude_enterprise.claude_enterprise_users.yaml create mode 100644 src/ingestion/tests/e2e/metrics/templates/claude_enterprise_users.yaml diff --git a/src/ingestion/connectors/ai/claude-enterprise/dbt/claude_enterprise__ai_dev_usage.sql b/src/ingestion/connectors/ai/claude-enterprise/dbt/claude_enterprise__ai_dev_usage.sql index 35600bfa9..95169756e 100644 --- a/src/ingestion/connectors/ai/claude-enterprise/dbt/claude_enterprise__ai_dev_usage.sql +++ b/src/ingestion/connectors/ai/claude-enterprise/dbt/claude_enterprise__ai_dev_usage.sql @@ -113,8 +113,16 @@ WHERE user_email IS NOT NULL OR coalesce(code_commit_count, 0) > 0 OR coalesce(code_pull_request_count, 0) > 0) {% if is_incremental() %} - AND toDate(parseDateTimeBestEffortOrNull(date)) > ( - SELECT coalesce(max(day), toDate('1970-01-01')) - INTERVAL 3 DAY - FROM {{ this }} + -- Empty-table guard. Over an empty `this` (the e2e rig resets staging between + -- tests) max(day) is the Date epoch (1970-01-01) and `- INTERVAL 3 DAY` + -- underflows the Date range, wrapping to ~2149-06-04 — which filters out every + -- row and leaves the model empty. Short-circuit when empty so the full set is + -- (re)loaded. Mirrors the cursor / claude_team / m365__collab_* guard. + AND ( + (SELECT count() FROM {{ this }}) = 0 + OR toDate(parseDateTimeBestEffortOrNull(date)) > ( + SELECT coalesce(max(day), toDate('1970-01-01')) - INTERVAL 3 DAY + FROM {{ this }} + ) ) {% endif %} diff --git a/src/ingestion/scripts/create-bronze-placeholders.sh b/src/ingestion/scripts/create-bronze-placeholders.sh index cb688d5cd..bf840b168 100644 --- a/src/ingestion/scripts/create-bronze-placeholders.sh +++ b/src/ingestion/scripts/create-bronze-placeholders.sh @@ -60,6 +60,7 @@ CREATE DATABASE IF NOT EXISTS bronze_bamboohr; CREATE DATABASE IF NOT EXISTS bronze_bitbucket_cloud; CREATE DATABASE IF NOT EXISTS bronze_zulip_proxy; CREATE DATABASE IF NOT EXISTS bronze_claude_team; +CREATE DATABASE IF NOT EXISTS bronze_claude_enterprise; SQL # --------------------------------------------------------------------------- @@ -1219,4 +1220,59 @@ CREATE TABLE IF NOT EXISTS bronze_zulip_proxy.users ( SQL fi +# bronze_claude_enterprise.claude_enterprise_users — per-user/day Claude Enterprise +# usage (admin Analytics API). claude_enterprise__ai_dev_usage (tool='claude_code') +# reads user_email/date/code_*; tool_use_accepted ← code_tool_accepted_count, +# tool_use_offered ← code_tool_accepted_count + code_tool_rejected_count. Mirrors +# the connector InlineSchemaLoader (unique_key + date are non-null String). +if ! ch_table_exists bronze_claude_enterprise claude_enterprise_users; then + echo " Creating placeholder: bronze_claude_enterprise.claude_enterprise_users" + run_ch <<'SQL' +CREATE TABLE IF NOT EXISTS bronze_claude_enterprise.claude_enterprise_users ( + unique_key String, + tenant_id Nullable(String), + source_id Nullable(String), + date String, + user_id Nullable(String), + user_email Nullable(String), + chat_conversation_count Nullable(Int64), + chat_message_count Nullable(Int64), + chat_projects_created_count Nullable(Int64), + chat_projects_used_count Nullable(Int64), + chat_files_uploaded_count Nullable(Int64), + chat_artifacts_created_count Nullable(Int64), + chat_thinking_message_count Nullable(Int64), + chat_skills_used_count Nullable(Int64), + chat_connectors_used_count Nullable(Int64), + code_commit_count Nullable(Int64), + code_pull_request_count Nullable(Int64), + code_lines_added Nullable(Int64), + code_lines_removed Nullable(Int64), + code_session_count Nullable(Int64), + code_tool_accepted_count Nullable(Int64), + code_tool_rejected_count Nullable(Int64), + web_search_count Nullable(Int64), + excel_session_count Nullable(Int64), + excel_message_count Nullable(Int64), + powerpoint_session_count Nullable(Int64), + powerpoint_message_count Nullable(Int64), + cowork_session_count Nullable(Int64), + cowork_message_count Nullable(Int64), + cowork_action_count Nullable(Int64), + cowork_dispatch_turn_count Nullable(Int64), + cowork_skills_used_count Nullable(Int64), + chat_metrics_json Nullable(String), + claude_code_metrics_json Nullable(String), + office_metrics_json Nullable(String), + cowork_metrics_json Nullable(String), + collected_at Nullable(String), + data_source Nullable(String), + _airbyte_raw_id String DEFAULT toString(generateUUIDv4()), + _airbyte_extracted_at DateTime64(3) DEFAULT now64(3), + _airbyte_meta String DEFAULT '{}', + _airbyte_generation_id UInt32 DEFAULT 0 +) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key; +SQL +fi + echo "=== Placeholders: done ===" diff --git a/src/ingestion/tests/e2e/conftest.py b/src/ingestion/tests/e2e/conftest.py index d3640b23a..1f5679553 100644 --- a/src/ingestion/tests/e2e/conftest.py +++ b/src/ingestion/tests/e2e/conftest.py @@ -145,6 +145,11 @@ def compose_stack(session_cfg: SessionConfig): # claude_team__ai_overage (cc_overage) is also incremental `append` with a # dbt `unique` test — reset it too for warm-rerun determinism. ("staging", "claude_team__ai_overage"), + # claude_enterprise specs build staging.claude_enterprise__ai_dev_usage — an + # incremental `append` model with a dbt `unique` test on unique_key. + # Session-start reset keeps warm re-runs (reused CH volume, no `./e2e.sh + # down`) from accumulating duplicate keys. + ("staging", "claude_enterprise__ai_dev_usage"), ] diff --git a/src/ingestion/tests/e2e/metrics/ai_cc_tool_accept.test.yaml b/src/ingestion/tests/e2e/metrics/ai_cc_tool_accept.test.yaml new file mode 100644 index 000000000..e934ce55c --- /dev/null +++ b/src/ingestion/tests/e2e/metrics/ai_cc_tool_accept.test.yaml @@ -0,0 +1,54 @@ +spec_version: 1 +description: > + Metric: cc_tool_accept — IC Bullet AI (…0013), #1440. + How it's computed (bronze → silver → gold): + • bronze: Claude Enterprise per-user/day usage (code tool suggestions accepted) + • silver: deduped to one row per user/day carrying the accepted-suggestions count + • gold: metric = sum of accepted Claude Code tool-use suggestions over the window + + Team (median/range = the person's department): + alice 50 · bob 20 · carol 10 (alice row re-synced once → must not double) + → median 20, range [10, 50]. + +bronze: + bronze_bamboohr.employees: + - $ref: templates/people.yaml#/templates/alice + - $ref: templates/people.yaml#/templates/bob + - $ref: templates/people.yaml#/templates/carol + bronze_claude_enterprise.claude_enterprise_users: + - $ref: templates/claude_enterprise_users.yaml#/templates/alice + unique_key: ent-alice-20260105 + date: "2026-01-05" + code_tool_accepted_count: 50 + - $ref: templates/claude_enterprise_users.yaml#/templates/alice # re-sync dup → must NOT double + unique_key: ent-alice-20260105 + date: "2026-01-05" + code_tool_accepted_count: 50 + - $ref: templates/claude_enterprise_users.yaml#/templates/bob + unique_key: ent-bob-20260105 + date: "2026-01-05" + code_tool_accepted_count: 20 + - $ref: templates/claude_enterprise_users.yaml#/templates/carol + unique_key: ent-carol-20260105 + date: "2026-01-05" + code_tool_accepted_count: 10 + +cases: + - name: cc_tool_accept — IC bullet (alice value vs team median, dedup holds) + request: + url: /v1/metrics/queries + method: POST + body: + queries: + - id: ai + metric_id: 00000000-0000-0000-0001-000000000013 + $top: 50 + $filter: "person_id eq 'alice@example.com' and metric_date ge '2026-01-01' and metric_date le '2026-01-31'" + $orderby: metric_key + expect: + - assert: "status == 200" + - in: ai + assert: "result.status == 'ok'" + - in: ai + find: { metric_key: cc_tool_accept } + equal: { value: 50, median: 20, range_min: 10, range_max: 50 } diff --git a/src/ingestion/tests/e2e/metrics/ai_cc_tool_acceptance.test.yaml b/src/ingestion/tests/e2e/metrics/ai_cc_tool_acceptance.test.yaml new file mode 100644 index 000000000..181452645 --- /dev/null +++ b/src/ingestion/tests/e2e/metrics/ai_cc_tool_acceptance.test.yaml @@ -0,0 +1,58 @@ +spec_version: 1 +description: > + Metric: cc_tool_acceptance — IC Bullet AI (…0013), #1440. + How it's computed (bronze → silver → gold): + • bronze: Claude Enterprise per-user/day usage (code tool suggestions accepted & rejected) + • silver: deduped to one row per user/day with accepted + offered (accepted + rejected) + • gold: a reconstructed ratio — 100 × Σ accepted ÷ Σ offered, rounded to 0.1 + (computed at serve time, not stored) + + Team (value = the requested person): + alice accepted 50 of 80 offered (50 accepted + 30 rejected) → 100×50/80 = 62.5. + +bronze: + bronze_bamboohr.employees: + - $ref: templates/people.yaml#/templates/alice + - $ref: templates/people.yaml#/templates/bob + - $ref: templates/people.yaml#/templates/carol + bronze_claude_enterprise.claude_enterprise_users: + - $ref: templates/claude_enterprise_users.yaml#/templates/alice + unique_key: ent-alice-20260105 + date: "2026-01-05" + code_tool_accepted_count: 50 + code_tool_rejected_count: 30 + - $ref: templates/claude_enterprise_users.yaml#/templates/alice # re-sync dup → ratio unchanged + unique_key: ent-alice-20260105 + date: "2026-01-05" + code_tool_accepted_count: 50 + code_tool_rejected_count: 30 + - $ref: templates/claude_enterprise_users.yaml#/templates/bob + unique_key: ent-bob-20260105 + date: "2026-01-05" + code_tool_accepted_count: 20 + code_tool_rejected_count: 60 + - $ref: templates/claude_enterprise_users.yaml#/templates/carol + unique_key: ent-carol-20260105 + date: "2026-01-05" + code_tool_accepted_count: 10 + code_tool_rejected_count: 15 + +cases: + - name: cc_tool_acceptance — IC bullet (reconstructed ratio, alice 50/80 = 62.5) + request: + url: /v1/metrics/queries + method: POST + body: + queries: + - id: ai + metric_id: 00000000-0000-0000-0001-000000000013 + $top: 50 + $filter: "person_id eq 'alice@example.com' and metric_date ge '2026-01-01' and metric_date le '2026-01-31'" + $orderby: metric_key + expect: + - assert: "status == 200" + - in: ai + assert: "result.status == 'ok'" + - in: ai + find: { metric_key: cc_tool_acceptance } + equal: { value: 62.5 } diff --git a/src/ingestion/tests/e2e/metrics/schemas/bronze_claude_enterprise.claude_enterprise_users.yaml b/src/ingestion/tests/e2e/metrics/schemas/bronze_claude_enterprise.claude_enterprise_users.yaml new file mode 100644 index 000000000..dcebf391f --- /dev/null +++ b/src/ingestion/tests/e2e/metrics/schemas/bronze_claude_enterprise.claude_enterprise_users.yaml @@ -0,0 +1,53 @@ +# JSON schema for bronze_claude_enterprise.claude_enterprise_users (all real columns). +# Mirrors the connector InlineSchemaLoader (connectors/ai/claude-enterprise/connector.yaml +# — stream claude_enterprise_users) plus the 4 _airbyte_* CDK columns. NOTE: `date` is +# a parseable date string; tool_use_accepted ← code_tool_accepted_count, tool_use_offered +# ← accepted + rejected. user_email (not email) is the identity column. +schemas: + bronze_claude_enterprise.claude_enterprise_users: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: false + properties: + _airbyte_raw_id: { type: string } + _airbyte_extracted_at: { type: string, format: date-time } + _airbyte_meta: { type: string } + _airbyte_generation_id: { type: integer } + unique_key: { type: string } + tenant_id: { type: [string, "null"] } + source_id: { type: [string, "null"] } + date: { type: string } + user_id: { type: [string, "null"] } + user_email: { type: [string, "null"] } + chat_conversation_count: { type: [integer, "null"] } + chat_message_count: { type: [integer, "null"] } + chat_projects_created_count: { type: [integer, "null"] } + chat_projects_used_count: { type: [integer, "null"] } + chat_files_uploaded_count: { type: [integer, "null"] } + chat_artifacts_created_count: { type: [integer, "null"] } + chat_thinking_message_count: { type: [integer, "null"] } + chat_skills_used_count: { type: [integer, "null"] } + chat_connectors_used_count: { type: [integer, "null"] } + code_commit_count: { type: [integer, "null"] } + code_pull_request_count: { type: [integer, "null"] } + code_lines_added: { type: [integer, "null"] } + code_lines_removed: { type: [integer, "null"] } + code_session_count: { type: [integer, "null"] } + code_tool_accepted_count: { type: [integer, "null"] } + code_tool_rejected_count: { type: [integer, "null"] } + web_search_count: { type: [integer, "null"] } + excel_session_count: { type: [integer, "null"] } + excel_message_count: { type: [integer, "null"] } + powerpoint_session_count: { type: [integer, "null"] } + powerpoint_message_count: { type: [integer, "null"] } + cowork_session_count: { type: [integer, "null"] } + cowork_message_count: { type: [integer, "null"] } + cowork_action_count: { type: [integer, "null"] } + cowork_dispatch_turn_count: { type: [integer, "null"] } + cowork_skills_used_count: { type: [integer, "null"] } + chat_metrics_json: { type: [string, "null"] } + claude_code_metrics_json: { type: [string, "null"] } + office_metrics_json: { type: [string, "null"] } + cowork_metrics_json: { type: [string, "null"] } + collected_at: { type: [string, "null"] } + data_source: { type: [string, "null"] } diff --git a/src/ingestion/tests/e2e/metrics/templates/claude_enterprise_users.yaml b/src/ingestion/tests/e2e/metrics/templates/claude_enterprise_users.yaml new file mode 100644 index 000000000..7b001b435 --- /dev/null +++ b/src/ingestion/tests/e2e/metrics/templates/claude_enterprise_users.yaml @@ -0,0 +1,69 @@ +# Reusable Claude Enterprise per-user/day usage records +# (bronze_claude_enterprise.claude_enterprise_users). Base carries every schema column +# (unused = null) incl. the 4 _airbyte_* CDK columns. claude_enterprise__ai_dev_usage +# (tool='claude_code') emits a row when code activity > 0; the fields under test +# (date, unique_key, code_tool_accepted_count, code_tool_rejected_count) are set per test. +# tool_use_accepted ← code_tool_accepted_count; tool_use_offered ← accepted + rejected. +# Variants override identity (user_email) only. +templates: + ent_user: + _airbyte_raw_id: "00000000-0000-0000-0000-000000000000" + _airbyte_extracted_at: "2026-01-05T00:00:00Z" + _airbyte_meta: "{}" + _airbyte_generation_id: 0 + unique_key: null + tenant_id: "00000000-0000-0000-0000-000000000000" + source_id: "claude-enterprise-test" + date: null + user_id: null + user_email: null + chat_conversation_count: null + chat_message_count: null + chat_projects_created_count: null + chat_projects_used_count: null + chat_files_uploaded_count: null + chat_artifacts_created_count: null + chat_thinking_message_count: null + chat_skills_used_count: null + chat_connectors_used_count: null + code_commit_count: null + code_pull_request_count: null + code_lines_added: null + code_lines_removed: null + code_session_count: null + code_tool_accepted_count: null + code_tool_rejected_count: null + web_search_count: null + excel_session_count: null + excel_message_count: null + powerpoint_session_count: null + powerpoint_message_count: null + cowork_session_count: null + cowork_message_count: null + cowork_action_count: null + cowork_dispatch_turn_count: null + cowork_skills_used_count: null + chat_metrics_json: null + claude_code_metrics_json: null + office_metrics_json: null + cowork_metrics_json: null + collected_at: "2026-01-05T00:00:00Z" + data_source: "insight_claude_enterprise" + + # user_id MUST be distinct per person: the staging model dedups with + # `LIMIT 1 BY tenant_id, source_id, user_id, date`, so a shared/NULL user_id + # collapses all three seats into one row (cohort of 1 → wrong team median). + alice: + $ref: "#/templates/ent_user" + user_email: alice@example.com + user_id: user-alice + + bob: + $ref: "#/templates/ent_user" + user_email: bob@example.com + user_id: user-bob + + carol: + $ref: "#/templates/ent_user" + user_email: carol@example.com + user_id: user-carol