diff --git a/src/ingestion/connectors/git/bitbucket-cloud/dbt/bitbucket_cloud__file_changes.sql b/src/ingestion/connectors/git/bitbucket-cloud/dbt/bitbucket_cloud__file_changes.sql index a62aafe731..085c611914 100644 --- a/src/ingestion/connectors/git/bitbucket-cloud/dbt/bitbucket_cloud__file_changes.sql +++ b/src/ingestion/connectors/git/bitbucket-cloud/dbt/bitbucket_cloud__file_changes.sql @@ -42,21 +42,7 @@ SELECT COALESCE(change.repo_slug, '') AS repo_slug, COALESCE(change.sha, '') AS commit_hash, COALESCE(change.filename, '') AS file_path, - -- File extension: last segment after the final '.', empty when none. - -- Earlier shape (issue #494) used `position('.', filename) > 0` as the - -- guard — but ClickHouse `position` is function-style - -- `position(haystack, needle)`, so this asked "is the string `filename` - -- present inside the single character '.'?" — always false. Result: - -- `file_extension` was empty for 100% of rows. Length check on the - -- split array is more robust than a fixed `position(filename, '.') > 0` - -- swap because it correctly returns '' for extensionless paths like - -- `Makefile` (where the position-based guard would also fire 0 by - -- accident, but the array-length guard is the explicit predicate). - if( - length(splitByChar('.', COALESCE(change.filename, ''))) > 1, - arrayElement(splitByChar('.', COALESCE(change.filename, '')), -1), - '' - ) AS file_extension, + {{ git_file_extension("COALESCE(change.filename, '')") }} AS file_extension, COALESCE(change.status, '') AS change_type, change.additions AS lines_added, change.deletions AS lines_removed, diff --git a/src/ingestion/connectors/git/github/README.md b/src/ingestion/connectors/git/github/README.md index 3499e9be2c..71549fea6e 100644 --- a/src/ingestion/connectors/git/github/README.md +++ b/src/ingestion/connectors/git/github/README.md @@ -6,9 +6,8 @@ the proxy instead of one vendor API call per commit; everything git cannot carry (PRs, reviews, comments, issues, projects, CI, deployments) stays on api.github.com. -Supersedes the retired `github-v2` CDK connector: same `data_source` -(`insight_github`), same bronze namespace (`bronze_github`), same silver -staging models. +Writes `data_source` `insight_github` into the `bronze_github` namespace, and +feeds the shared `class_git_*` silver models. ## Streams @@ -31,11 +30,54 @@ staging models. | deployment_statuses | `/deployments/{id}/statuses` | windowed deployments parent | | pull_request_timeline_events | GraphQL `pullRequest.timelineItems` | windowed PR parent, per PR | | issue_timeline_events | GraphQL `issue.timelineItems` | updated_at issues parent, per issue | +| commit_authors | proxy `/v1/authors` → `/repos/{r}/commits/{sha}` | last_committed_date authors parent, one lookup per author | **org_members is deliberately absent**: the deployed `git/github-directory` connector already syncs the org roster for identity resolution. Folding it in here is a separate migration. +## Commit attribution + +A commit carries a name and an e-mail; it never carries an account, because git +has no concept of one. Gold attributes activity to a person by e-mail, and the +roster can only claim an e-mail for a member who publishes one — so a member who +keeps their address private gets no commit attribution from the roster alone. + +Two streams close that gap. `commit_authors` asks GitHub who each distinct +commit author is — the proxy enumerates them from the clone, one vendor lookup +per author — and reaches authors who never open a pull request. +`pull_request_commits` carries the same match inline on every PR commit, +including for addresses that are not public. `github__account_emails` collects +the pairs from both, adds the ones a noreply address names by construction, and +`github__identity_inputs` publishes them as e-mail claims against the account. + +The claims feed the identity service's persons-seed, which decides what an +account is. No roster connector is required for attribution to work: an +account whose claimed e-mail matches a person the roster already knows — from +`github-directory`, BambooHR, Entra, or any other source — is linked to that +person and gets its binding minted right there, which is how a git-only +source joins an HR-rostered organization. + +Three things to know before deploying: + +- **When `github-directory` is deployed, both connectors must share one + `insight_source_id`.** The seed and the resolver key an account on (source + type, source id, login); under different ids the roster bindings and these + claims describe different accounts, and a member with no published e-mail + can end up as two persons. +- **An unmatched active account is minted as a new person.** An outside + contributor, or a member committing under an address no roster carries, + becomes a fresh person and shows up in the seed's `accounts_minted_new` + counter. Merging a minted person into the right one (or excluding it) is the + operator's manual-resolution workflow, and an operator-authored binding wins + every later seed. +- **Only `value_type='email'` rows are emitted, never the `value_type='id'` + binding.** Bindings are the seed's decision to make; emitting them here + would assert membership this connector cannot know. + +Commits GitHub cannot match at all (`author` comes back null — CI and service +identities) reach no account and stay unattributed. + ## Error policy - GitHub reports secondary rate limits as **403**, so throttle predicates diff --git a/src/ingestion/connectors/git/github/connector.yaml b/src/ingestion/connectors/git/github/connector.yaml index 87697e8c2c..d06c65edec 100644 --- a/src/ingestion/connectors/git/github/connector.yaml +++ b/src/ingestion/connectors/git/github/connector.yaml @@ -278,7 +278,9 @@ definitions: field_path: [] record_filter: type: RecordFilter - condition: "{{ not record.get('archived', false) }}" + # A fork's clone carries its whole upstream history, so admitting one + # attributes every upstream commit to this organization. + condition: "{{ not record.get('archived', false) and not record.get('fork', false) }}" paginator: type: DefaultPaginator pagination_strategy: @@ -334,7 +336,9 @@ definitions: field_path: [] record_filter: type: RecordFilter - condition: "{{ not record.get('archived', false) }}" + # A fork's clone carries its whole upstream history, so admitting one + # attributes every upstream commit to this organization. + condition: "{{ not record.get('archived', false) and not record.get('fork', false) }}" paginator: type: DefaultPaginator pagination_strategy: @@ -388,6 +392,20 @@ streams: private: {type: [boolean, "null"]} has_issues: {type: [boolean, "null"]} has_wiki: {type: [boolean, "null"]} + has_projects: {type: [boolean, "null"]} + has_discussions: {type: [boolean, "null"]} + has_pages: {type: [boolean, "null"]} + is_template: {type: [boolean, "null"]} + disabled: {type: [boolean, "null"]} + web_commit_signoff_required: {type: [boolean, "null"]} + visibility: {type: [string, "null"]} + license_spdx_id: {type: [string, "null"]} + topics: {type: [string, "null"]} + homepage: {type: [string, "null"]} + stargazers_count: {type: [integer, "null"]} + forks_count: {type: [integer, "null"]} + watchers_count: {type: [integer, "null"]} + open_issues_count: {type: [integer, "null"]} language: {type: [string, "null"]} size: {type: [integer, "null"]} clone_url: {type: [string, "null"]} @@ -414,7 +432,9 @@ streams: field_path: [] record_filter: type: RecordFilter - condition: "{{ not record.get('archived', false) }}" + # A fork's clone carries its whole upstream history, so admitting one + # attributes every upstream commit to this organization. + condition: "{{ not record.get('archived', false) and not record.get('fork', false) }}" paginator: type: DefaultPaginator pagination_strategy: @@ -440,10 +460,31 @@ streams: path: [description] value: "{{ (record.get('description') or '')[:1024] }}" value_type: string + # `private` is a boolean, so an organization's internal repositories + # are indistinguishable from private ones without this. + - type: AddedFieldDefinition + path: [visibility] + value: "{{ record.get('visibility') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [license_spdx_id] + value: "{{ (record.get('license') or {}).get('spdx_id') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [topics] + value: "{{ (record.get('topics') or []) | tojson }}" + value_type: string - type: AddedFieldDefinition path: [unique_key] value: >- {{ config['insight_tenant_id'] }}:{{ config['insight_source_id'] }}:{{ record['id'] }} + - type: RemoveFields + # Hoisted above; the nested originals stay out of bronze. + field_pointers: + - [license] + - [owner] + - [permissions] + - [custom_properties] # ── commits (git-cli-proxy) ──────────────────────────────────────── - type: DeclarativeStream @@ -729,6 +770,195 @@ streams: value: >- {{ config['insight_tenant_id'] }}:{{ config['insight_source_id'] }}:{{ stream_partition.repo_clone_url | replace(':', '%3A') }}:{{ record['name'] | replace(':', '%3A') }} + # ── Commit authors (git-cli-proxy + GitHub API) ───────────────────────── + # A commit names an e-mail and no account, so a commit reaches a person only + # once something claims that e-mail for one. The proxy enumerates the + # distinct authors of a repository from the clone it already holds, and this + # stream asks GitHub who each of them is — one call per AUTHOR, not per + # commit, which is the whole reason the proxy answers first. + # + # `/repos/{repo}/commits/{sha}` reports the account GitHub matched the + # commit's e-mail to, including addresses kept off the public profile. It + # reports `author: null` where it can match none — a CI or service identity — + # and those records are dropped rather than stored as an unresolved row. + - type: DeclarativeStream + name: commit_authors + primary_key: + - unique_key + schema_loader: + type: InlineSchemaLoader + schema: + type: object + $schema: http://json-schema.org/schema# + properties: + unique_key: {type: string} + tenant_id: {type: [string, "null"]} + source_id: {type: [string, "null"]} + data_source: {type: [string, "null"]} + collected_at: {type: [string, "null"]} + repo_full_name: {type: [string, "null"]} + author_email: {type: [string, "null"]} + author_login: {type: [string, "null"]} + author_id: {type: [integer, "null"]} + author_type: {type: [string, "null"]} + sample_sha: {type: [string, "null"]} + required: + - unique_key + additionalProperties: true + retriever: + type: SimpleRetriever + requester: + <<: *rest_requester + error_handler: *rest_error_handler_repo_scoped + path: "/repos/{{ stream_slice.extra_fields['repo_full_name'] }}/commits/{{ stream_partition.commit_sha }}" + record_selector: + type: RecordSelector + extractor: + # The response is one commit object, not a list. + type: DpathExtractor + field_path: [] + record_filter: + type: RecordFilter + # No account matched the e-mail; there is no claim to make. Compared + # against '' rather than tested for truth: the interpolation renders a + # missing login as the STRING "None", which is not falsy. + condition: "{{ ((record.get('author') or {}).get('login') or '') != '' }}" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sample_sha + partition_field: commit_sha + extra_fields: + - [repo_full_name] + - [author_email] + stream: + type: DeclarativeStream + name: repository_authors + primary_key: + - author_email + schema_loader: + type: InlineSchemaLoader + schema: + type: object + $schema: http://json-schema.org/schema# + properties: + author_email: {type: [string, "null"]} + author_name: {type: [string, "null"]} + sample_sha: {type: [string, "null"]} + last_committed_date: {type: [string, "null"]} + commit_count: {type: [integer, "null"]} + repo_full_name: {type: [string, "null"]} + additionalProperties: true + retriever: + type: SimpleRetriever + requester: + <<: *proxy_requester + path: /v1/authors + request_parameters: + repo: "{{ stream_partition.repo_clone_url }}" + page_size: "500" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [items] + paginator: + type: DefaultPaginator + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response['next_page_token'] }}" + stop_condition: "{{ not response.get('next_page_token') }}" + page_token_option: + type: RequestOption + field_name: page_token + inject_into: request_parameter + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: clone_url + partition_field: repo_clone_url + extra_fields: + - [full_name] + incremental_dependency: true + stream: *repositories_pushed_parent + incremental_sync: + type: DatetimeBasedCursor + cursor_field: last_committed_date + datetime_format: "%Y-%m-%dT%H:%M:%S%z" + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S%z" + - "%Y-%m-%dT%H:%M:%SZ" + # An author already resolved keeps their claim, so a later sync + # only needs the ones who have committed since — which is what + # keeps this O(active authors) rather than O(authors). + start_datetime: + type: MinMaxDatetime + datetime: "{{ config['github_start_date'] }}" + datetime_format: "%Y-%m-%d" + start_time_option: + type: RequestOption + field_name: since + inject_into: request_parameter + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: [repo_full_name] + value: "{{ stream_slice.extra_fields['full_name'] }}" + value_type: string + transformations: + - type: AddFields + fields: *standard_fields + - type: AddFields + fields: + - type: AddedFieldDefinition + path: [repo_full_name] + value: "{{ stream_slice.extra_fields['repo_full_name'] }}" + value_type: string + # The e-mail the account was resolved FOR: the git ident the proxy + # enumerated, which is what the commit rows carry and therefore what + # a claim has to be keyed on. + - type: AddedFieldDefinition + path: [author_email] + value: "{{ stream_slice.extra_fields['author_email'] }}" + value_type: string + - type: AddedFieldDefinition + path: [sample_sha] + value: "{{ stream_partition.commit_sha }}" + value_type: string + - type: AddedFieldDefinition + path: [author_login] + value: "{{ (record.get('author') or {}).get('login') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [author_id] + value: "{{ (record.get('author') or {}).get('id') or 0 }}" + value_type: integer + - type: AddedFieldDefinition + path: [author_type] + value: "{{ (record.get('author') or {}).get('type') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [unique_key] + value: >- + {{ config['insight_tenant_id'] }}:{{ config['insight_source_id'] }}:{{ stream_slice.extra_fields['repo_full_name'] | replace(':', '%3A') }}:author:{{ stream_slice.extra_fields['author_email'] | replace(':', '%3A') }} + - type: RemoveFields + # Hoisted above; the commit payload itself belongs to the proxy streams. + field_pointers: + - [commit] + - [author] + - [committer] + - [parents] + - [files] + - [stats] + - [node_id] + - [sha] + - [url] + - [html_url] + - [comments_url] + # ── Pull requests (GitHub API) ────────────────────────────────────────── # LIST endpoint only: /pulls has no `since`, so newest-first plus the # data-feed stop halts pagination at the stored per-repo state, and the @@ -757,16 +987,30 @@ streams: draft: {type: [boolean, "null"]} title: {type: [string, "null"]} author_login: {type: [string, "null"]} + author_id: {type: [integer, "null"]} + author_type: {type: [string, "null"]} author_association: {type: [string, "null"]} head_sha: {type: [string, "null"]} head_ref: {type: [string, "null"]} + head_label: {type: [string, "null"]} + head_repo_full_name: {type: [string, "null"]} base_ref: {type: [string, "null"]} + base_sha: {type: [string, "null"]} merge_commit_sha: {type: [string, "null"]} created_at: {type: [string, "null"]} updated_at: {type: [string, "null"]} closed_at: {type: [string, "null"]} merged_at: {type: [string, "null"]} body: {type: [string, "null"]} + locked: {type: [boolean, "null"]} + active_lock_reason: {type: [string, "null"]} + auto_merge_enabled: {type: [boolean, "null"]} + label_names: {type: [string, "null"]} + assignee_logins: {type: [string, "null"]} + requested_reviewer_logins: {type: [string, "null"]} + requested_team_slugs: {type: [string, "null"]} + milestone_title: {type: [string, "null"]} + milestone_number: {type: [integer, "null"]} required: - unique_key additionalProperties: true @@ -832,6 +1076,14 @@ streams: path: [author_login] value: "{{ (record.get('user') or {}).get('login') or '' }}" value_type: string + - type: AddedFieldDefinition + path: [author_id] + value: "{{ (record.get('user') or {}).get('id') or 0 }}" + value_type: integer + - type: AddedFieldDefinition + path: [author_type] + value: "{{ (record.get('user') or {}).get('type') or '' }}" + value_type: string - type: AddedFieldDefinition path: [head_sha] value: "{{ (record.get('head') or {}).get('sha') or '' }}" @@ -840,10 +1092,25 @@ streams: path: [head_ref] value: "{{ (record.get('head') or {}).get('ref') or '' }}" value_type: string + - type: AddedFieldDefinition + path: [head_label] + value: "{{ (record.get('head') or {}).get('label') or '' }}" + value_type: string + # Differs from repo_full_name exactly when the branch lives in a fork, + # which is the only signal in this payload for an external + # contribution. Empty where the fork has since been deleted. + - type: AddedFieldDefinition + path: [head_repo_full_name] + value: "{{ ((record.get('head') or {}).get('repo') or {}).get('full_name') or '' }}" + value_type: string - type: AddedFieldDefinition path: [base_ref] value: "{{ (record.get('base') or {}).get('ref') or '' }}" value_type: string + - type: AddedFieldDefinition + path: [base_sha] + value: "{{ (record.get('base') or {}).get('sha') or '' }}" + value_type: string - type: AddedFieldDefinition path: [title] value: "{{ (record.get('title') or '')[:1024] }}" @@ -852,6 +1119,38 @@ streams: path: [body] value: "{{ (record.get('body') or '')[:2048] }}" value_type: string + # `auto_merge` is an object describing who armed it, or null; only + # whether it was armed survives here. + - type: AddedFieldDefinition + path: [auto_merge_enabled] + value: "{{ record.get('auto_merge') is not none }}" + value_type: boolean + - type: AddedFieldDefinition + path: [label_names] + value: "{{ ((record.get('labels') or []) | map(attribute='name') | list) | tojson }}" + value_type: string + - type: AddedFieldDefinition + path: [assignee_logins] + value: "{{ ((record.get('assignees') or []) | map(attribute='login') | list) | tojson }}" + value_type: string + # Reviewers still owing a review: the request events say a review was + # asked for, this says the ask is still outstanding. + - type: AddedFieldDefinition + path: [requested_reviewer_logins] + value: "{{ ((record.get('requested_reviewers') or []) | map(attribute='login') | list) | tojson }}" + value_type: string + - type: AddedFieldDefinition + path: [requested_team_slugs] + value: "{{ ((record.get('requested_teams') or []) | map(attribute='slug') | list) | tojson }}" + value_type: string + - type: AddedFieldDefinition + path: [milestone_title] + value: "{{ ((record.get('milestone') or {}).get('title') or '')[:1024] }}" + value_type: string + - type: AddedFieldDefinition + path: [milestone_number] + value: "{{ (record.get('milestone') or {}).get('number') or 0 }}" + value_type: integer - type: AddedFieldDefinition path: [unique_key] value: >- @@ -862,6 +1161,13 @@ streams: - [user] - [head] - [base] + - [labels] + - [assignees] + - [requested_reviewers] + - [requested_teams] + - [milestone] + - [auto_merge] + - [_links] # ── Pull request reviews (GitHub API) ─────────────────────────────────── # No repo-level reviews endpoint exists in REST, so this is one call per PR @@ -890,6 +1196,8 @@ streams: commit_id: {type: [string, "null"]} submitted_at: {type: [string, "null"]} author_id: {type: [integer, "null"]} + author_type: {type: [string, "null"]} + body: {type: [string, "null"]} required: - unique_key additionalProperties: true @@ -1014,6 +1322,16 @@ streams: - type: AddedFieldDefinition path: [author_id] value: "{{ (record.get('user') or {}).get('id') }}" + - type: AddedFieldDefinition + path: [author_type] + value: "{{ (record.get('user') or {}).get('type') or '' }}" + value_type: string + # The reviewer's verdict text. Review COMMENTS are a separate stream; + # this is the summary that accompanies the approval or rejection. + - type: AddedFieldDefinition + path: [body] + value: "{{ (record.get('body') or '')[:2048] }}" + value_type: string - type: AddedFieldDefinition path: [unique_key] value: >- @@ -1022,13 +1340,15 @@ streams: # Hoisted above; the nested originals stay out of bronze. field_pointers: - [user] + - [_links] # ── Pull request commits (GitHub API) ─────────────────────────────────── # PR<->commit membership is vendor metadata: it does not exist in git, and a # squash- or rebase-merged PR leaves its original commits unreachable from - # every ref the proxy clones. One light call per PR in the window — the - # commit payloads themselves stay on the proxy path, so this stream stores - # only the edge. + # every ref the proxy clones. One light call per PR in the window. + # A PR's head commits live under refs/pull/*, which a clone does not fetch, + # so fork and force-pushed commits reach no other stream: this one carries + # their commit metadata as well as the membership edge. # VERIFIED CAVEAT: this endpoint returns at most 250 commits per PR. - type: DeclarativeStream name: pull_request_commits @@ -1048,6 +1368,22 @@ streams: sha: {type: [string, "null"]} pull_number: {type: [integer, "null"]} repo_full_name: {type: [string, "null"]} + author_login: {type: [string, "null"]} + author_id: {type: [integer, "null"]} + author_type: {type: [string, "null"]} + author_name: {type: [string, "null"]} + author_email: {type: [string, "null"]} + authored_date: {type: [string, "null"]} + committer_login: {type: [string, "null"]} + committer_id: {type: [integer, "null"]} + committer_name: {type: [string, "null"]} + committer_email: {type: [string, "null"]} + committed_date: {type: [string, "null"]} + message: {type: [string, "null"]} + parent_shas: {type: [string, "null"]} + is_merge: {type: [boolean, "null"]} + is_verified: {type: [boolean, "null"]} + verification_reason: {type: [string, "null"]} required: - unique_key additionalProperties: true @@ -1093,12 +1429,85 @@ streams: path: [repo_full_name] value: "{{ stream_slice.extra_fields['repo_full_name'] }}" value_type: string + # The GitHub account behind a commit. Git carries only a name and an + # e-mail, so this response is the one place a commit's e-mail and its + # GitHub login appear together — the edge identity resolution needs. + # `author` is null wherever GitHub cannot match the e-mail to an + # account, which is how CI and service identities present. + - type: AddedFieldDefinition + path: [author_login] + value: "{{ (record.get('author') or {}).get('login') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [author_id] + value: "{{ (record.get('author') or {}).get('id') or 0 }}" + value_type: integer + - type: AddedFieldDefinition + path: [author_type] + value: "{{ (record.get('author') or {}).get('type') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [committer_login] + value: "{{ (record.get('committer') or {}).get('login') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [committer_id] + value: "{{ (record.get('committer') or {}).get('id') or 0 }}" + value_type: integer + # A PR's head commits live under refs/pull/*, which a clone does not + # fetch, so the proxy commit walk cannot see fork or force-pushed + # commits at all. For those this stream is the only carrier of the + # commit's own metadata, not a duplicate of it. + - type: AddedFieldDefinition + path: [author_name] + value: "{{ ((record.get('commit') or {}).get('author') or {}).get('name') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [author_email] + value: "{{ ((record.get('commit') or {}).get('author') or {}).get('email') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [authored_date] + value: "{{ ((record.get('commit') or {}).get('author') or {}).get('date') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [committer_name] + value: "{{ ((record.get('commit') or {}).get('committer') or {}).get('name') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [committer_email] + value: "{{ ((record.get('commit') or {}).get('committer') or {}).get('email') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [committed_date] + value: "{{ ((record.get('commit') or {}).get('committer') or {}).get('date') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [message] + value: "{{ ((record.get('commit') or {}).get('message') or '')[:2048] }}" + value_type: string + - type: AddedFieldDefinition + path: [parent_shas] + value: "{{ ((record.get('parents') or []) | map(attribute='sha') | list) | tojson }}" + value_type: string + - type: AddedFieldDefinition + path: [is_merge] + value: "{{ ((record.get('parents') or []) | length) > 1 }}" + value_type: boolean + - type: AddedFieldDefinition + path: [is_verified] + value: "{{ ((record.get('commit') or {}).get('verification') or {}).get('verified') or false }}" + value_type: boolean + - type: AddedFieldDefinition + path: [verification_reason] + value: "{{ ((record.get('commit') or {}).get('verification') or {}).get('reason') or '' }}" + value_type: string - type: AddedFieldDefinition path: [unique_key] value: >- {{ config['insight_tenant_id'] }}:{{ config['insight_source_id'] }}:{{ stream_slice.extra_fields['repo_full_name'] | replace(':', '%3A') }}:{{ stream_partition.pull_number }}:{{ record['sha'] }} - type: RemoveFields - # The commit payload comes from the proxy; only the edge is stored. + # Hoisted above; the nested originals stay out of bronze. field_pointers: - [commit] - [author] @@ -1139,6 +1548,13 @@ streams: changed_files: {type: [integer, "null"]} updated_at: {type: [string, "null"]} author_email: {type: [string, "null"]} + author_login: {type: [string, "null"]} + author_id: {type: [integer, "null"]} + merged_by_login: {type: [string, "null"]} + merged_by_id: {type: [integer, "null"]} + review_decision: {type: [string, "null"]} + total_comments_count: {type: [integer, "null"]} + is_cross_repository: {type: [boolean, "null"]} required: - unique_key additionalProperties: true @@ -1155,7 +1571,7 @@ streams: repository(owner: $owner, name: $name) { pullRequests(first: 100, after: $cursor, states: [OPEN, CLOSED, MERGED], orderBy: {field: UPDATED_AT, direction: DESC}) { pageInfo { hasNextPage endCursor } - nodes { number updatedAt additions deletions changedFiles author { ... on User { email } } } + nodes { number updatedAt additions deletions changedFiles reviewDecision totalCommentsCount isCrossRepository author { login ... on User { databaseId email } } mergedBy { login ... on User { databaseId } } } } } } @@ -1227,6 +1643,39 @@ streams: path: [author_email] value: "{{ (record.get('author') or {}).get('email') or '' }}" value_type: string + # The account the e-mail above belongs to, so a published profile + # e-mail is a claim against a known account rather than a loose value. + - type: AddedFieldDefinition + path: [author_login] + value: "{{ (record.get('author') or {}).get('login') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [author_id] + value: "{{ (record.get('author') or {}).get('databaseId') or 0 }}" + value_type: integer + # Who merged it — REST reports merged_at but never the merger, and a + # MergedEvent is only collected for pull requests inside the timeline + # window. + - type: AddedFieldDefinition + path: [merged_by_login] + value: "{{ (record.get('mergedBy') or {}).get('login') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [merged_by_id] + value: "{{ (record.get('mergedBy') or {}).get('databaseId') or 0 }}" + value_type: integer + - type: AddedFieldDefinition + path: [review_decision] + value: "{{ record.get('reviewDecision') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [total_comments_count] + value: "{{ record.get('totalCommentsCount') or 0 }}" + value_type: integer + - type: AddedFieldDefinition + path: [is_cross_repository] + value: "{{ record.get('isCrossRepository') or false }}" + value_type: boolean - type: AddedFieldDefinition path: [unique_key] value: >- @@ -1238,6 +1687,10 @@ streams: - [changedFiles] - [updatedAt] - [author] + - [mergedBy] + - [reviewDecision] + - [totalCommentsCount] + - [isCrossRepository] # ── Issue & PR conversation comments (GitHub API) ─────────────────────── # ONE repo-level endpoint covers both: PR conversation comments are issue @@ -1267,6 +1720,9 @@ streams: updated_at: {type: [string, "null"]} body: {type: [string, "null"]} author_id: {type: [integer, "null"]} + author_type: {type: [string, "null"]} + is_via_github_app: {type: [boolean, "null"]} + reactions_total: {type: [integer, "null"]} required: - unique_key additionalProperties: true @@ -1339,6 +1795,22 @@ streams: - type: AddedFieldDefinition path: [author_id] value: "{{ (record.get('user') or {}).get('id') }}" + # `Bot` marks an automated commenter; login naming conventions are the + # only other signal and they are not reliable. + - type: AddedFieldDefinition + path: [author_type] + value: "{{ (record.get('user') or {}).get('type') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [is_via_github_app] + value: "{{ record.get('performed_via_github_app') is not none }}" + value_type: boolean + # Engagement on the comment. The per-emoji breakdown is dropped; only + # the total carries a signal worth serving. + - type: AddedFieldDefinition + path: [reactions_total] + value: "{{ (record.get('reactions') or {}).get('total_count') or 0 }}" + value_type: integer - type: AddedFieldDefinition path: [unique_key] value: >- @@ -1348,6 +1820,8 @@ streams: field_pointers: - [user] - [issue_url] + - [reactions] + - [performed_via_github_app] # ── Inline code-review comments (GitHub API) ──────────────────────────── # Repo-level, so one partition per repo rather than one call per PR. These @@ -1377,8 +1851,19 @@ streams: updated_at: {type: [string, "null"]} body: {type: [string, "null"]} author_id: {type: [integer, "null"]} + author_type: {type: [string, "null"]} path: {type: [string, "null"]} line: {type: [integer, "null"]} + start_line: {type: [integer, "null"]} + side: {type: [string, "null"]} + subject_type: {type: [string, "null"]} + commit_id: {type: [string, "null"]} + original_commit_id: {type: [string, "null"]} + review_id: {type: [integer, "null"]} + in_reply_to_id: {type: [integer, "null"]} + diff_hunk: {type: [string, "null"]} + is_via_github_app: {type: [boolean, "null"]} + reactions_total: {type: [integer, "null"]} required: - unique_key additionalProperties: true @@ -1451,11 +1936,44 @@ streams: - type: AddedFieldDefinition path: [author_id] value: "{{ (record.get('user') or {}).get('id') }}" + - type: AddedFieldDefinition + path: [author_type] + value: "{{ (record.get('user') or {}).get('type') or '' }}" + value_type: string # An outdated comment loses `line`; `original_line` keeps the # position it was written against. - type: AddedFieldDefinition path: [line] value: "{{ record.get('line') or record.get('original_line') }}" + - type: AddedFieldDefinition + path: [start_line] + value: "{{ record.get('start_line') or record.get('original_start_line') or 0 }}" + value_type: integer + # Which review this comment was submitted under, and which comment it + # answers: without them every inline comment reads as a standalone + # remark and a review thread cannot be reconstructed. + - type: AddedFieldDefinition + path: [review_id] + value: "{{ record.get('pull_request_review_id') or 0 }}" + value_type: integer + - type: AddedFieldDefinition + path: [in_reply_to_id] + value: "{{ record.get('in_reply_to_id') or 0 }}" + value_type: integer + # The code the remark was made about, so a comment can be read without + # refetching the diff. + - type: AddedFieldDefinition + path: [diff_hunk] + value: "{{ (record.get('diff_hunk') or '')[:2048] }}" + value_type: string + - type: AddedFieldDefinition + path: [is_via_github_app] + value: "{{ record.get('performed_via_github_app') is not none }}" + value_type: boolean + - type: AddedFieldDefinition + path: [reactions_total] + value: "{{ (record.get('reactions') or {}).get('total_count') or 0 }}" + value_type: integer - type: AddedFieldDefinition path: [unique_key] value: >- @@ -1466,6 +1984,11 @@ streams: - [user] - [pull_request_url] - [original_line] + - [original_start_line] + - [pull_request_review_id] + - [reactions] + - [performed_via_github_app] + - [_links] # ── Issues (GitHub API) ───────────────────────────────────────────────── # The endpoint returns PRs too; the record filter drops them (a PR carries @@ -1491,9 +2014,21 @@ streams: state: {type: [string, "null"]} state_reason: {type: [string, "null"]} title: {type: [string, "null"]} + body: {type: [string, "null"]} author_login: {type: [string, "null"]} + author_id: {type: [integer, "null"]} + author_type: {type: [string, "null"]} + author_association: {type: [string, "null"]} assignee_logins: {type: [string, "null"]} + assignee_ids: {type: [string, "null"]} label_names: {type: [string, "null"]} + issue_type: {type: [string, "null"]} + milestone_title: {type: [string, "null"]} + milestone_number: {type: [integer, "null"]} + closed_by_login: {type: [string, "null"]} + closed_by_id: {type: [integer, "null"]} + locked: {type: [boolean, "null"]} + reactions_total: {type: [integer, "null"]} comments: {type: [integer, "null"]} created_at: {type: [string, "null"]} updated_at: {type: [string, "null"]} @@ -1563,18 +2098,61 @@ streams: path: [author_login] value: "{{ (record.get('user') or {}).get('login') or '' }}" value_type: string + - type: AddedFieldDefinition + path: [author_id] + value: "{{ (record.get('user') or {}).get('id') or 0 }}" + value_type: integer + - type: AddedFieldDefinition + path: [author_type] + value: "{{ (record.get('user') or {}).get('type') or '' }}" + value_type: string - type: AddedFieldDefinition path: [assignee_logins] value: "{{ ((record.get('assignees') or []) | map(attribute='login') | list) | tojson }}" value_type: string + - type: AddedFieldDefinition + path: [assignee_ids] + value: "{{ ((record.get('assignees') or []) | map(attribute='id') | list) | tojson }}" + value_type: string - type: AddedFieldDefinition path: [label_names] value: "{{ ((record.get('labels') or []) | map(attribute='name') | list) | tojson }}" value_type: string + # GitHub's native issue type (Bug, Task, Feature), distinct from a + # label. Null on repositories where the org has not enabled types. + - type: AddedFieldDefinition + path: [issue_type] + value: "{{ (record.get('type') or {}).get('name') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [milestone_title] + value: "{{ ((record.get('milestone') or {}).get('title') or '')[:1024] }}" + value_type: string + - type: AddedFieldDefinition + path: [milestone_number] + value: "{{ (record.get('milestone') or {}).get('number') or 0 }}" + value_type: integer + # Who closed it, which the state and state_reason do not say. + - type: AddedFieldDefinition + path: [closed_by_login] + value: "{{ (record.get('closed_by') or {}).get('login') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [closed_by_id] + value: "{{ (record.get('closed_by') or {}).get('id') or 0 }}" + value_type: integer + - type: AddedFieldDefinition + path: [reactions_total] + value: "{{ (record.get('reactions') or {}).get('total_count') or 0 }}" + value_type: integer - type: AddedFieldDefinition path: [title] value: "{{ (record.get('title') or '')[:1024] }}" value_type: string + - type: AddedFieldDefinition + path: [body] + value: "{{ (record.get('body') or '')[:2048] }}" + value_type: string - type: AddedFieldDefinition path: [unique_key] value: >- @@ -1585,6 +2163,13 @@ streams: - [user] - [assignees] - [labels] + - [type] + - [milestone] + - [closed_by] + - [reactions] + - [performed_via_github_app] + - [sub_issues_summary] + - [issue_field_values] # ── Projects V2 (GitHub GraphQL) ──────────────────────────────────────── # Projects V2 has no REST API. Cursor rides in the request body — the @@ -1730,6 +2315,18 @@ streams: head_branch: {type: [string, "null"]} head_sha: {type: [string, "null"]} actor_login: {type: [string, "null"]} + actor_id: {type: [integer, "null"]} + triggering_actor_login: {type: [string, "null"]} + triggering_actor_id: {type: [integer, "null"]} + run_number: {type: [integer, "null"]} + workflow_path: {type: [string, "null"]} + display_title: {type: [string, "null"]} + check_suite_id: {type: [integer, "null"]} + pull_request_numbers: {type: [string, "null"]} + head_commit_message: {type: [string, "null"]} + head_commit_timestamp: {type: [string, "null"]} + head_commit_author_name: {type: [string, "null"]} + head_commit_author_email: {type: [string, "null"]} run_started_at: {type: [string, "null"]} created_at: {type: [string, "null"]} updated_at: {type: [string, "null"]} @@ -1796,6 +2393,52 @@ streams: path: [actor_login] value: "{{ (record.get('actor') or {}).get('login') or '' }}" value_type: string + - type: AddedFieldDefinition + path: [actor_id] + value: "{{ (record.get('actor') or {}).get('id') or 0 }}" + value_type: integer + # Diverges from `actor` on a re-run: the actor stays whoever caused + # the original run, so without this a re-run is credited to them. + - type: AddedFieldDefinition + path: [triggering_actor_login] + value: "{{ (record.get('triggering_actor') or {}).get('login') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [triggering_actor_id] + value: "{{ (record.get('triggering_actor') or {}).get('id') or 0 }}" + value_type: integer + # `name` is the workflow's display name, which is not unique and can be + # edited; the path identifies the workflow file itself. + - type: AddedFieldDefinition + path: [workflow_path] + value: "{{ record.get('path') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [display_title] + value: "{{ (record.get('display_title') or '')[:1024] }}" + value_type: string + # The pull requests this run was for, so CI outcome joins to review + # without matching on head_sha. + - type: AddedFieldDefinition + path: [pull_request_numbers] + value: "{{ ((record.get('pull_requests') or []) | map(attribute='number') | list) | tojson }}" + value_type: string + - type: AddedFieldDefinition + path: [head_commit_message] + value: "{{ ((record.get('head_commit') or {}).get('message') or '')[:2048] }}" + value_type: string + - type: AddedFieldDefinition + path: [head_commit_timestamp] + value: "{{ (record.get('head_commit') or {}).get('timestamp') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [head_commit_author_name] + value: "{{ ((record.get('head_commit') or {}).get('author') or {}).get('name') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [head_commit_author_email] + value: "{{ ((record.get('head_commit') or {}).get('author') or {}).get('email') or '' }}" + value_type: string - type: AddedFieldDefinition path: [unique_key] value: >- @@ -1804,6 +2447,12 @@ streams: # Hoisted above; the nested originals stay out of bronze. field_pointers: - [actor] + - [triggering_actor] + - [head_commit] + - [pull_requests] + - [repository] + - [head_repository] + - [referenced_workflows] # ── Deployments (GitHub API) ──────────────────────────────────────────── # No date filter and no outcome on this endpoint: newest-first + data-feed @@ -1829,8 +2478,12 @@ streams: ref: {type: [string, "null"]} task: {type: [string, "null"]} environment: {type: [string, "null"]} + original_environment: {type: [string, "null"]} + is_transient_environment: {type: [boolean, "null"]} + is_production_environment: {type: [boolean, "null"]} description: {type: [string, "null"]} creator_login: {type: [string, "null"]} + creator_id: {type: [integer, "null"]} created_at: {type: [string, "null"]} updated_at: {type: [string, "null"]} required: @@ -1896,19 +2549,42 @@ streams: path: [creator_login] value: "{{ (record.get('creator') or {}).get('login') or '' }}" value_type: string + - type: AddedFieldDefinition + path: [creator_id] + value: "{{ (record.get('creator') or {}).get('id') or 0 }}" + value_type: integer - type: AddedFieldDefinition path: [description] value: "{{ (record.get('description') or '')[:1024] }}" value_type: string + # `environment` is the current name; a promotion rewrites it, and + # `original_environment` is where the deploy actually started. The two + # flags say what KIND of environment it is — the name alone cannot, + # since naming is per-repository convention. + - type: AddedFieldDefinition + path: [original_environment] + value: "{{ record.get('original_environment') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [is_transient_environment] + value: "{{ record.get('transient_environment') or false }}" + value_type: boolean + - type: AddedFieldDefinition + path: [is_production_environment] + value: "{{ record.get('production_environment') or false }}" + value_type: boolean - type: AddedFieldDefinition path: [unique_key] value: >- {{ config['insight_tenant_id'] }}:{{ config['insight_source_id'] }}:{{ stream_partition.repo_full_name | replace(':', '%3A') }}:deploy:{{ record['id'] }} - type: RemoveFields - # Hoisted above; the nested originals stay out of bronze. + # Hoisted above; the nested and renamed originals stay out of bronze. field_pointers: - [creator] - [payload] + - [transient_environment] + - [production_environment] + - [performed_via_github_app] # ── Deployment statuses (GitHub API) ──────────────────────────────────── # A deployment record has no outcome; this child fetches success/failure @@ -1934,8 +2610,14 @@ streams: repo_full_name: {type: [string, "null"]} state: {type: [string, "null"]} environment: {type: [string, "null"]} + description: {type: [string, "null"]} creator_login: {type: [string, "null"]} + creator_id: {type: [integer, "null"]} + target_url: {type: [string, "null"]} + log_url: {type: [string, "null"]} + environment_url: {type: [string, "null"]} created_at: {type: [string, "null"]} + updated_at: {type: [string, "null"]} required: - unique_key additionalProperties: true @@ -2053,6 +2735,28 @@ streams: path: [creator_login] value: "{{ (record.get('creator') or {}).get('login') or '' }}" value_type: string + - type: AddedFieldDefinition + path: [creator_id] + value: "{{ (record.get('creator') or {}).get('id') or 0 }}" + value_type: integer + - type: AddedFieldDefinition + path: [description] + value: "{{ (record.get('description') or '')[:1024] }}" + value_type: string + # Where the run and the deployed environment can be reached, which is + # the evidence a failed status points at. + - type: AddedFieldDefinition + path: [target_url] + value: "{{ record.get('target_url') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [log_url] + value: "{{ record.get('log_url') or '' }}" + value_type: string + - type: AddedFieldDefinition + path: [environment_url] + value: "{{ record.get('environment_url') or '' }}" + value_type: string - type: AddedFieldDefinition path: [unique_key] value: >- @@ -2061,6 +2765,7 @@ streams: # Hoisted above; the nested originals stay out of bronze. field_pointers: - [creator] + - [performed_via_github_app] # ── Pull request timeline events (GitHub GraphQL) ─────────────────────── # One call per pull request in the sliding window rather than one per page: @@ -2088,9 +2793,12 @@ streams: item_number: {type: [integer, "null"]} repo_full_name: {type: [string, "null"]} actor_login: {type: [string, "null"]} + actor_id: {type: [integer, "null"]} target_login: {type: [string, "null"]} + target_id: {type: [integer, "null"]} label_name: {type: [string, "null"]} state_reason: {type: [string, "null"]} + merge_commit_sha: {type: [string, "null"]} required: - unique_key additionalProperties: true @@ -2110,17 +2818,17 @@ streams: pageInfo { hasNextPage endCursor } nodes { __typename - ... on ReadyForReviewEvent { id createdAt actor { login } } - ... on ConvertToDraftEvent { id createdAt actor { login } } - ... on ReviewRequestedEvent { id createdAt actor { login } requestedReviewer { ... on User { login } ... on Team { slug } ... on Bot { login } ... on Mannequin { login } } } - ... on ReviewRequestRemovedEvent { id createdAt actor { login } requestedReviewer { ... on User { login } ... on Team { slug } ... on Bot { login } ... on Mannequin { login } } } - ... on AssignedEvent { id createdAt actor { login } assignee { ... on User { login } ... on Bot { login } ... on Mannequin { login } ... on Organization { login } } } - ... on UnassignedEvent { id createdAt actor { login } assignee { ... on User { login } ... on Bot { login } ... on Mannequin { login } ... on Organization { login } } } - ... on LabeledEvent { id createdAt actor { login } label { name } } - ... on UnlabeledEvent { id createdAt actor { login } label { name } } - ... on ClosedEvent { id createdAt actor { login } stateReason } - ... on ReopenedEvent { id createdAt actor { login } } - ... on MergedEvent { id createdAt actor { login } } + ... on ReadyForReviewEvent { id createdAt actor { login ... on User { databaseId } } } + ... on ConvertToDraftEvent { id createdAt actor { login ... on User { databaseId } } } + ... on ReviewRequestedEvent { id createdAt actor { login ... on User { databaseId } } requestedReviewer { ... on User { login databaseId } ... on Team { slug } ... on Bot { login } ... on Mannequin { login } } } + ... on ReviewRequestRemovedEvent { id createdAt actor { login ... on User { databaseId } } requestedReviewer { ... on User { login databaseId } ... on Team { slug } ... on Bot { login } ... on Mannequin { login } } } + ... on AssignedEvent { id createdAt actor { login ... on User { databaseId } } assignee { ... on User { login databaseId } ... on Bot { login } ... on Mannequin { login } ... on Organization { login } } } + ... on UnassignedEvent { id createdAt actor { login ... on User { databaseId } } assignee { ... on User { login databaseId } ... on Bot { login } ... on Mannequin { login } ... on Organization { login } } } + ... on LabeledEvent { id createdAt actor { login ... on User { databaseId } } label { name } } + ... on UnlabeledEvent { id createdAt actor { login ... on User { databaseId } } label { name } } + ... on ClosedEvent { id createdAt actor { login ... on User { databaseId } } stateReason } + ... on ReopenedEvent { id createdAt actor { login ... on User { databaseId } } } + ... on MergedEvent { id createdAt actor { login ... on User { databaseId } } commit { oid } } } } } @@ -2182,6 +2890,12 @@ streams: path: [actor_login] value: "{{ (record.get('actor') or {}).get('login') or '' }}" value_type: string + # Teams, bots and mannequins have a login but no databaseId, so this + # is 0 for them while actor_login still names them. + - type: AddedFieldDefinition + path: [actor_id] + value: "{{ (record.get('actor') or {}).get('databaseId') or 0 }}" + value_type: integer # Assignment and review-request events name a second person; the # field differs by event type and only one is ever present. - type: AddedFieldDefinition @@ -2189,6 +2903,11 @@ streams: value: >- {{ (record.get('assignee') or {}).get('login') or (record.get('requestedReviewer') or {}).get('login') or (record.get('requestedReviewer') or {}).get('slug') or '' }} value_type: string + - type: AddedFieldDefinition + path: [target_id] + value: >- + {{ (record.get('assignee') or {}).get('databaseId') or (record.get('requestedReviewer') or {}).get('databaseId') or 0 }} + value_type: integer - type: AddedFieldDefinition path: [label_name] value: "{{ (record.get('label') or {}).get('name') or '' }}" @@ -2197,6 +2916,12 @@ streams: path: [state_reason] value: "{{ record.get('stateReason') or '' }}" value_type: string + # Present on MergedEvent alone: the commit the merge produced, which + # ties the lifecycle event to the resulting history. + - type: AddedFieldDefinition + path: [merge_commit_sha] + value: "{{ (record.get('commit') or {}).get('oid') or '' }}" + value_type: string - type: AddedFieldDefinition path: [unique_key] value: >- @@ -2212,6 +2937,7 @@ streams: - [requestedReviewer] - [label] - [stateReason] + - [commit] # ── Issue timeline events (GitHub GraphQL) ────────────────────────────── # Same per-item shape as the pull-request stream, but a different item-type @@ -2239,7 +2965,9 @@ streams: item_number: {type: [integer, "null"]} repo_full_name: {type: [string, "null"]} actor_login: {type: [string, "null"]} + actor_id: {type: [integer, "null"]} target_login: {type: [string, "null"]} + target_id: {type: [integer, "null"]} label_name: {type: [string, "null"]} state_reason: {type: [string, "null"]} field_name: {type: [string, "null"]} @@ -2264,15 +2992,15 @@ streams: pageInfo { hasNextPage endCursor } nodes { __typename - ... on AssignedEvent { id createdAt actor { login } assignee { ... on User { login } ... on Bot { login } ... on Mannequin { login } ... on Organization { login } } } - ... on UnassignedEvent { id createdAt actor { login } assignee { ... on User { login } ... on Bot { login } ... on Mannequin { login } ... on Organization { login } } } - ... on LabeledEvent { id createdAt actor { login } label { name } } - ... on UnlabeledEvent { id createdAt actor { login } label { name } } - ... on ClosedEvent { id createdAt actor { login } stateReason } - ... on ReopenedEvent { id createdAt actor { login } } - ... on IssueTypeChangedEvent { id createdAt actor { login } issueType { name } prevIssueType { name } } + ... on AssignedEvent { id createdAt actor { login ... on User { databaseId } } assignee { ... on User { login databaseId } ... on Bot { login } ... on Mannequin { login } ... on Organization { login } } } + ... on UnassignedEvent { id createdAt actor { login ... on User { databaseId } } assignee { ... on User { login databaseId } ... on Bot { login } ... on Mannequin { login } ... on Organization { login } } } + ... on LabeledEvent { id createdAt actor { login ... on User { databaseId } } label { name } } + ... on UnlabeledEvent { id createdAt actor { login ... on User { databaseId } } label { name } } + ... on ClosedEvent { id createdAt actor { login ... on User { databaseId } } stateReason } + ... on ReopenedEvent { id createdAt actor { login ... on User { databaseId } } } + ... on IssueTypeChangedEvent { id createdAt actor { login ... on User { databaseId } } issueType { name } prevIssueType { name } } ... on IssueFieldChangedEvent { - id createdAt actor { login } previousValue newValue + id createdAt actor { login ... on User { databaseId } } previousValue newValue issueField { ... on IssueFieldSingleSelect { name } ... on IssueFieldMultiSelect { name } @@ -2281,7 +3009,7 @@ streams: ... on IssueFieldText { name } } } - ... on ProjectV2ItemStatusChangedEvent { id createdAt actor { login } previousStatus status } + ... on ProjectV2ItemStatusChangedEvent { id createdAt actor { login ... on User { databaseId } } previousStatus status } } } } @@ -2418,10 +3146,20 @@ streams: path: [actor_login] value: "{{ (record.get('actor') or {}).get('login') or '' }}" value_type: string + # Teams, bots and mannequins have a login but no databaseId, so this + # is 0 for them while actor_login still names them. + - type: AddedFieldDefinition + path: [actor_id] + value: "{{ (record.get('actor') or {}).get('databaseId') or 0 }}" + value_type: integer - type: AddedFieldDefinition path: [target_login] value: "{{ (record.get('assignee') or {}).get('login') or '' }}" value_type: string + - type: AddedFieldDefinition + path: [target_id] + value: "{{ (record.get('assignee') or {}).get('databaseId') or 0 }}" + value_type: integer - type: AddedFieldDefinition path: [label_name] value: "{{ (record.get('label') or {}).get('name') or '' }}" diff --git a/src/ingestion/connectors/git/github/dbt/github__account_emails.sql b/src/ingestion/connectors/git/github/dbt/github__account_emails.sql new file mode 100644 index 0000000000..96ccb19f52 --- /dev/null +++ b/src/ingestion/connectors/git/github/dbt/github__account_emails.sql @@ -0,0 +1,137 @@ +-- depends_on: {{ ref('github__bronze_promoted') }} +{{ config( + materialized='table', + schema='staging', + tags=['github'] +) }} + +-- Every (GitHub account, e-mail) pair the connector has seen, one row per pair. +-- +-- Commits carry an e-mail and no account; the roster carries an account and, +-- only where a member publishes one, an e-mail. Neither side alone lets a +-- commit reach a person, so this model collects the pairs that name both. +-- +-- Four independent sources, weakest last: +-- 1. the commit author lookup, which resolves an e-mail GitHub could match +-- to an account whether or not the person ever opened a pull request +-- 2. the commit's own account on a pull-request commit, as GitHub matched it +-- 3. the profile e-mail on the pull-request author +-- 4. the noreply address, which encodes the login it was issued to and so +-- names its own account without any lookup +-- +-- Several e-mails per account is normal and every one of them is kept: a person +-- who changes address still owns what they committed under the old one. + +WITH resolved_authors AS ( + SELECT + lower(trimBoth(COALESCE(author_login, ''))) AS login, + lower(trimBoth(COALESCE(author_email, ''))) AS email, + tenant_id, + source_id, + 'bronze_github.commit_authors.author_email' AS observed_in, + max(parseDateTimeBestEffortOrNull(collected_at)) AS seen_at + FROM {{ source('bronze_github', 'commit_authors') }} FINAL + WHERE COALESCE(author_login, '') != '' + AND COALESCE(author_email, '') != '' + GROUP BY login, email, tenant_id, source_id, observed_in +), + +commit_accounts AS ( + SELECT + lower(trimBoth(COALESCE(author_login, ''))) AS login, + lower(trimBoth(COALESCE(author_email, ''))) AS email, + tenant_id, + source_id, + 'bronze_github.pull_request_commits.author_email' AS observed_in, + max(parseDateTimeBestEffortOrNull(authored_date)) AS seen_at + FROM {{ source('bronze_github', 'pull_request_commits') }} FINAL + WHERE COALESCE(author_login, '') != '' + AND COALESCE(author_email, '') != '' + GROUP BY login, email, tenant_id, source_id, observed_in +), + +committer_accounts AS ( + SELECT + lower(trimBoth(COALESCE(committer_login, ''))) AS login, + lower(trimBoth(COALESCE(committer_email, ''))) AS email, + tenant_id, + source_id, + 'bronze_github.pull_request_commits.committer_email' AS observed_in, + max(parseDateTimeBestEffortOrNull(committed_date)) AS seen_at + FROM {{ source('bronze_github', 'pull_request_commits') }} FINAL + WHERE COALESCE(committer_login, '') != '' + AND COALESCE(committer_email, '') != '' + GROUP BY login, email, tenant_id, source_id, observed_in +), + +profile_emails AS ( + SELECT + lower(trimBoth(COALESCE(author_login, ''))) AS login, + lower(trimBoth(COALESCE(author_email, ''))) AS email, + tenant_id, + source_id, + 'bronze_github.pull_request_diff_stats.author_email' AS observed_in, + max(parseDateTimeBestEffortOrNull(updated_at)) AS seen_at + FROM {{ source('bronze_github', 'pull_request_diff_stats') }} FINAL + WHERE COALESCE(author_login, '') != '' + AND COALESCE(author_email, '') != '' + GROUP BY login, email, tenant_id, source_id, observed_in +), + +-- `12345678+octocat@users.noreply.github.com`, and the older form without the +-- account id. The login sits between an optional `id+` and the host, so the +-- address states which account it belongs to. +noreply_commits AS ( + SELECT + lower(trimBoth(extract(COALESCE(author_email, ''), '^(?:[0-9]+\\+)?([^@]+)@users\\.noreply\\.github\\.com$'))) AS login, + lower(trimBoth(COALESCE(author_email, ''))) AS email, + tenant_id, + source_id, + 'bronze_github.commits.author_email' AS observed_in, + max(parseDateTimeBestEffortOrNull(authored_date)) AS seen_at + FROM {{ source('bronze_github', 'commits') }} FINAL + WHERE author_email LIKE '%@users.noreply.github.com' + GROUP BY login, email, tenant_id, source_id, observed_in +), + +observations AS ( + SELECT * FROM resolved_authors + UNION ALL + SELECT * FROM commit_accounts + UNION ALL + SELECT * FROM committer_accounts + UNION ALL + SELECT * FROM profile_emails + UNION ALL + SELECT * FROM noreply_commits +), + +-- The noreply pattern yields '' for an address that does not match it, and a +-- row whose date will not parse carries no usable observation. +every_pair AS ( + SELECT * + FROM observations + WHERE login != '' + AND email != '' + AND seen_at IS NOT NULL +) + +SELECT + login, + email, + tenant_id, + source_id, + -- Which source named the pair first, for provenance on the claim. + -- assumeNotNull because argMin and min inherit the ordering column's + -- nullability: these two feed silver.identity_inputs.value_field_name and + -- its `_version`, where a Nullable would widen a column every connector + -- shares and ReplacingMergeTree rejects a nullable version outright. Every + -- row reaching here passed the IS NOT NULL guard above. + assumeNotNull(argMin(observed_in, seen_at)) AS observed_in, + assumeNotNull(min(seen_at)) AS observed_at +FROM every_pair +-- The full connection scope, not just the pair: several GitHub connections +-- share this bronze namespace, and the same login can name different accounts +-- on different hosts — a pair collapsed across scopes lands its claim under an +-- arbitrary one. +GROUP BY login, email, tenant_id, source_id diff --git a/src/ingestion/connectors/git/github/dbt/github__bronze_promoted.sql b/src/ingestion/connectors/git/github/dbt/github__bronze_promoted.sql index 27dcb67831..43423df2ae 100644 --- a/src/ingestion/connectors/git/github/dbt/github__bronze_promoted.sql +++ b/src/ingestion/connectors/git/github/dbt/github__bronze_promoted.sql @@ -32,5 +32,6 @@ {% do promote_bronze_to_rmt(table='bronze_github.deployment_statuses', order_by='unique_key') %} {% do promote_bronze_to_rmt(table='bronze_github.pull_request_timeline_events', order_by='unique_key') %} {% do promote_bronze_to_rmt(table='bronze_github.issue_timeline_events', order_by='unique_key') %} +{% do promote_bronze_to_rmt(table='bronze_github.commit_authors', order_by='unique_key') %} SELECT 1 AS promoted diff --git a/src/ingestion/connectors/git/github/dbt/github__file_changes.sql b/src/ingestion/connectors/git/github/dbt/github__file_changes.sql index 595ff5b06e..0d6e42a444 100644 --- a/src/ingestion/connectors/git/github/dbt/github__file_changes.sql +++ b/src/ingestion/connectors/git/github/dbt/github__file_changes.sql @@ -16,21 +16,7 @@ SELECT replaceRegexpOne(arrayElement(splitByChar('/', COALESCE(repository, '')), -1), '\\.git$', '') AS repo_slug, COALESCE(sha, '') AS commit_hash, COALESCE(filename, '') AS file_path, - -- File extension: last segment after the final '.', empty when none. - -- Earlier shape (issue #494) used `position('.', filename) > 0` as the - -- guard — but ClickHouse `position` is function-style - -- `position(haystack, needle)`, so this asked "is the string `filename` - -- present inside the single character '.'?" — always false. Result: - -- `file_extension` was empty for 100% of rows. Length check on the - -- split array is more robust than a fixed `position(filename, '.') > 0` - -- swap because it correctly returns '' for extensionless paths like - -- `Makefile` (where the position-based guard would also fire 0 by - -- accident, but the array-length guard is the explicit predicate). - if( - length(splitByChar('.', COALESCE(filename, ''))) > 1, - arrayElement(splitByChar('.', COALESCE(filename, '')), -1), - '' - ) AS file_extension, + {{ git_file_extension("COALESCE(filename, '')") }} AS file_extension, COALESCE(status, '') AS change_type, toNullable(COALESCE(additions, 0)) AS lines_added, toNullable(COALESCE(deletions, 0)) AS lines_removed, diff --git a/src/ingestion/connectors/git/github/dbt/github__identity_inputs.sql b/src/ingestion/connectors/git/github/dbt/github__identity_inputs.sql new file mode 100644 index 0000000000..136bc09705 --- /dev/null +++ b/src/ingestion/connectors/git/github/dbt/github__identity_inputs.sql @@ -0,0 +1,74 @@ +{{ config( + materialized='incremental', + incremental_strategy='append', + schema='staging', + tags=['github', 'silver', 'silver:identity_inputs'] +) }} + +-- E-mail claims against GitHub accounts, unioned into silver.identity_inputs +-- via the `silver:identity_inputs` tag. +-- +-- source_type is 'github', matching the roster connector: both describe the +-- same accounts, so both must name the same vendor. The account id is the +-- lowercased login (ADR-0002). +-- +-- Only `value_type='email'` rows, never the `value_type='id'` binding the +-- shared macro also emits: what an account means is the persons-seed's decision, +-- not this model's. The seed attaches a claimed e-mail to whichever person its +-- roster binding or e-mail match names — which is how an organization rostered +-- from an HR source and using only git still gets commit attribution — and mints +-- a new person for an unmatched active account. +-- +-- Hand-rolled rather than built with identity_inputs_from_history: that macro +-- keys a row on (account, value_type, instant) without the value, which holds +-- for a roster carrying one e-mail per member but not here, where an account +-- legitimately has several e-mails at once and all of them must survive. +-- +-- Column order matches the macro's output — silver.identity_inputs is a +-- positional UNION ALL, and check-field-parity.py audits the shape. + +WITH observations AS ( + SELECT + toUUID(UUIDNumToString(sipHash128(coalesce(tenant_id, '')))) AS insight_tenant_id, + toUUID(UUIDNumToString(sipHash128(coalesce(source_id, '')))) AS insight_source_id, + 'github' AS insight_source_type, + login AS source_account_id, + 'email' AS value_type, + email AS value, + observed_in AS value_field_name, + 'UPSERT' AS operation_type, + -- Insertion time, NOT the historical observation time: the shared + -- identity_inputs model admits only rows above its global max(_version) + -- watermark, so a claim versioned by a past commit date would be + -- silently dropped whenever any other feeder has already stamped a + -- newer version. The observation time survives on + -- github__account_emails.observed_at. + now64(3) AS _synced_at + FROM {{ ref('github__account_emails') }} +) + +SELECT + -- The value and the source are part of the key: two e-mails on one account + -- stay two claims, and the same pair under two connections stays two + -- accounts. + CAST(concat( + toString(o.insight_tenant_id), '-', + toString(o.insight_source_id), '-', + o.insight_source_type, '-', + o.source_account_id, '-', + o.value_type, '-', + o.operation_type, '-', + hex(sipHash64(o.value)) + ) AS String) AS unique_key, + o.*, + toUnixTimestamp64Milli(o._synced_at) AS _version +FROM observations AS o +{% if is_incremental() %} +LEFT ANTI JOIN {{ this }} AS existing + ON o.value_type = existing.value_type + AND o.value = existing.value + AND o.source_account_id = existing.source_account_id + AND existing.insight_source_type = 'github' + AND existing.insight_tenant_id = o.insight_tenant_id + AND existing.insight_source_id = o.insight_source_id +{% endif %} diff --git a/src/ingestion/connectors/git/github/dbt/schema.yml b/src/ingestion/connectors/git/github/dbt/schema.yml index 2688c562d5..adfef53f67 100644 --- a/src/ingestion/connectors/git/github/dbt/schema.yml +++ b/src/ingestion/connectors/git/github/dbt/schema.yml @@ -27,6 +27,7 @@ sources: - name: deployment_statuses - name: pull_request_timeline_events - name: issue_timeline_events + - name: commit_authors models: - name: github__bronze_promoted @@ -58,3 +59,32 @@ models: - name: github__pull_requests_commits description: "GitHub PR commits -> staging, feeds class_git_pull_requests_commits" + + - name: github__account_emails + description: "Every (GitHub account, e-mail) pair the connector observed, one row per pair" + columns: + - name: login + description: "Lowercased GitHub login the e-mail was observed under (ADR-0002 account id)" + data_tests: + - not_null + - name: email + description: "Lowercased e-mail address the account committed under or published" + data_tests: + - not_null + - name: observed_in + description: "Bronze field that first named this pair, carried onto the claim as provenance" + - name: observed_at + description: "Earliest time the pair was observed" + + - name: github__identity_inputs + description: "E-mail claims against GitHub accounts -> staging, feeds silver.identity_inputs" + columns: + - name: unique_key + description: "One claim per (tenant, account, value_type, operation, e-mail)" + data_tests: + - not_null + - unique + - name: source_account_id + description: "Lowercased GitHub login the claim is made against" + - name: value + description: "The claimed e-mail address" diff --git a/src/ingestion/connectors/git/github/descriptor.yaml b/src/ingestion/connectors/git/github/descriptor.yaml index 452f47bcda..d01bfb8854 100644 --- a/src/ingestion/connectors/git/github/descriptor.yaml +++ b/src/ingestion/connectors/git/github/descriptor.yaml @@ -1,5 +1,5 @@ name: github -version: "2.1.0" +version: "2.3.0" schedule: "0 4 * * *" workflow: sync diff --git a/src/ingestion/connectors/git/github/tests/test_github_streams.py b/src/ingestion/connectors/git/github/tests/test_github_streams.py index 22fb98984a..062bb7b5ab 100644 --- a/src/ingestion/connectors/git/github/tests/test_github_streams.py +++ b/src/ingestion/connectors/git/github/tests/test_github_streams.py @@ -90,6 +90,26 @@ def test_repositories_full_refresh_and_stamping(http_mocker: HttpMocker) -> None assert_records_conform(output.records, _CONNECTOR, "repositories", strict=True) +@freezegun.freeze_time(_FROZEN) +def test_repository_roster_admits_neither_forks_nor_archived(http_mocker: HttpMocker) -> None: + """A fork's clone carries its whole upstream history, so admitting one + credits every upstream commit to this organization; an archived repository + is not active work. Both stay out of the roster every other stream + partitions over.""" + config = GithubConfigBuilder().build() + fork = _repo() | {"id": 43, "full_name": "acme/forked", "name": "forked", "fork": True} + archived = _repo() | {"id": 44, "full_name": "acme/old", "name": "old", "archived": True} + http_mocker.get( + HttpRequest(_REPOS_URL, query_params=ANY_QUERY_PARAMS), + HttpResponse(body=json.dumps([_repo(), fork, archived]), status_code=200), + ) + + output = read_stream(_CONNECTOR, "repositories", config) + + assert not output.errors + assert [r.record.data["full_name"] for r in output.records] == ["acme/app"] + + @freezegun.freeze_time(_FROZEN) def test_secondary_rate_limit_403_retries_then_succeeds(http_mocker: HttpMocker) -> None: """GitHub reports secondary limits as 403; the predicate must classify it @@ -173,10 +193,22 @@ def test_pull_requests_trim_body_and_hoist_author(http_mocker: HttpMocker) -> No "draft": False, "title": "t" * 3000, "body": "b" * 3000, - "user": {"login": "alice"}, - "head": {"ref": "feat", "sha": "e" * 40}, - "base": {"ref": "main"}, - "author_association": "MEMBER", + "user": {"login": "alice", "id": 4242, "type": "User"}, + "head": { + "ref": "feat", + "sha": "e" * 40, + "label": "contributor:feat", + "repo": {"full_name": "contributor/app"}, + }, + "base": {"ref": "main", "sha": "f" * 40}, + "author_association": "CONTRIBUTOR", + "labels": [{"name": "bug"}, {"name": "urgent"}], + "assignees": [{"login": "bob"}], + "requested_reviewers": [{"login": "carol"}], + "requested_teams": [{"slug": "platform"}], + "milestone": {"title": "v2", "number": 9}, + "auto_merge": None, + "locked": False, "created_at": "2026-06-10T00:00:00Z", "updated_at": "2026-06-20T00:00:00Z", } @@ -193,6 +225,17 @@ def test_pull_requests_trim_body_and_hoist_author(http_mocker: HttpMocker) -> No assert len(rec["body"]) == 2048 assert len(rec["title"]) == 1024 assert rec["author_login"] == "alice" + assert (rec["author_id"], rec["author_type"]) == (4242, "User") + # A head branch in another repository is the only fork signal here. + assert rec["head_repo_full_name"] == "contributor/app" + assert rec["head_label"] == "contributor:feat" + assert rec["base_sha"] == "f" * 40 + assert json.loads(rec["label_names"]) == ["bug", "urgent"] + assert json.loads(rec["assignee_logins"]) == ["bob"] + assert json.loads(rec["requested_reviewer_logins"]) == ["carol"] + assert json.loads(rec["requested_team_slugs"]) == ["platform"] + assert (rec["milestone_title"], rec["milestone_number"]) == ("v2", 9) + assert rec["auto_merge_enabled"] is False _no_literal_none(output.records) assert_records_conform(output.records, _CONNECTOR, "pull_requests", strict=True) @@ -234,9 +277,12 @@ def _pr(num: int, updated: str) -> dict: @freezegun.freeze_time(_FROZEN) -def test_pull_request_commits_store_only_the_edge(http_mocker: HttpMocker) -> None: - """PR<->commit membership is vendor-only; the commit payload belongs to - the proxy streams, so bronze keeps the sha and the PR identity alone.""" +def test_pull_request_commits_carry_the_commit_email_and_its_account( + http_mocker: HttpMocker, +) -> None: + """A commit's e-mail and the GitHub account it belongs to appear together + nowhere else, so both survive alongside the membership edge and the commit's + own metadata — the proxy cannot see a fork's head commits at all.""" config = GithubConfigBuilder().build() http_mocker.get(HttpRequest(_REPOS_URL, query_params=ANY_QUERY_PARAMS), _repos_page()) http_mocker.get( @@ -271,13 +317,26 @@ def test_pull_request_commits_store_only_the_edge(http_mocker: HttpMocker) -> No { "sha": "a" * 40, "node_id": "C_1", - "commit": {"message": "feat: x", "author": {"name": "Alice"}}, + "commit": { + "message": "feat: x", + "author": { + "name": "Alice", + "email": "alice@example.com", + "date": "2026-06-11T09:00:00Z", + }, + "committer": { + "name": "Alice", + "email": "1234+alice@users.noreply.github.com", + "date": "2026-06-11T09:05:00Z", + }, + "verification": {"verified": True, "reason": "valid"}, + }, "url": "https://api.github.com/repos/acme/app/commits/" + "a" * 40, "html_url": "https://github.com/acme/app/commit/" + "a" * 40, "comments_url": "https://api.github.com/repos/acme/app/commits/x/comments", - "author": {"login": "alice"}, - "committer": {"login": "alice"}, - "parents": [{"sha": "b" * 40}], + "author": {"login": "alice", "id": 4242, "type": "User"}, + "committer": {"login": "alice", "id": 4242, "type": "User"}, + "parents": [{"sha": "b" * 40}, {"sha": "c" * 40}], } ] ), @@ -293,6 +352,20 @@ def test_pull_request_commits_store_only_the_edge(http_mocker: HttpMocker) -> No assert rec["pull_number"] == 31 assert rec["repo_full_name"] == "acme/app" assert rec["unique_key"].endswith(f":acme/app:31:{'a' * 40}") + assert rec["author_login"] == "alice" + assert rec["author_id"] == 4242 + assert rec["author_type"] == "User" + assert rec["author_email"] == "alice@example.com" + assert rec["author_name"] == "Alice" + assert rec["authored_date"] == "2026-06-11T09:00:00Z" + assert rec["committer_login"] == "alice" + assert rec["committer_email"] == "1234+alice@users.noreply.github.com" + assert rec["committed_date"] == "2026-06-11T09:05:00Z" + assert rec["message"] == "feat: x" + assert rec["is_verified"] is True + assert rec["verification_reason"] == "valid" + assert json.loads(rec["parent_shas"]) == ["b" * 40, "c" * 40] + assert rec["is_merge"] is True assert "commit" not in rec and "parents" not in rec _no_literal_none(output.records) assert_records_conform(output.records, _CONNECTOR, "pull_request_commits", strict=True) @@ -324,7 +397,15 @@ def test_pull_request_diff_stats_come_from_the_list_node(http_mocker: HttpMocker "additions": 120, "deletions": 7, "changedFiles": 3, - "author": {"email": "alice@example.com"}, + "author": { + "login": "alice", + "databaseId": 4242, + "email": "alice@example.com", + }, + "mergedBy": {"login": "bob", "databaseId": 77}, + "reviewDecision": "APPROVED", + "totalCommentsCount": 5, + "isCrossRepository": False, } ], } @@ -343,6 +424,12 @@ def test_pull_request_diff_stats_come_from_the_list_node(http_mocker: HttpMocker assert (rec["additions"], rec["deletions"], rec["changed_files"]) == (120, 7, 3) assert rec["pull_number"] == 31 assert rec["repo_full_name"] == "acme/app" + # REST reports merged_at but never who merged it. + assert (rec["merged_by_login"], rec["merged_by_id"]) == ("bob", 77) + assert (rec["author_login"], rec["author_id"]) == ("alice", 4242) + assert rec["review_decision"] == "APPROVED" + assert rec["total_comments_count"] == 5 + assert rec["is_cross_repository"] is False assert rec["unique_key"].endswith(":acme/app:31") assert rec["author_email"] == "alice@example.com" assert "changedFiles" not in rec and "updatedAt" not in rec @@ -727,3 +814,96 @@ def test_projects_v2_graphql_pagination_cursor_in_body(http_mocker: HttpMocker) assert rec["title"] == "Roadmap" assert rec["short_description"] == "" _no_literal_none(output.records) + + +def _authors_page(*rows: dict) -> HttpResponse: + return HttpResponse( + body=json.dumps({"items": list(rows), "next_page_token": None}), status_code=200 + ) + + +def _author(email: str, sha: str, name: str = "Dev") -> dict: + return { + "author_email": email, + "author_name": name, + "sample_sha": sha, + "last_committed_date": "2026-06-15T10:00:00+00:00", + "commit_count": 3, + } + + +@freezegun.freeze_time(_FROZEN) +def test_commit_authors_resolve_a_proxy_author_to_an_account(http_mocker: HttpMocker) -> None: + """The proxy names the distinct authors; GitHub names the account behind + each one. One vendor call per author, keyed on the git e-mail so the claim + matches what the commit rows carry.""" + config = GithubConfigBuilder().build() + http_mocker.get(HttpRequest(_REPOS_URL, query_params=ANY_QUERY_PARAMS), _repos_page()) + http_mocker.get( + HttpRequest(f"{PROXY_URL}/v1/authors", query_params=ANY_QUERY_PARAMS), + _authors_page(_author("ada@example.com", "a" * 40)), + ) + http_mocker.get( + HttpRequest(f"{GH_URL}/repos/acme/app/commits/{'a' * 40}", query_params=ANY_QUERY_PARAMS), + HttpResponse( + body=json.dumps( + { + "sha": "a" * 40, + "node_id": "C_1", + "commit": {"author": {"name": "Ada", "email": "ada@example.com"}}, + "author": {"login": "ada", "id": 4242, "type": "User"}, + "committer": {"login": "ada", "id": 4242, "type": "User"}, + "parents": [], + } + ), + status_code=200, + ), + ) + + output = read_stream(_CONNECTOR, "commit_authors", config) + + assert not output.errors + rec = output.records[0].record.data + assert rec["author_email"] == "ada@example.com", "keyed on the git ident, not the profile" + assert (rec["author_login"], rec["author_id"], rec["author_type"]) == ("ada", 4242, "User") + assert rec["repo_full_name"] == "acme/app" + assert rec["sample_sha"] == "a" * 40 + assert rec["unique_key"].endswith(":acme/app:author:ada@example.com") + assert "commit" not in rec and "author" not in rec + _no_literal_none(output.records) + assert_records_conform(output.records, _CONNECTOR, "commit_authors", strict=True) + + +@freezegun.freeze_time(_FROZEN) +def test_commit_authors_drops_an_email_github_matches_to_nobody( + http_mocker: HttpMocker, +) -> None: + """GitHub answers `author: null` for an e-mail verified on no account — a + CI or service identity. There is no account to claim the e-mail, so the + row is dropped rather than stored as an unresolved one.""" + config = GithubConfigBuilder().build() + http_mocker.get(HttpRequest(_REPOS_URL, query_params=ANY_QUERY_PARAMS), _repos_page()) + http_mocker.get( + HttpRequest(f"{PROXY_URL}/v1/authors", query_params=ANY_QUERY_PARAMS), + _authors_page(_author("ci@build.local", "b" * 40, name="CI")), + ) + http_mocker.get( + HttpRequest(f"{GH_URL}/repos/acme/app/commits/{'b' * 40}", query_params=ANY_QUERY_PARAMS), + HttpResponse( + body=json.dumps( + { + "sha": "b" * 40, + "commit": {"author": {"name": "CI", "email": "ci@build.local"}}, + "author": None, + "committer": None, + "parents": [], + } + ), + status_code=200, + ), + ) + + output = read_stream(_CONNECTOR, "commit_authors", config) + + assert not output.errors + assert len(output.records) == 0, "an unmatched e-mail claims no account" diff --git a/src/ingestion/connectors/git/gitlab/dbt/gitlab__file_changes.sql b/src/ingestion/connectors/git/gitlab/dbt/gitlab__file_changes.sql index 64610ad724..4064e86907 100644 --- a/src/ingestion/connectors/git/gitlab/dbt/gitlab__file_changes.sql +++ b/src/ingestion/connectors/git/gitlab/dbt/gitlab__file_changes.sql @@ -28,11 +28,7 @@ SELECT COALESCE(p.repo_slug, '') AS repo_slug, COALESCE(fc.commit_sha, '') AS commit_hash, COALESCE(fc.new_path, fc.old_path, '') AS file_path, - if( - length(splitByChar('.', COALESCE(fc.new_path, fc.old_path, ''))) > 1, - arrayElement(splitByChar('.', COALESCE(fc.new_path, fc.old_path, '')), -1), - '' - ) AS file_extension, + {{ git_file_extension("COALESCE(fc.new_path, fc.old_path, '')") }} AS file_extension, multiIf( fc.new_file = true, 'added', fc.deleted_file = true, 'deleted', diff --git a/src/ingestion/dbt/macros/git_file_extension.sql b/src/ingestion/dbt/macros/git_file_extension.sql new file mode 100644 index 0000000000..2fbec7dec9 --- /dev/null +++ b/src/ingestion/dbt/macros/git_file_extension.sql @@ -0,0 +1,17 @@ +{# The extension of a git path: the segment after the final '.' of the file + NAME, empty when the name carries none. + + Splitting the whole path would read a dot in a directory (`github.com/x/y`, + `v1.2/setup`) as the extension separator and return the rest of the path, so + the basename is taken first. Dotfiles (`.gitignore`) and extensionless names + (`Makefile`) have no extension and yield ''. #} + +{% macro git_file_extension(path_expr) -%} +{%- set basename = "arrayElement(splitByChar('/', " ~ path_expr ~ "), -1)" -%} +if( + length(splitByChar('.', {{ basename }})) > 1 + AND arrayElement(splitByChar('.', {{ basename }}), 1) != '', + arrayElement(splitByChar('.', {{ basename }}), -1), + '' + ) +{%- endmacro %} diff --git a/src/ingestion/scripts/connectors-ddl/github.sql b/src/ingestion/scripts/connectors-ddl/github.sql index 5afcdd13cf..980af6d55c 100644 --- a/src/ingestion/scripts/connectors-ddl/github.sql +++ b/src/ingestion/scripts/connectors-ddl/github.sql @@ -22,6 +22,29 @@ ORDER BY unique_key SETTINGS allow_nullable_key = 1, index_granularity = 8192 ; +CREATE TABLE IF NOT EXISTS bronze_github.commit_authors +( + `_airbyte_raw_id` String, + `_airbyte_extracted_at` DateTime64(3), + `_airbyte_meta` String, + `_airbyte_generation_id` UInt32, + `unique_key` Nullable(String), + `tenant_id` Nullable(String), + `source_id` Nullable(String), + `data_source` Nullable(String), + `collected_at` Nullable(String), + `repo_full_name` Nullable(String), + `author_email` Nullable(String), + `author_login` Nullable(String), + `author_id` Nullable(Int64), + `author_type` Nullable(String), + `sample_sha` Nullable(String) +) +ENGINE = ReplacingMergeTree(_airbyte_extracted_at) +ORDER BY unique_key +SETTINGS allow_nullable_key = 1, index_granularity = 8192 +; + CREATE TABLE IF NOT EXISTS bronze_github.commits ( `_airbyte_raw_id` String, @@ -71,8 +94,14 @@ CREATE TABLE IF NOT EXISTS bronze_github.deployment_statuses `repo_full_name` Nullable(String), `state` Nullable(String), `environment` Nullable(String), + `description` Nullable(String), `creator_login` Nullable(String), - `created_at` Nullable(String) + `creator_id` Nullable(Int64), + `target_url` Nullable(String), + `log_url` Nullable(String), + `environment_url` Nullable(String), + `created_at` Nullable(String), + `updated_at` Nullable(String) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -96,8 +125,12 @@ CREATE TABLE IF NOT EXISTS bronze_github.deployments `ref` Nullable(String), `task` Nullable(String), `environment` Nullable(String), + `original_environment` Nullable(String), + `is_transient_environment` Nullable(Bool), + `is_production_environment` Nullable(Bool), `description` Nullable(String), `creator_login` Nullable(String), + `creator_id` Nullable(Int64), `created_at` Nullable(String), `updated_at` Nullable(String) ) @@ -152,7 +185,9 @@ CREATE TABLE IF NOT EXISTS bronze_github.issue_timeline_events `item_number` Nullable(Int64), `repo_full_name` Nullable(String), `actor_login` Nullable(String), + `actor_id` Nullable(Int64), `target_login` Nullable(String), + `target_id` Nullable(Int64), `label_name` Nullable(String), `state_reason` Nullable(String), `field_name` Nullable(String), @@ -181,9 +216,21 @@ CREATE TABLE IF NOT EXISTS bronze_github.issues `state` Nullable(String), `state_reason` Nullable(String), `title` Nullable(String), + `body` Nullable(String), `author_login` Nullable(String), + `author_id` Nullable(Int64), + `author_type` Nullable(String), + `author_association` Nullable(String), `assignee_logins` Nullable(String), + `assignee_ids` Nullable(String), `label_names` Nullable(String), + `issue_type` Nullable(String), + `milestone_title` Nullable(String), + `milestone_number` Nullable(Int64), + `closed_by_login` Nullable(String), + `closed_by_id` Nullable(Int64), + `locked` Nullable(Bool), + `reactions_total` Nullable(Int64), `comments` Nullable(Int64), `created_at` Nullable(String), `updated_at` Nullable(String), @@ -239,7 +286,10 @@ CREATE TABLE IF NOT EXISTS bronze_github.pull_request_comments `created_at` Nullable(String), `updated_at` Nullable(String), `body` Nullable(String), - `author_id` Nullable(Int64) + `author_id` Nullable(Int64), + `author_type` Nullable(String), + `is_via_github_app` Nullable(Bool), + `reactions_total` Nullable(Int64) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -259,7 +309,23 @@ CREATE TABLE IF NOT EXISTS bronze_github.pull_request_commits `collected_at` Nullable(String), `sha` Nullable(String), `pull_number` Nullable(Int64), - `repo_full_name` Nullable(String) + `repo_full_name` Nullable(String), + `author_login` Nullable(String), + `author_id` Nullable(Int64), + `author_type` Nullable(String), + `author_name` Nullable(String), + `author_email` Nullable(String), + `authored_date` Nullable(String), + `committer_login` Nullable(String), + `committer_id` Nullable(Int64), + `committer_name` Nullable(String), + `committer_email` Nullable(String), + `committed_date` Nullable(String), + `message` Nullable(String), + `parent_shas` Nullable(String), + `is_merge` Nullable(Bool), + `is_verified` Nullable(Bool), + `verification_reason` Nullable(String) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -283,7 +349,14 @@ CREATE TABLE IF NOT EXISTS bronze_github.pull_request_diff_stats `deletions` Nullable(Int64), `changed_files` Nullable(Int64), `updated_at` Nullable(String), - `author_email` Nullable(String) + `author_email` Nullable(String), + `author_login` Nullable(String), + `author_id` Nullable(Int64), + `merged_by_login` Nullable(String), + `merged_by_id` Nullable(Int64), + `review_decision` Nullable(String), + `total_comments_count` Nullable(Int64), + `is_cross_repository` Nullable(Bool) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -310,8 +383,19 @@ CREATE TABLE IF NOT EXISTS bronze_github.pull_request_review_comments `updated_at` Nullable(String), `body` Nullable(String), `author_id` Nullable(Int64), + `author_type` Nullable(String), `path` Nullable(String), - `line` Nullable(Int64) + `line` Nullable(Int64), + `start_line` Nullable(Int64), + `side` Nullable(String), + `subject_type` Nullable(String), + `commit_id` Nullable(String), + `original_commit_id` Nullable(String), + `review_id` Nullable(Int64), + `in_reply_to_id` Nullable(Int64), + `diff_hunk` Nullable(String), + `is_via_github_app` Nullable(Bool), + `reactions_total` Nullable(Int64) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -337,7 +421,9 @@ CREATE TABLE IF NOT EXISTS bronze_github.pull_request_reviews `state` Nullable(String), `commit_id` Nullable(String), `submitted_at` Nullable(String), - `author_id` Nullable(Int64) + `author_id` Nullable(Int64), + `author_type` Nullable(String), + `body` Nullable(String) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -361,9 +447,12 @@ CREATE TABLE IF NOT EXISTS bronze_github.pull_request_timeline_events `item_number` Nullable(Int64), `repo_full_name` Nullable(String), `actor_login` Nullable(String), + `actor_id` Nullable(Int64), `target_login` Nullable(String), + `target_id` Nullable(Int64), `label_name` Nullable(String), - `state_reason` Nullable(String) + `state_reason` Nullable(String), + `merge_commit_sha` Nullable(String) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -388,16 +477,30 @@ CREATE TABLE IF NOT EXISTS bronze_github.pull_requests `draft` Nullable(Bool), `title` Nullable(String), `author_login` Nullable(String), + `author_id` Nullable(Int64), + `author_type` Nullable(String), `author_association` Nullable(String), `head_sha` Nullable(String), `head_ref` Nullable(String), + `head_label` Nullable(String), + `head_repo_full_name` Nullable(String), `base_ref` Nullable(String), + `base_sha` Nullable(String), `merge_commit_sha` Nullable(String), `created_at` Nullable(String), `updated_at` Nullable(String), `closed_at` Nullable(String), `merged_at` Nullable(String), - `body` Nullable(String) + `body` Nullable(String), + `locked` Nullable(Bool), + `active_lock_reason` Nullable(String), + `auto_merge_enabled` Nullable(Bool), + `label_names` Nullable(String), + `assignee_logins` Nullable(String), + `requested_reviewer_logins` Nullable(String), + `requested_team_slugs` Nullable(String), + `milestone_title` Nullable(String), + `milestone_number` Nullable(Int64) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -425,6 +528,20 @@ CREATE TABLE IF NOT EXISTS bronze_github.repositories `private` Nullable(Bool), `has_issues` Nullable(Bool), `has_wiki` Nullable(Bool), + `has_projects` Nullable(Bool), + `has_discussions` Nullable(Bool), + `has_pages` Nullable(Bool), + `is_template` Nullable(Bool), + `disabled` Nullable(Bool), + `web_commit_signoff_required` Nullable(Bool), + `visibility` Nullable(String), + `license_spdx_id` Nullable(String), + `topics` Nullable(String), + `homepage` Nullable(String), + `stargazers_count` Nullable(Int64), + `forks_count` Nullable(Int64), + `watchers_count` Nullable(Int64), + `open_issues_count` Nullable(Int64), `language` Nullable(String), `size` Nullable(Int64), `clone_url` Nullable(String), @@ -461,6 +578,18 @@ CREATE TABLE IF NOT EXISTS bronze_github.workflow_runs `head_branch` Nullable(String), `head_sha` Nullable(String), `actor_login` Nullable(String), + `actor_id` Nullable(Int64), + `triggering_actor_login` Nullable(String), + `triggering_actor_id` Nullable(Int64), + `run_number` Nullable(Int64), + `workflow_path` Nullable(String), + `display_title` Nullable(String), + `check_suite_id` Nullable(Int64), + `pull_request_numbers` Nullable(String), + `head_commit_message` Nullable(String), + `head_commit_timestamp` Nullable(String), + `head_commit_author_name` Nullable(String), + `head_commit_author_email` Nullable(String), `run_started_at` Nullable(String), `created_at` Nullable(String), `updated_at` Nullable(String) diff --git a/src/ingestion/secrets/connectors/github.yaml.example b/src/ingestion/secrets/connectors/github.yaml.example index dc4e27f2fe..0301586787 100644 --- a/src/ingestion/secrets/connectors/github.yaml.example +++ b/src/ingestion/secrets/connectors/github.yaml.example @@ -11,9 +11,10 @@ metadata: annotations: insight.cyberfabric.com/connector: github insight.cyberfabric.com/source-id: github-main +type: Opaque stringData: github_token: CHANGE_ME # PAT: repo, read:org, read:project (classic) or fine-grained equivalent github_organizations: '["acme"]' # JSON array of org logins git_proxy_url: http://insight-git-cli-proxy:8085 git_proxy_token: CHANGE_ME - github_start_date: "2020-01-01" + github_start_date: "2026-01-01" diff --git a/src/ingestion/silver/_shared/identity_inputs.sql b/src/ingestion/silver/_shared/identity_inputs.sql index d14f31dc4f..40cf88fa09 100644 --- a/src/ingestion/silver/_shared/identity_inputs.sql +++ b/src/ingestion/silver/_shared/identity_inputs.sql @@ -8,6 +8,7 @@ -- depends_on: {{ ref('zulip_proxy__identity_inputs') }} -- depends_on: {{ ref('zoom__identity_inputs') }} -- depends_on: {{ ref('github_directory__identity_inputs') }} +-- depends_on: {{ ref('github__identity_inputs') }} -- @cpt-principle:cpt-dataflow-principle-rmt-with-version:p1 {{ config( materialized='incremental',