diff --git a/.claude/skills/connector/workflows/create.md b/.claude/skills/connector/workflows/create.md index 20a4715b6..db3572b37 100644 --- a/.claude/skills/connector/workflows/create.md +++ b/.claude/skills/connector/workflows/create.md @@ -676,13 +676,11 @@ cd src/ingestion/scripts/bootstrap-db ./generate-connectors-config.sh '/' ``` -**NEVER regenerate the whole file** (`./generate-connectors-config.sh` with no -argument): it overwrites the four `env:` credential references -(`HUBSPOT_ACCESS_TOKEN`, `SALESFORCE_CLIENT_ID`, `SALESFORCE_CLIENT_SECRET`, -`SALESFORCE_INSTANCE_URL`) with fake `value:` entries. +Regenerate only your own fragment: a whole-file regeneration rewrites every +connector's entry and buries your change in unrelated churn. -Fake credentials in your own fragment are fine — bootstrap only calls -`discover`, which reads the static spec, not the live API. +Fake credentials are fine — bootstrap only calls `discover`, which builds the +catalog from the connector's static schemas, not from the live API. ### 2. Keep shared silver class column types identical diff --git a/.github/workflows/connectors-ddl.yml b/.github/workflows/connectors-ddl.yml index 53d072a2b..6d08b6569 100644 --- a/.github/workflows/connectors-ddl.yml +++ b/.github/workflows/connectors-ddl.yml @@ -22,11 +22,9 @@ name: connectors-ddl # are each green apart — one adding a gold table, one regenerating the snapshot — # can still leave main drifted, and only a run on the merged tree sees that. # -# PRs are gated on the branch living in this repository: the job runs PR code and -# needs the HubSpot / Salesforce credentials whose CDK `discover` calls a live -# API, and secrets are not available to fork PRs. The trigger is `pull_request`, -# never `pull_request_target` — that would run fork code with access to those -# secrets. Pushes always carry secrets, so they need no such guard. +# Every connector's `discover` runs on fake config values, so the gate needs no +# secrets and fork PRs validate like any other. The trigger is `pull_request`, +# never `pull_request_target`. # # The gate job validates; it never commits. On PR drift the separate regen-pr # job delivers the regenerated snapshot as a reviewable stacked PR against the @@ -51,10 +49,6 @@ permissions: jobs: connectors-ddl: name: connectors-ddl snapshot + field parity - # Fork PR: no secrets, so `discover` for hubspot/salesforce cannot run. Skip - # up front instead of dying 20 minutes into the bootstrap. A push to main is - # always in-repo, so it has no head repository to compare. - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest timeout-minutes: 60 outputs: @@ -92,28 +86,6 @@ jobs: yq --version jq --version - - name: Require connector credentials - env: - HUBSPOT_ACCESS_TOKEN: ${{ secrets.HUBSPOT_ACCESS_TOKEN }} - SALESFORCE_CLIENT_ID: ${{ secrets.SALESFORCE_CLIENT_ID }} - SALESFORCE_CLIENT_SECRET: ${{ secrets.SALESFORCE_CLIENT_SECRET }} - SALESFORCE_INSTANCE_URL: ${{ secrets.SALESFORCE_INSTANCE_URL }} - run: | - set -euo pipefail - # HubSpot and Salesforce build their catalogue from a live API, so their - # `discover` cannot run on fake values. Without them seed-connectors.sh - # skips both connectors, their bronze never lands, and dbt dies — 20 - # minutes later, with a confusing error. Fail in 5 seconds instead, - # naming exactly what is missing. - missing=() - for var in HUBSPOT_ACCESS_TOKEN SALESFORCE_CLIENT_ID SALESFORCE_CLIENT_SECRET SALESFORCE_INSTANCE_URL; do - [[ -n "${!var:-}" ]] || missing+=("$var") - done - if (( ${#missing[@]} > 0 )); then - echo "::error title=Missing repository secrets::${missing[*]} — hubspot/salesforce discover calls a live API and cannot run on fake values. Add them as repository secrets, or narrow connectors-config.yaml." - exit 1 - fi - - name: Start an empty ClickHouse run: | set -euo pipefail @@ -128,11 +100,6 @@ jobs: - name: Bootstrap from connectors + dbt + migrations id: bootstrap - env: - HUBSPOT_ACCESS_TOKEN: ${{ secrets.HUBSPOT_ACCESS_TOKEN }} - SALESFORCE_CLIENT_ID: ${{ secrets.SALESFORCE_CLIENT_ID }} - SALESFORCE_CLIENT_SECRET: ${{ secrets.SALESFORCE_CLIENT_SECRET }} - SALESFORCE_INSTANCE_URL: ${{ secrets.SALESFORCE_INSTANCE_URL }} run: | set -euo pipefail "${BOOTSTRAP_DIR}/bootstrap-db.sh" "${BOOTSTRAP_DIR}/connectors-config.yaml" @@ -195,11 +162,10 @@ jobs: run: docker logs --tail 200 "${CLICKHOUSE_CONTAINER}" 2>&1 || true # On snapshot drift in a PR, deliver the regenerated snapshot as a REVIEWABLE - # stacked pull request against the PR's own branch, instead of telling the - # author to regenerate locally — which requires the HubSpot/Salesforce - # credentials most contributors do not have. The author reviews the DDL diff - # and merges it; that merge is a human push, so the required checks re-run - # normally (a bot push with GITHUB_TOKEN would not trigger them). + # stacked pull request against the PR's own branch, saving the author a local + # bootstrap run. The author reviews the DDL diff and merges it; that merge is + # a human push, so the required checks re-run normally (a bot push with + # GITHUB_TOKEN would not trigger them). # # This job never executes code from the PR: it checks out the head branch only # as a git base and replaces the snapshot directory with the artifact the gate @@ -211,8 +177,11 @@ jobs: regen-pr: name: Open a regen PR on snapshot drift needs: connectors-ddl + # Same-repo only: this job pushes a branch and opens a PR, neither of which + # it can do against a fork's head. A fork PR still gets the gate result and + # the regenerated snapshot as a downloadable artifact. if: >- - always() && github.event_name == 'pull_request' && needs.connectors-ddl.result == 'failure' && needs.connectors-ddl.outputs.drift == 'true' + always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && needs.connectors-ddl.result == 'failure' && needs.connectors-ddl.outputs.drift == 'true' runs-on: ubuntu-latest timeout-minutes: 10 permissions: diff --git a/docs/domain/connector/specs/ADR/0004-static-stream-schemas-with-raw-data.md b/docs/domain/connector/specs/ADR/0004-static-stream-schemas-with-raw-data.md new file mode 100644 index 000000000..494a71a7e --- /dev/null +++ b/docs/domain/connector/specs/ADR/0004-static-stream-schemas-with-raw-data.md @@ -0,0 +1,168 @@ +--- +status: proposed +date: 2026-08-05 +--- + +# ADR-0004: Static Stream Schemas with Full-Record `raw_data` + +**ID**: `cpt-insightspec-adr-connector-static-schema-raw-data` + + + +- [Context and Problem Statement](#context-and-problem-statement) +- [Decision Drivers](#decision-drivers) +- [Considered Options](#considered-options) +- [Decision Outcome](#decision-outcome) + - [Consequences](#consequences) + - [Confirmation](#confirmation) +- [Pros and Cons of the Options](#pros-and-cons-of-the-options) + - [Option 1: Static columns plus full-record `raw_data`](#option-1-static-columns-plus-full-record-raw_data) + - [Option 2: Discovery-derived columns](#option-2-discovery-derived-columns) + - [Option 3: Static columns, residual-only overflow blob](#option-3-static-columns-residual-only-overflow-blob) + - [Option 4: `raw_data` only, no typed columns](#option-4-raw_data-only-no-typed-columns) +- [More Information](#more-information) +- [Traceability](#traceability) + + + +## Context and Problem Statement + +A connector whose advertised schema is computed from a live discovery call +against the source instance makes the Bronze table shape a function of that +instance. Two consequences follow. Bronze DDL cannot be produced without +credentials, so tables cannot be created before the first sync. And two +instances of the same source yield different column sets for the same stream, +so downstream models cannot rely on a column existing. + +Restricting columns to a curated list fixes the shape but discards every field +outside the list — including fields the source adds later. Fields that were +never captured cannot be recovered retroactively, which forecloses metrics +that would otherwise be derivable from data already collected. + +What schema contract should a connector advertise so that Bronze is +instance-independent and no source field is lost? + +## Decision Drivers + +* Offline DDL — bronze tables must be creatable from the repository alone, with no credentials and no discovery call +* Instance independence — the same stream must produce the same columns against any instance of the source +* No field loss — a field the source returns must reach Bronze even when no column is declared for it +* Retroactive analysis — a metric conceived after ingestion must be answerable from history already stored +* Query ergonomics — the fields models actually consume should be plain typed columns, not JSON extraction + +## Considered Options + +* Option 1: Static columns plus full-record `raw_data` +* Option 2: Discovery-derived columns +* Option 3: Static columns, residual-only overflow blob +* Option 4: `raw_data` only, no typed columns + +## Decision Outcome + +Chosen option: "Static columns plus full-record `raw_data`". + +A connector declares each stream's columns statically, in the repository. The +declaration is the source of truth for both the advertised catalog and the +generated Bronze DDL. Every record additionally carries `raw_data`: the whole +source record, serialized as a compact JSON string. + +Rules: + +1. The advertised schema is loaded from a static declaration. Discovery calls + may build fetch lists (which fields to request from the API), never the + schema. +2. `raw_data` holds the record as received, minus source metadata envelopes + that carry no data. It is present on every stream. +3. Fields the stream does not declare are emitted only inside `raw_data`, never + as top-level keys. Emitting them would let the destination create columns + for them and restore instance-dependent drift. +4. String values inside `raw_data` are capped per value. The serialized blob is + never truncated, so it always parses. +5. Adding a column is a repository change: extend the static declaration. + +### Consequences + +* Good, because Bronze DDL derives from the repository and needs no credentials +* Good, because the column set is identical across instances of a source +* Good, because a field with no declared column still reaches Bronze and stays + available to models written later +* Good, because the fields models consume stay typed columns, so existing + queries need no JSON extraction +* Bad, because declared values are stored twice, once as a column and once + inside `raw_data` +* Bad, because a source that adds a field no longer surfaces it as a column + automatically; promoting it is a deliberate repository change +* Neutral, because `raw_data` is a JSON string rather than a native JSON + column — the destination materializes both identically today + +### Confirmation + +* Building a stream catalog performs no network call, and repeated builds + against different instances produce byte-identical schemas +* Every stream's advertised properties equal the static declaration plus the + envelope fields +* A record carrying a field with no declared column emits no top-level key for + it, and the field is present in the parsed `raw_data` + +## Pros and Cons of the Options + +### Option 1: Static columns plus full-record `raw_data` + +Columns declared in the repository; whole record additionally preserved as JSON. + +* Good, because it satisfies offline DDL and no-field-loss simultaneously +* Good, because typed columns keep the common query path ergonomic +* Bad, because declared values are duplicated inside the blob + +### Option 2: Discovery-derived columns + +The advertised schema is computed per instance from a live discovery call. + +* Good, because a newly added source field becomes a column with no code change +* Bad, because Bronze DDL cannot be produced without credentials +* Bad, because column sets differ across instances of the same source +* Bad, because a fetch failure during discovery fails the whole sync + +### Option 3: Static columns, residual-only overflow blob + +Only undeclared fields go to the blob; declared values are not duplicated. + +* Good, because it avoids duplicate storage +* Bad, because the record cannot be reconstructed from one column; consumers + must join columns and blob and know which is which +* Bad, because promoting a field to a column changes where historical values + live, so a query must read both shapes + +### Option 4: `raw_data` only, no typed columns + +Envelope plus one JSON column; all fields extracted downstream. + +* Good, because Bronze DDL becomes identical for every stream +* Good, because it stores each value once +* Bad, because every existing downstream model must be rewritten to extract + from JSON +* Bad, because wide blobs flowing through sort buffers are a known source of + memory exhaustion in downstream aggregation + +## More Information + +Connectors whose column set is already curated and instance-independent satisfy +this ADR by adding `raw_data`; their existing declaration becomes the static +schema. Connectors that compute schemas from discovery must move the +declaration into the repository. + +`raw_data` is the only overflow carrier. A connector emits no second blob for a +subset of the record — a column holding just the instance-defined fields is +contained in `raw_data`, and two representations of the same values drift: +they are written by different rules and can disagree on truncation, on null +handling, and on which fields they consider in scope. A consumer that wants +only the instance-defined subset filters `raw_data` at read time. + +## Traceability + +This decision directly addresses the following requirements or design elements: + +* `cpt-insightspec-fr-cn-custom-fields` — a field with no declared column is + preserved in `raw_data` +* `cpt-insightspec-adr-connector-responsibility-scope` — the connector emits the + full payload alongside extracted fields, as that ADR requires diff --git a/docs/domain/ingestion-data-flow/specs/DESIGN.md b/docs/domain/ingestion-data-flow/specs/DESIGN.md index 798253616..5e2c59d8d 100644 --- a/docs/domain/ingestion-data-flow/specs/DESIGN.md +++ b/docs/domain/ingestion-data-flow/specs/DESIGN.md @@ -276,7 +276,7 @@ Raw API ingestion. Connector writes minimally-transformed JSON-decoded rows into ##### Why this component exists -Per-connector cleanup, type coercion, projection, and dedup. Staging models hide bronze idiosyncrasies (Airbyte JSON envelopes, raw timestamp strings, custom_fields blobs) behind a normalized per-connector schema that silver can union without further per-source logic. +Per-connector cleanup, type coercion, projection, and dedup. Staging models hide bronze idiosyncrasies (Airbyte JSON envelopes, raw timestamp strings, raw_data blobs) behind a normalized per-connector schema that silver can union without further per-source logic. ##### Responsibility scope diff --git a/src/ingestion/connectors/crm/hubspot/README.md b/src/ingestion/connectors/crm/hubspot/README.md index a3176f42d..978e85992 100644 --- a/src/ingestion/connectors/crm/hubspot/README.md +++ b/src/ingestion/connectors/crm/hubspot/README.md @@ -1,6 +1,6 @@ # HubSpot Connector -CDK-based Python connector for HubSpot CRM. Pulls live data via CRM v3 Search API with v4 associations and archived data via list + batch_read; only an allowlisted subset of `hubspotDefined` standard properties (the curated `ALLOWED_PROPERTIES_BY_OBJECT`) surfaces as typed Bronze columns. Tenant-defined (`hubspotDefined=false`) properties are folded into a single `custom_fields` JSON column so Bronze stays stable across portals and bounded in width regardless of customization depth. +CDK-based Python connector for HubSpot CRM. Pulls live data via CRM v3 Search API with v4 associations and archived data via list + batch_read; the allowlist in `ALLOWED_PROPERTIES_BY_OBJECT` surfaces as dedicated Bronze columns and defines the advertised schema statically, so the table shape is identical across portals. Every other property still ships in the `raw_data` JSON column, keeping Bronze width bounded without losing values. Streams sync sequentially. HubSpot's search endpoint is rate-limited to 4 rps portal-wide so a single thread saturates the cap; concurrency would only redistribute the same 4 rps across more 429 retries. @@ -114,7 +114,9 @@ HubSpot's CRM Search endpoint caps at `after = 10,000`. The connector sorts ever - `5xx`, chunked-encoding, connection resets — retried with exponential backoff. ### Property scope -Bronze advertises **only the curated `ALLOWED_PROPERTIES_BY_OBJECT` allowlist** of `hubspotDefined` standard properties as typed `properties_*` columns; standard properties outside the allowlist are skipped. Tenant-defined (`hubspotDefined=False`) properties land in the `custom_fields` JSON column with null/empty values dropped and per-value byte cap applied (see envelope). This keeps Bronze width bounded regardless of portal customization depth — typical width is 5–15 typed columns per object plus `custom_fields`, instead of the 50–250+ you'd get from projecting every standard property. To project a new standard column, add it to `ALLOWED_PROPERTIES_BY_OBJECT[object_type]` in `constants.py`. +Bronze advertises **exactly the `ALLOWED_PROPERTIES_BY_OBJECT` allowlist** as `properties_*` columns. The advertised schema is static — derived from `constants.py`, not from a portal describe — so every portal produces the same table shape and an allowlisted property the portal doesn't define is simply NULL. To project a new column, add it to `ALLOWED_PROPERTIES_BY_OBJECT[object_type]` in `constants.py`. + +Syncs still request **every** property the portal defines. Nothing is discarded: the full record — nested `properties` object plus association ids — is serialized into the `raw_data` JSON column, so a property with no dedicated column stays recoverable downstream. The per-value byte cap (see envelope) applies before serializing, so the blob itself stays parseable. This keeps the typed width bounded regardless of portal customization depth. ### Deleted / archived records Each archived stream runs as **client-side incremental on `archivedAt`** — page the full archived set, drop records at-or-below the prior cursor state, batch_read full properties for the survivors. After the first sync, only newly-archived rows write to Bronze. Silver UNIONs the live and archived sources and ranks rows by `greatest(updatedAt, archivedAt)` so an archive event outranks the prior live update. The `archived: true` flag is still surfaced on Silver rows via the `metadata` JSON column. diff --git a/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_accounts.sql b/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_accounts.sql index 685ffdbf4..c2471a464 100644 --- a/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_accounts.sql +++ b/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_accounts.sql @@ -44,8 +44,6 @@ WITH src AS ( 'annualrevenue', coalesce(toString(properties_annualrevenue), ''), 'archived', toString(coalesce(archived, false)) )) AS metadata, - -- Envelope parity with salesforce__crm_* (no HubSpot custom-fields blob). - '{}' AS custom_fields, createdAt AS created_at, updatedAt AS updated_at, data_source, diff --git a/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_activities.sql b/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_activities.sql index c87557360..a9705a7b6 100644 --- a/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_activities.sql +++ b/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_activities.sql @@ -70,8 +70,6 @@ WITH calls AS ( 'direction', coalesce(toString(properties_hs_call_direction), ''), 'archived', toString(coalesce(archived, false)) )) AS metadata, - -- Envelope parity with salesforce__crm_* (no HubSpot custom-fields blob). - '{}' AS custom_fields, createdAt AS created_at, data_source, greatest( @@ -114,7 +112,6 @@ emails AS ( 'direction', coalesce(toString(properties_hs_email_direction), ''), 'archived', toString(coalesce(archived, false)) )) AS metadata, - '{}' AS custom_fields, createdAt AS created_at, data_source, greatest( @@ -170,7 +167,6 @@ meetings AS ( 'location', coalesce(toString(properties_hs_meeting_location), ''), 'archived', toString(coalesce(archived, false)) )) AS metadata, - '{}' AS custom_fields, createdAt AS created_at, data_source, coalesce( @@ -212,7 +208,6 @@ tasks AS ( 'type', coalesce(toString(properties_hs_task_type), ''), 'archived', toString(coalesce(archived, false)) )) AS metadata, - '{}' AS custom_fields, createdAt AS created_at, data_source, greatest( diff --git a/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_contacts.sql b/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_contacts.sql index 2d4093440..4ad99011f 100644 --- a/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_contacts.sql +++ b/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_contacts.sql @@ -48,8 +48,6 @@ WITH src AS ( 'hs_analytics_source', coalesce(toString(properties_hs_analytics_source), ''), 'archived', toString(coalesce(archived, false)) )) AS metadata, - -- Envelope parity with salesforce__crm_* (no HubSpot custom-fields blob). - '{}' AS custom_fields, createdAt AS created_at, updatedAt AS updated_at, data_source, diff --git a/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_deals.sql b/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_deals.sql index 927d18d86..fa54b0800 100644 --- a/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_deals.sql +++ b/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_deals.sql @@ -70,8 +70,6 @@ WITH src AS ( 'deal_type', coalesce(toString(properties_dealtype), ''), 'archived', toString(coalesce(archived, false)) )) AS metadata, - -- Envelope parity with salesforce__crm_* (no HubSpot custom-fields blob). - '{}' AS custom_fields, createdAt AS created_at, updatedAt AS updated_at, data_source, diff --git a/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_users.sql b/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_users.sql index 705c94fc2..ba1eee18e 100644 --- a/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_users.sql +++ b/src/ingestion/connectors/crm/hubspot/dbt/hubspot__crm_users.sql @@ -54,8 +54,6 @@ WITH src AS ( 'userId', coalesce(toString(userId), ''), 'archived', toString(coalesce(archived, false)) )) AS metadata, - -- Envelope parity with salesforce__crm_* (no HubSpot custom-fields blob). - '{}' AS custom_fields, collected_at, data_source, greatest( diff --git a/src/ingestion/connectors/crm/hubspot/dbt/schema.yml b/src/ingestion/connectors/crm/hubspot/dbt/schema.yml index ccf0873c6..6aa484c9c 100644 --- a/src/ingestion/connectors/crm/hubspot/dbt/schema.yml +++ b/src/ingestion/connectors/crm/hubspot/dbt/schema.yml @@ -4,7 +4,7 @@ version: 2 # Table names match the HubSpot CRM object path segment (lowercase plural) # as emitted by the Airbyte ClickHouse destination. Every table has the # Insight envelope columns: -# tenant_id, source_id, unique_key, data_source, collected_at, custom_fields +# tenant_id, source_id, unique_key, data_source, collected_at, raw_data # (JSON blob of all properties where hubspotDefined=false). sources: diff --git a/src/ingestion/connectors/crm/hubspot/descriptor.yaml b/src/ingestion/connectors/crm/hubspot/descriptor.yaml index 4e65a12c4..b449b39c0 100644 --- a/src/ingestion/connectors/crm/hubspot/descriptor.yaml +++ b/src/ingestion/connectors/crm/hubspot/descriptor.yaml @@ -1,5 +1,5 @@ name: hubspot -version: "2.10.0" +version: "2.11.0" type: cdk schedule: "0 6 * * *" workflow: sync diff --git a/src/ingestion/connectors/crm/hubspot/source_hubspot/api.py b/src/ingestion/connectors/crm/hubspot/source_hubspot/api.py index 38dddead2..a380949be 100644 --- a/src/ingestion/connectors/crm/hubspot/source_hubspot/api.py +++ b/src/ingestion/connectors/crm/hubspot/source_hubspot/api.py @@ -20,11 +20,7 @@ from airbyte_cdk.sources.streams.http import HttpClient from airbyte_cdk.utils import AirbyteTracedException -from source_hubspot.constants import ( - ALLOWED_PROPERTIES_BY_OBJECT, - BASE_URL, - HUBSPOT_TYPE_TO_JSON_SCHEMA, -) +from source_hubspot.constants import ALLOWED_PROPERTIES_BY_OBJECT, BASE_URL from source_hubspot.rate_limiting import HubspotErrorHandler @@ -75,8 +71,8 @@ def __init__(self, access_token: str) -> None: ) # Per-entity describe cache: {object_type: (property dict, ...)}. - # Populated by ``properties_for()`` so ``custom_property_names()`` and - # schema generation share a single describe call per object. + # Populated by ``properties_for()`` so repeated lookups share a single + # describe call per object. self._properties_cache: Dict[str, Tuple[Mapping[str, Any], ...]] = {} # ------- Check connection (scope validation) ----------------------------- @@ -154,32 +150,14 @@ def properties_for(self, object_type: str) -> Tuple[Mapping[str, Any], ...]: self._properties_cache[object_type] = payload return payload - def custom_property_names(self, object_type: str) -> frozenset: - """Names of properties where ``hubspotDefined`` is False. + def property_names(self, object_type: str) -> Tuple[str, ...]: + """Every property the portal defines for ``object_type``. - These get routed into the ``custom_fields`` JSON blob by the envelope, - keeping Bronze schema stable across portals with different customizations. + Both consumers send this list in a JSON request body (search, archived + batch_read), so there's no URL-length ceiling to ration against. """ props = self.properties_for(object_type) - return frozenset( - p["name"] - for p in props - if p.get("name") and not p.get("hubspotDefined") - ) - - def property_names(self, object_type: str) -> Tuple[str, ...]: - """Curated standard properties + all custom (tenant-defined) properties.""" - props = self.properties_for(object_type) - curated = ALLOWED_PROPERTIES_BY_OBJECT.get(object_type, frozenset()) - selected = [] - for p in props: - name = p.get("name") - if not name: - continue - if p.get("hubspotDefined") and name not in curated: - continue - selected.append(name) - return tuple(selected) + return tuple(p["name"] for p in props if p.get("name")) def probe_association_scope(self) -> Optional[str]: """Verify the token has association read scope. @@ -211,8 +189,12 @@ def probe_association_scope(self) -> Optional[str]: return None def generate_schema(self, object_type: str) -> Mapping[str, Any]: - """Build a JSON schema from the curated property descriptors.""" - props = self.properties_for(object_type) + """Build the advertised JSON schema for ``object_type``. + + Derived entirely from ``ALLOWED_PROPERTIES_BY_OBJECT`` — no describe + call — so the Bronze table shape is identical across portals. An + allowlisted property a portal doesn't define simply stays NULL. + """ schema: Dict[str, Any] = { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", @@ -225,37 +207,10 @@ def generate_schema(self, object_type: str) -> Mapping[str, Any]: "archivedAt": {"type": ["string", "null"], "format": "date-time"}, }, } - curated = ALLOWED_PROPERTIES_BY_OBJECT.get(object_type, frozenset()) - warned_unknown: set = set() - for prop in props: - name = prop.get("name") - if not name or not prop.get("hubspotDefined"): - continue - if name not in curated: - continue - schema["properties"][f"properties_{name}"] = _prop_to_json_schema( - prop, warned_unknown - ) - return schema - + # Every property column is a string: HubSpot returns all property + # values as JSON strings, so a typed column would make the destination + # NULL every row it can't deserialize. dbt coerces downstream. + for name in sorted(ALLOWED_PROPERTIES_BY_OBJECT.get(object_type, frozenset())): + schema["properties"][f"properties_{name}"] = {"type": ["string", "null"]} -def _prop_to_json_schema( - prop: Mapping[str, Any], warned_unknown: set -) -> Mapping[str, Any]: - """Map a HubSpot property descriptor to a JSON-schema property.""" - hs_type = (prop.get("type") or "string").lower() - mapped = HUBSPOT_TYPE_TO_JSON_SCHEMA.get(hs_type) - if not mapped: - if hs_type not in warned_unknown: - logger.warning( - "Unknown HubSpot property type %r on %r; falling back to string", - hs_type, - prop.get("name"), - ) - warned_unknown.add(hs_type) - mapped = ("string", None) - json_type, fmt = mapped - out: Dict[str, Any] = {"type": [json_type, "null"]} - if fmt: - out["format"] = fmt - return out + return schema diff --git a/src/ingestion/connectors/crm/hubspot/source_hubspot/constants.py b/src/ingestion/connectors/crm/hubspot/source_hubspot/constants.py index 85a3e01e5..a4c06c8f4 100644 --- a/src/ingestion/connectors/crm/hubspot/source_hubspot/constants.py +++ b/src/ingestion/connectors/crm/hubspot/source_hubspot/constants.py @@ -1,4 +1,4 @@ -"""HubSpot stream registry, property-type mapping, and API limits.""" +"""HubSpot stream registry, Bronze column allowlist, and API limits.""" from typing import FrozenSet, Mapping @@ -27,39 +27,6 @@ # request bodies small enough that a 429 retry doesn't replay a big payload. ASSOCIATIONS_BATCH_SIZE = 100 -# ------- Property-type mapping (describe -> JSON schema) --------------------- - -# HubSpot property type -> (json-schema type, optional format). -# Any type not listed falls back to string with a one-time warning. -HUBSPOT_TYPE_TO_JSON_SCHEMA: Mapping[str, tuple] = { - # HubSpot's CRM v3 Search API returns every property value as a JSON - # string — booleans come back as "true"/"false", numbers as "1234.56", - # datetimes as ISO strings (sometimes epoch-millis strings on legacy - # properties). Declaring anything other than ("string", None) makes - # the destination build a typed column (Bool/Decimal/DateTime64) and - # silently NULL every row whose value can't deserialize, with - # _airbyte_meta.changes recording DESTINATION_SERIALIZATION_ERROR. - # Observed losses: ~100% on deals.hs_is_closed/won, all values on - # companies.numberofemployees range strings ("500-1000"), tasks - # legacy completion dates, etc. - # - # Bronze stays as Nullable(String) for every property; dbt coerces - # downstream (toInt64OrNull, toFloat64OrNull, - # parseDateTime64BestEffortOrNull). Lossless storage, - # parser-failure isolation per row instead of silent NULL. - "string": ("string", None), - "bool": ("string", None), - "boolean": ("string", None), - "enumeration": ("string", None), - "date": ("string", None), - "datetime": ("string", None), - "date-time": ("string", None), - "number": ("string", None), - "json": ("string", None), - "object_coordinates": ("string", None), - "phone_number": ("string", None), -} - # ------- Cloudflare oddity --------------------------------------------------- # HubSpot fronts the API via Cloudflare; an invalid token format (e.g. wrong @@ -212,10 +179,11 @@ def _derive_archived(name: str, entry: Mapping) -> Mapping: # Stream-name suffix used by the source to derive archived siblings. ARCHIVED_STREAM_SUFFIX = _ARCHIVED_STREAM_SUFFIX -# Property scope: standard (``hubspotDefined``) properties are filtered -# through ``ALLOWED_PROPERTIES_BY_OBJECT`` so Bronze width stays bounded. -# Tenant-defined (``hubspotDefined=False``) properties always pass through -# and ride in ``custom_fields`` JSON. +# Bronze column scope: these properties get a dedicated +# ``properties_{name}`` column, and they alone define the advertised schema — +# it is identical for every portal, and a property the portal doesn't define +# stays NULL. Every other property the portal returns still ships, inside +# ``raw_data``. ALLOWED_PROPERTIES_BY_OBJECT: Mapping[str, FrozenSet[str]] = { "contacts": frozenset({ "email", "firstname", "lastname", "hubspot_owner_id", "lifecyclestage", diff --git a/src/ingestion/connectors/crm/hubspot/source_hubspot/envelope.py b/src/ingestion/connectors/crm/hubspot/source_hubspot/envelope.py index 8e6fb5802..e5c0abf1a 100644 --- a/src/ingestion/connectors/crm/hubspot/source_hubspot/envelope.py +++ b/src/ingestion/connectors/crm/hubspot/source_hubspot/envelope.py @@ -2,9 +2,9 @@ Every record emitted to Bronze is augmented with tenant / source scope and a deterministic ``unique_key`` so downstream dbt models can key off a single -stable identifier. HubSpot custom properties (``hubspotDefined=false``) are -pulled out into a single JSON blob so the Bronze schema stays stable across -portals with different customizations. +stable identifier. Only allowlisted properties get a top-level column, so the +Bronze schema stays identical across portals with different customizations; +everything else the portal returns survives inside ``raw_data``. """ import hashlib @@ -21,7 +21,14 @@ # (unlikely but possible with custom flat-top properties) would otherwise be # silently overwritten; we log and drop it instead. _RESERVED_FIELD_NAMES = frozenset( - {"tenant_id", "source_id", "unique_key", "data_source", "collected_at", "custom_fields"} + { + "tenant_id", + "source_id", + "unique_key", + "data_source", + "collected_at", + "raw_data", + } ) # Per-property string truncation cap. Bounds the worst-case row width @@ -50,6 +57,15 @@ def _truncate(value: Any) -> Any: return encoded[:allowed].decode("utf-8", errors="ignore") + _TRUNCATED_SUFFIX +def _truncate_deep(value: Any) -> Any: + """Copy ``value``, truncating every string it contains.""" + if isinstance(value, Mapping): + return {k: _truncate_deep(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_truncate_deep(v) for v in value] + return _truncate(value) + + def _now_iso() -> str: return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") @@ -59,7 +75,7 @@ def envelope( *, tenant_id: str, source_id: str, - custom_property_names: frozenset, + allowed_property_names: frozenset, collision_seen: Optional[MutableSet[str]] = None, ) -> MutableMapping[str, Any]: """Return a copy of ``record`` with Insight metadata injected. @@ -70,11 +86,13 @@ def envelope( "properties": {...}, "associations": {...}} The envelope: - - Flattens ``properties`` into top-level ``properties_{name}`` columns. - - Splits custom properties (names in ``custom_property_names``) into the - ``custom_fields`` JSON blob, keeping Bronze schema stable across portals. + - Flattens the properties named in ``allowed_property_names`` into + top-level ``properties_{name}`` columns — and only those, so the emitted + key set matches the advertised schema on every portal. - Keeps ``associations`` as-is (already flattened to id-array form by the association helper before this call). + - Serializes the whole incoming record into ``raw_data`` so properties + outside the advertised schema are still recoverable downstream. - Adds ``tenant_id`` / ``source_id`` / ``unique_key`` / ``data_source`` / ``collected_at``. @@ -82,7 +100,6 @@ def envelope( collisions. """ out: MutableMapping[str, Any] = {} - customs: dict = {} properties = record.get("properties") or {} for key, value in record.items(): @@ -94,23 +111,18 @@ def envelope( out[key] = value for prop_name, prop_value in properties.items(): - # HubSpot property names always land under a ``properties_`` prefix so - # they can't collide with the unprefixed envelope reserved names; no - # collision check needed here. - if prop_name in custom_property_names: - # Drop null/empty values — HubSpot returns every defined custom - # property on every record, so a portal with hundreds of custom - # fields (mostly empty per row) would otherwise bloat the JSON - # blob 10–20× with dead keys. - if prop_value is None or prop_value == "": - continue - customs[prop_name] = _truncate(prop_value) - else: - out[f"properties_{prop_name}"] = _truncate(prop_value) - - # ClickHouse stores JSON blobs as strings; serialize once. - out["custom_fields"] = ( - json.dumps(customs, separators=(",", ":"), default=str) if customs else "{}" + # A column the advertised schema never declared would be drift; + # ``raw_data`` carries what the allowlist leaves out. + if prop_name not in allowed_property_names: + continue + # The ``properties_`` prefix rules out collision with the unprefixed + # reserved names, so no collision check is needed here. + out[f"properties_{prop_name}"] = _truncate(prop_value) + + # Truncation is per value, never on the blob — clipping the serialized + # JSON would leave it unparseable. + out["raw_data"] = json.dumps( + _truncate_deep(record), separators=(",", ":"), default=str ) hs_id = record.get("id") @@ -154,7 +166,7 @@ def _warn_once(seen: Optional[MutableSet[str]], key: str) -> None: "unique_key": {"type": "string"}, "data_source": {"type": "string"}, "collected_at": {"type": "string", "format": "date-time"}, - "custom_fields": {"type": "string"}, + "raw_data": {"type": "string"}, } diff --git a/src/ingestion/connectors/crm/hubspot/source_hubspot/streams.py b/src/ingestion/connectors/crm/hubspot/source_hubspot/streams.py index e089148cc..c0b3a0e1a 100644 --- a/src/ingestion/connectors/crm/hubspot/source_hubspot/streams.py +++ b/src/ingestion/connectors/crm/hubspot/source_hubspot/streams.py @@ -43,6 +43,7 @@ from source_hubspot.api import Hubspot, _TimeoutSession from source_hubspot.associations import AssociationFetcher from source_hubspot.constants import ( + ALLOWED_PROPERTIES_BY_OBJECT, BASE_URL, BATCH_READ_LIMIT, LIST_PAGE_LIMIT, @@ -84,6 +85,11 @@ def __init__( self._hubspot = hubspot_api self._registry = STREAM_REGISTRY[stream_name] self._object_type = self._registry["object_type"] + # Same allowlist ``generate_schema`` declares columns from, so the + # ``properties_*`` keys a record carries can't drift from the schema. + self._allowed_property_names = ALLOWED_PROPERTIES_BY_OBJECT.get( + self._object_type, frozenset() + ) self._tenant_id = tenant_id self._source_id = source_id self._start_date = start_date @@ -194,13 +200,14 @@ def _record_cursor( def get_json_schema(self) -> Mapping[str, Any]: """Advertise per-stream schema to the destination. - - Start from describe-generated schema (every hubspotDefined property). + - Start from the static allowlist schema (no portal describe involved, + so every portal advertises the same columns). - Add the envelope fields so ClickHouse creates columns for them. - Add ``associations_{to_object_type}`` arrays when applicable. - - ``custom_fields`` JSON blob is added by inject_envelope_properties. + - The ``raw_data`` blob is added by inject_envelope_properties. """ - # Deep copy so envelope and association-props loop don't mutate the - # describe cache shared across streams. + # Deep copy so the mutations below can't reach anything the api client + # might hand out more than once. schema = copy.deepcopy(self._hubspot.generate_schema(self._object_type)) schema = inject_envelope_properties(schema) props = schema.setdefault("properties", {}) @@ -226,8 +233,6 @@ def read_records( if stream_state and self.cursor_field: self.state = stream_state # type: ignore[assignment] - custom_names = self._hubspot.custom_property_names(self._object_type) - latest_cursor: Optional[pendulum.DateTime] = None batch: List[MutableMapping[str, Any]] = [] for record in self._generate_records(sync_mode, stream_slice, stream_state): @@ -238,17 +243,16 @@ def read_records( latest_cursor = cursor_value batch.append(dict(record)) if len(batch) >= SEARCH_PAGE_LIMIT: - yield from self._finalize_batch(batch, custom_names) + yield from self._finalize_batch(batch) batch = [] if batch: - yield from self._finalize_batch(batch, custom_names) + yield from self._finalize_batch(batch) self._advance_state(latest_cursor) def _finalize_batch( self, batch: List[MutableMapping[str, Any]], - custom_names: frozenset, ) -> Iterable[MutableMapping[str, Any]]: if self._associations is not None: self._associations.enrich(batch) @@ -257,7 +261,7 @@ def _finalize_batch( record, tenant_id=self._tenant_id, source_id=self._source_id, - custom_property_names=custom_names, + allowed_property_names=self._allowed_property_names, collision_seen=self._envelope_collisions_seen, ) @@ -489,15 +493,12 @@ def read_records( stream_slice: Optional[Mapping[str, Any]] = None, stream_state: Optional[Mapping[str, Any]] = None, ) -> Iterable[StreamData]: - """Envelope owners without touching the CRM properties endpoint. - - Owners have no ``/crm/v3/properties/owners`` endpoint and no - custom-field surface, so the base :class:`HubspotStream.read_records` - path (which calls ``self._hubspot.custom_property_names`` and batches - through :func:`_finalize_batch`) doesn't apply. Stream directly from - :meth:`_generate_records`, envelope with an empty custom-field set, - and skip association enrichment (owners have none). State advance is - applied at the end via the same cursor-tracking pattern. + """Envelope owners one at a time, without the batch buffer. + + Owners carry no associations, so there is nothing to batch-enrich; + records stream straight from :meth:`_generate_records` to the + envelope. State advance follows the same cursor-tracking pattern as + the base class. """ if stream_state and self.cursor_field: self.state = stream_state # type: ignore[assignment] @@ -513,7 +514,7 @@ def read_records( record, tenant_id=self._tenant_id, source_id=self._source_id, - custom_property_names=frozenset(), + allowed_property_names=self._allowed_property_names, collision_seen=self._envelope_collisions_seen, ) self._advance_state(latest_cursor) diff --git a/src/ingestion/connectors/crm/hubspot/tests/conftest.py b/src/ingestion/connectors/crm/hubspot/tests/conftest.py index 0cc7495a7..9aad212dd 100644 --- a/src/ingestion/connectors/crm/hubspot/tests/conftest.py +++ b/src/ingestion/connectors/crm/hubspot/tests/conftest.py @@ -95,16 +95,12 @@ def send_request( class FakeHubspot: """Describe-time api stub: canned property descriptors, no HTTP.""" - def __init__(self, names: Iterable[str] = ("amount", "my_custom"), custom: Iterable[str] = ("my_custom",)): + def __init__(self, names: Iterable[str] = ("amount", "my_custom")): self._names = tuple(names) - self._custom = frozenset(custom) def property_names(self, object_type: str) -> tuple: return self._names - def custom_property_names(self, object_type: str) -> frozenset: - return self._custom - def generate_schema(self, object_type: str) -> Mapping[str, Any]: return { "$schema": "http://json-schema.org/draft-07/schema#", diff --git a/src/ingestion/connectors/crm/hubspot/tests/test_api.py b/src/ingestion/connectors/crm/hubspot/tests/test_api.py index 19fad01c5..0375138e1 100644 --- a/src/ingestion/connectors/crm/hubspot/tests/test_api.py +++ b/src/ingestion/connectors/crm/hubspot/tests/test_api.py @@ -8,9 +8,8 @@ import requests from airbyte_cdk.models import FailureType from airbyte_cdk.utils import AirbyteTracedException -from source_hubspot import api as api_mod -from source_hubspot.api import Hubspot, _prop_to_json_schema, _TimeoutSession -from source_hubspot.constants import BASE_URL +from source_hubspot.api import Hubspot, _TimeoutSession +from source_hubspot.constants import ALLOWED_PROPERTIES_BY_OBJECT, BASE_URL from tests.conftest import FakeHttpClient, FakeResponse @@ -118,77 +117,43 @@ def test_unexpected_shape_is_system_error(self): class TestPropertySelection: DESCRIPTORS = [ - prop("amount"), # hubspotDefined + curated → kept - prop("uncurated_std"), # hubspotDefined, not curated → dropped - prop("my_custom", hubspot_defined=False), # custom → always kept + prop("amount"), # allowlisted standard + prop("uncurated_std"), # standard outside the allowlist + prop("my_custom", hubspot_defined=False), {"hubspotDefined": True}, # nameless → dropped ] - def test_property_names_curated_plus_custom(self): + def test_every_named_portal_property_is_requested(self): hs = make_client([FakeResponse({"results": self.DESCRIPTORS})]) - assert hs.property_names("deals") == ("amount", "my_custom") + assert hs.property_names("deals") == ("amount", "uncurated_std", "my_custom") - def test_custom_property_names(self): - hs = make_client([FakeResponse({"results": self.DESCRIPTORS})]) - assert hs.custom_property_names("deals") == frozenset({"my_custom"}) - - def test_unknown_object_has_no_curated_names(self): + def test_property_names_independent_of_allowlist(self): hs = make_client([FakeResponse({"results": [prop("anything")]})]) - assert hs.property_names("unknown_object") == () + assert hs.property_names("unknown_object") == ("anything",) class TestGenerateSchema: - def test_curated_props_added_with_string_type(self): - hs = make_client( - [ - FakeResponse( - { - "results": [ - prop("amount", type_="number"), - prop("uncurated_std"), - prop("my_custom", hubspot_defined=False), - ] - } - ) - ] - ) - schema = hs.generate_schema("deals") - props = schema["properties"] - # Base record fields always present. + def test_allowlist_is_the_whole_property_column_set(self): + props = make_client().generate_schema("deals")["properties"] + expected = {f"properties_{name}" for name in ALLOWED_PROPERTIES_BY_OBJECT["deals"]} + assert {k for k in props if k.startswith("properties_")} == expected + assert all(props[k] == {"type": ["string", "null"]} for k in expected) + + def test_schema_needs_no_portal_describe(self): + hs = make_client() # no queued responses: any HTTP call raises + hs.generate_schema("deals") + assert hs._http_client.calls == [] + + def test_base_record_fields_always_present(self): + props = make_client().generate_schema("leads")["properties"] assert props["id"] == {"type": ["string", "null"]} + assert props["archived"] == {"type": ["boolean", "null"]} assert props["archivedAt"]["format"] == "date-time" - # number maps to string on purpose (Bronze stays Nullable(String)). - assert props["properties_amount"] == {"type": ["string", "null"]} - assert "properties_uncurated_std" not in props - assert "properties_my_custom" not in props # customs ride in custom_fields - - def test_unknown_type_warns_once(self, caplog): - hs = make_client( - [ - FakeResponse( - { - "results": [ - {"name": "amount", "hubspotDefined": True, "type": "alien"}, - {"name": "closedate", "hubspotDefined": True, "type": "alien"}, - ] - } - ) - ] - ) - with caplog.at_level(logging.WARNING, logger="airbyte"): - schema = hs.generate_schema("deals") - assert schema["properties"]["properties_amount"] == {"type": ["string", "null"]} - assert caplog.text.count("Unknown HubSpot property type") == 1 - - def test_prop_to_json_schema_format_passthrough(self, monkeypatch): - # No current mapping carries a format — patch one in to cover the - # format branch. - monkeypatch.setitem(api_mod.HUBSPOT_TYPE_TO_JSON_SCHEMA, "datetime", ("string", "date-time")) - out = _prop_to_json_schema({"name": "x", "type": "datetime"}, set()) - assert out == {"type": ["string", "null"], "format": "date-time"} - - def test_prop_to_json_schema_defaults_missing_type_to_string(self): - assert _prop_to_json_schema({"name": "x"}, set()) == {"type": ["string", "null"]} + + @pytest.mark.parametrize("object_type", ["leads", "unknown_object"]) + def test_empty_allowlist_yields_base_fields_only(self, object_type): + props = make_client().generate_schema(object_type)["properties"] + assert not [k for k in props if k.startswith("properties_")], f"object: {object_type}" class TestProbeAssociationScope: diff --git a/src/ingestion/connectors/crm/hubspot/tests/test_archived_stream.py b/src/ingestion/connectors/crm/hubspot/tests/test_archived_stream.py index 67fee8dc1..5b9f6c413 100644 --- a/src/ingestion/connectors/crm/hubspot/tests/test_archived_stream.py +++ b/src/ingestion/connectors/crm/hubspot/tests/test_archived_stream.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json + from source_hubspot import streams as streams_mod from source_hubspot.constants import BASE_URL from tests.conftest import SOURCE, TENANT, FakeResponse, wire @@ -129,15 +131,17 @@ def test_envelope_and_state_advance(self, companies_archived_stream): list_page([stub(1, archived_at="2024-06-05T00:00:00Z"), stub(2, archived_at="2024-06-07T00:00:00Z")]), batch_result( [ - {"id": "1", "properties": {"amount": "5", "my_custom": "c"}}, - {"id": "2", "properties": {"amount": "6"}}, + {"id": "1", "properties": {"name": "Example Corp", "my_custom": "c"}}, + {"id": "2", "properties": {"name": "Other Corp"}}, ] ), ], ) out = list(companies_archived_stream.read_records(sync_mode=None)) assert out[0]["unique_key"] == f"{TENANT}-{SOURCE}-1" - assert out[0]["properties_amount"] == "5" - assert out[0]["custom_fields"] == '{"my_custom":"c"}' + assert out[0]["properties_name"] == "Example Corp" + # ``my_custom`` is outside the companies allowlist — raw_data only. + assert "properties_my_custom" not in out[0] + assert json.loads(out[0]["raw_data"])["properties"]["my_custom"] == "c" # State advanced to max archivedAt (from the Pass-1 overlay). assert companies_archived_stream.state == {"archivedAt": "2024-06-07T00:00:00Z"} diff --git a/src/ingestion/connectors/crm/hubspot/tests/test_base_stream.py b/src/ingestion/connectors/crm/hubspot/tests/test_base_stream.py index 7dc4835bd..0c318b9be 100644 --- a/src/ingestion/connectors/crm/hubspot/tests/test_base_stream.py +++ b/src/ingestion/connectors/crm/hubspot/tests/test_base_stream.py @@ -94,17 +94,25 @@ def test_schema_has_envelope_and_association_columns(self, deals_stream): schema = deals_stream.get_json_schema() props = schema["properties"] assert props["properties_amount"] == {"type": ["string", "null"]} - for field in ("tenant_id", "source_id", "unique_key", "data_source", "collected_at", "custom_fields"): - assert field in props + for field in ( + "tenant_id", + "source_id", + "unique_key", + "data_source", + "collected_at", + "raw_data", + ): + assert field in props, f"missing envelope column: {field}" + assert "custom_fields" not in props # deals registry declares [companies, contacts] assert props["associations_companies"]["type"] == ["array", "null"] assert props["associations_contacts"]["items"] == {"type": "string"} - def test_schema_does_not_mutate_describe_cache(self, deals_stream): - cached = deals_stream._hubspot.generate_schema("deals") - before = dict(cached["properties"]) + def test_schema_does_not_mutate_the_api_result(self, deals_stream): + source = deals_stream._hubspot.generate_schema("deals") + before = dict(source["properties"]) deals_stream.get_json_schema() - assert cached["properties"] == before # deepcopy protects the cache + assert source["properties"] == before # deepcopy keeps mutations local def test_no_association_columns_without_associations(self, companies_stream): props = companies_stream.get_json_schema()["properties"] @@ -153,9 +161,9 @@ def test_flushes_in_page_limit_batches(self, companies_stream, monkeypatch): flushed: list[int] = [] original = companies_stream._finalize_batch - def spy(batch, custom_names): + def spy(batch): flushed.append(len(batch)) - return original(batch, custom_names) + return original(batch) companies_stream._finalize_batch = spy records = [{"id": str(i), "updatedAt": f"2024-06-0{i}T00:00:00Z", "properties": {}} for i in (1, 2, 3)] diff --git a/src/ingestion/connectors/crm/hubspot/tests/test_envelope.py b/src/ingestion/connectors/crm/hubspot/tests/test_envelope.py index f1e237868..2f4dda810 100644 --- a/src/ingestion/connectors/crm/hubspot/tests/test_envelope.py +++ b/src/ingestion/connectors/crm/hubspot/tests/test_envelope.py @@ -1,4 +1,4 @@ -"""Envelope: property flattening, custom-field routing, truncation, unique_key.""" +"""Envelope: allowlisted property flattening, truncation, raw_data, unique_key.""" from __future__ import annotations @@ -6,11 +6,13 @@ import logging from source_hubspot import envelope as envelope_mod +from source_hubspot.api import Hubspot +from source_hubspot.constants import ALLOWED_PROPERTIES_BY_OBJECT from source_hubspot.envelope import _truncate, envelope, inject_envelope_properties -def wrap(record, custom=frozenset(), seen=None): - return envelope(record, tenant_id="T", source_id="S", custom_property_names=custom, collision_seen=seen) +def wrap(record, allowed=frozenset({"amount"}), seen=None): + return envelope(record, tenant_id="T", source_id="S", allowed_property_names=allowed, collision_seen=seen) class TestEnvelope: @@ -26,18 +28,33 @@ def test_flattens_properties_and_adds_metadata(self): # collected_at is a UTC second-precision ISO timestamp. assert out["collected_at"].endswith("Z") - def test_custom_properties_go_to_json_blob(self): - out = wrap({"id": "1", "properties": {"amount": "10", "my_custom": "x"}}, custom=frozenset({"my_custom"})) + def test_property_outside_allowlist_reaches_raw_data_only(self): + out = wrap({"id": "1", "properties": {"amount": "10", "my_custom": "x", "uncurated_std": "y"}}) + assert out["properties_amount"] == "10" assert "properties_my_custom" not in out - assert json.loads(out["custom_fields"]) == {"my_custom": "x"} - - def test_empty_custom_values_dropped(self): - out = wrap({"id": "1", "properties": {"a": None, "b": "", "c": "kept"}}, custom=frozenset({"a", "b", "c"})) - assert json.loads(out["custom_fields"]) == {"c": "kept"} - - def test_no_customs_serializes_empty_object(self): - out = wrap({"id": "1", "properties": {}}) - assert out["custom_fields"] == "{}" + assert "properties_uncurated_std" not in out + raw_properties = json.loads(out["raw_data"])["properties"] + assert raw_properties["my_custom"] == "x" + assert raw_properties["uncurated_std"] == "y" + + def test_allowlisted_property_keeps_its_column_when_empty(self): + out = wrap({"id": "1", "properties": {"amount": None, "dealname": ""}}, allowed=frozenset({"amount", "dealname"})) + assert out["properties_amount"] is None + assert out["properties_dealname"] == "" + + def test_allowlisted_property_absent_from_record_gets_no_column(self): + out = wrap({"id": "1", "properties": {"amount": "10"}}, allowed=frozenset({"amount", "dealname"})) + assert "properties_dealname" not in out + + def test_emitted_property_keys_are_a_subset_of_the_declared_schema(self): + hubspot = Hubspot("pat-test-token") + for object_type, allowlist in ALLOWED_PROPERTIES_BY_OBJECT.items(): + declared = set(hubspot.generate_schema(object_type)["properties"]) + record = {"id": "1", "properties": {name: "v" for name in allowlist} | {"undeclared": "v"}} + emitted = {k for k in wrap(record, allowed=allowlist) if k.startswith("properties_")} + + assert emitted == {f"properties_{name}" for name in allowlist}, f"object: {object_type}" + assert emitted <= declared, f"object: {object_type}" def test_missing_properties_key_tolerated(self): out = wrap({"id": "1"}) @@ -104,11 +121,47 @@ def test_tiny_cap_returns_suffix_only(self, monkeypatch): monkeypatch.setattr(envelope_mod, "_VALUE_MAX_BYTES", 5) assert _truncate("x" * 100) == "…[truncated]" - def test_applied_to_flat_and_custom_properties(self): + def test_applied_to_property_columns(self): long = "y" * 5000 - out = wrap({"id": "1", "properties": {"amount": long, "my_custom": long}}, custom=frozenset({"my_custom"})) + out = wrap({"id": "1", "properties": {"amount": long}}) assert out["properties_amount"].endswith("…[truncated]") - assert json.loads(out["custom_fields"])["my_custom"].endswith("…[truncated]") + + +class TestRawData: + def test_keeps_record_shape_as_received(self): + record = { + "id": "1", + "updatedAt": "2024-06-01T00:00:00Z", + "properties": {"amount": "10", "uncurated_std": "kept", "my_custom": "x"}, + "associations_companies": ["7", "8"], + } + raw = json.loads(wrap(record)["raw_data"]) + assert raw["id"] == "1" + assert raw["properties"] == {"amount": "10", "uncurated_std": "kept", "my_custom": "x"} + assert raw["associations_companies"] == ["7", "8"] + + def test_present_without_properties_or_associations(self): + assert json.loads(wrap({"id": "1"})["raw_data"]) == {"id": "1"} + + def test_truncates_values_not_the_blob(self): + long = "y" * 5000 + out = wrap({"id": "1", "properties": {"amount": long}, "note": long}) + raw = json.loads(out["raw_data"]) # must stay parseable + assert raw["properties"]["amount"].endswith("…[truncated]") + assert raw["note"].endswith("…[truncated]") + # Two capped values plus the record scaffolding exceed the per-value cap. + assert len(out["raw_data"].encode("utf-8")) > 2048 + + def test_source_record_left_unmodified(self): + record = {"id": "1", "properties": {"amount": "y" * 5000}} + wrap(record) + assert record["properties"]["amount"] == "y" * 5000 + + def test_colliding_source_field_still_captured(self, caplog): + with caplog.at_level(logging.WARNING, logger="airbyte"): + out = wrap({"id": "1", "raw_data": "SOURCE"}) + assert json.loads(out["raw_data"])["raw_data"] == "SOURCE" + assert "collides with Insight envelope field" in caplog.text class TestInjectEnvelopeProperties: @@ -116,8 +169,16 @@ def test_adds_envelope_fields(self): schema = {"type": "object", "properties": {"id": {"type": "string"}}} out = inject_envelope_properties(schema) assert out is schema # mutates and returns the same mapping - for field in ("tenant_id", "source_id", "unique_key", "data_source", "collected_at", "custom_fields"): - assert field in schema["properties"] + for field in ( + "tenant_id", + "source_id", + "unique_key", + "data_source", + "collected_at", + "raw_data", + ): + assert field in schema["properties"], f"missing envelope field: {field}" + assert "custom_fields" not in schema["properties"] assert schema["properties"]["id"] == {"type": "string"} def test_creates_properties_when_absent(self): diff --git a/src/ingestion/connectors/crm/hubspot/tests/test_owners_streams.py b/src/ingestion/connectors/crm/hubspot/tests/test_owners_streams.py index 22f2ec8d9..296924c99 100644 --- a/src/ingestion/connectors/crm/hubspot/tests/test_owners_streams.py +++ b/src/ingestion/connectors/crm/hubspot/tests/test_owners_streams.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json + from source_hubspot.constants import BASE_URL from tests.conftest import SOURCE, TENANT, FakeResponse, wire @@ -22,7 +24,9 @@ def test_hardcoded_schema_with_envelope(self, owners_stream): props = owners_stream.get_json_schema()["properties"] assert props["email"] == {"type": ["string", "null"]} assert props["archivedAt"]["format"] == "date-time" - assert "unique_key" in props and "custom_fields" in props + for field in ("unique_key", "raw_data"): + assert field in props, f"missing envelope column: {field}" + assert "custom_fields" not in props def test_archived_stream_inherits_schema(self, owners_archived_stream): assert "userId" in owners_archived_stream.get_json_schema()["properties"] @@ -84,7 +88,7 @@ def test_envelope_and_state_advance(self, owners_stream): out = list(owners_stream.read_records(sync_mode=None)) assert out[0]["unique_key"] == f"{TENANT}-{SOURCE}-1" assert out[0]["tenant_id"] == TENANT - assert out[0]["custom_fields"] == "{}" # owners have no custom fields + assert json.loads(out[0]["raw_data"])["email"] == "o1@x" assert owners_stream.state == {"updatedAt": "2024-06-03T00:00:00Z"} def test_incoming_stream_state_applied(self, owners_stream): diff --git a/src/ingestion/connectors/crm/hubspot/tests/test_search_stream.py b/src/ingestion/connectors/crm/hubspot/tests/test_search_stream.py index 2f356603a..a469940bb 100644 --- a/src/ingestion/connectors/crm/hubspot/tests/test_search_stream.py +++ b/src/ingestion/connectors/crm/hubspot/tests/test_search_stream.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import logging from source_hubspot.constants import BASE_URL, SEARCH_PAGE_LIMIT @@ -161,9 +162,9 @@ def test_envelope_and_association_enrichment(self, deals_stream): assert first["unique_key"] == f"{TENANT}-{SOURCE}-1" assert first["data_source"] == "hubspot" assert first["properties_amount"] == "10" - # custom property routed into the JSON blob, not a flat column + # property outside the allowlist gets no column — raw_data carries it assert "properties_my_custom" not in first - assert first["custom_fields"] == '{"my_custom":"x"}' + assert json.loads(first["raw_data"])["properties"]["my_custom"] == "x" assert first["associations_companies"] == ["900", "901"] assert first["associations_contacts"] == [] # record 2 absent from the association response keeps empty arrays diff --git a/src/ingestion/connectors/crm/hubspot/tests/test_source.py b/src/ingestion/connectors/crm/hubspot/tests/test_source.py index 8103ba9f5..5c835c14f 100644 --- a/src/ingestion/connectors/crm/hubspot/tests/test_source.py +++ b/src/ingestion/connectors/crm/hubspot/tests/test_source.py @@ -47,9 +47,6 @@ def probe_association_scope(self): def property_names(self, object_type): return () - def custom_property_names(self, object_type): - return frozenset() - def generate_schema(self, object_type): return {"type": "object", "properties": {}} diff --git a/src/ingestion/connectors/crm/salesforce/README.md b/src/ingestion/connectors/crm/salesforce/README.md index 6df5ac43b..5f68db088 100644 --- a/src/ingestion/connectors/crm/salesforce/README.md +++ b/src/ingestion/connectors/crm/salesforce/README.md @@ -1,6 +1,6 @@ # Salesforce Connector -CDK-based Python connector for Salesforce CRM. Pulls data via the REST `/queryAll` API; describe-driven field discovery means no SOQL maintenance as SF orgs evolve; custom (`__c`) fields are captured into a single `custom_fields` JSON column so Bronze stays stable across orgs. +CDK-based Python connector for Salesforce CRM. Pulls data via the REST `/queryAll` API; describe-driven field discovery means no SOQL maintenance as SF orgs evolve; static per-stream schemas keep Bronze identical across orgs, and every field outside a stream's schema — custom (`__c`) fields included — is preserved in the `raw_data` JSON column. ## Prerequisites @@ -102,7 +102,7 @@ Every stream's Bronze table has: - **`unique_key`** — `{tenant_id}-{source_id}-{Id}` — stable surrogate PK. - **`data_source`** — literal `"salesforce"`. - **`collected_at`** — UTC ISO-8601 timestamp of the sync. -- **`custom_fields`** — JSON string containing every `__c` field. Access in dbt via `JSONExtractString(custom_fields, 'MyField__c')`. +- **`raw_data`** — JSON string containing the whole source record, including every field with no dedicated column. Access in dbt via `JSONExtractString(raw_data, 'MyField__c')`. ## Silver targets diff --git a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_accounts.sql b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_accounts.sql index 72eac4119..d18af51bf 100644 --- a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_accounts.sql +++ b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_accounts.sql @@ -29,7 +29,6 @@ WITH src AS ( 'AnnualRevenue', coalesce(toString(AnnualRevenue), ''), 'IsDeleted', toString(coalesce(IsDeleted, false)) )) AS metadata, - coalesce(custom_fields, '{}') AS custom_fields, CreatedDate AS created_at, LastModifiedDate AS updated_at, data_source, diff --git a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_activities.sql b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_activities.sql index 6ab316e92..10e7b4a71 100644 --- a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_activities.sql +++ b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_activities.sql @@ -48,7 +48,6 @@ WITH tasks AS ( 'CallType', coalesce(toString(CallType), ''), 'IsDeleted', toString(coalesce(IsDeleted, false)) )) AS metadata, - coalesce(custom_fields, '{}') AS custom_fields, CreatedDate AS created_at, data_source, coalesce(toUnixTimestamp64Milli(SystemModstamp), 0) AS _version @@ -93,7 +92,6 @@ events AS ( 'EventSubtype', coalesce(toString(EventSubtype), ''), 'IsDeleted', toString(coalesce(IsDeleted, false)) )) AS metadata, - coalesce(custom_fields, '{}') AS custom_fields, CreatedDate AS created_at, data_source, coalesce(toUnixTimestamp64Milli(SystemModstamp), 0) AS _version diff --git a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_contacts.sql b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_contacts.sql index 4c6738802..37c93897a 100644 --- a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_contacts.sql +++ b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_contacts.sql @@ -27,7 +27,6 @@ WITH src AS ( 'LeadSource', coalesce(toString(LeadSource), ''), 'IsDeleted', toString(coalesce(IsDeleted, false)) )) AS metadata, - coalesce(custom_fields, '{}') AS custom_fields, CreatedDate AS created_at, LastModifiedDate AS updated_at, data_source, diff --git a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_deals.sql b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_deals.sql index d484ed91e..086cbba52 100644 --- a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_deals.sql +++ b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_deals.sql @@ -55,7 +55,6 @@ WITH src AS ( 'Type', coalesce(toString(Type), ''), 'IsDeleted', if(coalesce(IsDeleted, false), 'true', 'false') )) AS metadata, - coalesce(custom_fields, '{}') AS custom_fields, CreatedDate AS created_at, LastModifiedDate AS updated_at, data_source, diff --git a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_users.sql b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_users.sql index 8b40b157a..f10a5e5bb 100644 --- a/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_users.sql +++ b/src/ingestion/connectors/crm/salesforce/dbt/salesforce__crm_users.sql @@ -29,7 +29,6 @@ WITH src AS ( 'Username', coalesce(toString(Username), ''), 'UserRoleId', coalesce(toString(UserRoleId), '') )) AS metadata, - coalesce(custom_fields, '{}') AS custom_fields, collected_at, data_source, coalesce(toUnixTimestamp64Milli(SystemModstamp), 0) AS _version diff --git a/src/ingestion/connectors/crm/salesforce/dbt/schema.yml b/src/ingestion/connectors/crm/salesforce/dbt/schema.yml index 9a8777f72..d701bc18c 100644 --- a/src/ingestion/connectors/crm/salesforce/dbt/schema.yml +++ b/src/ingestion/connectors/crm/salesforce/dbt/schema.yml @@ -3,7 +3,7 @@ version: 2 # Bronze sources for the Salesforce CDK connector. # Table names match the SF sobject name (PascalCase) as emitted by the Airbyte # ClickHouse destination. Every table has the Insight envelope columns: -# tenant_id, source_id, unique_key, data_source, collected_at, custom_fields +# tenant_id, source_id, unique_key, data_source, collected_at, raw_data # (JSON blob of all __c fields). sources: diff --git a/src/ingestion/connectors/crm/salesforce/descriptor.yaml b/src/ingestion/connectors/crm/salesforce/descriptor.yaml index 60d4543ac..a723026be 100644 --- a/src/ingestion/connectors/crm/salesforce/descriptor.yaml +++ b/src/ingestion/connectors/crm/salesforce/descriptor.yaml @@ -1,5 +1,5 @@ name: salesforce -version: "2.8.0" +version: "2.9.0" type: cdk schedule: "0 5 * * *" workflow: sync diff --git a/src/ingestion/connectors/crm/salesforce/pyproject.toml b/src/ingestion/connectors/crm/salesforce/pyproject.toml index 1ebf8353c..ba538319c 100644 --- a/src/ingestion/connectors/crm/salesforce/pyproject.toml +++ b/src/ingestion/connectors/crm/salesforce/pyproject.toml @@ -37,4 +37,4 @@ where = ["."] include = ["source_salesforce*"] [tool.setuptools.package-data] -"source_salesforce" = ["spec.json"] +"source_salesforce" = ["spec.json", "stream_schemas/*.schema.json"] diff --git a/src/ingestion/connectors/crm/salesforce/source_salesforce/api.py b/src/ingestion/connectors/crm/salesforce/source_salesforce/api.py index 1db72436d..f35d8c937 100644 --- a/src/ingestion/connectors/crm/salesforce/source_salesforce/api.py +++ b/src/ingestion/connectors/crm/salesforce/source_salesforce/api.py @@ -1,13 +1,11 @@ -"""Salesforce REST client, OAuth token provider, and describe-based schema generation. +"""Salesforce REST client, OAuth token provider, and describe-based field discovery. The ``Salesforce`` class handles auth via OAuth 2.0 Client Credentials flow (operator supplies ``instance_url``, ``client_id``, ``client_secret``) and -exposes ``describe()`` plus ``generate_schema()`` used by streams to build -SOQL and advertise shapes to the destination. ``SalesforceTokenProvider`` -keeps the access token fresh across long syncs. +exposes ``describe()`` plus ``field_names()`` used by streams to build SOQL. +``SalesforceTokenProvider`` keeps the access token fresh across long syncs. """ -import concurrent.futures import logging import threading import time @@ -15,9 +13,8 @@ import requests from requests import adapters as request_adapters -from requests.exceptions import RequestException -from airbyte_cdk.models import ConfiguredAirbyteCatalog, FailureType, StreamDescriptor +from airbyte_cdk.models import FailureType, StreamDescriptor from airbyte_cdk.sources.declarative.auth.token_provider import TokenProvider from airbyte_cdk.sources.streams.http import HttpClient from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import ( @@ -28,18 +25,14 @@ from source_salesforce.constants import ( API_VERSION, CRM_STREAMS, - DATE_TYPES, - LOOSE_TYPES, - NUMBER_TYPES, PARALLEL_TASKS_SIZE, QUERY_INCOMPATIBLE_SALESFORCE_OBJECTS, QUERY_RESTRICTED_SALESFORCE_OBJECTS, - STRING_TYPES, TOKEN_REFRESH_INTERVAL_SECONDS, UNSUPPORTED_STREAMS, ) -from source_salesforce.exceptions import TypeSalesforceException from source_salesforce.rate_limiting import SalesforceErrorHandler +from source_salesforce.schema_loader import available_stream_names logger = logging.getLogger("airbyte") @@ -66,6 +59,16 @@ def __init__(self, sf_api: "Salesforce") -> None: self._lock = threading.Lock() def get_token(self) -> str: + if self._sf_api.access_token is None: + # Authenticate on demand: discover advertises static schemas and + # issues no request, so nothing should log in until a caller + # actually needs a bearer token. A failure here has no previous + # token to fall back on and must surface. + with self._lock: + if self._sf_api.access_token is None: + self._sf_api.login() + self._last_refresh_time = time.monotonic() + elapsed = time.monotonic() - self._last_refresh_time if elapsed >= TOKEN_REFRESH_INTERVAL_SECONDS: with self._lock: @@ -175,15 +178,16 @@ def __init__( error_handler=SalesforceErrorHandler(token_provider=self._token_provider), ) - # Cache of full describe() responses per sobject. Populated by - # generate_schemas(); read by get_custom_field_names() so callers can - # split records into (standard, custom) without a second describe call. + # Cache of full describe() responses per sobject, so building a + # stream's SOQL field list costs one describe call. self._sobject_describes: dict = {} # ------- Auth ------------------------------------------------------------ def _get_standard_headers(self) -> Mapping[str, str]: - return {"Authorization": f"Bearer {self.access_token}"} + # Through the provider, not the raw attribute: describe is often the + # first authenticated call of a sync, and nothing has logged in for it. + return {"Authorization": f"Bearer {self._token_provider.get_token()}"} def login(self) -> None: """Obtain an access token via OAuth 2.0 Client Credentials flow. @@ -250,17 +254,22 @@ def describe( self, sobject: Optional[str] = None, sobject_options: Optional[Mapping[str, Any]] = None, - ) -> Mapping[str, Any]: + allow_missing: bool = False, + ) -> Optional[Mapping[str, Any]]: """Describe all sobjects (``sobject`` None) or a specific sobject. Raises on 404 for a named sobject rather than returning a bad payload — - callers depend on ``fields``/``sobjects`` keys being present. + callers depend on ``fields``/``sobjects`` keys being present. With + ``allow_missing`` a 404 returns None instead, for callers that treat an + absent sobject as a per-org fact rather than a failure. """ headers = self._get_standard_headers() endpoint = "sobjects" if not sobject else f"sobjects/{sobject}/describe" url = f"{self.instance_url}/services/data/{self.version}/{endpoint}" resp = self._make_request("GET", url, headers=headers) if resp.status_code == 404 and sobject: + if allow_missing: + return None raise AirbyteTracedException( message=( f"Salesforce sobject '{sobject}' not found. Check the " @@ -278,85 +287,33 @@ def describe( ) return resp.json() - def generate_schema( - self, - stream_name: Optional[str] = None, - stream_options: Optional[Mapping[str, Any]] = None, - ) -> Mapping[str, Any]: - response = self.describe(stream_name, stream_options) - if stream_name: - self._sobject_describes[stream_name] = response - schema = { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "additionalProperties": True, - "properties": {}, - } - for field in response["fields"]: - schema["properties"][field["name"]] = self.field_to_property_schema(field) - return schema - - def get_custom_field_names(self, sobject: str) -> frozenset: - """Return the set of field names for which describe reports ``custom=True``. + def sobject_describe(self, sobject: str) -> Optional[Mapping[str, Any]]: + """Cached describe for one sobject; None when the org does not expose it. - Requires that ``generate_schema(sobject)`` (or ``generate_schemas`` - bulk-parallel variant) has been called first; results come from the - per-sobject describe cache populated by :meth:`generate_schema`. + A 404 means the sobject is absent from this org or the Run-As user has + no object access. Both are per-org facts the connector reports and + works around, not errors that should fail a sync. """ - desc = self._sobject_describes.get(sobject) - if not desc: - # Fallback: fetch on demand. Keeps the call site simple even if - # generate_schemas wasn't called for this sobject for any reason. - desc = self.describe(sobject) - self._sobject_describes[sobject] = desc - return frozenset( - f["name"] for f in desc.get("fields", []) if f.get("custom") is True - ) + if sobject not in self._sobject_describes: + self._sobject_describes[sobject] = self.describe(sobject, allow_missing=True) + return self._sobject_describes[sobject] - def generate_schemas( - self, stream_objects: Mapping[str, Any] - ) -> Mapping[str, Any]: - """Describe-driven schema generation, parallelized via ThreadPoolExecutor. + def is_queryable(self, sobject: str) -> bool: + """Whether this org exposes ``sobject`` to SOQL.""" + desc = self.sobject_describe(sobject) + return bool(desc) and bool(desc.get("queryable", True)) - Chunks stream names into batches of ``parallel_tasks_size`` so we don't - open more sockets than our connection pool can hold. - """ + def field_names(self, sobject: str) -> Tuple[str, ...]: + """Every field the org exposes on ``sobject``, standard and custom. - def load_schema( - name: str, stream_options: Mapping[str, Any] - ) -> Tuple[str, Optional[Mapping[str, Any]], Optional[str]]: - try: - result = self.generate_schema( - stream_name=name, stream_options=stream_options - ) - except RequestException as e: - return name, None, str(e) - return name, result, None - - stream_names = list(stream_objects.keys()) - stream_schemas: dict = {} - for i in range(0, len(stream_names), self.parallel_tasks_size): - chunk = stream_names[i : i + self.parallel_tasks_size] - with concurrent.futures.ThreadPoolExecutor( - max_workers=min(len(chunk), self.parallel_tasks_size) - ) as executor: - for name, schema, err in executor.map( - lambda args: load_schema(*args), - [(n, stream_objects[n]) for n in chunk], - ): - if err: - self.logger.error(f"Loading error for {name} schema: {err}") - raise AirbyteTracedException( - message=( - f"Schema could not be extracted for stream {name}. " - "Please retry later." - ), - internal_message=str(err), - failure_type=FailureType.system_error, - stream_descriptor=StreamDescriptor(name=name), - ) - stream_schemas[name] = schema - return stream_schemas + SOQL selects the full set so custom and undeclared standard values still + reach the record envelope, which preserves them in ``raw_data``. Empty + when the org does not expose the sobject. + """ + desc = self.sobject_describe(sobject) + if desc is None: + return () + return tuple(f["name"] for f in desc.get("fields", []) if f.get("name")) # ------- Stream discovery ----------------------------------------------- @@ -369,54 +326,39 @@ def get_streams_black_list(self) -> List[str]: def filter_streams(self, stream_name: str) -> bool: if stream_name.endswith("ChangeEvent") or stream_name in self.get_streams_black_list(): return False + if stream_name not in available_stream_names(): + self.logger.warning( + "Stream %s has no static schema and is skipped.", stream_name + ) + return False return True - def get_validated_streams( - self, - catalog: Optional[ConfiguredAirbyteCatalog] = None, - ) -> Mapping[str, Any]: - """Return ``{stream_name: sobject_options}`` for streams to sync. - - Selection precedence: - 1. If catalog is provided (incremental sync), honor it intersected with - queryable sobjects. - 2. Else use :data:`CRM_STREAMS` (curated list; changing it is a code - change so Silver/dbt coverage ships alongside). - - In every case the full global describe is used to filter out sobjects - that are not queryable, are ChangeEvents, or are on our blocklists. + def syncable_streams(self) -> List[str]: + """The curated stream set, minus anything this connector cannot sync. + + Org-independent by construction: :data:`CRM_STREAMS` is a code-level + contract and the remaining filters read only local state, so the + advertised catalog costs no API call. Whether a given org exposes a + sobject is settled per stream at read time, off the describe the stream + already makes to build its SOQL field list. """ - stream_objects: dict = {} - for so in self.describe()["sobjects"]: - if so["name"] in UNSUPPORTED_STREAMS: - self.logger.warning( - f"Stream {so['name']} needs an object ID and is skipped." - ) - continue - if so["queryable"]: - stream_objects[so.pop("name")] = so - else: - self.logger.warning(f"Stream {so['name']} is not queryable; skipped.") - - if catalog: - return { - cs.stream.name: stream_objects[cs.stream.name] - for cs in catalog.streams - if cs.stream.name in stream_objects - } - - requested: List[str] = list(CRM_STREAMS) - missing = [n for n in requested if n not in stream_objects] - if missing: - self.logger.warning( - "Requested streams not queryable in this org (skipped): %s", - ", ".join(missing), - ) + return [ + name + for name in CRM_STREAMS + if name not in UNSUPPORTED_STREAMS and self.filter_streams(name) + ] - validated = [n for n in requested if n in stream_objects and self.filter_streams(n)] - return {name: stream_objects[name] for name in validated} + def unavailable_streams(self) -> List[str]: + """Curated streams this org does not expose to SOQL, from one describe. - # ------- Field-type -> JSON-schema mapping ------------------------------- + Reported by ``check`` so an operator sees the gaps while configuring the + connection instead of discovering them in sync logs. + """ + global_describe = self.describe() or {} + queryable = { + so["name"] for so in global_describe.get("sobjects", []) if so.get("queryable") + } + return [name for name in self.syncable_streams() if name not in queryable] @staticmethod def get_pk_and_replication_key( @@ -434,50 +376,3 @@ def get_pk_and_replication_key( return pk, cand return pk, None - @staticmethod - def field_to_property_schema(field_params: Mapping[str, Any]) -> Mapping[str, Any]: - """Map a describe() field entry to a JSON-schema property.""" - sf_type = field_params["type"] - - if sf_type in STRING_TYPES: - return {"type": ["string", "null"]} - if sf_type in DATE_TYPES: - return { - "type": ["string", "null"], - "format": "date-time" if sf_type == "datetime" else "date", - } - if sf_type in NUMBER_TYPES: - return {"type": ["number", "null"]} - if sf_type == "int": - return {"type": ["integer", "null"]} - if sf_type == "boolean": - return {"type": ["boolean", "null"]} - if sf_type == "base64": - return {"type": ["string", "null"], "format": "base64"} - if sf_type == "address": - return { - "type": ["object", "null"], - "properties": { - "street": {"type": ["null", "string"]}, - "state": {"type": ["null", "string"]}, - "postalCode": {"type": ["null", "string"]}, - "city": {"type": ["null", "string"]}, - "country": {"type": ["null", "string"]}, - "longitude": {"type": ["null", "number"]}, - "latitude": {"type": ["null", "number"]}, - "geocodeAccuracy": {"type": ["null", "string"]}, - }, - } - if sf_type == "location": - return { - "type": ["object", "null"], - "properties": { - "longitude": {"type": ["null", "number"]}, - "latitude": {"type": ["null", "number"]}, - }, - } - if sf_type in LOOSE_TYPES: - # >99% of values are strings; normalize to string to avoid - # destination type conflicts. - return {"type": ["string", "null"]} - raise TypeSalesforceException(f"Unsupported Salesforce field type: {sf_type}") diff --git a/src/ingestion/connectors/crm/salesforce/source_salesforce/constants.py b/src/ingestion/connectors/crm/salesforce/source_salesforce/constants.py index 77f1265b5..09f4d18b7 100644 --- a/src/ingestion/connectors/crm/salesforce/source_salesforce/constants.py +++ b/src/ingestion/connectors/crm/salesforce/source_salesforce/constants.py @@ -1,4 +1,4 @@ -"""Salesforce sobject and field-type constants. +"""Salesforce sobject constants. The blocklists (``QUERY_RESTRICTED_SALESFORCE_OBJECTS`` etc.) reflect platform-wide API limitations maintained by Salesforce; our curated @@ -9,36 +9,6 @@ API_VERSION = "v62.0" -# ------- Field-type buckets used in describe -> JSON-schema mapping ----------- - -STRING_TYPES = [ - "byte", - "combobox", - "complexvalue", - "datacategorygroupreference", - "email", - "encryptedstring", - "id", - "json", - "masterrecord", - "multipicklist", - "phone", - "picklist", - "reference", - "string", - "textarea", - "time", - "url", -] -NUMBER_TYPES = ["currency", "double", "long", "percent"] -DATE_TYPES = ["date", "datetime"] -LOOSE_TYPES = [ - "anyType", - # A calculated field's type can be any formula data type. Docs: - # https://developer.salesforce.com/docs/atlas.en-us.api.meta/api/field_types.htm - "calculated", -] - # ------- Sobject blocklists --------------------------------------------------- QUERY_RESTRICTED_SALESFORCE_OBJECTS = [ @@ -148,18 +118,6 @@ UNSUPPORTED_STREAMS = ["ActivityMetric", "ActivityMetricRollup"] -PARENT_SALESFORCE_OBJECTS = { - "ContentDocumentLink": { - "parent_name": "ContentDocument", - "field": "Id", - "schema_minimal": { - "properties": { - "Id": {"type": ["string", "null"]}, - "SystemModstamp": {"type": ["string", "null"], "format": "date-time"}, - } - }, - } -} # ------- Token / request limits ----------------------------------------------- diff --git a/src/ingestion/connectors/crm/salesforce/source_salesforce/envelope.py b/src/ingestion/connectors/crm/salesforce/source_salesforce/envelope.py index fdd91b90e..58f83f36f 100644 --- a/src/ingestion/connectors/crm/salesforce/source_salesforce/envelope.py +++ b/src/ingestion/connectors/crm/salesforce/source_salesforce/envelope.py @@ -2,16 +2,16 @@ Every record emitted to Bronze is augmented with tenant / source scope and a deterministic ``unique_key`` so downstream dbt models can key off a single -stable identifier. Custom ``__c`` fields are pulled out into a single JSON -blob so the Bronze schema stays stable across orgs with different SF -customizations. +stable identifier. Columns are limited to the stream's declared fields so the +Bronze schema is identical across orgs; the unabridged record is preserved in +``raw_data``. """ import hashlib import json import logging from datetime import datetime, timezone -from typing import Any, Mapping, MutableMapping, MutableSet, Optional +from typing import Any, FrozenSet, Mapping, MutableMapping, MutableSet, Optional logger = logging.getLogger("airbyte") @@ -20,9 +20,40 @@ # Field names injected by the envelope. A real SF field that collides with one # of these would otherwise be silently overwritten; we log and drop it instead. _RESERVED_FIELD_NAMES = frozenset( - {"tenant_id", "source_id", "unique_key", "data_source", "collected_at", "custom_fields"} + {"tenant_id", "source_id", "unique_key", "data_source", "collected_at", "raw_data"} ) +# Per-value string cap inside ``raw_data``. Bounds worst-case row width however +# an org customizes its objects — long-text and rich-text fields are otherwise +# unbounded. The suffix keeps a clipped value distinguishable from a whole one. +_VALUE_MAX_BYTES = 2048 +_TRUNCATED_SUFFIX = "…[truncated]" +_TRUNCATED_SUFFIX_BYTES = _TRUNCATED_SUFFIX.encode("utf-8") + + +def _truncate(value: Any) -> Any: + if not isinstance(value, str): + return value + + encoded = value.encode("utf-8") + if len(encoded) <= _VALUE_MAX_BYTES: + return value + + allowed = _VALUE_MAX_BYTES - len(_TRUNCATED_SUFFIX_BYTES) + if allowed <= 0: + return _TRUNCATED_SUFFIX + # Slice on bytes; ``errors="ignore"`` drops a partial multi-byte char at the + # boundary so the result stays valid UTF-8. + return encoded[:allowed].decode("utf-8", errors="ignore") + _TRUNCATED_SUFFIX + + +def _truncated_copy(value: Any) -> Any: + if isinstance(value, Mapping): + return {k: _truncated_copy(v) for k, v in value.items()} + if isinstance(value, list): + return [_truncated_copy(item) for item in value] + return _truncate(value) + def _now_iso() -> str: return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") @@ -33,21 +64,23 @@ def envelope( *, tenant_id: str, source_id: str, - custom_field_names: frozenset, + declared_fields: FrozenSet[str], collision_seen: Optional[MutableSet[str]] = None, ) -> MutableMapping[str, Any]: """Return a copy of ``record`` with Insight metadata injected. - Splits the record: standard fields stay at top level, every name in - ``custom_field_names`` is packed into a single ``custom_fields`` JSON string, - and ``tenant_id`` / ``source_id`` / ``unique_key`` / ``data_source`` / - ``collected_at`` are added. + Fields in ``declared_fields`` stay at top level, the whole record is + preserved in ``raw_data``, and ``tenant_id`` / ``source_id`` / + ``unique_key`` / ``data_source`` / ``collected_at`` are added. A field the + stream does not declare — a custom ``__c`` field, or a standard field + outside the stream's schema — reaches Bronze through ``raw_data`` alone; + emitting it top-level would make the table shape follow the org's field set. If ``collision_seen`` is provided, collision warnings are emitted only once per offending field name across the stream's lifetime. """ out: MutableMapping[str, Any] = {} - customs: dict = {} + raw: dict = {} for key, value in record.items(): # Salesforce always returns an ``attributes`` metadata dict — drop it. @@ -62,15 +95,13 @@ def envelope( if collision_seen is not None: collision_seen.add(key) continue - if key in custom_field_names: - customs[key] = value - else: + + raw[key] = _truncated_copy(value) + if key in declared_fields: out[key] = value # ClickHouse stores JSON blobs as strings; serialize once. - out["custom_fields"] = ( - json.dumps(customs, separators=(",", ":"), default=str) if customs else "{}" - ) + out["raw_data"] = json.dumps(raw, separators=(",", ":"), default=str) if raw else "{}" sf_id = record.get("Id") or record.get("id") or "" if not sf_id: @@ -103,12 +134,12 @@ def envelope( "unique_key": {"type": "string"}, "data_source": {"type": "string"}, "collected_at": {"type": "string", "format": "date-time"}, - "custom_fields": {"type": "string"}, + "raw_data": {"type": "string"}, } def inject_envelope_properties(schema: MutableMapping[str, Any]) -> MutableMapping[str, Any]: - """Add envelope field definitions to a JSON schema generated from describe(). + """Add envelope field definitions to a stream's JSON schema. Used when advertising per-stream schemas so the destination creates columns for the envelope fields alongside the SF fields. diff --git a/src/ingestion/connectors/crm/salesforce/source_salesforce/exceptions.py b/src/ingestion/connectors/crm/salesforce/source_salesforce/exceptions.py deleted file mode 100644 index 23163d70b..000000000 --- a/src/ingestion/connectors/crm/salesforce/source_salesforce/exceptions.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Exception hierarchy for the Salesforce connector.""" - - -class SalesforceException(Exception): - """Base class for Salesforce-specific errors.""" - - -class TypeSalesforceException(SalesforceException): - """Unknown Salesforce field type encountered during schema generation.""" diff --git a/src/ingestion/connectors/crm/salesforce/source_salesforce/schema_loader.py b/src/ingestion/connectors/crm/salesforce/source_salesforce/schema_loader.py new file mode 100644 index 000000000..1e322a86c --- /dev/null +++ b/src/ingestion/connectors/crm/salesforce/source_salesforce/schema_loader.py @@ -0,0 +1,43 @@ +import json +from copy import deepcopy +from functools import lru_cache +from pathlib import Path +from typing import Any, FrozenSet, Mapping, MutableMapping + +from source_salesforce.envelope import inject_envelope_properties + +# Not ``schemas/`` — src/ingestion/.gitignore reserves that name for generated, +# regenerable catalogs. These files are hand-maintained source of truth. +_SCHEMA_DIR = Path(__file__).parent / "stream_schemas" +_SCHEMA_SUFFIX = ".schema.json" + + +class UnknownStreamSchemaError(Exception): + pass + + +@lru_cache(maxsize=None) +def available_stream_names() -> FrozenSet[str]: + return frozenset(p.name[: -len(_SCHEMA_SUFFIX)] for p in _SCHEMA_DIR.glob(f"*{_SCHEMA_SUFFIX}")) + + +@lru_cache(maxsize=None) +def _load(stream_name: str) -> Mapping[str, Any]: + path = _SCHEMA_DIR / f"{stream_name}{_SCHEMA_SUFFIX}" + if not path.is_file(): + raise UnknownStreamSchemaError( + f"No static schema for stream '{stream_name}'; expected {path.name}" + ) + schema: Mapping[str, Any] = json.loads(path.read_text()) + return schema + + +def declared_field_names(stream_name: str) -> FrozenSet[str]: + """Salesforce fields that become Bronze columns, envelope fields excluded.""" + return frozenset(_load(stream_name)["properties"]) + + +def stream_schema(stream_name: str) -> MutableMapping[str, Any]: + # Deep copy: the CDK hands advertised schemas to callers that may mutate them. + schema = deepcopy(dict(_load(stream_name))) + return inject_envelope_properties(schema) diff --git a/src/ingestion/connectors/crm/salesforce/source_salesforce/source.py b/src/ingestion/connectors/crm/salesforce/source_salesforce/source.py index f5997fdec..b9a3642b4 100644 --- a/src/ingestion/connectors/crm/salesforce/source_salesforce/source.py +++ b/src/ingestion/connectors/crm/salesforce/source_salesforce/source.py @@ -43,15 +43,12 @@ from airbyte_cdk.models import ConnectorSpecification from source_salesforce.api import Salesforce, SalesforceAuthenticator -from source_salesforce.constants import ( - PARENT_SALESFORCE_OBJECTS, - UNSUPPORTED_FILTERING_STREAMS, -) +from source_salesforce.constants import UNSUPPORTED_FILTERING_STREAMS +from source_salesforce.schema_loader import stream_schema from source_salesforce.streams import ( DEFAULT_LOOKBACK_SECONDS, IncrementalRestSalesforceStream, RestSalesforceStream, - RestSalesforceSubStream, ) @@ -90,18 +87,24 @@ def spec(self, logger_: logging.Logger) -> ConnectorSpecification: return ConnectorSpecification(**json.loads(spec_path.read_text())) @staticmethod - def _get_sf_object(config: Mapping[str, Any]) -> Salesforce: - """Instantiate the Salesforce client and authenticate. + def _build_sf_client(config: Mapping[str, Any]) -> Salesforce: + """Instantiate the Salesforce client without contacting Salesforce. Config keys are prefixed to avoid collisions in shared K8s Secrets. - Only the ``salesforce_*`` keys are passed to the client. + Only the ``salesforce_*`` keys are passed to the client. The client + authenticates on demand, when a caller first needs a bearer token. """ - sf = Salesforce( + return Salesforce( instance_url=config["salesforce_instance_url"], client_id=config["salesforce_client_id"], client_secret=config["salesforce_client_secret"], start_date=config.get("salesforce_start_date"), ) + + @staticmethod + def _get_sf_object(config: Mapping[str, Any]) -> Salesforce: + """Instantiate the Salesforce client and authenticate eagerly.""" + sf = SourceSalesforce._build_sf_client(config) sf.login() return sf @@ -163,38 +166,39 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> self._validate_stream_slice_step(config.get("salesforce_stream_slice_step")) self._validate_lookback_window(config.get("salesforce_lookback_window")) salesforce = self._get_sf_object(config) - salesforce.describe() + unavailable = salesforce.unavailable_streams() + if unavailable: + # Surfaced here rather than failing: an org that does not license or + # expose an object still syncs the rest. Reporting at check time is + # what keeps the gap visible now that discover is org-independent. + logger.warning( + "Streams not exposed by this org (they will sync as empty): %s", + ", ".join(unavailable), + ) return True, None @classmethod def _get_stream_type(cls, stream_name: str): - """Get proper stream class: full_refresh, incremental or substream. + """Get proper stream class: full_refresh or incremental. - Every stream uses the REST ``/queryAll`` API. SubStreams (like - ContentDocumentLink) do not support incremental sync because of query - restrictions: https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/sforce_api_objects_contentdocumentlink.htm + Every stream uses the REST ``/queryAll`` API. """ - parent_name = PARENT_SALESFORCE_OBJECTS.get(stream_name, {}).get("parent_name") - full_refresh = RestSalesforceSubStream if parent_name else RestSalesforceStream - incremental = IncrementalRestSalesforceStream - return full_refresh, incremental + return RestSalesforceStream, IncrementalRestSalesforceStream def prepare_stream(self, stream_name: str, json_schema, sobject_options, sf_object, authenticator, config): - """Choose proper stream class: syncMode (full_refresh/incremental), REST API, SubStream.""" + """Choose proper stream class by sync mode (full_refresh / incremental).""" pk, replication_key = sf_object.get_pk_and_replication_key(json_schema) stream_kwargs = { "stream_name": stream_name, - "schema": json_schema, "pk": pk, "sobject_options": sobject_options, "sf_api": sf_object, "authenticator": authenticator, "start_date": config.get("salesforce_start_date"), "message_repository": self.message_repository, - # Envelope context — tenant_id / source_id / custom_fields split. + # Envelope context — tenant / source scope on every record. "tenant_id": config["insight_tenant_id"], "source_id": config["insight_source_id"], - "custom_field_names": sf_object.get_custom_field_names(stream_name), } full_refresh, incremental = self._get_stream_type(stream_name) @@ -217,27 +221,14 @@ def generate_streams( ) -> List[Stream]: """Generates a list of stream by their names. It can be used for different tests too""" authenticator = SalesforceAuthenticator(sf_object._token_provider) - schemas = sf_object.generate_schemas(stream_objects) default_args = [sf_object, authenticator, config] streams = [] state_manager = ConnectorStateManager(state=self.state) for stream_name, sobject_options in stream_objects.items(): - json_schema = schemas.get(stream_name, {}) + json_schema = stream_schema(stream_name) stream_class, kwargs = self.prepare_stream(stream_name, json_schema, sobject_options, *default_args) - parent_name = PARENT_SALESFORCE_OBJECTS.get(stream_name, {}).get("parent_name") - if parent_name: - # Minimal schema + sobject_options specific to the parent (not - # the child's). Child-specific permission flags should not - # shape the parent stream. - parent_schema = PARENT_SALESFORCE_OBJECTS.get(stream_name, {}).get("schema_minimal") - parent_sobject_options = stream_objects.get(parent_name) or {} - parent_class, parent_kwargs = self.prepare_stream( - parent_name, parent_schema, parent_sobject_options, *default_args - ) - kwargs["parent"] = parent_class(**parent_kwargs) - stream = stream_class(**kwargs) streams.append(self._wrap_for_concurrency(config, stream, state_manager)) # The Describe meta-stream is intentionally omitted — @@ -270,15 +261,21 @@ def _wrap_for_concurrency(self, config, stream, state_manager): return StreamFacade.create_from_stream(stream, self, logger, state, cursor) def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """Build the stream set without contacting Salesforce. + + Every stream's schema is a repository artifact and the stream set is a + code-level contract, so the catalog is the same for every org and costs + no API call. The client authenticates lazily, when a read issues its + first request. + """ if not config.get("salesforce_start_date"): config = dict(config) config["salesforce_start_date"] = ( datetime.now() - relativedelta(years=self.START_DATE_OFFSET_IN_YEARS) ).strftime(self.DATETIME_FORMAT) - sf = self._get_sf_object(config) - stream_objects = sf.get_validated_streams(catalog=self.catalog) - streams = self.generate_streams(config, stream_objects, sf) - return streams + sf = self._build_sf_client(config) + stream_objects = {name: {} for name in sf.syncable_streams()} + return self.generate_streams(config, stream_objects, sf) def _create_stream_slicer_cursor( self, config: Mapping[str, Any], state_manager: ConnectorStateManager, stream: Stream diff --git a/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Account.schema.json b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Account.schema.json new file mode 100644 index 000000000..83a783669 --- /dev/null +++ b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Account.schema.json @@ -0,0 +1,394 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "required": [ + "unique_key" + ], + "properties": { + "Id": { + "type": [ + "string", + "null" + ] + }, + "IsDeleted": { + "type": [ + "boolean", + "null" + ] + }, + "MasterRecordId": { + "type": [ + "string", + "null" + ] + }, + "Name": { + "type": [ + "string", + "null" + ] + }, + "Type": { + "type": [ + "string", + "null" + ] + }, + "ParentId": { + "type": [ + "string", + "null" + ] + }, + "BillingStreet": { + "type": [ + "string", + "null" + ] + }, + "BillingCity": { + "type": [ + "string", + "null" + ] + }, + "BillingState": { + "type": [ + "string", + "null" + ] + }, + "BillingPostalCode": { + "type": [ + "string", + "null" + ] + }, + "BillingCountry": { + "type": [ + "string", + "null" + ] + }, + "BillingStateCode": { + "type": [ + "string", + "null" + ] + }, + "BillingCountryCode": { + "type": [ + "string", + "null" + ] + }, + "BillingLatitude": { + "type": [ + "number", + "null" + ] + }, + "BillingLongitude": { + "type": [ + "number", + "null" + ] + }, + "BillingGeocodeAccuracy": { + "type": [ + "string", + "null" + ] + }, + "BillingAddress": { + "type": [ + "string", + "null" + ] + }, + "ShippingStreet": { + "type": [ + "string", + "null" + ] + }, + "ShippingCity": { + "type": [ + "string", + "null" + ] + }, + "ShippingState": { + "type": [ + "string", + "null" + ] + }, + "ShippingPostalCode": { + "type": [ + "string", + "null" + ] + }, + "ShippingCountry": { + "type": [ + "string", + "null" + ] + }, + "ShippingStateCode": { + "type": [ + "string", + "null" + ] + }, + "ShippingCountryCode": { + "type": [ + "string", + "null" + ] + }, + "ShippingLatitude": { + "type": [ + "number", + "null" + ] + }, + "ShippingLongitude": { + "type": [ + "number", + "null" + ] + }, + "ShippingGeocodeAccuracy": { + "type": [ + "string", + "null" + ] + }, + "ShippingAddress": { + "type": [ + "string", + "null" + ] + }, + "Phone": { + "type": [ + "string", + "null" + ] + }, + "Fax": { + "type": [ + "string", + "null" + ] + }, + "AccountNumber": { + "type": [ + "string", + "null" + ] + }, + "Website": { + "type": [ + "string", + "null" + ] + }, + "PhotoUrl": { + "type": [ + "string", + "null" + ] + }, + "Sic": { + "type": [ + "string", + "null" + ] + }, + "Industry": { + "type": [ + "string", + "null" + ] + }, + "AnnualRevenue": { + "type": [ + "number", + "null" + ] + }, + "NumberOfEmployees": { + "type": [ + "integer", + "null" + ] + }, + "Ownership": { + "type": [ + "string", + "null" + ] + }, + "TickerSymbol": { + "type": [ + "string", + "null" + ] + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "Rating": { + "type": [ + "string", + "null" + ] + }, + "Site": { + "type": [ + "string", + "null" + ] + }, + "OwnerId": { + "type": [ + "string", + "null" + ] + }, + "CreatedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "CreatedById": { + "type": [ + "string", + "null" + ] + }, + "LastModifiedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastModifiedById": { + "type": [ + "string", + "null" + ] + }, + "SystemModstamp": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastActivityDate": { + "type": [ + "string", + "null" + ], + "format": "date" + }, + "LastViewedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastReferencedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "Jigsaw": { + "type": [ + "string", + "null" + ] + }, + "JigsawCompanyId": { + "type": [ + "string", + "null" + ] + }, + "CleanStatus": { + "type": [ + "string", + "null" + ] + }, + "AccountSource": { + "type": [ + "string", + "null" + ] + }, + "DunsNumber": { + "type": [ + "string", + "null" + ] + }, + "Tradestyle": { + "type": [ + "string", + "null" + ] + }, + "NaicsCode": { + "type": [ + "string", + "null" + ] + }, + "NaicsDesc": { + "type": [ + "string", + "null" + ] + }, + "YearStarted": { + "type": [ + "string", + "null" + ] + }, + "SicDesc": { + "type": [ + "string", + "null" + ] + }, + "DandbCompanyId": { + "type": [ + "string", + "null" + ] + }, + "OperatingHoursId": { + "type": [ + "string", + "null" + ] + } + } +} diff --git a/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Case.schema.json b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Case.schema.json new file mode 100644 index 000000000..f8db30168 --- /dev/null +++ b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Case.schema.json @@ -0,0 +1,232 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "required": [ + "unique_key" + ], + "properties": { + "Id": { + "type": [ + "string", + "null" + ] + }, + "IsDeleted": { + "type": [ + "boolean", + "null" + ] + }, + "MasterRecordId": { + "type": [ + "string", + "null" + ] + }, + "CaseNumber": { + "type": [ + "string", + "null" + ] + }, + "ContactId": { + "type": [ + "string", + "null" + ] + }, + "AccountId": { + "type": [ + "string", + "null" + ] + }, + "AssetId": { + "type": [ + "string", + "null" + ] + }, + "SourceId": { + "type": [ + "string", + "null" + ] + }, + "ParentId": { + "type": [ + "string", + "null" + ] + }, + "SuppliedName": { + "type": [ + "string", + "null" + ] + }, + "SuppliedEmail": { + "type": [ + "string", + "null" + ] + }, + "SuppliedPhone": { + "type": [ + "string", + "null" + ] + }, + "SuppliedCompany": { + "type": [ + "string", + "null" + ] + }, + "Type": { + "type": [ + "string", + "null" + ] + }, + "Status": { + "type": [ + "string", + "null" + ] + }, + "Reason": { + "type": [ + "string", + "null" + ] + }, + "Origin": { + "type": [ + "string", + "null" + ] + }, + "Subject": { + "type": [ + "string", + "null" + ] + }, + "Priority": { + "type": [ + "string", + "null" + ] + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "IsClosed": { + "type": [ + "boolean", + "null" + ] + }, + "ClosedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "IsEscalated": { + "type": [ + "boolean", + "null" + ] + }, + "OwnerId": { + "type": [ + "string", + "null" + ] + }, + "CreatedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "CreatedById": { + "type": [ + "string", + "null" + ] + }, + "LastModifiedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastModifiedById": { + "type": [ + "string", + "null" + ] + }, + "SystemModstamp": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "ContactPhone": { + "type": [ + "string", + "null" + ] + }, + "ContactMobile": { + "type": [ + "string", + "null" + ] + }, + "ContactEmail": { + "type": [ + "string", + "null" + ] + }, + "ContactFax": { + "type": [ + "string", + "null" + ] + }, + "Comments": { + "type": [ + "string", + "null" + ] + }, + "LastViewedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastReferencedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + } + } +} diff --git a/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Contact.schema.json b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Contact.schema.json new file mode 100644 index 000000000..60a3b20f8 --- /dev/null +++ b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Contact.schema.json @@ -0,0 +1,410 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "required": [ + "unique_key" + ], + "properties": { + "Id": { + "type": [ + "string", + "null" + ] + }, + "IsDeleted": { + "type": [ + "boolean", + "null" + ] + }, + "MasterRecordId": { + "type": [ + "string", + "null" + ] + }, + "AccountId": { + "type": [ + "string", + "null" + ] + }, + "LastName": { + "type": [ + "string", + "null" + ] + }, + "FirstName": { + "type": [ + "string", + "null" + ] + }, + "Salutation": { + "type": [ + "string", + "null" + ] + }, + "Name": { + "type": [ + "string", + "null" + ] + }, + "OtherStreet": { + "type": [ + "string", + "null" + ] + }, + "OtherCity": { + "type": [ + "string", + "null" + ] + }, + "OtherState": { + "type": [ + "string", + "null" + ] + }, + "OtherPostalCode": { + "type": [ + "string", + "null" + ] + }, + "OtherCountry": { + "type": [ + "string", + "null" + ] + }, + "OtherStateCode": { + "type": [ + "string", + "null" + ] + }, + "OtherCountryCode": { + "type": [ + "string", + "null" + ] + }, + "OtherLatitude": { + "type": [ + "number", + "null" + ] + }, + "OtherLongitude": { + "type": [ + "number", + "null" + ] + }, + "OtherGeocodeAccuracy": { + "type": [ + "string", + "null" + ] + }, + "OtherAddress": { + "type": [ + "string", + "null" + ] + }, + "MailingStreet": { + "type": [ + "string", + "null" + ] + }, + "MailingCity": { + "type": [ + "string", + "null" + ] + }, + "MailingState": { + "type": [ + "string", + "null" + ] + }, + "MailingPostalCode": { + "type": [ + "string", + "null" + ] + }, + "MailingCountry": { + "type": [ + "string", + "null" + ] + }, + "MailingStateCode": { + "type": [ + "string", + "null" + ] + }, + "MailingCountryCode": { + "type": [ + "string", + "null" + ] + }, + "MailingLatitude": { + "type": [ + "number", + "null" + ] + }, + "MailingLongitude": { + "type": [ + "number", + "null" + ] + }, + "MailingGeocodeAccuracy": { + "type": [ + "string", + "null" + ] + }, + "MailingAddress": { + "type": [ + "string", + "null" + ] + }, + "Phone": { + "type": [ + "string", + "null" + ] + }, + "Fax": { + "type": [ + "string", + "null" + ] + }, + "MobilePhone": { + "type": [ + "string", + "null" + ] + }, + "HomePhone": { + "type": [ + "string", + "null" + ] + }, + "OtherPhone": { + "type": [ + "string", + "null" + ] + }, + "AssistantPhone": { + "type": [ + "string", + "null" + ] + }, + "ReportsToId": { + "type": [ + "string", + "null" + ] + }, + "Email": { + "type": [ + "string", + "null" + ] + }, + "Title": { + "type": [ + "string", + "null" + ] + }, + "Department": { + "type": [ + "string", + "null" + ] + }, + "AssistantName": { + "type": [ + "string", + "null" + ] + }, + "LeadSource": { + "type": [ + "string", + "null" + ] + }, + "Birthdate": { + "type": [ + "string", + "null" + ], + "format": "date" + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "OwnerId": { + "type": [ + "string", + "null" + ] + }, + "CreatedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "CreatedById": { + "type": [ + "string", + "null" + ] + }, + "LastModifiedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastModifiedById": { + "type": [ + "string", + "null" + ] + }, + "SystemModstamp": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastActivityDate": { + "type": [ + "string", + "null" + ], + "format": "date" + }, + "LastCURequestDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastCUUpdateDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastViewedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastReferencedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "EmailBouncedReason": { + "type": [ + "string", + "null" + ] + }, + "EmailBouncedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "IsEmailBounced": { + "type": [ + "boolean", + "null" + ] + }, + "PhotoUrl": { + "type": [ + "string", + "null" + ] + }, + "Jigsaw": { + "type": [ + "string", + "null" + ] + }, + "JigsawContactId": { + "type": [ + "string", + "null" + ] + }, + "CleanStatus": { + "type": [ + "string", + "null" + ] + }, + "IndividualId": { + "type": [ + "string", + "null" + ] + }, + "IsPriorityRecord": { + "type": [ + "boolean", + "null" + ] + }, + "ContactSource": { + "type": [ + "string", + "null" + ] + } + } +} diff --git a/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Event.schema.json b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Event.schema.json new file mode 100644 index 000000000..b432d0565 --- /dev/null +++ b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Event.schema.json @@ -0,0 +1,316 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "required": [ + "unique_key" + ], + "properties": { + "Id": { + "type": [ + "string", + "null" + ] + }, + "WhoId": { + "type": [ + "string", + "null" + ] + }, + "WhatId": { + "type": [ + "string", + "null" + ] + }, + "Subject": { + "type": [ + "string", + "null" + ] + }, + "Location": { + "type": [ + "string", + "null" + ] + }, + "IsAllDayEvent": { + "type": [ + "boolean", + "null" + ] + }, + "ActivityDateTime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "ActivityDate": { + "type": [ + "string", + "null" + ], + "format": "date" + }, + "DurationInMinutes": { + "type": [ + "integer", + "null" + ] + }, + "StartDateTime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "EndDateTime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "EndDate": { + "type": [ + "string", + "null" + ], + "format": "date" + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "AccountId": { + "type": [ + "string", + "null" + ] + }, + "OwnerId": { + "type": [ + "string", + "null" + ] + }, + "IsPrivate": { + "type": [ + "boolean", + "null" + ] + }, + "ShowAs": { + "type": [ + "string", + "null" + ] + }, + "IsDeleted": { + "type": [ + "boolean", + "null" + ] + }, + "IsChild": { + "type": [ + "boolean", + "null" + ] + }, + "IsGroupEvent": { + "type": [ + "boolean", + "null" + ] + }, + "GroupEventType": { + "type": [ + "string", + "null" + ] + }, + "CreatedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "CreatedById": { + "type": [ + "string", + "null" + ] + }, + "LastModifiedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastModifiedById": { + "type": [ + "string", + "null" + ] + }, + "SystemModstamp": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "IsArchived": { + "type": [ + "boolean", + "null" + ] + }, + "RecurrenceActivityId": { + "type": [ + "string", + "null" + ] + }, + "IsRecurrence": { + "type": [ + "boolean", + "null" + ] + }, + "RecurrenceStartDateTime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "RecurrenceEndDateOnly": { + "type": [ + "string", + "null" + ], + "format": "date" + }, + "RecurrenceTimeZoneSidKey": { + "type": [ + "string", + "null" + ] + }, + "RecurrenceType": { + "type": [ + "string", + "null" + ] + }, + "RecurrenceInterval": { + "type": [ + "integer", + "null" + ] + }, + "RecurrenceDayOfWeekMask": { + "type": [ + "integer", + "null" + ] + }, + "RecurrenceDayOfMonth": { + "type": [ + "integer", + "null" + ] + }, + "RecurrenceInstance": { + "type": [ + "string", + "null" + ] + }, + "RecurrenceMonthOfYear": { + "type": [ + "string", + "null" + ] + }, + "ReminderDateTime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "IsReminderSet": { + "type": [ + "boolean", + "null" + ] + }, + "EventSubtype": { + "type": [ + "string", + "null" + ] + }, + "IsRecurrence2Exclusion": { + "type": [ + "boolean", + "null" + ] + }, + "Recurrence2PatternText": { + "type": [ + "string", + "null" + ] + }, + "Recurrence2PatternVersion": { + "type": [ + "string", + "null" + ] + }, + "IsRecurrence2": { + "type": [ + "boolean", + "null" + ] + }, + "IsRecurrence2Exception": { + "type": [ + "boolean", + "null" + ] + }, + "Recurrence2PatternStartDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "Recurrence2PatternTimeZone": { + "type": [ + "string", + "null" + ] + }, + "ServiceAppointmentId": { + "type": [ + "string", + "null" + ] + } + } +} diff --git a/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Lead.schema.json b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Lead.schema.json new file mode 100644 index 000000000..3762c07e1 --- /dev/null +++ b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Lead.schema.json @@ -0,0 +1,360 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "required": [ + "unique_key" + ], + "properties": { + "Id": { + "type": [ + "string", + "null" + ] + }, + "IsDeleted": { + "type": [ + "boolean", + "null" + ] + }, + "MasterRecordId": { + "type": [ + "string", + "null" + ] + }, + "LastName": { + "type": [ + "string", + "null" + ] + }, + "FirstName": { + "type": [ + "string", + "null" + ] + }, + "Salutation": { + "type": [ + "string", + "null" + ] + }, + "Name": { + "type": [ + "string", + "null" + ] + }, + "Title": { + "type": [ + "string", + "null" + ] + }, + "Company": { + "type": [ + "string", + "null" + ] + }, + "Street": { + "type": [ + "string", + "null" + ] + }, + "City": { + "type": [ + "string", + "null" + ] + }, + "State": { + "type": [ + "string", + "null" + ] + }, + "PostalCode": { + "type": [ + "string", + "null" + ] + }, + "Country": { + "type": [ + "string", + "null" + ] + }, + "StateCode": { + "type": [ + "string", + "null" + ] + }, + "CountryCode": { + "type": [ + "string", + "null" + ] + }, + "Latitude": { + "type": [ + "number", + "null" + ] + }, + "Longitude": { + "type": [ + "number", + "null" + ] + }, + "GeocodeAccuracy": { + "type": [ + "string", + "null" + ] + }, + "Address": { + "type": [ + "string", + "null" + ] + }, + "Phone": { + "type": [ + "string", + "null" + ] + }, + "MobilePhone": { + "type": [ + "string", + "null" + ] + }, + "Fax": { + "type": [ + "string", + "null" + ] + }, + "Email": { + "type": [ + "string", + "null" + ] + }, + "Website": { + "type": [ + "string", + "null" + ] + }, + "PhotoUrl": { + "type": [ + "string", + "null" + ] + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "LeadSource": { + "type": [ + "string", + "null" + ] + }, + "Status": { + "type": [ + "string", + "null" + ] + }, + "Industry": { + "type": [ + "string", + "null" + ] + }, + "Rating": { + "type": [ + "string", + "null" + ] + }, + "AnnualRevenue": { + "type": [ + "number", + "null" + ] + }, + "NumberOfEmployees": { + "type": [ + "integer", + "null" + ] + }, + "OwnerId": { + "type": [ + "string", + "null" + ] + }, + "IsConverted": { + "type": [ + "boolean", + "null" + ] + }, + "ConvertedDate": { + "type": [ + "string", + "null" + ], + "format": "date" + }, + "ConvertedAccountId": { + "type": [ + "string", + "null" + ] + }, + "ConvertedContactId": { + "type": [ + "string", + "null" + ] + }, + "ConvertedOpportunityId": { + "type": [ + "string", + "null" + ] + }, + "IsUnreadByOwner": { + "type": [ + "boolean", + "null" + ] + }, + "CreatedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "CreatedById": { + "type": [ + "string", + "null" + ] + }, + "LastModifiedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastModifiedById": { + "type": [ + "string", + "null" + ] + }, + "SystemModstamp": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastActivityDate": { + "type": [ + "string", + "null" + ], + "format": "date" + }, + "LastViewedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastReferencedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "Jigsaw": { + "type": [ + "string", + "null" + ] + }, + "JigsawContactId": { + "type": [ + "string", + "null" + ] + }, + "CleanStatus": { + "type": [ + "string", + "null" + ] + }, + "CompanyDunsNumber": { + "type": [ + "string", + "null" + ] + }, + "DandbCompanyId": { + "type": [ + "string", + "null" + ] + }, + "EmailBouncedReason": { + "type": [ + "string", + "null" + ] + }, + "EmailBouncedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "IndividualId": { + "type": [ + "string", + "null" + ] + }, + "IsPriorityRecord": { + "type": [ + "boolean", + "null" + ] + } + } +} diff --git a/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Opportunity.schema.json b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Opportunity.schema.json new file mode 100644 index 000000000..ad193b665 --- /dev/null +++ b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Opportunity.schema.json @@ -0,0 +1,264 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "required": [ + "unique_key" + ], + "properties": { + "Id": { + "type": [ + "string", + "null" + ] + }, + "IsDeleted": { + "type": [ + "boolean", + "null" + ] + }, + "AccountId": { + "type": [ + "string", + "null" + ] + }, + "IsPrivate": { + "type": [ + "boolean", + "null" + ] + }, + "Name": { + "type": [ + "string", + "null" + ] + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "StageName": { + "type": [ + "string", + "null" + ] + }, + "Amount": { + "type": [ + "number", + "null" + ] + }, + "Probability": { + "type": [ + "number", + "null" + ] + }, + "ExpectedRevenue": { + "type": [ + "number", + "null" + ] + }, + "TotalOpportunityQuantity": { + "type": [ + "number", + "null" + ] + }, + "CloseDate": { + "type": [ + "string", + "null" + ], + "format": "date" + }, + "Type": { + "type": [ + "string", + "null" + ] + }, + "NextStep": { + "type": [ + "string", + "null" + ] + }, + "LeadSource": { + "type": [ + "string", + "null" + ] + }, + "IsClosed": { + "type": [ + "boolean", + "null" + ] + }, + "IsWon": { + "type": [ + "boolean", + "null" + ] + }, + "ForecastCategory": { + "type": [ + "string", + "null" + ] + }, + "ForecastCategoryName": { + "type": [ + "string", + "null" + ] + }, + "CampaignId": { + "type": [ + "string", + "null" + ] + }, + "HasOpportunityLineItem": { + "type": [ + "boolean", + "null" + ] + }, + "Pricebook2Id": { + "type": [ + "string", + "null" + ] + }, + "OwnerId": { + "type": [ + "string", + "null" + ] + }, + "CreatedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "CreatedById": { + "type": [ + "string", + "null" + ] + }, + "LastModifiedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastModifiedById": { + "type": [ + "string", + "null" + ] + }, + "SystemModstamp": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastActivityDate": { + "type": [ + "string", + "null" + ], + "format": "date" + }, + "PushCount": { + "type": [ + "integer", + "null" + ] + }, + "LastStageChangeDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "FiscalQuarter": { + "type": [ + "integer", + "null" + ] + }, + "FiscalYear": { + "type": [ + "integer", + "null" + ] + }, + "Fiscal": { + "type": [ + "string", + "null" + ] + }, + "ContactId": { + "type": [ + "string", + "null" + ] + }, + "LastViewedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastReferencedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "HasOpenActivity": { + "type": [ + "boolean", + "null" + ] + }, + "HasOverdueTask": { + "type": [ + "boolean", + "null" + ] + }, + "LastAmountChangedHistoryId": { + "type": [ + "string", + "null" + ] + }, + "LastCloseDateChangedHistoryId": { + "type": [ + "string", + "null" + ] + } + } +} diff --git a/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/OpportunityContactRole.schema.json b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/OpportunityContactRole.schema.json new file mode 100644 index 000000000..7fb39089c --- /dev/null +++ b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/OpportunityContactRole.schema.json @@ -0,0 +1,79 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "required": [ + "unique_key" + ], + "properties": { + "Id": { + "type": [ + "string", + "null" + ] + }, + "OpportunityId": { + "type": [ + "string", + "null" + ] + }, + "ContactId": { + "type": [ + "string", + "null" + ] + }, + "Role": { + "type": [ + "string", + "null" + ] + }, + "IsPrimary": { + "type": [ + "boolean", + "null" + ] + }, + "CreatedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "CreatedById": { + "type": [ + "string", + "null" + ] + }, + "LastModifiedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastModifiedById": { + "type": [ + "string", + "null" + ] + }, + "SystemModstamp": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "IsDeleted": { + "type": [ + "boolean", + "null" + ] + } + } +} diff --git a/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/OpportunityHistory.schema.json b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/OpportunityHistory.schema.json new file mode 100644 index 000000000..335ddc6eb --- /dev/null +++ b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/OpportunityHistory.schema.json @@ -0,0 +1,98 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "required": [ + "unique_key" + ], + "properties": { + "Id": { + "type": [ + "string", + "null" + ] + }, + "OpportunityId": { + "type": [ + "string", + "null" + ] + }, + "CreatedById": { + "type": [ + "string", + "null" + ] + }, + "CreatedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "StageName": { + "type": [ + "string", + "null" + ] + }, + "Amount": { + "type": [ + "number", + "null" + ] + }, + "ExpectedRevenue": { + "type": [ + "number", + "null" + ] + }, + "CloseDate": { + "type": [ + "string", + "null" + ], + "format": "date" + }, + "Probability": { + "type": [ + "number", + "null" + ] + }, + "ForecastCategory": { + "type": [ + "string", + "null" + ] + }, + "SystemModstamp": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "IsDeleted": { + "type": [ + "boolean", + "null" + ] + }, + "PrevAmount": { + "type": [ + "number", + "null" + ] + }, + "PrevCloseDate": { + "type": [ + "string", + "null" + ], + "format": "date" + } + } +} diff --git a/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Task.schema.json b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Task.schema.json new file mode 100644 index 000000000..88f2037ff --- /dev/null +++ b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/Task.schema.json @@ -0,0 +1,252 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "required": [ + "unique_key" + ], + "properties": { + "Id": { + "type": [ + "string", + "null" + ] + }, + "WhoId": { + "type": [ + "string", + "null" + ] + }, + "WhatId": { + "type": [ + "string", + "null" + ] + }, + "Subject": { + "type": [ + "string", + "null" + ] + }, + "ActivityDate": { + "type": [ + "string", + "null" + ], + "format": "date" + }, + "Status": { + "type": [ + "string", + "null" + ] + }, + "Priority": { + "type": [ + "string", + "null" + ] + }, + "IsHighPriority": { + "type": [ + "boolean", + "null" + ] + }, + "OwnerId": { + "type": [ + "string", + "null" + ] + }, + "Description": { + "type": [ + "string", + "null" + ] + }, + "IsDeleted": { + "type": [ + "boolean", + "null" + ] + }, + "AccountId": { + "type": [ + "string", + "null" + ] + }, + "IsClosed": { + "type": [ + "boolean", + "null" + ] + }, + "CreatedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "CreatedById": { + "type": [ + "string", + "null" + ] + }, + "LastModifiedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastModifiedById": { + "type": [ + "string", + "null" + ] + }, + "SystemModstamp": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "IsArchived": { + "type": [ + "boolean", + "null" + ] + }, + "CallDurationInSeconds": { + "type": [ + "integer", + "null" + ] + }, + "CallType": { + "type": [ + "string", + "null" + ] + }, + "CallDisposition": { + "type": [ + "string", + "null" + ] + }, + "CallObject": { + "type": [ + "string", + "null" + ] + }, + "ReminderDateTime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "IsReminderSet": { + "type": [ + "boolean", + "null" + ] + }, + "RecurrenceActivityId": { + "type": [ + "string", + "null" + ] + }, + "IsRecurrence": { + "type": [ + "boolean", + "null" + ] + }, + "RecurrenceStartDateOnly": { + "type": [ + "string", + "null" + ], + "format": "date" + }, + "RecurrenceEndDateOnly": { + "type": [ + "string", + "null" + ], + "format": "date" + }, + "RecurrenceTimeZoneSidKey": { + "type": [ + "string", + "null" + ] + }, + "RecurrenceType": { + "type": [ + "string", + "null" + ] + }, + "RecurrenceInterval": { + "type": [ + "integer", + "null" + ] + }, + "RecurrenceDayOfWeekMask": { + "type": [ + "integer", + "null" + ] + }, + "RecurrenceDayOfMonth": { + "type": [ + "integer", + "null" + ] + }, + "RecurrenceInstance": { + "type": [ + "string", + "null" + ] + }, + "RecurrenceMonthOfYear": { + "type": [ + "string", + "null" + ] + }, + "RecurrenceRegeneratedType": { + "type": [ + "string", + "null" + ] + }, + "TaskSubtype": { + "type": [ + "string", + "null" + ] + }, + "CompletedDateTime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + } + } +} diff --git a/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/User.schema.json b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/User.schema.json new file mode 100644 index 000000000..f0b0ea8bf --- /dev/null +++ b/src/ingestion/connectors/crm/salesforce/source_salesforce/stream_schemas/User.schema.json @@ -0,0 +1,1131 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "required": [ + "unique_key" + ], + "properties": { + "Id": { + "type": [ + "string", + "null" + ] + }, + "Username": { + "type": [ + "string", + "null" + ] + }, + "LastName": { + "type": [ + "string", + "null" + ] + }, + "FirstName": { + "type": [ + "string", + "null" + ] + }, + "Name": { + "type": [ + "string", + "null" + ] + }, + "CompanyName": { + "type": [ + "string", + "null" + ] + }, + "Division": { + "type": [ + "string", + "null" + ] + }, + "Department": { + "type": [ + "string", + "null" + ] + }, + "Title": { + "type": [ + "string", + "null" + ] + }, + "Street": { + "type": [ + "string", + "null" + ] + }, + "City": { + "type": [ + "string", + "null" + ] + }, + "State": { + "type": [ + "string", + "null" + ] + }, + "PostalCode": { + "type": [ + "string", + "null" + ] + }, + "Country": { + "type": [ + "string", + "null" + ] + }, + "StateCode": { + "type": [ + "string", + "null" + ] + }, + "CountryCode": { + "type": [ + "string", + "null" + ] + }, + "Latitude": { + "type": [ + "number", + "null" + ] + }, + "Longitude": { + "type": [ + "number", + "null" + ] + }, + "GeocodeAccuracy": { + "type": [ + "string", + "null" + ] + }, + "Address": { + "type": [ + "string", + "null" + ] + }, + "Email": { + "type": [ + "string", + "null" + ] + }, + "EmailPreferencesAutoBcc": { + "type": [ + "boolean", + "null" + ] + }, + "EmailPreferencesAutoBccStayInTouch": { + "type": [ + "boolean", + "null" + ] + }, + "EmailPreferencesStayInTouchReminder": { + "type": [ + "boolean", + "null" + ] + }, + "SenderEmail": { + "type": [ + "string", + "null" + ] + }, + "SenderName": { + "type": [ + "string", + "null" + ] + }, + "Signature": { + "type": [ + "string", + "null" + ] + }, + "StayInTouchSubject": { + "type": [ + "string", + "null" + ] + }, + "StayInTouchSignature": { + "type": [ + "string", + "null" + ] + }, + "StayInTouchNote": { + "type": [ + "string", + "null" + ] + }, + "Phone": { + "type": [ + "string", + "null" + ] + }, + "Fax": { + "type": [ + "string", + "null" + ] + }, + "MobilePhone": { + "type": [ + "string", + "null" + ] + }, + "Alias": { + "type": [ + "string", + "null" + ] + }, + "CommunityNickname": { + "type": [ + "string", + "null" + ] + }, + "BadgeText": { + "type": [ + "string", + "null" + ] + }, + "IsActive": { + "type": [ + "boolean", + "null" + ] + }, + "TimeZoneSidKey": { + "type": [ + "string", + "null" + ] + }, + "UserRoleId": { + "type": [ + "string", + "null" + ] + }, + "LocaleSidKey": { + "type": [ + "string", + "null" + ] + }, + "ReceivesInfoEmails": { + "type": [ + "boolean", + "null" + ] + }, + "ReceivesAdminInfoEmails": { + "type": [ + "boolean", + "null" + ] + }, + "EmailEncodingKey": { + "type": [ + "string", + "null" + ] + }, + "ProfileId": { + "type": [ + "string", + "null" + ] + }, + "UserType": { + "type": [ + "string", + "null" + ] + }, + "StartDay": { + "type": [ + "string", + "null" + ] + }, + "EndDay": { + "type": [ + "string", + "null" + ] + }, + "LanguageLocaleKey": { + "type": [ + "string", + "null" + ] + }, + "EmployeeNumber": { + "type": [ + "string", + "null" + ] + }, + "DelegatedApproverId": { + "type": [ + "string", + "null" + ] + }, + "ManagerId": { + "type": [ + "string", + "null" + ] + }, + "LastLoginDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastPasswordChangeDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "CreatedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "CreatedById": { + "type": [ + "string", + "null" + ] + }, + "LastModifiedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastModifiedById": { + "type": [ + "string", + "null" + ] + }, + "SystemModstamp": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "PasswordExpirationDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "NumberOfFailedLogins": { + "type": [ + "integer", + "null" + ] + }, + "SuAccessExpirationDate": { + "type": [ + "string", + "null" + ], + "format": "date" + }, + "OfflineTrialExpirationDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "OfflinePdaTrialExpirationDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "UserPermissionsMarketingUser": { + "type": [ + "boolean", + "null" + ] + }, + "UserPermissionsOfflineUser": { + "type": [ + "boolean", + "null" + ] + }, + "UserPermissionsCallCenterAutoLogin": { + "type": [ + "boolean", + "null" + ] + }, + "UserPermissionsSFContentUser": { + "type": [ + "boolean", + "null" + ] + }, + "UserPermissionsKnowledgeUser": { + "type": [ + "boolean", + "null" + ] + }, + "UserPermissionsInteractionUser": { + "type": [ + "boolean", + "null" + ] + }, + "UserPermissionsSupportUser": { + "type": [ + "boolean", + "null" + ] + }, + "UserPermissionsJigsawProspectingUser": { + "type": [ + "boolean", + "null" + ] + }, + "UserPermissionsSiteforceContributorUser": { + "type": [ + "boolean", + "null" + ] + }, + "UserPermissionsSiteforcePublisherUser": { + "type": [ + "boolean", + "null" + ] + }, + "UserPermissionsWorkDotComUserFeature": { + "type": [ + "boolean", + "null" + ] + }, + "ForecastEnabled": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesActivityRemindersPopup": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesEventRemindersCheckboxDefault": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesTaskRemindersCheckboxDefault": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesReminderSoundOff": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesDisableAllFeedsEmail": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesDisableFollowersEmail": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesDisableProfilePostEmail": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesDisableChangeCommentEmail": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesDisableLaterCommentEmail": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesDisProfPostCommentEmail": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesContentNoEmail": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesContentEmailAsAndWhen": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesApexPagesDeveloperMode": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesReceiveNoNotificationsAsApprover": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesReceiveNotificationsAsDelegatedApprover": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesHideCSNGetChatterMobileTask": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesDisableMentionsPostEmail": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesDisMentionsCommentEmail": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesHideCSNDesktopTask": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesHideChatterOnboardingSplash": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesHideSecondChatterOnboardingSplash": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesDisCommentAfterLikeEmail": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesDisableLikeEmail": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesSortFeedByComment": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesDisableMessageEmail": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesJigsawListUser": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesDisableBookmarkEmail": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesDisableSharePostEmail": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesActionLauncherEinsteinGptConsent": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesAssistiveActionsEnabledInActionLauncher": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesEnableAutoSubForFeeds": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesDisableFileShareNotificationsForApi": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowTitleToExternalUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowManagerToExternalUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowEmailToExternalUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowWorkPhoneToExternalUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowMobilePhoneToExternalUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowFaxToExternalUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowStreetAddressToExternalUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowCityToExternalUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowStateToExternalUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowPostalCodeToExternalUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowCountryToExternalUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowProfilePicToGuestUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowTitleToGuestUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowCityToGuestUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowStateToGuestUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowPostalCodeToGuestUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowCountryToGuestUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowForecastingChangeSignals": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesLiveAgentMiawSetupDeflection": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesHideS1BrowserUI": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesDisableEndorsementEmail": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesPathAssistantCollapsed": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesCacheDiagnostics": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowEmailToGuestUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowManagerToGuestUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowWorkPhoneToGuestUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowMobilePhoneToGuestUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowFaxToGuestUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowStreetAddressToGuestUsers": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesLightningExperiencePreferred": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesPreviewLightning": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesHideEndUserOnboardingAssistantModal": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesHideLightningMigrationModal": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesHideSfxWelcomeMat": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesHideBiggerPhotoCallout": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesGlobalNavBarWTShown": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesGlobalNavGridMenuWTShown": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesCreateLEXAppsWTShown": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesFavoritesWTShown": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesRecordHomeSectionCollapseWTShown": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesRecordHomeReservedWTShown": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesFavoritesShowTopFavorites": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesExcludeMailAppAttachments": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesSuppressTaskSFXReminders": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesSuppressEventSFXReminders": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesPreviewCustomTheme": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesHasCelebrationBadge": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesUserDebugModePref": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesSRHOverrideActivities": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesNewLightningReportRunPageEnabled": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesReverseOpenActivitiesView": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowTerritoryTimeZoneShifts": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesHasSentWarningEmail": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesHasSentWarningEmail238": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesHasSentWarningEmail240": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesNativeEmailClient": { + "type": [ + "boolean", + "null" + ] + }, + "UserPreferencesShowForecastingRoundedAmounts": { + "type": [ + "boolean", + "null" + ] + }, + "ContactId": { + "type": [ + "string", + "null" + ] + }, + "AccountId": { + "type": [ + "string", + "null" + ] + }, + "CallCenterId": { + "type": [ + "string", + "null" + ] + }, + "Extension": { + "type": [ + "string", + "null" + ] + }, + "FederationIdentifier": { + "type": [ + "string", + "null" + ] + }, + "AboutMe": { + "type": [ + "string", + "null" + ] + }, + "FullPhotoUrl": { + "type": [ + "string", + "null" + ] + }, + "SmallPhotoUrl": { + "type": [ + "string", + "null" + ] + }, + "IsExtIndicatorVisible": { + "type": [ + "boolean", + "null" + ] + }, + "OutOfOfficeMessage": { + "type": [ + "string", + "null" + ] + }, + "MediumPhotoUrl": { + "type": [ + "string", + "null" + ] + }, + "DigestFrequency": { + "type": [ + "string", + "null" + ] + }, + "DefaultGroupNotificationFrequency": { + "type": [ + "string", + "null" + ] + }, + "JigsawImportLimitOverride": { + "type": [ + "integer", + "null" + ] + }, + "LastViewedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "LastReferencedDate": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "BannerPhotoUrl": { + "type": [ + "string", + "null" + ] + }, + "SmallBannerPhotoUrl": { + "type": [ + "string", + "null" + ] + }, + "MediumBannerPhotoUrl": { + "type": [ + "string", + "null" + ] + }, + "IsProfilePhotoActive": { + "type": [ + "boolean", + "null" + ] + }, + "IndividualId": { + "type": [ + "string", + "null" + ] + } + } +} diff --git a/src/ingestion/connectors/crm/salesforce/source_salesforce/streams.py b/src/ingestion/connectors/crm/salesforce/source_salesforce/streams.py index 42e40f89c..6309d0108 100644 --- a/src/ingestion/connectors/crm/salesforce/source_salesforce/streams.py +++ b/src/ingestion/connectors/crm/salesforce/source_salesforce/streams.py @@ -1,8 +1,8 @@ """Stream classes: REST API (``/queryAll``), incremental via ``ConcurrentCursor``. Each stream emits records through :func:`envelope.envelope` so Bronze rows -carry ``tenant_id`` / ``source_id`` / ``unique_key`` / ``custom_fields`` in -addition to the raw SF fields. +carry ``tenant_id`` / ``source_id`` / ``unique_key`` / ``raw_data`` in addition +to the stream's declared SF fields. """ import logging @@ -11,6 +11,7 @@ from typing import ( Any, Callable, + FrozenSet, Iterable, List, Mapping, @@ -32,12 +33,13 @@ IsoMillisConcurrentStreamStateConverter, ) from airbyte_cdk.sources.streams.core import CheckpointMixin, StreamData -from airbyte_cdk.sources.streams.http import HttpClient, HttpStream, HttpSubStream +from airbyte_cdk.sources.streams.http import HttpClient, HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from source_salesforce.api import Salesforce -from source_salesforce.constants import PARENT_SALESFORCE_OBJECTS, UNSUPPORTED_FILTERING_STREAMS -from source_salesforce.envelope import envelope, inject_envelope_properties +from source_salesforce.constants import UNSUPPORTED_FILTERING_STREAMS +from source_salesforce.envelope import envelope +from source_salesforce.schema_loader import declared_field_names, stream_schema from source_salesforce.rate_limiting import ( SalesforceErrorHandler, default_backoff_handler, @@ -60,26 +62,24 @@ def __init__( stream_name: str, message_repository: MessageRepository, sobject_options: Mapping[str, Any] = None, - schema: dict = None, start_date=None, tenant_id: str = "", source_id: str = "", - custom_field_names: Optional[frozenset] = None, **kwargs, ): self.stream_name = stream_name self.pk = pk self.sf_api = sf_api super().__init__(**kwargs) - self.schema: Mapping[str, Any] = schema # type: ignore[assignment] self.sobject_options = sobject_options self.start_date = self.format_start_date(start_date) self._message_repository = message_repository + self._sf_field_names: Optional[Tuple[str, ...]] = None + self._unavailable_warned = False # Insight envelope context — used in read_records() to inject tenant / - # source / unique_key / custom_fields onto every emitted record. + # source / unique_key / raw_data onto every record. self._tenant_id = tenant_id self._source_id = source_id - self._custom_field_names: frozenset = custom_field_names or frozenset() # Tracks envelope-key collisions so we only warn once per offender # per stream instead of every record. self._envelope_collisions_seen: set = set() @@ -105,57 +105,59 @@ def read_records( Every record yielded by the upstream reader is passed through :func:`envelope.envelope` so Bronze gets tenant_id / source_id / - unique_key / data_source / collected_at / custom_fields. + unique_key / data_source / collected_at / raw_data. """ + if not self._sobject_available(): + return + for record in super().read_records(sync_mode, cursor_field, stream_slice, stream_state): if isinstance(record, Mapping): yield envelope( record, tenant_id=self._tenant_id, source_id=self._source_id, - custom_field_names=self._custom_field_names, + declared_fields=self.declared_fields, collision_seen=self._envelope_collisions_seen, ) else: # State / log / trace messages pass through untouched. yield record - def _sf_properties(self) -> Mapping[str, Any]: + @property + def declared_fields(self) -> FrozenSet[str]: + """SF fields this stream emits as Bronze columns; the rest ride in ``raw_data``.""" + return declared_field_names(self.name) + + def _sobject_available(self) -> bool: + """Whether this org exposes the sobject, warning once when it does not. + + The catalog is org-independent, so a stream can be advertised on an org + that does not license or expose its object. That is a per-org fact, not + a failure: the stream completes empty and the rest of the sync runs. + The answer comes off the describe the stream already needs for SOQL. + """ + available = self.sf_api.is_queryable(self.name) + if not available and not self._unavailable_warned: + self._unavailable_warned = True + self.logger.warning( + "Stream %s is not exposed by this org; syncing it as empty.", self.name + ) + return available + + def _sf_properties(self) -> Tuple[str, ...]: """All describe-reported fields (standard + custom). Used for SOQL. - SOQL needs every field present on the sobject so custom values can - reach :func:`envelope.envelope`, which routes them into the - ``custom_fields`` blob. + SOQL needs every field present on the sobject so custom and undeclared + standard values can reach :func:`envelope.envelope`, which preserves + them in ``raw_data``. """ - if not self.schema: - self.schema = self.sf_api.generate_schema(self.name) - return self.schema.get("properties", {}) + if self._sf_field_names is None: + self._sf_field_names = self.sf_api.field_names(self.name) + return self._sf_field_names def get_json_schema(self) -> Mapping[str, Any]: - """Advertise schema to the destination. - - - Start from describe-generated properties. - - Strip ``__c`` custom fields — their values are routed into the - ``custom_fields`` JSON blob by :func:`envelope.envelope`, so top-level - columns would always be NULL and create per-org schema drift in - Bronze. This is the main reason our Bronze stays stable across orgs. - - Add the Insight envelope fields (``tenant_id`` / ``source_id`` / - ``unique_key`` / ``data_source`` / ``collected_at`` / ``custom_fields``). - """ - if not self.schema: - self.schema = self.sf_api.generate_schema(self.name) - schema = { - "$schema": self.schema.get("$schema", "http://json-schema.org/draft-07/schema#"), - "type": self.schema.get("type", "object"), - "additionalProperties": self.schema.get("additionalProperties", True), - "properties": { - k: v - for k, v in self.schema.get("properties", {}).items() - if k not in self._custom_field_names - }, - } - inject_envelope_properties(schema) - return schema + """Advertise the stream's static schema to the destination.""" + return stream_schema(self.name) @staticmethod def format_start_date(start_date: Optional[str]) -> Optional[str]: @@ -187,15 +189,12 @@ def url_base(self) -> str: @property def too_many_properties(self): # Size check uses the full SF field list (what actually goes into SOQL). - selected_properties = self._sf_properties() - properties_length = len(urllib.parse.quote(",".join(p for p in selected_properties))) + properties_length = len(urllib.parse.quote(",".join(self._sf_properties()))) return properties_length > self.max_properties_length def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: yield from response.json()["records"] - # get_json_schema() is overridden above to inject envelope fields. - def get_error_display_message(self, exception: BaseException) -> Optional[str]: if isinstance(exception, exceptions.ConnectionError): return f"After {self.max_retries} retries the connector has failed with a network error. It looks like Salesforce API experienced temporary instability, please try again later." @@ -206,12 +205,12 @@ class PropertyChunk: Object that is used to keep track of the current state of a chunk of properties for the stream of records being synced. """ - properties: Mapping[str, Any] + properties: Tuple[str, ...] first_time: bool record_counter: int next_page: Optional[Mapping[str, Any]] - def __init__(self, properties: Mapping[str, Any]): + def __init__(self, properties: Tuple[str, ...]): self.properties = properties self.first_time = True self.record_counter = 0 @@ -221,12 +220,16 @@ def __init__(self, properties: Mapping[str, Any]): class RestSalesforceStream(SalesforceStream): state_converter = IsoMillisConcurrentStreamStateConverter(is_sequential_state=False) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - # Property chunking needs a natural key (self.pk, i.e. SF Id) to - # reassemble split records before envelope runs. Raise rather than - # assert so the failure survives `python -O` and gives operators a - # clear message instead of a bare AssertionError. + def _check_chunking_is_reassemblable(self) -> None: + """Guard the chunked-read precondition, at read time. + + Property chunking needs a natural key (self.pk, i.e. SF Id) to + reassemble split records before envelope runs. Checked here rather than + in ``__init__`` because the field count comes from describe, and + constructing a stream must stay free of API calls. Raise rather than + assert so the failure survives ``python -O`` and gives operators a clear + message instead of a bare AssertionError. + """ if self.too_many_properties and not self.pk: raise RuntimeError( f"Stream '{self.name}' has too many properties for REST " @@ -253,7 +256,7 @@ def request_params( stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, - property_chunk: Mapping[str, Any] = None, + property_chunk: Tuple[str, ...] = (), ) -> MutableMapping[str, Any]: """ Salesforce SOQL Query: https://developer.salesforce.com/docs/atlas.en-us.232.0.api_rest.meta/api_rest/dome_queryall.htm @@ -262,14 +265,8 @@ def request_params( # If `next_page_token` is set, subsequent requests use `nextRecordsUrl`, and do not include any parameters. return {} - property_chunk = property_chunk or {} - query = f"SELECT {','.join(property_chunk.keys())} FROM {self.name} " - - if self.name in PARENT_SALESFORCE_OBJECTS: - # add where clause: " WHERE ContentDocumentId IN ('06905000000NMXXXXX', ...)" - parent_field = PARENT_SALESFORCE_OBJECTS[self.name]["field"] - parent_ids = [f"'{parent_record[parent_field]}'" for parent_record in stream_slice["parents"]] - query += f" WHERE ContentDocumentId IN ({','.join(parent_ids)})" + property_chunk = property_chunk or () + query = f"SELECT {','.join(property_chunk)} FROM {self.name} " if self.pk and self.name not in UNSUPPORTED_FILTERING_STREAMS: # ORDER BY the SF natural key (Id), not the Insight unique_key — @@ -279,31 +276,32 @@ def request_params( return {"q": query} - def chunk_properties(self) -> Iterable[Mapping[str, Any]]: - # Use the full describe-derived field list (standard + custom). Custom - # fields are NOT in the destination schema but we still need them in - # SOQL so envelope() can route them into the ``custom_fields`` blob. - selected_properties = dict(self._sf_properties()) + def chunk_properties(self) -> Iterable[Tuple[str, ...]]: + # Use the full describe-derived field list (standard + custom). Fields + # outside the static schema are NOT destination columns but we still + # need them in SOQL so envelope() can preserve them in ``raw_data``. + selected_properties = self._sf_properties() - def empty_props_with_pk_if_present(): + def empty_props_with_pk_if_present() -> List[str]: # Chunk reassembly keys by SF Id (self.pk), not the Insight # unique_key which doesn't exist on the SF response. - return {self.pk: selected_properties[self.pk]} if self.pk else {} + return [self.pk] if self.pk and self.pk in selected_properties else [] summary_length = 0 local_properties = empty_props_with_pk_if_present() - for property_name, value in selected_properties.items(): + for property_name in selected_properties: current_property_length = len(urllib.parse.quote(f"{property_name},")) if current_property_length + summary_length >= self.max_properties_length: - yield local_properties + yield tuple(local_properties) local_properties = empty_props_with_pk_if_present() summary_length = 0 - local_properties[property_name] = value + if property_name not in local_properties: + local_properties.append(property_name) summary_length += current_property_length if local_properties: - yield local_properties + yield tuple(local_properties) @staticmethod def _next_chunk_id(property_chunks: Mapping[int, PropertyChunk]) -> Optional[int]: @@ -336,6 +334,7 @@ def _read_pages( stream_state: Mapping[str, Any] = None, ) -> Iterable[StreamData]: stream_state = stream_state or {} + self._check_chunking_is_reassemblable() records_by_primary_key = {} property_chunks: Mapping[int, PropertyChunk] = { index: PropertyChunk(properties=properties) for index, properties in enumerate(self.chunk_properties()) @@ -407,7 +406,7 @@ def _fetch_next_page_for_chunk( stream_slice: Mapping[str, Any] = None, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, - property_chunk: Mapping[str, Any] = None, + property_chunk: Tuple[str, ...] = (), ) -> Tuple[requests.PreparedRequest, requests.Response]: request_headers = self.request_headers( stream_state=stream_state, @@ -445,34 +444,6 @@ def _fetch_next_page_for_chunk( ) -class BatchedSubStream(HttpSubStream): - state_converter = IsoMillisConcurrentStreamStateConverter(is_sequential_state=False) - SLICE_BATCH_SIZE = 200 - - def stream_slices( - self, - sync_mode: SyncMode, - cursor_field: Optional[List[str]] = None, - stream_state: Optional[Mapping[str, Any]] = None, - ) -> Iterable[Optional[Mapping[str, Any]]]: - """Instead of yielding one parent record at a time, make stream slice contain a batch of parent records. - - It allows to get records by one requests (instead of only one). - """ - batched_slice = [] - for stream_slice in super().stream_slices(sync_mode, cursor_field, stream_state): - if len(batched_slice) == self.SLICE_BATCH_SIZE: - yield {"parents": batched_slice} - batched_slice = [] - batched_slice.append(stream_slice["parent"]) - if batched_slice: - yield {"parents": batched_slice} - - -class RestSalesforceSubStream(BatchedSubStream, RestSalesforceStream): - pass - - class IncrementalRestSalesforceStream(RestSalesforceStream, CheckpointMixin, ABC): def __init__(self, replication_key: str, stream_slice_step: str = "P30D", **kwargs): self.replication_key = replication_key @@ -513,7 +484,7 @@ def request_params( stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, - property_chunk: Mapping[str, Any] = None, + property_chunk: Tuple[str, ...] = (), ) -> MutableMapping[str, Any]: if next_page_token: """ @@ -521,8 +492,8 @@ def request_params( """ return {} - property_chunk = property_chunk or {} - select_fields = ",".join(property_chunk.keys()) + property_chunk = property_chunk or () + select_fields = ",".join(property_chunk) table_name = self.name if not self._stream_slicer_cursor: diff --git a/src/ingestion/connectors/crm/salesforce/tests/conftest.py b/src/ingestion/connectors/crm/salesforce/tests/conftest.py index f1406056f..3f20f2676 100644 --- a/src/ingestion/connectors/crm/salesforce/tests/conftest.py +++ b/src/ingestion/connectors/crm/salesforce/tests/conftest.py @@ -11,6 +11,7 @@ import json from typing import Any +from unittest.mock import Mock import pytest import requests @@ -31,19 +32,9 @@ "insight_source_id": SOURCE, } -# Small describe-derived schema: standard fields + one custom (``__c``) field. -ACCOUNT_SCHEMA = { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "additionalProperties": True, - "properties": { - "Id": {"type": ["string", "null"]}, - "Name": {"type": ["string", "null"]}, - "SystemModstamp": {"type": ["string", "null"], "format": "date-time"}, - "Custom__c": {"type": ["string", "null"]}, - }, -} -CUSTOM_FIELDS = frozenset({"Custom__c"}) +# Fields describe reports for the Account sobject in these tests: declared +# standard fields plus one custom (``__c``) field. +ACCOUNT_FIELDS = ("Id", "Name", "SystemModstamp", "Custom__c") class FakeResponse: @@ -102,29 +93,37 @@ def make_sf(**overrides: Any) -> Salesforce: return Salesforce(**kwargs) -_DEFAULT_SCHEMA = object() # sentinel: allows passing schema=None explicitly +_DEFAULT_FIELDS = object() # sentinel: allows passing sf_fields=None explicitly def make_stream( cls=RestSalesforceStream, stream_name: str = "Account", - schema: Any = _DEFAULT_SCHEMA, + sf_fields: Any = _DEFAULT_FIELDS, pk: str = "Id", sf: Salesforce | None = None, **extra: Any, ): - """Construct a stream with a real (offline) Salesforce client.""" + """Construct a stream with a real (offline) Salesforce client. + + ``sf_fields`` stubs the describe-reported field list the stream builds SOQL + from, and marks the sobject available; ``None`` leaves the real + (network-bound) lookups in place. + """ sf = sf or make_sf() + if sf_fields is not None: + fields = ACCOUNT_FIELDS if sf_fields is _DEFAULT_FIELDS else tuple(sf_fields) + sf.field_names = Mock(return_value=fields) + sf.is_queryable = Mock(return_value=True) + kwargs = dict( sf_api=sf, pk=pk, stream_name=stream_name, message_repository=InMemoryMessageRepository(), - schema=dict(ACCOUNT_SCHEMA) if schema is _DEFAULT_SCHEMA else schema, authenticator=SalesforceAuthenticator(sf._token_provider), tenant_id=TENANT, source_id=SOURCE, - custom_field_names=CUSTOM_FIELDS, ) kwargs.update(extra) return cls(**kwargs) diff --git a/src/ingestion/connectors/crm/salesforce/tests/test_api.py b/src/ingestion/connectors/crm/salesforce/tests/test_api.py index 1be6916bd..c27ff8e33 100644 --- a/src/ingestion/connectors/crm/salesforce/tests/test_api.py +++ b/src/ingestion/connectors/crm/salesforce/tests/test_api.py @@ -6,18 +6,10 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.models import ( - AirbyteStream, - ConfiguredAirbyteCatalog, - ConfiguredAirbyteStream, - DestinationSyncMode, - SyncMode, -) from airbyte_cdk.utils import AirbyteTracedException from requests.exceptions import RequestException from source_salesforce.api import Salesforce, SalesforceAuthenticator, SalesforceTokenProvider from source_salesforce.constants import CRM_STREAMS, TOKEN_REFRESH_INTERVAL_SECONDS -from source_salesforce.exceptions import TypeSalesforceException from tests.conftest import INSTANCE_URL, FakeResponse, make_sf @@ -40,6 +32,16 @@ def send_request(http_method, url, **kwargs): class TestTokenProvider: + def test_first_use_authenticates(self, sf): + sf.login = Mock(side_effect=lambda: setattr(sf, "access_token", "tok")) + assert sf._token_provider.get_token() == "tok" + sf.login.assert_called_once() + + def test_first_login_failure_surfaces(self, sf): + sf.login = Mock(side_effect=RequestException("no creds")) + with pytest.raises(RequestException): + sf._token_provider.get_token() + def test_fresh_token_not_refreshed(self, sf): sf.access_token = "tok" sf.login = Mock() @@ -158,7 +160,7 @@ def test_missing_access_token_raises(self, sf): # --------------------------------------------------------------------------- -# describe + schema generation +# describe + field discovery # --------------------------------------------------------------------------- ACCOUNT_DESCRIBE = { @@ -174,10 +176,19 @@ def test_missing_access_token_raises(self, sf): class TestDescribe: def test_global_describe_url(self, sf): + sf.access_token = "tok" calls = _stub_send_request(sf, [FakeResponse({"sobjects": []})]) assert sf.describe() == {"sobjects": []} assert calls[0]["url"] == (f"{INSTANCE_URL}/services/data/{sf.version}/sobjects") + def test_describe_authenticates_when_no_token_yet(self, sf): + """Describe is often a sync's first authenticated call.""" + sf.login = Mock(side_effect=lambda: setattr(sf, "access_token", "tok")) + calls = _stub_send_request(sf, [FakeResponse(ACCOUNT_DESCRIBE)]) + sf.describe("Account") + sf.login.assert_called_once() + assert calls[0]["headers"]["Authorization"] == "Bearer tok" + def test_sobject_describe_url_and_auth_header(self, sf): sf.access_token = "tok" calls = _stub_send_request(sf, [FakeResponse(ACCOUNT_DESCRIBE)]) @@ -186,64 +197,45 @@ def test_sobject_describe_url_and_auth_header(self, sf): assert calls[0]["headers"]["Authorization"] == "Bearer tok" def test_404_for_named_sobject_is_config_error(self, sf): + sf.access_token = "tok" _stub_send_request(sf, [FakeResponse({}, status_code=404)]) with pytest.raises(AirbyteTracedException, match="'Missing' not found"): sf.describe("Missing") def test_other_error_is_system_error(self, sf): + sf.access_token = "tok" _stub_send_request(sf, [FakeResponse({}, status_code=500)]) with pytest.raises(AirbyteTracedException, match="describe\\('global'\\) failed"): sf.describe() -class TestGenerateSchema: - def test_properties_built_from_fields(self, sf): +class TestFieldNames: + def test_returns_standard_and_custom_fields(self, sf): sf.describe = Mock(return_value=ACCOUNT_DESCRIBE) - schema = sf.generate_schema("Account") - assert schema["type"] == "object" - assert schema["properties"]["Id"] == {"type": ["string", "null"]} - assert schema["properties"]["AnnualRevenue"] == {"type": ["number", "null"]} - # Describe response cached for get_custom_field_names(). - assert sf._sobject_describes["Account"] is ACCOUNT_DESCRIBE + names = sf.field_names("Account") + assert "Id" in names and "Custom__c" in names - def test_unnamed_schema_not_cached(self, sf): - sf.describe = Mock(return_value=ACCOUNT_DESCRIBE) - sf.generate_schema() - assert sf._sobject_describes == {} - - -class TestGetCustomFieldNames: - def test_from_cache_no_extra_describe(self, sf): + def test_cached_describe_reused(self, sf): sf._sobject_describes["Account"] = ACCOUNT_DESCRIBE sf.describe = Mock() - assert sf.get_custom_field_names("Account") == frozenset({"Custom__c"}) + assert "Custom__c" in sf.field_names("Account") sf.describe.assert_not_called() - def test_fallback_fetches_on_demand(self, sf): + def test_describe_fetched_once_per_sobject(self, sf): sf.describe = Mock(return_value=ACCOUNT_DESCRIBE) - assert sf.get_custom_field_names("Account") == frozenset({"Custom__c"}) - sf.describe.assert_called_once_with("Account") + sf.field_names("Account") + sf.field_names("Account") + assert sf.describe.call_count == 1 assert sf._sobject_describes["Account"] is ACCOUNT_DESCRIBE + def test_absent_sobject_yields_no_fields(self, sf): + sf.describe = Mock(return_value=None) + assert sf.field_names("Ghost") == () + assert sf.is_queryable("Ghost") is False -class TestGenerateSchemas: - def test_parallel_success(self, sf): - sf.describe = Mock(return_value=ACCOUNT_DESCRIBE) - schemas = sf.generate_schemas({"Account": {}, "Contact": {}}) - assert set(schemas) == {"Account", "Contact"} - assert schemas["Account"]["properties"]["Id"] == {"type": ["string", "null"]} - - def test_request_error_becomes_traced_exception(self, sf): - sf.generate_schema = Mock(side_effect=RequestException("timeout")) - with pytest.raises(AirbyteTracedException, match="Schema could not be extracted"): - sf.generate_schemas({"Account": {}}) - - def test_chunked_across_parallel_task_size(self, sf, monkeypatch): - # Force two sequential batches to cover the outer chunking loop. - monkeypatch.setattr(sf, "parallel_tasks_size", 1) - sf.describe = Mock(return_value=ACCOUNT_DESCRIBE) - schemas = sf.generate_schemas({"Account": {}, "Contact": {}}) - assert set(schemas) == {"Account", "Contact"} + def test_non_queryable_sobject_reported_unavailable(self, sf): + sf.describe = Mock(return_value={"fields": [{"name": "Id"}], "queryable": False}) + assert sf.is_queryable("Account") is False # --------------------------------------------------------------------------- @@ -266,44 +258,23 @@ def test_blacklist_is_union_of_both_lists(self, sf): blacklist = sf.get_streams_black_list() assert "Vote" in blacklist and "ContentBody" in blacklist - def test_default_selection_uses_crm_streams(self, sf): - sf.describe = Mock(return_value=_global_describe(CRM_STREAMS + ["Unrelated"])) - validated = sf.get_validated_streams() - assert set(validated) == set(CRM_STREAMS) - - def test_non_queryable_skipped(self, sf): - sf.describe = Mock(return_value=_global_describe(["Account"], queryable=False)) - assert sf.get_validated_streams() == {} + def test_syncable_streams_are_the_curated_set(self, sf): + sf.describe = Mock(side_effect=AssertionError("describe must not be called")) + assert sf.syncable_streams() == list(CRM_STREAMS) - def test_unsupported_streams_skipped(self, sf): - sf.describe = Mock(return_value=_global_describe(["ActivityMetric", "Account"])) - assert set(sf.get_validated_streams()) == {"Account"} + def test_syncable_streams_exclude_streams_needing_an_object_id(self, sf, monkeypatch): + monkeypatch.setattr("source_salesforce.api.CRM_STREAMS", ["Account", "ActivityMetric"]) + assert sf.syncable_streams() == ["Account"] - def test_missing_requested_streams_logged(self, sf, caplog): - sf.describe = Mock(return_value=_global_describe(["Account"])) - with caplog.at_level("WARNING", logger="airbyte"): - validated = sf.get_validated_streams() - assert set(validated) == {"Account"} - assert "not queryable in this org" in caplog.text - - def test_catalog_intersection_wins(self, sf): + def test_unavailable_streams_reports_what_the_org_lacks(self, sf): sf.describe = Mock(return_value=_global_describe(["Account", "Contact"])) - catalog = ConfiguredAirbyteCatalog( - streams=[ - ConfiguredAirbyteStream( - stream=AirbyteStream(name="Account", json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), - sync_mode=SyncMode.full_refresh, - destination_sync_mode=DestinationSyncMode.overwrite, - ), - ConfiguredAirbyteStream( - stream=AirbyteStream(name="Ghost", json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), - sync_mode=SyncMode.full_refresh, - destination_sync_mode=DestinationSyncMode.overwrite, - ), - ] - ) - validated = sf.get_validated_streams(catalog=catalog) - assert set(validated) == {"Account"} + unavailable = sf.unavailable_streams() + assert "Account" not in unavailable + assert "Opportunity" in unavailable + + def test_unavailable_streams_counts_non_queryable_as_missing(self, sf): + sf.describe = Mock(return_value=_global_describe(CRM_STREAMS, queryable=False)) + assert set(sf.unavailable_streams()) == set(CRM_STREAMS) # --------------------------------------------------------------------------- @@ -326,34 +297,3 @@ def test_fallback_chain(self): def test_no_cursor_no_pk(self): assert Salesforce.get_pk_and_replication_key({"properties": {"Name": {}}}) == (None, None) assert Salesforce.get_pk_and_replication_key({}) == (None, None) - - -class TestFieldToPropertySchema: - @pytest.mark.parametrize( - "sf_type,expected", - [ - ("string", {"type": ["string", "null"]}), - ("picklist", {"type": ["string", "null"]}), - ("datetime", {"type": ["string", "null"], "format": "date-time"}), - ("date", {"type": ["string", "null"], "format": "date"}), - ("currency", {"type": ["number", "null"]}), - ("int", {"type": ["integer", "null"]}), - ("boolean", {"type": ["boolean", "null"]}), - ("base64", {"type": ["string", "null"], "format": "base64"}), - ("anyType", {"type": ["string", "null"]}), - ("calculated", {"type": ["string", "null"]}), - ], - ) - def test_scalar_types(self, sf_type, expected): - assert Salesforce.field_to_property_schema({"type": sf_type}) == expected - - def test_address_and_location_are_objects(self): - address = Salesforce.field_to_property_schema({"type": "address"}) - assert address["type"] == ["object", "null"] - assert "street" in address["properties"] - location = Salesforce.field_to_property_schema({"type": "location"}) - assert set(location["properties"]) == {"longitude", "latitude"} - - def test_unknown_type_raises(self): - with pytest.raises(TypeSalesforceException, match="Unsupported Salesforce field type"): - Salesforce.field_to_property_schema({"type": "hologram"}) diff --git a/src/ingestion/connectors/crm/salesforce/tests/test_envelope.py b/src/ingestion/connectors/crm/salesforce/tests/test_envelope.py index 282416816..adbeca175 100644 --- a/src/ingestion/connectors/crm/salesforce/tests/test_envelope.py +++ b/src/ingestion/connectors/crm/salesforce/tests/test_envelope.py @@ -10,9 +10,16 @@ SOURCE = "S" -def _wrap(record, custom=frozenset(), collision_seen=None): +DECLARED = frozenset({"Id", "Name"}) + + +def _wrap(record, collision_seen=None, declared=DECLARED): return envelope( - record, tenant_id=TENANT, source_id=SOURCE, custom_field_names=custom, collision_seen=collision_seen + record, + tenant_id=TENANT, + source_id=SOURCE, + declared_fields=declared, + collision_seen=collision_seen, ) @@ -30,15 +37,10 @@ def test_attributes_metadata_dropped(self): out = _wrap({"Id": "001", "attributes": {"type": "Account"}}) assert "attributes" not in out - def test_custom_fields_packed_into_json_blob(self): - out = _wrap( - {"Id": "001", "Name": "Acme", "Custom__c": "x", "Other__c": 5}, custom=frozenset({"Custom__c", "Other__c"}) - ) + def test_custom_field_is_not_a_column(self): + out = _wrap({"Id": "001", "Name": "Acme", "Custom__c": "x", "Other__c": 5}) assert "Custom__c" not in out and "Other__c" not in out - assert json.loads(out["custom_fields"]) == {"Custom__c": "x", "Other__c": 5} - - def test_no_custom_fields_yields_empty_blob(self): - assert _wrap({"Id": "001"})["custom_fields"] == "{}" + assert json.loads(out["raw_data"])["Custom__c"] == "x" def test_reserved_field_collision_dropped_and_warned_once(self, caplog): seen: set = set() @@ -76,6 +78,45 @@ def test_missing_id_derives_content_hash(self, caplog): assert other["unique_key"] != out["unique_key"] +class TestRawData: + def test_holds_every_source_field(self): + out = _wrap({"Id": "001", "Name": "Acme", "Undeclared": "u", "Custom__c": "c"}) + assert json.loads(out["raw_data"]) == { + "Id": "001", + "Name": "Acme", + "Undeclared": "u", + "Custom__c": "c", + } + + def test_undeclared_field_is_not_a_column(self): + out = _wrap({"Id": "001", "Undeclared": "u"}) + assert "Undeclared" not in out + assert json.loads(out["raw_data"])["Undeclared"] == "u" + + def test_excludes_attributes_metadata_and_envelope_collisions(self): + out = _wrap({"Id": "001", "attributes": {"type": "Account"}, "tenant_id": "EVIL"}) + assert json.loads(out["raw_data"]) == {"Id": "001"} + + def test_record_without_source_fields_yields_empty_blob(self): + assert _wrap({"attributes": {"type": "Account"}})["raw_data"] == "{}" + + def test_long_values_truncated_but_blob_stays_valid_json(self): + out = _wrap({"Id": "001", "Description": "x" * 5000}) + raw = json.loads(out["raw_data"]) + assert raw["Description"].endswith("…[truncated]") + assert len(raw["Description"].encode("utf-8")) <= 2048 + + def test_truncation_reaches_nested_values(self): + out = _wrap({"Id": "001", "Nested": {"Inner": ["y" * 5000]}}) + inner = json.loads(out["raw_data"])["Nested"]["Inner"][0] + assert inner.endswith("…[truncated]") + + def test_source_record_not_mutated(self): + record = {"Id": "001", "Description": "x" * 5000} + _wrap(record) + assert len(record["Description"]) == 5000 + + class TestInjectEnvelopeProperties: def test_adds_all_envelope_fields(self): schema = {"properties": {"Id": {"type": ["string", "null"]}}} diff --git a/src/ingestion/connectors/crm/salesforce/tests/test_source.py b/src/ingestion/connectors/crm/salesforce/tests/test_source.py index 0502134bc..b13be3d56 100644 --- a/src/ingestion/connectors/crm/salesforce/tests/test_source.py +++ b/src/ingestion/connectors/crm/salesforce/tests/test_source.py @@ -18,9 +18,10 @@ ) from airbyte_cdk.utils.traced_exception import AirbyteTracedException from requests import exceptions +from source_salesforce.schema_loader import stream_schema from source_salesforce.source import SourceSalesforce -from source_salesforce.streams import IncrementalRestSalesforceStream, RestSalesforceStream, RestSalesforceSubStream -from tests.conftest import ACCOUNT_SCHEMA, CONFIG, make_http_response, make_sf +from source_salesforce.streams import IncrementalRestSalesforceStream, RestSalesforceStream +from tests.conftest import CONFIG, make_http_response, make_sf logger = logging.getLogger("test") @@ -191,10 +192,20 @@ def login(self): class TestCheckConnection: def test_success(self, monkeypatch): sf = Mock() + sf.unavailable_streams.return_value = [] monkeypatch.setattr(SourceSalesforce, "_get_sf_object", staticmethod(lambda config: sf)) source = make_source() assert source.check_connection(logger, CONFIG) == (True, None) - sf.describe.assert_called_once() + sf.unavailable_streams.assert_called_once() + + def test_reports_streams_the_org_does_not_expose(self, monkeypatch, caplog): + sf = Mock() + sf.unavailable_streams.return_value = ["Case", "Lead"] + monkeypatch.setattr(SourceSalesforce, "_get_sf_object", staticmethod(lambda config: sf)) + source = make_source() + with caplog.at_level("WARNING"): + assert source.check_connection(logger, CONFIG) == (True, None) + assert "Case, Lead" in caplog.text def test_invalid_slice_step_fails_before_login(self, monkeypatch): get_sf = Mock() @@ -219,23 +230,16 @@ def test_invalid_lookback_fails_before_login(self, monkeypatch): class TestGetStreamType: - def test_substream_for_parented_object(self): - full_refresh, incremental = SourceSalesforce._get_stream_type("ContentDocumentLink") - assert full_refresh is RestSalesforceSubStream - assert incremental is IncrementalRestSalesforceStream - - def test_plain_rest_otherwise(self): - full_refresh, _ = SourceSalesforce._get_stream_type("Account") + def test_rest_classes_for_every_stream(self): + full_refresh, incremental = SourceSalesforce._get_stream_type("Account") assert full_refresh is RestSalesforceStream + assert incremental is IncrementalRestSalesforceStream def _sf_stub(): """Real Salesforce client with describe-time HTTP monkeypatched out.""" sf = make_sf() - sf.generate_schemas = Mock( - side_effect=lambda stream_objects: {name: dict(ACCOUNT_SCHEMA) for name in stream_objects} - ) - sf.get_custom_field_names = Mock(return_value=frozenset({"Custom__c"})) + sf.field_names = Mock(return_value=("Id", "Name", "SystemModstamp", "Custom__c")) return sf @@ -243,14 +247,13 @@ class TestPrepareStream: def test_incremental_when_replication_key_present(self): source = make_source() stream_class, kwargs = source.prepare_stream( - "Account", ACCOUNT_SCHEMA, {"queryable": True}, _sf_stub(), Mock(), CONFIG + "Account", stream_schema("Account"), {"queryable": True}, _sf_stub(), Mock(), CONFIG ) assert stream_class is IncrementalRestSalesforceStream assert kwargs["replication_key"] == "SystemModstamp" assert kwargs["stream_slice_step"] == "P30D" assert kwargs["tenant_id"] == CONFIG["insight_tenant_id"] assert kwargs["source_id"] == CONFIG["insight_source_id"] - assert kwargs["custom_field_names"] == frozenset({"Custom__c"}) def test_full_refresh_without_replication_key(self): source = make_source() @@ -261,13 +264,13 @@ def test_full_refresh_without_replication_key(self): def test_unsupported_filtering_forces_full_refresh(self): source = make_source() - stream_class, _ = source.prepare_stream("LoginEvent", ACCOUNT_SCHEMA, {}, _sf_stub(), Mock(), CONFIG) + stream_class, _ = source.prepare_stream("LoginEvent", stream_schema("Account"), {}, _sf_stub(), Mock(), CONFIG) assert stream_class is RestSalesforceStream def test_slice_step_from_config(self): source = make_source() _, kwargs = source.prepare_stream( - "Account", ACCOUNT_SCHEMA, {}, _sf_stub(), Mock(), {**CONFIG, "salesforce_stream_slice_step": "P7D"} + "Account", stream_schema("Account"), {}, _sf_stub(), Mock(), {**CONFIG, "salesforce_stream_slice_step": "P7D"} ) assert kwargs["stream_slice_step"] == "P7D" @@ -280,15 +283,6 @@ def test_incremental_stream_wrapped_with_cursor(self): # Facade wraps the legacy stream; the slicer cursor must be attached. assert streams[0].cursor_field == "SystemModstamp" - def test_substream_gets_parent_stream(self): - sf = _sf_stub() - source = make_source() - streams = source.generate_streams( - CONFIG, {"ContentDocumentLink": {}, "ContentDocument": {"queryable": True}}, sf - ) - names = {s.name for s in streams} - assert names == {"ContentDocumentLink", "ContentDocument"} - def test_full_refresh_catalog_disables_cursor(self): source = make_source(catalog=make_catalog("Account", SyncMode.full_refresh)) streams = source.generate_streams(CONFIG, {"Account": {}}, _sf_stub()) @@ -306,8 +300,8 @@ def test_lookback_and_slice_step_from_config(self): class TestStreams: def test_streams_injects_default_start_date(self, monkeypatch): sf = Mock() - sf.get_validated_streams.return_value = {} - monkeypatch.setattr(SourceSalesforce, "_get_sf_object", staticmethod(lambda config: sf)) + sf.syncable_streams.return_value = [] + monkeypatch.setattr(SourceSalesforce, "_build_sf_client", staticmethod(lambda config: sf)) seen = {} def fake_generate(config, stream_objects, sf_object): @@ -325,12 +319,26 @@ def fake_generate(config, stream_objects, sf_object): def test_streams_keeps_explicit_start_date(self, monkeypatch): sf = Mock() - sf.get_validated_streams.return_value = {} - monkeypatch.setattr(SourceSalesforce, "_get_sf_object", staticmethod(lambda config: sf)) + sf.syncable_streams.return_value = ["Account"] + monkeypatch.setattr(SourceSalesforce, "_build_sf_client", staticmethod(lambda config: sf)) source = make_source() - monkeypatch.setattr(source, "generate_streams", lambda config, objs, sf_obj: []) + seen = {} + monkeypatch.setattr( + source, "generate_streams", lambda config, objs, sf_obj: seen.update(objs=objs) or [] + ) source.streams(CONFIG) - sf.get_validated_streams.assert_called_once_with(catalog=None) + assert seen["objs"] == {"Account": {}} + + def test_streams_never_authenticates(self, monkeypatch): + """Discover advertises static schemas, so it must issue no API call.""" + sf = Mock() + sf.syncable_streams.return_value = [] + sf.login.side_effect = AssertionError("login must not be called") + sf.describe.side_effect = AssertionError("describe must not be called") + monkeypatch.setattr(SourceSalesforce, "_build_sf_client", staticmethod(lambda config: sf)) + source = make_source() + monkeypatch.setattr(source, "generate_streams", lambda config, objs, sf_obj: []) + assert source.streams(CONFIG) == [] class TestCreateStreamSlicerCursor: diff --git a/src/ingestion/connectors/crm/salesforce/tests/test_streams.py b/src/ingestion/connectors/crm/salesforce/tests/test_streams.py index d4378dcb7..abc2a19e9 100644 --- a/src/ingestion/connectors/crm/salesforce/tests/test_streams.py +++ b/src/ingestion/connectors/crm/salesforce/tests/test_streams.py @@ -1,9 +1,10 @@ """Tests for source_salesforce.streams: SOQL construction, pagination, -property chunking and record reassembly, envelope injection, substream batching. +property chunking and record reassembly, envelope injection. """ from __future__ import annotations +import json import urllib.parse from unittest.mock import Mock @@ -12,21 +13,16 @@ from airbyte_cdk.sources.streams.http import HttpStream from requests import exceptions from source_salesforce.streams import ( - BatchedSubStream, PropertyChunk, RestSalesforceStream, - RestSalesforceSubStream, SalesforceStream, ) -from tests.conftest import ACCOUNT_SCHEMA, INSTANCE_URL, FakeResponse, make_sf, make_stream +from tests.conftest import ACCOUNT_FIELDS, INSTANCE_URL, FakeResponse, make_sf, make_stream -def big_schema(n_fields: int = 4000): - """Schema wide enough to trip the SOQL length limit (forces chunking).""" - props = {"Id": {"type": ["string", "null"]}} - for i in range(n_fields): - props[f"Field{i:05d}"] = {"type": ["string", "null"]} - return {"type": "object", "properties": props} +def big_field_list(n_fields: int = 4000) -> tuple[str, ...]: + """Field list wide enough to trip the SOQL length limit (forces chunking).""" + return ("Id",) + tuple(f"Field{i:05d}" for i in range(n_fields)) # --------------------------------------------------------------------------- @@ -51,7 +47,7 @@ def test_max_properties_length(self, stream): def test_too_many_properties(self): assert make_stream().too_many_properties is False - assert make_stream(schema=big_schema()).too_many_properties is True + assert make_stream(sf_fields=big_field_list()).too_many_properties is True def test_parse_response_yields_records(self, stream): response = FakeResponse({"records": [{"Id": "1"}, {"Id": "2"}], "done": True}) @@ -62,31 +58,37 @@ def test_connection_error_display_message(self, stream): assert "network error" in message assert stream.get_error_display_message(ValueError("x")) is None - def test_sf_properties_lazily_generated(self): + def test_sf_properties_fetched_once_from_describe(self): sf = make_sf() - sf.generate_schema = Mock(return_value=ACCOUNT_SCHEMA) - stream = make_stream(sf=sf, schema=None) - assert stream._sf_properties() == ACCOUNT_SCHEMA["properties"] - sf.generate_schema.assert_called_once_with("Account") + sf.field_names = Mock(return_value=ACCOUNT_FIELDS) + stream = make_stream(sf=sf, sf_fields=None) + assert stream._sf_properties() == ACCOUNT_FIELDS + assert stream._sf_properties() == ACCOUNT_FIELDS + sf.field_names.assert_called_once_with("Account") class TestGetJsonSchema: - def test_strips_custom_fields_and_injects_envelope(self, stream): + def test_schema_is_the_static_file_plus_envelope(self, stream): schema = stream.get_json_schema() - # Custom __c fields are routed into the custom_fields blob instead. - assert "Custom__c" not in schema["properties"] assert "Id" in schema["properties"] - for envelope_field in ("tenant_id", "source_id", "unique_key", "data_source", "collected_at", "custom_fields"): + assert schema["additionalProperties"] is False + for envelope_field in ( + "tenant_id", + "source_id", + "unique_key", + "data_source", + "collected_at", + "raw_data", + ): assert envelope_field in schema["properties"] - def test_generates_schema_when_missing(self): - sf = make_sf() - sf.generate_schema = Mock(return_value=dict(ACCOUNT_SCHEMA)) - stream = make_stream(sf=sf) - stream.schema = None # force the lazy describe path - schema = stream.get_json_schema() - sf.generate_schema.assert_called_once_with("Account") - assert "unique_key" in schema["properties"] + def test_schema_needs_no_describe_call(self, stream): + stream.sf_api.field_names = Mock(side_effect=AssertionError("describe must not be called")) + assert "unique_key" in stream.get_json_schema()["properties"] + + def test_schema_is_isolated_between_callers(self, stream): + stream.get_json_schema()["properties"].pop("Id") + assert "Id" in stream.get_json_schema()["properties"] class TestReadRecordsEnvelope: @@ -99,10 +101,27 @@ def test_mappings_enveloped_others_passed_through(self, stream, monkeypatch): ) out = list(stream.read_records(SyncMode.full_refresh)) assert out[0]["unique_key"] == "T-S-001" - assert out[0]["custom_fields"] == '{"Custom__c":"x"}' assert "Custom__c" not in out[0] + assert json.loads(out[0]["raw_data"])["Custom__c"] == "x" assert out[1] is state_marker + def test_unavailable_sobject_yields_no_records_and_warns_once(self, stream, caplog): + stream.sf_api.is_queryable = Mock(return_value=False) + with caplog.at_level("WARNING"): + assert list(stream.read_records(SyncMode.full_refresh)) == [] + assert list(stream.read_records(SyncMode.full_refresh)) == [] + assert len([r for r in caplog.records if "not exposed by this org" in r.getMessage()]) == 1 + + def test_undeclared_field_reaches_raw_data_only(self, stream, monkeypatch): + monkeypatch.setattr( + HttpStream, + "read_records", + lambda self, *args, **kwargs: iter([{"Id": "001", "NotInStaticSchema": "v"}]), + ) + record = next(iter(stream.read_records(SyncMode.full_refresh))) + assert "NotInStaticSchema" not in record + assert json.loads(record["raw_data"])["NotInStaticSchema"] == "v" + # --------------------------------------------------------------------------- # RestSalesforceStream: pagination + SOQL @@ -110,9 +129,15 @@ def test_mappings_enveloped_others_passed_through(self, stream, monkeypatch): class TestChunkingGuard: - def test_too_many_properties_without_pk_raises(self): + def test_too_many_properties_without_pk_raises_at_read(self): + stream = make_stream(sf_fields=big_field_list(), pk=None) with pytest.raises(RuntimeError, match="no primary key"): - make_stream(schema=big_schema(), pk=None) + list(stream._read_pages(_records_generator)) + + def test_construction_makes_no_api_call(self): + sf = make_sf() + sf.field_names = Mock(side_effect=AssertionError("describe must not be called")) + assert make_stream(sf=sf, sf_fields=None, pk=None).pk is None def test_small_schema_without_pk_is_fine(self): assert make_stream(pk=None).pk is None @@ -138,7 +163,7 @@ def test_none_when_done(self, stream): class TestRequestParams: def test_soql_select_with_order_by(self, stream): - params = stream.request_params(stream_state={}, property_chunk={"Id": {}, "Name": {}}) + params = stream.request_params(stream_state={}, property_chunk=("Id", "Name")) assert params == {"q": "SELECT Id,Name FROM Account ORDER BY Id ASC"} def test_next_page_token_suppresses_params(self, stream): @@ -146,26 +171,18 @@ def test_next_page_token_suppresses_params(self, stream): def test_unsupported_filtering_stream_has_no_order_by(self): stream = make_stream(stream_name="TabDefinition") - params = stream.request_params(stream_state={}, property_chunk={"Id": {}}) + params = stream.request_params(stream_state={}, property_chunk=("Id",)) assert "ORDER BY" not in params["q"] - def test_parent_object_gets_where_in_clause(self): - stream = make_stream(cls=RestSalesforceStream, stream_name="ContentDocumentLink") - params = stream.request_params( - stream_state={}, stream_slice={"parents": [{"Id": "069A"}, {"Id": "069B"}]}, property_chunk={"Id": {}} - ) - # ContentDocumentLink is both parent-scoped and unsupported-filtering. - assert params["q"] == ("SELECT Id FROM ContentDocumentLink WHERE ContentDocumentId IN ('069A','069B')") - class TestChunkProperties: def test_single_chunk_for_small_schema(self, stream): chunks = list(stream.chunk_properties()) assert len(chunks) == 1 - assert set(chunks[0]) == set(ACCOUNT_SCHEMA["properties"]) + assert set(chunks[0]) == set(ACCOUNT_FIELDS) def test_wide_schema_split_with_pk_in_every_chunk(self): - stream = make_stream(schema=big_schema()) + stream = make_stream(sf_fields=big_field_list()) chunks = list(stream.chunk_properties()) assert len(chunks) > 1 for chunk in chunks: @@ -183,18 +200,18 @@ def test_no_pk_chunks_have_no_key_prefix(self): class TestNextChunkId: def test_picks_least_read_non_exhausted(self): - chunk_a = PropertyChunk({"Id": {}}) + chunk_a = PropertyChunk(("Id",)) chunk_a.first_time = False chunk_a.next_page = {"next_token": "/q"} chunk_a.record_counter = 10 - chunk_b = PropertyChunk({"Id": {}}) + chunk_b = PropertyChunk(("Id",)) chunk_b.first_time = False chunk_b.next_page = {"next_token": "/q"} chunk_b.record_counter = 3 assert RestSalesforceStream._next_chunk_id({0: chunk_a, 1: chunk_b}) == 1 def test_none_when_all_exhausted(self): - chunk = PropertyChunk({"Id": {}}) + chunk = PropertyChunk(("Id",)) chunk.first_time = False chunk.next_page = None assert RestSalesforceStream._next_chunk_id({0: chunk}) is None @@ -225,7 +242,7 @@ def fake_fetch(stream_slice, stream_state, next_page, properties): def test_chunked_records_reassembled_by_pk(self, monkeypatch): """Wide schema: each chunk returns a partial record; parts merge on Id.""" - stream = make_stream(schema=big_schema()) + stream = make_stream(sf_fields=big_field_list()) n_chunks = len(list(stream.chunk_properties())) assert n_chunks > 1 calls = [] @@ -245,7 +262,7 @@ def fake_fetch(stream_slice, stream_state, next_page, properties): def test_inconsistent_records_skipped_with_warning(self, monkeypatch, caplog): """A record seen by only one chunk is dropped, not emitted half-empty.""" - stream = make_stream(schema=big_schema()) + stream = make_stream(sf_fields=big_field_list()) calls = [] def fake_fetch(stream_slice, stream_state, next_page, properties): @@ -271,50 +288,8 @@ def send_request(**kwargs): return "req", FakeResponse({"records": []}) stream._http_client = Mock(send_request=send_request) - request, response = stream._fetch_next_page_for_chunk(property_chunk={"Id": {}, "Name": {}}) + request, response = stream._fetch_next_page_for_chunk(property_chunk=("Id", "Name")) assert request == "req" assert sent["http_method"] == "GET" assert sent["url"].startswith(INSTANCE_URL) assert sent["params"]["q"].startswith("SELECT Id,Name FROM Account") - - -# --------------------------------------------------------------------------- -# BatchedSubStream -# --------------------------------------------------------------------------- - - -class FakeParentStream: - """Duck-typed parent: HttpSubStream.stream_slices only calls read_only_records.""" - - def __init__(self, records): - self._records = records - - def read_only_records(self, stream_state=None): - yield from self._records - - -class TestBatchedSubStream: - def _substream(self, parent_records, batch_size=2): - stream = make_stream( - cls=RestSalesforceSubStream, stream_name="ContentDocumentLink", parent=FakeParentStream(parent_records) - ) - stream.SLICE_BATCH_SIZE = batch_size - return stream - - def test_parents_batched_into_slices(self): - parents = [{"Id": f"069{i}"} for i in range(5)] - stream = self._substream(parents, batch_size=2) - slices = list(stream.stream_slices(SyncMode.full_refresh)) - assert [len(s["parents"]) for s in slices] == [2, 2, 1] - assert slices[0]["parents"][0] == {"Id": "0690"} - - def test_exact_multiple_has_no_empty_tail(self): - parents = [{"Id": "A"}, {"Id": "B"}] - slices = list(self._substream(parents, batch_size=2).stream_slices(SyncMode.full_refresh)) - assert len(slices) == 1 - - def test_no_parents_yields_nothing(self): - assert list(self._substream([]).stream_slices(SyncMode.full_refresh)) == [] - - def test_default_batch_size(self): - assert BatchedSubStream.SLICE_BATCH_SIZE == 200 diff --git a/src/ingestion/scripts/apply-ch-migrations.sh b/src/ingestion/scripts/apply-ch-migrations.sh index bb1f8423b..92975490e 100755 --- a/src/ingestion/scripts/apply-ch-migrations.sh +++ b/src/ingestion/scripts/apply-ch-migrations.sh @@ -93,13 +93,32 @@ heal_ai_dev_staging chatgpt_team__ai_dev_usage heal_ai_assistant_staging claude_enterprise__ai_assistant_usage heal_ai_assistant_staging chatgpt_team__ai_assistant_usage -echo "=== Healing collab-chat and CRM contract schemas ===" +echo "=== Healing CRM staging contract schemas ===" +# The CRM overflow blob left the contract — the connectors carry the +# unabridged record in raw_data — so the column must leave the physical +# tables too, or the positional incremental insert misaligns. The silver +# side drops in migrations/*.sql; staging drops here because these tables +# exist only after the connector's first run. Idempotent. +heal_crm_staging() { + local table="$1" + ch_table_exists staging "${table}" || return 0 + echo " staging.${table}" + run_ch < dbt-clickhouse==`). - ClickHouse reachable under `CLICKHOUSE_HOST` both from this machine (dbt) and from inside docker containers (destination connector). For a ClickHouse running on this machine use the machine's LAN IP (`ipconfig getifaddr en0`) — `host.docker.internal` resolves inside containers but not on the macOS host itself. -- Real HubSpot and Salesforce credentials in `.env` — their CDK `discover` calls the live APIs, so fake values fail. Without credentials, seed their bronze from the committed snapshot instead: apply `../connectors-ddl/{hubspot,salesforce}.sql` (paths relative to this directory), run `./run-dbt.sh --select hubspot__bronze_promoted salesforce__bronze_promoted`, and continue from the dbt step. ## Local ClickHouse for testing @@ -44,19 +43,10 @@ Throw it away with `docker rm -f bootstrap-db-clickhouse`. ./generate-connectors-config.sh 'bitbucket-cloud' > one.yaml ``` -2. Review the file. Every required config field gets a fake value; that is enough for connectors with static stream schemas. Connectors that build schemas from a live API (`hubspot`, `salesforce`) need real credentials — replace `value` with `env` to take the value from an environment variable at run time, so secrets never land in the file: +2. Review the file. Every required config field gets a fake value, which is all `discover` needs. Should a future connector build its catalog from a live API, replace `value` with `env` to take that field from an environment variable at run time, so secrets never land in the file: ```yaml connectors: - hubspot: - path: crm/hubspot - config: - hubspot_access_token: - env: HUBSPOT_ACCESS_TOKEN - insight_source_id: - value: hubspot-acme-prod - insight_tenant_id: - value: fake salesforce: path: crm/salesforce config: @@ -65,13 +55,18 @@ connectors: insight_tenant_id: value: fake salesforce_client_id: - env: SALESFORCE_CLIENT_ID + value: fake salesforce_client_secret: - env: SALESFORCE_CLIENT_SECRET + value: fake salesforce_instance_url: - env: SALESFORCE_INSTANCE_URL + value: https://mycompany.my.salesforce.com salesforce_start_date: value: "2024-01-01" + example-live-catalog-connector: + path: category/name + config: + api_token: + env: EXAMPLE_API_TOKEN ``` The file contains no secrets and can be committed to the repository. @@ -88,14 +83,11 @@ connectors: ## Everything from scratch, one block -The full cycle — throwaway ClickHouse, fresh `.env`, bootstrap, snapshot re-dump, field-parity audit, cleanup — as a single copy-paste. Only prerequisite: `HUBSPOT_ACCESS_TOKEN`, `SALESFORCE_INSTANCE_URL`, `SALESFORCE_CLIENT_ID`, `SALESFORCE_CLIENT_SECRET` exported in the current shell (their `discover` calls the live APIs). **Overwrites `.env`** next to the scripts. +The full cycle — throwaway ClickHouse, fresh `.env`, bootstrap, snapshot re-dump, field-parity audit, cleanup — as a single copy-paste. No credentials needed: every connector discovers on fake config values. **Overwrites `.env`** next to the scripts. ```bash cd src/ingestion/scripts/bootstrap-db -: "${HUBSPOT_ACCESS_TOKEN:?export real credentials first}" -: "${SALESFORCE_INSTANCE_URL:?}" "${SALESFORCE_CLIENT_ID:?}" "${SALESFORCE_CLIENT_SECRET:?}" - source pins.env docker rm -f bootstrap-db-clickhouse 2>/dev/null docker run -d --name bootstrap-db-clickhouse -p 8123:8123 \ @@ -114,10 +106,6 @@ CLICKHOUSE_PROTOCOL=http CLICKHOUSE_USER=insight CLICKHOUSE_PASSWORD=insight CLICKHOUSE_DATABASE=insight -HUBSPOT_ACCESS_TOKEN=${HUBSPOT_ACCESS_TOKEN} -SALESFORCE_INSTANCE_URL=${SALESFORCE_INSTANCE_URL} -SALESFORCE_CLIENT_ID=${SALESFORCE_CLIENT_ID} -SALESFORCE_CLIENT_SECRET=${SALESFORCE_CLIENT_SECRET} EOF until curl -sf "http://localhost:8123/ping" >/dev/null; do sleep 1; done @@ -161,7 +149,7 @@ Contributors whose physical table is not owned by dbt are covered too. `jira__ta | `bootstrap-db.sh ` | Sources `pins.env` and `.env` (if present), runs `seed-connectors.sh`, runs all dbt models, runs `../apply-ch-migrations.sh`. | | `run-dbt.sh [dbt args]` | Helper: generates a profiles.yml from the `CLICKHOUSE_*` variables and runs `dbt run` in `src/ingestion/dbt`. | | `check-field-parity.py [--manifest PATH]` | Audits every staging contributor against its silver union target (column set, positional order, exact type) plus manifest-vs-warehouse coverage. Same `CLICKHOUSE_*` env contract as the other scripts. Non-zero exit on any finding. | -| `dump-ddl.sh` | Dumps `SHOW CREATE` for every `bronze_*` table, the `person`/`identity`/`silver`/`insight` databases (tables and views), and the gold-referenced `staging` tables into `../connectors-ddl/*.sql` — the committed snapshot that `../create-bronze-placeholders.sh` applies on fresh clusters. **Run it manually** after `bootstrap-db.sh` (see step above) whenever a schema changes, and commit the diff. `.github/workflows/connectors-ddl.yml` re-runs the whole pipeline on every same-repository PR and on every commit to `main`, and fails when the committed snapshot no longer matches. On PR drift its `regen-pr` job opens a stacked PR against the PR's branch with the regenerated snapshot (and links it in a sticky comment) — review the DDL diff there and merge, no local regeneration or CRM credentials needed. Drift on `main` stays a red run. | +| `dump-ddl.sh` | Dumps `SHOW CREATE` for every `bronze_*` table, the `person`/`identity`/`silver`/`insight` databases (tables and views), and the gold-referenced `staging` tables into `../connectors-ddl/*.sql` — the committed snapshot that `../create-bronze-placeholders.sh` applies on fresh clusters. **Run it manually** after `bootstrap-db.sh` (see step above) whenever a schema changes, and commit the diff. `.github/workflows/connectors-ddl.yml` re-runs the whole pipeline on every PR and on every commit to `main`, and fails when the committed snapshot no longer matches. On PR drift its `regen-pr` job opens a stacked PR against the PR's branch with the regenerated snapshot (and links it in a sticky comment) — review the DDL diff there and merge, no local regeneration needed. That job is same-repository only; a fork PR gets the regenerated snapshot as a downloadable artifact instead. Drift on `main` stays a red run. | ## Image pins (pins.env) diff --git a/src/ingestion/scripts/bootstrap-db/connectors-config.yaml b/src/ingestion/scripts/bootstrap-db/connectors-config.yaml index d8adb6a5d..306ca2b5e 100644 --- a/src/ingestion/scripts/bootstrap-db/connectors-config.yaml +++ b/src/ingestion/scripts/bootstrap-db/connectors-config.yaml @@ -126,7 +126,7 @@ connectors: path: crm/hubspot config: hubspot_access_token: - env: HUBSPOT_ACCESS_TOKEN + value: fake insight_source_id: value: hubspot-acme-prod insight_tenant_id: @@ -139,11 +139,11 @@ connectors: insight_tenant_id: value: fake salesforce_client_id: - env: SALESFORCE_CLIENT_ID + value: fake salesforce_client_secret: - env: SALESFORCE_CLIENT_SECRET + value: fake salesforce_instance_url: - env: SALESFORCE_INSTANCE_URL + value: https://mycompany.my.salesforce.com salesforce_start_date: value: "2024-01-01" bitbucket-cloud: diff --git a/src/ingestion/scripts/connectors-ddl/hubspot.sql b/src/ingestion/scripts/connectors-ddl/hubspot.sql index d0efe3937..6c530ec46 100644 --- a/src/ingestion/scripts/connectors-ddl/hubspot.sql +++ b/src/ingestion/scripts/connectors-ddl/hubspot.sql @@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.companies `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String) + `raw_data` Nullable(String) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -63,7 +63,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.companies_archived `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String) + `raw_data` Nullable(String) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -98,7 +98,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.contacts `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String), + `raw_data` Nullable(String), `associations_companies` Nullable(String), `associations_deals` Nullable(String) ) @@ -135,7 +135,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.contacts_archived `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String), + `raw_data` Nullable(String), `associations_companies` Nullable(String), `associations_deals` Nullable(String) ) @@ -180,7 +180,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.deals `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String), + `raw_data` Nullable(String), `associations_companies` Nullable(String), `associations_contacts` Nullable(String) ) @@ -225,7 +225,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.deals_archived `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String), + `raw_data` Nullable(String), `associations_companies` Nullable(String), `associations_contacts` Nullable(String) ) @@ -258,7 +258,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.engagements_calls `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String), + `raw_data` Nullable(String), `associations_contacts` Nullable(String), `associations_companies` Nullable(String), `associations_deals` Nullable(String) @@ -292,7 +292,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.engagements_calls_archived `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String), + `raw_data` Nullable(String), `associations_contacts` Nullable(String), `associations_companies` Nullable(String), `associations_deals` Nullable(String) @@ -324,7 +324,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.engagements_emails `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String), + `raw_data` Nullable(String), `associations_contacts` Nullable(String), `associations_companies` Nullable(String), `associations_deals` Nullable(String) @@ -356,7 +356,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.engagements_emails_archived `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String), + `raw_data` Nullable(String), `associations_contacts` Nullable(String), `associations_companies` Nullable(String), `associations_deals` Nullable(String) @@ -392,7 +392,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.engagements_meetings `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String), + `raw_data` Nullable(String), `associations_contacts` Nullable(String), `associations_companies` Nullable(String), `associations_deals` Nullable(String) @@ -426,7 +426,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.engagements_tasks `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String), + `raw_data` Nullable(String), `associations_contacts` Nullable(String), `associations_companies` Nullable(String), `associations_deals` Nullable(String) @@ -460,7 +460,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.engagements_tasks_archived `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String), + `raw_data` Nullable(String), `associations_contacts` Nullable(String), `associations_companies` Nullable(String), `associations_deals` Nullable(String) @@ -486,7 +486,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.leads `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String), + `raw_data` Nullable(String), `associations_contacts` Nullable(String), `associations_companies` Nullable(String) ) @@ -511,7 +511,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.leads_archived `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String), + `raw_data` Nullable(String), `associations_contacts` Nullable(String), `associations_companies` Nullable(String) ) @@ -540,7 +540,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.owners `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String) + `raw_data` Nullable(String) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -567,7 +567,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.owners_archived `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String) + `raw_data` Nullable(String) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -590,7 +590,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.tickets `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String), + `raw_data` Nullable(String), `associations_contacts` Nullable(String), `associations_companies` Nullable(String), `associations_deals` Nullable(String) @@ -616,7 +616,7 @@ CREATE TABLE IF NOT EXISTS bronze_hubspot.tickets_archived `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String), + `raw_data` Nullable(String), `associations_contacts` Nullable(String), `associations_companies` Nullable(String), `associations_deals` Nullable(String) diff --git a/src/ingestion/scripts/connectors-ddl/salesforce.sql b/src/ingestion/scripts/connectors-ddl/salesforce.sql index a3f239e12..e8f1b073f 100644 --- a/src/ingestion/scripts/connectors-ddl/salesforce.sql +++ b/src/ingestion/scripts/connectors-ddl/salesforce.sql @@ -74,7 +74,7 @@ CREATE TABLE IF NOT EXISTS bronze_salesforce.Account `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String) + `raw_data` Nullable(String) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -128,7 +128,7 @@ CREATE TABLE IF NOT EXISTS bronze_salesforce.Case `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String) + `raw_data` Nullable(String) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -211,7 +211,7 @@ CREATE TABLE IF NOT EXISTS bronze_salesforce.Contact `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String) + `raw_data` Nullable(String) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -278,7 +278,7 @@ CREATE TABLE IF NOT EXISTS bronze_salesforce.Event `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String) + `raw_data` Nullable(String) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -353,7 +353,7 @@ CREATE TABLE IF NOT EXISTS bronze_salesforce.Lead `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String) + `raw_data` Nullable(String) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -412,7 +412,7 @@ CREATE TABLE IF NOT EXISTS bronze_salesforce.Opportunity `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String) + `raw_data` Nullable(String) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -441,7 +441,7 @@ CREATE TABLE IF NOT EXISTS bronze_salesforce.OpportunityContactRole `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String) + `raw_data` Nullable(String) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -473,7 +473,7 @@ CREATE TABLE IF NOT EXISTS bronze_salesforce.OpportunityHistory `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String) + `raw_data` Nullable(String) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -530,7 +530,7 @@ CREATE TABLE IF NOT EXISTS bronze_salesforce.Task `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String) + `raw_data` Nullable(String) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key @@ -733,7 +733,7 @@ CREATE TABLE IF NOT EXISTS bronze_salesforce.User `unique_key` Nullable(String), `data_source` Nullable(String), `collected_at` Nullable(DateTime64(3)), - `custom_fields` Nullable(String) + `raw_data` Nullable(String) ) ENGINE = ReplacingMergeTree(_airbyte_extracted_at) ORDER BY unique_key diff --git a/src/ingestion/scripts/connectors-ddl/silver.sql b/src/ingestion/scripts/connectors-ddl/silver.sql index eda6ae7dd..1ce597693 100644 --- a/src/ingestion/scripts/connectors-ddl/silver.sql +++ b/src/ingestion/scripts/connectors-ddl/silver.sql @@ -243,7 +243,6 @@ CREATE TABLE IF NOT EXISTS silver.class_crm_accounts `owner_id` Nullable(String), `parent_account_id` Nullable(String), `metadata` String, - `custom_fields` String DEFAULT '{}', `created_at` Nullable(DateTime64(3)), `updated_at` Nullable(DateTime64(3)), `data_source` Nullable(String), @@ -270,7 +269,6 @@ CREATE TABLE IF NOT EXISTS silver.class_crm_activities `duration_seconds` Nullable(Int64), `outcome` Nullable(String), `metadata` String, - `custom_fields` String DEFAULT '{}', `created_at` Nullable(DateTime64(3)), `data_source` Nullable(String), `_version` Int64 @@ -293,7 +291,6 @@ CREATE TABLE IF NOT EXISTS silver.class_crm_contacts `account_id` Nullable(String), `lifecycle_stage` Nullable(String), `metadata` String, - `custom_fields` String DEFAULT '{}', `created_at` Nullable(DateTime64(3)), `updated_at` Nullable(DateTime64(3)), `data_source` Nullable(String), @@ -330,7 +327,6 @@ CREATE TABLE IF NOT EXISTS silver.class_crm_deals `lost_reason` Nullable(String), `pipeline_id` Nullable(String), `metadata` String, - `custom_fields` String DEFAULT '{}', `created_at` Nullable(DateTime64(3)), `updated_at` Nullable(DateTime64(3)), `data_source` Nullable(String), @@ -355,7 +351,6 @@ CREATE TABLE IF NOT EXISTS silver.class_crm_users `department` Nullable(String), `is_active` Nullable(Int64), `metadata` String, - `custom_fields` String DEFAULT '{}', `collected_at` Nullable(DateTime64(3)), `data_source` Nullable(String), `_version` Int64 diff --git a/src/ingestion/scripts/migrations/20260805000000_crm-drop-custom-fields.sql b/src/ingestion/scripts/migrations/20260805000000_crm-drop-custom-fields.sql new file mode 100644 index 000000000..5c44faec9 --- /dev/null +++ b/src/ingestion/scripts/migrations/20260805000000_crm-drop-custom-fields.sql @@ -0,0 +1,20 @@ +-- Drop the CRM overflow blob from the class contract. +-- +-- dbt-clickhouse incremental inserts are positional and union_by_tag is a +-- positional SELECT * UNION ALL, so physical column order must equal the +-- model's SELECT order. The CRM staging models no longer project +-- custom_fields — the connectors carry the unabridged record in raw_data +-- instead — so the column leaves the contract here in the same change. +-- DROP preserves the order of the remaining columns. +-- +-- Idempotent: this channel has no ledger and re-runs on every deploy. +-- The class tables always exist here (placeholders precede migrations). +ALTER TABLE silver.class_crm_accounts DROP COLUMN IF EXISTS custom_fields; + +ALTER TABLE silver.class_crm_activities DROP COLUMN IF EXISTS custom_fields; + +ALTER TABLE silver.class_crm_contacts DROP COLUMN IF EXISTS custom_fields; + +ALTER TABLE silver.class_crm_deals DROP COLUMN IF EXISTS custom_fields; + +ALTER TABLE silver.class_crm_users DROP COLUMN IF EXISTS custom_fields;