From bad2fb14e3b4444881691baa49efee865372f524 Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi Date: Mon, 14 Jun 2021 13:46:32 +0530 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20New=20source:=20Google=20Ads=20A?= =?UTF-8?q?PI=20(#3842)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * New source: Google Ads API Co-authored-by: Sherif Nada --- .github/workflows/publish-command.yml | 3 +- .github/workflows/test-command.yml | 1 + .../source-google-ads/.dockerignore | 7 + .../connectors/source-google-ads/Dockerfile | 15 + .../connectors/source-google-ads/README.md | 129 +++ .../acceptance-test-config.yml | 27 + .../acceptance-test-docker.sh | 7 + .../connectors/source-google-ads/build.gradle | 13 + .../integration_tests/__init__.py | 0 .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 36 + .../integration_tests/configured_catalog.json | 34 + .../integration_tests/conftest.py | 33 + .../integration_tests/invalid_config.json | 8 + .../integration_tests/test_incremental.py | 112 +++ .../connectors/source-google-ads/main.py | 33 + .../source-google-ads/requirements.txt | 2 + .../connectors/source-google-ads/setup.py | 46 ++ .../source_google_ads/__init__.py | 27 + .../source_google_ads/google_ads.py | 105 +++ .../schemas/ad_group_ad_report.json | 754 ++++++++++++++++++ .../source_google_ads/source.py | 132 +++ .../source_google_ads/spec.json | 59 ++ .../source_google_ads/utils.py | 39 + .../source-google-ads/unit_tests/conftest.py | 33 + .../unit_tests/test_google_ads.py | 147 ++++ .../unit_tests/test_source.py | 50 ++ .../unit_tests/test_utils.py | 39 + tools/bin/ci_credentials.sh | 1 + 29 files changed, 1896 insertions(+), 1 deletion(-) create mode 100644 airbyte-integrations/connectors/source-google-ads/.dockerignore create mode 100644 airbyte-integrations/connectors/source-google-ads/Dockerfile create mode 100644 airbyte-integrations/connectors/source-google-ads/README.md create mode 100644 airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml create mode 100755 airbyte-integrations/connectors/source-google-ads/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-google-ads/build.gradle create mode 100644 airbyte-integrations/connectors/source-google-ads/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-google-ads/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-google-ads/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-google-ads/integration_tests/conftest.py create mode 100644 airbyte-integrations/connectors/source-google-ads/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py create mode 100644 airbyte-integrations/connectors/source-google-ads/main.py create mode 100644 airbyte-integrations/connectors/source-google-ads/requirements.txt create mode 100644 airbyte-integrations/connectors/source-google-ads/setup.py create mode 100644 airbyte-integrations/connectors/source-google-ads/source_google_ads/__init__.py create mode 100644 airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py create mode 100644 airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_report.json create mode 100644 airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py create mode 100644 airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json create mode 100644 airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py create mode 100644 airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py create mode 100644 airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py create mode 100644 airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 866ffc26c66a1..5182801bb62f1 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -64,6 +64,7 @@ jobs: FRESHDESK_TEST_CREDS: ${{ secrets.FRESHDESK_TEST_CREDS }} GITLAB_INTEGRATION_TEST_CREDS: ${{ secrets.GITLAB_INTEGRATION_TEST_CREDS }} GH_INTEGRATION_TEST_CREDS: ${{ secrets.GH_INTEGRATION_TEST_CREDS }} + GOOGLE_ADS_TEST_CREDS: ${{ secrets.GOOGLE_ADS_TEST_CREDS }} GOOGLE_ANALYTICS_TEST_CREDS: ${{ secrets.GOOGLE_ANALYTICS_TEST_CREDS }} GOOGLE_ANALYTICS_TEST_TRACKING_ID: ${{ secrets.GOOGLE_ANALYTICS_TEST_TRACKING_ID }} GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} @@ -79,7 +80,6 @@ jobs: ITERABLE_INTEGRATION_TEST_CREDS: ${{ secrets.ITERABLE_INTEGRATION_TEST_CREDS }} JIRA_INTEGRATION_TEST_CREDS: ${{ secrets.JIRA_INTEGRATION_TEST_CREDS }} KLAVIYO_TEST_CREDS: ${{ secrets.KLAVIYO_TEST_CREDS }} - SOURCE_ASANA_TEST_CREDS: ${{ secrets.SOURCE_ASANA_TEST_CREDS }} LOOKER_INTEGRATION_TEST_CREDS: ${{ secrets.LOOKER_INTEGRATION_TEST_CREDS }} MAILCHIMP_TEST_CREDS: ${{ secrets.MAILCHIMP_TEST_CREDS }} MICROSOFT_TEAMS_TEST_CREDS: ${{ secrets.MICROSOFT_TEAMS_TEST_CREDS }} @@ -92,6 +92,7 @@ jobs: SENDGRID_INTEGRATION_TEST_CREDS: ${{ secrets.SENDGRID_INTEGRATION_TEST_CREDS }} SHOPIFY_INTEGRATION_TEST_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_CREDS }} SLACK_TEST_CREDS: ${{ secrets.SLACK_TEST_CREDS }} + SOURCE_ASANA_TEST_CREDS: ${{ secrets.SOURCE_ASANA_TEST_CREDS }} SOURCE_OKTA_TEST_CREDS: ${{ secrets.SOURCE_OKTA_TEST_CREDS }} SOURCE_SLACK_TEST_CREDS: ${{ secrets.SOURCE_SLACK_TEST_CREDS }} SMARTSHEETS_TEST_CREDS: ${{ secrets.SMARTSHEETS_TEST_CREDS }} diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index a9fdfbeaaab32..afcff0c278be2 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -64,6 +64,7 @@ jobs: FRESHDESK_TEST_CREDS: ${{ secrets.FRESHDESK_TEST_CREDS }} GITLAB_INTEGRATION_TEST_CREDS: ${{ secrets.GITLAB_INTEGRATION_TEST_CREDS }} GH_INTEGRATION_TEST_CREDS: ${{ secrets.GH_INTEGRATION_TEST_CREDS }} + GOOGLE_ADS_TEST_CREDS: ${{ secrets.GOOGLE_ADS_TEST_CREDS }} GOOGLE_ANALYTICS_TEST_CREDS: ${{ secrets.GOOGLE_ANALYTICS_TEST_CREDS }} GOOGLE_ANALYTICS_TEST_TRACKING_ID: ${{ secrets.GOOGLE_ANALYTICS_TEST_TRACKING_ID }} GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }} diff --git a/airbyte-integrations/connectors/source-google-ads/.dockerignore b/airbyte-integrations/connectors/source-google-ads/.dockerignore new file mode 100644 index 0000000000000..409db6ca0ee6a --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_google_ads +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-google-ads/Dockerfile b/airbyte-integrations/connectors/source-google-ads/Dockerfile new file mode 100644 index 0000000000000..bc0aa467c0431 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.7-slim + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +WORKDIR /airbyte/integration_code +COPY source_google_ads ./source_google_ads +COPY main.py ./ +COPY setup.py ./ +RUN pip install . + +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-google-ads diff --git a/airbyte-integrations/connectors/source-google-ads/README.md b/airbyte-integrations/connectors/source-google-ads/README.md new file mode 100644 index 0000000000000..a8f7c9a0a596f --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/README.md @@ -0,0 +1,129 @@ +# Google Ads Source + +This is the repository for the Google Ads source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/google-ads). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-google-ads:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-ads) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_ads/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source google-ads test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-google-ads:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-google-ads:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-google-ads:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-ads:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-ads:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-google-ads:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-google-ads:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-google-ads:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml new file mode 100644 index 0000000000000..7f153ddf9b791 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml @@ -0,0 +1,27 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/source-acceptance-tests.md) +# for more information about how to configure these tests +connector_image: airbyte/source-google-ads:dev +tests: + spec: + - spec_path: "source_google_ads/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + validate_output_from_all_streams: yes +# TODO incremental test is disabled because records output from the report streams can be up to 14 days older than the input state +# incremental: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state_path: "integration_tests/abnormal_state.json" +# cursor_paths: +# ad_group_ad_report: ["segments.date"] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-google-ads/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-google-ads/acceptance-test-docker.sh new file mode 100755 index 0000000000000..1425ff74f1511 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/acceptance-test-docker.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input diff --git a/airbyte-integrations/connectors/source-google-ads/build.gradle b/airbyte-integrations/connectors/source-google-ads/build.gradle new file mode 100644 index 0000000000000..996905bf785a0 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/build.gradle @@ -0,0 +1,13 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_google_ads' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/__init__.py b/airbyte-integrations/connectors/source-google-ads/integration_tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..5f7c2ed7bcaea --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "ad_group_ad_report": { + "segments.date": "2021-06-07" + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-ads/integration_tests/acceptance.py new file mode 100644 index 0000000000000..eeb4a2d3e02e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/acceptance.py @@ -0,0 +1,36 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """ This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..32b4c05b39838 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json @@ -0,0 +1,34 @@ +{ + "streams": [ + { + "stream": { + "name": "ad_group_ad_report", + "json_schema": { + "type": "object", + "title": "Ad Group Ad Report", + "description": "An ad group ad.", + "properties": { + "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": { + "type": ["null", "string"] + }, + "customer.currency_code": { + "type": ["null", "string"] + }, + "customer.descriptive_name": { + "type": ["null", "string"] + }, + "segments.date": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["segments.date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["segments.date"] + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/conftest.py b/airbyte-integrations/connectors/source-google-ads/integration_tests/conftest.py new file mode 100644 index 0000000000000..707729a1f2826 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/conftest.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +import json + +import pytest + + +@pytest.fixture(scope="session", name="config") +def config_fixture(): + with open("secrets/config.json", "r") as config_file: + return json.load(config_file) diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..283ec10d56b6a --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/invalid_config.json @@ -0,0 +1,8 @@ +{ + "developer_token": "developer_token", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token", + "start_date": "start_date", + "customer_id": "customer_id" +} diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py b/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py new file mode 100644 index 0000000000000..f6ee9fd0c55ea --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py @@ -0,0 +1,112 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +import pendulum +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import ConfiguredAirbyteCatalog, Type + +from source_google_ads.source import SourceGoogleAds + +SAMPLE_CATALOG = { + "streams": [ + { + "stream": { + "name": "ad_group_ad_report", + "json_schema": { + "type": "object", + "title": "Ad Group Ad Report", + "description": "An ad group ad.", + "properties": { + "accent_color": { + "description": "AccentColor", + "type": ["null", "string"], + "field": "ad_group_ad.ad.legacy_responsive_display_ad.accent_color", + }, + "account_currency_code": { + "description": "AccountCurrencyCode", + "type": ["null", "string"], + "field": "customer.currency_code", + }, + "account_descriptive_name": { + "description": "AccountDescriptiveName", + "type": ["null", "string"], + "field": "customer.descriptive_name", + }, + "segments.date": {"description": "Date", "type": ["null", "string"], "field": "segments.date"}, + }, + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": True, + "default_cursor_field": ["segments.date"], + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["segments.date"], + } + ] +} + +def test_incremental_sync(config): + google_ads_client = SourceGoogleAds() + state = "2021-05-24" + records = google_ads_client.read( + AirbyteLogger(), config, ConfiguredAirbyteCatalog.parse_obj(SAMPLE_CATALOG), {"ad_group_ad_report": {"segments.date": state}} + ) + current_state = pendulum.parse(state).subtract(days=14).to_date_string() + + for record in records: + if record and record.type == Type.STATE: + print(record) + current_state = record.state.data["ad_group_ad_report"]["segments.date"] + if record and record.type == Type.RECORD: + assert record.record.data["segments.date"] >= current_state + + # Next sync + state = "2021-06-04" + records = google_ads_client.read( + AirbyteLogger(), config, ConfiguredAirbyteCatalog.parse_obj(SAMPLE_CATALOG), {"ad_group_ad_report": {"segments.date": state}} + ) + current_state = pendulum.parse(state).subtract(days=14).to_date_string() + + for record in records: + if record and record.type == Type.STATE: + current_state = record.state.data["ad_group_ad_report"]["segments.date"] + if record and record.type == Type.RECORD: + assert record.record.data["segments.date"] >= current_state + + # Abnormal state + state = "2029-06-04" + records = google_ads_client.read( + AirbyteLogger(), config, ConfiguredAirbyteCatalog.parse_obj(SAMPLE_CATALOG), {"ad_group_ad_report": {"segments.date": state}} + ) + current_state = pendulum.parse(state).subtract(days=14).to_date_string() + + no_records = True + for record in records: + if record and record.type == Type.STATE: + assert record.state.data["ad_group_ad_report"]["segments.date"] == state + if record and record.type == Type.RECORD: + no_records = False + + assert no_records diff --git a/airbyte-integrations/connectors/source-google-ads/main.py b/airbyte-integrations/connectors/source-google-ads/main.py new file mode 100644 index 0000000000000..fffca148626fc --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/main.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_ads import SourceGoogleAds + +if __name__ == "__main__": + source = SourceGoogleAds() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-ads/requirements.txt b/airbyte-integrations/connectors/source-google-ads/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-google-ads/setup.py b/airbyte-integrations/connectors/source-google-ads/setup.py new file mode 100644 index 0000000000000..28c9d13dde0a0 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/setup.py @@ -0,0 +1,46 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "google-ads", "pendulum"] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock", +] + +setup( + name="source_google_ads", + description="Source implementation for Google Ads.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/__init__.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/__init__.py new file mode 100644 index 0000000000000..39f872c4c095f --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/__init__.py @@ -0,0 +1,27 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .source import SourceGoogleAds + +__all__ = ["SourceGoogleAds"] diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py new file mode 100644 index 0000000000000..5f8a418f44ce9 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py @@ -0,0 +1,105 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from enum import Enum +from string import Template +from typing import Any, List, Mapping + +from google.ads.googleads.client import GoogleAdsClient +from google.ads.googleads.v7.services.types.google_ads_service import GoogleAdsRow, SearchGoogleAdsResponse + +REPORT_MAPPING = {"ad_group_ad_report": "ad_group_ad"} + + +class GoogleAds: + DEFAULT_PAGE_SIZE = 1000 + + def __init__(self, developer_token: str, refresh_token: str, client_id: str, client_secret: str, customer_id: str, **kwargs): + credentials = { + "developer_token": developer_token, + "refresh_token": refresh_token, + "client_id": client_id, + "client_secret": client_secret, + } + self.client = GoogleAdsClient.load_from_dict(credentials) + self.customer_id = customer_id + self.ga_service = self.client.get_service("GoogleAdsService") + + def send_request(self, query: str) -> SearchGoogleAdsResponse: + client = self.client + search_request = client.get_type("SearchGoogleAdsRequest") + search_request.customer_id = self.customer_id + search_request.query = query + search_request.page_size = self.DEFAULT_PAGE_SIZE + + return self.ga_service.search(search_request) + + @staticmethod + def get_fields_from_schema(schema: Mapping[str, Any]) -> List[str]: + properties = schema.get("properties") + + return [*properties] + + @staticmethod + def convert_schema_into_query(schema: Mapping[str, Any], report_name: str, from_date: str, to_date: str) -> str: + from_category = REPORT_MAPPING[report_name] + fields = GoogleAds.get_fields_from_schema(schema) + fields = ",\n".join(fields) + + query = Template( + """ + SELECT + $fields + FROM $from_category + WHERE segments.date > '$from_date' + AND segments.date < '$to_date' + ORDER BY segments.date + """ + ) + query = query.substitute(fields=fields, from_category=from_category, from_date=from_date, to_date=to_date) + + return query + + @staticmethod + def get_field_value(result: GoogleAdsRow, field: str) -> str: + field_name = field.split(".") + try: + field_value = result + for level_attr in field_name: + field_value = field_value.__getattr__(level_attr) + if isinstance(field_value, Enum): + field_value = field_value.name + field_value = str(field_value) + except Exception: + field_value = None + + return field_value + + @staticmethod + def parse_single_result(schema: Mapping[str, Any], result: GoogleAdsRow): + fields = GoogleAds.get_fields_from_schema(schema) + single_record = {} + for field in fields: + single_record[field] = GoogleAds.get_field_value(result, field) + return single_record diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_report.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_report.json new file mode 100644 index 0000000000000..a0bda1f6ec07e --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_report.json @@ -0,0 +1,754 @@ +{ + "properties": { + "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": { + "description": "AccentColor", + "type": ["null", "string"], + "title": "ad_group_ad.ad.legacy_responsive_display_ad.accent_color" + }, + "customer.currency_code": { + "description": "AccountCurrencyCode", + "type": ["null", "string"], + "title": "customer.currency_code" + }, + "customer.descriptive_name": { + "description": "CustomerDescriptiveName", + "type": ["null", "string"], + "title": "customer.descriptive_name" + }, + "customer.time_zone": { + "description": "AccountTimeZone", + "type": ["null", "string"], + "title": "customer.time_zone" + }, + "metrics.active_view_cpm": { + "description": "ActiveViewCpm", + "type": ["null", "string"], + "title": "metrics.active_view_cpm" + }, + "metrics.active_view_ctr": { + "description": "ActiveViewCtr", + "type": ["null", "string"], + "title": "metrics.active_view_ctr" + }, + "metrics.active_view_impressions": { + "description": "ActiveViewImpressions", + "type": ["null", "string"], + "title": "metrics.active_view_impressions" + }, + "metrics.active_view_measurability": { + "description": "ActiveViewMeasurability", + "type": ["null", "string"], + "title": "metrics.active_view_measurability" + }, + "metrics.active_view_measurable_cost_micros": { + "description": "ActiveViewMeasurableCost", + "type": ["null", "string"], + "title": "metrics.active_view_measurable_cost_micros" + }, + "metrics.active_view_measurable_impressions": { + "description": "ActiveViewMeasurableImpressions", + "type": ["null", "string"], + "title": "metrics.active_view_measurable_impressions" + }, + "metrics.active_view_viewability": { + "description": "ActiveViewViewability", + "type": ["null", "string"], + "title": "metrics.active_view_viewability" + }, + "ad_group_ad.ad_group": { + "description": "AdGroupId", + "type": ["null", "string"], + "title": "ad_group_ad.ad_group" + }, + "ad_group.name": { + "description": "AdGroupName", + "type": ["null", "string"], + "title": "ad_group.name" + }, + "ad_group.status": { + "description": "AdGroupStatus", + "type": ["null", "string"], + "title": "ad_group.status" + }, + "segments.ad_network_type": { + "description": "AdNetworkType2", + "type": ["null", "string"], + "title": "segments.ad_network_type" + }, + "ad_group_ad.ad_strength": { + "description": "AdStrengthInfo", + "type": ["null", "string"], + "title": "ad_group_ad.ad_strength" + }, + "ad_group_ad.ad.type": { + "description": "AdType", + "type": ["null", "string"], + "title": "ad_group_ad.ad.type" + }, + "metrics.all_conversions_from_interactions_rate": { + "description": "AllConversionRate", + "type": ["null", "string"], + "title": "metrics.all_conversions_from_interactions_rate" + }, + "metrics.all_conversions_value": { + "description": "AllConversionValue", + "type": ["null", "string"], + "title": "metrics.all_conversions_value" + }, + "metrics.all_conversions": { + "description": "AllConversions", + "type": ["null", "string"], + "title": "metrics.all_conversions" + }, + "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": { + "description": "AllowFlexibleColor", + "type": ["null", "string"], + "title": "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color" + }, + "ad_group_ad.ad.added_by_google_ads": { + "description": "Automated", + "type": ["null", "string"], + "title": "ad_group_ad.ad.added_by_google_ads" + }, + "metrics.average_cost": { + "description": "AverageCost", + "type": ["null", "string"], + "title": "metrics.average_cost" + }, + "metrics.average_cpc": { + "description": "AverageCpc", + "type": ["null", "string"], + "title": "metrics.average_cpc" + }, + "metrics.average_cpe": { + "description": "AverageCpe", + "type": ["null", "string"], + "title": "metrics.average_cpe" + }, + "metrics.average_cpm": { + "description": "AverageCpm", + "type": ["null", "string"], + "title": "metrics.average_cpm" + }, + "metrics.average_cpv": { + "description": "AverageCpv", + "type": ["null", "string"], + "title": "metrics.average_cpv" + }, + "metrics.average_page_views": { + "description": "AveragePageviews", + "type": ["null", "string"], + "title": "metrics.average_page_views" + }, + "metrics.average_time_on_site": { + "description": "AverageTimeOnSite", + "type": ["null", "string"], + "title": "metrics.average_time_on_site" + }, + "ad_group.base_ad_group": { + "description": "BaseAdGroupId", + "type": ["null", "string"], + "title": "ad_group.base_ad_group" + }, + "campaign.base_campaign": { + "description": "BaseCampaignId", + "type": ["null", "string"], + "title": "campaign.base_campaign" + }, + "metrics.bounce_rate": { + "description": "BounceRate", + "type": ["null", "string"], + "title": "metrics.bounce_rate" + }, + "ad_group_ad.ad.legacy_responsive_display_ad.business_name": { + "description": "BusinessName", + "type": ["null", "string"], + "title": "ad_group_ad.ad.legacy_responsive_display_ad.business_name" + }, + "ad_group_ad.ad.call_only_ad.phone_number": { + "description": "CallOnlyPhoneNumber", + "type": ["null", "string"], + "title": "ad_group_ad.ad.call_only_ad.phone_number" + }, + "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": { + "description": "CallToActionText", + "type": ["null", "string"], + "title": "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text" + }, + "campaign.id": { + "description": "CampaignId", + "type": ["null", "string"], + "title": "campaign.id" + }, + "campaign.name": { + "description": "CampaignName", + "type": ["null", "string"], + "title": "campaign.name" + }, + "campaign.status": { + "description": "CampaignStatus", + "type": ["null", "string"], + "title": "campaign.status" + }, + "metrics.clicks": { + "description": "Clicks", + "type": ["null", "string"], + "title": "metrics.clicks" + }, + "ad_group_ad.policy_summary.approval_status": { + "description": "CombinedApprovalStatus", + "type": ["null", "string"], + "title": "ad_group_ad.policy_summary.approval_status" + }, + "metrics.conversions_from_interactions_rate": { + "description": "ConversionRate", + "type": ["null", "string"], + "title": "metrics.conversions_from_interactions_rate" + }, + "metrics.conversions_value": { + "description": "ConversionValue", + "type": ["null", "string"], + "title": "metrics.conversions_value" + }, + "metrics.conversions": { + "description": "Conversions", + "type": ["null", "string"], + "title": "metrics.conversions" + }, + "metrics.cost_micros": { + "description": "Cost", + "type": ["null", "string"], + "title": "metrics.cost_micros" + }, + "metrics.cost_per_all_conversions": { + "description": "CostPerAllConversion", + "type": ["null", "string"], + "title": "metrics.cost_per_all_conversions" + }, + "metrics.cost_per_conversion": { + "description": "CostPerConversion", + "type": ["null", "string"], + "title": "metrics.cost_per_conversion" + }, + "metrics.cost_per_current_model_attributed_conversion": { + "description": "CostPerCurrentModelAttributedConversion", + "type": ["null", "string"], + "title": "metrics.cost_per_current_model_attributed_conversion" + }, + "ad_group_ad.ad.final_mobile_urls": { + "description": "CreativeFinalMobileUrls", + "type": ["null", "string"], + "title": "ad_group_ad.ad.final_mobile_urls" + }, + "ad_group_ad.ad.final_urls": { + "description": "CreativeFinalUrls", + "type": ["null", "string"], + "title": "ad_group_ad.ad.final_urls" + }, + "ad_group_ad.ad.tracking_url_template": { + "description": "CreativeTrackingUrlTemplate", + "type": ["null", "string"], + "title": "ad_group_ad.ad.tracking_url_template" + }, + "ad_group_ad.ad.url_custom_parameters": { + "description": "CreativeUrlCustomParameters", + "type": ["null", "string"], + "title": "ad_group_ad.ad.url_custom_parameters" + }, + "metrics.cross_device_conversions": { + "description": "CrossDeviceConversions", + "type": ["null", "string"], + "title": "metrics.cross_device_conversions" + }, + "metrics.ctr": { + "description": "Ctr", + "type": ["null", "string"], + "title": "metrics.ctr" + }, + "metrics.current_model_attributed_conversions_value": { + "description": "CurrentModelAttributedConversionValue", + "type": ["null", "string"], + "title": "metrics.current_model_attributed_conversions_value" + }, + "metrics.current_model_attributed_conversions": { + "description": "CurrentModelAttributedConversions", + "type": ["null", "string"], + "title": "metrics.current_model_attributed_conversions" + }, + "segments.date": { + "description": "Date", + "type": ["null", "string"], + "title": "segments.date" + }, + "segments.day_of_week": { + "description": "DayOfWeek", + "type": ["null", "string"], + "title": "segments.day_of_week" + }, + "ad_group_ad.ad.expanded_text_ad.description": { + "description": "Description", + "type": ["null", "string"], + "title": "ad_group_ad.ad.expanded_text_ad.description" + }, + "ad_group_ad.ad.text_ad.description1": { + "description": "Description1", + "type": ["null", "string"], + "title": "ad_group_ad.ad.text_ad.description1" + }, + "ad_group_ad.ad.text_ad.description2": { + "description": "Description2", + "type": ["null", "string"], + "title": "ad_group_ad.ad.text_ad.description2" + }, + "ad_group_ad.ad.device_preference": { + "description": "DevicePreference", + "type": ["null", "string"], + "title": "ad_group_ad.ad.device_preference" + }, + "ad_group_ad.ad.display_url": { + "description": "DisplayUrl", + "type": ["null", "string"], + "title": "ad_group_ad.ad.display_url" + }, + "metrics.engagement_rate": { + "description": "EngagementRate", + "type": ["null", "string"], + "title": "metrics.engagement_rate" + }, + "metrics.engagements": { + "description": "Engagements", + "type": ["null", "string"], + "title": "metrics.engagements" + }, + "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": { + "description": "EnhancedDisplayCreativeLandscapeLogoImageMediaId", + "type": ["null", "string"], + "title": "ad_group_ad.ad.legacy_responsive_display_ad.logo_image" + }, + "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": { + "description": "EnhancedDisplayCreativeLogoImageMediaId", + "type": ["null", "string"], + "title": "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image" + }, + "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": { + "description": "EnhancedDisplayCreativeMarketingImageMediaId", + "type": ["null", "string"], + "title": "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image" + }, + "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": { + "description": "EnhancedDisplayCreativeMarketingImageSquareMediaId", + "type": ["null", "string"], + "title": "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image" + }, + "ad_group_ad.ad.expanded_dynamic_search_ad.description": { + "description": "ExpandedDynamicSearchCreativeDescription2", + "type": ["null", "string"], + "title": "ad_group_ad.ad.expanded_dynamic_search_ad.description" + }, + "ad_group_ad.ad.expanded_text_ad.description2": { + "description": "ExpandedTextAdDescription2", + "type": ["null", "string"], + "title": "ad_group_ad.ad.expanded_text_ad.description2" + }, + "ad_group_ad.ad.expanded_text_ad.headline_part3": { + "description": "ExpandedTextAdHeadlinePart3", + "type": ["null", "string"], + "title": "ad_group_ad.ad.expanded_text_ad.headline_part3" + }, + "customer.id": { + "description": "ExternalCustomerId", + "type": ["null", "string"], + "title": "customer.id" + }, + "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": { + "description": "FormatSetting", + "type": ["null", "string"], + "title": "ad_group_ad.ad.legacy_responsive_display_ad.format_setting" + }, + "ad_group_ad.ad.gmail_ad.header_image": { + "description": "GmailCreativeHeaderImageMediaId", + "type": ["null", "string"], + "title": "ad_group_ad.ad.gmail_ad.header_image" + }, + "ad_group_ad.ad.gmail_ad.teaser.logo_image": { + "description": "GmailCreativeLogoImageMediaId", + "type": ["null", "string"], + "title": "ad_group_ad.ad.gmail_ad.teaser.logo_image" + }, + "ad_group_ad.ad.gmail_ad.marketing_image": { + "description": "GmailCreativeMarketingImageMediaId", + "type": ["null", "string"], + "title": "ad_group_ad.ad.gmail_ad.marketing_image" + }, + "metrics.gmail_forwards": { + "description": "GmailForwards", + "type": ["null", "string"], + "title": "metrics.gmail_forwards" + }, + "metrics.gmail_saves": { + "description": "GmailSaves", + "type": ["null", "string"], + "title": "metrics.gmail_saves" + }, + "metrics.gmail_secondary_clicks": { + "description": "GmailSecondaryClicks", + "type": ["null", "string"], + "title": "metrics.gmail_secondary_clicks" + }, + "ad_group_ad.ad.gmail_ad.teaser.business_name": { + "description": "GmailTeaserBusinessName", + "type": ["null", "string"], + "title": "ad_group_ad.ad.gmail_ad.teaser.business_name" + }, + "ad_group_ad.ad.gmail_ad.teaser.description": { + "description": "GmailTeaserDescription", + "type": ["null", "string"], + "title": "ad_group_ad.ad.gmail_ad.teaser.description" + }, + "ad_group_ad.ad.gmail_ad.teaser.headline": { + "description": "GmailTeaserHeadline", + "type": ["null", "string"], + "title": "ad_group_ad.ad.gmail_ad.teaser.headline" + }, + "ad_group_ad.ad.text_ad.headline": { + "description": "Headline", + "type": ["null", "string"], + "title": "ad_group_ad.ad.text_ad.headline" + }, + "ad_group_ad.ad.expanded_text_ad.headline_part1": { + "description": "HeadlinePart1", + "type": ["null", "string"], + "title": "ad_group_ad.ad.expanded_text_ad.headline_part1" + }, + "ad_group_ad.ad.expanded_text_ad.headline_part2": { + "description": "HeadlinePart2", + "type": ["null", "string"], + "title": "ad_group_ad.ad.expanded_text_ad.headline_part2" + }, + "ad_group_ad.ad.id": { + "description": "Id", + "type": ["null", "string"], + "title": "ad_group_ad.ad.id" + }, + "ad_group_ad.ad.image_ad.image_url": { + "description": "ImageAdUrl", + "type": ["null", "string"], + "title": "ad_group_ad.ad.image_ad.image_url" + }, + "ad_group_ad.ad.image_ad.pixel_height": { + "description": "ImageCreativeImageHeight", + "type": ["null", "string"], + "title": "ad_group_ad.ad.image_ad.pixel_height" + }, + "ad_group_ad.ad.image_ad.pixel_width": { + "description": "ImageCreativeImageWidth", + "type": ["null", "string"], + "title": "ad_group_ad.ad.image_ad.pixel_width" + }, + "ad_group_ad.ad.image_ad.mime_type": { + "description": "ImageCreativeMimeType", + "type": ["null", "string"], + "title": "ad_group_ad.ad.image_ad.mime_type" + }, + "ad_group_ad.ad.image_ad.name": { + "description": "ImageCreativeName", + "type": ["null", "string"], + "title": "ad_group_ad.ad.image_ad.name" + }, + "metrics.impressions": { + "description": "Impressions", + "type": ["null", "string"], + "title": "metrics.impressions" + }, + "metrics.interaction_rate": { + "description": "InteractionRate", + "type": ["null", "string"], + "title": "metrics.interaction_rate" + }, + "metrics.interaction_event_types": { + "description": "InteractionTypes", + "type": ["null", "string"], + "title": "metrics.interaction_event_types" + }, + "metrics.interactions": { + "description": "Interactions", + "type": ["null", "string"], + "title": "metrics.interactions" + }, + "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": { + "description": "LongHeadline", + "type": ["null", "string"], + "title": "ad_group_ad.ad.legacy_responsive_display_ad.long_headline" + }, + "ad_group_ad.ad.legacy_responsive_display_ad.main_color": { + "description": "MainColor", + "type": ["null", "string"], + "title": "ad_group_ad.ad.legacy_responsive_display_ad.main_color" + }, + "ad_group_ad.ad.gmail_ad.marketing_image_display_call_to_action.text": { + "description": "MarketingImageCallToActionText", + "type": ["null", "string"], + "title": "ad_group_ad.ad.gmail_ad.marketing_image_display_call_to_action.text" + }, + "ad_group_ad.ad.gmail_ad.marketing_image_display_call_to_action.text_color": { + "description": "MarketingImageCallToActionTextColor", + "type": ["null", "string"], + "title": "ad_group_ad.ad.gmail_ad.marketing_image_display_call_to_action.text_color" + }, + "ad_group_ad.ad.gmail_ad.marketing_image_headline": { + "description": "MarketingImageDescription", + "type": ["null", "string"], + "title": "ad_group_ad.ad.gmail_ad.marketing_image_headline" + }, + "ad_group_ad.ad.gmail_ad.marketing_image_description": { + "description": "MarketingImageHeadline", + "type": ["null", "string"], + "title": "ad_group_ad.ad.gmail_ad.marketing_image_description" + }, + "segments.month": { + "description": "Month", + "type": ["null", "string"], + "title": "segments.month" + }, + "ad_group_ad.ad.responsive_display_ad.accent_color": { + "description": "MultiAssetResponsiveDisplayAdAccentColor", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_display_ad.accent_color" + }, + "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": { + "description": "MultiAssetResponsiveDisplayAdAllowFlexibleColor", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_display_ad.allow_flexible_color" + }, + "ad_group_ad.ad.responsive_display_ad.business_name": { + "description": "MultiAssetResponsiveDisplayAdBusinessName", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_display_ad.business_name" + }, + "ad_group_ad.ad.responsive_display_ad.call_to_action_text": { + "description": "MultiAssetResponsiveDisplayAdCallToActionText", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_display_ad.call_to_action_text" + }, + "ad_group_ad.ad.responsive_display_ad.descriptions": { + "description": "MultiAssetResponsiveDisplayAdDescriptions", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_display_ad.descriptions" + }, + "ad_group_ad.ad.responsive_display_ad.price_prefix": { + "description": "MultiAssetResponsiveDisplayAdDynamicSettingsPricePrefix", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_display_ad.price_prefix" + }, + "ad_group_ad.ad.responsive_display_ad.promo_text": { + "description": "MultiAssetResponsiveDisplayAdDynamicSettingsPromoText", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_display_ad.promo_text" + }, + "ad_group_ad.ad.responsive_display_ad.format_setting": { + "description": "MultiAssetResponsiveDisplayAdFormatSetting", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_display_ad.format_setting" + }, + "ad_group_ad.ad.responsive_display_ad.headlines": { + "description": "MultiAssetResponsiveDisplayAdHeadlines", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_display_ad.headlines" + }, + "ad_group_ad.ad.responsive_display_ad.logo_images": { + "description": "MultiAssetResponsiveDisplayAdLandscapeLogoImages", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_display_ad.logo_images" + }, + "ad_group_ad.ad.responsive_display_ad.square_logo_images": { + "description": "MultiAssetResponsiveDisplayAdLogoImages", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_display_ad.square_logo_images" + }, + "ad_group_ad.ad.responsive_display_ad.long_headline": { + "description": "MultiAssetResponsiveDisplayAdLongHeadline", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_display_ad.long_headline" + }, + "ad_group_ad.ad.responsive_display_ad.main_color": { + "description": "MultiAssetResponsiveDisplayAdMainColor", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_display_ad.main_color" + }, + "ad_group_ad.ad.responsive_display_ad.marketing_images": { + "description": "MultiAssetResponsiveDisplayAdMarketingImages", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_display_ad.marketing_images" + }, + "ad_group_ad.ad.responsive_display_ad.square_marketing_images": { + "description": "MultiAssetResponsiveDisplayAdSquareMarketingImages", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_display_ad.square_marketing_images" + }, + "ad_group_ad.ad.responsive_display_ad.youtube_videos": { + "description": "MultiAssetResponsiveDisplayAdYouTubeVideos", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_display_ad.youtube_videos" + }, + "ad_group_ad.ad.expanded_text_ad.path1": { + "description": "Path1", + "type": ["null", "string"], + "title": "ad_group_ad.ad.expanded_text_ad.path1" + }, + "ad_group_ad.ad.expanded_text_ad.path2": { + "description": "Path2", + "type": ["null", "string"], + "title": "ad_group_ad.ad.expanded_text_ad.path2" + }, + "metrics.percent_new_visitors": { + "description": "PercentNewVisitors", + "type": ["null", "string"], + "title": "metrics.percent_new_visitors" + }, + "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": { + "description": "PricePrefix", + "type": ["null", "string"], + "title": "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix" + }, + "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": { + "description": "PromoText", + "type": ["null", "string"], + "title": "ad_group_ad.ad.legacy_responsive_display_ad.promo_text" + }, + "segments.quarter": { + "description": "Quarter", + "type": ["null", "string"], + "title": "segments.quarter" + }, + "ad_group_ad.ad.responsive_search_ad.descriptions": { + "description": "ResponsiveSearchAdDescriptions", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_search_ad.descriptions" + }, + "ad_group_ad.ad.responsive_search_ad.headlines": { + "description": "ResponsiveSearchAdHeadlines", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_search_ad.headlines" + }, + "ad_group_ad.ad.responsive_search_ad.path1": { + "description": "ResponsiveSearchAdPath1", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_search_ad.path1" + }, + "ad_group_ad.ad.responsive_search_ad.path2": { + "description": "ResponsiveSearchAdPath2", + "type": ["null", "string"], + "title": "ad_group_ad.ad.responsive_search_ad.path2" + }, + "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": { + "description": "ShortHeadline", + "type": ["null", "string"], + "title": "ad_group_ad.ad.legacy_responsive_display_ad.short_headline" + }, + "ad_group_ad.status": { + "description": "Status", + "type": ["null", "string"], + "title": "ad_group_ad.status" + }, + "ad_group_ad.ad.system_managed_resource_source": { + "description": "SystemManagedEntitySource", + "type": ["null", "string"], + "title": "ad_group_ad.ad.system_managed_resource_source" + }, + "metrics.top_impression_percentage": { + "description": "TopImpressionPercentage", + "type": ["null", "string"], + "title": "metrics.top_impression_percentage" + }, + "ad_group_ad.ad.app_ad.descriptions": { + "description": "UniversalAppAdDescriptions", + "type": ["null", "string"], + "title": "ad_group_ad.ad.app_ad.descriptions" + }, + "ad_group_ad.ad.app_ad.headlines": { + "description": "UniversalAppAdHeadlines", + "type": ["null", "string"], + "title": "ad_group_ad.ad.app_ad.headlines" + }, + "ad_group_ad.ad.app_ad.html5_media_bundles": { + "description": "UniversalAppAdHtml5MediaBundles", + "type": ["null", "string"], + "title": "ad_group_ad.ad.app_ad.html5_media_bundles" + }, + "ad_group_ad.ad.app_ad.images": { + "description": "UniversalAppAdImages", + "type": ["null", "string"], + "title": "ad_group_ad.ad.app_ad.images" + }, + "ad_group_ad.ad.app_ad.mandatory_ad_text": { + "description": "UniversalAppAdMandatoryAdText", + "type": ["null", "string"], + "title": "ad_group_ad.ad.app_ad.mandatory_ad_text" + }, + "ad_group_ad.ad.app_ad.youtube_videos": { + "description": "UniversalAppAdYouTubeVideos", + "type": ["null", "string"], + "title": "ad_group_ad.ad.app_ad.youtube_videos" + }, + "metrics.value_per_all_conversions": { + "description": "ValuePerAllConversion", + "type": ["null", "string"], + "title": "metrics.value_per_all_conversions" + }, + "metrics.value_per_conversion": { + "description": "ValuePerConversion", + "type": ["null", "string"], + "title": "metrics.value_per_conversion" + }, + "metrics.value_per_current_model_attributed_conversion": { + "description": "ValuePerCurrentModelAttributedConversion", + "type": ["null", "string"], + "title": "metrics.value_per_current_model_attributed_conversion" + }, + "metrics.video_quartile_p100_rate": { + "description": "VideoQuartile100Rate", + "type": ["null", "string"], + "title": "metrics.video_quartile_p100_rate" + }, + "metrics.video_quartile_p25_rate": { + "description": "VideoQuartile25Rate", + "type": ["null", "string"], + "title": "metrics.video_quartile_p25_rate" + }, + "metrics.video_quartile_p50_rate": { + "description": "VideoQuartile50Rate", + "type": ["null", "string"], + "title": "metrics.video_quartile_p50_rate" + }, + "metrics.video_quartile_p75_rate": { + "description": "VideoQuartile75Rate", + "type": ["null", "string"], + "title": "metrics.video_quartile_p75_rate" + }, + "metrics.video_view_rate": { + "description": "VideoViewRate", + "type": ["null", "string"], + "title": "metrics.video_view_rate" + }, + "metrics.video_views": { + "description": "VideoViews", + "type": ["null", "string"], + "title": "metrics.video_views" + }, + "metrics.view_through_conversions": { + "description": "ViewThroughConversions", + "type": ["null", "string"], + "title": "metrics.view_through_conversions" + }, + "segments.week": { + "description": "Week", + "type": ["null", "string"], + "title": "segments.week" + }, + "segments.year": { + "description": "Year", + "type": ["null", "string"], + "title": "segments.year" + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py new file mode 100644 index 0000000000000..3e24ae2434dcb --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py @@ -0,0 +1,132 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import pendulum +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from google.ads.googleads.v7.services.types.google_ads_service import SearchGoogleAdsResponse + +from .google_ads import GoogleAds +from .utils import Utils + + +def chunk_date_range(start_date: str, end_date: str, conversion_window: Optional[int], field: str) -> Iterable[Mapping[str, any]]: + """ + Passing optional parameter end_date for testing + Returns a list of the beginning and ending timetsamps of each month between the start date and now. + The return value is a list of dicts {'date': str} which can be used directly with the Slack API + """ + intervals = [] + end_date = pendulum.parse(end_date) if end_date else pendulum.now() + start_date = pendulum.parse(start_date) + + # As in to return some state when state in abnormal + if start_date > end_date: + return [{field: start_date.to_date_string()}] + + # applying conversion windoe + start_date = start_date.subtract(days=conversion_window) + + # Each stream_slice contains the beginning and ending timestamp for a 24 hour period + while start_date < end_date: + intervals.append({field: start_date.to_date_string()}) + start_date = start_date.add(months=1) + + return intervals + + +class GoogleAdsStream(Stream, ABC): + DEFAULT_CONVERSION_WINDOW_DAYS = 14 + + def __init__(self, config): + self.config = config + self.google_ads_client = GoogleAds(**config) + self.conversion_window_days = self.config.get('conversion_window_days', self.DEFAULT_CONVERSION_WINDOW_DAYS) + + def parse_response(self, response: SearchGoogleAdsResponse) -> Iterable[Mapping]: + for result in response: + record = GoogleAds.parse_single_result(self.get_json_schema(), result) + yield record + + def read_records( + self, + sync_mode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + start_date, end_date = Utils.get_date_params(stream_slice, self.cursor_field) + query = GoogleAds.convert_schema_into_query(self.get_json_schema(), self.name, start_date, end_date) + + response = self.google_ads_client.send_request(query) + yield from self.parse_response(response) + + +class IncrementalGoogleAdsStream(GoogleAdsStream, ABC): + state_checkpoint_interval = None + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + stream_state = stream_state or {} + start_date = stream_state.get(self.cursor_field) or self.config.get("start_date") + + return chunk_date_range(start_date, None, self.conversion_window_days, self.cursor_field) + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + current_stream_state = current_stream_state or {} + + # When state is none return date from latest record + if current_stream_state.get(self.cursor_field) is None: + current_stream_state[self.cursor_field] = latest_record[self.cursor_field] + + return current_stream_state + + date_in_current_stream = pendulum.parse(current_stream_state.get(self.cursor_field)) + date_in_latest_record = pendulum.parse(latest_record[self.cursor_field]) + + current_stream_state[self.cursor_field] = (max(date_in_current_stream, date_in_latest_record)).to_date_string() + + return current_stream_state + + +class AdGroupAdReport(IncrementalGoogleAdsStream): + cursor_field = "segments.date" + primary_key = None + + +# Source +class SourceGoogleAds(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + try: + logger.info("Checking the config") + GoogleAds(**config) + return True, None + except Exception as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + return [AdGroupAdReport(config)] diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json new file mode 100644 index 0000000000000..4fb0a09ae8341 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json @@ -0,0 +1,59 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-ads", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Google Ads Spec", + "type": "object", + "required": [ + "developer_token", + "client_id", + "client_secret", + "refresh_token", + "start_date", + "customer_id" + ], + "additionalProperties": false, + "properties": { + "developer_token": { + "type": "string", + "title": "Developer Token", + "description": "Developer token granted by Google to use their APIs. More instruction on how to find this value in our docs", + "airbyte_secret": true + }, + "client_id": { + "type": "string", + "title": "Client Id", + "description": "Google client id. More instruction on how to find this value in our docs" + }, + "client_secret": { + "type": "string", + "title": "Client Secret", + "description": "Google client secret. More instruction on how to find this value in our docs", + "airbyte_secret": true + }, + "refresh_token": { + "type": "string", + "title": "Refresh Token", + "description": "Refresh token generated using developer_token, oauth_client_id, and oauth_client_secret. More instruction on how to find this value in our docs", + "airbyte_secret": true + }, + "start_date": { + "type": "string", + "title": "Start Date", + "description": "UTC date and time in the format 2017-01-25. Any data before this date will not be replicated.", + "examples": ["2017-01-25"] + }, + "customer_id": { + "title": "Customer Id", + "type": "string", + "description": "Customer id must be specified as a 10-digit number without dashes. More instruction on how to find this value in our docs" + }, + "conversion_window_days": { + "title": "Conversion Window", + "type": "integer", + "description": "Define the historical replication lookback window in days", + "examples": [14] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py new file mode 100644 index 0000000000000..b233a6f26025e --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py @@ -0,0 +1,39 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from typing import Any, Mapping + +import pendulum +from pendulum import DateTime + + +class Utils: + @staticmethod + def get_date_params(stream_slice: Mapping[str, Any], cursor_field: str, end_date: DateTime = pendulum.yesterday()): + start_date = pendulum.parse(stream_slice.get(cursor_field)) + if start_date > pendulum.now(): + return start_date.to_date_string(), start_date.add(days=1).to_date_string() + + end_date = min(end_date, pendulum.parse(stream_slice.get(cursor_field)).add(months=1)) + return start_date.add(days=1).to_date_string(), end_date.to_date_string() diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py new file mode 100644 index 0000000000000..707729a1f2826 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +import json + +import pytest + + +@pytest.fixture(scope="session", name="config") +def config_fixture(): + with open("secrets/config.json", "r") as config_file: + return json.load(config_file) diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py new file mode 100644 index 0000000000000..55db571ec7719 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py @@ -0,0 +1,147 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from datetime import date +from string import Template + +from source_google_ads.google_ads import GoogleAds + +SAMPLE_SCHEMA = { + "properties": { + "segment.date": { + "type": ["null", "string"], + } + } +} + + +# Mocking Classes +class MockGoogleAdsService: + def search(self, search_request): + return search_request + + +class MockedDateSegment: + def __init__(self, date: str): + self._mock_date = date + + def __getattr__(self, attr): + if attr == "date": + return date.fromisoformat(self._mock_date) + return MockedDateSegment(self._mock_date) + + +class MockSearchRequest: + customer_id = "12345" + query = None + page_size = 100 + page_token = None + + +class MockGoogleAdsClient: + def __init__(self, config): + self.config = config + + def get_type(self, type): + return MockSearchRequest() + + def get_service(self, service): + return MockGoogleAdsService() + + @staticmethod + def load_from_dict(config): + return MockGoogleAdsClient(config) + + +SAMPLE_CONFIG = { + "developer_token": "developer_token", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token", + "customer_id": "customer_id", +} + +EXPECTED_CRED = { + "developer_token": "developer_token", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token", +} + + +def test_google_ads_init(mocker): + google_client_mocker = mocker.patch("source_google_ads.google_ads.GoogleAdsClient", return_value=MockGoogleAdsClient) + google_ads_client = GoogleAds(**SAMPLE_CONFIG) + assert google_ads_client.customer_id == SAMPLE_CONFIG["customer_id"] + assert google_client_mocker.load_from_dict.call_args[0][0] == EXPECTED_CRED + + +def test_send_request(mocker): + mocker.patch("source_google_ads.google_ads.GoogleAdsClient.load_from_dict", return_value=MockGoogleAdsClient(SAMPLE_CONFIG)) + mocker.patch("source_google_ads.google_ads.GoogleAdsClient.get_service", return_value=MockGoogleAdsService()) + google_ads_client = GoogleAds(**SAMPLE_CONFIG) + query = "Query" + page_size = 1000 + response = google_ads_client.send_request(query) + + assert response.customer_id == SAMPLE_CONFIG["customer_id"] + assert response.query == query + assert response.page_size == page_size + + +def test_get_fields_from_schema(): + response = GoogleAds.get_fields_from_schema(SAMPLE_SCHEMA) + assert response == ["segment.date"] + + +def test_convert_schema_into_query(): + report_name = "ad_group_ad_report" + query = Template( + """ + SELECT + segment.date + FROM ad_group_ad + WHERE segments.date > '2020-01-01' + AND segments.date < '2020-03-01' + ORDER BY segments.date + """ + ) + response = GoogleAds.convert_schema_into_query(SAMPLE_SCHEMA, report_name, "2020-01-01", "2020-03-01") + assert response == query.substitute() + + +def test_get_field_value(): + field = "segment.date" + date = "2001-01-01" + response = GoogleAds.get_field_value(MockedDateSegment(date), field) + assert response == date + date = "2020" + response = GoogleAds.get_field_value(MockedDateSegment(date), field) + assert response is None + + +def test_parse_single_result(): + date = "2001-01-01" + response = GoogleAds.parse_single_result(SAMPLE_SCHEMA, MockedDateSegment(date)) + assert response == response diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py new file mode 100644 index 0000000000000..6effa2fd52aa3 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py @@ -0,0 +1,50 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from source_google_ads.source import AdGroupAdReport, chunk_date_range + + +def test_chunk_date_range(): + start_date = "2021-03-04" + end_date = "2021-05-04" + conversion_window = 14 + field = "date" + response = chunk_date_range(start_date, end_date, conversion_window, field) + assert [{"date": "2021-02-18"}, {"date": "2021-03-18"}, {"date": "2021-04-18"}] == response + + +# this requires the config because instantiating a stream creates a google client. TODO refactor so client can be mocked. +def test_get_updated_state(config): + client = AdGroupAdReport(config) + current_state_stream = {} + latest_record = {"segments.date": "2020-01-01"} + + new_stream_state = client.get_updated_state(current_state_stream, latest_record) + assert new_stream_state == {"segments.date": "2020-01-01"} + + current_state_stream = {"segments.date": "2020-01-01"} + latest_record = {"segments.date": "2020-02-01"} + new_stream_state = client.get_updated_state(current_state_stream, latest_record) + assert new_stream_state == {"segments.date": "2020-02-01"} + diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py new file mode 100644 index 0000000000000..f86ca17171ca7 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py @@ -0,0 +1,39 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from source_google_ads.utils import Utils +import pendulum + +def test_get_date_params(): + stream_slice = {"date": "2020-01-01"} + cursor_field = "date" + + response = Utils.get_date_params(stream_slice, cursor_field, pendulum.parse("2020-02-15")) + assert response == ("2020-01-02", "2020-02-01") + + stream_slice = {"date": "2029-01-01"} + cursor_field = "date" + + response = Utils.get_date_params(stream_slice, cursor_field) + assert response == ("2029-01-01", "2029-01-02") \ No newline at end of file diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 3cfb6cd5119e0..95c0540f32c2f 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -43,6 +43,7 @@ write_standard_creds source-freshdesk "$FRESHDESK_TEST_CREDS" write_standard_creds source-facebook-marketing "$FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS" write_standard_creds source-gitlab-singer "$GITLAB_INTEGRATION_TEST_CREDS" write_standard_creds source-github-singer "$GH_INTEGRATION_TEST_CREDS" +write_standard_creds source-google-ads "$GOOGLE_ADS_TEST_CREDS" write_standard_creds source-google-adwords-singer "$ADWORDS_INTEGRATION_TEST_CREDS" write_standard_creds source-googleanalytics-singer "$GOOGLE_ANALYTICS_TEST_CREDS" write_standard_creds source-googleanalytics-singer "$GOOGLE_ANALYTICS_TEST_TRACKING_ID" "tracker.txt"