Skip to content

Externalize configuration of testing AWS resources #17952

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
findepi merged 3 commits into from
Jun 19, 2023
Merged

Externalize configuration of testing AWS resources #17952

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
aa2b82b
Externalize configuration of testing AWS resources
findepi Jun 19, 2023
e1a77ed
Use one GCS bucket for GCP tests
findepi Jun 19, 2023
64c0580
Externalize configuration of testing GCP resources
findepi Jun 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 22 additions & 18 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,8 +312,8 @@ jobs:
env:
AWS_ACCESS_KEY_ID: ${{ secrets.TRINO_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
S3_BUCKET: "trino-ci-test"
S3_BUCKET_ENDPOINT: "https://s3.us-east-2.amazonaws.com"
S3_BUCKET: ${{ vars.TRINO_S3_BUCKET }}
S3_BUCKET_ENDPOINT: "https://s3.${{ vars.TRINO_AWS_REGION }}.amazonaws.com"
run: |
if [ "${AWS_ACCESS_KEY_ID}" != "" ]; then
source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
Expand All @@ -327,9 +327,9 @@ jobs:
env:
AWS_ACCESS_KEY_ID: ${{ secrets.TRINO_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: us-east-2
S3_BUCKET: "trino-ci-test"
S3_BUCKET_ENDPOINT: "s3.us-east-2.amazonaws.com"
AWS_REGION: ${{ vars.TRINO_AWS_REGION }}
S3_BUCKET: ${{ vars.TRINO_S3_BUCKET }}
S3_BUCKET_ENDPOINT: "s3.${{ vars.TRINO_AWS_REGION }}.amazonaws.com"
run: |
if [ "${AWS_ACCESS_KEY_ID}" != "" ]; then
$MAVEN test ${MAVEN_TEST} -pl :trino-hive -P aws-tests
Expand Down Expand Up @@ -642,8 +642,8 @@ jobs:
env:
AWS_ACCESS_KEY_ID: ${{ secrets.TRINO_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: us-east-2
S3_BUCKET: trino-ci-s3fs-test
AWS_REGION: ${{ vars.TRINO_AWS_REGION }}
S3_BUCKET: ${{ vars.TRINO_S3_S3FS_TESTS_BUCKET }}
if: >-
contains(matrix.modules, 'trino-filesystem-s3') && contains(matrix.profile, 'cloud-tests') &&
(env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.AWS_ACCESS_KEY_ID != '' || env.AWS_SECRET_ACCESS_KEY != '')
Expand All @@ -657,9 +657,10 @@ jobs:
ABFS_ACCESSKEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }}
AWS_ACCESS_KEY_ID: ${{ secrets.TRINO_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: us-east-2
S3_BUCKET: trino-ci-test
AWS_REGION: ${{ vars.TRINO_AWS_REGION }}
S3_BUCKET: ${{ vars.TRINO_S3_BUCKET }}
GCP_CREDENTIALS_KEY: ${{ secrets.GCP_CREDENTIALS_KEY }}
GCP_STORAGE_BUCKET: ${{ vars.GCP_STORAGE_BUCKET }}
# Run tests if any of the secrets is present. Do not skip tests when one secret renamed, or secret name has a typo.
if: >-
contains(matrix.modules, 'trino-delta-lake') && contains(matrix.profile, 'cloud-tests') &&
Expand All @@ -669,7 +670,7 @@ jobs:
-Dhive.hadoop2.azure-abfs-container="${ABFS_CONTAINER}" \
-Dhive.hadoop2.azure-abfs-account="${ABFS_ACCOUNT}" \
-Dhive.hadoop2.azure-abfs-access-key="${ABFS_ACCESSKEY}" \
-Dtesting.gcp-storage-bucket="trino-ci-test" \
-Dtesting.gcp-storage-bucket="${GCP_STORAGE_BUCKET}" \
-Dtesting.gcp-credentials-key="${GCP_CREDENTIALS_KEY}"
- name: Memsql Tests
env:
Expand All @@ -680,20 +681,22 @@ jobs:
- name: Cloud BigQuery Tests
env:
BIGQUERY_CREDENTIALS_KEY: ${{ secrets.BIGQUERY_CREDENTIALS_KEY }}
GCP_STORAGE_BUCKET: ${{ vars.GCP_STORAGE_BUCKET }}
if: matrix.modules == 'plugin/trino-bigquery' && !contains(matrix.profile, 'cloud-tests-arrow') && (env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.BIGQUERY_CREDENTIALS_KEY != '')
run: |
$MAVEN test ${MAVEN_TEST} -pl :trino-bigquery -Pcloud-tests \
-Dbigquery.credentials-key="${BIGQUERY_CREDENTIALS_KEY}" \
-Dtesting.gcp-storage-bucket="trino-ci-test" \
-Dtesting.gcp-storage-bucket="${GCP_STORAGE_BUCKET}" \
-Dtesting.alternate-bq-project-id=bigquery-cicd-alternate
- name: Cloud BigQuery Arrow Serialization Tests
env:
BIGQUERY_CREDENTIALS_KEY: ${{ secrets.BIGQUERY_CREDENTIALS_KEY }}
GCP_STORAGE_BUCKET: ${{ vars.GCP_STORAGE_BUCKET }}
if: matrix.modules == 'plugin/trino-bigquery' && contains(matrix.profile, 'cloud-tests-arrow') && (env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.BIGQUERY_CREDENTIALS_KEY != '')
run: |
$MAVEN test ${MAVEN_TEST} -pl :trino-bigquery -Pcloud-tests-arrow \
-Dbigquery.credentials-key="${BIGQUERY_CREDENTIALS_KEY}" \
-Dtesting.gcp-storage-bucket="trino-ci-test"
-Dtesting.gcp-storage-bucket="${GCP_STORAGE_BUCKET}"
- name: Cloud BigQuery Case Insensitive Mapping Tests
env:
BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY: ${{ secrets.BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY }}
Expand All @@ -704,9 +707,10 @@ jobs:
env:
AWS_ACCESS_KEY_ID: ${{ secrets.TRINO_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: us-east-2
S3_BUCKET: trino-ci-test
AWS_REGION: ${{ vars.TRINO_AWS_REGION }}
S3_BUCKET: ${{ vars.TRINO_S3_BUCKET }}
GCP_CREDENTIALS_KEY: ${{ secrets.GCP_CREDENTIALS_KEY }}
GCP_STORAGE_BUCKET: ${{ vars.GCP_STORAGE_BUCKET }}
ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }}
ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }}
ABFS_ACCESS_KEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }}
Expand All @@ -716,7 +720,7 @@ jobs:
run: |
$MAVEN test ${MAVEN_TEST} -pl :trino-iceberg ${{ format('-P {0}', matrix.profile) }} \
-Ds3.bucket=${S3_BUCKET} \
-Dtesting.gcp-storage-bucket="trino-ci-test-us-east" \
-Dtesting.gcp-storage-bucket="${GCP_STORAGE_BUCKET}" \
-Dtesting.gcp-credentials-key="${GCP_CREDENTIALS_KEY}" \
-Dhive.hadoop2.azure-abfs-container="${ABFS_CONTAINER}" \
-Dhive.hadoop2.azure-abfs-account="${ABFS_ACCOUNT}" \
Expand Down Expand Up @@ -1090,8 +1094,8 @@ jobs:
ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }}
ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }}
ABFS_ACCESS_KEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }}
S3_BUCKET: trino-ci-test
AWS_REGION: us-east-2
S3_BUCKET: ${{ vars.TRINO_S3_BUCKET }}
AWS_REGION: ${{ vars.TRINO_AWS_REGION }}
TRINO_AWS_ACCESS_KEY_ID: ${{ secrets.TRINO_AWS_ACCESS_KEY_ID }}
TRINO_AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
DATABRICKS_73_JDBC_URL: ${{ secrets.DATABRICKS_73_JDBC_URL }}
Expand All @@ -1102,7 +1106,7 @@ jobs:
DATABRICKS_LOGIN: token
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
GCP_CREDENTIALS_KEY: ${{ secrets.GCP_CREDENTIALS_KEY }}
GCP_STORAGE_BUCKET: trino-ci-test-us-east
GCP_STORAGE_BUCKET: ${{ vars.GCP_STORAGE_BUCKET }}
run: |
exec testing/trino-product-tests-launcher/target/trino-product-tests-launcher-*-executable.jar suite run \
--suite ${{ matrix.suite }} \
Expand Down