diff --git a/plugin/trino-bigquery/README.md b/plugin/trino-bigquery/README.md index e4e63f67ba30..85f4b560f08a 100644 --- a/plugin/trino-bigquery/README.md +++ b/plugin/trino-bigquery/README.md @@ -1,7 +1,7 @@ # BigQuery Connector Developer Notes The BigQuery connector module has both unit tests and integration tests. -The integration tests require access to a BigQuery instance in Google Cloud seeded with TPCH data. +The integration tests require access to a BigQuery instance in Google Cloud. You can follow the steps below to be able to run the integration tests locally. ## Requirements @@ -15,17 +15,13 @@ You can follow the steps below to be able to run the integration tests locally. * [Enable BigQuery in your Google Cloud account](https://console.cloud.google.com/flows/enableapi?apiid=bigquery). * Build the project by following the instructions [here](../../README.md). -* Run Trino with the TPCH connector installed using Docker as `docker run --rm --name trino -it -p 8080:8080 - trinodb/trino:latest`. -* Run the script `plugin/trino-bigquery/bin/import-tpch-to-bigquery.sh` and pass your Google Cloud project id to it as - an argument. e.g. `plugin/trino-bigquery/bin/import-tpch-to-bigquery.sh trino-bigquery-07` where `trino-bigquery-07` - is your Google Cloud project id. -* Run `gsutil cp src/test/resources/region.csv gs://DESTINATION_BUCKET_NAME/tpch/tiny/region.csv` +* Create a Google Cloud Storage bucket using `gsutil mb gs://DESTINATION_BUCKET_NAME` +* Run `gsutil cp plugin/trino-bigquery/src/test/resources/region.csv gs://DESTINATION_BUCKET_NAME/tpch/tiny/region.csv` (replace `DESTINATION_BUCKET_NAME` with the target bucket name). -* [Create a service account](https://cloud.google.com/docs/authentication/getting-started) in Google Cloud with the +* [Create a service account](https://cloud.google.com/iam/docs/creating-managing-service-accounts#iam-service-accounts-create-console) in Google Cloud with the **BigQuery Admin** role assigned. * Get the base64 encoded text of the service account credentials file using `base64 /path/to/service_account_credentials.json`. -* Set the VM option `bigquery.credentials-key` in the IntelliJ "Run Configuration" (or on the CLI if using Maven - directly). It should look something like `-Dbigquery.credentials-key=base64-text`. +* Set the VM option `bigquery.credentials-key` and `testing.gcp-storage-bucket` in the IntelliJ "Run Configuration" (or on the CLI if using Maven + directly). It should look something like `-Dbigquery.credentials-key=base64-text -Dtesting.gcp-storage-bucket=DESTINATION_BUCKET_NAME`. * Run any test of your choice. diff --git a/plugin/trino-bigquery/bin/import-tpch-to-bigquery.sh b/plugin/trino-bigquery/bin/import-tpch-to-bigquery.sh deleted file mode 100755 index e1a079d97abd..000000000000 --- a/plugin/trino-bigquery/bin/import-tpch-to-bigquery.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -set -euo pipefail -cd "${BASH_SOURCE%/*}" - -if [[ "$#" -ne 1 ]]; then - printf '%s\n' "Usage: $0 " - exit 1 -fi - -target_project="$1" -trino_cli="../../../client/trino-cli/target/trino-cli-*-executable.jar" - -for file in $trino_cli; do - if [[ -x "$file" ]]; then - trino_cli="$file" - break - else - printf '%s\n' "Couldn't find $trino_cli. Please build the project first following the steps in the README." - exit 1 - fi -done - -source_catalog=tpch -source_schema=tiny - -tables=('customer' 'lineitem' 'nation' 'orders' 'part' 'partsupp' 'region' 'supplier') - -function trino_cli() { - ${trino_cli} "$@" -} - -printf '%s\n' "Creating datasets... You can safely ignore any messages about a dataset already existing." -bq mk --dataset --description 'TPCH dataset for Trino tests' "$target_project:tpch" || true -bq mk --dataset --description 'Test dataset for Trino tests' "$target_project:test" || true - -for table in "${tables[@]}"; do - printf '%s\n' "Exporting ${source_catalog}.${source_schema}.${table}" - trino_cli --execute "SELECT * FROM ${source_catalog}.${source_schema}.${table}" --output-format=CSV > "/tmp/${table}.csv" - printf '%s\n' "Importing ${table}.csv to tpch.${table}" - # if the tables already exist they will be overwritten due to --replace - bq --project_id="${target_project}" load --replace --source_format=CSV "tpch.${table}" "/tmp/${table}.csv" "./schema/${table}.json" -done diff --git a/plugin/trino-bigquery/bin/schema/customer.json b/plugin/trino-bigquery/bin/schema/customer.json deleted file mode 100644 index 75176c0b0c0c..000000000000 --- a/plugin/trino-bigquery/bin/schema/customer.json +++ /dev/null @@ -1,42 +0,0 @@ -[ - { - "name": "custkey", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "name", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "address", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "nationkey", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "phone", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "acctbal", - "type": "FLOAT", - "mode": "REQUIRED" - }, - { - "name": "mktsegment", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "comment", - "type": "STRING", - "mode": "REQUIRED" - } -] diff --git a/plugin/trino-bigquery/bin/schema/lineitem.json b/plugin/trino-bigquery/bin/schema/lineitem.json deleted file mode 100644 index 5f27318b3d5f..000000000000 --- a/plugin/trino-bigquery/bin/schema/lineitem.json +++ /dev/null @@ -1,82 +0,0 @@ -[ - { - "name": "orderkey", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "partkey", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "suppkey", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "linenumber", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "quantity", - "type": "FLOAT", - "mode": "REQUIRED" - }, - { - "name": "extendedprice", - "type": "FLOAT", - "mode": "REQUIRED" - }, - { - "name": "discount", - "type": "FLOAT", - "mode": "REQUIRED" - }, - { - "name": "tax", - "type": "FLOAT", - "mode": "REQUIRED" - }, - { - "name": "returnflag", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "linestatus", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "shipdate", - "type": "DATE", - "mode": "REQUIRED" - }, - { - "name": "commitdate", - "type": "DATE", - "mode": "REQUIRED" - }, - { - "name": "receiptdate", - "type": "DATE", - "mode": "REQUIRED" - }, - { - "name": "shipinstruct", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "shipmode", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "comment", - "type": "STRING", - "mode": "REQUIRED" - } -] diff --git a/plugin/trino-bigquery/bin/schema/nation.json b/plugin/trino-bigquery/bin/schema/nation.json deleted file mode 100644 index fca528300caf..000000000000 --- a/plugin/trino-bigquery/bin/schema/nation.json +++ /dev/null @@ -1,22 +0,0 @@ -[ - { - "name": "nationkey", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "name", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "regionkey", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "comment", - "type": "STRING", - "mode": "REQUIRED" - } -] diff --git a/plugin/trino-bigquery/bin/schema/orders.json b/plugin/trino-bigquery/bin/schema/orders.json deleted file mode 100644 index c39320bb27ef..000000000000 --- a/plugin/trino-bigquery/bin/schema/orders.json +++ /dev/null @@ -1,47 +0,0 @@ -[ - { - "name": "orderkey", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "custkey", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "orderstatus", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "totalprice", - "type": "FLOAT", - "mode": "REQUIRED" - }, - { - "name": "orderdate", - "type": "DATE", - "mode": "REQUIRED" - }, - { - "name": "orderpriority", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "clerk", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "shippriority", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "comment", - "type": "STRING", - "mode": "REQUIRED" - } -] diff --git a/plugin/trino-bigquery/bin/schema/part.json b/plugin/trino-bigquery/bin/schema/part.json deleted file mode 100644 index 839acea275d0..000000000000 --- a/plugin/trino-bigquery/bin/schema/part.json +++ /dev/null @@ -1,47 +0,0 @@ -[ - { - "name": "partkey", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "name", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "mfgr", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "brand", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "type", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "size", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "container", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "retailprice", - "type": "FLOAT", - "mode": "REQUIRED" - }, - { - "name": "comment", - "type": "STRING", - "mode": "REQUIRED" - } -] diff --git a/plugin/trino-bigquery/bin/schema/partsupp.json b/plugin/trino-bigquery/bin/schema/partsupp.json deleted file mode 100644 index 3ae99471f7c1..000000000000 --- a/plugin/trino-bigquery/bin/schema/partsupp.json +++ /dev/null @@ -1,27 +0,0 @@ -[ - { - "name": "partkey", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "suppkey", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "availqty", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "supplycost", - "type": "FLOAT", - "mode": "REQUIRED" - }, - { - "name": "comment", - "type": "STRING", - "mode": "REQUIRED" - } -] diff --git a/plugin/trino-bigquery/bin/schema/region.json b/plugin/trino-bigquery/bin/schema/region.json deleted file mode 100644 index 4c0e8e465a9d..000000000000 --- a/plugin/trino-bigquery/bin/schema/region.json +++ /dev/null @@ -1,17 +0,0 @@ -[ - { - "name": "regionkey", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "name", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "comment", - "type": "STRING", - "mode": "REQUIRED" - } -] diff --git a/plugin/trino-bigquery/bin/schema/supplier.json b/plugin/trino-bigquery/bin/schema/supplier.json deleted file mode 100644 index 98a112c1b62a..000000000000 --- a/plugin/trino-bigquery/bin/schema/supplier.json +++ /dev/null @@ -1,37 +0,0 @@ -[ - { - "name": "suppkey", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "name", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "address", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "nationkey", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "phone", - "type": "STRING", - "mode": "REQUIRED" - }, - { - "name": "acctbal", - "type": "FLOAT", - "mode": "REQUIRED" - }, - { - "name": "comment", - "type": "STRING", - "mode": "REQUIRED" - } -] diff --git a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BigQueryQueryRunner.java b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BigQueryQueryRunner.java index b00e02231cd0..e490db5d78cc 100644 --- a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BigQueryQueryRunner.java +++ b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BigQueryQueryRunner.java @@ -31,6 +31,7 @@ import io.trino.plugin.tpch.TpchPlugin; import io.trino.testing.DistributedQueryRunner; import io.trino.testing.sql.SqlExecutor; +import io.trino.tpch.TpchTable; import org.intellij.lang.annotations.Language; import java.io.ByteArrayInputStream; @@ -45,6 +46,8 @@ import static com.google.cloud.bigquery.BigQuery.DatasetDeleteOption.deleteContents; import static com.google.cloud.bigquery.BigQuery.DatasetListOption.labelFilter; import static io.airlift.testing.Closeables.closeAllSuppress; +import static io.trino.plugin.tpch.TpchMetadata.TINY_SCHEMA_NAME; +import static io.trino.testing.QueryAssertions.copyTpchTables; import static io.trino.testing.TestingSession.testSessionBuilder; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -57,7 +60,10 @@ public final class BigQueryQueryRunner private BigQueryQueryRunner() {} - public static DistributedQueryRunner createQueryRunner(Map extraProperties, Map connectorProperties) + public static DistributedQueryRunner createQueryRunner( + Map extraProperties, + Map connectorProperties, + Iterable> tables) throws Exception { DistributedQueryRunner queryRunner = null; @@ -80,6 +86,10 @@ public static DistributedQueryRunner createQueryRunner(Map extra "bigquery", connectorProperties); + queryRunner.execute(createSession(), "CREATE SCHEMA IF NOT EXISTS " + TPCH_SCHEMA); + queryRunner.execute(createSession(), "CREATE SCHEMA IF NOT EXISTS " + TEST_SCHEMA); + copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, createSession(), tables); + return queryRunner; } catch (Throwable e) { @@ -182,7 +192,10 @@ private static BigQuery createBigQueryClient() public static void main(String[] args) throws Exception { - DistributedQueryRunner queryRunner = createQueryRunner(ImmutableMap.of("http-server.http.port", "8080"), ImmutableMap.of()); + DistributedQueryRunner queryRunner = createQueryRunner( + ImmutableMap.of("http-server.http.port", "8080"), + ImmutableMap.of(), + TpchTable.getTables()); Thread.sleep(10); Logger log = Logger.get(BigQueryQueryRunner.class); log.info("======== SERVER STARTED ========"); diff --git a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryCaseInsensitiveMapping.java b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryCaseInsensitiveMapping.java index 03b833f78125..a2f835877113 100644 --- a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryCaseInsensitiveMapping.java +++ b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryCaseInsensitiveMapping.java @@ -13,6 +13,7 @@ */ package io.trino.plugin.bigquery; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import io.trino.plugin.bigquery.BigQueryQueryRunner.BigQuerySqlExecutor; @@ -53,7 +54,8 @@ protected QueryRunner createQueryRunner() { return BigQueryQueryRunner.createQueryRunner( ImmutableMap.of(), - ImmutableMap.of("bigquery.case-insensitive-name-matching", "true")); + ImmutableMap.of("bigquery.case-insensitive-name-matching", "true"), + ImmutableList.of()); } @Test diff --git a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryConnectorTest.java b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryConnectorTest.java index ab5800d0870e..ad776ce79ad4 100644 --- a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryConnectorTest.java +++ b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryConnectorTest.java @@ -60,7 +60,8 @@ protected QueryRunner createQueryRunner() { return BigQueryQueryRunner.createQueryRunner( ImmutableMap.of(), - ImmutableMap.of()); + ImmutableMap.of(), + REQUIRED_TPCH_TABLES); } @SuppressWarnings("DuplicateBranchesInSwitch") diff --git a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryMetadataCaching.java b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryMetadataCaching.java index 4bf645a0397d..1962a8815a17 100644 --- a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryMetadataCaching.java +++ b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryMetadataCaching.java @@ -13,6 +13,7 @@ */ package io.trino.plugin.bigquery; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.QueryRunner; @@ -34,7 +35,8 @@ protected QueryRunner createQueryRunner() this.bigQuerySqlExecutor = new BigQuerySqlExecutor(); return BigQueryQueryRunner.createQueryRunner( ImmutableMap.of(), - ImmutableMap.of("bigquery.metadata.cache-ttl", "5m")); + ImmutableMap.of("bigquery.metadata.cache-ttl", "5m"), + ImmutableList.of()); } @Test diff --git a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryTypeMapping.java b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryTypeMapping.java index 79c27c92bbb1..460787eb2b9a 100644 --- a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryTypeMapping.java +++ b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryTypeMapping.java @@ -68,7 +68,8 @@ protected QueryRunner createQueryRunner() { return BigQueryQueryRunner.createQueryRunner( ImmutableMap.of(), - ImmutableMap.of("bigquery.skip-view-materialization", "true")); + ImmutableMap.of("bigquery.skip-view-materialization", "true"), + ImmutableList.of()); } @Test