Skip to content

Support reading wildcard tables in BigQuery #12428

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ebyhr merged 1 commit into from
May 23, 2022
Merged

Support reading wildcard tables in BigQuery #12428

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryErrorCode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import io.trino.spi.ErrorType;

import static io.trino.spi.ErrorType.EXTERNAL;
import static io.trino.spi.ErrorType.USER_ERROR;

public enum BigQueryErrorCode
implements ErrorCodeSupplier
Expand All @@ -27,6 +28,7 @@ public enum BigQueryErrorCode
BIGQUERY_FAILED_TO_EXECUTE_QUERY(2, EXTERNAL),
BIGQUERY_AMBIGUOUS_OBJECT_NAME(3, EXTERNAL),
BIGQUERY_LISTING_DATASET_ERROR(4, EXTERNAL),
BIGQUERY_UNSUPPORTED_OPERATION(5, USER_ERROR),
/**/;

private final ErrorCode errorCode;
Expand Down
5 changes: 5 additions & 0 deletions plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,10 +73,12 @@
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static io.trino.plugin.bigquery.BigQueryErrorCode.BIGQUERY_LISTING_DATASET_ERROR;
import static io.trino.plugin.bigquery.BigQueryErrorCode.BIGQUERY_UNSUPPORTED_OPERATION;
import static io.trino.plugin.bigquery.BigQueryPseudoColumn.PARTITION_DATE;
import static io.trino.plugin.bigquery.BigQueryPseudoColumn.PARTITION_TIME;
import static io.trino.plugin.bigquery.BigQueryTableHandle.BigQueryPartitionType.INGESTION;
import static io.trino.plugin.bigquery.BigQueryType.toField;
import static io.trino.plugin.bigquery.BigQueryUtil.isWildcardTable;
import static java.util.Locale.ENGLISH;
import static java.util.Objects.requireNonNull;
import static java.util.function.Function.identity;
Expand Down Expand Up @@ -406,6 +408,9 @@ public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle
{
BigQueryClient client = bigQueryClientFactory.create(session);
BigQueryTableHandle bigQueryTable = (BigQueryTableHandle) tableHandle;
if (isWildcardTable(TableDefinition.Type.valueOf(bigQueryTable.getType()), bigQueryTable.getRemoteTableName().getTableName())) {
throw new TrinoException(BIGQUERY_UNSUPPORTED_OPERATION, "This connector does not support dropping wildcard tables");
}
TableId tableId = bigQueryTable.getRemoteTableName().toTableId();
client.dropTable(tableId);
}
Expand Down
5 changes: 5 additions & 0 deletions plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQuerySplitManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import static io.trino.plugin.bigquery.BigQuerySessionProperties.createDisposition;
import static io.trino.plugin.bigquery.BigQuerySessionProperties.isQueryResultsCacheEnabled;
import static io.trino.plugin.bigquery.BigQuerySessionProperties.isSkipViewMaterialization;
import static io.trino.plugin.bigquery.BigQueryUtil.isWildcardTable;
import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toList;
Expand Down Expand Up @@ -117,6 +118,10 @@ private List<BigQuerySplit> readFromBigQuery(ConnectorSession session, TableDefi
.map(column -> ((BigQueryColumnHandle) column).getName())
.collect(toImmutableList());

if (isWildcardTable(type, remoteTableId.getTable())) {
// Storage API doesn't support reading wildcard tables
return ImmutableList.of(BigQuerySplit.forViewStream(columns, filter));
}
if (type == MATERIALIZED_VIEW) {
// Storage API doesn't support reading materialized views
return ImmutableList.of(BigQuerySplit.forViewStream(columns, filter));
Expand Down
7 changes: 7 additions & 0 deletions plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

import com.google.cloud.bigquery.BigQueryError;
import com.google.cloud.bigquery.BigQueryException;
import com.google.cloud.bigquery.TableDefinition;
import com.google.common.collect.ImmutableSet;
import io.grpc.Status;
import io.grpc.StatusRuntimeException;
Expand All @@ -23,6 +24,7 @@
import java.util.Optional;
import java.util.Set;

import static com.google.cloud.bigquery.TableDefinition.Type.TABLE;
import static com.google.cloud.http.BaseHttpServiceException.UNKNOWN_CODE;
import static com.google.common.base.Throwables.getCausalChain;

Expand Down Expand Up @@ -66,4 +68,9 @@ public static String toBigQueryColumnName(String columnName)
}
return columnName;
}

public static boolean isWildcardTable(TableDefinition.Type type, String tableName)
{
return type == TABLE && tableName.contains("*");
Copy link
Member

@findepi findepi May 17, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Before the change, could a user create a table with asterisk in its name?

Copy link
Member Author

@ebyhr ebyhr May 17, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No. Table name must be alphanumeric and underscores.

}
}
55 changes: 55 additions & 0 deletions plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryConnectorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -651,6 +651,61 @@ public void testQueryCache()
}
}

@Test
public void testWildcardTable()
{
String suffix = randomTableSuffix();
String firstTable = format("test_wildcard_%s_1", suffix);
String secondTable = format("test_wildcard_%s_2", suffix);
String wildcardTable = format("test_wildcard_%s_*", suffix);
try {
onBigQuery("CREATE TABLE test." + firstTable + " AS SELECT 1 AS value");
onBigQuery("CREATE TABLE test." + secondTable + " AS SELECT 2 AS value");

assertQuery("DESCRIBE test.\"" + wildcardTable + "\"", "VALUES ('value', 'bigint', '', '')");
assertQuery("SELECT * FROM test.\"" + wildcardTable + "\"", "VALUES (1), (2)");

// Unsupported operations
assertQueryFails("DROP TABLE test.\"" + wildcardTable + "\"", "This connector does not support dropping wildcard tables");
assertQueryFails("INSERT INTO test.\"" + wildcardTable + "\" VALUES (1)", "This connector does not support inserts");
assertQueryFails("ALTER TABLE test.\"" + wildcardTable + "\" ADD COLUMN new_column INT", "This connector does not support adding columns");
assertQueryFails("ALTER TABLE test.\"" + wildcardTable + "\" RENAME TO test.new_wildcard_table", "This connector does not support renaming tables");
}
finally {
onBigQuery("DROP TABLE IF EXISTS test." + firstTable);
onBigQuery("DROP TABLE IF EXISTS test." + secondTable);
}
}

@Test
public void testWildcardTableWithDifferentColumnDefinition()
{
String suffix = randomTableSuffix();
String firstTable = format("test_invalid_wildcard_%s_1", suffix);
String secondTable = format("test_invalid_wildcard_%s_2", suffix);
String wildcardTable = format("test_invalid_wildcard_%s_*", suffix);
try {
onBigQuery("CREATE TABLE test." + firstTable + " AS SELECT 1 AS value");
onBigQuery("CREATE TABLE test." + secondTable + " AS SELECT 'string' AS value");

assertQuery("DESCRIBE test.\"" + wildcardTable + "\"", "VALUES ('value', 'varchar', '', '')");

assertThatThrownBy(() -> query("SELECT * FROM test.\"" + wildcardTable + "\""))
.hasMessageContaining("Cannot read field of type INT64 as STRING Field: value");
}
finally {
onBigQuery("DROP TABLE IF EXISTS test." + firstTable);
onBigQuery("DROP TABLE IF EXISTS test." + secondTable);
}
}

@Test
public void testMissingWildcardTable()
{
assertThatThrownBy(() -> query("SELECT * FROM test.\"test_missing_wildcard_table_*\""))
.hasMessageEndingWith("does not match any table.");
}

private void onBigQuery(@Language("SQL") String sql)
{
bigQuerySqlExecutor.execute(sql);
Expand Down