Skip to content

feat(bigquery): Add custom ExceptionHandler to BigQueryOptions #3937

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Aug 29, 2025
Merged

feat(bigquery): Add custom ExceptionHandler to BigQueryOptions #3937

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
84a10b7
feat(bigquery): Add custom ExceptionHandler to BigQueryOptions
whuffman36 Aug 13, 2025
669e643
add default project id to tests
whuffman36 Aug 13, 2025
a57704d
Add dummy project id to tests
whuffman36 Aug 13, 2025
1353913
Remove defaultInstance() from test
whuffman36 Aug 13, 2025
2b0152f
add option to specify retry algo instead of just abortOn/retryOn
whuffman36 Aug 26, 2025
12f463b
fix formatting
whuffman36 Aug 26, 2025
2918cb4
remove custom exception handler from bq options
whuffman36 Aug 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryBaseService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ protected BigQueryBaseService(ServiceOptions options) {
super(options);
}

public static final ExceptionHandler BIGQUERY_EXCEPTION_HANDLER =
public static final ExceptionHandler DEFAULT_BIGQUERY_EXCEPTION_HANDLER =
ExceptionHandler.newBuilder()
.abortOn(RuntimeException.class)
.retryOn(java.net.ConnectException.class) // retry on Connection Exception
Expand Down
62 changes: 31 additions & 31 deletions google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ public com.google.api.services.bigquery.model.Dataset call() throws IOException
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -340,7 +340,7 @@ public com.google.api.services.bigquery.model.Table call() throws IOException {
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -396,7 +396,7 @@ public com.google.api.services.bigquery.model.Routine call() throws IOException
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -490,7 +490,7 @@ public com.google.api.services.bigquery.model.Job call() throws IOException {
? RetryOption.mergeToSettings(
getOptions().getRetrySettings(), getRetryOptions(optionsMap))
: getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
getBigQueryRetryConfig(optionsMap) != null
? getBigQueryRetryConfig(optionsMap)
Expand Down Expand Up @@ -586,7 +586,7 @@ public com.google.api.services.bigquery.model.Dataset call() throws IOException
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -652,7 +652,7 @@ private static Page<Dataset> listDatasets(
}
},
serviceOptions.getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
serviceOptions.getExceptionHandler(),
serviceOptions.getClock(),
EMPTY_RETRY_CONFIG,
serviceOptions.isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -704,7 +704,7 @@ public Boolean call() throws IOException {
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -755,7 +755,7 @@ public Boolean call() throws IOException {
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -801,7 +801,7 @@ public Boolean call() throws IOException {
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -847,7 +847,7 @@ public Boolean call() throws IOException {
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -891,7 +891,7 @@ public Boolean call() throws IOException {
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -932,7 +932,7 @@ public com.google.api.services.bigquery.model.Dataset call() throws IOException
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -979,7 +979,7 @@ public com.google.api.services.bigquery.model.Table call() throws IOException {
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -1025,7 +1025,7 @@ public com.google.api.services.bigquery.model.Model call() throws IOException {
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -1071,7 +1071,7 @@ public com.google.api.services.bigquery.model.Routine call() throws IOException
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -1125,7 +1125,7 @@ public com.google.api.services.bigquery.model.Table call() throws IOException {
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -1184,7 +1184,7 @@ public com.google.api.services.bigquery.model.Model call() throws IOException {
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -1243,7 +1243,7 @@ public com.google.api.services.bigquery.model.Routine call() throws IOException
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -1461,7 +1461,7 @@ public Tuple<String, Iterable<com.google.api.services.bigquery.model.Table>> cal
}
},
serviceOptions.getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
serviceOptions.getExceptionHandler(),
serviceOptions.getClock(),
EMPTY_RETRY_CONFIG,
serviceOptions.isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -1502,7 +1502,7 @@ public Tuple<String, Iterable<com.google.api.services.bigquery.model.Model>> cal
}
},
serviceOptions.getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
serviceOptions.getExceptionHandler(),
serviceOptions.getClock(),
EMPTY_RETRY_CONFIG,
serviceOptions.isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -1543,7 +1543,7 @@ private static Page<Routine> listRoutines(
}
},
serviceOptions.getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
serviceOptions.getExceptionHandler(),
serviceOptions.getClock(),
EMPTY_RETRY_CONFIG,
serviceOptions.isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -1625,7 +1625,7 @@ public TableDataInsertAllResponse call() throws Exception {
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -1719,7 +1719,7 @@ public TableDataList call() throws IOException {
}
},
serviceOptions.getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
serviceOptions.getExceptionHandler(),
serviceOptions.getClock(),
EMPTY_RETRY_CONFIG,
serviceOptions.isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -1793,7 +1793,7 @@ public com.google.api.services.bigquery.model.Job call() throws IOException {
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -1850,7 +1850,7 @@ public Tuple<String, Iterable<com.google.api.services.bigquery.model.Job>> call(
}
},
serviceOptions.getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
serviceOptions.getExceptionHandler(),
serviceOptions.getClock(),
EMPTY_RETRY_CONFIG,
serviceOptions.isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -1905,7 +1905,7 @@ public Boolean call() throws IOException {
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -1992,7 +1992,7 @@ public com.google.api.services.bigquery.model.QueryResponse call()
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
DEFAULT_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -2169,7 +2169,7 @@ public GetQueryResultsResponse call() throws IOException {
}
},
serviceOptions.getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
serviceOptions.getExceptionHandler(),
serviceOptions.getClock(),
DEFAULT_RETRY_CONFIG,
serviceOptions.isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -2240,7 +2240,7 @@ public com.google.api.services.bigquery.model.Policy call() throws IOException {
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -2286,7 +2286,7 @@ public com.google.api.services.bigquery.model.Policy call() throws IOException {
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down Expand Up @@ -2334,7 +2334,7 @@ public com.google.api.services.bigquery.model.TestIamPermissionsResponse call()
}
},
getOptions().getRetrySettings(),
BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER,
getOptions().getExceptionHandler(),
getOptions().getClock(),
EMPTY_RETRY_CONFIG,
getOptions().isOpenTelemetryTracingEnabled(),
Expand Down
37 changes: 37 additions & 0 deletions google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryOptions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
package com.google.cloud.bigquery;

import com.google.api.core.BetaApi;
import com.google.cloud.BaseService;
import com.google.cloud.ExceptionHandler;
import com.google.cloud.ServiceDefaults;
import com.google.cloud.ServiceOptions;
import com.google.cloud.ServiceRpc;
Expand All @@ -43,6 +45,7 @@ public class BigQueryOptions extends ServiceOptions<BigQuery, BigQueryOptions> {
private JobCreationMode defaultJobCreationMode = JobCreationMode.JOB_CREATION_MODE_UNSPECIFIED;
private boolean enableOpenTelemetryTracing;
private Tracer openTelemetryTracer;
private ExceptionHandler exceptionHandler;

public static class DefaultBigQueryFactory implements BigQueryFactory {

Expand Down Expand Up @@ -70,11 +73,14 @@ public static class Builder extends ServiceOptions.Builder<BigQuery, BigQueryOpt
private boolean useInt64Timestamps;
private boolean enableOpenTelemetryTracing;
private Tracer openTelemetryTracer;
private boolean customExceptionHandler;
Copy link
Contributor

@PhongChuong PhongChuong Aug 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

customExceptionHandler is no longer needed.

Copy link
Contributor Author

@whuffman36 whuffman36 Aug 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oops, good catch thanks! Removed customExceptionHandler from bq options.

private ExceptionHandler.Builder exceptionHandlerBuilder;

private Builder() {}

private Builder(BigQueryOptions options) {
super(options);
customExceptionHandler = false;
}

@Override
Expand Down Expand Up @@ -118,6 +124,24 @@ public Builder setOpenTelemetryTracer(Tracer tracer) {
return this;
}

public Builder abortOn(Class<? extends Exception> exception) {
Copy link
Contributor

@PhongChuong PhongChuong Aug 18, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Providing the helper methods abortOn and retryOn in BigQueryOptions is probably easier to use for the user at the cost of flexibility. The sample code to use current feature would look something like:

    BigQuery bigQuery =
        BigQueryOptions.newBuilder()
            .abortOn(RuntimeException.class)
            .retryOn(java.net.UnknownHostException.class)
            ...
            .build()
            .getService();

Consider allowing users to directly set the ResultRetryAlgorithm in BigQueryOption so it might look like this:

    BigQuery bigQuery =
        BigQueryOptions.newBuilder()
            .setResultRetryAlgorithm(
                ExceptionHandler.newBuilder().
                    .abortOn(RuntimeException.class)
                    .retryOn(java.net.UnknownHostException.class)
                    .addInterceptors(EXCEPTION_HANDLER_INTERCEPTOR))
            ...
            .build()
            .getService();

We can also consider adding a BigQueryExceptionHandler class to help the user not have to set addInterceptors() by providing a BigQueryExceptionHandler so the final code might look like this:

    BigQuery bigQuery =
        BigQueryOptions.newBuilder()
            .setResultRetryAlgorithm(
                BigQueryExceptionHandler.newBuilder().
                    .abortOn(RuntimeException.class)
                    .retryOn(java.net.UnknownHostException.class)
                    .build())
            ...
            .build()
            .getService();

Let me know what you think,

Copy link
Contributor Author

@whuffman36 whuffman36 Aug 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I still prefer to just set the retry/abort exceptions directly within the BigQueryOptions builder. Unless the user defines their own ResultRetryAlgorithm object, they are gaining no new flexibility or customization compared to setting the abort/retry settings directly in the options as there are no other builder options to set in ExceptionHandler. Having them create an ExceptionHandler object in the builder accomplishes the same thing but with more verbosity (though you could argue this makes the code more readable).

If we do decide to make them specify the algorithm instead of just the .abortOn()/.retryOn() helpers, I think the BigQueryExceptionHandler is the better option since the Interceptor interface is not very clear. This means writing and managing a new class that has essentially the same functionality as ExceptionHandler without providing any extra functionality or ease of use compared to the .abortOn()/.retryOn() helpers in BigQueryOptions.

This change already enables more flexibility to the user than they had by choosing which exceptions to abort and retry on. I suppose I don't expect users to want (or need) to define their own ResultRetryAlgorithm objects since it comes from gax and feels like an internal implementation detail to me. But I could be very wrong in this regard, do you foresee users wanting to specify their own ResultRetryAlgorithm objects?

if (!customExceptionHandler) {
exceptionHandlerBuilder = ExceptionHandler.newBuilder();
customExceptionHandler = true;
}
this.exceptionHandlerBuilder.abortOn(exception);
return this;
}

public Builder retryOn(Class<? extends Exception> exception) {
if (!customExceptionHandler) {
exceptionHandlerBuilder = ExceptionHandler.newBuilder();
customExceptionHandler = true;
}
this.exceptionHandlerBuilder.retryOn(exception);
return this;
}

@Override
public BigQueryOptions build() {
return new BigQueryOptions(this);
Expand All @@ -130,6 +154,15 @@ private BigQueryOptions(Builder builder) {
this.useInt64Timestamps = builder.useInt64Timestamps;
this.enableOpenTelemetryTracing = builder.enableOpenTelemetryTracing;
this.openTelemetryTracer = builder.openTelemetryTracer;
if (builder.customExceptionHandler) {
this.exceptionHandler =
builder
.exceptionHandlerBuilder
.addInterceptors(BaseService.EXCEPTION_HANDLER_INTERCEPTOR)
.build();
} else {
this.exceptionHandler = BigQueryBaseService.DEFAULT_BIGQUERY_EXCEPTION_HANDLER;
}
}

private static class BigQueryDefaults implements ServiceDefaults<BigQuery, BigQueryOptions> {
Expand Down Expand Up @@ -221,6 +254,10 @@ public Tracer getOpenTelemetryTracer() {
return openTelemetryTracer;
}

public ExceptionHandler getExceptionHandler() {
return exceptionHandler;
}

@SuppressWarnings("unchecked")
@Override
public Builder toBuilder() {
Expand Down
Loading
Loading