From dfe2da2d06232addf6d781ffc97a43d92e1a5926 Mon Sep 17 00:00:00 2001 From: stephwang Date: Wed, 23 Jun 2021 14:48:08 -0400 Subject: [PATCH 1/4] feat: add support for jobs.delete --- .../clirr-ignored-differences.xml | 12 ++++---- .../com/google/cloud/bigquery/BigQuery.java | 8 +++++ .../google/cloud/bigquery/BigQueryImpl.java | 29 +++++++++++++++++++ .../cloud/bigquery/spi/v2/BigQueryRpc.java | 8 +++++ .../bigquery/spi/v2/HttpBigQueryRpc.java | 15 ++++++++++ .../cloud/bigquery/it/ITBigQueryTest.java | 13 +++++++++ 6 files changed, 79 insertions(+), 6 deletions(-) diff --git a/google-cloud-bigquery/clirr-ignored-differences.xml b/google-cloud-bigquery/clirr-ignored-differences.xml index 2801782f4..4d66f0620 100644 --- a/google-cloud-bigquery/clirr-ignored-differences.xml +++ b/google-cloud-bigquery/clirr-ignored-differences.xml @@ -3,13 +3,13 @@ - 7013 - com/google/cloud/bigquery/ExternalTableDefinition - com.google.common.collect.ImmutableList getDecimalTargetTypes() + 7012 + com/google/cloud/bigquery/BigQuery + boolean delete(com.google.cloud.bigquery.JobId) - 7013 - com/google/cloud/bigquery/ExternalTableDefinition$Builder - com.google.cloud.bigquery.ExternalTableDefinition$Builder setDecimalTargetTypes(java.util.List) + 7012 + com/google/cloud/bigquery/spi/v2/BigQueryRpc + boolean deleteJob(java.lang.String, java.lang.String, java.lang.String) \ No newline at end of file diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java index 70a29fd9f..4e88f000f 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQuery.java @@ -982,6 +982,14 @@ public int hashCode() { */ boolean delete(RoutineId routineId); + /** + * Deletes the requested job. + * + * @return {@code true} if job was deleted, {@code false} if it was not found + * @throws BigQueryException upon failure + */ + boolean delete(JobId jobId); + /** * Updates dataset information. * diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java index 12eb09c5a..735be9ad3 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java @@ -585,6 +585,35 @@ public Boolean call() { } } + @Override + public boolean delete(JobId jobId) { + final JobId completeJobId = + JobId.of() + .setLocation( + Strings.isNullOrEmpty(jobId.getLocation()) + ? getOptions().getLocation() + : jobId.getLocation()) + .setProjectId( + Strings.isNullOrEmpty(jobId.getProject()) + ? getOptions().getProjectId() + : jobId.getProject()); + try { + return runWithRetries( + new Callable() { + @Override + public Boolean call() { + return bigQueryRpc.deleteJob( + completeJobId.getProject(), completeJobId.getJob(), completeJobId.getLocation()); + } + }, + getOptions().getRetrySettings(), + EXCEPTION_HANDLER, + getOptions().getClock()); + } catch (RetryHelper.RetryHelperException e) { + throw BigQueryException.translateAndThrow(e); + } + } + @Override public Dataset update(DatasetInfo datasetInfo, DatasetOption... options) { final com.google.api.services.bigquery.model.Dataset datasetPb = diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java index c9b0f0a21..96422b8be 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java @@ -270,6 +270,14 @@ TableDataList listTableData( */ boolean cancel(String projectId, String jobId, String location); + /** + * Sends a job delete request. + * + * @return {@code true} if delete was successful, {@code false} if the job was not found + * @throws BigQueryException upon failure + */ + boolean deleteJob(String projectId, String jobId, String location); + /** * Returns results of the query associated with the provided job. * diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index 4d9558454..01fb9a898 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -607,6 +607,21 @@ public boolean cancel(String projectId, String jobId, String location) { } } + @Override + public boolean deleteJob(String projectId, String jobId, String location) { + try { + bigquery + .jobs() + .delete(projectId, jobId) + .setLocation(location) + .setPrettyPrint(false) + .execute(); + return true; + } catch (IOException ex) { + throw translate(ex); + } + } + @Override public GetQueryResultsResponse getQueryResults( String projectId, String jobId, String location, Map options) { diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index b151ad22c..2cad6ff3a 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -1243,6 +1243,19 @@ public void testDeleteNonExistingTable() { assertFalse(bigquery.delete("test_delete_non_existing_table")); } + @Test + public void testDeleteJob() { + String query = "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID.getTable(); + QueryJobConfiguration config = + QueryJobConfiguration.newBuilder(query).setDefaultDataset(DatasetId.of(DATASET)).build(); + String jobName = "jobId_" + UUID.randomUUID().toString(); + JobId jobId = JobId.newBuilder().setLocation("us-east1").setJob(jobName).build(); + bigquery.create(JobInfo.of(jobId, config)); + assertNotNull(bigquery.getJob(jobId)); + assertTrue(bigquery.delete(jobId)); + assertNull(bigquery.getJob(jobId)); + } + @Test public void testInsertAll() throws IOException { String tableName = "test_insert_all_table"; From ad92340b708488177e3b4d99a8c733d33f3ee1ad Mon Sep 17 00:00:00 2001 From: stephwang Date: Wed, 23 Jun 2021 15:49:50 -0400 Subject: [PATCH 2/4] update IT --- .../java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 2cad6ff3a..84bc93dd8 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -1245,9 +1245,8 @@ public void testDeleteNonExistingTable() { @Test public void testDeleteJob() { - String query = "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID.getTable(); - QueryJobConfiguration config = - QueryJobConfiguration.newBuilder(query).setDefaultDataset(DatasetId.of(DATASET)).build(); + String query = "SELECT 17 as foo"; + QueryJobConfiguration config = QueryJobConfiguration.of(query); String jobName = "jobId_" + UUID.randomUUID().toString(); JobId jobId = JobId.newBuilder().setLocation("us-east1").setJob(jobName).build(); bigquery.create(JobInfo.of(jobId, config)); @@ -1255,8 +1254,7 @@ public void testDeleteJob() { assertTrue(bigquery.delete(jobId)); assertNull(bigquery.getJob(jobId)); } - - @Test + @Test public void testInsertAll() throws IOException { String tableName = "test_insert_all_table"; StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA); From c35bd062028b4c1e6e1d13c5b1c0d0fbbb3d6e18 Mon Sep 17 00:00:00 2001 From: stephwang Date: Wed, 23 Jun 2021 15:51:45 -0400 Subject: [PATCH 3/4] lint --- .../test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 84bc93dd8..47d8aa41b 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -1254,7 +1254,8 @@ public void testDeleteJob() { assertTrue(bigquery.delete(jobId)); assertNull(bigquery.getJob(jobId)); } - @Test + + @Test public void testInsertAll() throws IOException { String tableName = "test_insert_all_table"; StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA); From d21358e5eb1306f9b180a3f3c650ad413d7714d6 Mon Sep 17 00:00:00 2001 From: stephwang Date: Thu, 24 Jun 2021 12:35:18 -0400 Subject: [PATCH 4/4] updates --- .../com/google/cloud/bigquery/BigQueryImpl.java | 13 ++++--------- .../google/cloud/bigquery/spi/v2/BigQueryRpc.java | 2 +- .../cloud/bigquery/spi/v2/HttpBigQueryRpc.java | 4 ++-- .../google/cloud/bigquery/it/ITBigQueryTest.java | 8 +++++--- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java index 735be9ad3..4086c6a73 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java @@ -588,15 +588,10 @@ public Boolean call() { @Override public boolean delete(JobId jobId) { final JobId completeJobId = - JobId.of() - .setLocation( - Strings.isNullOrEmpty(jobId.getLocation()) - ? getOptions().getLocation() - : jobId.getLocation()) - .setProjectId( - Strings.isNullOrEmpty(jobId.getProject()) - ? getOptions().getProjectId() - : jobId.getProject()); + jobId.setProjectId( + Strings.isNullOrEmpty(jobId.getProject()) + ? getOptions().getProjectId() + : jobId.getProject()); try { return runWithRetries( new Callable() { diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java index 96422b8be..06488c5b4 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/BigQueryRpc.java @@ -276,7 +276,7 @@ TableDataList listTableData( * @return {@code true} if delete was successful, {@code false} if the job was not found * @throws BigQueryException upon failure */ - boolean deleteJob(String projectId, String jobId, String location); + boolean deleteJob(String projectId, String jobName, String location); /** * Returns results of the query associated with the provided job. diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index 01fb9a898..24d7dd6b0 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -608,11 +608,11 @@ public boolean cancel(String projectId, String jobId, String location) { } @Override - public boolean deleteJob(String projectId, String jobId, String location) { + public boolean deleteJob(String projectId, String jobName, String location) { try { bigquery .jobs() - .delete(projectId, jobId) + .delete(projectId, jobName) .setLocation(location) .setPrettyPrint(false) .execute(); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 47d8aa41b..e072cf0f7 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -1248,9 +1248,11 @@ public void testDeleteJob() { String query = "SELECT 17 as foo"; QueryJobConfiguration config = QueryJobConfiguration.of(query); String jobName = "jobId_" + UUID.randomUUID().toString(); - JobId jobId = JobId.newBuilder().setLocation("us-east1").setJob(jobName).build(); - bigquery.create(JobInfo.of(jobId, config)); - assertNotNull(bigquery.getJob(jobId)); + JobId jobId = + JobId.newBuilder().setLocation("us-east1").setJob(jobName).setProject(PROJECT_ID).build(); + Job createdJob = bigquery.create(JobInfo.of(jobId, config)); + Job remoteJob = bigquery.getJob(createdJob.getJobId()); + assertEquals(createdJob.getJobId(), remoteJob.getJobId()); assertTrue(bigquery.delete(jobId)); assertNull(bigquery.getJob(jobId)); }