Skip to content

Commit fb477a3

Browse files
PhongChuongPhongChuong
committed
Add max staleness to StandardTableDefinition and MaterializedViewDefinition
1 parent 19b7c3a commit fb477a3

File tree

5 files changed

+100
-3
lines changed

5 files changed

+100
-3
lines changed

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/MaterializedViewDefinition.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,12 @@ public abstract static class Builder
3535
*/
3636
abstract Builder setLastRefreshTime(Long lastRefreshTime);
3737

38+
/**
39+
* Sets the max staleness of data that could be returned when materizlized view is queried
40+
* (formatted as Google SQL Interval type).
41+
*/
42+
public abstract Builder setMaxStaleness(String query);
43+
3844
/** Sets the query whose result is persisted. */
3945
public abstract Builder setQuery(String query);
4046

@@ -87,6 +93,10 @@ public abstract static class Builder
8793
@Nullable
8894
public abstract Long getLastRefreshTime();
8995

96+
/** Returns max stalness of this materialized view. */
97+
@Nullable
98+
public abstract String getMaxStaleness();
99+
90100
/** Returns a query whose result is persisted. */
91101
@Nullable
92102
public abstract String getQuery();
@@ -146,6 +156,9 @@ Table toPb() {
146156
if (getRefreshIntervalMs() != null) {
147157
materializedViewDefinition.setRefreshIntervalMs(getRefreshIntervalMs());
148158
}
159+
if (getMaxStaleness() != null) {
160+
materializedViewDefinition.setMaxStaleness(getMaxStaleness());
161+
}
149162
tablePb.setMaterializedView(materializedViewDefinition);
150163
if (getTimePartitioning() != null) {
151164
tablePb.setTimePartitioning(getTimePartitioning().toPb());
@@ -198,6 +211,10 @@ static MaterializedViewDefinition fromPb(Table tablePb) {
198211
if (materializedViewDefinition.getRefreshIntervalMs() != null) {
199212
builder.setRefreshIntervalMs(materializedViewDefinition.getRefreshIntervalMs());
200213
}
214+
builder.setMaxStaleness(materializedViewDefinition.getMaxStaleness());
215+
// if (materializedViewDefinition.getMaxStaleness() != null) {
216+
// builder.setMaxStaleness(materializedViewDefinition.getMaxStaleness());
217+
// }
201218
if (tablePb.getTimePartitioning() != null) {
202219
builder.setTimePartitioning(TimePartitioning.fromPb(tablePb.getTimePartitioning()));
203220
}

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/StandardTableDefinition.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,8 @@ public abstract static class Builder
150150

151151
public abstract Builder setLocation(String location);
152152

153+
public abstract Builder setMaxStaleness(String maxStaleness);
154+
153155
public abstract Builder setStreamingBuffer(StreamingBuffer streamingBuffer);
154156

155157
public abstract Builder setType(Type type);
@@ -271,6 +273,10 @@ public abstract static class Builder
271273
@Nullable
272274
public abstract String getLocation();
273275

276+
/** Returns the max staleness of the table. */
277+
@Nullable
278+
public abstract String getMaxStaleness();
279+
274280
/**
275281
* Returns information on the table's streaming buffer if any exists. Returns {@code null} if no
276282
* streaming buffer exists.
@@ -346,6 +352,7 @@ Table toPb() {
346352
tablePb.setNumActivePhysicalBytes(getNumActivePhysicalBytes());
347353
tablePb.setNumLongTermPhysicalBytes(getNumLongTermPhysicalBytes());
348354
tablePb.setLocation(getLocation());
355+
tablePb.setMaxStaleness(getMaxStaleness());
349356
if (getStreamingBuffer() != null) {
350357
tablePb.setStreamingBuffer(getStreamingBuffer().toPb());
351358
}
@@ -430,6 +437,10 @@ static StandardTableDefinition fromPb(Table tablePb) {
430437
BigLakeConfiguration.fromPb(tablePb.getBiglakeConfiguration()));
431438
}
432439

433-
return builder.setNumBytes(tablePb.getNumBytes()).setLocation(tablePb.getLocation()).build();
440+
builder.setNumBytes(tablePb.getNumBytes());
441+
builder.setLocation(tablePb.getLocation());
442+
builder.setMaxStaleness(tablePb.getMaxStaleness());
443+
444+
return builder.build();
434445
}
435446
}

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/MaterializedViewDefinitionTest.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ public class MaterializedViewDefinitionTest {
2828
private static final Long LAST_REFRESH_TIME = 1580302008L;
2929
private static final Boolean ENABLE_REFRESH = false;
3030
private static final Long REFRESH_INTERVAL_MS = 60000L;
31+
32+
private static final String MAX_STALENESS = "INTERVAL 1 MINUTE";
33+
3134
private static final Schema SCHEMA = Schema.of();
3235
private static final TimePartitioning TIME_PARTITIONING =
3336
TimePartitioning.of(TimePartitioning.Type.DAY, 42);
@@ -40,6 +43,7 @@ public class MaterializedViewDefinitionTest {
4043
.setLastRefreshTime(LAST_REFRESH_TIME)
4144
.setEnableRefresh(ENABLE_REFRESH)
4245
.setRefreshIntervalMs(REFRESH_INTERVAL_MS)
46+
.setMaxStaleness(MAX_STALENESS)
4347
.setClustering(CLUSTERING)
4448
.setTimePartitioning(TIME_PARTITIONING)
4549
.build();
@@ -75,6 +79,7 @@ public void testBuilder() {
7579
.setLastRefreshTime(LAST_REFRESH_TIME)
7680
.setEnableRefresh(ENABLE_REFRESH)
7781
.setRefreshIntervalMs(REFRESH_INTERVAL_MS)
82+
.setMaxStaleness(MAX_STALENESS)
7883
.setClustering(CLUSTERING)
7984
.setTimePartitioning(TIME_PARTITIONING)
8085
.build();
@@ -101,6 +106,7 @@ private void compareMaterializedView(
101106
assertEquals(expected.getLastRefreshTime(), actual.getLastRefreshTime());
102107
assertEquals(expected.getEnableRefresh(), actual.getEnableRefresh());
103108
assertEquals(expected.getRefreshIntervalMs(), actual.getRefreshIntervalMs());
109+
assertEquals(expected.getMaxStaleness(), actual.getMaxStaleness());
104110
assertEquals(expected.getClustering(), actual.getClustering());
105111
assertEquals(expected.getTimePartitioning(), actual.getTimePartitioning());
106112
assertEquals(expected.toString(), actual.toString());

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/StandardTableDefinitionTest.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ public class StandardTableDefinitionTest {
6060
private static final Long NUM_LONG_TERM_PHYSICAL_BYTES = 27L;
6161
private static final Long NUM_ROWS = 43L;
6262
private static final String LOCATION = "US";
63+
64+
private static final String MAX_STALENESS = "INTERVAL 1 MINUTE";
65+
6366
private static final StreamingBuffer STREAMING_BUFFER = new StreamingBuffer(1L, 2L, 3L);
6467
private static final TimePartitioning TIME_PARTITIONING =
6568
TimePartitioning.of(TimePartitioning.Type.DAY, 42);
@@ -75,6 +78,7 @@ public class StandardTableDefinitionTest {
7578
private static final StandardTableDefinition TABLE_DEFINITION =
7679
StandardTableDefinition.newBuilder()
7780
.setLocation(LOCATION)
81+
.setMaxStaleness(MAX_STALENESS)
7882
.setNumBytes(NUM_BYTES)
7983
.setNumRows(NUM_ROWS)
8084
.setNumLongTermBytes(NUM_LONG_TERM_BYTES)
@@ -113,6 +117,7 @@ public void testBuilder() {
113117
assertEquals(TableDefinition.Type.TABLE, TABLE_DEFINITION.getType());
114118
assertEquals(TABLE_SCHEMA, TABLE_DEFINITION.getSchema());
115119
assertEquals(LOCATION, TABLE_DEFINITION.getLocation());
120+
assertEquals(MAX_STALENESS, TABLE_DEFINITION.getMaxStaleness());
116121
assertEquals(NUM_BYTES, TABLE_DEFINITION.getNumBytes());
117122
assertEquals(NUM_LONG_TERM_BYTES, TABLE_DEFINITION.getNumLongTermBytes());
118123
assertEquals(NUM_TIME_TRAVEL_PHYSICAL_BYTES, TABLE_DEFINITION.getNumTimeTravelPhysicalBytes());
@@ -221,6 +226,7 @@ private void compareStandardTableDefinition(
221226
assertEquals(expected.getNumLongTermPhysicalBytes(), value.getNumLongTermPhysicalBytes());
222227
assertEquals(expected.getNumRows(), value.getNumRows());
223228
assertEquals(expected.getLocation(), value.getLocation());
229+
assertEquals(expected.getMaxStaleness(), value.getMaxStaleness());
224230
assertEquals(expected.getStreamingBuffer(), value.getStreamingBuffer());
225231
assertEquals(expected.getType(), value.getType());
226232
assertEquals(expected.getTimePartitioning(), value.getTimePartitioning());

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java‎

Lines changed: 59 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1749,6 +1749,53 @@ public void testCreateAndGetTableWithSelectedField() {
17491749
assertTrue(remoteTable.delete());
17501750
}
17511751

1752+
@Test
1753+
public void testCreateAndGetTableWithMaxStaleness() {
1754+
// TODO(DNM): DO NOT MERGE. Add integration test here.
1755+
1756+
String tableName = "test_create_and_get_table_with_max_staleness";
1757+
TableId tableId = TableId.of(DATASET, tableName);
1758+
TimePartitioning partitioning = TimePartitioning.of(Type.DAY);
1759+
Clustering clustering =
1760+
Clustering.newBuilder().setFields(ImmutableList.of(STRING_FIELD_SCHEMA.getName())).build();
1761+
StandardTableDefinition tableDefinition =
1762+
StandardTableDefinition.newBuilder()
1763+
.setSchema(TABLE_SCHEMA)
1764+
.setTimePartitioning(partitioning)
1765+
.setClustering(clustering)
1766+
.setMaxStaleness("INTERVAL 1 MINUTE")
1767+
.build();
1768+
Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition));
1769+
assertNotNull(createdTable);
1770+
assertEquals(DATASET, createdTable.getTableId().getDataset());
1771+
assertEquals(tableName, createdTable.getTableId().getTable());
1772+
Table remoteTable = bigquery.getTable(DATASET, tableName);
1773+
assertNotNull(remoteTable);
1774+
assertTrue(remoteTable.getDefinition() instanceof StandardTableDefinition);
1775+
assertEquals(createdTable.getTableId(), remoteTable.getTableId());
1776+
assertEquals(TableDefinition.Type.TABLE, remoteTable.getDefinition().getType());
1777+
assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema());
1778+
assertNotNull(remoteTable.getCreationTime());
1779+
assertNotNull(remoteTable.getLastModifiedTime());
1780+
assertNotNull(remoteTable.<StandardTableDefinition>getDefinition().getNumBytes());
1781+
assertNotNull(remoteTable.<StandardTableDefinition>getDefinition().getNumLongTermBytes());
1782+
assertNotNull(remoteTable.<StandardTableDefinition>getDefinition().getNumTotalLogicalBytes());
1783+
assertNotNull(remoteTable.<StandardTableDefinition>getDefinition().getNumActiveLogicalBytes());
1784+
assertEquals(
1785+
"INTERVAL 1 MINUTE",
1786+
remoteTable.<StandardTableDefinition>getDefinition().getMaxStaleness());
1787+
1788+
assertNotNull(
1789+
remoteTable.<StandardTableDefinition>getDefinition().getNumLongTermLogicalBytes());
1790+
assertNotNull(remoteTable.<StandardTableDefinition>getDefinition().getNumRows());
1791+
assertNotNull(remoteTable.<StandardTableDefinition>getDefinition().getNumRows());
1792+
1793+
assertEquals(
1794+
partitioning, remoteTable.<StandardTableDefinition>getDefinition().getTimePartitioning());
1795+
assertEquals(clustering, remoteTable.<StandardTableDefinition>getDefinition().getClustering());
1796+
assertTrue(remoteTable.delete());
1797+
}
1798+
17521799
@Test
17531800
public void testCreateExternalTable() throws InterruptedException {
17541801
String tableName = "test_create_external_table";
@@ -1921,6 +1968,7 @@ public void testCreateMaterializedViewTable() {
19211968
String.format(
19221969
"SELECT MAX(TimestampField) AS TimestampField,StringField, MAX(BooleanField) AS BooleanField FROM %s.%s.%s GROUP BY StringField",
19231970
PROJECT_ID, DATASET, TABLE_ID.getTable()))
1971+
.setMaxStaleness("INTERVAL 0 MINUTE")
19241972
.build();
19251973
TableInfo tableInfo = TableInfo.of(tableId, viewDefinition);
19261974
Table createdTable = bigquery.create(tableInfo);
@@ -3756,6 +3804,7 @@ public void testProjectIDFastSQLQueryWithJobId() throws InterruptedException {
37563804

37573805
@Test
37583806
public void testLocationFastSQLQueryWithJobId() throws InterruptedException {
3807+
// TODO(DNM): DO NOT MERGE. Example of Integration tests.
37593808
DatasetInfo infoUK =
37603809
DatasetInfo.newBuilder(UK_DATASET)
37613810
.setDescription(DESCRIPTION)
@@ -3764,9 +3813,17 @@ public void testLocationFastSQLQueryWithJobId() throws InterruptedException {
37643813
.build();
37653814
bigquery.create(infoUK);
37663815

3767-
TableDefinition tableDefinition = StandardTableDefinition.of(SIMPLE_SCHEMA);
3816+
// TableDefinition tableDefinition = StandardTableDefinition.of(SIMPLE_SCHEMA);
3817+
StandardTableDefinition tableDefinition =
3818+
StandardTableDefinition.newBuilder()
3819+
.setSchema(SIMPLE_SCHEMA)
3820+
.setMaxStaleness("INTERVAL 1 MINUTE")
3821+
.build();
3822+
37683823
TableInfo tableInfo = TableInfo.newBuilder(TABLE_ID_FASTQUERY_UK, tableDefinition).build();
3769-
bigquery.create(tableInfo);
3824+
Table createdTable = bigquery.create(tableInfo);
3825+
3826+
assertEquals("CHUONGPH111", createdTable.<StandardTableDefinition>getDefinition().toString());
37703827

37713828
String insert =
37723829
"INSERT " + UK_DATASET + "." + TABLE_ID_FASTQUERY_UK.getTable() + " VALUES('Anna');";

0 commit comments

Comments
 (0)