Skip to content

Commit

Permalink
feat: add materialize view (#174)
Browse files Browse the repository at this point in the history
* feat: add materialize view

* feat: add unit test case

* feat: modified comment

* feat: add additional fields and code coverage

* feat: add integration test

* feat: run formatter
  • Loading branch information
Praful Makani authored Mar 20, 2020
1 parent 6213ea9 commit d33b645
Show file tree
Hide file tree
Showing 4 changed files with 303 additions and 12 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.google.cloud.bigquery;

import com.google.api.services.bigquery.model.Table;
import com.google.auto.value.AutoValue;
import javax.annotation.Nullable;

@AutoValue
public abstract class MaterializedViewDefinition extends TableDefinition {

private static final long serialVersionUID = 5898696389126164276L;

@AutoValue.Builder
public abstract static class Builder
extends TableDefinition.Builder<MaterializedViewDefinition, Builder> {

/**
* [Output-only] The time when this materialized view was last modified, in milliseconds since
* the epoch.
*/
abstract Builder setLastRefreshTime(Long lastRefreshTime);

/** Sets the query whose result is persisted. */
public abstract Builder setQuery(String query);

/**
* Set enable automatic refresh of the materialized view when the base table is updated. The
* default value is "true".
*/
public abstract Builder setEnableRefresh(Boolean enableRefresh);

/**
* Set a maximum frequency at which this materialized view will be refreshed. The default value
* is "1800000" (30 minutes).
*/
public abstract Builder setRefreshIntervalMs(Long refreshIntervalMs);

/** Sets the table schema. */
@Override
public abstract Builder setSchema(Schema schema);

@Override
public abstract Builder setType(Type type);

/** Creates a {@code MaterializedViewDefinition} object. */
@Override
public abstract MaterializedViewDefinition build();
}

/**
* Returns time when this materialized view was last modified, in milliseconds since the epoch.
*/
@Nullable
public abstract Long getLastRefreshTime();

/** Returns a query whose result is persisted. */
@Nullable
public abstract String getQuery();

/**
* Returns enable automatic refresh of the materialized view when the base table is updated. The
* default value is "true".
*/
@Nullable
public abstract Boolean getEnableRefresh();

/**
* Returns a maximum frequency at which this materialized view will be refreshed. The default
* value is "1800000" (30 minutes).
*/
@Nullable
public abstract Long getRefreshIntervalMs();

/** Returns a builder for the {@code MaterializedViewDefinition} object. */
public abstract Builder toBuilder();

@Override
Table toPb() {
Table tablePb = super.toPb();
com.google.api.services.bigquery.model.MaterializedViewDefinition materializedViewDefinition =
new com.google.api.services.bigquery.model.MaterializedViewDefinition();
if (getQuery() != null) {
materializedViewDefinition.setQuery(getQuery());
}
if (getLastRefreshTime() != null) {
materializedViewDefinition.setLastRefreshTime(getLastRefreshTime());
}
if (getEnableRefresh() != null) {
materializedViewDefinition.setEnableRefresh(getEnableRefresh());
}
if (getRefreshIntervalMs() != null) {
materializedViewDefinition.setRefreshIntervalMs(getRefreshIntervalMs());
}
tablePb.setMaterializedView(materializedViewDefinition);
return tablePb;
}

static Builder newBuilder() {
return new AutoValue_MaterializedViewDefinition.Builder().setType(Type.MATERIALIZED_VIEW);
}

/**
* Returns a builder for a BigQuery materialized view definition.
*
* @param query the query used to generate the materialized view
*/
public static Builder newBuilder(String query) {
return newBuilder().setQuery(query);
}

/**
* Returns a builder for a BigQuery materialized view definition.
*
* @param query the query used to generate the materialized view
*/
public static MaterializedViewDefinition of(String query) {
return newBuilder(query).build();
}

static MaterializedViewDefinition fromPb(Table tablePb) {
Builder builder = newBuilder().table(tablePb);
if (tablePb.getMaterializedView() != null) {
com.google.api.services.bigquery.model.MaterializedViewDefinition materializedViewDefinition =
tablePb.getMaterializedView();
if (materializedViewDefinition.getQuery() != null) {
builder.setQuery(materializedViewDefinition.getQuery());
}
if (materializedViewDefinition.getLastRefreshTime() != null) {
builder.setLastRefreshTime(materializedViewDefinition.getLastRefreshTime());
}
if (materializedViewDefinition.getEnableRefresh() != null) {
builder.setEnableRefresh(materializedViewDefinition.getEnableRefresh());
}
if (materializedViewDefinition.getRefreshIntervalMs() != null) {
builder.setRefreshIntervalMs(materializedViewDefinition.getRefreshIntervalMs());
}
}
return builder.build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,14 @@ public Type apply(String constant) {
*/
public static final Type VIEW = type.createAndRegister("VIEW");

/**
* SQL query whose result is persisted. Instances of {@code MaterializedViewDefinition} for this
* type are implemented by {@link MaterializedViewDefinition}.
*
* @see <a href="https://cloud.google.com/bigquery/querying-data#views">Views</a>
*/
public static final Type MATERIALIZED_VIEW = type.createAndRegister("MATERIALIZED_VIEW");

/**
* A BigQuery table backed by external data. Instances of {@code TableDefinition} for this type
* are implemented by {@link ExternalTableDefinition}.
Expand Down Expand Up @@ -151,6 +159,8 @@ static <T extends TableDefinition> T fromPb(Table tablePb) {
return (T) StandardTableDefinition.fromPb(tablePb);
case "VIEW":
return (T) ViewDefinition.fromPb(tablePb);
case "MATERIALIZED_VIEW":
return (T) MaterializedViewDefinition.fromPb(tablePb);
case "EXTERNAL":
return (T) ExternalTableDefinition.fromPb(tablePb);
case "MODEL":
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.google.cloud.bigquery;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import org.junit.Test;

public class MaterializedViewDefinitionTest {

private static final String MATERIALIZED_VIEW_QUERY = "MATERIALIZED_VIEW_QUERY";
private static final Long LAST_REFRESH_TIME = 1580302008L;
private static final Boolean ENABLE_REFRESH = false;
private static final Long REFRESH_INTERVAL_MS = 60000L;
private static final Schema SCHEMA = Schema.of();
private static final MaterializedViewDefinition MATERIALIZED_VIEW_DEFINITION =
MaterializedViewDefinition.newBuilder()
.setSchema(SCHEMA)
.setQuery(MATERIALIZED_VIEW_QUERY)
.setLastRefreshTime(LAST_REFRESH_TIME)
.setEnableRefresh(ENABLE_REFRESH)
.setRefreshIntervalMs(REFRESH_INTERVAL_MS)
.build();

@Test
public void testToBuilder() {
compareMaterializedView(
MATERIALIZED_VIEW_DEFINITION, MATERIALIZED_VIEW_DEFINITION.toBuilder().build());
MaterializedViewDefinition materializedViewDefinition =
MATERIALIZED_VIEW_DEFINITION.toBuilder().setQuery("NEW QUERY").build();
assertEquals("NEW QUERY", materializedViewDefinition.getQuery());
materializedViewDefinition =
materializedViewDefinition.toBuilder().setQuery(MATERIALIZED_VIEW_QUERY).build();
compareMaterializedView(MATERIALIZED_VIEW_DEFINITION, materializedViewDefinition);
}

@Test
public void testToBuilderIncomplete() {
TableDefinition materializedViewDefinition =
MaterializedViewDefinition.of(MATERIALIZED_VIEW_QUERY);
assertEquals(materializedViewDefinition, materializedViewDefinition.toBuilder().build());
}

@Test
public void testBuilder() {
assertEquals(MATERIALIZED_VIEW_QUERY, MATERIALIZED_VIEW_DEFINITION.getQuery());
assertEquals(TableDefinition.Type.MATERIALIZED_VIEW, MATERIALIZED_VIEW_DEFINITION.getType());
assertEquals(LAST_REFRESH_TIME, MATERIALIZED_VIEW_DEFINITION.getLastRefreshTime());
MaterializedViewDefinition materializedViewDefinition =
MaterializedViewDefinition.newBuilder()
.setSchema(SCHEMA)
.setQuery(MATERIALIZED_VIEW_QUERY)
.setLastRefreshTime(LAST_REFRESH_TIME)
.setEnableRefresh(ENABLE_REFRESH)
.setRefreshIntervalMs(REFRESH_INTERVAL_MS)
.build();
assertEquals(MATERIALIZED_VIEW_DEFINITION, materializedViewDefinition);
}

@Test
public void testToAndFromPb() {
MaterializedViewDefinition materializedViewDefinition =
MATERIALIZED_VIEW_DEFINITION.toBuilder().build();
assertTrue(
TableDefinition.fromPb(materializedViewDefinition.toPb())
instanceof MaterializedViewDefinition);
compareMaterializedView(
materializedViewDefinition,
TableDefinition.<MaterializedViewDefinition>fromPb(materializedViewDefinition.toPb()));
}

private void compareMaterializedView(
MaterializedViewDefinition expected, MaterializedViewDefinition actual) {
assertEquals(expected.getType(), actual.getType());
assertEquals(expected.getSchema(), actual.getSchema());
assertEquals(expected.getQuery(), actual.getQuery());
assertEquals(expected.getLastRefreshTime(), actual.getLastRefreshTime());
assertEquals(expected.getEnableRefresh(), actual.getEnableRefresh());
assertEquals(expected.getRefreshIntervalMs(), actual.getRefreshIntervalMs());
assertEquals(expected.toString(), actual.toString());
assertEquals(expected.hashCode(), actual.hashCode());
assertEquals(expected, actual);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import com.google.auth.oauth2.ServiceAccountCredentials;
import com.google.cloud.Date;
import com.google.cloud.RetryOption;
import com.google.cloud.ServiceOptions;
import com.google.cloud.bigquery.Acl;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption;
Expand Down Expand Up @@ -65,6 +66,7 @@
import com.google.cloud.bigquery.JobStatistics.LoadStatistics;
import com.google.cloud.bigquery.LegacySQLTypeName;
import com.google.cloud.bigquery.LoadJobConfiguration;
import com.google.cloud.bigquery.MaterializedViewDefinition;
import com.google.cloud.bigquery.Model;
import com.google.cloud.bigquery.ModelId;
import com.google.cloud.bigquery.ModelInfo;
Expand Down Expand Up @@ -134,6 +136,7 @@ public class ITBigQueryTest {
private static final String OTHER_DATASET = RemoteBigQueryHelper.generateDatasetName();
private static final String MODEL_DATASET = RemoteBigQueryHelper.generateDatasetName();
private static final String ROUTINE_DATASET = RemoteBigQueryHelper.generateDatasetName();
private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId();
private static final Map<String, String> LABELS =
ImmutableMap.of(
"example-label1", "example-value1",
Expand Down Expand Up @@ -219,6 +222,17 @@ public class ITBigQueryTest {
Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN)
.setMode(Field.Mode.NULLABLE)
.build());
private static final Schema VIEW_SCHEMA =
Schema.of(
Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP)
.setMode(Field.Mode.NULLABLE)
.build(),
Field.newBuilder("StringField", LegacySQLTypeName.STRING)
.setMode(Field.Mode.NULLABLE)
.build(),
Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN)
.setMode(Field.Mode.NULLABLE)
.build());
private static final RangePartitioning.Range RANGE =
RangePartitioning.Range.newBuilder().setStart(1L).setInterval(2L).setEnd(20L).build();
private static final RangePartitioning RANGE_PARTITIONING =
Expand Down Expand Up @@ -640,18 +654,7 @@ public void testCreateViewTable() throws InterruptedException {
assertNotNull(remoteTable);
assertEquals(createdTable.getTableId(), remoteTable.getTableId());
assertTrue(remoteTable.getDefinition() instanceof ViewDefinition);
Schema expectedSchema =
Schema.of(
Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP)
.setMode(Field.Mode.NULLABLE)
.build(),
Field.newBuilder("StringField", LegacySQLTypeName.STRING)
.setMode(Field.Mode.NULLABLE)
.build(),
Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN)
.setMode(Field.Mode.NULLABLE)
.build());
assertEquals(expectedSchema, remoteTable.getDefinition().getSchema());
assertEquals(VIEW_SCHEMA, remoteTable.getDefinition().getSchema());
QueryJobConfiguration config =
QueryJobConfiguration.newBuilder("SELECT * FROM " + tableName)
.setDefaultDataset(DatasetId.of(DATASET))
Expand All @@ -678,6 +681,30 @@ public void testCreateViewTable() throws InterruptedException {
assertTrue(remoteTable.delete());
}

@Test
public void testCreateMaterializedViewTable() {
String tableName = "test_materialized_view_table";
TableId tableId = TableId.of(DATASET, tableName);
MaterializedViewDefinition viewDefinition =
MaterializedViewDefinition.newBuilder(
String.format(
"SELECT MAX(TimestampField) AS TimestampField,StringField, MAX(BooleanField) AS BooleanField FROM %s.%s.%s GROUP BY StringField",
PROJECT_ID, DATASET, TABLE_ID.getTable()))
.build();
TableInfo tableInfo = TableInfo.of(tableId, viewDefinition);
Table createdTable = bigquery.create(tableInfo);
assertNotNull(createdTable);
assertEquals(DATASET, createdTable.getTableId().getDataset());
assertEquals(tableName, createdTable.getTableId().getTable());
Table remoteTable = bigquery.getTable(DATASET, tableName);
assertNotNull(remoteTable);
assertEquals(createdTable.getTableId(), remoteTable.getTableId());
assertEquals(createdTable.getTableId(), remoteTable.getTableId());
assertTrue(remoteTable.getDefinition() instanceof MaterializedViewDefinition);
assertEquals(VIEW_SCHEMA, remoteTable.getDefinition().getSchema());
assertTrue(remoteTable.delete());
}

@Test
public void testListTables() {
String tableName = "test_list_tables";
Expand Down

0 comments on commit d33b645

Please sign in to comment.