diff --git a/presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java index 25c69d0a47edb..f77f130efdb97 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main-base/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -244,6 +244,7 @@ public final class SystemSessionProperties public static final String MATERIALIZED_VIEW_DATA_CONSISTENCY_ENABLED = "materialized_view_data_consistency_enabled"; public static final String CONSIDER_QUERY_FILTERS_FOR_MATERIALIZED_VIEW_PARTITIONS = "consider-query-filters-for-materialized-view-partitions"; public static final String QUERY_OPTIMIZATION_WITH_MATERIALIZED_VIEW_ENABLED = "query_optimization_with_materialized_view_enabled"; + public static final String LEGACY_MATERIALIZED_VIEWS = "legacy_materialized_views"; public static final String AGGREGATION_IF_TO_FILTER_REWRITE_STRATEGY = "aggregation_if_to_filter_rewrite_strategy"; public static final String JOINS_NOT_NULL_INFERENCE_STRATEGY = "joins_not_null_inference_strategy"; public static final String RESOURCE_AWARE_SCHEDULING_STRATEGY = "resource_aware_scheduling_strategy"; @@ -1353,6 +1354,12 @@ public SystemSessionProperties( "Enable query optimization with materialized view", featuresConfig.isQueryOptimizationWithMaterializedViewEnabled(), true), + booleanProperty( + LEGACY_MATERIALIZED_VIEWS, + "Experimental: Use legacy materialized views. This feature is under active development and may change" + + "or be removed at any time. Do not disable in production environments.", + featuresConfig.isLegacyMaterializedViews(), + true), stringProperty( DISTRIBUTED_TRACING_MODE, "Mode for distributed tracing. NO_TRACE, ALWAYS_TRACE, or SAMPLE_BASED", @@ -2882,6 +2889,11 @@ public static boolean isQueryOptimizationWithMaterializedViewEnabled(Session ses return session.getSystemProperty(QUERY_OPTIMIZATION_WITH_MATERIALIZED_VIEW_ENABLED, Boolean.class); } + public static boolean isLegacyMaterializedViews(Session session) + { + return session.getSystemProperty(LEGACY_MATERIALIZED_VIEWS, Boolean.class); + } + public static boolean isVerboseRuntimeStatsEnabled(Session session) { return session.getSystemProperty(VERBOSE_RUNTIME_STATS_ENABLED, Boolean.class); diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/MaterializedViewUtils.java b/presto-main-base/src/main/java/com/facebook/presto/sql/MaterializedViewUtils.java index eea0746d1841d..a978bfdd2aa32 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/MaterializedViewUtils.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/MaterializedViewUtils.java @@ -19,6 +19,7 @@ import com.facebook.presto.common.predicate.TupleDomain; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.SessionPropertyManager; +import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.MaterializedViewStatus; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.relation.DomainTranslator; @@ -90,7 +91,7 @@ public static Session buildOwnerSession(Session session, Optional owner, { Identity identity = getOwnerIdentity(owner, session); - return Session.builder(sessionPropertyManager) + Session.SessionBuilder builder = Session.builder(sessionPropertyManager) .setQueryId(session.getQueryId()) .setTransactionId(session.getTransactionId().orElse(null)) .setIdentity(identity) @@ -102,8 +103,20 @@ public static Session buildOwnerSession(Session session, Optional owner, .setRemoteUserAddress(session.getRemoteUserAddress().orElse(null)) .setUserAgent(session.getUserAgent().orElse(null)) .setClientInfo(session.getClientInfo().orElse(null)) - .setStartTime(session.getStartTime()) - .build(); + .setStartTime(session.getStartTime()); + + for (Map.Entry property : session.getSystemProperties().entrySet()) { + builder.setSystemProperty(property.getKey(), property.getValue()); + } + + for (Map.Entry> connectorEntry : session.getConnectorProperties().entrySet()) { + String catalogName = connectorEntry.getKey().getCatalogName(); + for (Map.Entry property : connectorEntry.getValue().entrySet()) { + builder.setCatalogSessionProperty(catalogName, property.getKey(), property.getValue()); + } + } + + return builder.build(); } public static Identity getOwnerIdentity(Optional owner, Session session) diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java index 7214485a7d7a6..1cc73d325053e 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java @@ -225,6 +225,7 @@ public class FeaturesConfig private boolean materializedViewDataConsistencyEnabled = true; private boolean materializedViewPartitionFilteringEnabled = true; private boolean queryOptimizationWithMaterializedViewEnabled; + private boolean legacyMaterializedViewRefresh = true; private AggregationIfToFilterRewriteStrategy aggregationIfToFilterRewriteStrategy = AggregationIfToFilterRewriteStrategy.DISABLED; private String analyzerType = "BUILTIN"; @@ -2154,6 +2155,20 @@ public FeaturesConfig setQueryOptimizationWithMaterializedViewEnabled(boolean va return this; } + public boolean isLegacyMaterializedViews() + { + return legacyMaterializedViewRefresh; + } + + @Config("experimental.legacy-materialized-views") + @ConfigDescription("Experimental: Use legacy materialized views. This feature is under active development and may change" + + "or be removed at any time. Do not disable in production environments.") + public FeaturesConfig setLegacyMaterializedViews(boolean value) + { + this.legacyMaterializedViewRefresh = value; + return this; + } + public boolean isVerboseRuntimeStatsEnabled() { return verboseRuntimeStatsEnabled; diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java index eb2dc037dcf46..7d9e8f196f1af 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java @@ -228,6 +228,7 @@ import static com.facebook.presto.SystemSessionProperties.getMaxGroupingSets; import static com.facebook.presto.SystemSessionProperties.isAllowWindowOrderByLiterals; +import static com.facebook.presto.SystemSessionProperties.isLegacyMaterializedViews; import static com.facebook.presto.SystemSessionProperties.isMaterializedViewDataConsistencyEnabled; import static com.facebook.presto.SystemSessionProperties.isMaterializedViewPartitionFilteringEnabled; import static com.facebook.presto.common.RuntimeMetricName.SKIP_READING_FROM_MATERIALIZED_VIEW_COUNT; @@ -864,10 +865,9 @@ protected Scope visitRefreshMaterializedView(RefreshMaterializedView node, Optio // Use AllowAllAccessControl; otherwise Analyzer will check SELECT permission on the materialized view, which is not necessary. StatementAnalyzer viewAnalyzer = new StatementAnalyzer(analysis, metadata, sqlParser, new AllowAllAccessControl(), session, warningCollector); Scope viewScope = viewAnalyzer.analyze(node.getTarget(), scope); - if (!node.getWhere().isPresent()) { - throw new SemanticException(NOT_SUPPORTED, node, "Refresh Materialized View without predicates is not supported."); - } - Map tablePredicates = extractTablePredicates(viewName, node.getWhere().get(), viewScope, metadata, session); + + Map tablePredicates = getTablePredicatesForMaterializedViewRefresh( + session, node, viewName, viewScope, metadata); Query viewQuery = parseView(view.getOriginalSql(), viewName, node); Query refreshQuery = tablePredicates.containsKey(toSchemaTableName(viewName)) ? @@ -910,10 +910,9 @@ private Optional analyzeBaseTableForRefreshMaterializedView(Table // Use AllowAllAccessControl; otherwise Analyzer will check SELECT permission on the materialized view, which is not necessary. StatementAnalyzer viewAnalyzer = new StatementAnalyzer(analysis, metadata, sqlParser, new AllowAllAccessControl(), session, warningCollector); Scope viewScope = viewAnalyzer.analyze(refreshMaterializedView.getTarget(), scope); - if (!refreshMaterializedView.getWhere().isPresent()) { - throw new SemanticException(NOT_SUPPORTED, "Refresh Materialized View without predicates is not supported."); - } - Map tablePredicates = extractTablePredicates(viewName, refreshMaterializedView.getWhere().get(), viewScope, metadata, session); + + Map tablePredicates = getTablePredicatesForMaterializedViewRefresh( + session, refreshMaterializedView, viewName, viewScope, metadata); SchemaTableName baseTableName = toSchemaTableName(createQualifiedObjectName(session, baseTable, baseTable.getName(), metadata)); if (tablePredicates.containsKey(baseTableName)) { @@ -928,6 +927,28 @@ private Optional analyzeBaseTableForRefreshMaterializedView(Table return Optional.empty(); } + private Map getTablePredicatesForMaterializedViewRefresh( + Session session, + RefreshMaterializedView node, + QualifiedObjectName viewName, + Scope viewScope, + Metadata metadata) + { + if (isLegacyMaterializedViews(session)) { + if (!node.getWhere().isPresent()) { + throw new SemanticException(NOT_SUPPORTED, node, "Refresh Materialized View without predicates is not supported."); + } + return extractTablePredicates(viewName, node.getWhere().get(), viewScope, metadata, session); + } + else { + if (node.getWhere().isPresent()) { + throw new SemanticException(NOT_SUPPORTED, node, "WHERE clause in REFRESH MATERIALIZED VIEW is not supported. " + + "Connectors automatically determine which data needs refreshing based on staleness detection."); + } + return ImmutableMap.of(); + } + } + private Query buildQueryWithPredicate(Table table, Expression predicate) { Query query = simpleQuery(selectList(new AllColumns()), table, predicate); diff --git a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java index 9f55c3b2eab19..ec24eba3a86d4 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java +++ b/presto-main-base/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java @@ -187,6 +187,7 @@ public void testDefaults() .setMaterializedViewDataConsistencyEnabled(true) .setMaterializedViewPartitionFilteringEnabled(true) .setQueryOptimizationWithMaterializedViewEnabled(false) + .setLegacyMaterializedViews(true) .setVerboseRuntimeStatsEnabled(false) .setAggregationIfToFilterRewriteStrategy(AggregationIfToFilterRewriteStrategy.DISABLED) .setAnalyzerType("BUILTIN") @@ -406,6 +407,7 @@ public void testExplicitPropertyMappings() .put("materialized-view-data-consistency-enabled", "false") .put("consider-query-filters-for-materialized-view-partitions", "false") .put("query-optimization-with-materialized-view-enabled", "true") + .put("experimental.legacy-materialized-views", "false") .put("analyzer-type", "CRUX") .put("pre-process-metadata-calls", "true") .put("verbose-runtime-stats-enabled", "true") @@ -622,6 +624,7 @@ public void testExplicitPropertyMappings() .setMaterializedViewDataConsistencyEnabled(false) .setMaterializedViewPartitionFilteringEnabled(false) .setQueryOptimizationWithMaterializedViewEnabled(true) + .setLegacyMaterializedViews(false) .setVerboseRuntimeStatsEnabled(true) .setAggregationIfToFilterRewriteStrategy(AggregationIfToFilterRewriteStrategy.FILTER_WITH_IF) .setAnalyzerType("CRUX") diff --git a/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryInsertTableHandle.java b/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryInsertTableHandle.java index fda8f2075aa8d..9d7bb488e2c02 100644 --- a/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryInsertTableHandle.java +++ b/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryInsertTableHandle.java @@ -27,14 +27,22 @@ public class MemoryInsertTableHandle { private final MemoryTableHandle table; private final Set activeTableIds; + private final boolean insertOverwrite; @JsonCreator public MemoryInsertTableHandle( @JsonProperty("table") MemoryTableHandle table, - @JsonProperty("activeTableIds") Set activeTableIds) + @JsonProperty("activeTableIds") Set activeTableIds, + @JsonProperty("insertOverwrite") boolean insertOverwrite) { this.table = requireNonNull(table, "table is null"); this.activeTableIds = requireNonNull(activeTableIds, "activeTableIds is null"); + this.insertOverwrite = insertOverwrite; + } + + public MemoryInsertTableHandle(MemoryTableHandle table, Set activeTableIds) + { + this(table, activeTableIds, false); } @JsonProperty @@ -49,6 +57,12 @@ public Set getActiveTableIds() return activeTableIds; } + @JsonProperty + public boolean isInsertOverwrite() + { + return insertOverwrite; + } + @Override public String toString() { diff --git a/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryMetadata.java b/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryMetadata.java index b9741caf478c9..42458bb165946 100644 --- a/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryMetadata.java +++ b/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryMetadata.java @@ -28,6 +28,8 @@ import com.facebook.presto.spi.ConnectorViewDefinition; import com.facebook.presto.spi.Constraint; import com.facebook.presto.spi.HostAddress; +import com.facebook.presto.spi.MaterializedViewDefinition; +import com.facebook.presto.spi.MaterializedViewStatus; import com.facebook.presto.spi.Node; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.PrestoException; @@ -39,6 +41,7 @@ import com.facebook.presto.spi.connector.ConnectorOutputMetadata; import com.facebook.presto.spi.statistics.ComputedStatistics; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.errorprone.annotations.ThreadSafe; import io.airlift.slice.Slice; @@ -82,6 +85,11 @@ public class MemoryMetadata private final Map> tableDataFragments = new HashMap<>(); private final Map views = new HashMap<>(); + private final Map materializedViews = new HashMap<>(); + private final Map tableVersions = new HashMap<>(); + private final Map> mvRefreshVersions = new HashMap<>(); + private final Map storageTableToMaterializedView = new HashMap<>(); + @Inject public MemoryMetadata(NodeManager nodeManager, MemoryConnectorId connectorId) { @@ -184,10 +192,17 @@ public ConnectorTableMetadata toTableMetadata(MemoryTableHandle memoryTableHandl public synchronized void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle) { MemoryTableHandle handle = (MemoryTableHandle) tableHandle; - Long tableId = tableIds.remove(handle.toSchemaTableName()); + SchemaTableName tableName = handle.toSchemaTableName(); + + if (storageTableToMaterializedView.containsKey(tableName)) { + throw new PrestoException(NOT_FOUND, format("Cannot drop table [%s] because it is a materialized view storage table. Use DROP MATERIALIZED VIEW instead.", tableName)); + } + + Long tableId = tableIds.remove(tableName); if (tableId != null) { tables.remove(tableId); tableDataFragments.remove(tableId); + tableVersions.remove(tableName); } } @@ -197,6 +212,11 @@ public synchronized void renameTable(ConnectorSession session, ConnectorTableHan checkSchemaExists(newTableName.getSchemaName()); checkTableNotExists(newTableName); MemoryTableHandle oldTableHandle = (MemoryTableHandle) tableHandle; + SchemaTableName oldTableName = oldTableHandle.toSchemaTableName(); + + if (storageTableToMaterializedView.containsKey(oldTableName)) { + throw new PrestoException(NOT_FOUND, format("Cannot rename table [%s] because it is a materialized view storage table", oldTableName)); + } MemoryTableHandle newTableHandle = new MemoryTableHandle( oldTableHandle.getConnectorId(), newTableName.getSchemaName(), @@ -262,6 +282,8 @@ public synchronized Optional finishCreateTable(Connecto MemoryOutputTableHandle memoryOutputHandle = (MemoryOutputTableHandle) tableHandle; updateRowsOnHosts(memoryOutputHandle.getTable(), fragments); + incrementTableVersion(memoryOutputHandle.getTable().toSchemaTableName()); + return Optional.empty(); } @@ -279,6 +301,8 @@ public synchronized Optional finishInsert(ConnectorSess MemoryInsertTableHandle memoryInsertHandle = (MemoryInsertTableHandle) insertHandle; updateRowsOnHosts(memoryInsertHandle.getTable(), fragments); + incrementTableVersion(memoryInsertHandle.getTable().toSchemaTableName()); + return Optional.empty(); } @@ -355,6 +379,11 @@ private void updateRowsOnHosts(MemoryTableHandle table, Collection fragme } } + private void incrementTableVersion(SchemaTableName tableName) + { + tableVersions.put(tableName, tableVersions.getOrDefault(tableName, 0L) + 1); + } + @Override public synchronized ConnectorTableLayoutResult getTableLayoutForConstraint( ConnectorSession session, @@ -390,4 +419,137 @@ public synchronized ConnectorTableLayout getTableLayout(ConnectorSession session Optional.empty(), ImmutableList.of()); } + + @Override + public synchronized void createMaterializedView( + ConnectorSession session, + ConnectorTableMetadata viewMetadata, + MaterializedViewDefinition viewDefinition, + boolean ignoreExisting) + { + SchemaTableName viewName = viewMetadata.getTable(); + checkSchemaExists(viewName.getSchemaName()); + + if (materializedViews.containsKey(viewName)) { + if (ignoreExisting) { + return; + } + throw new PrestoException(ALREADY_EXISTS, "Materialized view already exists: " + viewName); + } + + if (getTableHandle(session, viewName) != null) { + throw new PrestoException(ALREADY_EXISTS, "Table already exists: " + viewName); + } + + if (views.containsKey(viewName)) { + throw new PrestoException(ALREADY_EXISTS, "View already exists: " + viewName); + } + + SchemaTableName storageTableName = new SchemaTableName( + viewDefinition.getSchema(), + viewDefinition.getTable()); + + ConnectorTableMetadata storageTableMetadata = new ConnectorTableMetadata( + storageTableName, + viewMetadata.getColumns(), + viewMetadata.getProperties(), + viewMetadata.getComment()); + + createTable(session, storageTableMetadata, false); + + materializedViews.put(viewName, viewDefinition); + Map baseTableVersionSnapshot = new HashMap<>(); + for (SchemaTableName baseTable : viewDefinition.getBaseTables()) { + baseTableVersionSnapshot.put(baseTable, 0L); + } + mvRefreshVersions.put(viewName, baseTableVersionSnapshot); + storageTableToMaterializedView.put(storageTableName, viewName); + } + + @Override + public synchronized Optional getMaterializedView(ConnectorSession session, SchemaTableName viewName) + { + return Optional.ofNullable(materializedViews.get(viewName)); + } + + @Override + public synchronized void dropMaterializedView(ConnectorSession session, SchemaTableName viewName) + { + MaterializedViewDefinition removed = materializedViews.remove(viewName); + if (removed == null) { + throw new PrestoException(NOT_FOUND, "Materialized view not found: " + viewName); + } + mvRefreshVersions.remove(viewName); + + SchemaTableName storageTableName = new SchemaTableName( + removed.getSchema(), + removed.getTable()); + storageTableToMaterializedView.remove(storageTableName); + + ConnectorTableHandle storageTableHandle = getTableHandle(session, storageTableName); + if (storageTableHandle != null) { + dropTable(session, storageTableHandle); + } + } + + @Override + public synchronized MaterializedViewStatus getMaterializedViewStatus( + ConnectorSession session, + SchemaTableName materializedViewName, + TupleDomain baseQueryDomain) + { + MaterializedViewDefinition mvDefinition = materializedViews.get(materializedViewName); + if (mvDefinition == null) { + throw new PrestoException(NOT_FOUND, "Materialized view not found: " + materializedViewName); + } + + Map baseTableVersionSnapshot = mvRefreshVersions.getOrDefault(materializedViewName, ImmutableMap.of()); + + for (SchemaTableName baseTable : mvDefinition.getBaseTables()) { + long currentVersion = tableVersions.getOrDefault(baseTable, 0L); + long refreshedVersion = baseTableVersionSnapshot.getOrDefault(baseTable, 0L); + if (currentVersion != refreshedVersion) { + return new MaterializedViewStatus( + MaterializedViewStatus.MaterializedViewState.NOT_MATERIALIZED, + ImmutableMap.of()); + } + } + + return new MaterializedViewStatus(MaterializedViewStatus.MaterializedViewState.FULLY_MATERIALIZED); + } + + @Override + public synchronized ConnectorInsertTableHandle beginRefreshMaterializedView( + ConnectorSession session, + ConnectorTableHandle tableHandle) + { + MemoryTableHandle memoryTableHandle = (MemoryTableHandle) tableHandle; + tableDataFragments.put(memoryTableHandle.getTableId(), new HashMap<>()); + return new MemoryInsertTableHandle(memoryTableHandle, ImmutableSet.copyOf(tableIds.values()), true); + } + + @Override + public synchronized Optional finishRefreshMaterializedView( + ConnectorSession session, + ConnectorInsertTableHandle insertHandle, + Collection fragments, + Collection computedStatistics) + { + Optional result = finishInsert(session, insertHandle, fragments, computedStatistics); + + MemoryInsertTableHandle memoryInsertHandle = (MemoryInsertTableHandle) insertHandle; + SchemaTableName storageTableName = memoryInsertHandle.getTable().toSchemaTableName(); + + SchemaTableName materializedViewName = storageTableToMaterializedView.get(storageTableName); + checkState(materializedViewName != null, "No materialized view found for storage table: %s", storageTableName); + + MaterializedViewDefinition mvDefinition = materializedViews.get(materializedViewName); + Map baseTableVersionSnapshot = new HashMap<>(); + for (SchemaTableName baseTable : mvDefinition.getBaseTables()) { + baseTableVersionSnapshot.put(baseTable, tableVersions.getOrDefault(baseTable, 0L)); + } + mvRefreshVersions.put(materializedViewName, baseTableVersionSnapshot); + + return result; + } } diff --git a/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryPageSinkProvider.java b/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryPageSinkProvider.java index 24ecb8d8947d4..9cddad538e0d7 100644 --- a/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryPageSinkProvider.java +++ b/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryPageSinkProvider.java @@ -81,6 +81,9 @@ public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHa checkState(memoryInsertTableHandle.getActiveTableIds().contains(tableId)); pagesStore.cleanUp(memoryInsertTableHandle.getActiveTableIds()); + if (memoryInsertTableHandle.isInsertOverwrite()) { + pagesStore.clearTable(tableId); + } pagesStore.initialize(tableId); return new MemoryPageSink(pagesStore, currentHostAddress, tableId); } diff --git a/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryPagesStore.java b/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryPagesStore.java index 17b1f40db3e44..f913d768efa58 100644 --- a/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryPagesStore.java +++ b/presto-memory/src/main/java/com/facebook/presto/plugin/memory/MemoryPagesStore.java @@ -104,6 +104,17 @@ public synchronized boolean contains(Long tableId) return tables.containsKey(tableId); } + public synchronized void clearTable(Long tableId) + { + TableData tableData = tables.get(tableId); + if (tableData != null) { + for (Page page : tableData.getPages()) { + currentBytes -= page.getRetainedSizeInBytes(); + } + tables.put(tableId, new TableData()); + } + } + public synchronized void cleanUp(Set activeTableIds) { // We have to remember that there might be some race conditions when there are two tables created at once. diff --git a/presto-memory/src/test/java/com/facebook/presto/plugin/memory/TestMemoryMaterializedViews.java b/presto-memory/src/test/java/com/facebook/presto/plugin/memory/TestMemoryMaterializedViews.java new file mode 100644 index 0000000000000..ea9423cd94bfe --- /dev/null +++ b/presto-memory/src/test/java/com/facebook/presto/plugin/memory/TestMemoryMaterializedViews.java @@ -0,0 +1,612 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.memory; + +import com.facebook.presto.Session; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestQueryFramework; +import com.facebook.presto.tests.DistributedQueryRunner; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; + +@Test(singleThreaded = true) +public class TestMemoryMaterializedViews + extends AbstractTestQueryFramework +{ + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + Session session = testSessionBuilder() + .setCatalog("memory") + .setSchema("default") + .setSystemProperty("legacy_materialized_views", "false") + .build(); + + DistributedQueryRunner queryRunner = DistributedQueryRunner.builder(session) + .setNodeCount(4) + .build(); + + queryRunner.installPlugin(new MemoryPlugin()); + queryRunner.createCatalog("memory", "memory", ImmutableMap.of()); + + return queryRunner; + } + + @Test + public void testCreateMaterializedView() + { + assertUpdate("CREATE TABLE base_table (id BIGINT, name VARCHAR, value BIGINT)"); + assertUpdate("INSERT INTO base_table VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)", 3); + + assertUpdate("CREATE MATERIALIZED VIEW mv_simple AS SELECT id, name, value FROM base_table"); + + assertQuery("SELECT COUNT(*) FROM mv_simple", "SELECT 3"); + assertQuery("SELECT * FROM mv_simple ORDER BY id", + "VALUES (1, 'Alice', 100), (2, 'Bob', 200), (3, 'Charlie', 300)"); + + assertUpdate("DROP MATERIALIZED VIEW mv_simple"); + assertUpdate("DROP TABLE base_table"); + } + + @Test + public void testCreateMaterializedViewDuplicateName() + { + assertUpdate("CREATE TABLE dup_base (id BIGINT, value VARCHAR)"); + assertUpdate("INSERT INTO dup_base VALUES (1, 'test')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW mv_dup AS SELECT id, value FROM dup_base"); + + assertQueryFails("CREATE MATERIALIZED VIEW mv_dup AS SELECT id FROM dup_base", + ".*Materialized view .* already exists.*"); + + assertUpdate("DROP MATERIALIZED VIEW mv_dup"); + assertUpdate("DROP TABLE dup_base"); + } + + @Test + public void testCreateMaterializedViewWithFilter() + { + assertUpdate("CREATE TABLE filtered_base (id BIGINT, status VARCHAR, amount BIGINT)"); + assertUpdate("INSERT INTO filtered_base VALUES (1, 'active', 100), (2, 'inactive', 200), (3, 'active', 300)", 3); + + assertUpdate("CREATE MATERIALIZED VIEW mv_filtered AS SELECT id, amount FROM filtered_base WHERE status = 'active'"); + + assertQuery("SELECT COUNT(*) FROM mv_filtered", "SELECT 2"); + assertQuery("SELECT * FROM mv_filtered ORDER BY id", + "VALUES (1, 100), (3, 300)"); + + assertUpdate("DROP MATERIALIZED VIEW mv_filtered"); + assertUpdate("DROP TABLE filtered_base"); + } + + @Test + public void testCreateMaterializedViewWithComplexFilter() + { + assertUpdate("CREATE TABLE complex_filter_base (id BIGINT, status VARCHAR, amount BIGINT, priority INTEGER)"); + assertUpdate("INSERT INTO complex_filter_base VALUES (1, 'active', 100, 1), (2, 'inactive', 200, 2), (3, 'active', 50, 3), (4, 'active', 150, 1)", 4); + + assertUpdate("CREATE MATERIALIZED VIEW mv_complex_filter AS " + + "SELECT id, amount, priority FROM complex_filter_base " + + "WHERE status = 'active' AND amount > 75 AND priority = 1"); + + assertQuery("SELECT COUNT(*) FROM mv_complex_filter", "SELECT 2"); + assertQuery("SELECT * FROM mv_complex_filter ORDER BY id", + "VALUES (1, 100, 1), (4, 150, 1)"); + + assertUpdate("DROP MATERIALIZED VIEW mv_complex_filter"); + assertUpdate("DROP TABLE complex_filter_base"); + } + + @Test + public void testCreateMaterializedViewWithAggregation() + { + assertUpdate("CREATE TABLE sales (product_id BIGINT, category VARCHAR, revenue BIGINT)"); + assertUpdate("INSERT INTO sales VALUES (1, 'Electronics', 1000), (2, 'Electronics', 1500), (3, 'Books', 500), (4, 'Books', 300)", 4); + + assertUpdate("CREATE MATERIALIZED VIEW mv_category_sales AS " + + "SELECT category, COUNT(*) as product_count, SUM(revenue) as total_revenue " + + "FROM sales GROUP BY category"); + + assertQuery("SELECT COUNT(*) FROM mv_category_sales", "SELECT 2"); + assertQuery("SELECT * FROM mv_category_sales ORDER BY category", + "VALUES ('Books', 2, 800), ('Electronics', 2, 2500)"); + + assertUpdate("DROP MATERIALIZED VIEW mv_category_sales"); + assertUpdate("DROP TABLE sales"); + } + + @Test + public void testCreateMaterializedViewWithComputedColumns() + { + assertUpdate("CREATE TABLE transactions (trans_id BIGINT, amount BIGINT, tax_rate DOUBLE)"); + assertUpdate("INSERT INTO transactions VALUES (1, 100, 0.08), (2, 200, 0.08), (3, 150, 0.10)", 3); + + assertUpdate("CREATE MATERIALIZED VIEW mv_computed AS " + + "SELECT trans_id, amount, tax_rate, " + + "CAST(amount * tax_rate AS BIGINT) as tax_amount, " + + "CAST(amount * (1 + tax_rate) AS BIGINT) as total_amount " + + "FROM transactions"); + + assertQuery("SELECT COUNT(*) FROM mv_computed", "SELECT 3"); + assertQuery("SELECT trans_id, amount, tax_amount, total_amount FROM mv_computed ORDER BY trans_id", + "VALUES (1, 100, 8, 108), (2, 200, 16, 216), (3, 150, 15, 165)"); + + assertUpdate("DROP MATERIALIZED VIEW mv_computed"); + assertUpdate("DROP TABLE transactions"); + } + + @Test + public void testCreateMaterializedViewWithJoin() + { + assertUpdate("CREATE TABLE customer_orders (order_id BIGINT, customer_id BIGINT, amount BIGINT)"); + assertUpdate("CREATE TABLE customers (customer_id BIGINT, customer_name VARCHAR)"); + + assertUpdate("INSERT INTO customer_orders VALUES (1, 100, 50), (2, 200, 75), (3, 100, 25)", 3); + assertUpdate("INSERT INTO customers VALUES (100, 'Alice'), (200, 'Bob')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_customer_orders AS " + + "SELECT o.order_id, c.customer_name, o.amount " + + "FROM customer_orders o JOIN customers c ON o.customer_id = c.customer_id"); + + assertQuery("SELECT COUNT(*) FROM mv_customer_orders", "SELECT 3"); + assertQuery("SELECT * FROM mv_customer_orders ORDER BY order_id", + "VALUES (1, 'Alice', 50), (2, 'Bob', 75), (3, 'Alice', 25)"); + + assertUpdate("DROP MATERIALIZED VIEW mv_customer_orders"); + assertUpdate("DROP TABLE customers"); + assertUpdate("DROP TABLE customer_orders"); + } + + @Test + public void testRefreshMaterializedView() + { + assertUpdate("CREATE TABLE refresh_base (id BIGINT, value BIGINT)"); + assertUpdate("INSERT INTO refresh_base VALUES (1, 100), (2, 200)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_refresh AS SELECT id, value FROM refresh_base"); + + assertQuery("SELECT COUNT(*) FROM mv_refresh", "SELECT 2"); + assertQuery("SELECT * FROM mv_refresh ORDER BY id", "VALUES (1, 100), (2, 200)"); + + assertUpdate("INSERT INTO refresh_base VALUES (3, 300)", 1); + + assertQuery("SELECT COUNT(*) FROM mv_refresh", "SELECT 3"); + + assertUpdate("REFRESH MATERIALIZED VIEW mv_refresh", 3); + + assertQuery("SELECT COUNT(*) FROM mv_refresh", "SELECT 3"); + assertQuery("SELECT * FROM mv_refresh ORDER BY id", + "VALUES (1, 100), (2, 200), (3, 300)"); + + assertUpdate("DROP MATERIALIZED VIEW mv_refresh"); + assertUpdate("DROP TABLE refresh_base"); + } + + @Test + public void testRefreshMaterializedViewWithAggregation() + { + assertUpdate("CREATE TABLE agg_refresh_base (category VARCHAR, value BIGINT)"); + assertUpdate("INSERT INTO agg_refresh_base VALUES ('A', 10), ('B', 20), ('A', 15)", 3); + + assertUpdate("CREATE MATERIALIZED VIEW mv_agg_refresh AS " + + "SELECT category, SUM(value) as total FROM agg_refresh_base GROUP BY category"); + + assertQuery("SELECT * FROM mv_agg_refresh ORDER BY category", + "VALUES ('A', 25), ('B', 20)"); + + assertUpdate("INSERT INTO agg_refresh_base VALUES ('A', 5), ('C', 30)", 2); + + assertUpdate("REFRESH MATERIALIZED VIEW mv_agg_refresh", 3); + + assertQuery("SELECT * FROM mv_agg_refresh ORDER BY category", + "VALUES ('A', 30), ('B', 20), ('C', 30)"); + + assertUpdate("DROP MATERIALIZED VIEW mv_agg_refresh"); + assertUpdate("DROP TABLE agg_refresh_base"); + } + + @Test + public void testRefreshNonExistentMaterializedView() + { + assertQueryFails("REFRESH MATERIALIZED VIEW mv_nonexistent", + ".*Materialized view .* does not exist.*"); + } + + @Test + public void testDropMaterializedView() + { + assertUpdate("CREATE TABLE drop_base (id BIGINT, value VARCHAR)"); + assertUpdate("INSERT INTO drop_base VALUES (1, 'test')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW mv_drop AS SELECT id, value FROM drop_base"); + + assertQuery("SELECT COUNT(*) FROM mv_drop", "SELECT 1"); + + assertUpdate("DROP MATERIALIZED VIEW mv_drop"); + + assertQuery("SELECT COUNT(*) FROM drop_base", "SELECT 1"); + + assertUpdate("DROP TABLE drop_base"); + } + + @Test + public void testDropNonExistentMaterializedView() + { + assertQueryFails("DROP MATERIALIZED VIEW mv_nonexistent", + ".*Materialized view .* does not exist.*"); + } + + @Test + public void testCreateMaterializedViewWithEmptyBaseTable() + { + assertUpdate("CREATE TABLE empty_base (id BIGINT, value VARCHAR)"); + + assertUpdate("CREATE MATERIALIZED VIEW mv_empty AS SELECT id, value FROM empty_base"); + + assertQuery("SELECT COUNT(*) FROM mv_empty", "SELECT 0"); + + assertUpdate("DROP MATERIALIZED VIEW mv_empty"); + assertUpdate("DROP TABLE empty_base"); + } + + @Test + public void testMultipleMaterializedViews() + { + assertUpdate("CREATE TABLE multi_base (id BIGINT, category VARCHAR, value BIGINT)"); + assertUpdate("INSERT INTO multi_base VALUES (1, 'A', 100), (2, 'B', 200), (3, 'A', 150)", 3); + + assertUpdate("CREATE MATERIALIZED VIEW mv_multi_1 AS SELECT id, value FROM multi_base WHERE category = 'A'"); + assertUpdate("CREATE MATERIALIZED VIEW mv_multi_2 AS SELECT category, SUM(value) as total FROM multi_base GROUP BY category"); + + assertQuery("SELECT COUNT(*) FROM mv_multi_1", "SELECT 2"); + assertQuery("SELECT * FROM mv_multi_1 ORDER BY id", "VALUES (1, 100), (3, 150)"); + + assertQuery("SELECT COUNT(*) FROM mv_multi_2", "SELECT 2"); + assertQuery("SELECT * FROM mv_multi_2 ORDER BY category", + "VALUES ('A', 250), ('B', 200)"); + + assertUpdate("DROP MATERIALIZED VIEW mv_multi_1"); + assertUpdate("DROP MATERIALIZED VIEW mv_multi_2"); + assertUpdate("DROP TABLE multi_base"); + } + + @Test + public void testCreateMaterializedViewWithMultiTableJoin() + { + assertUpdate("CREATE TABLE orders (order_id BIGINT, customer_id BIGINT, product_id BIGINT, quantity BIGINT)"); + assertUpdate("CREATE TABLE customers (customer_id BIGINT, customer_name VARCHAR, region VARCHAR)"); + assertUpdate("CREATE TABLE products (product_id BIGINT, product_name VARCHAR, unit_price BIGINT)"); + + assertUpdate("INSERT INTO orders VALUES (1, 100, 1, 2), (2, 200, 2, 1), (3, 100, 2, 3)", 3); + assertUpdate("INSERT INTO customers VALUES (100, 'Alice', 'East'), (200, 'Bob', 'West')", 2); + assertUpdate("INSERT INTO products VALUES (1, 'Widget', 50), (2, 'Gadget', 75)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_order_details AS " + + "SELECT o.order_id, c.customer_name, c.region, p.product_name, o.quantity, " + + "CAST(p.unit_price * o.quantity AS BIGINT) as total_price " + + "FROM orders o " + + "JOIN customers c ON o.customer_id = c.customer_id " + + "JOIN products p ON o.product_id = p.product_id"); + + assertQuery("SELECT COUNT(*) FROM mv_order_details", "SELECT 3"); + assertQuery("SELECT order_id, customer_name, product_name, total_price FROM mv_order_details ORDER BY order_id", + "VALUES (1, 'Alice', 'Widget', 100), (2, 'Bob', 'Gadget', 75), (3, 'Alice', 'Gadget', 225)"); + + assertUpdate("DROP MATERIALIZED VIEW mv_order_details"); + assertUpdate("DROP TABLE products"); + assertUpdate("DROP TABLE customers"); + assertUpdate("DROP TABLE orders"); + } + + @Test + public void testRefreshMaterializedViewAfterBaseTableDropped() + { + assertUpdate("CREATE TABLE temp_base (id BIGINT, value VARCHAR)"); + assertUpdate("INSERT INTO temp_base VALUES (1, 'test'), (2, 'data')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_temp AS SELECT id, value FROM temp_base"); + + assertQuery("SELECT COUNT(*) FROM mv_temp", "SELECT 2"); + + assertUpdate("DROP TABLE temp_base"); + + assertQueryFails("REFRESH MATERIALIZED VIEW mv_temp", + ".*Table .* does not exist.*"); + + assertUpdate("DROP MATERIALIZED VIEW mv_temp"); + } + + @Test + public void testMaterializedViewBecomesUnqueryableAfterBaseTableDropped() + { + assertUpdate("CREATE TABLE persist_base (id BIGINT, value VARCHAR)"); + assertUpdate("INSERT INTO persist_base VALUES (1, 'test'), (2, 'data')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_persist AS SELECT id, value FROM persist_base"); + + assertQuery("SELECT COUNT(*) FROM mv_persist", "SELECT 2"); + assertQuery("SELECT * FROM mv_persist ORDER BY id", "VALUES (1, 'test'), (2, 'data')"); + + assertUpdate("INSERT INTO persist_base VALUES (3, 'more')", 1); + assertUpdate("REFRESH MATERIALIZED VIEW mv_persist", 3); + + assertQuery("SELECT COUNT(*) FROM mv_persist", "SELECT 3"); + assertQuery("SELECT * FROM mv_persist ORDER BY id", "VALUES (1, 'test'), (2, 'data'), (3, 'more')"); + + assertUpdate("DROP TABLE persist_base"); + + assertQueryFails("SELECT COUNT(*) FROM mv_persist", + ".*Table .* does not exist.*"); + + assertQueryFails("REFRESH MATERIALIZED VIEW mv_persist", + ".*Table .* does not exist.*"); + + assertUpdate("DROP MATERIALIZED VIEW mv_persist"); + } + + @Test + public void testMaterializedViewStalenessDetection() + { + assertUpdate("CREATE TABLE base (id BIGINT, value VARCHAR)"); + assertUpdate("INSERT INTO base VALUES (1, 'first')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW mv AS SELECT id, value FROM base"); + + assertUpdate("REFRESH MATERIALIZED VIEW mv", 1); + assertQuery("SELECT * FROM mv", "VALUES (1, 'first')"); + + assertUpdate("INSERT INTO base VALUES (2, 'second')", 1); + assertQuery("SELECT COUNT(*) FROM mv", "SELECT 2"); + assertUpdate("REFRESH MATERIALIZED VIEW mv", 2); + assertQuery("SELECT COUNT(*) FROM mv", "SELECT 2"); + + assertUpdate("INSERT INTO base VALUES (3, 'third')", 1); + assertQuery("SELECT COUNT(*) FROM mv", "SELECT 3"); + assertUpdate("REFRESH MATERIALIZED VIEW mv", 3); + assertQuery("SELECT COUNT(*) FROM mv", "SELECT 3"); + + assertUpdate("DROP MATERIALIZED VIEW mv"); + assertUpdate("DROP TABLE base"); + } + + @Test + public void testMaterializedViewWithMultipleBaseTables() + { + assertUpdate("CREATE TABLE orders (order_id BIGINT, customer_id BIGINT)"); + assertUpdate("CREATE TABLE customers (customer_id BIGINT, name VARCHAR)"); + + assertUpdate("INSERT INTO orders VALUES (1, 100)", 1); + assertUpdate("INSERT INTO customers VALUES (100, 'Alice')", 1); + + assertUpdate("CREATE MATERIALIZED VIEW mv_join AS " + + "SELECT o.order_id, c.name FROM orders o JOIN customers c ON o.customer_id = c.customer_id"); + assertQuery("SELECT * FROM mv_join", "VALUES (1, 'Alice')"); + assertUpdate("REFRESH MATERIALIZED VIEW mv_join", 1); + assertQuery("SELECT * FROM mv_join", "VALUES (1, 'Alice')"); + + assertUpdate("INSERT INTO orders VALUES (2, 100)", 1); + assertQuery("SELECT COUNT(*) FROM mv_join", "SELECT 2"); + assertUpdate("REFRESH MATERIALIZED VIEW mv_join", 2); + assertQuery("SELECT COUNT(*) FROM mv_join", "SELECT 2"); + + assertUpdate("INSERT INTO customers VALUES (200, 'Bob')", 1); + assertUpdate("INSERT INTO orders VALUES (3, 200)", 1); + assertQuery("SELECT COUNT(*) FROM mv_join", "SELECT 3"); + assertUpdate("REFRESH MATERIALIZED VIEW mv_join", 3); + assertQuery("SELECT COUNT(*) FROM mv_join", "SELECT 3"); + + assertUpdate("DROP MATERIALIZED VIEW mv_join"); + assertUpdate("DROP TABLE customers"); + assertUpdate("DROP TABLE orders"); + } + + @Test + public void testMultipleMaterializedViewsIndependentTracking() + { + assertUpdate("CREATE TABLE shared (id BIGINT, category VARCHAR)"); + assertUpdate("INSERT INTO shared VALUES (1, 'A'), (2, 'B')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv1 AS SELECT * FROM shared WHERE category = 'A'"); + assertUpdate("CREATE MATERIALIZED VIEW mv2 AS SELECT * FROM shared WHERE category = 'B'"); + + assertUpdate("REFRESH MATERIALIZED VIEW mv1", 1); + assertUpdate("REFRESH MATERIALIZED VIEW mv2", 1); + + assertUpdate("INSERT INTO shared VALUES (3, 'A'), (4, 'B')", 2); + + assertQuery("SELECT COUNT(*) FROM mv1", "SELECT 2"); + assertUpdate("REFRESH MATERIALIZED VIEW mv1", 2); + assertQuery("SELECT COUNT(*) FROM mv1", "SELECT 2"); + + assertQuery("SELECT COUNT(*) FROM mv2", "SELECT 2"); + assertUpdate("REFRESH MATERIALIZED VIEW mv2", 2); + assertQuery("SELECT COUNT(*) FROM mv2", "SELECT 2"); + + assertUpdate("DROP MATERIALIZED VIEW mv1"); + assertUpdate("DROP MATERIALIZED VIEW mv2"); + assertUpdate("DROP TABLE shared"); + } + + @Test + public void testMaterializedViewWithDataConsistencyDisabled() + { + assertUpdate("CREATE TABLE consistency_test (id BIGINT, value VARCHAR)"); + assertUpdate("INSERT INTO consistency_test VALUES (1, 'initial'), (2, 'data')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_consistency AS SELECT id, value FROM consistency_test"); + + Session session = Session.builder(getSession()) + .setSystemProperty("materialized_view_data_consistency_enabled", "false") + .build(); + + assertQuery(session, "SELECT COUNT(*) FROM mv_consistency", "SELECT 0"); + + assertUpdate("REFRESH MATERIALIZED VIEW mv_consistency", 2); + + assertQuery(session, "SELECT COUNT(*) FROM mv_consistency", "SELECT 2"); + assertQuery(session, "SELECT * FROM mv_consistency ORDER BY id", + "VALUES (1, 'initial'), (2, 'data')"); + + assertUpdate("INSERT INTO consistency_test VALUES (3, 'new')", 1); + + assertQuery(session, "SELECT COUNT(*) FROM mv_consistency", "SELECT 2"); + + assertUpdate("REFRESH MATERIALIZED VIEW mv_consistency", 3); + assertQuery(session, "SELECT COUNT(*) FROM mv_consistency", "SELECT 3"); + + assertUpdate("DROP MATERIALIZED VIEW mv_consistency"); + assertUpdate("DROP TABLE consistency_test"); + } + + @Test + public void testMaterializedViewStalenessWithDataConsistencyDisabled() + { + assertUpdate("CREATE TABLE stale_base (id BIGINT, category VARCHAR, amount BIGINT)"); + assertUpdate("INSERT INTO stale_base VALUES (1, 'A', 100), (2, 'B', 200)", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_stale AS " + + "SELECT category, SUM(amount) as total FROM stale_base GROUP BY category"); + + assertUpdate("REFRESH MATERIALIZED VIEW mv_stale", 2); + + Session sessionWithConsistencyDisabled = Session.builder(getSession()) + .setSystemProperty("materialized_view_data_consistency_enabled", "false") + .build(); + + assertQuery(sessionWithConsistencyDisabled, "SELECT * FROM mv_stale ORDER BY category", + "VALUES ('A', 100), ('B', 200)"); + + assertUpdate("INSERT INTO stale_base VALUES (3, 'A', 50), (4, 'C', 150)", 2); + + assertQuery(sessionWithConsistencyDisabled, "SELECT * FROM mv_stale ORDER BY category", + "VALUES ('A', 100), ('B', 200)"); + + assertUpdate("REFRESH MATERIALIZED VIEW mv_stale", 3); + assertQuery(sessionWithConsistencyDisabled, "SELECT * FROM mv_stale ORDER BY category", + "VALUES ('A', 150), ('B', 200), ('C', 150)"); + + assertUpdate("DROP MATERIALIZED VIEW mv_stale"); + assertUpdate("DROP TABLE stale_base"); + } + + @Test + public void testMaterializedViewBecomesUnqueryableAfterBaseTableRenamed() + { + assertUpdate("CREATE TABLE rename_base (id BIGINT, value VARCHAR)"); + assertUpdate("INSERT INTO rename_base VALUES (1, 'test'), (2, 'data')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_rename AS SELECT id, value FROM rename_base"); + + assertQuery("SELECT COUNT(*) FROM mv_rename", "SELECT 2"); + assertQuery("SELECT * FROM mv_rename ORDER BY id", "VALUES (1, 'test'), (2, 'data')"); + + assertUpdate("INSERT INTO rename_base VALUES (3, 'more')", 1); + assertUpdate("REFRESH MATERIALIZED VIEW mv_rename", 3); + + assertQuery("SELECT COUNT(*) FROM mv_rename", "SELECT 3"); + assertQuery("SELECT * FROM mv_rename ORDER BY id", "VALUES (1, 'test'), (2, 'data'), (3, 'more')"); + + assertUpdate("ALTER TABLE rename_base RENAME TO rename_base_new"); + + assertQueryFails("SELECT COUNT(*) FROM mv_rename", + ".*Table .* does not exist.*"); + + assertQueryFails("REFRESH MATERIALIZED VIEW mv_rename", + ".*Table .* does not exist.*"); + + assertUpdate("DROP MATERIALIZED VIEW mv_rename"); + assertUpdate("DROP TABLE rename_base_new"); + } + + @Test + public void testMaterializedViewWithDataConsistencyDisabledAfterBaseTableDropped() + { + assertUpdate("CREATE TABLE drop_consistency_test (id BIGINT, value VARCHAR)"); + assertUpdate("INSERT INTO drop_consistency_test VALUES (1, 'initial'), (2, 'data')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_drop_consistency AS SELECT id, value FROM drop_consistency_test"); + + assertUpdate("REFRESH MATERIALIZED VIEW mv_drop_consistency", 2); + + Session session = Session.builder(getSession()) + .setSystemProperty("materialized_view_data_consistency_enabled", "false") + .build(); + + assertQuery(session, "SELECT COUNT(*) FROM mv_drop_consistency", "SELECT 2"); + assertQuery(session, "SELECT * FROM mv_drop_consistency ORDER BY id", + "VALUES (1, 'initial'), (2, 'data')"); + + assertUpdate("DROP TABLE drop_consistency_test"); + + assertQuery(session, "SELECT COUNT(*) FROM mv_drop_consistency", "SELECT 2"); + assertQuery(session, "SELECT * FROM mv_drop_consistency ORDER BY id", + "VALUES (1, 'initial'), (2, 'data')"); + + assertQueryFails("REFRESH MATERIALIZED VIEW mv_drop_consistency", + ".*Table .* does not exist.*"); + + assertUpdate("DROP MATERIALIZED VIEW mv_drop_consistency"); + } + + @Test + public void testMaterializedViewWithDataConsistencyDisabledAfterBaseTableRenamed() + { + assertUpdate("CREATE TABLE rename_consistency_test (id BIGINT, value VARCHAR)"); + assertUpdate("INSERT INTO rename_consistency_test VALUES (1, 'initial'), (2, 'data')", 2); + + assertUpdate("CREATE MATERIALIZED VIEW mv_rename_consistency AS SELECT id, value FROM rename_consistency_test"); + + assertUpdate("REFRESH MATERIALIZED VIEW mv_rename_consistency", 2); + + Session session = Session.builder(getSession()) + .setSystemProperty("materialized_view_data_consistency_enabled", "false") + .build(); + + assertQuery(session, "SELECT COUNT(*) FROM mv_rename_consistency", "SELECT 2"); + assertQuery(session, "SELECT * FROM mv_rename_consistency ORDER BY id", + "VALUES (1, 'initial'), (2, 'data')"); + + assertUpdate("ALTER TABLE rename_consistency_test RENAME TO rename_consistency_test_new"); + + assertQuery(session, "SELECT COUNT(*) FROM mv_rename_consistency", "SELECT 2"); + assertQuery(session, "SELECT * FROM mv_rename_consistency ORDER BY id", + "VALUES (1, 'initial'), (2, 'data')"); + + assertQueryFails("REFRESH MATERIALIZED VIEW mv_rename_consistency", + ".*Table .* does not exist.*"); + + assertUpdate("DROP MATERIALIZED VIEW mv_rename_consistency"); + assertUpdate("DROP TABLE rename_consistency_test_new"); + } + + @Test + public void testRefreshMaterializedViewWithWhereClause() + { + assertUpdate("CREATE TABLE where_base (id BIGINT, category VARCHAR, value BIGINT)"); + assertUpdate("INSERT INTO where_base VALUES (1, 'A', 100), (2, 'B', 200), (3, 'A', 150)", 3); + + assertUpdate("CREATE MATERIALIZED VIEW mv_where AS SELECT id, category, value FROM where_base"); + + assertUpdate("REFRESH MATERIALIZED VIEW mv_where", 3); + + assertQueryFails("REFRESH MATERIALIZED VIEW mv_where WHERE category = 'A'", + ".*WHERE clause in REFRESH MATERIALIZED VIEW is not supported.*"); + + assertUpdate("DROP MATERIALIZED VIEW mv_where"); + assertUpdate("DROP TABLE where_base"); + } +}