diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobMgr.java b/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobMgr.java index 2b41b8cf06a45d..09a08d25b9e6a6 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobMgr.java @@ -37,10 +37,13 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.starrocks.analysis.IntLiteral; import com.starrocks.analysis.StringLiteral; import com.starrocks.analysis.TableName; import com.starrocks.analysis.TableRef; +import com.starrocks.authentication.AuthenticationManager; +import com.starrocks.catalog.BaseTableInfo; import com.starrocks.catalog.ColocateTableIndex; import com.starrocks.catalog.Column; import com.starrocks.catalog.DataProperty; @@ -68,6 +71,7 @@ import com.starrocks.common.UserException; import com.starrocks.common.util.DynamicPartitionUtil; import com.starrocks.common.util.PropertyAnalyzer; +import com.starrocks.persist.AlterMaterializedViewStatusLog; import com.starrocks.persist.AlterViewInfo; import com.starrocks.persist.BatchModifyPartitionsInfo; import com.starrocks.persist.ChangeMaterializedViewRefreshSchemeLog; @@ -79,6 +83,7 @@ import com.starrocks.persist.metablock.SRMetaBlockException; import com.starrocks.persist.metablock.SRMetaBlockReader; import com.starrocks.persist.metablock.SRMetaBlockWriter; +import com.starrocks.privilege.PrivilegeBuiltinConstants; import com.starrocks.qe.ConnectContext; import com.starrocks.qe.ShowResultSet; import com.starrocks.scheduler.Constants; @@ -87,6 +92,9 @@ import com.starrocks.scheduler.TaskManager; import com.starrocks.scheduler.mv.MVManager; import com.starrocks.server.GlobalStateMgr; +import com.starrocks.sql.analyzer.Analyzer; +import com.starrocks.sql.analyzer.MaterializedViewAnalyzer; +import com.starrocks.sql.analyzer.SemanticException; import com.starrocks.sql.analyzer.SetStmtAnalyzer; import com.starrocks.sql.ast.AddPartitionClause; import com.starrocks.sql.ast.AlterClause; @@ -106,17 +114,21 @@ import com.starrocks.sql.ast.ModifyPartitionClause; import com.starrocks.sql.ast.ModifyTablePropertiesClause; import com.starrocks.sql.ast.PartitionRenameClause; +import com.starrocks.sql.ast.QueryStatement; import com.starrocks.sql.ast.RefreshSchemeDesc; import com.starrocks.sql.ast.ReplacePartitionClause; import com.starrocks.sql.ast.RollupRenameClause; import com.starrocks.sql.ast.SetListItem; import com.starrocks.sql.ast.SetStmt; +import com.starrocks.sql.ast.StatementBase; import com.starrocks.sql.ast.SwapTableClause; import com.starrocks.sql.ast.SystemVariable; import com.starrocks.sql.ast.TableRenameClause; import com.starrocks.sql.ast.TruncatePartitionClause; import com.starrocks.sql.ast.TruncateTableStmt; +import com.starrocks.sql.ast.UserIdentity; import com.starrocks.sql.optimizer.Utils; +import com.starrocks.sql.parser.SqlParser; import com.starrocks.thrift.TTabletMetaType; import com.starrocks.thrift.TTabletType; import org.apache.logging.log4j.LogManager; @@ -264,6 +276,7 @@ public void processAlterMaterializedView(AlterMaterializedViewStmt stmt) final String oldMvName = mvName.getTbl(); final String newMvName = stmt.getNewMvName(); final RefreshSchemeDesc refreshSchemeDesc = stmt.getRefreshSchemeDesc(); + final String status = stmt.getStatus(); ModifyTablePropertiesClause modifyTablePropertiesClause = stmt.getModifyTablePropertiesClause(); String dbName = mvName.getDb(); Database db = GlobalStateMgr.getCurrentState().getDb(dbName); @@ -303,6 +316,29 @@ public void processAlterMaterializedView(AlterMaterializedViewStmt stmt) } catch (AnalysisException ae) { throw new DdlException(ae.getMessage()); } + } else if (status != null) { + if (AlterMaterializedViewStmt.ACTIVE.equalsIgnoreCase(status)) { + if (materializedView.isActive()) { + return; + } + processChangeMaterializedViewStatus(materializedView, status); + GlobalStateMgr.getCurrentState().getLocalMetastore() + .refreshMaterializedView(dbName, materializedView.getName(), true, null, + Constants.TaskRunPriority.NORMAL.value(), true, false); + } else if (AlterMaterializedViewStmt.INACTIVE.equalsIgnoreCase(status)) { + if (!materializedView.isActive()) { + return; + } + LOG.warn("Setting the materialized view {}({}) to inactive because " + + "user use alter materialized view set status to inactive", + materializedView.getName(), materializedView.getId()); + processChangeMaterializedViewStatus(materializedView, status); + } else { + throw new DdlException("Unsupported modification materialized view status:" + status); + } + AlterMaterializedViewStatusLog log = new AlterMaterializedViewStatusLog(materializedView.getDbId(), + materializedView.getId(), status); + GlobalStateMgr.getCurrentState().getEditLog().logAlterMvStatus(log); } else { throw new DdlException("Unsupported modification for materialized view"); } @@ -313,6 +349,48 @@ public void processAlterMaterializedView(AlterMaterializedViewStmt stmt) } } + private void processChangeMaterializedViewStatus(MaterializedView materializedView, String status) { + if (AlterMaterializedViewStmt.ACTIVE.equalsIgnoreCase(status)) { + String viewDefineSql = materializedView.getViewDefineSql(); + ConnectContext context = new ConnectContext(); + context.setQualifiedUser(AuthenticationManager.ROOT_USER); + context.setCurrentUserIdentity(UserIdentity.ROOT); + context.setCurrentRoleIds(Sets.newHashSet(PrivilegeBuiltinConstants.ROOT_ROLE_ID)); + + List statementBaseList = SqlParser.parse(viewDefineSql, context.getSessionVariable()); + QueryStatement queryStatement = (QueryStatement) statementBaseList.get(0); + try { + Analyzer.analyze(queryStatement, context); + } catch (SemanticException e) { + throw new SemanticException("Can not active materialized view [" + materializedView.getName() + + "] because analyze materialized view define sql: \n\n" + viewDefineSql + + "\n\nCause an error: " + e.getDetailMsg()); + } + + List baseTableInfos = MaterializedViewAnalyzer.getAndCheckBaseTables(queryStatement); + materializedView.setBaseTableInfos(baseTableInfos); + materializedView.getRefreshScheme().getAsyncRefreshContext().clearVisibleVersionMap(); + GlobalStateMgr.getCurrentState().updateBaseTableRelatedMv(materializedView.getDbId(), + materializedView, baseTableInfos); + materializedView.setActive(true); + } else if (AlterMaterializedViewStmt.INACTIVE.equalsIgnoreCase(status)) { + materializedView.setInactiveAndReason("user use alter materialized view set status to inactive"); + } + } + + public void replayAlterMaterializedViewStatus(AlterMaterializedViewStatusLog log) { + long dbId = log.getDbId(); + long tableId = log.getTableId(); + Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + db.writeLock(); + try { + MaterializedView mv = (MaterializedView) db.getTable(tableId); + processChangeMaterializedViewStatus(mv, log.getStatus()); + } finally { + db.writeUnlock(); + } + } + private void processModifyTableProperties(ModifyTablePropertiesClause modifyTablePropertiesClause, Database db, MaterializedView materializedView) throws AnalysisException { diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java b/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java index ba1db1616788d7..8ce4d1e04d0f62 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedView.java @@ -177,6 +177,11 @@ public Map> getBaseTableInfoVisibl return baseTableInfoVisibleVersionMap; } + public void clearVisibleVersionMap() { + this.baseTableInfoVisibleVersionMap.clear(); + this.baseTableVisibleVersionMap.clear(); + } + public boolean isDefineStartTime() { return defineStartTime; } diff --git a/fe/fe-core/src/main/java/com/starrocks/journal/JournalEntity.java b/fe/fe-core/src/main/java/com/starrocks/journal/JournalEntity.java index a16468f8aebe68..16be85afeaa216 100644 --- a/fe/fe-core/src/main/java/com/starrocks/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/com/starrocks/journal/JournalEntity.java @@ -67,6 +67,7 @@ import com.starrocks.persist.AddPartitionsInfo; import com.starrocks.persist.AddPartitionsInfoV2; import com.starrocks.persist.AlterLoadJobOperationLog; +import com.starrocks.persist.AlterMaterializedViewStatusLog; import com.starrocks.persist.AlterRoutineLoadJobOperationLog; import com.starrocks.persist.AlterUserInfo; import com.starrocks.persist.AlterViewInfo; @@ -323,6 +324,10 @@ public void readFields(DataInput in) throws IOException { data = RenameMaterializedViewLog.read(in); isRead = true; break; + case OperationType.OP_ALTER_MATERIALIZED_VIEW_STATUS: + data = AlterMaterializedViewStatusLog.read(in); + isRead = true; + break; case OperationType.OP_BACKUP_JOB: { data = AbstractJob.read(in); isRead = true; diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/AlterMaterializedViewStatusLog.java b/fe/fe-core/src/main/java/com/starrocks/persist/AlterMaterializedViewStatusLog.java new file mode 100644 index 00000000000000..dd938634b50b4e --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/persist/AlterMaterializedViewStatusLog.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.starrocks.persist; + +import com.google.gson.annotations.SerializedName; +import com.starrocks.common.io.Text; +import com.starrocks.common.io.Writable; +import com.starrocks.persist.gson.GsonUtils; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +public class AlterMaterializedViewStatusLog implements Writable { + + @SerializedName(value = "dbId") + private long dbId; + @SerializedName(value = "tableId") + private long tableId; + @SerializedName(value = "status") + private String status; + + public AlterMaterializedViewStatusLog(long dbId, long tableId, String status) { + this.dbId = dbId; + this.tableId = tableId; + this.status = status; + } + + public long getDbId() { + return dbId; + } + + public void setDbId(long dbId) { + this.dbId = dbId; + } + + public long getTableId() { + return tableId; + } + + public void setTableId(long tableId) { + this.tableId = tableId; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, GsonUtils.GSON.toJson(this)); + } + + public static AlterMaterializedViewStatusLog read(DataInput in) throws IOException { + return GsonUtils.GSON.fromJson(Text.readString(in), AlterMaterializedViewStatusLog.class); + } + +} \ No newline at end of file diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java b/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java index 42ef71831cb7b3..c6003d480afaf4 100644 --- a/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java +++ b/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java @@ -327,6 +327,12 @@ public static void loadJournal(GlobalStateMgr globalStateMgr, JournalEntity jour globalStateMgr.replayAlterMaterializedViewProperties(opCode, log); break; } + case OperationType.OP_ALTER_MATERIALIZED_VIEW_STATUS: { + AlterMaterializedViewStatusLog log = + (AlterMaterializedViewStatusLog) journal.getData(); + globalStateMgr.replayAlterMaterializedViewStatus(log); + break; + } case OperationType.OP_RENAME_MATERIALIZED_VIEW: { RenameMaterializedViewLog log = (RenameMaterializedViewLog) journal.getData(); globalStateMgr.replayRenameMaterializedView(log); @@ -1629,6 +1635,10 @@ public void logInsertOverwriteStateChange(InsertOverwriteStateChangeInfo info) { logEdit(OperationType.OP_INSERT_OVERWRITE_STATE_CHANGE, info); } + public void logAlterMvStatus(AlterMaterializedViewStatusLog log) { + logEdit(OperationType.OP_ALTER_MATERIALIZED_VIEW_STATUS, log); + } + public void logMvRename(RenameMaterializedViewLog log) { logEdit(OperationType.OP_RENAME_MATERIALIZED_VIEW, log); } diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java b/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java index a96f119c19e7ec..929aed63f4623b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java +++ b/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java @@ -266,6 +266,7 @@ public class OperationType { public static final short OP_CREATE_MATERIALIZED_VIEW = 10094; public static final short OP_CREATE_INSERT_OVERWRITE = 10095; public static final short OP_INSERT_OVERWRITE_STATE_CHANGE = 10096; + public static final short OP_ALTER_MATERIALIZED_VIEW_STATUS = 10097; // manage system node info 10101 ~ 10120 public static final short OP_UPDATE_FRONTEND = 10101; diff --git a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java index b67ea890a0cfdb..9c1dd4f871d465 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java @@ -179,6 +179,7 @@ import com.starrocks.mysql.privilege.Auth; import com.starrocks.mysql.privilege.AuthUpgrader; import com.starrocks.mysql.privilege.PrivPredicate; +import com.starrocks.persist.AlterMaterializedViewStatusLog; import com.starrocks.persist.AuthUpgradeInfo; import com.starrocks.persist.BackendIdsUpdateInfo; import com.starrocks.persist.BackendTabletsInfo; @@ -1566,31 +1567,8 @@ private void processMvRelatedMeta() { for (String dbName : dbNames) { Database db = metadataMgr.getDb(InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME, dbName); for (MaterializedView mv : db.getMaterializedViews()) { - for (BaseTableInfo baseTableInfo : mv.getBaseTableInfos()) { - Table table = baseTableInfo.getTable(); - if (table == null) { - LOG.warn("Setting the materialized view {}({}) to invalid because " + - "the table {} was not exist.", mv.getName(), mv.getId(), baseTableInfo.getTableName()); - mv.setInactiveAndReason("base table dropped: " + baseTableInfo.getTableId()); - continue; - } - if (table instanceof MaterializedView && !((MaterializedView) table).isActive()) { - MaterializedView baseMv = (MaterializedView) table; - LOG.warn("Setting the materialized view {}({}) to invalid because " + - "the materialized view{}({}) is invalid.", mv.getName(), mv.getId(), - baseMv.getName(), baseMv.getId()); - mv.setInactiveAndReason("base mv is not active: " + baseMv.getName()); - continue; - } - MvId mvId = new MvId(db.getId(), mv.getId()); - table.addRelatedMaterializedView(mvId); - if (!table.isNativeTableOrMaterializedView()) { - connectorTblMetaInfoMgr.addConnectorTableInfo(baseTableInfo.getCatalogName(), - baseTableInfo.getDbName(), baseTableInfo.getTableIdentifier(), - ConnectorTableInfo.builder().setRelatedMaterializedViews( - Sets.newHashSet(mvId)).build()); - } - } + List baseTableInfos = mv.getBaseTableInfos(); + updateBaseTableRelatedMv(db.getId(), mv, baseTableInfos); } } @@ -1598,6 +1576,34 @@ private void processMvRelatedMeta() { LOG.info("finish processing all tables' related materialized views in {}ms", duration); } + public void updateBaseTableRelatedMv(Long dbId, MaterializedView mv, List baseTableInfos) { + for (BaseTableInfo baseTableInfo : baseTableInfos) { + Table table = baseTableInfo.getTable(); + if (table == null) { + LOG.warn("Setting the materialized view {}({}) to invalid because " + + "the table {} was not exist.", mv.getName(), mv.getId(), baseTableInfo.getTableName()); + mv.setInactiveAndReason("base table dropped: " + baseTableInfo.getTableId()); + continue; + } + if (table instanceof MaterializedView && !((MaterializedView) table).isActive()) { + MaterializedView baseMv = (MaterializedView) table; + LOG.warn("Setting the materialized view {}({}) to invalid because " + + "the materialized view{}({}) is invalid.", mv.getName(), mv.getId(), + baseMv.getName(), baseMv.getId()); + mv.setInactiveAndReason("base mv is not active: " + baseMv.getName()); + continue; + } + MvId mvId = new MvId(dbId, mv.getId()); + table.addRelatedMaterializedView(mvId); + if (!table.isNativeTableOrMaterializedView()) { + connectorTblMetaInfoMgr.addConnectorTableInfo(baseTableInfo.getCatalogName(), + baseTableInfo.getDbName(), baseTableInfo.getTableIdentifier(), + ConnectorTableInfo.builder().setRelatedMaterializedViews( + Sets.newHashSet(mvId)).build()); + } + } + } + public long loadVersion(DataInputStream dis, long checksum) throws IOException { // for new format, version schema is [starrocksMetaVersion], and the int value must be positive // for old format, version schema is [-1, metaVersion, starrocksMetaVersion] @@ -3291,6 +3297,12 @@ public void replayAlterMaterializedViewProperties(short opCode, ModifyTablePrope this.alterJobMgr.replayAlterMaterializedViewProperties(opCode, log); } + + public void replayAlterMaterializedViewStatus(AlterMaterializedViewStatusLog log) { + this.alterJobMgr.replayAlterMaterializedViewStatus(log); + } + + /* * used for handling CancelAlterStmt (for client is the CANCEL ALTER * command). including SchemaChangeHandler and RollupHandler diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java index 5aa7b688f4bb4c..f44c207ece63fb 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java @@ -124,6 +124,84 @@ public static void analyze(StatementBase stmt, ConnectContext session) { new MaterializedViewAnalyzerVisitor().visit(stmt, session); } + public static List getAndCheckBaseTables(QueryStatement queryStatement) { + List baseTableInfos = Lists.newArrayList(); + processBaseTables(queryStatement, baseTableInfos); + Set baseTableInfoSet = Sets.newHashSet(baseTableInfos); + baseTableInfos.clear(); + baseTableInfos.addAll(baseTableInfoSet); + return baseTableInfos; + } + + private static void processBaseTables(QueryStatement queryStatement, List baseTableInfos) { + Map tableNameTableMap = AnalyzerUtils.collectAllConnectorTableAndView(queryStatement); + tableNameTableMap.forEach((tableNameInfo, table) -> { + Preconditions.checkState(table != null, "Materialized view base table is null"); + if (!isSupportBasedOnTable(table)) { + throw new SemanticException("Create materialized view do not support the table type: " + + table.getType(), tableNameInfo.getPos()); + } + if (table instanceof MaterializedView && !((MaterializedView) table).isActive()) { + throw new SemanticException( + "Create materialized view from inactive materialized view: " + table.getName(), + tableNameInfo.getPos()); + } + + if (table.isView()) { + return; + } + + if (isExternalTableFromResource(table)) { + throw new SemanticException( + "Only supports creating materialized views based on the external table " + + "which created by catalog", tableNameInfo.getPos()); + } + Database database = GlobalStateMgr.getCurrentState().getMetadataMgr().getDb(tableNameInfo.getCatalog(), + tableNameInfo.getDb()); + if (isInternalCatalog(tableNameInfo.getCatalog())) { + baseTableInfos.add(new BaseTableInfo(database.getId(), database.getFullName(), + table.getId())); + } else { + baseTableInfos.add(new BaseTableInfo(tableNameInfo.getCatalog(), + tableNameInfo.getDb(), table.getTableIdentifier())); + } + }); + processViews(queryStatement, baseTableInfos); + } + + private static boolean isSupportBasedOnTable(Table table) { + return SUPPORTED_TABLE_TYPE.contains(table.getType()) || table instanceof OlapTable; + } + + private static boolean isExternalTableFromResource(Table table) { + if (table instanceof OlapTable) { + return false; + } else if (table instanceof JDBCTable || table instanceof MysqlTable) { + return false; + } else if (table instanceof HiveTable || table instanceof HudiTable) { + HiveMetaStoreTable hiveMetaStoreTable = (HiveMetaStoreTable) table; + String catalogName = hiveMetaStoreTable.getCatalogName(); + return Strings.isBlank(catalogName) || isResourceMappingCatalog(catalogName); + } else if (table instanceof IcebergTable) { + IcebergTable icebergTable = (IcebergTable) table; + String catalogName = icebergTable.getCatalogName(); + return Strings.isBlank(catalogName) || isResourceMappingCatalog(catalogName); + } else { + return true; + } + } + + private static void processViews(QueryStatement queryStatement, List baseTableInfos) { + List viewRelations = AnalyzerUtils.collectViewRelations(queryStatement); + if (viewRelations.isEmpty()) { + return; + } + Set viewRelationSet = Sets.newHashSet(viewRelations); + for (ViewRelation viewRelation : viewRelationSet) { + processBaseTables(viewRelation.getQueryStatement(), baseTableInfos); + } + } + static class MaterializedViewAnalyzerVisitor extends AstVisitor { public enum RefreshTimeUnit { @@ -133,28 +211,6 @@ public enum RefreshTimeUnit { SECOND } - private boolean isSupportBasedOnTable(Table table) { - return SUPPORTED_TABLE_TYPE.contains(table.getType()) || table instanceof OlapTable; - } - - private boolean isExternalTableFromResource(Table table) { - if (table instanceof OlapTable) { - return false; - } else if (table instanceof JDBCTable || table instanceof MysqlTable) { - return false; - } else if (table instanceof HiveTable || table instanceof HudiTable) { - HiveMetaStoreTable hiveMetaStoreTable = (HiveMetaStoreTable) table; - String catalogName = hiveMetaStoreTable.getCatalogName(); - return Strings.isBlank(catalogName) || isResourceMappingCatalog(catalogName); - } else if (table instanceof IcebergTable) { - IcebergTable icebergTable = (IcebergTable) table; - String catalogName = icebergTable.getCatalogName(); - return Strings.isBlank(catalogName) || isResourceMappingCatalog(catalogName); - } else { - return true; - } - } - @Override public Void visitCreateMaterializedViewStatement(CreateMaterializedViewStatement statement, ConnectContext context) { @@ -241,62 +297,6 @@ private Map getAllBaseTables(QueryStatement queryStatement, Co return result; } - private List getAndCheckBaseTables(QueryStatement queryStatement) { - List baseTableInfos = Lists.newArrayList(); - processBaseTables(queryStatement, baseTableInfos); - Set baseTableInfoSet = Sets.newHashSet(baseTableInfos); - baseTableInfos.clear(); - baseTableInfos.addAll(baseTableInfoSet); - return baseTableInfos; - } - - private void processBaseTables(QueryStatement queryStatement, List baseTableInfos) { - Map tableNameTableMap = AnalyzerUtils.collectAllConnectorTableAndView(queryStatement); - tableNameTableMap.forEach((tableNameInfo, table) -> { - Preconditions.checkState(table != null, "Materialized view base table is null"); - if (!isSupportBasedOnTable(table)) { - throw new SemanticException("Create materialized view do not support the table type: " + - table.getType(), tableNameInfo.getPos()); - } - if (table instanceof MaterializedView && !((MaterializedView) table).isActive()) { - throw new SemanticException( - "Create materialized view from inactive materialized view: " + table.getName(), - tableNameInfo.getPos()); - } - - if (table.isView()) { - return; - } - - if (isExternalTableFromResource(table)) { - throw new SemanticException( - "Only supports creating materialized views based on the external table " + - "which created by catalog", tableNameInfo.getPos()); - } - Database database = GlobalStateMgr.getCurrentState().getMetadataMgr().getDb(tableNameInfo.getCatalog(), - tableNameInfo.getDb()); - if (isInternalCatalog(tableNameInfo.getCatalog())) { - baseTableInfos.add(new BaseTableInfo(database.getId(), database.getFullName(), - table.getId())); - } else { - baseTableInfos.add(new BaseTableInfo(tableNameInfo.getCatalog(), - tableNameInfo.getDb(), table.getTableIdentifier())); - } - }); - processViews(queryStatement, baseTableInfos); - } - - private void processViews(QueryStatement queryStatement, List baseTableInfos) { - List viewRelations = AnalyzerUtils.collectViewRelations(queryStatement); - if (viewRelations.isEmpty()) { - return; - } - Set viewRelationSet = Sets.newHashSet(viewRelations); - for (ViewRelation viewRelation : viewRelationSet) { - processBaseTables(viewRelation.getQueryStatement(), baseTableInfos); - } - } - // TODO(murphy) implement // Plan the query statement and store in memory private void planMVQuery(CreateMaterializedViewStatement createStmt, QueryStatement query, ConnectContext ctx) { @@ -872,6 +872,11 @@ public Void visitAlterMaterializedViewStatement(AlterMaterializedViewStmt statem if (!(table instanceof MaterializedView)) { throw new SemanticException(mvName.getTbl() + " is not async materialized view", mvName.getPos()); } + } else if (statement.getStatus() != null) { + String status = statement.getStatus(); + if (!AlterMaterializedViewStmt.SUPPORTED_MV_STATUS.contains(status)) { + throw new SemanticException("Unsupported modification for materialized view status:" + status); + } } else { throw new SemanticException("Unsupported modification for materialized view"); } @@ -894,8 +899,9 @@ public Void visitRefreshMaterializedViewStatement(RefreshMaterializedViewStateme Preconditions.checkState(table instanceof MaterializedView); MaterializedView mv = (MaterializedView) table; if (!mv.isActive()) { - throw new SemanticException( - "Refresh materialized view failed because " + mv.getName() + " is not active", mvName.getPos()); + throw new SemanticException("Refresh materialized view failed because [" + mv.getName() + + "] is not active. You can try to active it with ALTER MATERIALIZED VIEW " + mv.getName() + + " ACTIVE; ", mvName.getPos()); } if (statement.getPartitionRangeDesc() == null) { return null; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/SemanticException.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/SemanticException.java index 1bf3f62b63c9e5..61dcfdb928ed1b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/SemanticException.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/SemanticException.java @@ -68,4 +68,8 @@ public String getMessage() { return builder.toString(); } + public String getDetailMsg() { + return detailMsg; + } + } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/AlterMaterializedViewStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AlterMaterializedViewStmt.java index e198bb7507bd41..4db3c1a1ca15a1 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/AlterMaterializedViewStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AlterMaterializedViewStmt.java @@ -15,9 +15,12 @@ package com.starrocks.sql.ast; +import com.google.common.collect.Sets; import com.starrocks.analysis.TableName; import com.starrocks.sql.parser.NodePosition; +import java.util.Set; + /** * 1.Support for modifying the way of refresh and the cycle of asynchronous refresh; * 2.Support for modifying the name of a materialized view; @@ -28,24 +31,28 @@ public class AlterMaterializedViewStmt extends DdlStmt { private final TableName mvName; private final String newMvName; private final RefreshSchemeDesc refreshSchemeDesc; - private final ModifyTablePropertiesClause modifyTablePropertiesClause; + private final String status; + public static final String ACTIVE = "active"; + public static final String INACTIVE = "inactive"; + public static final Set SUPPORTED_MV_STATUS = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); - public AlterMaterializedViewStmt(TableName mvName, String newMvName, - RefreshSchemeDesc refreshSchemeDesc, - ModifyTablePropertiesClause modifyTablePropertiesClause) { - this(mvName, newMvName, refreshSchemeDesc, modifyTablePropertiesClause, NodePosition.ZERO); + static { + SUPPORTED_MV_STATUS.add(ACTIVE); + SUPPORTED_MV_STATUS.add(INACTIVE); } public AlterMaterializedViewStmt(TableName mvName, String newMvName, RefreshSchemeDesc refreshSchemeDesc, ModifyTablePropertiesClause modifyTablePropertiesClause, + String status, NodePosition pos) { super(pos); this.mvName = mvName; this.newMvName = newMvName; this.refreshSchemeDesc = refreshSchemeDesc; this.modifyTablePropertiesClause = modifyTablePropertiesClause; + this.status = status; } public TableName getMvName() { @@ -64,6 +71,10 @@ public ModifyTablePropertiesClause getModifyTablePropertiesClause() { return modifyTablePropertiesClause; } + public String getStatus() { + return status; + } + @Override public R accept(AstVisitor visitor, C context) { return visitor.visitAlterMaterializedViewStatement(this, context); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java index 02b099747eae58..c614f227e72248 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java @@ -1637,7 +1637,11 @@ public ParseNode visitAlterMaterializedViewStatement( if (context.modifyTablePropertiesClause() != null) { modifyTablePropertiesClause = (ModifyTablePropertiesClause) visit(context.modifyTablePropertiesClause()); } - return new AlterMaterializedViewStmt(mvName, newMvName, refreshSchemeDesc, modifyTablePropertiesClause, + String status = null; + if (context.statusDesc() != null) { + status = context.statusDesc().getText(); + } + return new AlterMaterializedViewStmt(mvName, newMvName, refreshSchemeDesc, modifyTablePropertiesClause, status, createPos(context)); } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 index 28c647d337dd41..24173a1a099bae 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 @@ -583,6 +583,7 @@ dropMaterializedViewStatement alterMaterializedViewStatement : ALTER MATERIALIZED VIEW mvName=qualifiedName (refreshSchemeDesc | tableRenameClause | modifyTablePropertiesClause) + | ALTER MATERIALIZED VIEW mvName=qualifiedName statusDesc ; refreshMaterializedViewStatement @@ -2249,6 +2250,11 @@ refreshSchemeDesc | MANUAL) ; +statusDesc + : ACTIVE + | INACTIVE + ; + properties : PROPERTIES '(' property (',' property)* ')' ; @@ -2435,7 +2441,7 @@ number ; nonReserved - : ACCESS | AFTER | AGGREGATE | APPLY | ASYNC | AUTHORS | AVG | ADMIN | ANTI | AUTHENTICATION | AUTO_INCREMENT + : ACCESS | ACTIVE | AFTER | AGGREGATE | APPLY | ASYNC | AUTHORS | AVG | ADMIN | ANTI | AUTHENTICATION | AUTO_INCREMENT | BACKEND | BACKENDS | BACKUP | BEGIN | BITMAP_UNION | BLACKLIST | BINARY | BODY | BOOLEAN | BROKER | BUCKETS | BUILTIN | BASE | CAST | CANCEL | CATALOG | CATALOGS | CEIL | CHAIN | CHARSET | CLEAN | CLUSTER | CLUSTERS | CURRENT | COLLATION | COLUMNS @@ -2447,7 +2453,7 @@ nonReserved | FUNCTIONS | GLOBAL | GRANTS | HASH | HISTOGRAM | HELP | HLL_UNION | HOST | HOUR | HUB - | IDENTIFIED | IMAGE | IMPERSONATE | INCREMENTAL | INDEXES | INSTALL | INTEGRATION | INTEGRATIONS | INTERMEDIATE + | IDENTIFIED | IMAGE | IMPERSONATE | INACTIVE | INCREMENTAL | INDEXES | INSTALL | INTEGRATION | INTEGRATIONS | INTERMEDIATE | INTERVAL | ISOLATION | JOB | LABEL | LAST | LESS | LEVEL | LIST | LOCAL | LOCATION | LOGICAL | LOW_PRIORITY | LOCK | LOCATIONS diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocksLex.g4 b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocksLex.g4 index eba5208e666db9..54b805617d795c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocksLex.g4 +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocksLex.g4 @@ -25,6 +25,7 @@ tokens { } ACCESS: 'ACCESS'; +ACTIVE: 'ACTIVE'; ADD: 'ADD'; ADMIN: 'ADMIN'; AFTER: 'AFTER'; @@ -188,6 +189,7 @@ IMMEDIATE: 'IMMEDIATE'; IGNORE: 'IGNORE'; IMAGE: 'IMAGE'; IN: 'IN'; +INACTIVE: 'INACTIVE'; INCREMENTAL: 'INCREMENTAL'; INDEX: 'INDEX'; INDEXES: 'INDEXES'; diff --git a/test/sql/test_materialized_column/R/test_materialized_column b/test/sql/test_materialized_column/R/test_materialized_column index b5fab46d894446..7e9b6d58ac4afc 100644 --- a/test/sql/test_materialized_column/R/test_materialized_column +++ b/test/sql/test_materialized_column/R/test_materialized_column @@ -157,7 +157,7 @@ None -- !result REFRESH MATERIALIZED VIEW mv1; -- result: -E: (1064, 'Getting analyzing error at line 1, column 26. Detail message: Refresh materialized view failed because mv1 is not active.') +E: (1064, 'Getting analyzing error at line 1, column 26. Detail message: Refresh materialized view failed because [mv1] is not active. You can try to active it with ALTER MATERIALIZED VIEW mv1 ACTIVE; .') -- !result select sleep(5); -- result: diff --git a/test/sql/test_materialized_view/R/test_materialized_view_status b/test/sql/test_materialized_view/R/test_materialized_view_status new file mode 100644 index 00000000000000..0a774f839ff96e --- /dev/null +++ b/test/sql/test_materialized_view/R/test_materialized_view_status @@ -0,0 +1,105 @@ +-- name: test_materialized_view_status +CREATE TABLE `t1` ( + `k1` date NULL COMMENT "", + `v1` int(11) NULL COMMENT "", + `v2` int(11) NULL COMMENT "" +) ENGINE=OLAP +DUPLICATE KEY(`k1`) +COMMENT "OLAP" +PARTITION BY RANGE(`k1`) +(PARTITION p1 VALUES [("0000-01-01"), ("2023-01-01")), +PARTITION p2 VALUES [("2023-01-01"), ("2023-02-01")), +PARTITION p3 VALUES [("2023-02-01"), ("2023-03-01")), +PARTITION p4 VALUES [("2023-03-01"), ("2023-04-01")), +PARTITION p5 VALUES [("2023-04-01"), ("2023-05-01")), +PARTITION p6 VALUES [("2023-05-01"), ("2023-06-01"))) +DISTRIBUTED BY HASH(`k1`) BUCKETS 2 +PROPERTIES ( +"replication_num" = "1" +); +-- result: +-- !result +insert into t1 values ("2019-01-01",1,1),("2019-01-01",1,2),("2019-01-01",2,1),("2019-01-01",2,2), + ("2023-01-11",1,1),("2023-01-11",1,2),("2023-02-11",2,1),("2023-01-11",2,2), + ("2023-03-22",1,1),("2023-05-22",1,2),("2023-04-22",2,1),("2023-05-01",2,2); +-- result: +-- !result +CREATE MATERIALIZED VIEW mv1 + PARTITION BY k1 + DISTRIBUTED BY HASH(k1) BUCKETS 10 + REFRESH ASYNC + AS SELECT k1, sum(v1) as sum_v1 FROM t1 group by k1; +-- result: +-- !result +select sleep(2); +-- result: +1 +-- !result +drop table t1; +-- result: +-- !result +refresh materialized view mv1; +-- result: +E: (1064, 'Getting analyzing error at line 1, column 26. Detail message: Refresh materialized view failed because [mv1] is not active. You can try to active it with ALTER MATERIALIZED VIEW mv1 ACTIVE; .') +-- !result +CREATE TABLE `t1` ( + `k1` date NULL COMMENT "", + `v1` int(11) NULL COMMENT "", + `v2` int(11) NULL COMMENT "" +) ENGINE=OLAP +DUPLICATE KEY(`k1`) +COMMENT "OLAP" +PARTITION BY RANGE(`k1`) +(PARTITION p1 VALUES [("0000-01-01"), ("2023-01-01")), +PARTITION p2 VALUES [("2023-01-01"), ("2023-02-01")), +PARTITION p3 VALUES [("2023-02-01"), ("2023-03-01")), +PARTITION p4 VALUES [("2023-03-01"), ("2023-04-01")), +PARTITION p5 VALUES [("2023-04-01"), ("2023-05-01")), +PARTITION p6 VALUES [("2023-05-01"), ("2023-06-01"))) +DISTRIBUTED BY HASH(`k1`) BUCKETS 2 +PROPERTIES ( +"replication_num" = "1" +); +-- result: +-- !result +ALTER MATERIALIZED VIEW mv1 ACTIVE; +-- result: +-- !result +REFRESH MATERIALIZED VIEW mv1; +-- !result +select sleep(2); +-- result: +1 +-- !result +select * from mv1; +-- result: +-- !result +ALTER MATERIALIZED VIEW mv1 ACTIVE; +-- result: +-- !result +insert into t1 values ("2019-01-01",1,1),("2019-01-01",1,2),("2019-01-01",2,1),("2019-01-01",2,2), + ("2023-01-11",1,1),("2023-01-11",1,2),("2023-02-11",2,1),("2023-01-11",2,2), + ("2023-03-11",1,1),("2023-05-11",1,2),("2023-04-11",2,1),("2023-05-01",2,2); +-- result: +-- !result +select sleep(2); +-- result: +1 +-- !result +select * from mv1; +-- result: +2019-01-01 6 +2023-01-11 4 +2023-02-11 2 +2023-05-11 1 +2023-04-11 2 +2023-03-11 1 +2023-05-01 2 +-- !result +ALTER MATERIALIZED VIEW mv1 INACTIVE; +-- result: +-- !result +REFRESH MATERIALIZED VIEW mv1; +-- result: +E: (1064, 'Getting analyzing error at line 1, column 26. Detail message: Refresh materialized view failed because [mv1] is not active. You can try to active it with ALTER MATERIALIZED VIEW mv1 ACTIVE; .') +-- !result \ No newline at end of file diff --git a/test/sql/test_materialized_view/T/test_materialized_view_status b/test/sql/test_materialized_view/T/test_materialized_view_status new file mode 100644 index 00000000000000..5dffccf0dcb20b --- /dev/null +++ b/test/sql/test_materialized_view/T/test_materialized_view_status @@ -0,0 +1,60 @@ +-- name: test_materialized_view_status +CREATE TABLE `t1` ( + `k1` date NULL COMMENT "", + `v1` int(11) NULL COMMENT "", + `v2` int(11) NULL COMMENT "" +) ENGINE=OLAP +DUPLICATE KEY(`k1`) +COMMENT "OLAP" +PARTITION BY RANGE(`k1`) +(PARTITION p1 VALUES [("0000-01-01"), ("2023-01-01")), +PARTITION p2 VALUES [("2023-01-01"), ("2023-02-01")), +PARTITION p3 VALUES [("2023-02-01"), ("2023-03-01")), +PARTITION p4 VALUES [("2023-03-01"), ("2023-04-01")), +PARTITION p5 VALUES [("2023-04-01"), ("2023-05-01")), +PARTITION p6 VALUES [("2023-05-01"), ("2023-06-01"))) +DISTRIBUTED BY HASH(`k1`) BUCKETS 2 +PROPERTIES ( +"replication_num" = "1" +); +insert into t1 values ("2019-01-01",1,1),("2019-01-01",1,2),("2019-01-01",2,1),("2019-01-01",2,2), + ("2023-01-11",1,1),("2023-01-11",1,2),("2023-02-11",2,1),("2023-01-11",2,2), + ("2023-03-22",1,1),("2023-05-22",1,2),("2023-04-22",2,1),("2023-05-01",2,2); +CREATE MATERIALIZED VIEW mv1 + PARTITION BY k1 + DISTRIBUTED BY HASH(k1) BUCKETS 10 + REFRESH ASYNC + AS SELECT k1, sum(v1) as sum_v1 FROM t1 group by k1; +select sleep(2); +drop table t1; +refresh materialized view mv1; +CREATE TABLE `t1` ( + `k1` date NULL COMMENT "", + `v1` int(11) NULL COMMENT "", + `v2` int(11) NULL COMMENT "" +) ENGINE=OLAP +DUPLICATE KEY(`k1`) +COMMENT "OLAP" +PARTITION BY RANGE(`k1`) +(PARTITION p1 VALUES [("0000-01-01"), ("2023-01-01")), +PARTITION p2 VALUES [("2023-01-01"), ("2023-02-01")), +PARTITION p3 VALUES [("2023-02-01"), ("2023-03-01")), +PARTITION p4 VALUES [("2023-03-01"), ("2023-04-01")), +PARTITION p5 VALUES [("2023-04-01"), ("2023-05-01")), +PARTITION p6 VALUES [("2023-05-01"), ("2023-06-01"))) +DISTRIBUTED BY HASH(`k1`) BUCKETS 2 +PROPERTIES ( +"replication_num" = "1" +); +ALTER MATERIALIZED VIEW mv1 ACTIVE; +REFRESH MATERIALIZED VIEW mv1; +select sleep(2); +select * from mv1; +ALTER MATERIALIZED VIEW mv1 ACTIVE; +insert into t1 values ("2019-01-01",1,1),("2019-01-01",1,2),("2019-01-01",2,1),("2019-01-01",2,2), + ("2023-01-11",1,1),("2023-01-11",1,2),("2023-02-11",2,1),("2023-01-11",2,2), + ("2023-03-11",1,1),("2023-05-11",1,2),("2023-04-11",2,1),("2023-05-01",2,2); +select sleep(2); +select * from mv1; +ALTER MATERIALIZED VIEW mv1 INACTIVE; +REFRESH MATERIALIZED VIEW mv1; \ No newline at end of file diff --git a/test/sql/test_mv/R/basic b/test/sql/test_mv/R/basic index ab43c3adf1b9e0..5d9da7dc69934d 100644 --- a/test/sql/test_mv/R/basic +++ b/test/sql/test_mv/R/basic @@ -40,5 +40,5 @@ false base-table dropped: ss -- !result REFRESH MATERIALIZED VIEW mv1; -- result: -E: (1064, 'Getting analyzing error at line 1, column 26. Detail message: Refresh materialized view failed because mv1 is not active.') +E: (1064, 'Getting analyzing error at line 1, column 26. Detail message: Refresh materialized view failed because [mv1] is not active. You can try to active it with ALTER MATERIALIZED VIEW mv1 ACTIVE; .') -- !result \ No newline at end of file