HIVE-28908: From clause supports catalog.db.tbl pattern

parser/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g

@@ -233,6 +233,35 @@ uniqueJoinTableSource
 
 tableName
 @init { gParent.pushMsg("table name", state); }
+@after { gParent.popMsg(state); }
+    :
+    // case 1：catalog.db.table(.meta)?
+    (cat=identifier DOT db=identifier DOT tab=identifier (DOT meta=identifier)?)
+    =>
+    cat=identifier DOT db=identifier DOT tab=identifier (DOT meta=identifier)?
+    {
+        tables.add(new ImmutablePair<>($cat.text + "." + $db.text, $tab.text));
+    }
+    -> ^(TOK_TABNAME $cat $db $tab $meta?)
+
+    // case 2：db.table
+    | db=identifier DOT tab=identifier
+    {
+        tables.add(new ImmutablePair<>($db.text, $tab.text));
+    }
+    -> ^(TOK_TABNAME $db $tab)
+
+    // case 3：table
+    | tab=identifier
+    {
+        tables.add(new ImmutablePair<>(null, $tab.text));
+    }
+    -> ^(TOK_TABNAME $tab)
+    ;
+
+
+tableName_bak
+@init { gParent.pushMsg("table name", state); }
 @after { gParent.popMsg(state); }
     :
     db=identifier DOT tab=identifier (DOT meta=identifier)?

ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLUtils.java

@@ -195,6 +195,7 @@ public static void addDbAndTableToOutputs(Database database, TableName tableName
     outputs.add(new WriteEntity(database, WriteEntity.WriteType.DDL_SHARED));
 
     Table table = new Table(tableName.getDb(), tableName.getTable());
+    table.setCatName(tableName.getCat());
     table.setParameters(properties);
     table.setTableType(type);
     table.setTemporary(isTemporary);

ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java

@@ -117,7 +117,7 @@ public int execute() throws Exception {
   }
 
   private Table getTable() throws HiveException {
-    Table table = context.getDb().getTable(desc.getTableName().getDb(), desc.getTableName().getTable(),
+    Table table = context.getDb().getTable(null, desc.getTableName().getDb(), desc.getTableName().getTable(),
         desc.getTableName().getTableMetaRef(), false, false, false);
     if (table == null) {
       throw new HiveException(ErrorMsg.INVALID_TABLE, desc.getDbTableName());

ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java

@@ -2326,7 +2326,7 @@ public static List<String> getColumnTypes(Properties props) {
    */
   @Deprecated
   public static String[] getDbTableName(String dbtable) throws SemanticException {
-    return getDbTableName(SessionState.get().getCurrentDatabase(), dbtable);
+    return getDbTableName(SessionState.get().getCurrentCatalog(), SessionState.get().getCurrentDatabase(), dbtable);
   }
 
   /**
@@ -2338,17 +2338,19 @@ public static String[] getDbTableName(String dbtable) throws SemanticException {
    * @deprecated use {@link TableName} or {@link org.apache.hadoop.hive.ql.parse.HiveTableName} instead
    */
   @Deprecated
-  public static String[] getDbTableName(String defaultDb, String dbtable) throws SemanticException {
+  public static String[] getDbTableName(String defaultCatalog, String defaultDb, String dbtable) throws SemanticException {
     if (dbtable == null) {
       return new String[2];
     }
     String[] names =  dbtable.split("\\.");
     switch (names.length) {
+      case 4:
       case 3:
-      case 2:
         return names;
+      case 2:
+        return new String [] {defaultCatalog, names[0], names[1]};
       case 1:
-        return new String [] {defaultDb, dbtable};
+        return new String [] {defaultCatalog, defaultDb, dbtable};
       default:
         throw new SemanticException(ErrorMsg.INVALID_TABLE_NAME, dbtable);
     }
@@ -2381,7 +2383,7 @@ public static void validateColumnNames(List<String> colNames, List<String> check
    * @param dbTableName
    * @return a {@link TableName}
    * @throws SemanticException
-   * @deprecated handle null values and use {@link TableName#fromString(String, String, String)}
+   * @deprecated handle null values and use {@link TableName#fromString(String, String, String)} (why deprecated)
    */
   @Deprecated
   public static TableName getNullableTableName(String dbTableName) throws SemanticException {
@@ -2396,7 +2398,7 @@ public static TableName getNullableTableName(String dbTableName) throws Semantic
    * @param defaultDb
    * @return a {@link TableName}
    * @throws SemanticException
-   * @deprecated handle null values and use {@link TableName#fromString(String, String, String)}
+   * @deprecated handle null values and use {@link TableName#fromString(String, String, String)}(why deprecated)
    */
   @Deprecated
   public static TableName getNullableTableName(String dbTableName, String defaultDb) throws SemanticException {

ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java

@@ -1609,12 +1609,12 @@ public Table getTable(final String tableName) throws HiveException {
    */
   public Table getTable(final String tableName, boolean throwException) throws HiveException {
     String[] nameParts = tableName.split("\\.");
-    if (nameParts.length == 3) {
-      Table table = this.getTable(nameParts[0], nameParts[1], nameParts[2], throwException);
+    if (nameParts.length == 4) {
+      Table table = this.getTable(nameParts[0], nameParts[1], nameParts[2], nameParts[3], throwException);
       return table;
     } else {
       String[] names = Utilities.getDbTableName(tableName);
-      Table table = this.getTable(names[0], names[1], null, throwException);
+      Table table = this.getTable(names[0], names[1], names[2], null,  throwException);
       return table;
     }
   }
@@ -1634,9 +1634,9 @@ public Table getTable(final String dbName, final String tableName) throws HiveEx
      // TODO: catalog... etc everywhere
     if (tableName.contains(".")) {
       String[] names = Utilities.getDbTableName(tableName);
-      return this.getTable(names[0], names[1], null, true);
+      return this.getTable(names[0], names[1], names[2], null, true);
     } else {
-      return this.getTable(dbName, tableName, null, true);
+      return this.getTable(SessionState.get().getCurrentCatalog(), dbName, tableName, null, true);
     }
   }
 
@@ -1650,7 +1650,7 @@ public Table getTable(final String dbName, final String tableName) throws HiveEx
    *              if there's an internal error or if the table doesn't exist
    */
   public Table getTable(TableName tableName) throws HiveException {
-    return this.getTable(ObjectUtils.firstNonNull(tableName.getDb(), SessionState.get().getCurrentDatabase()),
+    return this.getTable(SessionState.get().getCurrentCatalog(), ObjectUtils.firstNonNull(tableName.getDb(), SessionState.get().getCurrentDatabase()),
         tableName.getTable(), tableName.getTableMetaRef(), true);
   }
 
@@ -1668,9 +1668,9 @@ public Table getTable(TableName tableName) throws HiveException {
    * @return the table or if throwException is false a null value.
    * @throws HiveException
    */
-  public Table getTable(final String dbName, final String tableName,
+  public Table getTable(final String catName, final String dbName, final String tableName,
                         final String tableMetaRef, boolean throwException) throws HiveException {
-    return this.getTable(dbName, tableName, tableMetaRef, throwException, false);
+    return this.getTable(catName, dbName, tableName, tableMetaRef, throwException, false);
   }
 
   /**
@@ -1686,8 +1686,8 @@ public Table getTable(final String dbName, final String tableName,
    * @throws HiveException
    */
   public Table getTable(final String dbName, final String tableName, boolean throwException) throws HiveException {
-    return this.getTable(dbName, tableName, null, throwException);
-  }
+    return this.getTable(SessionState.get().getCurrentCatalog(), dbName, tableName, null, throwException);
+  } //TODO get the correct catalog
 
   /**
    * Returns metadata of the table.
@@ -1706,7 +1706,7 @@ public Table getTable(final String dbName, final String tableName, boolean throw
    */
   public Table getTable(final String dbName, final String tableName, boolean throwException, boolean checkTransactional)
       throws HiveException {
-    return getTable(dbName, tableName, null, throwException, checkTransactional, false);
+    return getTable(null, dbName, tableName, null, throwException, checkTransactional, false);
   }
 
   /**
@@ -1726,9 +1726,9 @@ public Table getTable(final String dbName, final String tableName, boolean throw
    * @return the table or if throwException is false a null value.
    * @throws HiveException
    */
-  public Table getTable(final String dbName, final String tableName, String tableMetaRef, boolean throwException,
+  public Table getTable(final String catName, final String dbName, final String tableName, String tableMetaRef, boolean throwException,
                         boolean checkTransactional) throws HiveException {
-    return getTable(dbName, tableName, tableMetaRef, throwException, checkTransactional, false);
+    return getTable(catName, dbName, tableName, tableMetaRef, throwException, checkTransactional, false);
   }
 
   /**
@@ -1750,8 +1750,8 @@ public Table getTable(final String dbName, final String tableName, String tableM
    * @return the table or if throwException is false a null value.
    * @throws HiveException
    */
-  public Table getTable(final String dbName, final String tableName, String tableMetaRef, boolean throwException,
-                        boolean checkTransactional, boolean getColumnStats) throws HiveException {
+  public Table getTable(final String catalogName, final String dbName, final String tableName, String tableMetaRef,
+                        boolean throwException, boolean checkTransactional, boolean getColumnStats) throws HiveException {
 
     if (tableName == null || tableName.equals("")) {
       throw new HiveException("empty table creation??");
@@ -1762,7 +1762,7 @@ public Table getTable(final String dbName, final String tableName, String tableM
     try {
       // Note: this is currently called w/true from StatsOptimizer only.
       GetTableRequest request = new GetTableRequest(dbName, tableName);
-      request.setCatName(getDefaultCatalog(conf));
+      request.setCatName(catalogName != null ? catalogName : SessionState.get().getCurrentCatalog());
       request.setGetColumnStats(getColumnStats);
       request.setEngine(Constants.HIVE_ENGINE);
       if (checkTransactional) {

ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java

@@ -170,7 +170,7 @@ protected void initialize(org.apache.hadoop.hive.metastore.api.Table table) {
     // performance. Since those fields are null/cache-check by their accessors
     // anyway, that's not a concern.
   }
-
+  // TODO add catalog after calling this constructor
   public Table(String databaseName, String tableName) {
     this(getEmptyTable(databaseName, tableName));
   }
@@ -928,6 +928,10 @@ public void setDbName(String databaseName) {
     tTable.setDbName(databaseName);
   }
 
+  public void setCatName(String catName) {
+    tTable.setCatName(catName);
+  }
+
   public List<FieldSchema> getPartitionKeys() {
     return tTable.getPartitionKeys();
   }

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java

@@ -81,6 +81,7 @@ public static ASTNode table(final RelNode scan) {
       return cte(hTbl.getHiveTableMD().getTableName(), hts.getTableAlias());
     }
     ASTBuilder tableNameBuilder = ASTBuilder.construct(HiveParser.TOK_TABNAME, "TOK_TABNAME")
+        .add(HiveParser.Identifier, hTbl.getHiveTableMD().getCatName())
         .add(HiveParser.Identifier, hTbl.getHiveTableMD().getDbName())
         .add(HiveParser.Identifier, hTbl.getHiveTableMD().getTableName());
     if (hTbl.getHiveTableMD().getMetaTable() != null) {

ql/src/java/org/apache/hadoop/hive/ql/parse/AnalyzeCommandUtils.java

@@ -20,6 +20,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.hadoop.hive.common.TableName;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.metadata.Table;
@@ -47,9 +48,8 @@ public static boolean isPartitionLevelStats(ASTNode tree) {
 
   public static Table getTable(ASTNode tree, BaseSemanticAnalyzer sa) throws SemanticException {
     String tableName = ColumnStatsSemanticAnalyzer.getUnescapedName((ASTNode) tree.getChild(0).getChild(0));
-    String currentDb = SessionState.get().getCurrentDatabase();
-    String [] names = Utilities.getDbTableName(currentDb, tableName);
-    return sa.getTable(names[0], names[1], true);
+    TableName tName = HiveTableName.of(tableName);
+    return sa.getTable(tName);
   }
 
   public static Map<String,String> getPartKeyValuePairsFromAST(Table tbl, ASTNode tree,

ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java

@@ -40,6 +40,7 @@
 import org.antlr.runtime.tree.Tree;
 import org.apache.calcite.sql.SqlKind;
 import org.apache.commons.lang3.tuple.Pair;
+import org.apache.commons.lang3.tuple.Triple;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.FileUtils;
@@ -431,6 +432,43 @@ public static String getUnescapedName(ASTNode tableOrColumnNode) throws Semantic
     return getUnescapedName(tableOrColumnNode, null);
   }
 
+
+  public static Triple<String, String, String> getDbTableNameTriple(ASTNode tableNameNode) throws SemanticException {
+
+    if (tableNameNode.getType() != HiveParser.TOK_TABNAME ||
+        (tableNameNode.getChildCount() < 1 || tableNameNode.getChildCount() > 4)) {
+      throw new SemanticException(ASTErrorUtils.getMsg(ErrorMsg.INVALID_TABLE_NAME.getMsg(), tableNameNode));
+    }
+
+    List<String> parts = new ArrayList<>();
+    for (int i = 0; i < tableNameNode.getChildCount(); i++) {
+      String part = unescapeIdentifier(tableNameNode.getChild(i).getText());
+      if (part != null && part.contains(".")) {
+        throw new SemanticException(ASTErrorUtils.getMsg(ErrorMsg.OBJECTNAME_CONTAINS_DOT.getMsg(), tableNameNode));
+      }
+      parts.add(part);
+    }
+
+    String catalog = null, db = null, table = null;
+
+    if (parts.size() == 1) {
+      table = parts.get(0);
+    } else if (parts.size() == 2) {
+      db = parts.get(0);
+      table = parts.get(1);
+    } else if (parts.size() == 3) {
+      catalog = parts.get(0);
+      db = parts.get(1);
+      table = parts.get(2);
+    } else if (parts.size() == 4) {
+      catalog = parts.get(0);
+      db = parts.get(1);
+      table = parts.get(2) + "." + parts.get(3);  // meta table
+    }
+
+    return Triple.of(catalog, db, table);
+  }
+  // todo remove this function
   public static Map.Entry<String, String> getDbTableNamePair(ASTNode tableNameNode) throws SemanticException {
 
     if (tableNameNode.getType() != HiveParser.TOK_TABNAME ||
@@ -463,17 +501,17 @@ public static String getUnescapedName(ASTNode tableOrColumnNode, String currentD
     int tokenType = tableOrColumnNode.getToken().getType();
     if (tokenType == HiveParser.TOK_TABNAME) {
       // table node
-      Map.Entry<String,String> dbTablePair = getDbTableNamePair(tableOrColumnNode);
-      String tableName = dbTablePair.getValue();
+      Triple<String, String, String> dbTablePair = getDbTableNameTriple(tableOrColumnNode);
+      String tableName = dbTablePair.getRight();
       String tableMetaRef = null;
       if (tableName.contains(".")) {
         String[] tmpNames = tableName.split("\\.");
         tableName = tmpNames[0];
         tableMetaRef = tmpNames[1];
       }
       return TableName.fromString(tableName,
-          null,
-          dbTablePair.getKey() == null ? currentDatabase : dbTablePair.getKey(),
+              dbTablePair.getLeft() == null ? SessionState.get().getCurrentCatalog() : dbTablePair.getLeft(),
+          dbTablePair.getMiddle() == null ? currentDatabase : dbTablePair.getMiddle(),
           tableMetaRef)
           .getNotEmptyDbTable();
     } else if (tokenType == HiveParser.StringLiteral) {
@@ -548,13 +586,13 @@ public static String getUnescapedUnqualifiedTableName(ASTNode node) throws Seman
     assert node.getChildCount() <= 3;
     assert node.getType() == HiveParser.TOK_TABNAME;
 
-    if (node.getChildCount() == 2 || node.getChildCount() == 3) {
+    if (node.getChildCount() == 2 || node.getChildCount() == 3 || node.getChildCount() == 4) {
       node = (ASTNode) node.getChild(1);
     }
 
     String tableName = getUnescapedName(node);
-    if (node.getChildCount() == 3) {
-      tableName = tableName + "." + node.getChild(2);
+    if (node.getChildCount() == 4) {
+      tableName = tableName + "." + node.getChild(3);
     }
     return tableName;
   }
@@ -1969,7 +2007,7 @@ protected Table getTable(String database, String tblName, String tableMetaRef, b
     try {
       String tableName = tableMetaRef == null ? tblName : tblName + "." + tableMetaRef;
       tab = database == null ? db.getTable(tableName, false)
-          : db.getTable(database, tblName, tableMetaRef, false);
+          : db.getTable(SessionState.get().getCurrentCatalog(), database, tblName, tableMetaRef, false);
     }
     catch (InvalidTableException e) {
       throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(TableName.fromString(tblName, null, database).getNotEmptyDbTable()), e);

ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java

@@ -5120,18 +5120,19 @@ private QBParseInfo getQBParseInfo(QB qb) throws CalciteSemanticException {
   @Override
   protected Table getTableObjectByName(String tabName, boolean throwException) throws HiveException {
     String[] names = Utilities.getDbTableName(tabName);
-    final String  tableName = names[1];
-    final String  dbName = names[0];
+    final String  catalogName = names[0];
+    final String  tableName = names[2];
+    final String  dbName = names[1];
     String tableMetaRef = null;
-    if (names.length == 3) {
-      tableMetaRef = names[2];
+    if (names.length == 4) {
+      tableMetaRef = names[3];
     }
-    String fullyQualName = dbName + "." + tableName;
+    String fullyQualName = catalogName + "." + dbName + "." + tableName;
     if (tableMetaRef != null) {
       fullyQualName += "." + tableMetaRef;
     }
     if (!tabNameToTabObject.containsKey(fullyQualName)) {
-      Table table = db.getTable(dbName, tableName, tableMetaRef, throwException, false, false);
+      Table table = db.getTable(catalogName, dbName, tableName, tableMetaRef, throwException, false, false);
       if (table != null) {
         tabNameToTabObject.put(fullyQualName, table);
       }

ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java

@@ -211,8 +211,8 @@ public boolean isDestToOpTypeInsertOverwrite(String clause) {
   /**
    * See also {@link #getInsertOverwriteTables()}
    */
-  public boolean isInsertIntoTable(String dbName, String table, String branchName) {
-    String  fullName = dbName + "." + table;
+  public boolean isInsertIntoTable(String catName, String dbName, String table, String branchName) {
+    String  fullName = catName + "." + dbName + "." + table;
     if (branchName != null) {
       fullName += "." + branchName;
     }

ql/src/java/org/apache/hadoop/hive/ql/parse/RewriteSemanticAnalyzer.java

@@ -198,7 +198,7 @@ protected static Table getTable(ASTNode tabRef, Hive db, boolean throwException)
 
     Table mTable;
     try {
-      mTable = db.getTable(tableName.getDb(), tableName.getTable(), tableName.getTableMetaRef(), throwException);
+      mTable = db.getTable(tableName.getCat(), tableName.getDb(), tableName.getTable(), tableName.getTableMetaRef(), throwException);
     } catch (InvalidTableException e) {
       LOG.error("Failed to find table " + tableName.getNotEmptyDbTable() + " got exception " + e.getMessage());
       throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tableName.getNotEmptyDbTable()), e);