diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index ded1d1943582..5d13551cf510 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -118,7 +118,6 @@ import org.apache.hadoop.hive.ql.security.authorization.HiveCustomStorageHandlerUtils; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionStateUtil; -import org.apache.hadoop.hive.ql.stats.Partish; import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.DefaultFetchFormatter; import org.apache.hadoop.hive.serde2.Deserializer; @@ -450,44 +449,45 @@ public void appendFiles(org.apache.hadoop.hive.metastore.api.Table table, URI fr } @Override - public Map getBasicStatistics(Partish partish) { - org.apache.hadoop.hive.ql.metadata.Table hmsTable = partish.getTable(); + public Map getBasicStatistics(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { // For write queries where rows got modified, don't fetch from cache as values could have changed. - Table table = getTable(hmsTable); Map stats = Maps.newHashMap(); - if (getStatsSource().equals(HiveMetaHook.ICEBERG)) { - if (table.currentSnapshot() != null) { - Map summary = table.currentSnapshot().summary(); - if (summary != null) { + if (!getStatsSource().equals(HiveMetaHook.ICEBERG)) { + return hmsTable.getParameters(); + } + Table table = getTable(hmsTable); - if (summary.containsKey(SnapshotSummary.TOTAL_DATA_FILES_PROP)) { - stats.put(StatsSetupConst.NUM_FILES, summary.get(SnapshotSummary.TOTAL_DATA_FILES_PROP)); - } + if (table.currentSnapshot() != null) { + Map summary = table.currentSnapshot().summary(); + if (summary != null) { - if (summary.containsKey(SnapshotSummary.TOTAL_RECORDS_PROP)) { - long totalRecords = Long.parseLong(summary.get(SnapshotSummary.TOTAL_RECORDS_PROP)); - if (summary.containsKey(SnapshotSummary.TOTAL_EQ_DELETES_PROP) && - summary.containsKey(SnapshotSummary.TOTAL_POS_DELETES_PROP)) { + if (summary.containsKey(SnapshotSummary.TOTAL_DATA_FILES_PROP)) { + stats.put(StatsSetupConst.NUM_FILES, summary.get(SnapshotSummary.TOTAL_DATA_FILES_PROP)); + } - long totalEqDeletes = Long.parseLong(summary.get(SnapshotSummary.TOTAL_EQ_DELETES_PROP)); - long totalPosDeletes = Long.parseLong(summary.get(SnapshotSummary.TOTAL_POS_DELETES_PROP)); + if (summary.containsKey(SnapshotSummary.TOTAL_RECORDS_PROP)) { + long totalRecords = Long.parseLong(summary.get(SnapshotSummary.TOTAL_RECORDS_PROP)); + if (summary.containsKey(SnapshotSummary.TOTAL_EQ_DELETES_PROP) && + summary.containsKey(SnapshotSummary.TOTAL_POS_DELETES_PROP)) { - long actualRecords = totalRecords - (totalEqDeletes > 0 ? 0 : totalPosDeletes); - totalRecords = actualRecords > 0 ? actualRecords : totalRecords; - // actualRecords maybe -ve in edge cases - } - stats.put(StatsSetupConst.ROW_COUNT, String.valueOf(totalRecords)); - } + long totalEqDeletes = Long.parseLong(summary.get(SnapshotSummary.TOTAL_EQ_DELETES_PROP)); + long totalPosDeletes = Long.parseLong(summary.get(SnapshotSummary.TOTAL_POS_DELETES_PROP)); - if (summary.containsKey(SnapshotSummary.TOTAL_FILE_SIZE_PROP)) { - stats.put(StatsSetupConst.TOTAL_SIZE, summary.get(SnapshotSummary.TOTAL_FILE_SIZE_PROP)); + long actualRecords = totalRecords - (totalEqDeletes > 0 ? 0 : totalPosDeletes); + totalRecords = actualRecords > 0 ? actualRecords : totalRecords; + // actualRecords maybe -ve in edge cases } + stats.put(StatsSetupConst.ROW_COUNT, String.valueOf(totalRecords)); + } + + if (summary.containsKey(SnapshotSummary.TOTAL_FILE_SIZE_PROP)) { + stats.put(StatsSetupConst.TOTAL_SIZE, summary.get(SnapshotSummary.TOTAL_FILE_SIZE_PROP)); } - } else { - stats.put(StatsSetupConst.NUM_FILES, "0"); - stats.put(StatsSetupConst.ROW_COUNT, "0"); - stats.put(StatsSetupConst.TOTAL_SIZE, "0"); } + } else { + stats.put(StatsSetupConst.NUM_FILES, "0"); + stats.put(StatsSetupConst.ROW_COUNT, "0"); + stats.put(StatsSetupConst.TOTAL_SIZE, "0"); } return stats; } @@ -600,7 +600,10 @@ private ColumnStatistics readColStats(Table table, Path statsPath) { @Override public boolean canComputeQueryUsingStats(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { - if (getStatsSource().equals(HiveMetaHook.ICEBERG) && hmsTable.getMetaTable() == null) { + if (hmsTable.getMetaTable() != null) { + return false; + } + if (getStatsSource().equals(HiveMetaHook.ICEBERG)) { Table table = getTable(hmsTable); if (table.currentSnapshot() != null) { Map summary = table.currentSnapshot().summary(); @@ -613,7 +616,7 @@ public boolean canComputeQueryUsingStats(org.apache.hadoop.hive.ql.metadata.Tabl } } } - return false; + return true; } private String getStatsSource() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java index 168a91f3c3de..bd2de511330d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java @@ -254,7 +254,12 @@ default Map getOperatorDescProperties(OperatorDesc operatorDesc, * @param partish a partish wrapper class * @return map of basic statistics, can be null */ + @Deprecated default Map getBasicStatistics(Partish partish) { + return getBasicStatistics(partish.getTable()); + } + + default Map getBasicStatistics(org.apache.hadoop.hive.ql.metadata.Table table) { return null; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java index 38d67660c639..a7939ed972c1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java @@ -945,7 +945,10 @@ private Long getRowCnt( if (!StatsUtils.areBasicStatsUptoDateForQueryAnswering(tbl, tbl.getParameters())) { return null; } - rowCnt = Long.valueOf(tbl.getProperty(StatsSetupConst.ROW_COUNT)); + Map basicStats = MetaStoreUtils.isNonNativeTable(tbl.getTTable()) ? + tbl.getStorageHandler().getBasicStatistics(tbl) : tbl.getParameters(); + + rowCnt = Long.valueOf(basicStats.get(StatsSetupConst.ROW_COUNT)); } return rowCnt; }