diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index 098928bb2043..37b7f01bff7b 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -633,9 +633,9 @@ public static List convertNameToMetastorePartition(org.apache.hadoop. return partitions; } - public static TableFetcher getTableFetcher(IMetaStoreClient msc, String catalogName, String dbPattern, + public static TableFetcher getTableFetcher(IMetaStoreClient msc, String catalogPattern, String dbPattern, String tablePattern) { - return new TableFetcher.Builder(msc, catalogName, dbPattern, tablePattern).tableTypes( + return new TableFetcher.Builder(msc, catalogPattern, dbPattern, tablePattern).tableTypes( "EXTERNAL_TABLE") .tableCondition( hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS + "table_type like \"ICEBERG\" ") diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergTableOptimizer.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergTableOptimizer.java index dfc352ee0dad..e7b1ac056c21 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergTableOptimizer.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergTableOptimizer.java @@ -121,7 +121,8 @@ public Set findPotentialCompactions(long lastChecked, ShowCompac private Iterable getTables(Set skipDBs, Set skipTables) { try { int maxBatchSize = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX); - return IcebergTableUtil.getTableFetcher(client, null, "*", null).getTables(skipDBs, skipTables, maxBatchSize); + return IcebergTableUtil.getTableFetcher(client, null, "*", null) + .getTables(null, skipDBs, skipTables, maxBatchSize); } catch (Exception e) { throw new RuntimeMetaException(e, "Error getting tables"); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/metastore/task/IcebergHouseKeeperService.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/metastore/task/IcebergHouseKeeperService.java index 13ab0ca1ab2b..38ee1f71cb06 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/metastore/task/IcebergHouseKeeperService.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/metastore/task/IcebergHouseKeeperService.java @@ -52,24 +52,24 @@ public long runFrequency(TimeUnit unit) { public void run() { LOG.debug("Running IcebergHouseKeeperService..."); - String catalogName = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.ICEBERG_TABLE_EXPIRY_CATALOG_NAME); + String catalogPattern = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.ICEBERG_TABLE_EXPIRY_CATALOG_PATTERN); String dbPattern = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.ICEBERG_TABLE_EXPIRY_DATABASE_PATTERN); String tablePattern = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.ICEBERG_TABLE_EXPIRY_TABLE_PATTERN); TxnStore.MutexAPI mutex = shouldUseMutex ? txnHandler.getMutexAPI() : new NoMutex(); try (AutoCloseable closeable = mutex.acquireLock(TxnStore.MUTEX_KEY.IcebergHouseKeeper.name())) { - expireTables(catalogName, dbPattern, tablePattern); + expireTables(catalogPattern, dbPattern, tablePattern); } catch (Exception e) { throw new RuntimeException(e); } } - private void expireTables(String catalogName, String dbPattern, String tablePattern) { + private void expireTables(String catalogPattern, String dbPattern, String tablePattern) { try (IMetaStoreClient msc = new HiveMetaStoreClient(conf)) { int maxBatchSize = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX); Iterable tables = - IcebergTableUtil.getTableFetcher(msc, catalogName, dbPattern, tablePattern).getTables(maxBatchSize); + IcebergTableUtil.getTableFetcher(msc, catalogPattern, dbPattern, tablePattern).getTables(maxBatchSize); // TODO : HIVE-29163 - Create client with cache in metastore package and then use it in TableFetcher // and HiveTableOperations to reduce the number of msc calls and fetch it from cache for (org.apache.hadoop.hive.metastore.api.Table table : tables) { diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/metastore/task/TestIcebergHouseKeeperService.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/metastore/task/TestIcebergHouseKeeperService.java index 4b56edbd07aa..c809c0264ecc 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/metastore/task/TestIcebergHouseKeeperService.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/metastore/task/TestIcebergHouseKeeperService.java @@ -68,7 +68,7 @@ public static void afterClass() { public void testIcebergTableFetched() throws Exception { createIcebergTable("iceberg_table"); - TableFetcher tableFetcher = IcebergTableUtil.getTableFetcher(db.getMSC(), null, "default", "*"); + TableFetcher tableFetcher = IcebergTableUtil.getTableFetcher(db.getMSC(), "*", "default", "*"); int maxBatchSize = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX); Iterator tables = tableFetcher.getTables(maxBatchSize).iterator(); @@ -82,7 +82,7 @@ public void testIcebergTableFetched() throws Exception { public void testExpireSnapshotsByServiceRun() throws Exception { String tableName = "iceberg_table_snapshot_expiry_e2e_test"; createIcebergTable(tableName); - IcebergHouseKeeperService service = getServiceForTable("default", tableName); + IcebergHouseKeeperService service = getServiceForTable("hive", "default", tableName); GetTableRequest request = new GetTableRequest("default", tableName); org.apache.iceberg.Table icebergTable = IcebergTableUtil.getTable(conf, db.getMSC().getTable(request)); @@ -128,9 +128,10 @@ private void createIcebergTable(String name) throws Exception { * @param tableName to be cleaned up * @return IcebergHouseKeeperService */ - private IcebergHouseKeeperService getServiceForTable(String dbName, String tableName) { + private IcebergHouseKeeperService getServiceForTable(String catalogName, String dbName, String tableName) { IcebergHouseKeeperService service = new IcebergHouseKeeperService(); HiveConf serviceConf = new HiveConf(conf); + serviceConf.set("hive.metastore.iceberg.table.expiry.catalog.pattern", catalogName); serviceConf.set("hive.metastore.iceberg.table.expiry.database.pattern", dbName); serviceConf.set("hive.metastore.iceberg.table.expiry.table.pattern", tableName); service.setConf(serviceConf); diff --git a/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index 793cfffceda6..efd0c466b56b 100644 --- a/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ b/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -131,6 +131,15 @@ void alterCatalog(String catalogName, Catalog newCatalog) */ List getCatalogs() throws MetaException, TException; + /** + * Get a list of all catalogs known to the system. + * @param catalogPattern pattern for the catalog name to match + * @return list of catalog names + * @throws MetaException something went wrong, usually in the database. + * @throws TException general thrift exception. + */ + List getCatalogs(String catalogPattern) throws MetaException, TException; + /** * Drop a catalog. Catalogs must be empty to be dropped, there is no cascade for dropping a * catalog. diff --git a/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/BaseMetaStoreClient.java b/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/BaseMetaStoreClient.java index 8a624735a14d..d1409a0657e9 100644 --- a/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/BaseMetaStoreClient.java +++ b/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/BaseMetaStoreClient.java @@ -64,6 +64,11 @@ public final void dropCatalog(String catName) dropCatalog(catName, true); } + @Override + public final List getCatalogs() throws MetaException, TException { + return getCatalogs(null); + } + @Override public final List getDatabases(String databasePattern) throws MetaException, TException { return getDatabases(getDefaultCatalog(conf), databasePattern); diff --git a/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientWrapper.java b/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientWrapper.java index 4d63e4f88124..dabda7b1db92 100644 --- a/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientWrapper.java +++ b/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/MetaStoreClientWrapper.java @@ -98,8 +98,8 @@ public Catalog getCatalog(String catName) throws NoSuchObjectException, MetaExce } @Override - public List getCatalogs() throws MetaException, TException { - return delegate.getCatalogs(); + public List getCatalogs(String catalogPattern) throws MetaException, TException { + return delegate.getCatalogs(catalogPattern); } @Override diff --git a/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/ThriftHiveMetaStoreClient.java b/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/ThriftHiveMetaStoreClient.java index 24a50145364a..30e22f581a59 100644 --- a/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/ThriftHiveMetaStoreClient.java +++ b/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/ThriftHiveMetaStoreClient.java @@ -966,8 +966,8 @@ public Catalog getCatalog(String catName) throws TException { } @Override - public List getCatalogs() throws TException { - GetCatalogsResponse rsp = client.get_catalogs(); + public List getCatalogs(String catalogPattern) throws TException { + GetCatalogsResponse rsp = client.get_catalogs(new GetCatalogRequest(catalogPattern)); if (rsp == null || rsp.getNames() == null) { return null; } else { diff --git a/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/utils/TableFetcher.java b/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/utils/TableFetcher.java index 8381fa032631..07acfe0a9b58 100644 --- a/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/utils/TableFetcher.java +++ b/standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/utils/TableFetcher.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.TableIterable; import org.apache.hadoop.hive.metastore.TableType; -import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; @@ -42,8 +41,8 @@ public class TableFetcher { // mandatory client passed to this fetcher, has to be closed from caller private final IMetaStoreClient client; - // mandatory catalogName - private final String catalogName; + // mandatory catalogPattern: use "*" to fetch all, empty to fetch none + private final String catalogPattern; // mandatory dbPattern: use "*" to fetch all, empty to fetch none private final String dbPattern; // optional tableTypes: comma separated table types to fetch, fetcher result is empty list if this is empty @@ -56,10 +55,7 @@ public class TableFetcher { private TableFetcher(Builder builder) { this.client = builder.client; - if ("*".equalsIgnoreCase(builder.catalogName)) { - LOG.warn("Invalid wildcard '*' parameter for catalogName, exact catalog name is expected instead of regexp"); - } - this.catalogName = Optional.ofNullable(builder.catalogName).orElse(Warehouse.DEFAULT_CATALOG_NAME); + this.catalogPattern = Optional.ofNullable(builder.catalogPattern).orElse(""); this.dbPattern = Optional.ofNullable(builder.dbPattern).orElse(""); String tablePattern = Optional.ofNullable(builder.tablePattern).orElse(""); String stringTableTypes = Optional.ofNullable(builder.tableTypes).orElse(""); @@ -103,42 +99,57 @@ public List getTableNames() throws Exception { return candidates; } - List databases = client.getDatabases(catalogName, dbPattern); + List catalogs = client.getCatalogs(catalogPattern); - for (String db : databases) { - List tablesNames = getTableNamesForDatabase(catalogName, db); - tablesNames.forEach(tablesName -> candidates.add(TableName.fromString(tablesName, catalogName, db))); + for (String catalogName : catalogs) { + List databases = client.getDatabases(catalogName, dbPattern); + for (String db : databases) { + List tablesNames = getTableNamesForDatabase(catalogName, db); + tablesNames.forEach(tablesName -> candidates.add(TableName.fromString(tablesName, catalogName, db))); + } } return candidates; } - public Iterable
getTables(Set skipDBs, Set skipTables, int maxBatchSize) throws Exception { + public Iterable
getTables(Set skipCatalogs, Set skipDBs, Set skipTables, int maxBatchSize) throws Exception { // if tableTypes is empty, then a list with single empty string has to specified to scan no tables. if (tableTypes.isEmpty()) { LOG.info("Table fetcher returns empty list as no table types specified"); return Collections.emptyList(); } - List databases = client.getDatabases(catalogName, dbPattern).stream() - .filter(dbName -> skipDBs == null || !skipDBs.contains(dbName)) + List catalogs = client.getCatalogs(catalogPattern).stream() + .filter(catalogName -> skipCatalogs == null || !skipCatalogs.contains(catalogName)) .toList(); return () -> Iterators.concat( - Iterators.transform(databases.iterator(), db -> { + Iterators.transform(catalogs.iterator(), catalogName -> { try { - List tableNames = getTableNamesForDatabase(catalogName, db).stream() - .filter(tableName -> skipTables == null || !skipTables.contains(TableName.getDbTable(db, tableName))) + List databases = client.getDatabases(catalogName, dbPattern).stream() + .filter(dbName -> skipDBs == null || !skipDBs.contains(dbName)) .toList(); - return new TableIterable(client, db, tableNames, maxBatchSize).iterator(); + return Iterators.concat( + Iterators.transform(databases.iterator(), dbName -> { + try { + List tableNames = getTableNamesForDatabase(catalogName, dbName).stream() + .filter(tableName -> skipTables == null || + !skipTables.contains(TableName.getDbTable(dbName, tableName))) + .toList(); + return new TableIterable(client, dbName, tableNames, maxBatchSize).iterator(); + } catch (Exception e) { + throw new RuntimeException("Failed to fetch tables for db: " + dbName, e); + } + }) + ); } catch (Exception e) { - throw new RuntimeException("Failed to fetch tables for db: " + db, e); + throw new RuntimeException("Failed to fetch database for catalog: " + catalogName, e); } }) ); } public Iterable
getTables(int maxBatchSize) throws Exception { - return getTables(null, null, maxBatchSize); + return getTables(null, null, null, maxBatchSize); } private List getTableNamesForDatabase(String catalogName, String dbName) throws Exception { @@ -158,15 +169,15 @@ private List getTableNamesForDatabase(String catalogName, String dbName) public static class Builder { private final IMetaStoreClient client; - private final String catalogName; + private final String catalogPattern; private final String dbPattern; private final String tablePattern; private final List tableConditions = new ArrayList<>(); private String tableTypes; - public Builder(IMetaStoreClient client, String catalogName, String dbPattern, String tablePattern) { + public Builder(IMetaStoreClient client, String catalogPattern, String dbPattern, String tablePattern) { this.client = client; - this.catalogName = catalogName; + this.catalogPattern = catalogPattern; this.dbPattern = dbPattern; this.tablePattern = tablePattern; } diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp index 11bdf1675b76..1c00fed2c0e1 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/ThriftHiveMetastore.cpp @@ -1310,7 +1310,20 @@ uint32_t ThriftHiveMetastore_get_catalogs_args::read(::apache::thrift::protocol: if (ftype == ::apache::thrift::protocol::T_STOP) { break; } - xfer += iprot->skip(ftype); + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->pattern.read(iprot); + this->__isset.pattern = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } xfer += iprot->readFieldEnd(); } @@ -1324,6 +1337,10 @@ uint32_t ThriftHiveMetastore_get_catalogs_args::write(::apache::thrift::protocol ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); xfer += oprot->writeStructBegin("ThriftHiveMetastore_get_catalogs_args"); + xfer += oprot->writeFieldBegin("pattern", ::apache::thrift::protocol::T_STRUCT, 1); + xfer += this->pattern.write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -1339,6 +1356,10 @@ uint32_t ThriftHiveMetastore_get_catalogs_pargs::write(::apache::thrift::protoco ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); xfer += oprot->writeStructBegin("ThriftHiveMetastore_get_catalogs_pargs"); + xfer += oprot->writeFieldBegin("pattern", ::apache::thrift::protocol::T_STRUCT, 1); + xfer += (*(this->pattern)).write(oprot); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -67317,18 +67338,19 @@ void ThriftHiveMetastoreClient::recv_get_catalog(GetCatalogResponse& _return) throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "get_catalog failed: unknown result"); } -void ThriftHiveMetastoreClient::get_catalogs(GetCatalogsResponse& _return) +void ThriftHiveMetastoreClient::get_catalogs(GetCatalogsResponse& _return, const GetCatalogRequest& pattern) { - send_get_catalogs(); + send_get_catalogs(pattern); recv_get_catalogs(_return); } -void ThriftHiveMetastoreClient::send_get_catalogs() +void ThriftHiveMetastoreClient::send_get_catalogs(const GetCatalogRequest& pattern) { int32_t cseqid = 0; oprot_->writeMessageBegin("get_catalogs", ::apache::thrift::protocol::T_CALL, cseqid); ThriftHiveMetastore_get_catalogs_pargs args; + args.pattern = &pattern; args.write(oprot_); oprot_->writeMessageEnd(); @@ -85144,7 +85166,7 @@ void ThriftHiveMetastoreProcessor::process_get_catalogs(int32_t seqid, ::apache: ThriftHiveMetastore_get_catalogs_result result; try { - iface_->get_catalogs(result.success); + iface_->get_catalogs(result.success, args.pattern); result.__isset.success = true; } catch (MetaException &o1) { result.o1 = std::move(o1); @@ -102124,19 +102146,20 @@ void ThriftHiveMetastoreConcurrentClient::recv_get_catalog(GetCatalogResponse& _ } // end while(true) } -void ThriftHiveMetastoreConcurrentClient::get_catalogs(GetCatalogsResponse& _return) +void ThriftHiveMetastoreConcurrentClient::get_catalogs(GetCatalogsResponse& _return, const GetCatalogRequest& pattern) { - int32_t seqid = send_get_catalogs(); + int32_t seqid = send_get_catalogs(pattern); recv_get_catalogs(_return, seqid); } -int32_t ThriftHiveMetastoreConcurrentClient::send_get_catalogs() +int32_t ThriftHiveMetastoreConcurrentClient::send_get_catalogs(const GetCatalogRequest& pattern) { int32_t cseqid = this->sync_->generateSeqId(); ::apache::thrift::async::TConcurrentSendSentry sentry(this->sync_.get()); oprot_->writeMessageBegin("get_catalogs", ::apache::thrift::protocol::T_CALL, cseqid); ThriftHiveMetastore_get_catalogs_pargs args; + args.pattern = &pattern; args.write(oprot_); oprot_->writeMessageEnd(); diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h index 765213815211..a3192ab726f2 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/ThriftHiveMetastore.h @@ -32,7 +32,7 @@ class ThriftHiveMetastoreIf : virtual public ::facebook::fb303::FacebookService virtual void create_catalog(const CreateCatalogRequest& catalog) = 0; virtual void alter_catalog(const AlterCatalogRequest& rqst) = 0; virtual void get_catalog(GetCatalogResponse& _return, const GetCatalogRequest& catName) = 0; - virtual void get_catalogs(GetCatalogsResponse& _return) = 0; + virtual void get_catalogs(GetCatalogsResponse& _return, const GetCatalogRequest& pattern) = 0; virtual void drop_catalog(const DropCatalogRequest& catName) = 0; virtual void create_database(const Database& database) = 0; virtual void create_database_req(const CreateDatabaseRequest& createDatabaseRequest) = 0; @@ -357,7 +357,7 @@ class ThriftHiveMetastoreNull : virtual public ThriftHiveMetastoreIf , virtual p void get_catalog(GetCatalogResponse& /* _return */, const GetCatalogRequest& /* catName */) override { return; } - void get_catalogs(GetCatalogsResponse& /* _return */) override { + void get_catalogs(GetCatalogsResponse& /* _return */, const GetCatalogRequest& /* pattern */) override { return; } void drop_catalog(const DropCatalogRequest& /* catName */) override { @@ -1918,19 +1918,30 @@ class ThriftHiveMetastore_get_catalog_presult { }; +typedef struct _ThriftHiveMetastore_get_catalogs_args__isset { + _ThriftHiveMetastore_get_catalogs_args__isset() : pattern(false) {} + bool pattern :1; +} _ThriftHiveMetastore_get_catalogs_args__isset; class ThriftHiveMetastore_get_catalogs_args { public: - ThriftHiveMetastore_get_catalogs_args(const ThriftHiveMetastore_get_catalogs_args&) noexcept; - ThriftHiveMetastore_get_catalogs_args& operator=(const ThriftHiveMetastore_get_catalogs_args&) noexcept; + ThriftHiveMetastore_get_catalogs_args(const ThriftHiveMetastore_get_catalogs_args&); + ThriftHiveMetastore_get_catalogs_args& operator=(const ThriftHiveMetastore_get_catalogs_args&); ThriftHiveMetastore_get_catalogs_args() noexcept { } virtual ~ThriftHiveMetastore_get_catalogs_args() noexcept; + GetCatalogRequest pattern; + + _ThriftHiveMetastore_get_catalogs_args__isset __isset; + + void __set_pattern(const GetCatalogRequest& val); - bool operator == (const ThriftHiveMetastore_get_catalogs_args & /* rhs */) const + bool operator == (const ThriftHiveMetastore_get_catalogs_args & rhs) const { + if (!(pattern == rhs.pattern)) + return false; return true; } bool operator != (const ThriftHiveMetastore_get_catalogs_args &rhs) const { @@ -1950,6 +1961,7 @@ class ThriftHiveMetastore_get_catalogs_pargs { virtual ~ThriftHiveMetastore_get_catalogs_pargs() noexcept; + const GetCatalogRequest* pattern; uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; @@ -35693,8 +35705,8 @@ class ThriftHiveMetastoreClient : virtual public ThriftHiveMetastoreIf, public void get_catalog(GetCatalogResponse& _return, const GetCatalogRequest& catName) override; void send_get_catalog(const GetCatalogRequest& catName); void recv_get_catalog(GetCatalogResponse& _return); - void get_catalogs(GetCatalogsResponse& _return) override; - void send_get_catalogs(); + void get_catalogs(GetCatalogsResponse& _return, const GetCatalogRequest& pattern) override; + void send_get_catalogs(const GetCatalogRequest& pattern); void recv_get_catalogs(GetCatalogsResponse& _return); void drop_catalog(const DropCatalogRequest& catName) override; void send_drop_catalog(const DropCatalogRequest& catName); @@ -37199,13 +37211,13 @@ class ThriftHiveMetastoreMultiface : virtual public ThriftHiveMetastoreIf, publi return; } - void get_catalogs(GetCatalogsResponse& _return) override { + void get_catalogs(GetCatalogsResponse& _return, const GetCatalogRequest& pattern) override { size_t sz = ifaces_.size(); size_t i = 0; for (; i < (sz - 1); ++i) { - ifaces_[i]->get_catalogs(_return); + ifaces_[i]->get_catalogs(_return, pattern); } - ifaces_[i]->get_catalogs(_return); + ifaces_[i]->get_catalogs(_return, pattern); return; } @@ -39903,8 +39915,8 @@ class ThriftHiveMetastoreConcurrentClient : virtual public ThriftHiveMetastoreIf void get_catalog(GetCatalogResponse& _return, const GetCatalogRequest& catName) override; int32_t send_get_catalog(const GetCatalogRequest& catName); void recv_get_catalog(GetCatalogResponse& _return, const int32_t seqid); - void get_catalogs(GetCatalogsResponse& _return) override; - int32_t send_get_catalogs(); + void get_catalogs(GetCatalogsResponse& _return, const GetCatalogRequest& pattern) override; + int32_t send_get_catalogs(const GetCatalogRequest& pattern); void recv_get_catalogs(GetCatalogsResponse& _return, const int32_t seqid); void drop_catalog(const DropCatalogRequest& catName) override; int32_t send_drop_catalog(const DropCatalogRequest& catName); diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp index 3f78ab375f73..7fbbdd38e815 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/ThriftHiveMetastore_server.skeleton.cpp @@ -50,7 +50,7 @@ class ThriftHiveMetastoreHandler : virtual public ThriftHiveMetastoreIf { printf("get_catalog\n"); } - void get_catalogs(GetCatalogsResponse& _return) { + void get_catalogs(GetCatalogsResponse& _return, const GetCatalogRequest& pattern) { // Your implementation goes here printf("get_catalogs\n"); } diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java index 0b4f3a04bdfc..9a30fdbcb98c 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ThriftHiveMetastore.java @@ -27,7 +27,7 @@ public GetCatalogResponse get_catalog(GetCatalogRequest catName) throws NoSuchObjectException, MetaException, org.apache.thrift.TException; - public GetCatalogsResponse get_catalogs() throws MetaException, org.apache.thrift.TException; + public GetCatalogsResponse get_catalogs(GetCatalogRequest pattern) throws MetaException, org.apache.thrift.TException; public void drop_catalog(DropCatalogRequest catName) throws NoSuchObjectException, InvalidOperationException, MetaException, org.apache.thrift.TException; @@ -599,7 +599,7 @@ public void get_catalog(GetCatalogRequest catName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; - public void get_catalogs(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + public void get_catalogs(GetCatalogRequest pattern, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; public void drop_catalog(DropCatalogRequest catName, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; @@ -1337,15 +1337,16 @@ public GetCatalogResponse recv_get_catalog() throws NoSuchObjectException, MetaE throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "get_catalog failed: unknown result"); } - public GetCatalogsResponse get_catalogs() throws MetaException, org.apache.thrift.TException + public GetCatalogsResponse get_catalogs(GetCatalogRequest pattern) throws MetaException, org.apache.thrift.TException { - send_get_catalogs(); + send_get_catalogs(pattern); return recv_get_catalogs(); } - public void send_get_catalogs() throws org.apache.thrift.TException + public void send_get_catalogs(GetCatalogRequest pattern) throws org.apache.thrift.TException { get_catalogs_args args = new get_catalogs_args(); + args.setPattern(pattern); sendBase("get_catalogs", args); } @@ -9415,21 +9416,24 @@ public GetCatalogResponse getResult() throws NoSuchObjectException, MetaExceptio } } - public void get_catalogs(org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + public void get_catalogs(GetCatalogRequest pattern, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { checkReady(); - get_catalogs_call method_call = new get_catalogs_call(resultHandler, this, ___protocolFactory, ___transport); + get_catalogs_call method_call = new get_catalogs_call(pattern, resultHandler, this, ___protocolFactory, ___transport); this.___currentMethod = method_call; ___manager.call(method_call); } @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class get_catalogs_call extends org.apache.thrift.async.TAsyncMethodCall { - public get_catalogs_call(org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + private GetCatalogRequest pattern; + public get_catalogs_call(GetCatalogRequest pattern, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { super(client, protocolFactory, transport, resultHandler, false); + this.pattern = pattern; } public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("get_catalogs", org.apache.thrift.protocol.TMessageType.CALL, 0)); get_catalogs_args args = new get_catalogs_args(); + args.setPattern(pattern); args.write(prot); prot.writeMessageEnd(); } @@ -19310,7 +19314,7 @@ protected boolean rethrowUnhandledExceptions() { public get_catalogs_result getResult(I iface, get_catalogs_args args) throws org.apache.thrift.TException { get_catalogs_result result = new get_catalogs_result(); try { - result.success = iface.get_catalogs(); + result.success = iface.get_catalogs(args.pattern); } catch (MetaException o1) { result.o1 = o1; } @@ -28547,7 +28551,7 @@ protected boolean isOneway() { } public void start(I iface, get_catalogs_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { - iface.get_catalogs(resultHandler); + iface.get_catalogs(args.pattern,resultHandler); } } @@ -52677,14 +52681,16 @@ private static S scheme(org.apache. @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public static class get_catalogs_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("get_catalogs_args"); + private static final org.apache.thrift.protocol.TField PATTERN_FIELD_DESC = new org.apache.thrift.protocol.TField("pattern", org.apache.thrift.protocol.TType.STRUCT, (short)1); private static final org.apache.thrift.scheme.SchemeFactory STANDARD_SCHEME_FACTORY = new get_catalogs_argsStandardSchemeFactory(); private static final org.apache.thrift.scheme.SchemeFactory TUPLE_SCHEME_FACTORY = new get_catalogs_argsTupleSchemeFactory(); + private @org.apache.thrift.annotation.Nullable GetCatalogRequest pattern; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { -; + PATTERN((short)1, "pattern"); private static final java.util.Map byName = new java.util.HashMap(); @@ -52700,6 +52706,8 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @org.apache.thrift.annotation.Nullable public static _Fields findByThriftId(int fieldId) { switch(fieldId) { + case 1: // PATTERN + return PATTERN; default: return null; } @@ -52739,9 +52747,13 @@ public java.lang.String getFieldName() { return _fieldName; } } + + // isset id assignments public static final java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new java.util.EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.PATTERN, new org.apache.thrift.meta_data.FieldMetaData("pattern", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, GetCatalogRequest.class))); metaDataMap = java.util.Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(get_catalogs_args.class, metaDataMap); } @@ -52749,10 +52761,20 @@ public java.lang.String getFieldName() { public get_catalogs_args() { } + public get_catalogs_args( + GetCatalogRequest pattern) + { + this(); + this.pattern = pattern; + } + /** * Performs a deep copy on other. */ public get_catalogs_args(get_catalogs_args other) { + if (other.isSetPattern()) { + this.pattern = new GetCatalogRequest(other.pattern); + } } public get_catalogs_args deepCopy() { @@ -52761,16 +52783,52 @@ public get_catalogs_args deepCopy() { @Override public void clear() { + this.pattern = null; + } + + @org.apache.thrift.annotation.Nullable + public GetCatalogRequest getPattern() { + return this.pattern; + } + + public void setPattern(@org.apache.thrift.annotation.Nullable GetCatalogRequest pattern) { + this.pattern = pattern; + } + + public void unsetPattern() { + this.pattern = null; + } + + /** Returns true if field pattern is set (has been assigned a value) and false otherwise */ + public boolean isSetPattern() { + return this.pattern != null; + } + + public void setPatternIsSet(boolean value) { + if (!value) { + this.pattern = null; + } } public void setFieldValue(_Fields field, @org.apache.thrift.annotation.Nullable java.lang.Object value) { switch (field) { + case PATTERN: + if (value == null) { + unsetPattern(); + } else { + setPattern((GetCatalogRequest)value); + } + break; + } } @org.apache.thrift.annotation.Nullable public java.lang.Object getFieldValue(_Fields field) { switch (field) { + case PATTERN: + return getPattern(); + } throw new java.lang.IllegalStateException(); } @@ -52782,6 +52840,8 @@ public boolean isSet(_Fields field) { } switch (field) { + case PATTERN: + return isSetPattern(); } throw new java.lang.IllegalStateException(); } @@ -52799,6 +52859,15 @@ public boolean equals(get_catalogs_args that) { if (this == that) return true; + boolean this_present_pattern = true && this.isSetPattern(); + boolean that_present_pattern = true && that.isSetPattern(); + if (this_present_pattern || that_present_pattern) { + if (!(this_present_pattern && that_present_pattern)) + return false; + if (!this.pattern.equals(that.pattern)) + return false; + } + return true; } @@ -52806,6 +52875,10 @@ public boolean equals(get_catalogs_args that) { public int hashCode() { int hashCode = 1; + hashCode = hashCode * 8191 + ((isSetPattern()) ? 131071 : 524287); + if (isSetPattern()) + hashCode = hashCode * 8191 + pattern.hashCode(); + return hashCode; } @@ -52817,6 +52890,16 @@ public int compareTo(get_catalogs_args other) { int lastComparison = 0; + lastComparison = java.lang.Boolean.compare(isSetPattern(), other.isSetPattern()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetPattern()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.pattern, other.pattern); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -52838,6 +52921,13 @@ public java.lang.String toString() { java.lang.StringBuilder sb = new java.lang.StringBuilder("get_catalogs_args("); boolean first = true; + sb.append("pattern:"); + if (this.pattern == null) { + sb.append("null"); + } else { + sb.append(this.pattern); + } + first = false; sb.append(")"); return sb.toString(); } @@ -52845,6 +52935,9 @@ public java.lang.String toString() { public void validate() throws org.apache.thrift.TException { // check for required fields // check for sub-struct validity + if (pattern != null) { + pattern.validate(); + } } private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { @@ -52881,6 +52974,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, get_catalogs_args s break; } switch (schemeField.id) { + case 1: // PATTERN + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.pattern = new GetCatalogRequest(); + struct.pattern.read(iprot); + struct.setPatternIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -52894,6 +52996,11 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, get_catalogs_args struct.validate(); oprot.writeStructBegin(STRUCT_DESC); + if (struct.pattern != null) { + oprot.writeFieldBegin(PATTERN_FIELD_DESC); + struct.pattern.write(oprot); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -52911,11 +53018,25 @@ private static class get_catalogs_argsTupleScheme extends org.apache.thrift.sche @Override public void write(org.apache.thrift.protocol.TProtocol prot, get_catalogs_args struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol oprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet optionals = new java.util.BitSet(); + if (struct.isSetPattern()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetPattern()) { + struct.pattern.write(oprot); + } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, get_catalogs_args struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TTupleProtocol iprot = (org.apache.thrift.protocol.TTupleProtocol) prot; + java.util.BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.pattern = new GetCatalogRequest(); + struct.pattern.read(iprot); + struct.setPatternIsSet(true); + } } } diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastoreClient.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastoreClient.php index 2b7e324917d1..78bcadb26f57 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastoreClient.php +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastoreClient.php @@ -400,15 +400,16 @@ public function recv_get_catalog() throw new \Exception("get_catalog failed: unknown result"); } - public function get_catalogs() + public function get_catalogs(\metastore\GetCatalogRequest $pattern) { - $this->send_get_catalogs(); + $this->send_get_catalogs($pattern); return $this->recv_get_catalogs(); } - public function send_get_catalogs() + public function send_get_catalogs(\metastore\GetCatalogRequest $pattern) { $args = new \metastore\ThriftHiveMetastore_get_catalogs_args(); + $args->pattern = $pattern; $bin_accel = ($this->output_ instanceof TBinaryProtocolAccelerated) && function_exists('thrift_protocol_write_binary'); if ($bin_accel) { thrift_protocol_write_binary( diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastoreIf.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastoreIf.php index bffe6f4a7de4..8fff1ad47570 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastoreIf.php +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastoreIf.php @@ -60,10 +60,11 @@ public function alter_catalog(\metastore\AlterCatalogRequest $rqst); */ public function get_catalog(\metastore\GetCatalogRequest $catName); /** + * @param \metastore\GetCatalogRequest $pattern * @return \metastore\GetCatalogsResponse * @throws \metastore\MetaException */ - public function get_catalogs(); + public function get_catalogs(\metastore\GetCatalogRequest $pattern); /** * @param \metastore\DropCatalogRequest $catName * @throws \metastore\NoSuchObjectException diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore_get_catalogs_args.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore_get_catalogs_args.php index b8a29f67f174..311ed9370495 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore_get_catalogs_args.php +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore_get_catalogs_args.php @@ -21,11 +21,26 @@ class ThriftHiveMetastore_get_catalogs_args static public $isValidate = false; static public $_TSPEC = array( + 1 => array( + 'var' => 'pattern', + 'isRequired' => false, + 'type' => TType::STRUCT, + 'class' => '\metastore\GetCatalogRequest', + ), ); + /** + * @var \metastore\GetCatalogRequest + */ + public $pattern = null; - public function __construct() + public function __construct($vals = null) { + if (is_array($vals)) { + if (isset($vals['pattern'])) { + $this->pattern = $vals['pattern']; + } + } } public function getName() @@ -47,6 +62,14 @@ public function read($input) break; } switch ($fid) { + case 1: + if ($ftype == TType::STRUCT) { + $this->pattern = new \metastore\GetCatalogRequest(); + $xfer += $this->pattern->read($input); + } else { + $xfer += $input->skip($ftype); + } + break; default: $xfer += $input->skip($ftype); break; @@ -61,6 +84,14 @@ public function write($output) { $xfer = 0; $xfer += $output->writeStructBegin('ThriftHiveMetastore_get_catalogs_args'); + if ($this->pattern !== null) { + if (!is_object($this->pattern)) { + throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA); + } + $xfer += $output->writeFieldBegin('pattern', TType::STRUCT, 1); + $xfer += $this->pattern->write($output); + $xfer += $output->writeFieldEnd(); + } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote index 51067bcd5f85..755536ccba27 100755 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote @@ -30,7 +30,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print(' void create_catalog(CreateCatalogRequest catalog)') print(' void alter_catalog(AlterCatalogRequest rqst)') print(' GetCatalogResponse get_catalog(GetCatalogRequest catName)') - print(' GetCatalogsResponse get_catalogs()') + print(' GetCatalogsResponse get_catalogs(GetCatalogRequest pattern)') print(' void drop_catalog(DropCatalogRequest catName)') print(' void create_database(Database database)') print(' void create_database_req(CreateDatabaseRequest createDatabaseRequest)') @@ -437,10 +437,10 @@ elif cmd == 'get_catalog': pp.pprint(client.get_catalog(eval(args[0]),)) elif cmd == 'get_catalogs': - if len(args) != 0: - print('get_catalogs requires 0 args') + if len(args) != 1: + print('get_catalogs requires 1 args') sys.exit(1) - pp.pprint(client.get_catalogs()) + pp.pprint(client.get_catalogs(eval(args[0]),)) elif cmd == 'drop_catalog': if len(args) != 1: diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py index 159db857053e..24fa2c333698 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py @@ -73,7 +73,12 @@ def get_catalog(self, catName): """ pass - def get_catalogs(self): + def get_catalogs(self, pattern): + """ + Parameters: + - pattern + + """ pass def drop_catalog(self, catName): @@ -2628,13 +2633,19 @@ def recv_get_catalog(self): raise result.o2 raise TApplicationException(TApplicationException.MISSING_RESULT, "get_catalog failed: unknown result") - def get_catalogs(self): - self.send_get_catalogs() + def get_catalogs(self, pattern): + """ + Parameters: + - pattern + + """ + self.send_get_catalogs(pattern) return self.recv_get_catalogs() - def send_get_catalogs(self): + def send_get_catalogs(self, pattern): self._oprot.writeMessageBegin('get_catalogs', TMessageType.CALL, self._seqid) args = get_catalogs_args() + args.pattern = pattern args.write(self._oprot) self._oprot.writeMessageEnd() self._oprot.trans.flush() @@ -13158,7 +13169,7 @@ def process_get_catalogs(self, seqid, iprot, oprot): iprot.readMessageEnd() result = get_catalogs_result() try: - result.success = self._handler.get_catalogs() + result.success = self._handler.get_catalogs(args.pattern) msg_type = TMessageType.REPLY except TTransport.TTransportException: raise @@ -21926,8 +21937,16 @@ def __ne__(self, other): class get_catalogs_args(object): + """ + Attributes: + - pattern + + """ + def __init__(self, pattern=None,): + self.pattern = pattern + def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) @@ -21937,6 +21956,12 @@ def read(self, iprot): (fname, ftype, fid) = iprot.readFieldBegin() if ftype == TType.STOP: break + if fid == 1: + if ftype == TType.STRUCT: + self.pattern = GetCatalogRequest() + self.pattern.read(iprot) + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -21947,6 +21972,10 @@ def write(self, oprot): oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return oprot.writeStructBegin('get_catalogs_args') + if self.pattern is not None: + oprot.writeFieldBegin('pattern', TType.STRUCT, 1) + self.pattern.write(oprot) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -21965,6 +21994,8 @@ def __ne__(self, other): return not (self == other) all_structs.append(get_catalogs_args) get_catalogs_args.thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'pattern', [GetCatalogRequest, None], None, ), # 1 ) diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/thrift_hive_metastore.rb b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/thrift_hive_metastore.rb index e622846a8100..69572de8b06e 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/thrift_hive_metastore.rb +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/thrift_hive_metastore.rb @@ -109,13 +109,13 @@ def recv_get_catalog() raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_catalog failed: unknown result') end - def get_catalogs() - send_get_catalogs() + def get_catalogs(pattern) + send_get_catalogs(pattern) return recv_get_catalogs() end - def send_get_catalogs() - send_message('get_catalogs', Get_catalogs_args) + def send_get_catalogs(pattern) + send_message('get_catalogs', Get_catalogs_args, :pattern => pattern) end def recv_get_catalogs() @@ -4795,7 +4795,7 @@ def process_get_catalogs(seqid, iprot, oprot) args = read_args(iprot, Get_catalogs_args) result = Get_catalogs_result.new() begin - result.success = @handler.get_catalogs() + result.success = @handler.get_catalogs(args.pattern) rescue ::MetaException => o1 result.o1 = o1 end @@ -8461,9 +8461,10 @@ def validate class Get_catalogs_args include ::Thrift::Struct, ::Thrift::Struct_Union + PATTERN = 1 FIELDS = { - + PATTERN => {:type => ::Thrift::Types::STRUCT, :name => 'pattern', :class => ::GetCatalogRequest} } def struct_fields; FIELDS; end diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/AbstractThriftHiveMetastore.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/AbstractThriftHiveMetastore.java index d3ec3b6bd828..b403ad9dcfda 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/AbstractThriftHiveMetastore.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/AbstractThriftHiveMetastore.java @@ -63,7 +63,7 @@ public GetCatalogResponse get_catalog(GetCatalogRequest catName) } @Override - public GetCatalogsResponse get_catalogs() throws MetaException, TException { + public GetCatalogsResponse get_catalogs(GetCatalogRequest catalogPattern) throws MetaException, TException { throw new UnsupportedOperationException("this method is not supported"); } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 91e68d7921a4..652fc1e4197a 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -926,10 +926,11 @@ public enum ConfVars { ICEBERG_TABLE_EXPIRY_INTERVAL("metastore.iceberg.table.expiry.interval", "hive.metastore.iceberg.table.expiry.interval", 3600, TimeUnit.SECONDS, "Time interval describing how often the iceberg table expiry service runs."), - // TODO: HIVE-28974: Implement pattern-based catalog retrieval in metastore client - ICEBERG_TABLE_EXPIRY_CATALOG_NAME("metastore.iceberg.table.expiry.catalog.name", - "hive.metastore.iceberg.table.expiry.catalog.name", "hive", - "Iceberg table expiry service looks for tables under the specified catalog name"), + ICEBERG_TABLE_EXPIRY_CATALOG_PATTERN("metastore.iceberg.table.expiry.catalog.pattern", + "hive.metastore.iceberg.table.expiry.catalog.pattern", "", + "Iceberg table expiry service searches for tables using the specified catalog pattern. " + + "By default, the pattern is set to empty string, which results in no matches (this is intentional" + + "to avoid expensive metastore calls unless explicitly configured by the user)."), ICEBERG_TABLE_EXPIRY_DATABASE_PATTERN("metastore.iceberg.table.expiry.database.pattern", "hive.metastore.iceberg.table.expiry.database.pattern", "", "Iceberg table expiry service searches for tables using the specified database pattern. " + diff --git a/standalone-metastore/metastore-common/src/main/protobuf/org/apache/hadoop/hive/metastore/hive_metastore.proto b/standalone-metastore/metastore-common/src/main/protobuf/org/apache/hadoop/hive/metastore/hive_metastore.proto index 271e097b9a06..4e9959bb53b9 100644 --- a/standalone-metastore/metastore-common/src/main/protobuf/org/apache/hadoop/hive/metastore/hive_metastore.proto +++ b/standalone-metastore/metastore-common/src/main/protobuf/org/apache/hadoop/hive/metastore/hive_metastore.proto @@ -66,6 +66,7 @@ message GetCatalogResponse { } message GetCatalogsRequest { + string pattern = 1; } message GetCatalogsResponse { diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift index df0721655295..05591fd2ee6c 100644 --- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift @@ -2619,7 +2619,7 @@ service ThriftHiveMetastore extends fb303.FacebookService void create_catalog(1: CreateCatalogRequest catalog) throws (1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3: MetaException o3) void alter_catalog(1: AlterCatalogRequest rqst) throws (1:NoSuchObjectException o1, 2:InvalidOperationException o2, 3:MetaException o3) GetCatalogResponse get_catalog(1: GetCatalogRequest catName) throws (1:NoSuchObjectException o1, 2:MetaException o2) - GetCatalogsResponse get_catalogs() throws (1:MetaException o1) + GetCatalogsResponse get_catalogs(1: GetCatalogRequest pattern) throws (1:MetaException o1) void drop_catalog(1: DropCatalogRequest catName) throws (1:NoSuchObjectException o1, 2:InvalidOperationException o2, 3:MetaException o3) void create_database(1:Database database) throws(1:AlreadyExistsException o1, 2:InvalidObjectException o2, 3:MetaException o3) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java index 086766de41bb..8aa646ee9a01 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java @@ -1038,13 +1038,14 @@ public GetCatalogResponse get_catalog(GetCatalogRequest rqst) } @Override - public GetCatalogsResponse get_catalogs() throws MetaException { + public GetCatalogsResponse get_catalogs(GetCatalogRequest rqst) throws MetaException { + String catalogPattern = rqst.getName(); startFunction("get_catalogs"); List ret = null; Exception ex = null; try { - ret = getMS().getCatalogs(); + ret = getMS().getCatalogs(catalogPattern); } catch (Exception e) { ex = e; throw e; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 688a1142efbc..6cb34dc2cac8 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -724,19 +724,27 @@ public Catalog getCatalog(String catalogName) throws NoSuchObjectException, Meta } @Override - public List getCatalogs() { - LOG.debug("Fetching all catalog names"); + public List getCatalogs(String catalogPattern) { + if (catalogPattern == null) { + catalogPattern = "*"; + LOG.debug("Fetching all catalog names"); + } else { + LOG.debug("Fetching catalog names with pattern " + catalogPattern); + } boolean commited = false; List catalogs = null; - String queryStr = "select name from org.apache.hadoop.hive.metastore.model.MCatalog"; Query query = null; openTransaction(); try { - query = pm.newQuery(queryStr); + StringBuilder filterBuilder = new StringBuilder(); + List parameterVals = new ArrayList<>(); + appendPatternCondition(filterBuilder, "name", catalogPattern, parameterVals); + query = pm.newQuery(MCatalog.class, filterBuilder.toString()); query.setResult("name"); - catalogs = new ArrayList<>((Collection) query.execute()); + query.setOrdering("name ascending"); + catalogs = new ArrayList<>((Collection) query.executeWithArray(parameterVals.toArray(new String[0]))); commited = commitTransaction(); } finally { rollbackAndCleanup(commited, query); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index d0e73d75ec92..48b8d3d0ff60 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -188,7 +188,7 @@ public interface RawStore extends Configurable { * Get all the catalogs. * @return list of names of all catalogs in the system */ - List getCatalogs(); + List getCatalogs(String catalogPattern); /** * Drop a catalog. The catalog must be empty. diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index eb25f7e4452d..35ce95142dc8 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -684,7 +684,7 @@ private static void initBlackListWhiteList(Configuration conf) { private static Collection catalogsToCache(RawStore rs) throws MetaException { Collection confValue = MetastoreConf.getStringCollection(rs.getConf(), ConfVars.CATALOGS_TO_CACHE); if (confValue == null || confValue.isEmpty() || (confValue.size() == 1 && confValue.contains(""))) { - return rs.getCatalogs(); + return rs.getCatalogs(null); } else { return confValue; } @@ -1105,10 +1105,10 @@ private void updateCacheUsingEvents() { return cat; } - @Override public List getCatalogs() { + @Override public List getCatalogs(String catalogPattern) { // in case of event based cache update, cache will not be updated for catalog. if (!sharedCache.isCatalogCachePrewarmed() || canUseEvents) { - return rawStore.getCatalogs(); + return rawStore.getCatalogs(catalogPattern); } return sharedCache.listCachedCatalogs(); } diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java index 4a2408b69e2a..51b506655d12 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -211,7 +211,7 @@ public void catalogs() throws MetaException, NoSuchObjectException { objectStore.createCatalog(cat); } - List fetchedNames = objectStore.getCatalogs(); + List fetchedNames = objectStore.getCatalogs(null); Assert.assertEquals(3, fetchedNames.size()); for (int i = 0; i < names.length - 1; i++) { Assert.assertEquals(names[i], fetchedNames.get(i)); @@ -226,7 +226,7 @@ public void catalogs() throws MetaException, NoSuchObjectException { // Location will vary by system. for (int i = 0; i < names.length; i++) objectStore.dropCatalog(names[i]); - fetchedNames = objectStore.getCatalogs(); + fetchedNames = objectStore.getCatalogs(null); Assert.assertEquals(1, fetchedNames.size()); } @@ -1247,7 +1247,7 @@ private static void dropAllStoreObjects(RawStore store) for (Function func : functions) { store.dropFunction(DEFAULT_CATALOG_NAME, func.getDbName(), func.getFunctionName()); } - for (String catName : store.getCatalogs()) { + for (String catName : store.getCatalogs(null)) { List dbs = store.getAllDatabases(catName); for (String db : dbs) { List tbls = store.getAllTables(DEFAULT_CATALOG_NAME, db); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java index 4b5e1813f0ba..be6a26a1c1af 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java @@ -117,7 +117,7 @@ MetaStoreTestUtils.setConfForStandloneMode(conf); ObjectStore objectStore = new ObjectStore(); objectStore.setConf(conf); - for (String clg : objectStore.getCatalogs()) { + for (String clg : objectStore.getCatalogs(null)) { for (String db : objectStore.getAllDatabases(clg)) { for (String tbl : objectStore.getAllTables(clg, db)) { List pts = objectStore.listPartitionNames(clg, db, tbl, Short.MAX_VALUE); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCatalogCaching.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCatalogCaching.java index 0e95538a2ed7..3ae00c13a3dc 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCatalogCaching.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCatalogCaching.java @@ -75,7 +75,7 @@ public void createObjectStore() throws MetaException, InvalidOperationException @After public void clearCatalogCache() throws MetaException, NoSuchObjectException { - List catalogs = objectStore.getCatalogs(); + List catalogs = objectStore.getCatalogs(null); for (String catalog : catalogs) objectStore.dropCatalog(catalog); } @@ -90,7 +90,7 @@ public void defaultHiveOnly() throws Exception { CachedStore.prewarm(objectStore); // Only the hive catalog should be cached - List cachedCatalogs = cachedStore.getCatalogs(); + List cachedCatalogs = cachedStore.getCatalogs(null); Assert.assertEquals(1, cachedCatalogs.size()); Assert.assertEquals(Warehouse.DEFAULT_CATALOG_NAME, cachedCatalogs.get(0)); } @@ -110,7 +110,7 @@ public void cacheAll() throws Exception { CachedStore.prewarm(objectStore); // All the catalogs should be cached - List cachedCatalogs = cachedStore.getCatalogs(); + List cachedCatalogs = cachedStore.getCatalogs(null); Assert.assertEquals(3, cachedCatalogs.size()); cachedCatalogs.sort(Comparator.naturalOrder()); Assert.assertEquals(CAT1_NAME, cachedCatalogs.get(0)); @@ -133,7 +133,7 @@ public void cacheSome() throws Exception { CachedStore.prewarm(objectStore); // All the catalogs should be cached - List cachedCatalogs = cachedStore.getCatalogs(); + List cachedCatalogs = cachedStore.getCatalogs(null); Assert.assertEquals(2, cachedCatalogs.size()); cachedCatalogs.sort(Comparator.naturalOrder()); Assert.assertEquals(CAT1_NAME, cachedCatalogs.get(0)); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestCatalogs.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestCatalogs.java index 6854ab9315e0..4ddd535dd95e 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestCatalogs.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestCatalogs.java @@ -139,6 +139,10 @@ public void catalogOperations() throws TException { List catalogs = client.getCatalogs(); Assert.assertEquals(4, catalogs.size()); + List catalogsByPattern = client.getCatalogs("cat*"); + Assert.assertEquals(2, catalogsByPattern.size()); + List catalogsByEmptyPattern = client.getCatalogs(""); + Assert.assertEquals(0, catalogsByEmptyPattern.size()); catalogs.sort(Comparator.naturalOrder()); List expected = new ArrayList<>(catNames.length + 1); expected.add(Warehouse.DEFAULT_CATALOG_NAME);