From 656248baf39137c80d24477e39465ef85cc9cb5e Mon Sep 17 00:00:00 2001 From: Reetika Agrawal Date: Tue, 12 Dec 2023 01:00:19 +0530 Subject: [PATCH] Set HadoopFileIO as default for HadoopCatalog --- .../apache/iceberg/hadoop/HadoopCatalog.java | 10 +++++----- .../iceberg/hadoop/TestHadoopCatalog.java | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/hadoop/HadoopCatalog.java b/core/src/main/java/org/apache/iceberg/hadoop/HadoopCatalog.java index 166551fe5575..e9ed4dcd280d 100644 --- a/core/src/main/java/org/apache/iceberg/hadoop/HadoopCatalog.java +++ b/core/src/main/java/org/apache/iceberg/hadoop/HadoopCatalog.java @@ -112,11 +112,11 @@ public void initialize(String name, Map properties) { this.warehouseLocation = LocationUtil.stripTrailingSlash(inputWarehouseLocation); this.fs = Util.getFs(new Path(warehouseLocation), conf); - String fileIOImpl = properties.get(CatalogProperties.FILE_IO_IMPL); - this.fileIO = - fileIOImpl == null - ? new HadoopFileIO(conf) - : CatalogUtil.loadFileIO(fileIOImpl, properties, conf); + String fileIOImpl = + properties.getOrDefault( + CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO"); + + this.fileIO = CatalogUtil.loadFileIO(fileIOImpl, properties, conf); this.lockManager = LockManagers.from(properties); diff --git a/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopCatalog.java b/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopCatalog.java index 4b01eead5d97..8365129a5615 100644 --- a/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopCatalog.java +++ b/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopCatalog.java @@ -221,6 +221,23 @@ public void testBasicCatalog() throws Exception { Assertions.assertThat(fs.isDirectory(new Path(metaLocation))).isFalse(); } + @Test + public void testHadoopFileIOProperties() { + TableIdentifier tableIdent = TableIdentifier.of("db", "ns1", "ns2", "tbl"); + ImmutableMap catalogProps = + ImmutableMap.of( + "warehouse", "/hive/testwarehouse", + "io.manifest.cache-enabled", "true"); + + HadoopCatalog catalog = new HadoopCatalog(); + catalog.setConf(new Configuration()); + catalog.initialize("hadoop", catalogProps); + FileIO fileIO = catalog.newTableOps(tableIdent).io(); + + Assertions.assertThat(fileIO.properties()).containsEntry("warehouse", "/hive/testwarehouse"); + Assertions.assertThat(fileIO.properties()).containsEntry("io.manifest.cache-enabled", "true"); + } + @Test public void testCreateAndDropTableWithoutNamespace() throws Exception { HadoopCatalog catalog = hadoopCatalog();