apache · ZihanLi58 · Jun 14, 2023 · May 20, 2023 · May 24, 2023 · Jun 12, 2023
diff --git a/gobblin-data-management/build.gradle b/gobblin-data-management/build.gradle
@@ -45,7 +45,7 @@ dependencies {
   compile externalDependency.junit
   compile externalDependency.jacksonMapperAsl
 
-  testCompile(group: 'org.apache.iceberg', name: 'iceberg-hive-metastore', version: '0.11.1', classifier: 'tests') {
+  testCompile(externalDependency.icebergHiveMetastoreTest) {
     transitive = false
   }
   testCompile('org.apache.hadoop:hadoop-common:2.6.0')

diff --git a/...anagement/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergTable.java b/...anagement/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergTable.java
@@ -161,7 +161,7 @@ protected IcebergSnapshotInfo createSnapshotInfo(Snapshot snapshot, Optional<Str
         metadataFileLocation,
         snapshot.manifestListLocation(),
         // NOTE: unable to `.stream().map(m -> calcManifestFileInfo(m, tableOps.io()))` due to checked exception
-        skipManifestFileInfo ? Lists.newArrayList() : calcAllManifestFileInfos(snapshot.allManifests(), tableOps.io())
+        skipManifestFileInfo ? Lists.newArrayList() : calcAllManifestFileInfos(snapshot.allManifests(tableOps.io()), tableOps.io())
       );
   }
 

diff --git a/gobblin-iceberg/build.gradle b/gobblin-iceberg/build.gradle
@@ -48,7 +48,7 @@ dependencies {
     compile externalDependency.findBugsAnnotations
     compile externalDependency.avroMapredH2
 
-    testCompile(group: 'org.apache.iceberg', name: 'iceberg-hive-metastore', version: '0.11.1', classifier: 'tests') {
+    testCompile(externalDependency.icebergHiveMetastoreTest) {
         transitive = false
     }
     testCompile('org.apache.hadoop:hadoop-common:2.6.0')

diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/Utils/IcebergUtils.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/Utils/IcebergUtils.java
@@ -248,6 +248,8 @@ public static DataFile getIcebergDataFileWithMetric(org.apache.gobblin.metadata.
         IcebergUtils.getMapFromIntegerLongPairs(file.getFileMetrics().getColumnSizes(), schemaIdMap),
         IcebergUtils.getMapFromIntegerLongPairs(file.getFileMetrics().getValueCounts(), schemaIdMap),
         IcebergUtils.getMapFromIntegerLongPairs(file.getFileMetrics().getNullValueCounts(), schemaIdMap),
+        // TODO: If required, handle NaN value count File metric conversion in ORC metrics with iceberg upgrade
+        IcebergUtils.getMapFromIntegerLongPairs(Lists.newArrayList(), schemaIdMap), // metric value will be null since Nan values are supported from avro version 1.10.*
         IcebergUtils.getMapFromIntegerBytesPairs(file.getFileMetrics().getLowerBounds(), schemaIdMap),
         IcebergUtils.getMapFromIntegerBytesPairs(file.getFileMetrics().getUpperBounds(), schemaIdMap));
     return dataFileBuilder.withMetrics(metrics).build();

diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/publisher/GobblinMCEPublisher.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/publisher/GobblinMCEPublisher.java
@@ -219,7 +219,7 @@ public static Metrics getMetrics(State state, Path path, Configuration conf, Nam
       }
       case AVRO: {
         try {
-          return new Metrics(100000000L, null, null, null);
+          return new Metrics(100000000L, null, null, null, null);
         } catch (Exception e) {
           throw new RuntimeException("Cannot get file information for file " + path.toString(), e);
         }

diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java
@@ -58,6 +58,7 @@
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils;
 import org.apache.iceberg.AppendFiles;
+import org.apache.iceberg.CatalogUtil;
 import org.apache.iceberg.DataFile;
 import org.apache.iceberg.DeleteFiles;
 import org.apache.iceberg.ExpireSnapshots;
@@ -77,7 +78,7 @@
 import org.apache.iceberg.exceptions.NoSuchTableException;
 import org.apache.iceberg.expressions.Expression;
 import org.apache.iceberg.expressions.Expressions;
-import org.apache.iceberg.hive.HiveCatalogs;
+import org.apache.iceberg.hive.HiveCatalog;
 import org.apache.iceberg.types.Types;
 import org.joda.time.DateTime;
 import org.joda.time.format.PeriodFormatter;
@@ -253,7 +254,7 @@ protected void setAuditCountVerifier(KafkaAuditCountVerifier verifier) {
   }
 
   protected void initializeCatalog() {
-    catalog = HiveCatalogs.loadCatalog(conf);
+    catalog = CatalogUtil.loadCatalog(HiveCatalog.class.getName(), "HiveCatalog", new HashMap<>(), conf);
   }
 
   private org.apache.iceberg.Table getIcebergTable(TableIdentifier tid) throws NoSuchTableException {

diff --git a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/HiveMetadataWriterTest.java b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/HiveMetadataWriterTest.java
@@ -128,7 +128,6 @@ public void clean() throws Exception {
   @BeforeSuite
   public void setUp() throws Exception {
     Class.forName("org.apache.derby.jdbc.EmbeddedDriver").newInstance();
-    startMetastore();
     State state = ConfigUtils.configToState(ConfigUtils.propertiesToConfig(hiveConf.getAllProperties()));
     Optional<String> metastoreUri = Optional.fromNullable(state.getProperties().getProperty(HiveRegister.HIVE_METASTORE_URI_KEY));
     hc = HiveMetastoreClientPool.get(state.getProperties(), metastoreUri);

diff --git a/...in-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java b/...in-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java
diff --git a/gradle/scripts/defaultBuildProperties.gradle b/gradle/scripts/defaultBuildProperties.gradle
@@ -31,7 +31,7 @@ def BuildProperties BUILD_PROPERTIES = new BuildProperties(project)
     .register(new BuildProperty("gobblinFlavor", "standard", "Build flavor (see http://gobblin.readthedocs.io/en/latest/developer-guide/GobblinModules/)"))
     .register(new BuildProperty("hadoopVersion", "2.10.0", "Hadoop dependencies version"))
     .register(new BuildProperty("hiveVersion", "1.0.1-avro", "Hive dependencies version"))
-    .register(new BuildProperty("icebergVersion", "0.11.1", "Iceberg dependencies version"))
+    .register(new BuildProperty("icebergVersion", "1.2.0", "Iceberg dependencies version"))
     .register(new BuildProperty("jdkVersion", JavaVersion.VERSION_1_8.toString(),
     "Java languange compatibility; supported versions: " + JavaVersion.VERSION_1_8))
     .register(new BuildProperty("kafka08Version", "0.8.2.2", "Kafka 0.8 dependencies version"))

diff --git a/gradle/scripts/dependencyDefinitions.gradle b/gradle/scripts/dependencyDefinitions.gradle
@@ -82,6 +82,7 @@ ext.externalDependency = [
     "httpcore": "org.apache.httpcomponents:httpcore:4.4.11",
     "httpasyncclient": "org.apache.httpcomponents:httpasyncclient:4.1.3",
     "icebergHive": "org.apache.iceberg:iceberg-hive-runtime:" + icebergVersion,
+    "icebergHiveMetastoreTest": "org.apache.iceberg:iceberg-hive-metastore:" + icebergVersion + ":tests",
     "jgit": "org.eclipse.jgit:org.eclipse.jgit:5.1.1.201809181055-r",
     "jmh": "org.openjdk.jmh:jmh-core:1.17.3",
     "jmhAnnotations": "org.openjdk.jmh:jmh-generator-annprocess:1.17.3",