Separate classes under hive legacy package to new hivelink module

build.gradle

@@ -495,6 +495,66 @@ project(':iceberg-hive-metastore') {
   }
 }
 
+project(':iceberg-hiveberg') {
+  dependencies {
+    compile project(':iceberg-hive-metastore')
+
+    compileOnly "org.apache.avro:avro"
+    compileOnly("org.apache.hadoop:hadoop-client") {
+      exclude group: 'org.apache.avro', module: 'avro'
+      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+    }
+
+    compileOnly("org.apache.hive:hive-metastore") {
+      exclude group: 'org.apache.avro', module: 'avro'
+      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+      exclude group: 'org.pentaho' // missing dependency
+      exclude group: 'org.apache.hbase'
+      exclude group: 'org.apache.logging.log4j'
+      exclude group: 'co.cask.tephra'
+      exclude group: 'com.google.code.findbugs', module: 'jsr305'
+      exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all'
+      exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet'
+      exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle'
+      exclude group: 'com.tdunning', module: 'json'
+      exclude group: 'javax.transaction', module: 'transaction-api'
+      exclude group: 'com.zaxxer', module: 'HikariCP'
+    }
+
+    // By default, hive-exec is a fat/uber jar and it exports a guava library
+    // that's really old. We use the core classifier to be able to override our guava
+    // version. Luckily, hive-exec seems to work okay so far with this version of guava
+    // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context.
+    testCompile("org.apache.hive:hive-exec::core") {
+      exclude group: 'org.apache.avro', module: 'avro'
+      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+      exclude group: 'org.pentaho' // missing dependency
+      exclude group: 'org.apache.hive', module: 'hive-llap-tez'
+      exclude group: 'org.apache.logging.log4j'
+      exclude group: 'com.google.protobuf', module: 'protobuf-java'
+      exclude group: 'org.apache.calcite'
+      exclude group: 'org.apache.calcite.avatica'
+      exclude group: 'com.google.code.findbugs', module: 'jsr305'
+    }
+
+    testCompile("org.apache.hive:hive-metastore") {
+      exclude group: 'org.apache.avro', module: 'avro'
+      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+      exclude group: 'org.pentaho' // missing dependency
+      exclude group: 'org.apache.hbase'
+      exclude group: 'org.apache.logging.log4j'
+      exclude group: 'co.cask.tephra'
+      exclude group: 'com.google.code.findbugs', module: 'jsr305'
+      exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all'
+      exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet'
+      exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle'
+      exclude group: 'com.tdunning', module: 'json'
+      exclude group: 'javax.transaction', module: 'transaction-api'
+      exclude group: 'com.zaxxer', module: 'HikariCP'
+    }
+  }
+}
+
 project(':iceberg-mr') {
   configurations {
     testCompile {
@@ -812,6 +872,7 @@ if (jdkVersion == '8') {
       compile project(':iceberg-arrow')
       compile project(':iceberg-hive-metastore')
       compile project(':iceberg-spark')
+      compile project(':iceberg-hiveberg')
 
       compileOnly "org.apache.avro:avro"
       compileOnly("org.apache.spark:spark-hive_2.11") {
@@ -921,6 +982,7 @@ project(':iceberg-spark3') {
     compile project(':iceberg-arrow')
     compile project(':iceberg-hive-metastore')
     compile project(':iceberg-spark')
+    compile project(':iceberg-hiveberg')
 
     compileOnly "org.apache.avro:avro"
     compileOnly("org.apache.spark:spark-hive_2.12") {

hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalogs.java

@@ -23,7 +23,6 @@
 import com.github.benmanes.caffeine.cache.Caffeine;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.iceberg.hive.legacy.LegacyHiveCatalog;
 
 public final class HiveCatalogs {
 
@@ -35,20 +34,12 @@ private HiveCatalogs() {
   private static final Cache<String, HiveMetadataPreservingCatalog> HIVE_METADATA_PRESERVING_CATALOG_CACHE =
       Caffeine.newBuilder().build();
 
-  private static final Cache<String, HiveCatalog> LEGACY_CATALOG_CACHE = Caffeine.newBuilder().build();
-
   public static HiveCatalog loadCatalog(Configuration conf) {
     // metastore URI can be null in local mode
     String metastoreUri = conf.get(HiveConf.ConfVars.METASTOREURIS.varname, "");
     return CATALOG_CACHE.get(metastoreUri, uri -> new HiveCatalog(conf));
   }
 
-  public static HiveCatalog loadLegacyCatalog(Configuration conf) {
-    // metastore URI can be null in local mode
-    String metastoreUri = conf.get(HiveConf.ConfVars.METASTOREURIS.varname, "");
-    return LEGACY_CATALOG_CACHE.get(metastoreUri, uri -> new LegacyHiveCatalog(conf));
-  }
-
   /**
    * @deprecated Use {@link #loadHiveMetadataPreservingCatalog(Configuration)} instead
    */

hive-metastore/src/main/java/org/apache/iceberg/hive/legacy/DirectoryInfo.java → hiveberg/src/main/java/org/apache/iceberg/hiveberg/DirectoryInfo.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.iceberg.hive.legacy;
+package org.apache.iceberg.hiveberg;
 
 import org.apache.iceberg.FileFormat;
 import org.apache.iceberg.StructLike;

hive-metastore/src/main/java/org/apache/iceberg/hive/legacy/FileSystemUtils.java → hiveberg/src/main/java/org/apache/iceberg/hiveberg/FileSystemUtils.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.iceberg.hive.legacy;
+package org.apache.iceberg.hiveberg;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;

hive-metastore/src/main/java/org/apache/iceberg/hive/legacy/HiveExpressions.java → hiveberg/src/main/java/org/apache/iceberg/hiveberg/HiveExpressions.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.iceberg.hive.legacy;
+package org.apache.iceberg.hiveberg;
 
 import java.time.Instant;
 import java.time.OffsetDateTime;

hive-metastore/src/main/java/org/apache/iceberg/hive/legacy/HiveSchemaWithPartnerVisitor.java → hiveberg/src/main/java/org/apache/iceberg/hiveberg/HiveSchemaWithPartnerVisitor.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.iceberg.hive.legacy;
+package org.apache.iceberg.hiveberg;
 
 import java.util.List;
 import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;

hive-metastore/src/main/java/org/apache/iceberg/hive/legacy/HiveTypeToIcebergType.java → hiveberg/src/main/java/org/apache/iceberg/hiveberg/HiveTypeToIcebergType.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.iceberg.hive.legacy;
+package org.apache.iceberg.hiveberg;
 
 import java.util.List;
 import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;

hive-metastore/src/main/java/org/apache/iceberg/hive/legacy/HiveTypeUtil.java → hiveberg/src/main/java/org/apache/iceberg/hiveberg/HiveTypeUtil.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.iceberg.hive.legacy;
+package org.apache.iceberg.hiveberg;
 
 import java.util.List;
 import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;

hive-metastore/src/main/java/org/apache/iceberg/hive/legacy/LegacyHiveCatalog.java → hiveberg/src/main/java/org/apache/iceberg/hiveberg/LegacyHiveCatalog.java

@@ -17,10 +17,13 @@
  * under the License.
  */
 
-package org.apache.iceberg.hive.legacy;
+package org.apache.iceberg.hiveberg;
 
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
 import java.util.Map;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.iceberg.PartitionSpec;
 import org.apache.iceberg.Schema;
 import org.apache.iceberg.Table;
@@ -41,10 +44,20 @@ public class LegacyHiveCatalog extends HiveCatalog {
 
   private static final Logger LOG = LoggerFactory.getLogger(LegacyHiveCatalog.class);
 
+  // Hiveberg refactoring: this is moved from HiveCatalogs
+  private static final Cache<String, HiveCatalog> LEGACY_CATALOG_CACHE = Caffeine.newBuilder().build();
+
   public LegacyHiveCatalog(Configuration conf) {
     super(conf);
   }
 
+  // Hiveberg refactoring: this is moved from HiveCatalogs
+  public static HiveCatalog loadLegacyCatalog(Configuration conf) {
+    // metastore URI can be null in local mode
+    String metastoreUri = conf.get(HiveConf.ConfVars.METASTOREURIS.varname, "");
+    return LEGACY_CATALOG_CACHE.get(metastoreUri, uri -> new LegacyHiveCatalog(conf));
+  }
+
   @Override
   @SuppressWarnings("CatchBlockLogException")
   public Table loadTable(TableIdentifier identifier) {

hive-metastore/src/main/java/org/apache/iceberg/hive/legacy/LegacyHiveTable.java → hiveberg/src/main/java/org/apache/iceberg/hiveberg/LegacyHiveTable.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.iceberg.hive.legacy;
+package org.apache.iceberg.hiveberg;
 
 import java.util.List;
 import java.util.Map;

hive-metastore/src/main/java/org/apache/iceberg/hive/legacy/LegacyHiveTableOperations.java → hiveberg/src/main/java/org/apache/iceberg/hiveberg/LegacyHiveTableOperations.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.iceberg.hive.legacy;
+package org.apache.iceberg.hiveberg;
 
 import java.time.LocalDate;
 import java.time.LocalDateTime;

hive-metastore/src/main/java/org/apache/iceberg/hive/legacy/LegacyHiveTableScan.java → hiveberg/src/main/java/org/apache/iceberg/hiveberg/LegacyHiveTableScan.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.iceberg.hive.legacy;
+package org.apache.iceberg.hiveberg;
 
 import org.apache.iceberg.BaseFileScanTask;
 import org.apache.iceberg.DataTableScan;

hive-metastore/src/main/java/org/apache/iceberg/hive/legacy/LegacyHiveTableUtils.java → hiveberg/src/main/java/org/apache/iceberg/hiveberg/LegacyHiveTableUtils.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.iceberg.hive.legacy;
+package org.apache.iceberg.hiveberg;
 
 import java.util.HashMap;
 import java.util.List;

hive-metastore/src/main/java/org/apache/iceberg/hive/legacy/MergeHiveSchemaWithAvro.java → hiveberg/src/main/java/org/apache/iceberg/hiveberg/MergeHiveSchemaWithAvro.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.iceberg.hive.legacy;
+package org.apache.iceberg.hiveberg;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -52,7 +52,7 @@
 class MergeHiveSchemaWithAvro extends HiveSchemaWithPartnerVisitor<Schema, Schema.Field, Schema, Schema.Field> {
 
   static Schema visit(StructTypeInfo typeInfo, Schema schema) {
-    return HiveSchemaWithPartnerVisitor.visit(typeInfo, schema, new MergeHiveSchemaWithAvro(),
+    return visit(typeInfo, schema, new MergeHiveSchemaWithAvro(),
         AvroPartnerAccessor.INSTANCE);
   }
 

hiveberg/src/test/java/org/apache/iceberg/hiveberg/HiveMetastoreTest.java

@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.hiveberg;
+
+import java.util.HashMap;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.iceberg.hive.HiveCatalog;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+
+/**
+ * Hiveberg refactoring:
+ * This class is copied from iceberg-hive-metastore module test code
+ */
+public abstract class HiveMetastoreTest {
+
+  protected static final String DB_NAME = "hivedb";
+
+  protected static HiveMetaStoreClient metastoreClient;
+  protected static HiveCatalog catalog;
+  protected static HiveConf hiveConf;
+  protected static TestHiveMetastore metastore;
+
+  @BeforeClass
+  public static void startMetastore() throws Exception {
+    HiveMetastoreTest.metastore = new TestHiveMetastore();
+    metastore.start();
+    HiveMetastoreTest.hiveConf = metastore.hiveConf();
+    HiveMetastoreTest.metastoreClient = new HiveMetaStoreClient(hiveConf);
+    String dbPath = metastore.getDatabasePath(DB_NAME);
+    Database db = new Database(DB_NAME, "description", dbPath, new HashMap<>());
+    metastoreClient.createDatabase(db);
+    HiveMetastoreTest.catalog = new HiveCatalog(hiveConf);
+  }
+
+  @AfterClass
+  public static void stopMetastore() {
+    catalog.close();
+    HiveMetastoreTest.catalog = null;
+
+    metastoreClient.close();
+    HiveMetastoreTest.metastoreClient = null;
+
+    metastore.stop();
+    HiveMetastoreTest.metastore = null;
+  }
+}