Skip to content

Move custom hive catalog to hivelink-core #99

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ljfgem merged 2 commits into from
Apr 21, 2022
Merged

Move custom hive catalog to hivelink-core #99

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
656e552
Move custom hive catalog to hivelink-core
ljfgem Apr 21, 2022
bb94901
Remove duplicate license
ljfgem Apr 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -495,7 +495,7 @@ project(':iceberg-hive-metastore') {
}
}

project(':iceberg-hivelink') {
project(':iceberg-hivelink-core') {
dependencies {
compile project(':iceberg-hive-metastore')

Expand Down Expand Up @@ -872,7 +872,7 @@ if (jdkVersion == '8') {
compile project(':iceberg-arrow')
compile project(':iceberg-hive-metastore')
compile project(':iceberg-spark')
compile project(':iceberg-hivelink')
compile project(':iceberg-hivelink-core')

compileOnly "org.apache.avro:avro"
compileOnly("org.apache.spark:spark-hive_2.11") {
Expand Down Expand Up @@ -982,7 +982,7 @@ project(':iceberg-spark3') {
compile project(':iceberg-arrow')
compile project(':iceberg-hive-metastore')
compile project(':iceberg-spark')
compile project(':iceberg-hivelink')
compile project(':iceberg-hivelink-core')

compileOnly "org.apache.avro:avro"
compileOnly("org.apache.spark:spark-hive_2.12") {
Expand Down
17 changes: 0 additions & 17 deletions hive-metastore/src/main/java/org/apache/iceberg/hive/HiveCatalogs.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,26 +31,9 @@ public final class HiveCatalogs {
private HiveCatalogs() {
}

private static final Cache<String, HiveMetadataPreservingCatalog> HIVE_METADATA_PRESERVING_CATALOG_CACHE =
Caffeine.newBuilder().build();

public static HiveCatalog loadCatalog(Configuration conf) {
// metastore URI can be null in local mode
String metastoreUri = conf.get(HiveConf.ConfVars.METASTOREURIS.varname, "");
return CATALOG_CACHE.get(metastoreUri, uri -> new HiveCatalog(conf));
}

/**
* @deprecated Use {@link #loadHiveMetadataPreservingCatalog(Configuration)} instead
*/
@Deprecated
public static HiveCatalog loadCustomCatalog(Configuration conf) {
return loadHiveMetadataPreservingCatalog(conf);
}

public static HiveCatalog loadHiveMetadataPreservingCatalog(Configuration conf) {
// metastore URI can be null in local mode
String metastoreUri = conf.get(HiveConf.ConfVars.METASTOREURIS.varname, "");
return HIVE_METADATA_PRESERVING_CATALOG_CACHE.get(metastoreUri, uri -> new HiveMetadataPreservingCatalog(conf));
}
}
2 changes: 1 addition & 1 deletion ...pache/iceberg/hivelink/DirectoryInfo.java → .../iceberg/hivelink/core/DirectoryInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

package org.apache.iceberg.hivelink;
package org.apache.iceberg.hivelink.core;

import org.apache.iceberg.FileFormat;
import org.apache.iceberg.StructLike;
Expand Down
2 changes: 1 addition & 1 deletion ...che/iceberg/hivelink/HiveExpressions.java → ...ceberg/hivelink/core/HiveExpressions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

package org.apache.iceberg.hivelink;
package org.apache.iceberg.hivelink.core;

import java.time.Instant;
import java.time.OffsetDateTime;
Expand Down
24 changes: 23 additions & 1 deletion ...g/hive/HiveMetadataPreservingCatalog.java → ...k/core/HiveMetadataPreservingCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,16 @@
* under the License.
*/

package org.apache.iceberg.hive;
package org.apache.iceberg.hivelink.core;

import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.iceberg.TableOperations;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.hadoop.HadoopFileIO;
import org.apache.iceberg.hive.HiveCatalog;


/**
Expand All @@ -37,11 +41,29 @@ public HiveMetadataPreservingCatalog(Configuration conf) {
super(conf);
}

private static final Cache<String, HiveMetadataPreservingCatalog> HIVE_METADATA_PRESERVING_CATALOG_CACHE =
Caffeine.newBuilder().build();

@Override
public TableOperations newTableOps(TableIdentifier tableIdentifier) {
String dbName = tableIdentifier.namespace().level(0);
String tableName = tableIdentifier.name();
return new HiveMetadataPreservingTableOperations(conf(), clientPool(), new HadoopFileIO(conf()), name(), dbName,
tableName);
}


/**
* @deprecated Use {@link #loadHiveMetadataPreservingCatalog(Configuration)} instead
*/
@Deprecated
public static HiveCatalog loadCustomCatalog(Configuration conf) {
return loadHiveMetadataPreservingCatalog(conf);
}

public static HiveCatalog loadHiveMetadataPreservingCatalog(Configuration conf) {
// metastore URI can be null in local mode
String metastoreUri = conf.get(HiveConf.ConfVars.METASTOREURIS.varname, "");
return HIVE_METADATA_PRESERVING_CATALOG_CACHE.get(metastoreUri, uri -> new HiveMetadataPreservingCatalog(conf));
}
}
4 changes: 3 additions & 1 deletion ...iveMetadataPreservingTableOperations.java → ...iveMetadataPreservingTableOperations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

package org.apache.iceberg.hive;
package org.apache.iceberg.hivelink.core;

import java.net.UnknownHostException;
import java.util.Collections;
Expand All @@ -37,6 +37,8 @@
import org.apache.iceberg.exceptions.CommitFailedException;
import org.apache.iceberg.exceptions.CommitStateUnknownException;
import org.apache.iceberg.exceptions.NoSuchTableException;
import org.apache.iceberg.hive.HiveClientPool;
import org.apache.iceberg.hive.HiveTableOperations;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
Expand Down
3 changes: 2 additions & 1 deletion ...eberg/hivelink/HiveTypeToIcebergType.java → .../hivelink/core/HiveTypeToIcebergType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

package org.apache.iceberg.hivelink;
package org.apache.iceberg.hivelink.core;

import java.util.List;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
Expand All @@ -26,6 +26,7 @@
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
import org.apache.iceberg.hivelink.core.utils.HiveTypeUtil;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
Expand Down
2 changes: 1 addition & 1 deletion ...e/iceberg/hivelink/LegacyHiveCatalog.java → ...berg/hivelink/core/LegacyHiveCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

package org.apache.iceberg.hivelink;
package org.apache.iceberg.hivelink.core;

import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
Expand Down
2 changes: 1 addition & 1 deletion ...che/iceberg/hivelink/LegacyHiveTable.java → ...ceberg/hivelink/core/LegacyHiveTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

package org.apache.iceberg.hivelink;
package org.apache.iceberg.hivelink.core;

import java.util.List;
import java.util.Map;
Expand Down
3 changes: 2 additions & 1 deletion ...g/hivelink/LegacyHiveTableOperations.java → ...elink/core/LegacyHiveTableOperations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

package org.apache.iceberg.hivelink;
package org.apache.iceberg.hivelink.core;

import java.time.LocalDate;
import java.time.LocalDateTime;
Expand Down Expand Up @@ -50,6 +50,7 @@
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.hadoop.HadoopFileIO;
import org.apache.iceberg.hive.HiveClientPool;
import org.apache.iceberg.hivelink.core.utils.FileSystemUtils;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.io.LocationProvider;
import org.apache.iceberg.mapping.MappingUtil;
Expand Down
2 changes: 1 addition & 1 deletion ...iceberg/hivelink/LegacyHiveTableScan.java → ...rg/hivelink/core/LegacyHiveTableScan.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

package org.apache.iceberg.hivelink;
package org.apache.iceberg.hivelink.core;

import org.apache.iceberg.BaseFileScanTask;
import org.apache.iceberg.DataTableScan;
Expand Down
4 changes: 3 additions & 1 deletion ...ceberg/hivelink/LegacyHiveTableUtils.java → ...g/hivelink/core/LegacyHiveTableUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

package org.apache.iceberg.hivelink;
package org.apache.iceberg.hivelink.core;

import java.util.HashMap;
import java.util.List;
Expand All @@ -37,6 +37,8 @@
import org.apache.iceberg.StructLike;
import org.apache.iceberg.avro.AvroSchemaUtil;
import org.apache.iceberg.avro.AvroSchemaVisitor;
import org.apache.iceberg.hivelink.core.schema.MergeHiveSchemaWithAvro;
import org.apache.iceberg.hivelink.core.utils.HiveTypeUtil;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.types.Conversions;
Expand Down
2 changes: 1 addition & 1 deletion ...ivelink/HiveSchemaWithPartnerVisitor.java → .../schema/HiveSchemaWithPartnerVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

package org.apache.iceberg.hivelink;
package org.apache.iceberg.hivelink.core.schema;

import java.util.List;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
Expand Down
6 changes: 3 additions & 3 deletions ...erg/hivelink/MergeHiveSchemaWithAvro.java → .../core/schema/MergeHiveSchemaWithAvro.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

package org.apache.iceberg.hivelink;
package org.apache.iceberg.hivelink.core.schema;

import java.util.ArrayList;
import java.util.List;
Expand Down Expand Up @@ -49,9 +49,9 @@
* 5. Fields found only in Hive schema are represented as optional fields in the resultant Avro schema
* 6. For fields found only in Hive schema, field names are sanitized to make them compatible with Avro identifier spec
*/
class MergeHiveSchemaWithAvro extends HiveSchemaWithPartnerVisitor<Schema, Schema.Field, Schema, Schema.Field> {
public class MergeHiveSchemaWithAvro extends HiveSchemaWithPartnerVisitor<Schema, Schema.Field, Schema, Schema.Field> {

static Schema visit(StructTypeInfo typeInfo, Schema schema) {
public static Schema visit(StructTypeInfo typeInfo, Schema schema) {
return visit(typeInfo, schema, new MergeHiveSchemaWithAvro(),
AvroPartnerAccessor.INSTANCE);
}
Expand Down
8 changes: 4 additions & 4 deletions ...che/iceberg/hivelink/FileSystemUtils.java → .../hivelink/core/utils/FileSystemUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

package org.apache.iceberg.hivelink;
package org.apache.iceberg.hivelink.core.utils;

import java.io.IOException;
import java.io.UncheckedIOException;
Expand All @@ -30,15 +30,15 @@
import org.apache.hadoop.fs.PathFilter;
import org.apache.iceberg.exceptions.RuntimeIOException;

class FileSystemUtils {
public class FileSystemUtils {

private FileSystemUtils() {
}

/**
* Lists all non-hidden files for the given directory
*/
static List<FileStatus> listFiles(String directory, Configuration conf) {
public static List<FileStatus> listFiles(String directory, Configuration conf) {

final Path directoryPath = new Path(directory);
final FileStatus[] files;
Expand All @@ -51,7 +51,7 @@ static List<FileStatus> listFiles(String directory, Configuration conf) {
return Arrays.asList(files);
}

static boolean exists(String file, Configuration conf) {
public static boolean exists(String file, Configuration conf) {
final Path filePath = new Path(file);
try {
FileSystem fs = filePath.getFileSystem(conf);
Expand Down
3 changes: 2 additions & 1 deletion ...apache/iceberg/hivelink/HiveTypeUtil.java → ...erg/hivelink/core/utils/HiveTypeUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

package org.apache.iceberg.hivelink;
package org.apache.iceberg.hivelink.core.utils;

import java.util.List;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
Expand All @@ -26,6 +26,7 @@
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
import org.apache.iceberg.hivelink.core.HiveTypeToIcebergType;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.types.Type;

Expand Down
2 changes: 1 addition & 1 deletion ...e/iceberg/hivelink/HiveMetastoreTest.java → ...berg/hivelink/core/HiveMetastoreTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

package org.apache.iceberg.hivelink;
package org.apache.iceberg.hivelink.core;

import java.util.HashMap;
import org.apache.hadoop.hive.conf.HiveConf;
Expand Down
2 changes: 1 addition & 1 deletion ...apache/iceberg/hivelink/ScriptRunner.java → ...e/iceberg/hivelink/core/ScriptRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

package org.apache.iceberg.hivelink;
package org.apache.iceberg.hivelink.core;

import java.io.IOException;
import java.io.LineNumberReader;
Expand Down
4 changes: 2 additions & 2 deletions ...iceberg/hivelink/TestHiveExpressions.java → ...rg/hivelink/core/TestHiveExpressions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

package org.apache.iceberg.hivelink;
package org.apache.iceberg.hivelink.core;

import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
Expand All @@ -35,7 +35,7 @@
import static org.apache.iceberg.expressions.Expressions.notIn;
import static org.apache.iceberg.expressions.Expressions.notNull;
import static org.apache.iceberg.expressions.Expressions.or;
import static org.apache.iceberg.hivelink.HiveExpressions.simplifyPartitionFilter;
import static org.apache.iceberg.hivelink.core.HiveExpressions.simplifyPartitionFilter;


public class TestHiveExpressions {
Expand Down
Loading