matchedIdentifiersPredicate;
+ if (identifierRegex == null) {
+ LOG.info("Collecting all the tables from all the namespaces of source catalog...");
+ matchedIdentifiersPredicate = tableIdentifier -> true;
+ } else {
+ LOG.info(
+ "Collecting all the tables from all the namespaces of source catalog"
+ + " which matches the regex pattern:{}",
+ identifierRegex);
+ Pattern pattern = Pattern.compile(identifierRegex);
+ matchedIdentifiersPredicate =
+ tableIdentifier -> pattern.matcher(tableIdentifier.toString()).matches();
+ }
+ return namespaces.stream()
+ .flatMap(
+ namespace -> {
+ try {
+ return sourceCatalog().listTables(namespace).stream()
+ .filter(matchedIdentifiersPredicate);
+ } catch (IllegalArgumentException | NoSuchNamespaceException exception) {
+ if (namespace.isEmpty()) {
+ // some catalogs don't support empty namespace.
+ // Hence, just log the warning and ignore the exception.
+ LOG.warn(
+ "Failed to identify tables from empty namespace : {}",
+ exception.getMessage());
+ return Stream.empty();
+ } else {
+ throw exception;
+ }
+ }
+ })
+ .collect(Collectors.toCollection(LinkedHashSet::new));
+ }
+
+ /**
+ * Register or Migrate a single table from one catalog(source catalog) to another catalog(target
+ * catalog).
+ *
+ * Users must make sure that no in-progress commits on the tables of source catalog during
+ * registration.
+ *
+ * @param identifier table identifier to register or migrate
+ * @return {@code this} for use in a chained invocation
+ */
+ public CatalogMigrator registerTable(TableIdentifier identifier) {
+ Preconditions.checkArgument(identifier != null, "Identifier is null");
+
+ boolean isRegistered = registerTableToTargetCatalog(identifier);
+ if (isRegistered) {
+ resultBuilder.addRegisteredTableIdentifiers(identifier);
+ } else {
+ resultBuilder.addFailedToRegisterTableIdentifiers(identifier);
+ }
+
+ try {
+ if (isRegistered
+ && deleteEntriesFromSourceCatalog()
+ && !sourceCatalog().dropTable(identifier, false)) {
+ resultBuilder.addFailedToDeleteTableIdentifiers(identifier);
+ }
+ } catch (Exception exception) {
+ resultBuilder.addFailedToDeleteTableIdentifiers(identifier);
+ if (enableStacktrace()) {
+ LOG.error("Failed to delete the table after migration {}", identifier, exception);
+ } else {
+ LOG.error(
+ "Failed to delete the table after migration {} : {}",
+ identifier,
+ exception.getMessage());
+ }
+ }
+ return this;
+ }
+
+ public CatalogMigrationResult result() {
+ processedNamespaces.clear();
+ return resultBuilder.build();
+ }
+
+ protected void createNamespacesIfNotExistOnTargetCatalog(Namespace identifierNamespace) {
+ if (!processedNamespaces.contains(identifierNamespace)) {
+ String[] levels = identifierNamespace.levels();
+ for (int index = 0; index < levels.length; index++) {
+ Namespace namespace = Namespace.of(Arrays.copyOfRange(levels, 0, index + 1));
+ if (processedNamespaces.add(namespace)) {
+ try {
+ ((SupportsNamespaces) targetCatalog()).createNamespace(namespace);
+ } catch (AlreadyExistsException ex) {
+ LOG.debug(
+ "{}.Ignoring the error as forcefully creating the namespace even if it exists to avoid "
+ + "namespaceExists() check.",
+ ex.getMessage());
+ }
+ }
+ }
+ }
+ }
+
+ protected void getAllNamespacesFromSourceCatalog(Namespace namespace, Set visited) {
+ if (!visited.add(namespace)) {
+ return;
+ }
+ List children = ((SupportsNamespaces) sourceCatalog()).listNamespaces(namespace);
+ for (Namespace child : children) {
+ getAllNamespacesFromSourceCatalog(child, visited);
+ }
+ }
+
+ private boolean registerTableToTargetCatalog(TableIdentifier tableIdentifier) {
+ try {
+ createNamespacesIfNotExistOnTargetCatalog(tableIdentifier.namespace());
+ // register the table to the target catalog
+ TableOperations ops = ((BaseTable) sourceCatalog().loadTable(tableIdentifier)).operations();
+ targetCatalog().registerTable(tableIdentifier, ops.current().metadataFileLocation());
+ LOG.info("Successfully registered the table {}", tableIdentifier);
+ return true;
+ } catch (Exception ex) {
+ if (enableStacktrace()) {
+ LOG.error("Unable to register the table {}", tableIdentifier, ex);
+ } else {
+ LOG.error("Unable to register the table {} : {}", tableIdentifier, ex.getMessage());
+ }
+ return false;
+ }
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/AbstractTestCatalogMigrator.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/AbstractTestCatalogMigrator.java
new file mode 100644
index 00000000..c41b5253
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/AbstractTestCatalogMigrator.java
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.IntStream;
+import nl.altindag.log.LogCaptor;
+import nl.altindag.log.model.LogEvent;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.polaris.iceberg.catalog.migrator.api.test.AbstractTest;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public abstract class AbstractTestCatalogMigrator extends AbstractTest {
+
+ protected static final Namespace NS1 = Namespace.of("ns1");
+ protected static final Namespace NS2 = Namespace.of("ns2");
+ protected static final Namespace NS3 = Namespace.of("ns3");
+ protected static final Namespace NS1_NS2 = Namespace.of("ns1", "ns2");
+ protected static final Namespace NS1_NS3 = Namespace.of("ns1", "ns3");
+ protected static final Namespace NS1_NS2_NS3 = Namespace.of("ns1", "ns2", "ns3");
+
+ protected static final TableIdentifier TBL = TableIdentifier.parse("tblz");
+ protected static final TableIdentifier NS1_TBL = TableIdentifier.of(NS1, "tblz");
+ protected static final TableIdentifier NS2_TBL = TableIdentifier.of(NS2, "tblz");
+ protected static final TableIdentifier NS3_TBL = TableIdentifier.of(NS3, "tblz");
+ protected static final TableIdentifier NS1_NS2_TBL = TableIdentifier.of(NS1_NS2, "tblz");
+ protected static final TableIdentifier NS1_NS3_TBL = TableIdentifier.of(NS1_NS3, "tblz");
+ protected static final TableIdentifier NS1_NS2_NS3_TBL = TableIdentifier.of(NS1_NS2_NS3, "tblz");
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+
+ @BeforeEach
+ protected void beforeEach() {
+ createTables();
+ }
+
+ @AfterEach
+ protected void afterEach() {
+ dropTables();
+ }
+
+ protected static void initializeSourceCatalog(
+ CatalogMigrationUtil.CatalogType catalogType, Map additionalProp) {
+ initializeCatalog(true, catalogType, additionalProp);
+ createNamespacesForSourceCatalog();
+ }
+
+ protected static void initializeTargetCatalog(
+ CatalogMigrationUtil.CatalogType catalogType, Map additionalProp) {
+ initializeCatalog(false, catalogType, additionalProp);
+ createNamespacesForTargetCatalog();
+ }
+
+ private static void initializeCatalog(
+ boolean isSourceCatalog,
+ CatalogMigrationUtil.CatalogType catalogType,
+ Map additionalProp) {
+ Map properties;
+ switch (catalogType) {
+ case HADOOP:
+ properties = hadoopCatalogProperties(isSourceCatalog);
+ break;
+ case NESSIE:
+ properties = nessieCatalogProperties(isSourceCatalog);
+ break;
+ case HIVE:
+ properties = hiveCatalogProperties(isSourceCatalog, additionalProp);
+ break;
+ default:
+ throw new UnsupportedOperationException(
+ String.format("Unsupported for catalog type: %s", catalogType));
+ }
+ Catalog catalog =
+ CatalogMigrationUtil.buildCatalog(
+ properties,
+ catalogType,
+ isSourceCatalog ? "sourceCatalog" : "targetCatalog" + "_" + catalogType,
+ null,
+ null);
+ if (isSourceCatalog) {
+ sourceCatalog = catalog;
+ } else {
+ targetCatalog = catalog;
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegister(boolean deleteSourceTables) {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ CatalogMigrationResult result = registerAllTables(deleteSourceTables);
+
+ Assertions.assertThat(result.registeredTableIdentifiers())
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2, BAR_TBL3, BAR_TBL4);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO))
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(targetCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+
+ if (deleteSourceTables) {
+ // table should be deleted after migration from source catalog
+ Assertions.assertThat(sourceCatalog.listTables(FOO)).isEmpty();
+ Assertions.assertThat(sourceCatalog.listTables(BAR)).isEmpty();
+ } else {
+ // tables should be present in source catalog.
+ Assertions.assertThat(sourceCatalog.listTables(FOO))
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(sourceCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterSelectedTables(boolean deleteSourceTables) {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ // using `--identifiers` option
+ CatalogMigrationResult result =
+ catalogMigratorWithDefaultArgs(deleteSourceTables).registerTable(BAR_TBL3).result();
+ Assertions.assertThat(result.registeredTableIdentifiers()).containsExactly(BAR_TBL3);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO)).isEmpty();
+ Assertions.assertThat(targetCatalog.listTables(BAR)).containsExactly(BAR_TBL3);
+
+ // using --identifiers-regex option which matches all the tables starts with "foo."
+ CatalogMigrator catalogMigrator = catalogMigratorWithDefaultArgs(deleteSourceTables);
+ catalogMigrator
+ .getMatchingTableIdentifiers("^foo\\..*")
+ .forEach(catalogMigrator::registerTable);
+ result = catalogMigrator.result();
+ Assertions.assertThat(result.registeredTableIdentifiers())
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO))
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(targetCatalog.listTables(BAR)).containsExactly(BAR_TBL3);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterError(boolean deleteSourceTables) {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ // use invalid namespace which leads to NoSuchTableException
+ TableIdentifier identifier = TableIdentifier.parse("dummy.tbl3");
+ CatalogMigrationResult result =
+ catalogMigratorWithDefaultArgs(deleteSourceTables).registerTable(identifier).result();
+ Assertions.assertThat(result.registeredTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).containsExactly(identifier);
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ // try to register same table twice which leads to AlreadyExistsException
+ result = catalogMigratorWithDefaultArgs(deleteSourceTables).registerTable(FOO_TBL2).result();
+ Assertions.assertThat(result.registeredTableIdentifiers()).containsExactly(FOO_TBL2);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ result = catalogMigratorWithDefaultArgs(deleteSourceTables).registerTable(FOO_TBL2).result();
+ Assertions.assertThat(result.registeredTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).contains(FOO_TBL2);
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterWithFewFailures(boolean deleteSourceTables) {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ // register only foo.tbl2
+ CatalogMigrationResult result =
+ catalogMigratorWithDefaultArgs(deleteSourceTables).registerTable(FOO_TBL2).result();
+ Assertions.assertThat(result.registeredTableIdentifiers()).containsExactly(FOO_TBL2);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ if (deleteSourceTables) {
+ // create a table with the same name in source catalog which got deleted.
+ sourceCatalog.createTable(FOO_TBL2, schema);
+ }
+
+ // register all the tables from source catalog again. So that `foo.tbl2` will fail to register.
+ result = registerAllTables(deleteSourceTables);
+ Assertions.assertThat(result.registeredTableIdentifiers())
+ .containsExactlyInAnyOrder(FOO_TBL1, BAR_TBL3, BAR_TBL4);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).contains(FOO_TBL2);
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO))
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(targetCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterLargeNumberOfTables(boolean deleteSourceTables) {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ // additionally create 240 tables along with 4 tables created in beforeEach()
+ IntStream.range(0, 240)
+ .forEach(val -> sourceCatalog.createTable(TableIdentifier.of(FOO, "tblx" + val), schema));
+
+ CatalogMigrationResult result;
+ result = registerAllTables(deleteSourceTables);
+
+ Assertions.assertThat(result.registeredTableIdentifiers()).hasSize(244);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO)).hasSize(242);
+ Assertions.assertThat(targetCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testListingTableIdentifiers(boolean deleteSourceTables) {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ CatalogMigrator catalogMigrator = catalogMigratorWithDefaultArgs(deleteSourceTables);
+
+ // should list all the tables from all the namespace when regex is null.
+ Set matchingTableIdentifiers =
+ catalogMigrator.getMatchingTableIdentifiers(null);
+ Assertions.assertThat(matchingTableIdentifiers)
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2, BAR_TBL3, BAR_TBL4);
+
+ // list the tables whose identifier starts with "foo."
+ matchingTableIdentifiers = catalogMigrator.getMatchingTableIdentifiers("^foo\\..*");
+ Assertions.assertThat(matchingTableIdentifiers).containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+
+ // test filter that doesn't match any table.
+ matchingTableIdentifiers = catalogMigrator.getMatchingTableIdentifiers("^dev\\..*");
+ Assertions.assertThat(matchingTableIdentifiers).isEmpty();
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterWithNewNamespace(boolean deleteSourceTables) {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ TableIdentifier tbl5 = TableIdentifier.of(DB1, "tbl5");
+ // namespace "db1" exists only in source catalog
+ sourceCatalog.createTable(tbl5, schema);
+
+ CatalogMigrationResult result =
+ catalogMigratorWithDefaultArgs(deleteSourceTables).registerTable(tbl5).result();
+
+ Assertions.assertThat(result.registeredTableIdentifiers()).containsExactly(tbl5);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ Assertions.assertThat(targetCatalog.listTables(DB1)).containsExactly(tbl5);
+ }
+
+ @ParameterizedTest
+ @CsvSource(value = {"false,false", "false,true", "true,false", "true,true"})
+ public void testStacktrace(boolean deleteSourceTables, boolean enableStacktrace) {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ TableIdentifier identifier = TableIdentifier.parse("db.dummy_table");
+
+ ImmutableCatalogMigrator migrator =
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(deleteSourceTables)
+ .enableStacktrace(enableStacktrace)
+ .build();
+ try (LogCaptor logCaptor = LogCaptor.forClass(CatalogMigrator.class)) {
+ CatalogMigrationResult result = migrator.registerTable(identifier).result();
+ Assertions.assertThat(result.registeredTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).containsExactly(identifier);
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ Assertions.assertThat(logCaptor.getLogEvents()).hasSize(1);
+ LogEvent logEvent = logCaptor.getLogEvents().get(0);
+ if (enableStacktrace) {
+ Assertions.assertThat(logEvent.getFormattedMessage())
+ .isEqualTo("Unable to register the table db.dummy_table");
+ Assertions.assertThat(logEvent.getThrowable())
+ .isPresent()
+ .get()
+ .isInstanceOf(NoSuchTableException.class);
+ } else {
+ Assertions.assertThat(logEvent.getFormattedMessage())
+ .isEqualTo(
+ "Unable to register the table db.dummy_table : Table does not exist: db.dummy_table");
+ Assertions.assertThat(logEvent.getThrowable()).isEmpty();
+ }
+ }
+ }
+
+ protected CatalogMigrator catalogMigratorWithDefaultArgs(boolean deleteSourceTables) {
+ return ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(deleteSourceTables)
+ .build();
+ }
+
+ private CatalogMigrationResult registerAllTables(boolean deleteSourceTables) {
+ CatalogMigrator catalogMigrator = catalogMigratorWithDefaultArgs(deleteSourceTables);
+ catalogMigrator.getMatchingTableIdentifiers(null).forEach(catalogMigrator::registerTable);
+ return catalogMigrator.result();
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/CatalogMigrationUtilTest.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/CatalogMigrationUtilTest.java
new file mode 100644
index 00000000..c4ed478b
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/CatalogMigrationUtilTest.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import static org.apache.polaris.iceberg.catalog.migrator.api.test.AbstractTest.FOO_TBL1;
+
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Stream;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.hadoop.HadoopCatalog;
+import org.apache.iceberg.hive.HiveCatalog;
+import org.apache.iceberg.nessie.NessieCatalog;
+import org.apache.iceberg.types.Types;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
+public class CatalogMigrationUtilTest {
+
+ private static @TempDir Path logDir;
+
+ private static @TempDir Path tempDir;
+
+ @BeforeAll
+ protected static void initLogDir() {
+ System.setProperty("catalog.migration.log.dir", logDir.toAbsolutePath().toString());
+ }
+
+ static Stream blankOrNullStrings() {
+ return Stream.of("", " ", null);
+ }
+
+ @ParameterizedTest()
+ @MethodSource("blankOrNullStrings")
+ public void testCustomCatalogWithoutImpl(String impl) {
+ Assertions.assertThatThrownBy(
+ () ->
+ CatalogMigrationUtil.buildCatalog(
+ Collections.emptyMap(),
+ CatalogMigrationUtil.CatalogType.CUSTOM,
+ "catalogName",
+ impl,
+ Collections.emptyMap()))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(
+ "Need to specify the fully qualified class name of the custom catalog impl");
+ }
+
+ @Test
+ public void testInvalidArgs() {
+ Assertions.assertThatThrownBy(
+ () -> CatalogMigrationUtil.buildCatalog(null, null, null, null, null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("catalog properties is null");
+
+ Assertions.assertThatThrownBy(
+ () -> CatalogMigrationUtil.buildCatalog(Collections.emptyMap(), null, null, null, null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("catalog type is null");
+
+ Assertions.assertThatThrownBy(
+ () ->
+ CatalogMigrationUtil.buildCatalog(
+ Collections.emptyMap(),
+ CatalogMigrationUtil.CatalogType.CUSTOM,
+ "catalogName",
+ "abc",
+ null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(
+ "Cannot initialize Catalog implementation abc: Cannot find constructor for interface");
+ }
+
+ @Test
+ public void testBuildHadoopCatalog() throws Exception {
+ Map properties = new HashMap<>();
+ properties.put("warehouse", tempDir.toAbsolutePath().toString());
+ properties.put("type", "hadoop");
+
+ Map conf = new HashMap<>();
+ conf.put("k1", "v1");
+
+ Catalog catalog =
+ CatalogMigrationUtil.buildCatalog(
+ properties, CatalogMigrationUtil.CatalogType.HADOOP, "catalogName", null, conf);
+
+ try {
+ Assertions.assertThat(catalog).isInstanceOf(HadoopCatalog.class);
+ Assertions.assertThat(catalog.name()).isEqualTo("catalogName");
+ Assertions.assertThat(((HadoopCatalog) catalog).getConf().get("k1")).isEqualTo("v1");
+ Schema schema =
+ new Schema(
+ Types.StructType.of(Types.NestedField.required(1, "id", Types.LongType.get()))
+ .fields());
+ Table table = catalog.createTable(FOO_TBL1, schema);
+ Assertions.assertThat(table.location()).contains(tempDir.toAbsolutePath().toString());
+ catalog.dropTable(FOO_TBL1);
+ } finally {
+ if (catalog instanceof AutoCloseable) {
+ ((AutoCloseable) catalog).close();
+ }
+ }
+ }
+
+ @Test
+ public void testBuildNessieCatalog() throws Exception {
+ Map properties = new HashMap<>();
+ properties.put("warehouse", tempDir.toAbsolutePath().toString());
+ properties.put("ref", "main");
+ properties.put("uri", "http://localhost:19120/api/v1");
+ properties.put("enable-api-compatibility-check", "false");
+
+ Catalog catalog =
+ CatalogMigrationUtil.buildCatalog(
+ properties, CatalogMigrationUtil.CatalogType.NESSIE, "catalogName", null, null);
+
+ try {
+ Assertions.assertThat(catalog).isInstanceOf(NessieCatalog.class);
+ Assertions.assertThat(catalog.name()).isEqualTo("catalogName");
+ } finally {
+ if (catalog instanceof AutoCloseable) {
+ ((AutoCloseable) catalog).close();
+ }
+ }
+ }
+
+ @Test
+ public void testBuildHiveCatalog() throws Exception {
+ Map properties = new HashMap<>();
+ properties.put("warehouse", tempDir.toAbsolutePath().toString());
+ properties.put("type", "hive");
+ properties.put("uri", "thrift://localhost:9083");
+
+ Catalog catalog =
+ CatalogMigrationUtil.buildCatalog(
+ properties, CatalogMigrationUtil.CatalogType.HIVE, "catalogName", null, null);
+
+ try {
+ Assertions.assertThat(catalog).isInstanceOf(HiveCatalog.class);
+ Assertions.assertThat(catalog.name()).isEqualTo("catalogName");
+ } finally {
+ if (catalog instanceof AutoCloseable) {
+ ((AutoCloseable) catalog).close();
+ }
+ }
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/CatalogMigratorParamsTest.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/CatalogMigratorParamsTest.java
new file mode 100644
index 00000000..fc66d07e
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/CatalogMigratorParamsTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.hadoop.HadoopCatalog;
+import org.apache.polaris.iceberg.catalog.migrator.api.test.AbstractTest;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class CatalogMigratorParamsTest extends AbstractTest {
+
+ @Test
+ public void testInvalidArgs() {
+ sourceCatalog =
+ CatalogUtil.loadCatalog(
+ HadoopCatalog.class.getName(),
+ "source",
+ hadoopCatalogProperties(true),
+ new Configuration());
+ targetCatalog =
+ CatalogUtil.loadCatalog(
+ HadoopCatalog.class.getName(),
+ "target",
+ hadoopCatalogProperties(true),
+ new Configuration());
+
+ Assertions.assertThatThrownBy(
+ () ->
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(false)
+ .build()
+ .registerTable(null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Identifier is null");
+
+ Assertions.assertThatThrownBy(
+ () ->
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(null) // target-catalog is null
+ .deleteEntriesFromSourceCatalog(true)
+ .build())
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("targetCatalog");
+
+ Assertions.assertThatThrownBy(
+ () ->
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(null) // source-catalog is null
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(true)
+ .build())
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("sourceCatalog");
+
+ // test source catalog as hadoop with `deleteEntriesFromSourceCatalog` as true.
+ Assertions.assertThatThrownBy(
+ () ->
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(true)
+ .build())
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessageContaining(
+ "Source catalog is a Hadoop catalog and it doesn't support deleting the table entries just from the catalog. "
+ + "Please configure `deleteEntriesFromSourceCatalog` as `false`");
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/CustomCatalogMigratorTest.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/CustomCatalogMigratorTest.java
new file mode 100644
index 00000000..81ac4d5d
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/CustomCatalogMigratorTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.hadoop.HadoopCatalog;
+import org.apache.polaris.iceberg.catalog.migrator.api.test.AbstractTest;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+public class CustomCatalogMigratorTest extends AbstractTest {
+
+ protected static @TempDir Path warehouse1;
+ protected static @TempDir Path warehouse2;
+
+ @BeforeAll
+ protected static void setup() {
+ sourceCatalog = createCustomCatalog(warehouse1.toAbsolutePath().toString(), "sourceCatalog");
+ targetCatalog = createCustomCatalog(warehouse2.toAbsolutePath().toString(), "targetCatalog");
+ }
+
+ @BeforeEach
+ protected void beforeEach() {
+ createTables();
+ }
+
+ @AfterEach
+ protected void afterEach() {
+ dropTables();
+ }
+
+ @Test
+ public void testRegister() {
+ CatalogMigrator catalogMigrator =
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(false)
+ .build();
+ // should fail to register as catalog doesn't support register table operations.
+ catalogMigrator.getMatchingTableIdentifiers(null).forEach(catalogMigrator::registerTable);
+ CatalogMigrationResult result = catalogMigrator.result();
+ Assertions.assertThat(result.registeredTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers())
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2, BAR_TBL3, BAR_TBL4);
+ }
+
+ private static Catalog createCustomCatalog(String warehousePath, String name) {
+
+ class TestCatalog extends HadoopCatalog {
+ @Override
+ public Table registerTable(TableIdentifier identifier, String metadataFileLocation) {
+ throw new UnsupportedOperationException("This catalog doesn't support register table");
+ }
+ }
+
+ Map properties = new HashMap<>();
+ properties.put("warehouse", warehousePath);
+ TestCatalog testCatalog = new TestCatalog();
+ testCatalog.setConf(new Configuration());
+ testCatalog.initialize(name, properties);
+ return testCatalog;
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/HadoopCatalogMigratorTest.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/HadoopCatalogMigratorTest.java
new file mode 100644
index 00000000..32429624
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/HadoopCatalogMigratorTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+public class HadoopCatalogMigratorTest extends AbstractTestCatalogMigrator {
+
+ @BeforeAll
+ protected static void setup() {
+ initializeSourceCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ initializeTargetCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ }
+
+ @Test
+ public void testRegisterWithNewNestedNamespaces() {
+ List namespaceList = Arrays.asList(NS1, NS2, NS3, NS1_NS2, NS1_NS3, NS1_NS2_NS3);
+
+ List identifiers =
+ Arrays.asList(TBL, NS1_TBL, NS2_TBL, NS3_TBL, NS1_NS2_TBL, NS1_NS3_TBL, NS1_NS2_NS3_TBL);
+
+ namespaceList.forEach(((SupportsNamespaces) sourceCatalog)::createNamespace);
+ identifiers.forEach(identifier -> sourceCatalog.createTable(identifier, schema));
+
+ CatalogMigrator catalogMigrator = catalogMigratorWithDefaultArgs(false);
+ Set matchingTableIdentifiers =
+ catalogMigrator.getMatchingTableIdentifiers(null);
+ // HadoopCatalog supports implicit namespaces.
+ // Hence, No concept of empty namespace too. So, cannot list the tables from empty
+ // namespaces.
+ // Can only load tables in empty namespace using identifiers.
+ Assertions.assertThat(matchingTableIdentifiers)
+ .containsAll(identifiers.subList(1, 7)); // without "tblz"
+ Assertions.assertThat(matchingTableIdentifiers).doesNotContain(identifiers.get(0));
+
+ matchingTableIdentifiers.forEach(catalogMigrator::registerTable);
+ CatalogMigrationResult result = catalogMigrator.result();
+ Assertions.assertThat(result.registeredTableIdentifiers())
+ .containsAll(identifiers.subList(1, 7)); // without "tblz"
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ // manually register the table from empty namespace
+ result = catalogMigratorWithDefaultArgs(false).registerTable(TBL).result();
+ Assertions.assertThat(result.registeredTableIdentifiers()).containsExactly(TBL);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ Collections.reverse(namespaceList);
+ identifiers.forEach(sourceCatalog::dropTable);
+ namespaceList.forEach(((SupportsNamespaces) sourceCatalog)::dropNamespace);
+ identifiers.forEach(targetCatalog::dropTable);
+ namespaceList.forEach(((SupportsNamespaces) targetCatalog)::dropNamespace);
+ }
+
+ @Test
+ public void testCreateAndListNamespaces() {
+ ImmutableCatalogMigrator catalogMigrator =
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(false)
+ .build();
+
+ List namespaceList =
+ Arrays.asList(NS_A, NS_A_B, NS_A_B_C, NS_A_B_C_D, NS_A_B_C_D_E, NS_A_C);
+ catalogMigrator.createNamespacesIfNotExistOnTargetCatalog(
+ namespaceList.get(4)); // try creating "a.b.c.d.e"
+ catalogMigrator.createNamespacesIfNotExistOnTargetCatalog(
+ namespaceList.get(5)); // try creating "a.c"
+ // should create all the levels of missing namespaces on target catalog
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces())
+ .contains(namespaceList.get(0))
+ .doesNotContainAnyElementsOf(namespaceList.subList(1, 6));
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(namespaceList.get(0)))
+ .containsExactlyInAnyOrder(namespaceList.get(1), namespaceList.get(5));
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(namespaceList.get(1)))
+ .containsExactly(namespaceList.get(2));
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(namespaceList.get(2)))
+ .containsExactly(namespaceList.get(3));
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(namespaceList.get(3)))
+ .containsExactly(namespaceList.get(4));
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(namespaceList.get(4)))
+ .isEmpty();
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(namespaceList.get(5)))
+ .isEmpty();
+
+ namespaceList.forEach(
+ namespace -> ((SupportsNamespaces) sourceCatalog).createNamespace(namespace));
+ Set listedNamespaces = new HashSet<>();
+ // collect all the namespaces from all levels
+ catalogMigrator.getAllNamespacesFromSourceCatalog(Namespace.empty(), listedNamespaces);
+ Assertions.assertThat(listedNamespaces).containsAll(namespaceList);
+
+ Collections.reverse(namespaceList);
+ namespaceList.forEach(((SupportsNamespaces) sourceCatalog)::dropNamespace);
+ namespaceList.forEach(((SupportsNamespaces) targetCatalog)::dropNamespace);
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHadoopToHiveCatalogMigrator.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHadoopToHiveCatalogMigrator.java
new file mode 100644
index 00000000..87708c23
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHadoopToHiveCatalogMigrator.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import java.util.Collections;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.hive.HiveMetastoreExtension;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+public class ITHadoopToHiveCatalogMigrator extends AbstractTestCatalogMigrator {
+
+ @RegisterExtension
+ public static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().build();
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+ initializeSourceCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ initializeTargetCatalog(
+ CatalogMigrationUtil.CatalogType.HIVE,
+ Collections.singletonMap(
+ "uri", HIVE_METASTORE_EXTENSION.hiveConf().get("hive.metastore.uris")));
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+
+ @Test
+ public void testRegisterWithNewNestedNamespace() {
+ TableIdentifier tableIdentifier = TableIdentifier.of(NS_A_B_C, "tbl5");
+ // create namespace "a.b.c" only in source catalog
+ ((SupportsNamespaces) sourceCatalog).createNamespace(NS_A_B_C);
+ sourceCatalog.createTable(tableIdentifier, schema);
+
+ CatalogMigrationResult result =
+ catalogMigratorWithDefaultArgs(false).registerTable(tableIdentifier).result();
+
+ // hive catalog doesn't support multipart namespace. Hence, table should fail to register.
+ Assertions.assertThat(result.registeredTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers())
+ .containsExactly(tableIdentifier);
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ sourceCatalog.dropTable(tableIdentifier);
+ ((SupportsNamespaces) sourceCatalog).dropNamespace(NS_A_B_C);
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHadoopToNessieCatalogMigrator.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHadoopToNessieCatalogMigrator.java
new file mode 100644
index 00000000..87e0034e
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHadoopToNessieCatalogMigrator.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+public class ITHadoopToNessieCatalogMigrator extends AbstractTestCatalogMigrator {
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+ initializeSourceCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ initializeTargetCatalog(CatalogMigrationUtil.CatalogType.NESSIE, Collections.emptyMap());
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+
+ @Test
+ public void testRegisterWithNewNestedNamespaces() {
+ List namespaceList = Arrays.asList(NS1, NS2, NS3, NS1_NS2, NS1_NS3, NS1_NS2_NS3);
+ List identifiers =
+ Arrays.asList(TBL, NS1_TBL, NS2_TBL, NS3_TBL, NS1_NS2_TBL, NS1_NS3_TBL, NS1_NS2_NS3_TBL);
+ namespaceList.forEach(((SupportsNamespaces) sourceCatalog)::createNamespace);
+ identifiers.forEach(identifier -> sourceCatalog.createTable(identifier, schema));
+
+ CatalogMigrator catalogMigrator = catalogMigratorWithDefaultArgs(false);
+ Set matchingTableIdentifiers =
+ catalogMigrator.getMatchingTableIdentifiers(null);
+ // HadoopCatalog supports implicit namespaces.
+ // Hence, No concept of empty namespace too. So, cannot list the tables from default
+ // namespaces.
+ // Can only load tables in empty namespace using identifiers.
+ Assertions.assertThat(matchingTableIdentifiers)
+ .containsAll(identifiers.subList(1, 7)); // without "tblz"
+ Assertions.assertThat(matchingTableIdentifiers).doesNotContain(identifiers.get(0));
+
+ matchingTableIdentifiers.forEach(catalogMigrator::registerTable);
+ CatalogMigrationResult result = catalogMigrator.result();
+ Assertions.assertThat(result.registeredTableIdentifiers())
+ .containsAll(identifiers.subList(1, 7)); // without "tblz"
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ // manually register the table from empty namespace
+ result = catalogMigratorWithDefaultArgs(false).registerTable(TBL).result();
+ Assertions.assertThat(result.registeredTableIdentifiers()).containsExactly(TBL);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+
+ Collections.reverse(namespaceList);
+ identifiers.forEach(sourceCatalog::dropTable);
+ namespaceList.forEach(((SupportsNamespaces) sourceCatalog)::dropNamespace);
+ identifiers.forEach(targetCatalog::dropTable);
+ namespaceList.forEach(((SupportsNamespaces) targetCatalog)::dropNamespace);
+ }
+
+ @Test
+ public void testCreateMissingNamespaces() {
+ ImmutableCatalogMigrator catalogMigrator =
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(false)
+ .build();
+
+ List namespaceList =
+ Arrays.asList(NS_A, NS_A_B, NS_A_B_C, NS_A_B_C_D, NS_A_B_C_D_E, NS_A_C);
+ catalogMigrator.createNamespacesIfNotExistOnTargetCatalog(
+ namespaceList.get(4)); // try creating "a.b.c.d.e"
+ catalogMigrator.createNamespacesIfNotExistOnTargetCatalog(
+ namespaceList.get(5)); // try creating "a.c"
+
+ // should create all the levels of missing namespaces on target catalog.
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces()).contains(NS_A);
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(NS_A))
+ .contains(NS_A_B, NS_A_C);
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(NS_A_B))
+ .contains(NS_A_B_C);
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(NS_A_B_C))
+ .contains(NS_A_B_C_D);
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).listNamespaces(NS_A_B_C_D))
+ .contains(NS_A_B_C_D_E);
+
+ namespaceList.forEach(
+ namespace -> ((SupportsNamespaces) sourceCatalog).createNamespace(namespace));
+ Set listedNamespaces = new HashSet<>();
+ // collect all the namespaces from all levels
+ catalogMigrator.getAllNamespacesFromSourceCatalog(Namespace.empty(), listedNamespaces);
+ Assertions.assertThat(listedNamespaces).containsAll(namespaceList);
+
+ Collections.reverse(namespaceList);
+ namespaceList.forEach(((SupportsNamespaces) targetCatalog)::dropNamespace);
+ namespaceList.forEach(((SupportsNamespaces) sourceCatalog)::dropNamespace);
+ }
+
+ @Test
+ public void testListingNamespacesFromNessie() {
+ Catalog nessie = targetCatalog;
+ Catalog hadoop = sourceCatalog;
+
+ ImmutableCatalogMigrator catalogMigrator =
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(nessie)
+ .targetCatalog(hadoop)
+ .deleteEntriesFromSourceCatalog(false)
+ .build();
+
+ List namespaceList =
+ Arrays.asList(NS_A, NS_A_B, NS_A_B_C, NS_A_B_C_D, NS_A_B_C_D_E, NS_A_C);
+
+ namespaceList.forEach(namespace -> ((SupportsNamespaces) nessie).createNamespace(namespace));
+ Set listedNamespaces = new HashSet<>();
+ // collect all the namespaces from all levels
+ catalogMigrator.getAllNamespacesFromSourceCatalog(Namespace.empty(), listedNamespaces);
+ Assertions.assertThat(listedNamespaces).containsAll(namespaceList);
+
+ Collections.reverse(namespaceList);
+ namespaceList.forEach(((SupportsNamespaces) nessie)::dropNamespace);
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHiveToHadoopCatalogMigrator.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHiveToHadoopCatalogMigrator.java
new file mode 100644
index 00000000..7ad97960
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHiveToHadoopCatalogMigrator.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import java.util.Collections;
+import org.apache.iceberg.hive.HiveMetastoreExtension;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+public class ITHiveToHadoopCatalogMigrator extends AbstractTestCatalogMigrator {
+
+ @RegisterExtension
+ public static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().build();
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+ initializeSourceCatalog(
+ CatalogMigrationUtil.CatalogType.HIVE,
+ Collections.singletonMap(
+ "uri", HIVE_METASTORE_EXTENSION.hiveConf().get("hive.metastore.uris")));
+ initializeTargetCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHiveToNessieCatalogMigrator.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHiveToNessieCatalogMigrator.java
new file mode 100644
index 00000000..c7ea8430
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITHiveToNessieCatalogMigrator.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import java.util.Collections;
+import org.apache.iceberg.hive.HiveMetastoreExtension;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+public class ITHiveToNessieCatalogMigrator extends AbstractTestCatalogMigrator {
+
+ @RegisterExtension
+ public static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().build();
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+ initializeSourceCatalog(
+ CatalogMigrationUtil.CatalogType.HIVE,
+ Collections.singletonMap(
+ "uri", HIVE_METASTORE_EXTENSION.hiveConf().get("hive.metastore.uris")));
+ initializeTargetCatalog(CatalogMigrationUtil.CatalogType.NESSIE, Collections.emptyMap());
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITNessieToHiveCatalogMigrator.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITNessieToHiveCatalogMigrator.java
new file mode 100644
index 00000000..cc6e0201
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/ITNessieToHiveCatalogMigrator.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import java.util.Collections;
+import java.util.Set;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.hive.HiveMetastoreExtension;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+public class ITNessieToHiveCatalogMigrator extends AbstractTestCatalogMigrator {
+
+ @RegisterExtension
+ public static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().build();
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+ initializeSourceCatalog(CatalogMigrationUtil.CatalogType.NESSIE, Collections.emptyMap());
+ initializeTargetCatalog(
+ CatalogMigrationUtil.CatalogType.HIVE,
+ Collections.singletonMap(
+ "uri", HIVE_METASTORE_EXTENSION.hiveConf().get("hive.metastore.uris")));
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+
+ @Test
+ public void testRegisterWithDefaultNamespace() {
+ sourceCatalog.createTable(TBL, schema);
+
+ CatalogMigrator catalogMigrator = catalogMigratorWithDefaultArgs(false);
+ // should also include table from empty namespace
+ Set matchingTableIdentifiers =
+ catalogMigrator.getMatchingTableIdentifiers(null);
+ Assertions.assertThat(matchingTableIdentifiers).contains(TBL);
+
+ matchingTableIdentifiers.forEach(catalogMigrator::registerTable);
+ CatalogMigrationResult result = catalogMigrator.result();
+ // hive will not support empty namespace (namespace with level = 0). Hence, register will
+ // fail.
+ Assertions.assertThat(result.registeredTableIdentifiers()).doesNotContain(TBL);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).containsExactly(TBL);
+ Assertions.assertThat(result.failedToDeleteTableIdentifiers()).isEmpty();
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/UnsupportedNamespaceTest.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/UnsupportedNamespaceTest.java
new file mode 100644
index 00000000..66127213
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/UnsupportedNamespaceTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.api;
+
+import java.nio.file.Path;
+import java.util.List;
+import org.apache.iceberg.BaseMetastoreCatalog;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.hadoop.HadoopCatalog;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+public class UnsupportedNamespaceTest {
+
+ protected static @TempDir Path tempDir;
+
+ @BeforeAll
+ protected static void initLogDir() {
+ System.setProperty("catalog.migration.log.dir", tempDir.toAbsolutePath().toString());
+ }
+
+ @Test
+ public void testUnsupportedNamespace() {
+
+ class TestCatalog extends BaseMetastoreCatalog {
+ // doesn't support namespaces
+ @Override
+ protected TableOperations newTableOps(TableIdentifier tableIdentifier) {
+ return null;
+ }
+
+ @Override
+ protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) {
+ return null;
+ }
+
+ @Override
+ public List listTables(Namespace namespace) {
+ return null;
+ }
+
+ @Override
+ public boolean dropTable(TableIdentifier identifier, boolean purge) {
+ return false;
+ }
+
+ @Override
+ public void renameTable(TableIdentifier from, TableIdentifier to) {}
+ }
+
+ Catalog sourceCatalog = new TestCatalog();
+ Catalog targetCatalog = new TestCatalog();
+
+ Assertions.assertThatThrownBy(
+ () ->
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(false)
+ .build())
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessageContaining(
+ "target catalog TestCatalog{} doesn't implement SupportsNamespaces to create missing namespaces.");
+
+ Assertions.assertThatThrownBy(
+ () ->
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(new HadoopCatalog())
+ .deleteEntriesFromSourceCatalog(false)
+ .build())
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessageContaining(
+ "source catalog TestCatalog{} doesn't implement SupportsNamespaces to list all namespaces.");
+ }
+}
diff --git a/iceberg-catalog-migrator/api/src/test/resources/logback.xml b/iceberg-catalog-migrator/api/src/test/resources/logback.xml
new file mode 100644
index 00000000..769b6180
--- /dev/null
+++ b/iceberg-catalog-migrator/api/src/test/resources/logback.xml
@@ -0,0 +1,34 @@
+
+
+
+
+ ${catalog.migration.log.dir}/catalog_migration.log
+ true
+
+ %date{ISO8601} [%thread] %-5level %logger{36} - %msg%n
+
+
+
+
+
+
diff --git a/iceberg-catalog-migrator/build.gradle.kts b/iceberg-catalog-migrator/build.gradle.kts
new file mode 100644
index 00000000..fbc0783c
--- /dev/null
+++ b/iceberg-catalog-migrator/build.gradle.kts
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+ `maven-publish`
+ signing
+ `build-conventions`
+}
+
+spotless {
+ kotlinGradle {
+ // Must be repeated :( - there's no "addTarget" or so
+ target("*.gradle.kts", "buildSrc/*.gradle.kts")
+ }
+}
diff --git a/iceberg-catalog-migrator/buildSrc/build.gradle.kts b/iceberg-catalog-migrator/buildSrc/build.gradle.kts
new file mode 100644
index 00000000..96798761
--- /dev/null
+++ b/iceberg-catalog-migrator/buildSrc/build.gradle.kts
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins { `kotlin-dsl` }
+
+repositories {
+ mavenCentral()
+ gradlePluginPortal()
+ if (System.getProperty("withMavenLocal").toBoolean()) {
+ mavenLocal()
+ }
+}
+
+dependencies {
+ implementation(gradleKotlinDsl())
+ implementation(baselibs.spotless)
+ implementation(baselibs.jandex)
+ implementation(baselibs.idea.ext)
+ implementation(baselibs.shadow)
+ implementation(baselibs.errorprone)
+}
+
+java {
+ sourceCompatibility = JavaVersion.VERSION_21
+ targetCompatibility = JavaVersion.VERSION_21
+}
+
+kotlinDslPluginOptions { jvmTarget.set(JavaVersion.VERSION_21.toString()) }
+
+tasks.withType().configureEach { useJUnitPlatform() }
diff --git a/iceberg-catalog-migrator/buildSrc/settings.gradle.kts b/iceberg-catalog-migrator/buildSrc/settings.gradle.kts
new file mode 100644
index 00000000..14085355
--- /dev/null
+++ b/iceberg-catalog-migrator/buildSrc/settings.gradle.kts
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+dependencyResolutionManagement {
+ versionCatalogs { create("baselibs") { from(files("../gradle/baselibs.versions.toml")) } }
+}
diff --git a/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Checkstyle.kt b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Checkstyle.kt
new file mode 100644
index 00000000..8d893cf4
--- /dev/null
+++ b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Checkstyle.kt
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.gradle.api.Project
+import org.gradle.api.plugins.quality.Checkstyle
+import org.gradle.api.plugins.quality.CheckstyleExtension
+import org.gradle.api.plugins.quality.CheckstylePlugin
+import org.gradle.kotlin.dsl.apply
+import org.gradle.kotlin.dsl.configure
+import org.gradle.kotlin.dsl.withType
+
+fun Project.configureCheckstyle() {
+ apply()
+ configure {
+ toolVersion = libsRequiredVersion("checkstyle")
+ config = resources.text.fromFile(rootProject.file("codestyle/checkstyle-config.xml"))
+ isShowViolations = true
+ isIgnoreFailures = false
+ }
+
+ tasks.withType().configureEach {
+ when (name) {
+ "checkstyleMain" -> dependsOn(tasks.named("processJandexIndex"))
+ "checkstyleTest" -> dependsOn(tasks.named("processTestJandexIndex"))
+ else -> {}
+ }
+ maxWarnings = 0 // treats warnings as errors
+ }
+}
diff --git a/iceberg-catalog-migrator/buildSrc/src/main/kotlin/CodeCoverage.kt b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/CodeCoverage.kt
new file mode 100644
index 00000000..26eb6317
--- /dev/null
+++ b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/CodeCoverage.kt
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.kotlin.dsl.apply
+import org.gradle.kotlin.dsl.configure
+import org.gradle.kotlin.dsl.withType
+import org.gradle.testing.jacoco.plugins.JacocoPlugin
+import org.gradle.testing.jacoco.plugins.JacocoPluginExtension
+import org.gradle.testing.jacoco.plugins.JacocoReportAggregationPlugin
+import org.gradle.testing.jacoco.tasks.JacocoReport
+
+class CodeCoveragePlugin : Plugin {
+ override fun apply(project: Project): Unit =
+ project.run {
+ apply()
+ apply()
+
+ tasks.withType().configureEach {
+ reports {
+ html.required.set(true)
+ xml.required.set(true)
+ }
+ }
+
+ configure { toolVersion = libsRequiredVersion("jacoco") }
+ }
+}
diff --git a/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Errorprone.kt b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Errorprone.kt
new file mode 100644
index 00000000..3220102e
--- /dev/null
+++ b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Errorprone.kt
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Properties
+import kotlin.collections.HashMap
+import net.ltgt.gradle.errorprone.CheckSeverity
+import net.ltgt.gradle.errorprone.ErrorPronePlugin
+import net.ltgt.gradle.errorprone.errorprone
+import org.gradle.api.Project
+import org.gradle.api.plugins.JavaPlugin
+import org.gradle.api.plugins.JavaPluginExtension
+import org.gradle.api.tasks.PathSensitivity
+import org.gradle.api.tasks.compile.JavaCompile
+import org.gradle.kotlin.dsl.apply
+import org.gradle.kotlin.dsl.configure
+import org.gradle.kotlin.dsl.dependencies
+import org.gradle.kotlin.dsl.withType
+
+fun Project.configureErrorprone() {
+ apply()
+ tasks.withType().configureEach {
+ options.errorprone.disableWarningsInGeneratedCode.set(true)
+
+ val errorproneRules = rootProject.projectDir.resolve("codestyle/errorprone-rules.properties")
+ inputs.file(errorproneRules).withPathSensitivity(PathSensitivity.RELATIVE)
+
+ val checksMapProperty =
+ objects
+ .mapProperty(String::class.java, CheckSeverity::class.java)
+ .convention(
+ provider {
+ val checksMap = HashMap()
+ errorproneRules.reader().use {
+ val rules = Properties()
+ rules.load(it)
+ rules.forEach { k, v ->
+ val key = k as String
+ val value = v as String
+ if (key.isNotEmpty() && value.isNotEmpty()) {
+ checksMap[key.trim()] = CheckSeverity.valueOf(value.trim())
+ }
+ }
+ }
+ checksMap
+ }
+ )
+
+ options.errorprone.checks.putAll(checksMapProperty)
+ options.errorprone.excludedPaths.set(".*/build/generated.*")
+ }
+ plugins.withType().configureEach {
+ configure {
+ sourceSets.configureEach {
+ dependencies {
+ add(
+ "errorprone",
+ "com.google.errorprone:error_prone_core:${libsRequiredVersion("errorprone")}",
+ )
+ add(
+ "errorprone",
+ "jp.skypencil.errorprone.slf4j:errorprone-slf4j:${libsRequiredVersion("errorproneSlf4j")}",
+ )
+ }
+ }
+ }
+ }
+}
diff --git a/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Ide.kt b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Ide.kt
new file mode 100644
index 00000000..3285c9d6
--- /dev/null
+++ b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Ide.kt
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.gradle.api.Project
+import org.gradle.kotlin.dsl.apply
+import org.gradle.kotlin.dsl.configure
+import org.gradle.plugins.ide.eclipse.EclipsePlugin
+import org.gradle.plugins.ide.eclipse.model.EclipseModel
+import org.gradle.plugins.ide.idea.model.IdeaModel
+import org.jetbrains.gradle.ext.ActionDelegationConfig
+import org.jetbrains.gradle.ext.IdeaExtPlugin
+import org.jetbrains.gradle.ext.copyright
+import org.jetbrains.gradle.ext.delegateActions
+import org.jetbrains.gradle.ext.encodings
+import org.jetbrains.gradle.ext.runConfigurations
+import org.jetbrains.gradle.ext.settings
+
+fun Project.configureIde() {
+ apply()
+
+ if (this == rootProject) {
+
+ val projectName = rootProject.file("ide-name.txt").readText().trim()
+ val ideName =
+ "$projectName ${rootProject.version.toString().replace(Regex("^([0-9.]+).*"), "$1")}"
+
+ apply()
+ configure {
+ module {
+ name = ideName
+ isDownloadSources = true // this is the default BTW
+ inheritOutputDirs = true
+ }
+
+ project.settings {
+ copyright {
+ useDefault = "Iceberg-catalog-migrator"
+ profiles.create("Iceberg-catalog-migrator") {
+ // strip trailing LF
+ val copyrightText =
+ rootProject.file("codestyle/copyright-header.txt").readLines().joinToString("\n")
+ notice = copyrightText
+ }
+ }
+
+ encodings.encoding = "UTF-8"
+ encodings.properties.encoding = "UTF-8"
+
+ runConfigurations.register("Gradle", org.jetbrains.gradle.ext.Gradle::class.java) {
+ defaults = true
+
+ jvmArgs =
+ rootProject.projectDir
+ .resolve("gradle.properties")
+ .reader()
+ .use {
+ val rules = java.util.Properties()
+ rules.load(it)
+ rules
+ }
+ .map { e -> "-D${e.key}=${e.value}" }
+ .joinToString(" ")
+ }
+
+ delegateActions.testRunner = ActionDelegationConfig.TestRunner.CHOOSE_PER_TEST
+ }
+ }
+
+ // There's no proper way to set the name of the IDEA project (when "just importing" or syncing
+ // the Gradle project)
+ val ideaDir = projectDir.resolve(".idea")
+
+ if (ideaDir.isDirectory) {
+ ideaDir.resolve(".name").writeText(ideName)
+ }
+
+ configure { project { name = ideName } }
+ }
+}
diff --git a/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Jandex.kt b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Jandex.kt
new file mode 100644
index 00000000..4780eabb
--- /dev/null
+++ b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Jandex.kt
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import com.github.vlsi.jandex.JandexExtension
+import com.github.vlsi.jandex.JandexPlugin
+import org.gradle.api.Project
+import org.gradle.api.tasks.testing.Test
+import org.gradle.kotlin.dsl.apply
+import org.gradle.kotlin.dsl.configure
+import org.gradle.kotlin.dsl.withType
+
+fun Project.configureJandex() {
+ apply()
+ configure { toolVersion.set(libsRequiredVersion("jandex")) }
+
+ tasks.withType().configureEach { dependsOn(tasks.named("processTestJandexIndex")) }
+}
diff --git a/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Java.kt b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Java.kt
new file mode 100644
index 00000000..b301c15a
--- /dev/null
+++ b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Java.kt
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.gradle.api.Project
+import org.gradle.api.file.DuplicatesStrategy
+import org.gradle.api.plugins.JavaPlugin
+import org.gradle.api.plugins.JavaPluginExtension
+import org.gradle.api.tasks.bundling.Jar
+import org.gradle.api.tasks.compile.AbstractCompile
+import org.gradle.api.tasks.compile.JavaCompile
+import org.gradle.api.tasks.javadoc.Javadoc
+import org.gradle.external.javadoc.CoreJavadocOptions
+import org.gradle.kotlin.dsl.configure
+import org.gradle.kotlin.dsl.repositories
+import org.gradle.kotlin.dsl.withType
+import org.gradle.language.jvm.tasks.ProcessResources
+
+fun Project.configureJava() {
+ tasks.withType().configureEach {
+ manifest {
+ attributes["Implementation-Title"] = "iceberg-catalog-migrator"
+ attributes["Implementation-Version"] = project.version
+ }
+ duplicatesStrategy = DuplicatesStrategy.WARN
+ }
+
+ repositories {
+ mavenCentral()
+ if (System.getProperty("withMavenLocal").toBoolean()) {
+ mavenLocal()
+ }
+ }
+
+ tasks.withType().configureEach {
+ options.encoding = "UTF-8"
+ options.compilerArgs.add("-parameters")
+ options.release.set(21)
+
+ // Required to enable incremental compilation w/ immutables, see
+ // https://github.com/immutables/immutables/pull/858 and
+ // https://github.com/immutables/immutables/issues/804#issuecomment-487366544
+ options.compilerArgs.add("-Aimmutables.gradle.incremental")
+ }
+
+ tasks.withType().configureEach {
+ val opt = options as CoreJavadocOptions
+ // don't spam log w/ "warning: no @param/@return"
+ opt.addStringOption("Xdoclint:-reference", "-quiet")
+ }
+
+ plugins.withType().configureEach {
+ configure {
+ withJavadocJar()
+ withSourcesJar()
+ }
+ }
+
+ tasks.register("compileAll").configure {
+ group = "build"
+ description = "Runs all compilation and jar tasks"
+ dependsOn(
+ tasks.withType(AbstractCompile::class.java),
+ tasks.withType(ProcessResources::class.java),
+ )
+ }
+}
diff --git a/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Spotless.kt b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Spotless.kt
new file mode 100644
index 00000000..a2662190
--- /dev/null
+++ b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Spotless.kt
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import com.diffplug.gradle.spotless.SpotlessExtension
+import com.diffplug.gradle.spotless.SpotlessPlugin
+import org.gradle.api.Project
+import org.gradle.kotlin.dsl.apply
+import org.gradle.kotlin.dsl.configure
+import org.gradle.kotlin.dsl.withType
+
+fun Project.configureSpotless() {
+
+ apply()
+ if (!java.lang.Boolean.getBoolean("idea.sync.active")) {
+ plugins.withType().configureEach {
+ configure {
+ format("xml") {
+ target("src/**/*.xml", "src/**/*.xsd")
+ eclipseWtp(com.diffplug.spotless.extra.wtp.EclipseWtpFormatterStep.XML)
+ .configFile(rootProject.projectDir.resolve("codestyle/org.eclipse.wst.xml.core.prefs"))
+ }
+ kotlinGradle {
+ ktfmt().googleStyle()
+ licenseHeaderFile(rootProject.file("codestyle/copyright-header-java.txt"), "$")
+ if (project == rootProject) {
+ target("*.gradle.kts", "buildSrc/*.gradle.kts")
+ }
+ }
+ if (project == rootProject) {
+ kotlin {
+ ktfmt().googleStyle()
+ licenseHeaderFile(rootProject.file("codestyle/copyright-header-java.txt"), "$")
+ target("buildSrc/src/**/kotlin/**")
+ targetExclude("buildSrc/build/**")
+ }
+ }
+
+ val dirsInSrc = projectDir.resolve("src").listFiles()
+ val sourceLangs =
+ if (dirsInSrc != null)
+ dirsInSrc
+ .filter { f -> f.isDirectory }
+ .map { f -> f.listFiles() }
+ .filterNotNull()
+ .flatMap { l -> l.filter { f -> f.isDirectory } }
+ .map { f -> f.name }
+ .distinct()
+ else listOf()
+
+ if (sourceLangs.contains("java")) {
+ java {
+ googleJavaFormat(libsRequiredVersion("googleJavaFormat"))
+ licenseHeaderFile(rootProject.file("codestyle/copyright-header-java.txt"))
+ target("src/**/java/**")
+ targetExclude("build/**")
+ }
+ }
+ if (sourceLangs.contains("kotlin")) {
+ kotlin {
+ ktfmt().googleStyle()
+ licenseHeaderFile(rootProject.file("codestyle/copyright-header-java.txt"), "$")
+ target("src/**/kotlin/**")
+ targetExclude("build/**")
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Testing.kt b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Testing.kt
new file mode 100644
index 00000000..236fb80f
--- /dev/null
+++ b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Testing.kt
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.gradle.api.Project
+import org.gradle.api.tasks.testing.Test
+import org.gradle.kotlin.dsl.named
+import org.gradle.kotlin.dsl.provideDelegate
+import org.gradle.kotlin.dsl.register
+import org.gradle.kotlin.dsl.withType
+
+fun Project.configureTestTasks() {
+ tasks.withType().configureEach {
+ useJUnitPlatform {}
+ val testJvmArgs: String? by project
+ val testHeapSize: String? by project
+ if (testJvmArgs != null) {
+ jvmArgs((testJvmArgs as String).split(" "))
+ }
+
+ systemProperty("file.encoding", "UTF-8")
+ systemProperty("user.language", "en")
+ systemProperty("user.country", "US")
+ systemProperty("user.variant", "")
+ filter {
+ isFailOnNoMatchingTests = false
+ when (name) {
+ "test" -> {
+ includeTestsMatching("*Test")
+ includeTestsMatching("Test*")
+ excludeTestsMatching("Abstract*")
+ excludeTestsMatching("IT*")
+ }
+ "intTest" -> includeTestsMatching("IT*")
+ }
+ }
+ if (name != "test") {
+ mustRunAfter(tasks.named("test"))
+ }
+
+ if (testHeapSize != null) {
+ setMinHeapSize(testHeapSize)
+ setMaxHeapSize(testHeapSize)
+ }
+ }
+ val intTest =
+ tasks.register("intTest") {
+ group = "verification"
+ description = "Runs the integration tests."
+ }
+ tasks.named("check") { dependsOn(intTest) }
+}
diff --git a/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Utilities.kt b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Utilities.kt
new file mode 100644
index 00000000..d00a236e
--- /dev/null
+++ b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/Utilities.kt
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+import org.gradle.api.Project
+import org.gradle.api.artifacts.VersionCatalogsExtension
+import org.gradle.api.tasks.bundling.Jar
+import org.gradle.kotlin.dsl.getByType
+import org.gradle.kotlin.dsl.named
+import org.gradle.kotlin.dsl.withType
+
+fun Project.libsRequiredVersion(name: String): String {
+ val libVer =
+ extensions.getByType().named("libs").findVersion(name).get()
+ val reqVer = libVer.requiredVersion
+ check(reqVer.isNotEmpty()) {
+ "libs-version for '$name' is empty, but must not be empty, version. strict: ${libVer.strictVersion}, required: ${libVer.requiredVersion}, preferred: ${libVer.preferredVersion}"
+ }
+ return reqVer
+}
+
+fun Project.applyShadowJar() {
+ plugins.apply(ShadowPlugin::class.java)
+
+ plugins.withType().configureEach {
+ val shadowJar =
+ tasks.named("shadowJar") {
+ outputs.cacheIf { false } // do not cache uber/shaded jars
+ archiveClassifier.set("")
+ mergeServiceFiles()
+ }
+
+ tasks.named("jar") {
+ dependsOn(shadowJar)
+ archiveClassifier.set("raw")
+ }
+ }
+}
diff --git a/iceberg-catalog-migrator/buildSrc/src/main/kotlin/build-conventions.gradle.kts b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/build-conventions.gradle.kts
new file mode 100644
index 00000000..4be6e6bc
--- /dev/null
+++ b/iceberg-catalog-migrator/buildSrc/src/main/kotlin/build-conventions.gradle.kts
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+val hasSrcMain = projectDir.resolve("src/main").exists()
+val hasSrcTest = projectDir.resolve("src/test").exists()
+
+configureIde()
+
+configureSpotless()
+
+configureJandex()
+
+configureJava()
+
+apply()
+
+if (hasSrcMain || hasSrcTest) {
+ configureCheckstyle()
+
+ configureErrorprone()
+
+ if (hasSrcTest) {
+ configureTestTasks()
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/build.gradle.kts b/iceberg-catalog-migrator/cli/build.gradle.kts
new file mode 100644
index 00000000..b7534127
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/build.gradle.kts
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+
+plugins {
+ `java-library`
+ `maven-publish`
+ signing
+ alias(libs.plugins.nessie.run)
+ `build-conventions`
+}
+
+java.sourceCompatibility = JavaVersion.VERSION_21
+
+applyShadowJar()
+
+dependencies {
+ implementation(project(":iceberg-catalog-migrator-api"))
+ implementation(libs.guava)
+ implementation(libs.slf4j)
+ runtimeOnly(libs.logback.classic)
+ implementation(libs.picocli)
+ implementation(libs.iceberg.spark.runtime)
+ implementation(libs.hadoop.aws) { exclude("com.amazonaws", "aws-java-sdk-bundle") }
+ // AWS dependencies based on https://iceberg.apache.org/docs/latest/aws/#enabling-aws-integration
+ runtimeOnly(libs.aws.sdk.apache.client)
+ runtimeOnly(libs.aws.sdk.auth)
+ runtimeOnly(libs.aws.sdk.glue)
+ runtimeOnly(libs.aws.sdk.s3)
+ runtimeOnly(libs.aws.sdk.dynamo)
+ runtimeOnly(libs.aws.sdk.kms)
+ runtimeOnly(libs.aws.sdk.lakeformation)
+ runtimeOnly(libs.aws.sdk.sts)
+ runtimeOnly(libs.aws.sdk.url.connection.client)
+
+ // needed for Hive catalog
+ runtimeOnly("org.apache.hive:hive-metastore:${libs.versions.hive.get()}") {
+ // these are taken from iceberg repo configurations
+ exclude("org.apache.avro", "avro")
+ exclude("org.slf4j", "slf4j-log4j12")
+ exclude("org.pentaho") // missing dependency
+ exclude("org.apache.hbase")
+ exclude("org.apache.logging.log4j")
+ exclude("co.cask.tephra")
+ exclude("com.google.code.findbugs", "jsr305")
+ exclude("org.eclipse.jetty.aggregate", "jetty-all")
+ exclude("org.eclipse.jetty.orbit", "javax.servlet")
+ exclude("org.apache.parquet", "parquet-hadoop-bundle")
+ exclude("com.tdunning", "json")
+ exclude("javax.transaction", "transaction-api")
+ exclude("com.zaxxer", "HikariCP")
+ }
+ runtimeOnly("org.apache.hive:hive-exec:${libs.versions.hive.get()}:core") {
+ // these are taken from iceberg repo configurations
+ exclude("org.apache.avro", "avro")
+ exclude("org.slf4j", "slf4j-log4j12")
+ exclude("org.pentaho") // missing dependency
+ exclude("org.apache.hive", "hive-llap-tez")
+ exclude("org.apache.logging.log4j")
+ exclude("com.google.protobuf", "protobuf-java")
+ exclude("org.apache.calcite")
+ exclude("org.apache.calcite.avatica")
+ exclude("com.google.code.findbugs", "jsr305")
+ }
+ runtimeOnly("org.apache.hadoop:hadoop-mapreduce-client-core:${libs.versions.hadoop.get()}")
+
+ testImplementation(platform(libs.junit.bom))
+ testImplementation("org.junit.jupiter:junit-jupiter-params")
+ testImplementation("org.junit.jupiter:junit-jupiter-api")
+ testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine")
+ testRuntimeOnly("org.junit.platform:junit-platform-launcher")
+ testImplementation(libs.assertj)
+ testImplementation(libs.logcaptor)
+
+ testImplementation(project(":iceberg-catalog-migrator-api-test"))
+
+ // for integration tests
+ testImplementation(
+ "org.apache.iceberg:iceberg-hive-metastore:${libs.versions.iceberg.get()}:tests"
+ )
+ // this junit4 dependency is needed for above Iceberg's TestHiveMetastore
+ testRuntimeOnly("junit:junit:4.13.2")
+
+ testImplementation("org.apache.hive:hive-metastore:${libs.versions.hive.get()}") {
+ // these are taken from iceberg repo configurations
+ exclude("org.apache.avro", "avro")
+ exclude("org.slf4j", "slf4j-log4j12")
+ exclude("org.pentaho") // missing dependency
+ exclude("org.apache.hbase")
+ exclude("org.apache.logging.log4j")
+ exclude("co.cask.tephra")
+ exclude("com.google.code.findbugs", "jsr305")
+ exclude("org.eclipse.jetty.aggregate", "jetty-all")
+ exclude("org.eclipse.jetty.orbit", "javax.servlet")
+ exclude("org.apache.parquet", "parquet-hadoop-bundle")
+ exclude("com.tdunning", "json")
+ exclude("javax.transaction", "transaction-api")
+ exclude("com.zaxxer", "HikariCP")
+ }
+ testImplementation("org.apache.hive:hive-exec:${libs.versions.hive.get()}:core") {
+ // these are taken from iceberg repo configurations
+ exclude("org.apache.avro", "avro")
+ exclude("org.slf4j", "slf4j-log4j12")
+ exclude("org.pentaho") // missing dependency
+ exclude("org.apache.hive", "hive-llap-tez")
+ exclude("org.apache.logging.log4j")
+ exclude("com.google.protobuf", "protobuf-java")
+ exclude("org.apache.calcite")
+ exclude("org.apache.calcite.avatica")
+ exclude("com.google.code.findbugs", "jsr305")
+ }
+ testImplementation("org.apache.hadoop:hadoop-mapreduce-client-core:${libs.versions.hadoop.get()}")
+
+ nessieQuarkusServer(
+ "org.projectnessie.nessie:nessie-quarkus:${libs.versions.nessie.get()}:runner"
+ )
+}
+
+nessieQuarkusApp { includeTask(tasks.named("intTest")) }
+
+tasks.named("test") { systemProperty("expectedCLIVersion", project.version) }
+
+val processResources =
+ tasks.named("processResources") {
+ inputs.property("projectVersion", project.version)
+ filter(
+ org.apache.tools.ant.filters.ReplaceTokens::class,
+ mapOf("tokens" to mapOf("projectVersion" to project.version)),
+ )
+ }
+
+val mainClassName = "org.apache.polaris.iceberg.catalog.migrator.cli.CatalogMigrationCLI"
+
+val shadowJar = tasks.named("shadowJar") { isZip64 = true }
+
+shadowJar { manifest { attributes["Main-Class"] = mainClassName } }
+
+tasks.withType().configureEach { systemProperty("java.security.manager", "allow") }
diff --git a/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/BaseRegisterCommand.java b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/BaseRegisterCommand.java
new file mode 100644
index 00000000..cef41dea
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/BaseRegisterCommand.java
@@ -0,0 +1,359 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import com.google.common.base.Preconditions;
+import java.io.Console;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationResult;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+public abstract class BaseRegisterCommand implements Callable {
+
+ @CommandLine.ArgGroup(
+ exclusive = false,
+ multiplicity = "1",
+ heading = "Source catalog options: %n")
+ protected SourceCatalogOptions sourceCatalogOptions;
+
+ @CommandLine.ArgGroup(
+ exclusive = false,
+ multiplicity = "1",
+ heading = "Target catalog options: %n")
+ private TargetCatalogOptions targetCatalogOptions;
+
+ @CommandLine.ArgGroup(heading = "Identifier options: %n")
+ private IdentifierOptions identifierOptions;
+
+ @CommandLine.Option(
+ names = {"--output-dir"},
+ defaultValue = "",
+ description = {
+ "Optional local output directory path to write CLI output files like `failed_identifiers.txt`, "
+ + "`failed_to_delete_at_source.txt`, `dry_run_identifiers.txt`. "
+ + "If not specified, uses the present working directory.",
+ "Example: --output-dir /tmp/output/",
+ " --output-dir $PWD/output_folder"
+ })
+ private Path outputDirPath;
+
+ @CommandLine.Option(
+ names = {"--dry-run"},
+ description =
+ "Optional configuration to simulate the registration without actually registering. Can learn about a list "
+ + "of tables that will be registered by running this.")
+ private boolean isDryRun;
+
+ @CommandLine.Option(
+ names = {"--disable-safety-prompts"},
+ description = "Optional configuration to disable safety prompts which needs console input.")
+ private boolean disablePrompts;
+
+ @CommandLine.Option(
+ names = {"--stacktrace"},
+ description =
+ "Optional configuration to enable capturing stacktrace in logs in case of failures.")
+ private boolean enableStackTrace;
+
+ private static final int BATCH_SIZE = 100;
+ public static final String FAILED_IDENTIFIERS_FILE = "failed_identifiers.txt";
+ public static final String FAILED_TO_DELETE_AT_SOURCE_FILE = "failed_to_delete_at_source.txt";
+ public static final String DRY_RUN_FILE = "dry_run_identifiers.txt";
+
+ private static final Logger consoleLog = LoggerFactory.getLogger("console-log");
+
+ public BaseRegisterCommand() {}
+
+ protected abstract CatalogMigrator catalogMigrator(
+ Catalog sourceCatalog, Catalog targetCatalog, boolean enableStackTrace);
+
+ protected abstract boolean canProceed(Catalog sourceCatalog);
+
+ protected abstract String operation();
+
+ protected abstract String operated();
+
+ protected abstract String operate();
+
+ @Override
+ public Integer call() {
+ Set identifiers = Collections.emptySet();
+ String identifierRegEx = identifierOptions != null ? identifierOptions.identifiersRegEx : null;
+
+ if (identifierOptions != null) {
+ identifiers = identifierOptions.processIdentifiersInput();
+ }
+ checkAndWarnAboutIdentifiers(identifiers, identifierRegEx);
+
+ validateOutputDir();
+
+ Catalog sourceCatalog = null;
+ Catalog targetCatalog = null;
+
+ try {
+ sourceCatalog = sourceCatalogOptions.build();
+ consoleLog.info("Configured source catalog: {}", sourceCatalog.name());
+
+ targetCatalog = targetCatalogOptions.build();
+ consoleLog.info("Configured target catalog: {}", targetCatalog.name());
+
+ if (!isDryRun && !disablePrompts && !canProceed(sourceCatalog)) {
+ return 1;
+ }
+
+ CatalogMigrator catalogMigrator =
+ catalogMigrator(sourceCatalog, targetCatalog, enableStackTrace);
+
+ if (identifiers.isEmpty()) {
+ consoleLog.info("Identifying tables for {} ...", operation());
+ identifiers = catalogMigrator.getMatchingTableIdentifiers(identifierRegEx);
+ if (identifiers.isEmpty()) {
+ consoleLog.warn(
+ "No tables were identified for {}. Please check `catalog_migration.log` file for more info.",
+ operation());
+ return 1;
+ }
+ }
+
+ if (isDryRun) {
+ consoleLog.info("Dry run is completed.");
+ handleDryRunResult(identifiers);
+ return 0;
+ }
+
+ consoleLog.info("Identified {} tables for {}.", identifiers.size(), operation());
+
+ consoleLog.info("Started {} ...", operation());
+
+ CatalogMigrationResult result;
+ try {
+ int processedIdentifiersCount = 0;
+ for (TableIdentifier identifier : identifiers) {
+ catalogMigrator.registerTable(identifier);
+ processedIdentifiersCount++;
+ if (processedIdentifiersCount % BATCH_SIZE == 0
+ || processedIdentifiersCount == identifiers.size()) {
+ consoleLog.info(
+ "Attempted {} for {} tables out of {} tables.",
+ operation(),
+ processedIdentifiersCount,
+ identifiers.size());
+ }
+ }
+ } finally {
+ consoleLog.info("Finished {} ...", operation());
+ result = catalogMigrator.result();
+ handleResults(result);
+ }
+
+ if (!result.failedToRegisterTableIdentifiers().isEmpty()
+ || !result.failedToDeleteTableIdentifiers().isEmpty()
+ || result.registeredTableIdentifiers().isEmpty()) {
+ return 1;
+ }
+
+ return 0;
+ } finally {
+ close(sourceCatalog);
+ close(targetCatalog);
+ }
+ }
+
+ private void close(Catalog catalog) {
+ if (catalog instanceof AutoCloseable) {
+ try {
+ ((AutoCloseable) catalog).close();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private void checkAndWarnAboutIdentifiers(
+ Set identifiers, String identifierRegEx) {
+ if (identifiers.isEmpty()) {
+ if (identifierRegEx != null) {
+ consoleLog.warn(
+ "User has not specified the table identifiers."
+ + " Will be selecting all the tables from all the namespaces from the source catalog "
+ + "which matches the regex pattern:{}",
+ identifierRegEx);
+ } else {
+ consoleLog.warn(
+ "User has not specified the table identifiers."
+ + " Will be selecting all the tables from all the namespaces from the source catalog.");
+ }
+ }
+ }
+
+ private void validateOutputDir() {
+ if (!Files.exists(outputDirPath)) {
+ try {
+ Files.createDirectories(outputDirPath);
+ } catch (IOException ex) {
+ throw new UncheckedIOException(
+ "Failed to create the output directory from the path specified in `--output-dir`", ex);
+ }
+ }
+ Preconditions.checkArgument(
+ Files.isWritable(outputDirPath), "Path specified in `--output-dir` is not writable");
+ }
+
+ private void handleResults(CatalogMigrationResult result) {
+ try {
+ writeToFile(
+ outputDirPath.resolve(FAILED_IDENTIFIERS_FILE),
+ result.failedToRegisterTableIdentifiers());
+ writeToFile(
+ outputDirPath.resolve(FAILED_TO_DELETE_AT_SOURCE_FILE),
+ result.failedToDeleteTableIdentifiers());
+ } finally {
+ printSummary(result);
+ printDetails(result);
+ }
+ }
+
+ private void handleDryRunResult(Set identifiers) {
+ try {
+ writeToFile(outputDirPath.resolve(DRY_RUN_FILE), identifiers);
+ } finally {
+ printDryRunResult(identifiers);
+ }
+ }
+
+ private void printSummary(CatalogMigrationResult result) {
+ consoleLog.info("Summary: ");
+ if (!result.registeredTableIdentifiers().isEmpty()) {
+ consoleLog.info(
+ "Successfully {} {} tables from {} catalog to {} catalog.",
+ operated(),
+ result.registeredTableIdentifiers().size(),
+ sourceCatalogOptions.type.name(),
+ targetCatalogOptions.type.name());
+ }
+ if (!result.failedToRegisterTableIdentifiers().isEmpty()) {
+ consoleLog.error(
+ "Failed to {} {} tables from {} catalog to {} catalog. "
+ + "Please check the `catalog_migration.log` file for the failure reason. "
+ + "Failed identifiers are written into `{}`. "
+ + "Retry with that file using `--identifiers-from-file` option "
+ + "if the failure is because of network/connection timeouts.",
+ operate(),
+ result.failedToRegisterTableIdentifiers().size(),
+ sourceCatalogOptions.type.name(),
+ targetCatalogOptions.type.name(),
+ FAILED_IDENTIFIERS_FILE);
+ }
+ if (!result.failedToDeleteTableIdentifiers().isEmpty()) {
+ consoleLog.error(
+ "Failed to delete {} tables from {} catalog. "
+ + "Please check the `catalog_migration.log` file for the failure reason. "
+ + "{}Failed to delete identifiers are written into `{}`.",
+ result.failedToDeleteTableIdentifiers().size(),
+ sourceCatalogOptions.type.name(),
+ System.lineSeparator(),
+ FAILED_TO_DELETE_AT_SOURCE_FILE);
+ }
+ }
+
+ private void printDetails(CatalogMigrationResult result) {
+ consoleLog.info("Details: ");
+ if (!result.registeredTableIdentifiers().isEmpty()) {
+ consoleLog.info(
+ "Successfully {} these tables:{}{}",
+ operated(),
+ System.lineSeparator(),
+ result.registeredTableIdentifiers());
+ }
+
+ if (!result.failedToRegisterTableIdentifiers().isEmpty()) {
+ consoleLog.error(
+ "Failed to {} these tables:{}{}",
+ operate(),
+ System.lineSeparator(),
+ result.failedToRegisterTableIdentifiers());
+ }
+
+ if (!result.failedToDeleteTableIdentifiers().isEmpty()) {
+ consoleLog.error(
+ "Failed to delete these tables from source catalog:{}{}",
+ System.lineSeparator(),
+ result.failedToDeleteTableIdentifiers());
+ }
+ }
+
+ private void printDryRunResult(Set result) {
+ consoleLog.info("Summary: ");
+ consoleLog.info(
+ "Identified {} tables for {} by dry-run. These identifiers are also written into {}. "
+ + "This file can be used with `--identifiers-from-file` option for an actual run.",
+ result.size(),
+ operation(),
+ DRY_RUN_FILE);
+ consoleLog.info(
+ "Details: {}Identified these tables for {} by dry-run:{}{}",
+ System.lineSeparator(),
+ operation(),
+ System.lineSeparator(),
+ result);
+ }
+
+ private static void writeToFile(Path filePath, Collection identifiers) {
+ List identifiersString =
+ identifiers.stream().map(TableIdentifier::toString).collect(Collectors.toList());
+ try {
+ Files.write(filePath, identifiersString);
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to write the file:" + filePath, e);
+ }
+ }
+
+ protected boolean proceed() {
+ Console console = System.console();
+ while (true) {
+ consoleLog.info(
+ "Are you certain that you wish to proceed, after reading the above warnings? (yes/no):");
+ String input = console.readLine();
+
+ if (input.equalsIgnoreCase("yes")) {
+ consoleLog.info("Continuing...");
+ return true;
+ } else if (input.equalsIgnoreCase("no")) {
+ consoleLog.info("Aborting...");
+ return false;
+ } else {
+ consoleLog.info("Invalid input. Please enter 'yes' or 'no'.");
+ }
+ }
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/CLIVersionProvider.java b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/CLIVersionProvider.java
new file mode 100644
index 00000000..3fecc2d4
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/CLIVersionProvider.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.io.InputStream;
+import java.util.Properties;
+import picocli.CommandLine.IVersionProvider;
+
+public class CLIVersionProvider implements IVersionProvider {
+ @Override
+ public String[] getVersion() throws Exception {
+ try (InputStream input =
+ CLIVersionProvider.class
+ .getResource("version.properties")
+ .openConnection()
+ .getInputStream()) {
+ Properties props = new Properties();
+ props.load(input);
+ return new String[] {props.getProperty("cli.version")};
+ }
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/CatalogMigrationCLI.java b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/CatalogMigrationCLI.java
new file mode 100644
index 00000000..6d4b0820
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/CatalogMigrationCLI.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+@CommandLine.Command(
+ name = "iceberg-catalog-migrator",
+ mixinStandardHelpOptions = true,
+ versionProvider = CLIVersionProvider.class,
+ subcommands = {MigrateCommand.class, RegisterCommand.class})
+public class CatalogMigrationCLI {
+
+ public CatalogMigrationCLI() {}
+
+ private static final Logger consoleLog = LoggerFactory.getLogger("console-log");
+
+ public static void main(String... args) {
+ CommandLine commandLine =
+ new CommandLine(new CatalogMigrationCLI())
+ .setExecutionExceptionHandler(
+ (ex, cmd, parseResult) -> {
+ if (enableStacktrace(args)) {
+ cmd.getErr().println(cmd.getColorScheme().richStackTraceString(ex));
+ } else {
+ consoleLog.error(
+ "Error during CLI execution: {}. Please check `catalog_migration.log` file for more info.",
+ ex.getMessage());
+ }
+ return 1;
+ });
+ commandLine.setUsageHelpWidth(150);
+ int exitCode = commandLine.execute(args);
+ System.exit(exitCode);
+ }
+
+ private static boolean enableStacktrace(String... args) {
+ for (String arg : args) {
+ if (arg.equalsIgnoreCase("--stacktrace")) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/IdentifierOptions.java b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/IdentifierOptions.java
new file mode 100644
index 00000000..7047935b
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/IdentifierOptions.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+import java.util.stream.Collectors;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+public class IdentifierOptions {
+
+ @CommandLine.Option(
+ names = {"--identifiers"},
+ split = ",",
+ description = {
+ "Optional selective set of identifiers to register. If not specified, all the tables will be registered. "
+ + "Use this when there are few identifiers that need to be registered. For a large number of identifiers, "
+ + "use the `--identifiers-from-file` or `--identifiers-regex` option.",
+ "Example: --identifiers foo.t1,bar.t2"
+ })
+ protected Set identifiers = new HashSet<>();
+
+ @CommandLine.Option(
+ names = {"--identifiers-from-file"},
+ description = {
+ "Optional text file path that contains a set of table identifiers (one per line) to register. Should not be "
+ + "used with `--identifiers` or `--identifiers-regex` option.",
+ "Example: --identifiers-from-file /tmp/files/ids.txt"
+ })
+ protected String identifiersFromFile;
+
+ @CommandLine.Option(
+ names = {"--identifiers-regex"},
+ description = {
+ "Optional regular expression pattern used to register only the tables whose identifiers match this pattern. "
+ + "Should not be used with `--identifiers` or '--identifiers-from-file' option.",
+ "Example: --identifiers-regex ^foo\\..*"
+ })
+ protected String identifiersRegEx;
+
+ private static final Logger consoleLog = LoggerFactory.getLogger("console-log");
+
+ protected Set processIdentifiersInput() {
+
+ if (!identifiers.isEmpty()) {
+ return identifiers.stream()
+ .map(TableIdentifier::parse)
+ .collect(Collectors.toCollection(LinkedHashSet::new));
+ } else if (identifiersFromFile != null) {
+ Preconditions.checkArgument(
+ Files.exists(Paths.get(identifiersFromFile)),
+ "File specified in `--identifiers-from-file` option does not exist");
+ try {
+ consoleLog.info("Collecting identifiers from the file {} ...", identifiersFromFile);
+ return Files.readAllLines(Paths.get(identifiersFromFile)).stream()
+ .map(String::trim)
+ .filter(string -> !string.isEmpty())
+ .map(TableIdentifier::parse)
+ .collect(Collectors.toCollection(LinkedHashSet::new));
+ } catch (IOException e) {
+ throw new UncheckedIOException(
+ String.format("Failed to read the file: %s", identifiersFromFile), e);
+ }
+ } else if (identifiersRegEx != null) {
+ Preconditions.checkArgument(
+ !identifiersRegEx.trim().isEmpty(), "--identifiers-regex should not be empty");
+ // check whether pattern is compilable
+ try {
+ Pattern.compile(identifiersRegEx);
+ } catch (PatternSyntaxException ex) {
+ throw new IllegalArgumentException("--identifiers-regex pattern is not compilable", ex);
+ }
+ }
+ return Sets.newHashSet();
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/MigrateCommand.java b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/MigrateCommand.java
new file mode 100644
index 00000000..5616e07c
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/MigrateCommand.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrator;
+import org.apache.polaris.iceberg.catalog.migrator.api.ImmutableCatalogMigrator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+@CommandLine.Command(
+ name = "migrate",
+ mixinStandardHelpOptions = true,
+ versionProvider = CLIVersionProvider.class,
+ // As both source and target catalog has similar configurations,
+ // documentation is easy to read if the target and source property is one after another instead
+ // of sorted order.
+ sortOptions = false,
+ description =
+ "Bulk migrate the iceberg tables from source catalog to target catalog without data copy."
+ + " Table entries from source catalog will be deleted after the successful migration to the target "
+ + "catalog.")
+public class MigrateCommand extends BaseRegisterCommand {
+
+ private static final String newLine = System.lineSeparator();
+ private static final Logger consoleLog = LoggerFactory.getLogger("console-log");
+
+ @Override
+ protected CatalogMigrator catalogMigrator(
+ Catalog sourceCatalog, Catalog targetCatalog, boolean enableStackTrace) {
+
+ return ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(true)
+ .enableStacktrace(enableStackTrace)
+ .build();
+ }
+
+ @Override
+ public Integer call() {
+ if (sourceCatalogOptions.type == CatalogMigrationUtil.CatalogType.HADOOP) {
+ consoleLog.error(
+ "Source catalog is a Hadoop catalog and it doesn't support deleting the table entries just from the catalog. "
+ + "Please use 'register' command instead.");
+ return 1;
+ }
+ return super.call();
+ }
+
+ @Override
+ protected boolean canProceed(Catalog sourceCatalog) {
+ consoleLog.warn(
+ "{}"
+ + "\ta) Executing catalog migration when the source catalog has some in-progress commits "
+ + "{}\tcan lead to a data loss as the in-progress commits will not be considered for migration. "
+ + "{}\tSo, while using this tool please make sure there are no in-progress commits for the source "
+ + "catalog.{}"
+ + "{}"
+ + "\tb) After the migration, successfully migrated tables will be deleted from the source catalog "
+ + "{}\tand can only be accessed from the target catalog.",
+ newLine,
+ newLine,
+ newLine,
+ newLine,
+ newLine,
+ newLine);
+ return proceed();
+ }
+
+ @Override
+ protected String operation() {
+ return "migration";
+ }
+
+ @Override
+ protected String operated() {
+ return "migrated";
+ }
+
+ @Override
+ protected String operate() {
+ return "migrate";
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/RegisterCommand.java b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/RegisterCommand.java
new file mode 100644
index 00000000..c482217a
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/RegisterCommand.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrator;
+import org.apache.polaris.iceberg.catalog.migrator.api.ImmutableCatalogMigrator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+@CommandLine.Command(
+ name = "register",
+ mixinStandardHelpOptions = true,
+ versionProvider = CLIVersionProvider.class,
+ // As both source and target catalog has similar configurations,
+ // documentation is easy to read if the target and source property is one after another instead
+ // of sorted order.
+ sortOptions = false,
+ description =
+ "Bulk register the iceberg tables from source catalog to target catalog without data copy.")
+public class RegisterCommand extends BaseRegisterCommand {
+
+ private static final String newLine = System.lineSeparator();
+ private static final Logger consoleLog = LoggerFactory.getLogger("console-log");
+
+ @Override
+ protected CatalogMigrator catalogMigrator(
+ Catalog sourceCatalog, Catalog targetCatalog, boolean enableStackTrace) {
+ return ImmutableCatalogMigrator.builder()
+ .sourceCatalog(sourceCatalog)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(false)
+ .enableStacktrace(enableStackTrace)
+ .build();
+ }
+
+ @Override
+ protected boolean canProceed(Catalog sourceCatalog) {
+ consoleLog.warn(
+ "{}"
+ + "\ta) Executing catalog migration when the source catalog has some in-progress commits "
+ + "{}\tcan lead to a data loss as the in-progress commits will not be considered for migration. "
+ + "{}\tSo, while using this tool please make sure there are no in-progress commits for the source "
+ + "catalog.{}"
+ + "{}"
+ + "\tb) After the registration, successfully registered tables will be present in both source and target "
+ + "catalog. "
+ + "{}\tHaving the same metadata.json registered in more than one catalog can lead to missing updates, "
+ + "loss of data, and table corruption. "
+ + "{}\tUse `migrate` command to automatically delete the table from source catalog after "
+ + "migration.",
+ newLine,
+ newLine,
+ newLine,
+ newLine,
+ newLine,
+ newLine,
+ newLine);
+ return proceed();
+ }
+
+ @Override
+ protected String operation() {
+ return "registration";
+ }
+
+ @Override
+ protected String operated() {
+ return "registered";
+ }
+
+ @Override
+ protected String operate() {
+ return "register";
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/SourceCatalogOptions.java b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/SourceCatalogOptions.java
new file mode 100644
index 00000000..31e2c248
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/SourceCatalogOptions.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import picocli.CommandLine;
+
+public class SourceCatalogOptions {
+
+ @CommandLine.Option(
+ names = "--source-catalog-type",
+ required = true,
+ description = {
+ "Source catalog type. Can be one of these [CUSTOM, DYNAMODB, ECS, GLUE, HADOOP, HIVE, JDBC, "
+ + "NESSIE, REST].",
+ "Example: --source-catalog-type GLUE",
+ " --source-catalog-type NESSIE"
+ })
+ protected CatalogMigrationUtil.CatalogType type;
+
+ @CommandLine.Option(
+ names = "--source-catalog-properties",
+ required = true,
+ split = ",",
+ description = {
+ "Iceberg catalog properties for source catalog (like uri, warehouse, etc).",
+ "Example: --source-catalog-properties uri=http://localhost:19120/api/v1,ref=main,warehouse=/tmp/warehouseNessie"
+ })
+ private Map properties;
+
+ @CommandLine.Option(
+ names = "--source-catalog-hadoop-conf",
+ split = ",",
+ description = {
+ "Optional source catalog Hadoop configurations required by the Iceberg catalog.",
+ "Example: --source-catalog-hadoop-conf key1=value1,key2=value2"
+ })
+ private final Map hadoopConf = new HashMap<>();
+
+ @CommandLine.Option(
+ names = {"--source-custom-catalog-impl"},
+ description = {
+ "Optional fully qualified class name of the custom catalog implementation of the source catalog. Required "
+ + "when the catalog type is CUSTOM.",
+ "Example: --source-custom-catalog-impl org.apache.iceberg.AwesomeCatalog"
+ })
+ private String customCatalogImpl;
+
+ Catalog build() {
+ return CatalogMigrationUtil.buildCatalog(
+ properties, type, "SOURCE_CATALOG_" + type.name(), customCatalogImpl, hadoopConf);
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/TargetCatalogOptions.java b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/TargetCatalogOptions.java
new file mode 100644
index 00000000..1ea9bc9d
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/java/org/apache/polaris/iceberg/catalog/migrator/cli/TargetCatalogOptions.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import picocli.CommandLine;
+
+public class TargetCatalogOptions {
+
+ @CommandLine.Option(
+ names = "--target-catalog-type",
+ required = true,
+ description = {
+ "Target catalog type. Can be one of these [CUSTOM, DYNAMODB, ECS, GLUE, HADOOP, HIVE, JDBC, "
+ + "NESSIE, REST].",
+ "Example: --target-catalog-type GLUE",
+ " --target-catalog-type NESSIE"
+ })
+ protected CatalogMigrationUtil.CatalogType type;
+
+ @CommandLine.Option(
+ names = "--target-catalog-properties",
+ required = true,
+ split = ",",
+ description = {
+ "Iceberg catalog properties for target catalog (like uri, warehouse, etc).",
+ "Example: --target-catalog-properties uri=http://localhost:19120/api/v1,ref=main,warehouse=/tmp/warehouseNessie"
+ })
+ private Map properties;
+
+ @CommandLine.Option(
+ names = "--target-catalog-hadoop-conf",
+ split = ",",
+ description = {
+ "Optional target catalog Hadoop configurations required by the Iceberg catalog.",
+ "Example: --target-catalog-hadoop-conf key1=value1,key2=value2"
+ })
+ private final Map hadoopConf = new HashMap<>();
+
+ @CommandLine.Option(
+ names = {"--target-custom-catalog-impl"},
+ description = {
+ "Optional fully qualified class name of the custom catalog implementation of the target catalog. Required "
+ + "when the catalog type is CUSTOM.",
+ "Example: --target-custom-catalog-impl org.apache.iceberg.AwesomeCatalog"
+ })
+ private String customCatalogImpl;
+
+ Catalog build() {
+ return CatalogMigrationUtil.buildCatalog(
+ properties, type, "TARGET_CATALOG_" + type.name(), customCatalogImpl, hadoopConf);
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/main/resources/logback.xml b/iceberg-catalog-migrator/cli/src/main/resources/logback.xml
new file mode 100644
index 00000000..b257baf3
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/resources/logback.xml
@@ -0,0 +1,44 @@
+
+
+
+
+ ${catalog.migration.log.dir}/catalog_migration.log
+ true
+
+ %date{ISO8601} [%thread] %-5level %logger{36} - %msg%n
+
+
+
+
+ true
+
+ %highlight(%-5level) - %msg%n
+
+
+
+
+
+
+
+
diff --git a/iceberg-catalog-migrator/cli/src/main/resources/org/apache/polaris/iceberg/catalog/migrator/cli/version.properties b/iceberg-catalog-migrator/cli/src/main/resources/org/apache/polaris/iceberg/catalog/migrator/cli/version.properties
new file mode 100644
index 00000000..45434cbe
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/main/resources/org/apache/polaris/iceberg/catalog/migrator/cli/version.properties
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+cli.version=@projectVersion@
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/AbstractCLIMigrationTest.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/AbstractCLIMigrationTest.java
new file mode 100644
index 00000000..57f85ac7
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/AbstractCLIMigrationTest.java
@@ -0,0 +1,496 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import static org.apache.polaris.iceberg.catalog.migrator.cli.BaseRegisterCommand.DRY_RUN_FILE;
+import static org.apache.polaris.iceberg.catalog.migrator.cli.BaseRegisterCommand.FAILED_IDENTIFIERS_FILE;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import nl.altindag.log.LogCaptor;
+import nl.altindag.log.model.LogEvent;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrator;
+import org.apache.polaris.iceberg.catalog.migrator.api.test.AbstractTest;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public abstract class AbstractCLIMigrationTest extends AbstractTest {
+
+ protected static @TempDir Path outputDir;
+
+ protected static String sourceCatalogProperties;
+ protected static String targetCatalogProperties;
+
+ protected static String sourceCatalogType;
+ protected static String targetCatalogType;
+
+ protected static void initializeSourceCatalog(
+ CatalogMigrationUtil.CatalogType catalogType, Map additionalProp) {
+ initializeCatalog(true, catalogType, additionalProp);
+ createNamespacesForSourceCatalog();
+ }
+
+ protected static void initializeTargetCatalog(
+ CatalogMigrationUtil.CatalogType catalogType, Map additionalProp) {
+ initializeCatalog(false, catalogType, additionalProp);
+ createNamespacesForTargetCatalog();
+ }
+
+ private static void initializeCatalog(
+ boolean isSourceCatalog,
+ CatalogMigrationUtil.CatalogType catalogType,
+ Map additionalProp) {
+ Map properties;
+ switch (catalogType) {
+ case HADOOP:
+ properties = hadoopCatalogProperties(isSourceCatalog);
+ break;
+ case NESSIE:
+ properties = nessieCatalogProperties(isSourceCatalog);
+ break;
+ case HIVE:
+ properties = hiveCatalogProperties(isSourceCatalog, additionalProp);
+ break;
+ default:
+ throw new UnsupportedOperationException(
+ String.format("Unsupported for catalog type: %s", catalogType));
+ }
+ Catalog catalog =
+ CatalogMigrationUtil.buildCatalog(
+ properties,
+ catalogType,
+ isSourceCatalog ? "sourceCatalog" : "targetCatalog" + "_" + catalogType,
+ null,
+ null);
+ String propertiesStr = Joiner.on(",").withKeyValueSeparator("=").join(properties);
+ if (isSourceCatalog) {
+ sourceCatalog = catalog;
+ sourceCatalogProperties = propertiesStr;
+ sourceCatalogType = catalogType.name();
+ } else {
+ targetCatalog = catalog;
+ targetCatalogProperties = propertiesStr;
+ targetCatalogType = catalogType.name();
+ }
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+
+ @BeforeEach
+ protected void beforeEach() {
+ createTables();
+ }
+
+ @AfterEach
+ protected void afterEach() {
+ // manually refreshing catalog due to missing refresh in Nessie catalog
+ // https://github.com/apache/iceberg/pull/6789
+ // create table will call refresh internally.
+ sourceCatalog.createTable(TableIdentifier.of(BAR, "tblx"), schema).refresh();
+ targetCatalog.createTable(TableIdentifier.of(BAR, "tblx"), schema).refresh();
+
+ dropTables();
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegister(boolean deleteSourceTables) throws Exception {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ String operation = deleteSourceTables ? "migration" : "registration";
+ String operated = deleteSourceTables ? "migrated" : "registered";
+
+ // register or migrate all the tables
+ RunCLI run = runCLI(deleteSourceTables, defaultArgs());
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(0);
+ Assertions.assertThat(run.getOut())
+ .contains(
+ "User has not specified the table identifiers. "
+ + "Will be selecting all the tables from all the namespaces from the source catalog.")
+ .contains(String.format("Identified 4 tables for %s.", operation))
+ .contains(
+ String.format(
+ "Summary: %nSuccessfully %s 4 tables from %s catalog to %s catalog.",
+ operated, sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nSuccessfully %s these tables:%n", operated));
+
+ // manually refreshing catalog due to missing refresh in Nessie catalog
+ // https://github.com/apache/iceberg/pull/6789
+ targetCatalog.loadTable(FOO_TBL1).refresh();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO))
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(targetCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+
+ // manually refreshing catalog due to missing refresh in Nessie catalog
+ // https://github.com/apache/iceberg/pull/6789
+ sourceCatalog.tableExists(FOO_TBL1);
+
+ if (deleteSourceTables) {
+ // table should be deleted after migration from source catalog
+ Assertions.assertThat(sourceCatalog.listTables(FOO)).isEmpty();
+ Assertions.assertThat(sourceCatalog.listTables(BAR)).isEmpty();
+ } else {
+ // tables should be present in source catalog.
+ Assertions.assertThat(sourceCatalog.listTables(FOO))
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(sourceCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterSelectedTables(boolean deleteSourceTables) throws Exception {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ String operation = deleteSourceTables ? "migration" : "registration";
+ String operated = deleteSourceTables ? "migrated" : "registered";
+
+ // using `--identifiers` option
+ List argsList = defaultArgs();
+ argsList.addAll(Arrays.asList("--identifiers", "bar.tbl3"));
+ RunCLI run = runCLI(deleteSourceTables, argsList);
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(0);
+ Assertions.assertThat(run.getOut())
+ .doesNotContain(
+ "User has not specified the table identifiers. "
+ + "Selecting all the tables from all the namespaces from the source catalog.")
+ .contains(String.format("Identified 1 tables for %s.", operation))
+ .contains(
+ String.format(
+ "Summary: %nSuccessfully %s 1 tables from %s catalog to %s catalog.",
+ operated, sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nSuccessfully %s these tables:%n[bar.tbl3]", operated));
+
+ // manually refreshing catalog due to missing refresh in Nessie catalog
+ // https://github.com/apache/iceberg/pull/6789
+ targetCatalog.loadTable(BAR_TBL3).refresh();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO)).isEmpty();
+ Assertions.assertThat(targetCatalog.listTables(BAR)).containsExactly(BAR_TBL3);
+
+ Path identifierFile = outputDir.resolve("ids.txt");
+
+ // using `--identifiers-from-file` option
+ Files.write(identifierFile, Collections.singletonList("bar.tbl4"));
+ argsList = defaultArgs();
+ argsList.addAll(
+ Arrays.asList("--identifiers-from-file", identifierFile.toAbsolutePath().toString()));
+ run = runCLI(deleteSourceTables, argsList);
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(0);
+ Assertions.assertThat(run.getOut())
+ .doesNotContain(
+ "User has not specified the table identifiers. "
+ + "Selecting all the tables from all the namespaces from the source catalog.")
+ .contains(String.format("Identified 1 tables for %s.", operation))
+ .contains(
+ String.format(
+ "Summary: %nSuccessfully %s 1 tables from %s catalog to %s catalog.",
+ operated, sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nSuccessfully %s these tables:%n", operated));
+
+ // manually refreshing catalog due to missing refresh in Nessie catalog
+ // https://github.com/apache/iceberg/pull/6789
+ targetCatalog.loadTable(BAR_TBL3).refresh();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO)).isEmpty();
+ Assertions.assertThat(targetCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL4, BAR_TBL3);
+ Files.delete(identifierFile);
+
+ // using `--identifiers-regex` option which matches all the tables starts with "foo."
+ argsList = defaultArgs();
+ argsList.addAll(Arrays.asList("--identifiers-regex", "^foo\\..*"));
+ run = runCLI(deleteSourceTables, argsList);
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(0);
+ Assertions.assertThat(run.getOut())
+ .contains(
+ "User has not specified the table identifiers. Will be selecting all the tables from all the namespaces "
+ + "from the source catalog which matches the regex pattern:^foo\\..*")
+ .contains(String.format("Identified 2 tables for %s.", operation))
+ .contains(
+ String.format(
+ "Summary: %nSuccessfully %s 2 tables from %s catalog to %s catalog.",
+ operated, sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nSuccessfully %s these tables:%n", operated));
+
+ // manually refreshing catalog due to missing refresh in Nessie catalog
+ // https://github.com/apache/iceberg/pull/6789
+ targetCatalog.loadTable(BAR_TBL3).refresh();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO))
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(targetCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterError(boolean deleteSourceTables) throws Exception {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ String operation = deleteSourceTables ? "migration" : "registration";
+ String operate = deleteSourceTables ? "migrate" : "register";
+
+ // use invalid namespace which leads to NoSuchTableException
+ List argsList = defaultArgs();
+ argsList.addAll(Arrays.asList("--identifiers", "dummy.tbl3"));
+ RunCLI run = runCLI(deleteSourceTables, argsList);
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(1);
+ Assertions.assertThat(run.getOut())
+ .contains(String.format("Identified 1 tables for %s.", operation))
+ .contains(
+ String.format(
+ "Summary: %nFailed to %s 1 tables from %s catalog to %s catalog."
+ + " Please check the `catalog_migration.log`",
+ operate, sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nFailed to %s these tables:%n[dummy.tbl3]", operate));
+
+ // try to register same table twice which leads to AlreadyExistsException
+ argsList = defaultArgs();
+ argsList.addAll(Arrays.asList("--identifiers", "foo.tbl2"));
+ runCLI(deleteSourceTables, argsList);
+ run = RunCLI.run(argsList.toArray(new String[0]));
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(1);
+ Assertions.assertThat(run.getOut())
+ .contains(String.format("Identified 1 tables for %s.", operation))
+ .contains(
+ String.format(
+ "Summary: %nFailed to %s 1 tables from %s catalog to %s catalog."
+ + " Please check the `catalog_migration.log`",
+ operate, sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nFailed to %s these tables:%n[foo.tbl2]", operate));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterWithFewFailures(boolean deleteSourceTables) throws Exception {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ String operation = deleteSourceTables ? "migration" : "registration";
+ String operated = deleteSourceTables ? "migrated" : "registered";
+ String operate = deleteSourceTables ? "migrate" : "register";
+
+ // register only foo.tbl2
+ List argsList = defaultArgs();
+ argsList.addAll(Arrays.asList("--identifiers", "foo.tbl2"));
+ RunCLI run = runCLI(deleteSourceTables, argsList);
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(0);
+ Assertions.assertThat(run.getOut())
+ .contains(String.format("Identified 1 tables for %s.", operation))
+ .contains(
+ String.format(
+ "Summary: %nSuccessfully %s 1 tables from %s catalog to %s catalog.",
+ operated, sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nSuccessfully %s these tables:%n[foo.tbl2]", operated));
+
+ if (deleteSourceTables) {
+ // create a table with the same name in source catalog which got deleted.
+ sourceCatalog.createTable(FOO_TBL2, schema);
+ }
+
+ // register all the tables from source catalog again. So that registering `foo.tbl2` will fail.
+ run = runCLI(deleteSourceTables, defaultArgs());
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(1);
+ Assertions.assertThat(run.getOut())
+ .contains(String.format("Identified 4 tables for %s.", operation))
+ .contains(
+ String.format(
+ "Summary: %n"
+ + "Successfully %s 3 tables from %s catalog to %s catalog.%n"
+ + "Failed to %s 1 tables from %s catalog to %s catalog. "
+ + "Please check the `catalog_migration.log` file for the failure reason. "
+ + "Failed identifiers are written into `failed_identifiers.txt`. "
+ + "Retry with that file using `--identifiers-from-file` option "
+ + "if the failure is because of network/connection timeouts.",
+ operated,
+ sourceCatalogType,
+ targetCatalogType,
+ operate,
+ sourceCatalogType,
+ targetCatalogType))
+ .contains(String.format("Details: %nSuccessfully %s these tables:%n", operated))
+ .contains(String.format("Failed to %s these tables:%n[foo.tbl2]", operate));
+
+ // manually refreshing catalog due to missing refresh in Nessie catalog
+ // https://github.com/apache/iceberg/pull/6789
+ targetCatalog.loadTable(BAR_TBL3).refresh();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO))
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(targetCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+
+ Path failedIdentifiersFile = outputDir.resolve(FAILED_IDENTIFIERS_FILE);
+
+ // retry the failed tables using `--identifiers-from-file`
+ argsList = defaultArgs();
+ argsList.addAll(
+ Arrays.asList(
+ "--identifiers-from-file", failedIdentifiersFile.toAbsolutePath().toString()));
+ run = runCLI(deleteSourceTables, argsList);
+
+ Assertions.assertThat(run.getOut())
+ .contains(
+ String.format(
+ "Summary: %n"
+ + "Failed to %s 1 tables from %s catalog to %s catalog. "
+ + "Please check the `catalog_migration.log` file for the failure reason. "
+ + "Failed identifiers are written into `failed_identifiers.txt`. "
+ + "Retry with that file using `--identifiers-from-file` option "
+ + "if the failure is because of network/connection timeouts.",
+ operate, sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nFailed to %s these tables:%n[foo.tbl2]", operate));
+ Assertions.assertThat(failedIdentifiersFile).exists();
+ Assertions.assertThat(Files.readAllLines(failedIdentifiersFile)).containsExactly("foo.tbl2");
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterNoTables(boolean deleteSourceTables) throws Exception {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ // clean up the default tables present in the source catalog.
+ dropTables();
+
+ RunCLI run = runCLI(deleteSourceTables, defaultArgs());
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(1);
+ String operation = deleteSourceTables ? "migration" : "registration";
+ Assertions.assertThat(run.getOut())
+ .contains(
+ String.format(
+ "No tables were identified for %s. Please check `catalog_migration.log` file for more info.",
+ operation));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testDryRun(boolean deleteSourceTables) throws Exception {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ List argsList = defaultArgs();
+ argsList.add("--dry-run");
+ RunCLI run = runCLI(deleteSourceTables, argsList);
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(0);
+ String operation = deleteSourceTables ? "migration" : "registration";
+ // should not prompt for dry run
+ Assertions.assertThat(run.getOut())
+ .doesNotContain(
+ "Are you certain that you wish to proceed, after reading the above warnings? (yes/no):")
+ .contains("Dry run is completed.")
+ .contains(
+ String.format(
+ "Summary: %n"
+ + "Identified 4 tables for %s by dry-run. "
+ + "These identifiers are also written into dry_run_identifiers.txt. "
+ + "This file can be used with `--identifiers-from-file` option for an actual run.",
+ operation))
+ .contains(
+ String.format("Details: %nIdentified these tables for %s by dry-run:%n", operation));
+ Path dryRunFile = outputDir.resolve(DRY_RUN_FILE);
+ Assertions.assertThat(dryRunFile).exists();
+ Assertions.assertThat(Files.readAllLines(dryRunFile))
+ .containsExactlyInAnyOrder("foo.tbl1", "foo.tbl2", "bar.tbl3", "bar.tbl4");
+ }
+
+ @ParameterizedTest
+ @CsvSource(value = {"false,false", "false,true", "true,false", "true,true"})
+ public void testStacktrace(boolean deleteSourceTables, boolean enableStacktrace)
+ throws Exception {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+ try (LogCaptor logCaptor = LogCaptor.forClass(CatalogMigrator.class)) {
+ List argsList = defaultArgs();
+ argsList.addAll(
+ Arrays.asList("--identifiers", "db.dummy_table", "--stacktrace=" + enableStacktrace));
+ runCLI(deleteSourceTables, argsList);
+
+ Assertions.assertThat(logCaptor.getLogEvents()).hasSize(1);
+ LogEvent logEvent = logCaptor.getLogEvents().get(0);
+ if (enableStacktrace) {
+ Assertions.assertThat(logEvent.getFormattedMessage())
+ .isEqualTo("Unable to register the table db.dummy_table");
+ Assertions.assertThat(logEvent.getThrowable())
+ .isPresent()
+ .get()
+ .isInstanceOf(NoSuchTableException.class);
+ } else {
+ Assertions.assertThat(logEvent.getFormattedMessage())
+ .isEqualTo(
+ "Unable to register the table db.dummy_table : Table does not exist: db.dummy_table");
+ Assertions.assertThat(logEvent.getThrowable()).isEmpty();
+ }
+ }
+ }
+
+ protected static List defaultArgs() {
+ return Lists.newArrayList(
+ "--source-catalog-type",
+ sourceCatalogType,
+ "--source-catalog-properties",
+ sourceCatalogProperties,
+ "--target-catalog-type",
+ targetCatalogType,
+ "--target-catalog-properties",
+ targetCatalogProperties,
+ "--output-dir",
+ outputDir.toAbsolutePath().toString(),
+ "--disable-safety-prompts");
+ }
+
+ protected static RunCLI runCLI(boolean deleteSourceTables, List argsList)
+ throws Exception {
+ if (!deleteSourceTables) {
+ argsList.add(0, "register");
+ } else {
+ argsList.add(0, "migrate");
+ }
+ return RunCLI.run(argsList.toArray(new String[0]));
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/CLIOptionsTest.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/CLIOptionsTest.java
new file mode 100644
index 00000000..0f63fbbc
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/CLIOptionsTest.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import static org.junit.jupiter.params.provider.Arguments.arguments;
+
+import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.UUID;
+import java.util.stream.Stream;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+public class CLIOptionsTest {
+
+ protected static @TempDir Path logDir;
+
+ @BeforeAll
+ protected static void initLogDir() {
+ System.setProperty("catalog.migration.log.dir", logDir.toAbsolutePath().toString());
+ }
+
+ private static Stream optionErrors() {
+ return Stream.of(
+ // no arguments
+ arguments(
+ Lists.newArrayList(),
+ "Error: Missing required argument(s): (--target-catalog-type= --target-catalog-properties=[,...] [--target-catalog-properties=[,...]]... [--target-catalog-hadoop-conf=[,...]]... [--target-custom-catalog-impl=])"),
+ // missing required arguments
+ arguments(Lists.newArrayList(""), "Unmatched argument at index 1: ''"),
+ // missing required arguments
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-properties", "properties1=ab", "--target-catalog-type", "NESSIE"),
+ "Error: Missing required argument(s): --source-catalog-type="),
+ // missing required arguments
+ arguments(
+ Lists.newArrayList("--source-catalog-type", "GLUE"),
+ "Error: Missing required argument(s): --source-catalog-properties="),
+ // missing required arguments
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HIVE",
+ "--source-catalog-properties",
+ "properties1=ab",
+ "--target-catalog-type",
+ "NESSIE"),
+ "Error: Missing required argument(s): --target-catalog-properties="),
+ // missing required arguments
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HIVE",
+ "--source-catalog-properties",
+ "properties1=ab",
+ "--target-catalog-properties",
+ "properties2=cd"),
+ "Error: Missing required argument(s): --target-catalog-type="),
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HADOOP",
+ "--source-catalog-properties",
+ "k1=v1,k2=v2",
+ "--target-catalog-type",
+ "HIVE",
+ "--target-catalog-properties",
+ "k3=v3, k4=v4",
+ "--identifiers",
+ "foo.tbl",
+ "--identifiers-from-file",
+ "file.txt",
+ "--identifiers-regex",
+ "^foo\\."),
+ "Error: --identifiers=, --identifiers-from-file=, --identifiers-regex= are mutually exclusive (specify only one)"),
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HADOOP",
+ "--source-catalog-properties",
+ "k1=v1,k2=v2",
+ "--target-catalog-type",
+ "HIVE",
+ "--target-catalog-properties",
+ "k3=v3, k4=v4",
+ "--identifiers",
+ "foo.tbl",
+ "--identifiers-from-file",
+ "file.txt"),
+ "Error: --identifiers=, --identifiers-from-file= are mutually exclusive (specify only one)"),
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HADOOP",
+ "--source-catalog-properties",
+ "k1=v1,k2=v2",
+ "--target-catalog-type",
+ "HIVE",
+ "--target-catalog-properties",
+ "k3=v3, k4=v4",
+ "--identifiers-regex",
+ "^foo\\.",
+ "--identifiers-from-file",
+ "file.txt"),
+ "Error: --identifiers-from-file=, --identifiers-regex= are mutually exclusive (specify only one)"),
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HADOOP",
+ "--source-catalog-properties",
+ "k1=v1,k2=v2",
+ "--target-catalog-type",
+ "HIVE",
+ "--target-catalog-properties",
+ "k3=v3, k4=v4",
+ "--identifiers",
+ "foo.tbl",
+ "--identifiers-regex",
+ "^foo\\."),
+ "Error: --identifiers=, --identifiers-regex= are mutually exclusive "
+ + "(specify only one)"));
+ }
+
+ @ParameterizedTest
+ @MethodSource("optionErrors")
+ public void testOptionErrorsForRegister(List args, String expectedMessage)
+ throws Exception {
+ executeAndValidateResults("register", args, expectedMessage, 2);
+ }
+
+ @ParameterizedTest
+ @MethodSource("optionErrors")
+ public void testOptionErrorsForMigrate(List args, String expectedMessage)
+ throws Exception {
+ executeAndValidateResults("migrate", args, expectedMessage, 2);
+ }
+
+ private static Stream invalidArgs() {
+ return Stream.of(
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HIVE",
+ "--source-catalog-properties",
+ "k1=v1,k2=v2",
+ "--target-catalog-type",
+ "HADOOP",
+ "--target-catalog-properties",
+ "k3=v3, k4=v4"),
+ "Error during CLI execution: Cannot initialize HadoopCatalog "
+ + "because warehousePath must not be null or empty"),
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HIVE",
+ "--source-catalog-properties",
+ "k1=v1,k2=v2",
+ "--target-catalog-type",
+ "HADOOP",
+ "--target-catalog-properties",
+ "k3=v3, k4=v4",
+ "--identifiers-from-file",
+ "file.txt"),
+ "Error during CLI execution: File specified in `--identifiers-from-file` option does not exist"),
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HIVE",
+ "--source-catalog-properties",
+ "k1=v1,k2=v2",
+ "--target-catalog-type",
+ "HADOOP",
+ "--target-catalog-properties",
+ "k3=v3, k4=v4",
+ "--output-dir",
+ "/path/to/file"),
+ "Error during CLI execution: Failed to create the output directory from the path specified in `--output-dir`"),
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HIVE",
+ "--source-catalog-properties",
+ "k1=v1,k2=v2",
+ "--target-catalog-type",
+ "HADOOP",
+ "--target-catalog-properties",
+ "k3=v3, k4=v4",
+ "--output-dir",
+ readOnlyDirLocation()),
+ "Error during CLI execution: Path specified in `--output-dir` is not writable"),
+ // test with stacktrace
+ arguments(
+ Lists.newArrayList(
+ "--source-catalog-type",
+ "HIVE",
+ "--source-catalog-properties",
+ "k1=v1,k2=v2",
+ "--target-catalog-type",
+ "HADOOP",
+ "--target-catalog-properties",
+ "k3=v3, k4=v4",
+ "--output-dir",
+ readOnlyDirLocation(),
+ "--stacktrace"),
+ "java.lang.IllegalArgumentException: Path specified in `--output-dir` is not writable"));
+ }
+
+ @ParameterizedTest
+ @MethodSource("invalidArgs")
+ public void testInvalidArgsForRegister(List args, String expectedMessage)
+ throws Exception {
+ executeAndValidateResults("register", args, expectedMessage, 1);
+ }
+
+ @ParameterizedTest
+ @MethodSource("invalidArgs")
+ public void testInvalidArgsForMigrate(List args, String expectedMessage)
+ throws Exception {
+ executeAndValidateResults("migrate", args, expectedMessage, 1);
+ }
+
+ @Test
+ public void version() throws Exception {
+ RunCLI run = RunCLI.runWithPrintWriter("--version");
+ Assertions.assertThat(run.getExitCode()).isEqualTo(0);
+ Assertions.assertThat(run.getOut()).startsWith(System.getProperty("expectedCLIVersion"));
+ }
+
+ private static void executeAndValidateResults(
+ String command, List args, String expectedMessage, int expectedErrorCode)
+ throws Exception {
+ args.add(0, command);
+ RunCLI run = RunCLI.run(args);
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(expectedErrorCode);
+ Assertions.assertThat(run.getErr()).contains(expectedMessage);
+ }
+
+ private static String readOnlyDirLocation() {
+ Path readOnly = logDir.resolve(UUID.randomUUID().toString());
+ try {
+ Files.createDirectory(readOnly);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ Assertions.assertThat(readOnly.toFile().setWritable(false)).isTrue();
+
+ return readOnly.toAbsolutePath().toString();
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/HadoopCLIMigrationTest.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/HadoopCLIMigrationTest.java
new file mode 100644
index 00000000..0e6d6c42
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/HadoopCLIMigrationTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.util.Collections;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.junit.jupiter.api.BeforeAll;
+
+public class HadoopCLIMigrationTest extends AbstractCLIMigrationTest {
+
+ @BeforeAll
+ protected static void setup() {
+ initializeSourceCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ initializeTargetCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToHiveCLIMigrationTest.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToHiveCLIMigrationTest.java
new file mode 100644
index 00000000..a7f4ee5c
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToHiveCLIMigrationTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.util.Collections;
+import org.apache.iceberg.hive.HiveMetastoreExtension;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+public class ITHadoopToHiveCLIMigrationTest extends AbstractCLIMigrationTest {
+
+ @RegisterExtension
+ public static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().build();
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+
+ initializeSourceCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ initializeTargetCatalog(
+ CatalogMigrationUtil.CatalogType.HIVE,
+ Collections.singletonMap(
+ "uri", HIVE_METASTORE_EXTENSION.hiveConf().get("hive.metastore.uris")));
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToNessieCLIMigrationTest.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToNessieCLIMigrationTest.java
new file mode 100644
index 00000000..938df448
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToNessieCLIMigrationTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.IntStream;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+public class ITHadoopToNessieCLIMigrationTest extends AbstractCLIMigrationTest {
+
+ @BeforeAll
+ protected static void setup() {
+ initializeSourceCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ initializeTargetCatalog(CatalogMigrationUtil.CatalogType.NESSIE, Collections.emptyMap());
+ }
+
+ @Test
+ public void testRegisterLargeNumberOfTablesWithNestedNamespaces() throws Exception {
+ List namespaceList =
+ Arrays.asList(NS_A, NS_A_B, NS_A_B_C, NS_A_B_C_D, NS_A_B_C_D_E, NS_A_C);
+
+ // additionally create 240 tables along with 4 tables created in beforeEach()
+ namespaceList.forEach(
+ namespace -> {
+ ((SupportsNamespaces) sourceCatalog).createNamespace(namespace);
+ IntStream.range(0, 40)
+ .forEach(
+ val ->
+ sourceCatalog.createTable(
+ TableIdentifier.of(namespace, "tblx" + val), schema));
+ });
+
+ // register or migrate all the tables
+ RunCLI run = runCLI(false, defaultArgs());
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(0);
+ Assertions.assertThat(run.getOut())
+ .contains("Identified 244 tables for registration.")
+ .contains(
+ String.format(
+ "Summary: %nSuccessfully registered 244 tables from %s catalog to" + " %s catalog.",
+ sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nSuccessfully registered these tables:%n"))
+ // validate intermediate output
+ .contains("Attempted registration for 100 tables out of 244 tables.")
+ .contains("Attempted registration for 200 tables out of 244 tables.")
+ .contains("Attempted registration for 244 tables out of 244 tables.");
+
+ // manually refreshing catalog due to missing refresh in Nessie catalog
+ // https://github.com/apache/iceberg/pull/6789
+ targetCatalog.loadTable(BAR_TBL3).refresh();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO))
+ .containsExactlyInAnyOrder(FOO_TBL1, FOO_TBL2);
+ Assertions.assertThat(targetCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+
+ Collections.reverse(namespaceList);
+ namespaceList.forEach(
+ namespace -> {
+ List identifiers = targetCatalog.listTables(namespace);
+
+ // validate tables count in each namespace.
+ Assertions.assertThat(identifiers).hasSize(40);
+
+ identifiers.forEach(
+ identifier -> {
+ targetCatalog.dropTable(identifier);
+ sourceCatalog.dropTable(identifier);
+ });
+ ((SupportsNamespaces) sourceCatalog).dropNamespace(namespace);
+ ((SupportsNamespaces) targetCatalog).dropNamespace(namespace);
+ });
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHiveToHadoopCLIMigrationTest.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHiveToHadoopCLIMigrationTest.java
new file mode 100644
index 00000000..7a8a4197
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHiveToHadoopCLIMigrationTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.util.Collections;
+import org.apache.iceberg.hive.HiveMetastoreExtension;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+public class ITHiveToHadoopCLIMigrationTest extends AbstractCLIMigrationTest {
+
+ @RegisterExtension
+ public static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().build();
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+ initializeSourceCatalog(
+ CatalogMigrationUtil.CatalogType.HIVE,
+ Collections.singletonMap(
+ "uri", HIVE_METASTORE_EXTENSION.hiveConf().get("hive.metastore.uris")));
+ initializeTargetCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHiveToNessieCLIMigrationTest.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHiveToNessieCLIMigrationTest.java
new file mode 100644
index 00000000..09876329
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHiveToNessieCLIMigrationTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.util.Collections;
+import java.util.stream.IntStream;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.hive.HiveMetastoreExtension;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class ITHiveToNessieCLIMigrationTest extends AbstractCLIMigrationTest {
+
+ @RegisterExtension
+ public static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().build();
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+ initializeSourceCatalog(
+ CatalogMigrationUtil.CatalogType.HIVE,
+ Collections.singletonMap(
+ "uri", HIVE_METASTORE_EXTENSION.hiveConf().get("hive.metastore.uris")));
+ initializeTargetCatalog(CatalogMigrationUtil.CatalogType.NESSIE, Collections.emptyMap());
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterLargeNumberOfTables(boolean deleteSourceTables) throws Exception {
+ validateAssumptionForHadoopCatalogAsSource(deleteSourceTables);
+
+ String operation = deleteSourceTables ? "migration" : "registration";
+ String operated = deleteSourceTables ? "migrated" : "registered";
+
+ // additionally create 240 tables along with 4 tables created in beforeEach()
+ IntStream.range(0, 240)
+ .forEach(val -> sourceCatalog.createTable(TableIdentifier.of(FOO, "tblx" + val), schema));
+
+ // register or migrate all the tables
+ RunCLI run = runCLI(deleteSourceTables, defaultArgs());
+
+ Assertions.assertThat(run.getExitCode()).isEqualTo(0);
+ Assertions.assertThat(run.getOut())
+ .contains(String.format("Identified 244 tables for %s.", operation))
+ .contains(
+ String.format(
+ "Summary: %nSuccessfully %s 244 tables from %s catalog to" + " %s catalog.",
+ operated, sourceCatalogType, targetCatalogType))
+ .contains(String.format("Details: %nSuccessfully %s these tables:%n", operated))
+ // validate intermediate output
+ .contains(String.format("Attempted %s for 100 tables out of 244 tables.", operation))
+ .contains(String.format("Attempted %s for 200 tables out of 244 tables.", operation))
+ .contains(String.format("Attempted %s for 244 tables out of 244 tables.", operation));
+
+ // manually refreshing catalog due to missing refresh in Nessie catalog
+ // https://github.com/apache/iceberg/pull/6789
+ targetCatalog.loadTable(BAR_TBL3).refresh();
+
+ Assertions.assertThat(targetCatalog.listTables(FOO)).hasSize(242);
+ Assertions.assertThat(targetCatalog.listTables(BAR))
+ .containsExactlyInAnyOrder(BAR_TBL3, BAR_TBL4);
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITNessieToHiveCLIMigrationTest.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITNessieToHiveCLIMigrationTest.java
new file mode 100644
index 00000000..d151aed4
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITNessieToHiveCLIMigrationTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.util.Collections;
+import org.apache.iceberg.hive.HiveMetastoreExtension;
+import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+public class ITNessieToHiveCLIMigrationTest extends AbstractCLIMigrationTest {
+
+ @RegisterExtension
+ public static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
+ HiveMetastoreExtension.builder().build();
+
+ @BeforeAll
+ protected static void setup() throws Exception {
+ initializeSourceCatalog(CatalogMigrationUtil.CatalogType.NESSIE, Collections.emptyMap());
+ initializeTargetCatalog(
+ CatalogMigrationUtil.CatalogType.HIVE,
+ Collections.singletonMap(
+ "uri", HIVE_METASTORE_EXTENSION.hiveConf().get("hive.metastore.uris")));
+ }
+
+ @AfterAll
+ protected static void tearDown() throws Exception {
+ dropNamespaces();
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ProcessIdentifiersTest.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ProcessIdentifiersTest.java
new file mode 100644
index 00000000..482cbfd1
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ProcessIdentifiersTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import com.google.common.collect.Sets;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Set;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+public class ProcessIdentifiersTest {
+
+ protected static @TempDir Path tempDir;
+
+ @BeforeAll
+ protected static void initLogDir() {
+ System.setProperty("catalog.migration.log.dir", tempDir.toAbsolutePath().toString());
+ }
+
+ @Test
+ public void testIdentifiersSet() {
+ // test empty set
+ Assertions.assertThat(new IdentifierOptions().processIdentifiersInput()).isEmpty();
+
+ // test valid elements
+ IdentifierOptions identifierOptions = new IdentifierOptions();
+ identifierOptions.identifiers = Sets.newHashSet("foo.abc", "bar.def");
+ Assertions.assertThat(identifierOptions.processIdentifiersInput())
+ .containsExactlyInAnyOrder(
+ TableIdentifier.parse("foo.abc"), TableIdentifier.parse("bar.def"));
+ }
+
+ @Test
+ public void testIdentifiersFromFile() throws Exception {
+ // valid file contents
+ Path identifierFile = tempDir.resolve("file_with_ids.txt");
+ Files.write(identifierFile, Arrays.asList("db1.t1", "db2.t2", "db123.t5"));
+ IdentifierOptions options = new IdentifierOptions();
+ options.identifiersFromFile = identifierFile.toAbsolutePath().toString();
+ Assertions.assertThat(options.processIdentifiersInput())
+ .containsExactlyInAnyOrder(
+ TableIdentifier.parse("db1.t1"),
+ TableIdentifier.parse("db2.t2"),
+ TableIdentifier.parse("db123.t5"));
+
+ // empty file
+ identifierFile = tempDir.resolve("ids1.txt");
+ Files.createFile(identifierFile);
+ options = new IdentifierOptions();
+ options.identifiersFromFile = identifierFile.toAbsolutePath().toString();
+ Assertions.assertThat(options.processIdentifiersInput()).isEmpty();
+
+ // file with some blanks contents
+ identifierFile = tempDir.resolve("ids2.txt");
+ String[] lines = {"abc. def", " abc 123 ", "", "", " xyz%n123"};
+ Files.writeString(identifierFile, String.join(System.lineSeparator(), lines));
+ options = new IdentifierOptions();
+ options.identifiersFromFile = identifierFile.toAbsolutePath().toString();
+ Set identifiers = options.processIdentifiersInput();
+ Assertions.assertThat(identifiers)
+ .containsExactlyInAnyOrder(
+ TableIdentifier.parse("abc. def"),
+ TableIdentifier.parse("abc 123"),
+ TableIdentifier.parse("xyz%n123"));
+
+ // with duplicate entries
+ identifierFile = tempDir.resolve("ids3.txt");
+ String[] ids = {"abc.def", "xx.yy", "abc.def", "abc.def", "abc.def ", " xx.yy"};
+ Files.writeString(identifierFile, String.join(System.lineSeparator(), ids));
+ options = new IdentifierOptions();
+ options.identifiersFromFile = identifierFile.toAbsolutePath().toString();
+ identifiers = options.processIdentifiersInput();
+ Assertions.assertThat(identifiers)
+ .containsExactlyInAnyOrder(
+ TableIdentifier.parse("abc.def"), TableIdentifier.parse("xx.yy"));
+ }
+
+ @Test
+ public void testIdentifiersFromFileInvalidInputs() throws Exception {
+ // file without permission to read
+ Path identifierFile = tempDir.resolve("non_readable_file.txt");
+ Files.createFile(identifierFile);
+ Assertions.assertThat(identifierFile.toFile().setReadable(false)).isTrue();
+ IdentifierOptions options = new IdentifierOptions();
+ options.identifiersFromFile = identifierFile.toAbsolutePath().toString();
+ Assertions.assertThatThrownBy(options::processIdentifiersInput)
+ .isInstanceOf(UncheckedIOException.class)
+ .hasMessageContaining("Failed to read the file: " + identifierFile);
+ Assertions.assertThat(identifierFile.toFile().setReadable(true)).isTrue();
+
+ // file doesn't exist
+ options.identifiersFromFile = "path/to/file";
+ Assertions.assertThatThrownBy(options::processIdentifiersInput)
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("File specified in `--identifiers-from-file` option does not exist");
+ }
+
+ @Test
+ public void testIdentifiersRegEx() {
+ // test valid regex
+ IdentifierOptions options = new IdentifierOptions();
+ options.identifiersRegEx = "^foo\\..*";
+ Assertions.assertThat(options.processIdentifiersInput()).isEmpty();
+
+ // test invalid regex
+ options.identifiersRegEx = "(23erf423!";
+ Assertions.assertThatThrownBy(options::processIdentifiersInput)
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("--identifiers-regex pattern is not compilable");
+
+ options = new IdentifierOptions();
+ options.identifiersRegEx = " ";
+ Assertions.assertThatThrownBy(options::processIdentifiersInput)
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("--identifiers-regex should not be empty");
+ }
+}
diff --git a/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/RunCLI.java b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/RunCLI.java
new file mode 100644
index 00000000..0e17c605
--- /dev/null
+++ b/iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/RunCLI.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.iceberg.catalog.migrator.cli;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.Arrays;
+import java.util.List;
+import nl.altindag.log.LogCaptor;
+import picocli.CommandLine;
+
+/** Helper class for tests. */
+public final class RunCLI {
+
+ private final String[] args;
+ private final int exitCode;
+ private final String out;
+ private final String err;
+
+ public RunCLI(int exitCode, String out, String err, String[] args) {
+ this.args = args;
+ this.exitCode = exitCode;
+ this.out = out;
+ this.err = err;
+ }
+
+ public static RunCLI run(List args) throws Exception {
+ return run(args.toArray(new String[0]));
+ }
+
+ public static RunCLI run(String... args) throws Exception {
+ try (LogCaptor logCaptor = LogCaptor.forName("console-log");
+ StringWriter err = new StringWriter();
+ PrintWriter errWriter = new PrintWriter(err)) {
+ int exitCode = runMain(null, errWriter, args);
+ String out = String.join(System.lineSeparator(), logCaptor.getLogs());
+ return new RunCLI(exitCode, out, err.toString(), args);
+ }
+ }
+
+ public static RunCLI runWithPrintWriter(String... args) throws Exception {
+ try (StringWriter out = new StringWriter();
+ PrintWriter outWriter = new PrintWriter(out);
+ StringWriter err = new StringWriter();
+ PrintWriter errWriter = new PrintWriter(err)) {
+ int exitCode = runMain(outWriter, errWriter, args);
+ return new RunCLI(exitCode, out.toString(), err.toString(), args);
+ }
+ }
+
+ private static int runMain(PrintWriter out, PrintWriter err, String... arguments) {
+ CommandLine commandLine =
+ new CommandLine(new CatalogMigrationCLI())
+ .setExecutionExceptionHandler(
+ (ex, cmd, parseResult) -> {
+ if (enableStacktrace(arguments)) {
+ cmd.getErr().println(cmd.getColorScheme().richStackTraceString(ex));
+ } else {
+ cmd.getErr().println("Error during CLI execution: " + ex.getMessage());
+ }
+ return cmd.getExitCodeExceptionMapper() != null
+ ? cmd.getExitCodeExceptionMapper().getExitCode(ex)
+ : cmd.getCommandSpec().exitCodeOnExecutionException();
+ });
+ if (null != out) {
+ commandLine = commandLine.setOut(out);
+ }
+ if (null != err) {
+ commandLine = commandLine.setErr(err);
+ }
+ try {
+ return commandLine.execute(arguments);
+ } finally {
+ commandLine.getErr().flush();
+ }
+ }
+
+ public int getExitCode() {
+ return exitCode;
+ }
+
+ public String getOut() {
+ return out;
+ }
+
+ public String getErr() {
+ return err;
+ }
+
+ private static boolean enableStacktrace(String... args) {
+ for (String arg : args) {
+ if (arg.equalsIgnoreCase("--stacktrace")) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "org.apache.polaris.catalogs.migrator.cli"
+ + ".RunCLI{args=%s%nexitCode=%d%n%nstdout:%n%s%n%nstderr:%n%s",
+ Arrays.toString(args), exitCode, out, err);
+ }
+}
diff --git a/iceberg-catalog-migrator/codestyle/checkstyle-config.xml b/iceberg-catalog-migrator/codestyle/checkstyle-config.xml
new file mode 100644
index 00000000..f895b4a3
--- /dev/null
+++ b/iceberg-catalog-migrator/codestyle/checkstyle-config.xml
@@ -0,0 +1,178 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/iceberg-catalog-migrator/codestyle/copyright-header-java.txt b/iceberg-catalog-migrator/codestyle/copyright-header-java.txt
new file mode 100644
index 00000000..042f3ce1
--- /dev/null
+++ b/iceberg-catalog-migrator/codestyle/copyright-header-java.txt
@@ -0,0 +1,18 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
diff --git a/iceberg-catalog-migrator/codestyle/copyright-header.txt b/iceberg-catalog-migrator/codestyle/copyright-header.txt
new file mode 100644
index 00000000..90705e02
--- /dev/null
+++ b/iceberg-catalog-migrator/codestyle/copyright-header.txt
@@ -0,0 +1,16 @@
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
\ No newline at end of file
diff --git a/iceberg-catalog-migrator/codestyle/errorprone-rules.properties b/iceberg-catalog-migrator/codestyle/errorprone-rules.properties
new file mode 100644
index 00000000..5f6a6222
--- /dev/null
+++ b/iceberg-catalog-migrator/codestyle/errorprone-rules.properties
@@ -0,0 +1,1655 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Contains bug patterns up to Error Prone 2.15.0
+
+####################################################################################################
+# On by default : ERROR
+# See https://errorprone.info/bugpatterns
+####################################################################################################
+
+#AlwaysThrows=ERROR
+# Detects calls that will fail at runtime
+
+#AndroidInjectionBeforeSuper=ERROR
+# AndroidInjection.inject() should always be invoked before calling super.lifecycleMethod()
+
+#ArrayEquals=ERROR
+# Reference equality used to compare arrays
+
+#ArrayFillIncompatibleType=ERROR
+# Arrays.fill(Object[], Object) called with incompatible types.
+
+#ArrayHashCode=ERROR
+# hashcode method on array does not hash array contents
+
+#ArrayToString=ERROR
+# Calling toString on an array does not provide useful information
+
+#ArraysAsListPrimitiveArray=ERROR
+# Arrays.asList does not autobox primitive arrays, as one might expect.
+
+#AsyncCallableReturnsNull=ERROR
+# AsyncCallable should not return a null Future, only a Future whose result is null.
+
+#AsyncFunctionReturnsNull=ERROR
+# AsyncFunction should not return a null Future, only a Future whose result is null.
+
+#AutoValueBuilderDefaultsInConstructor=ERROR
+# Defaults for AutoValue Builders should be set in the factory method returning Builder instances, not the constructor
+
+#AutoValueConstructorOrderChecker=ERROR
+# Arguments to AutoValue constructor are in the wrong order
+
+#BadAnnotationImplementation=ERROR
+# Classes that implement Annotation must override equals and hashCode. Consider using AutoAnnotation instead of implementing Annotation by hand.
+
+#BadShiftAmount=ERROR
+# Shift by an amount that is out of range
+
+#BanJNDI=ERROR
+# Using JNDI may deserialize user input via the `Serializable` API which is extremely dangerous
+
+#BoxedPrimitiveEquality=ERROR
+# Comparison using reference equality instead of value equality. Reference equality of boxed primitive types is usually not useful, as they are value objects, and it is bug-prone, as instances are cached for some values but not others.
+
+#BundleDeserializationCast=ERROR
+# Object serialized in Bundle may have been flattened to base type.
+
+#ChainingConstructorIgnoresParameter=ERROR
+# The called constructor accepts a parameter with the same name and type as one of its caller's parameters, but its caller doesn't pass that parameter to it. It's likely that it was intended to.
+
+#CheckNotNullMultipleTimes=ERROR
+# A variable was checkNotNulled multiple times. Did you mean to check something else?
+
+#CheckReturnValue=ERROR
+# Ignored return value of method that is annotated with @CheckReturnValue
+
+#CollectionIncompatibleType=ERROR
+# Incompatible type as argument to Object-accepting Java collections method
+
+#CollectionToArraySafeParameter=ERROR
+# The type of the array parameter of Collection.toArray needs to be compatible with the array type
+
+#ComparableType=ERROR
+# Implementing 'Comparable' where T is not the same as the implementing class is incorrect, since it violates the symmetry contract of compareTo.
+
+#ComparingThisWithNull=ERROR
+# this == null is always false, this != null is always true
+
+#ComparisonOutOfRange=ERROR
+# Comparison to value that is out of range for the compared type
+
+#CompatibleWithAnnotationMisuse=ERROR
+# @CompatibleWith's value is not a type argument.
+
+#CompileTimeConstant=ERROR
+# Non-compile-time constant expression passed to parameter with @CompileTimeConstant type annotation.
+
+#ComputeIfAbsentAmbiguousReference=ERROR
+# computeIfAbsent passes the map key to the provided class's constructor
+
+#ConditionalExpressionNumericPromotion=ERROR
+# A conditional expression with numeric operands of differing types will perform binary numeric promotion of the operands; when these operands are of reference types, the expression's result may not be of the expected type.
+
+#ConstantOverflow=ERROR
+# Compile-time constant expression overflows
+
+#DaggerProvidesNull=ERROR
+# Dagger @Provides methods may not return null unless annotated with @Nullable
+
+#DangerousLiteralNull=ERROR
+# This method is null-hostile: passing a null literal to it is always wrong
+
+#DeadException=ERROR
+# Exception created but not thrown
+
+#DeadThread=ERROR
+# Thread created but not started
+
+#DiscardedPostfixExpression=ERROR
+# The result of this unary operation on a lambda parameter is discarded
+
+#DoNotCall=ERROR
+# This method should not be called.
+
+#DoNotMock=ERROR
+# Identifies undesirable mocks.
+
+#DoubleBraceInitialization=ERROR
+# Prefer collection factory methods or builders to the double-brace initialization pattern.
+
+#DuplicateMapKeys=ERROR
+# Map#ofEntries will throw an IllegalArgumentException if there are any duplicate keys
+
+#DurationFrom=ERROR
+# Duration.from(Duration) returns itself; from(Period) throws a runtime exception.
+
+#DurationGetTemporalUnit=ERROR
+# Duration.get() only works with SECONDS or NANOS.
+
+#DurationTemporalUnit=ERROR
+# Duration APIs only work for DAYS or exact durations.
+
+#DurationToLongTimeUnit=ERROR
+# Unit mismatch when decomposing a Duration or Instant to call a API
+
+#EmptyTopLevelDeclaration=ERROR
+# Empty top-level type declarations should be omitted
+
+#EqualsHashCode=ERROR
+# Classes that override equals should also override hashCode.
+
+#EqualsNaN=ERROR
+# == NaN always returns false; use the isNaN methods instead
+
+#EqualsNull=ERROR
+# The contract of Object.equals() states that for any non-null reference value x, x.equals(null) should return false. If x is null, a NullPointerException is thrown. Consider replacing equals() with the == operator.
+
+#EqualsReference=ERROR
+# == must be used in equals method to check equality to itself or an infinite loop will occur.
+
+#EqualsWrongThing=ERROR
+# Comparing different pairs of fields/getters in an equals implementation is probably a mistake.
+
+#FloggerFormatString=ERROR
+# Invalid printf-style format string
+
+#FloggerLogString=ERROR
+# Arguments to log(String) must be compile-time constants or parameters annotated with @CompileTimeConstant. If possible, use Flogger's formatting log methods instead.
+
+#FloggerLogVarargs=ERROR
+# logVarargs should be used to pass through format strings and arguments.
+
+#FloggerSplitLogStatement=ERROR
+# Splitting log statements and using Api instances directly breaks logging.
+
+#ForOverride=ERROR
+# Method annotated @ForOverride must be protected or package-private and only invoked from declaring class, or from an override of the method
+
+#FormatString=ERROR
+# Invalid printf-style format string
+
+#FormatStringAnnotation=ERROR
+# Invalid format string passed to formatting method.
+
+#FromTemporalAccessor=ERROR
+# Certain combinations of javaTimeType.from(TemporalAccessor) will always throw a DateTimeException or return the parameter directly.
+
+#FunctionalInterfaceMethodChanged=ERROR
+# Casting a lambda to this @FunctionalInterface can cause a behavior change from casting to a functional superinterface, which is surprising to users. Prefer decorator methods to this surprising behavior.
+
+#FuturesGetCheckedIllegalExceptionType=ERROR
+# Futures.getChecked requires a checked exception type with a standard constructor.
+
+#FuzzyEqualsShouldNotBeUsedInEqualsMethod=ERROR
+# DoubleMath.fuzzyEquals should never be used in an Object.equals() method
+
+#GetClassOnAnnotation=ERROR
+# Calling getClass() on an annotation may return a proxy class
+
+#GetClassOnClass=ERROR
+# Calling getClass() on an object of type Class returns the Class object for java.lang.Class; you probably meant to operate on the object directly
+
+#GuardedBy=ERROR
+# Checks for unguarded accesses to fields and methods with @GuardedBy annotations
+
+#GuiceAssistedInjectScoping=ERROR
+# Scope annotation on implementation class of AssistedInject factory is not allowed
+
+#GuiceAssistedParameters=ERROR
+# A constructor cannot have two @Assisted parameters of the same type unless they are disambiguated with named @Assisted annotations.
+
+#GuiceInjectOnFinalField=ERROR
+# Although Guice allows injecting final fields, doing so is disallowed because the injected value may not be visible to other threads.
+
+#HashtableContains=ERROR
+# contains() is a legacy method that is equivalent to containsValue()
+
+#IdentityBinaryExpression=ERROR
+# A binary expression where both operands are the same is usually incorrect.
+
+#IdentityHashMapBoxing=ERROR
+# Using IdentityHashMap with a boxed type as the key is risky since boxing may produce distinct instances
+
+#IgnoredPureGetter=ERROR
+# Getters on AutoValues, AutoBuilders, and Protobuf Messages are side-effect free, so there is no point in calling them if the return value is ignored. While there are no side effects from the getter, the receiver may have side effects.
+
+#Immutable=ERROR
+# Type declaration annotated with @Immutable is not immutable
+
+#Incomparable=ERROR
+# Types contained in sorted collections must implement Comparable.
+
+#IncompatibleArgumentType=ERROR
+# Passing argument to a generic method with an incompatible type.
+
+#IncompatibleModifiers=ERROR
+# This annotation has incompatible modifiers as specified by its @IncompatibleModifiers annotation
+
+#IndexOfChar=ERROR
+# The first argument to indexOf is a Unicode code point, and the second is the index to start the search from
+
+#InexactVarargsConditional=ERROR
+# Conditional expression in varargs call contains array and non-array arguments
+
+#InfiniteRecursion=ERROR
+# This method always recurses, and will cause a StackOverflowError
+
+#InjectMoreThanOneScopeAnnotationOnClass=ERROR
+# A class can be annotated with at most one scope annotation.
+
+#InjectOnMemberAndConstructor=ERROR
+# Members shouldn't be annotated with @Inject if constructor is already annotated @Inject
+
+#InlineMeValidator=ERROR
+# Ensures that the @InlineMe annotation is used correctly.
+
+#InstantTemporalUnit=ERROR
+# Instant APIs only work for NANOS, MICROS, MILLIS, SECONDS, MINUTES, HOURS, HALF_DAYS and DAYS.
+
+#InvalidJavaTimeConstant=ERROR
+# This checker errors on calls to java.time methods using values that are guaranteed to throw a DateTimeException.
+
+#InvalidPatternSyntax=ERROR
+# Invalid syntax used for a regular expression
+
+#InvalidTimeZoneID=ERROR
+# Invalid time zone identifier. TimeZone.getTimeZone(String) will silently return GMT instead of the time zone you intended.
+
+#InvalidZoneId=ERROR
+# Invalid zone identifier. ZoneId.of(String) will throw exception at runtime.
+
+#IsInstanceIncompatibleType=ERROR
+# This use of isInstance will always evaluate to false.
+
+#IsInstanceOfClass=ERROR
+# The argument to Class#isInstance(Object) should not be a Class
+
+#IsLoggableTagLength=ERROR
+# Log tag too long, cannot exceed 23 characters.
+
+#JUnit3TestNotRun=ERROR
+# Test method will not be run; please correct method signature (Should be public, non-static, and method name should begin with "test").
+
+#JUnit4ClassAnnotationNonStatic=ERROR
+# This method should be static
+
+#JUnit4SetUpNotRun=ERROR
+# setUp() method will not be run; please add JUnit's @Before annotation
+
+#JUnit4TearDownNotRun=ERROR
+# tearDown() method will not be run; please add JUnit's @After annotation
+
+#JUnit4TestNotRun=ERROR
+# This looks like a test method but is not run; please add @Test and @Ignore, or, if this is a helper method, reduce its visibility.
+
+#JUnit4TestsNotRunWithinEnclosed=ERROR
+# This test is annotated @Test, but given it's within a class using the Enclosed runner, will not run.
+
+#JUnitAssertSameCheck=ERROR
+# An object is tested for reference equality to itself using JUnit library.
+
+#JUnitParameterMethodNotFound=ERROR
+# The method for providing parameters was not found.
+
+#JavaxInjectOnAbstractMethod=ERROR
+# Abstract and default methods are not injectable with javax.inject.Inject
+
+#JodaToSelf=ERROR
+# Use of Joda-Time's DateTime.toDateTime(), Duration.toDuration(), Instant.toInstant(), Interval.toInterval(), and Period.toPeriod() are not allowed.
+
+#LiteByteStringUtf8=ERROR
+# This pattern will silently corrupt certain byte sequences from the serialized protocol message. Use ByteString or byte[] directly
+
+#LocalDateTemporalAmount=ERROR
+# LocalDate.plus() and minus() does not work with Durations. LocalDate represents civil time (years/months/days), so java.time.Period is the appropriate thing to add or subtract instead.
+
+#LockOnBoxedPrimitive=ERROR
+# It is dangerous to use a boxed primitive as a lock as it can unintentionally lead to sharing a lock with another piece of code.
+
+#LoopConditionChecker=ERROR
+# Loop condition is never modified in loop body.
+
+#LossyPrimitiveCompare=ERROR
+# Using an unnecessarily-wide comparison method can lead to lossy comparison
+
+#MathRoundIntLong=ERROR
+# Math.round(Integer) results in truncation
+
+#MislabeledAndroidString=ERROR
+# Certain resources in `android.R.string` have names that do not match their content
+
+#MisplacedScopeAnnotations=ERROR
+# Scope annotations used as qualifier annotations don't have any effect. Move the scope annotation to the binding location or delete it.
+
+#MissingSuperCall=ERROR
+# Overriding method is missing a call to overridden super method
+
+#MissingTestCall=ERROR
+# A terminating method call is required for a test helper to have any effect.
+
+#MisusedDayOfYear=ERROR
+# Use of 'DD' (day of year) in a date pattern with 'MM' (month of year) is not likely to be intentional, as it would lead to dates like 'March 73rd'.
+
+#MisusedWeekYear=ERROR
+# Use of "YYYY" (week year) in a date pattern without "ww" (week in year). You probably meant to use "yyyy" (year) instead.
+
+#MixedDescriptors=ERROR
+# The field number passed into #getFieldByNumber belongs to a different proto to the Descriptor.
+
+#MockitoUsage=ERROR
+# Missing method call for verify(mock) here
+
+#ModifyingCollectionWithItself=ERROR
+# Using a collection function with itself as the argument.
+
+#MoreThanOneInjectableConstructor=ERROR
+# This class has more than one @Inject-annotated constructor. Please remove the @Inject annotation from all but one of them.
+
+MustBeClosedChecker=ERROR
+# This method returns a resource which must be managed carefully, not just left for garbage collection. If it is a constant that will persist for the lifetime of your program, move it to a private static final field. Otherwise, you should use it in a try-with-resources.
+
+#NCopiesOfChar=ERROR
+# The first argument to nCopies is the number of copies, and the second is the item to copy
+
+#NoCanIgnoreReturnValueOnClasses=ERROR
+# @CanIgnoreReturnValue should not be applied to classes as it almost always overmatches (as it
+# applies to constructors and all methods), and the CIRVness isn't conferred to its subclasses.
+
+#NonCanonicalStaticImport=ERROR
+# Static import of type uses non-canonical name
+
+#NonFinalCompileTimeConstant=ERROR
+# @CompileTimeConstant parameters should be final or effectively final
+
+#NonRuntimeAnnotation=ERROR
+# Calling getAnnotation on an annotation that is not retained at runtime.
+
+#NullArgumentForNonNullParameter=ERROR
+# Null is not permitted for this parameter.
+
+#NullTernary=ERROR
+# This conditional expression may evaluate to null, which will result in an NPE when the result is unboxed.
+
+#OptionalEquality=ERROR
+# Comparison using reference equality instead of value equality
+
+#OptionalMapUnusedValue=ERROR
+# Optional.ifPresent is preferred over Optional.map when the return value is unused
+
+#OptionalOfRedundantMethod=ERROR
+# Optional.of() always returns a non-empty optional. Using ifPresent/isPresent/orElse/orElseGet/orElseThrow/isPresent/or/orNull method on it is unnecessary and most probably a bug.
+
+#OverlappingQualifierAndScopeAnnotation=ERROR
+# Annotations cannot be both Scope annotations and Qualifier annotations: this causes confusion when trying to use them.
+
+#OverridesJavaxInjectableMethod=ERROR
+# This method is not annotated with @Inject, but it overrides a method that is annotated with @javax.inject.Inject. The method will not be Injected.
+
+#PackageInfo=ERROR
+# Declaring types inside package-info.java files is very bad form
+
+#ParametersButNotParameterized=ERROR
+# This test has @Parameters but is using the default JUnit4 runner. The parameters will have no effect.
+
+#ParcelableCreator=ERROR
+# Detects classes which implement Parcelable but don't have CREATOR
+
+#PeriodFrom=ERROR
+# Period.from(Period) returns itself; from(Duration) throws a runtime exception.
+
+#PeriodGetTemporalUnit=ERROR
+# Period.get() only works with YEARS, MONTHS, or DAYS.
+
+#PeriodTimeMath=ERROR
+# When adding or subtracting from a Period, Duration is incompatible.
+
+#PreconditionsInvalidPlaceholder=ERROR
+# Preconditions only accepts the %s placeholder in error message strings
+
+#PrivateSecurityContractProtoAccess=ERROR
+# Access to a private protocol buffer field is forbidden. This protocol buffer carries a security contract, and can only be created using an approved library. Direct access to the fields is forbidden.
+
+#ProtoBuilderReturnValueIgnored=ERROR
+# Unnecessary call to proto's #build() method. If you don't consume the return value of #build(), the result is discarded and the only effect is to verify that all required fields are set, which can be expressed more directly with #isInitialized().
+
+#ProtoFieldNullComparison=ERROR
+# Protobuf fields cannot be null.
+
+#ProtoStringFieldReferenceEquality=ERROR
+# Comparing protobuf fields of type String using reference equality
+
+#ProtoTruthMixedDescriptors=ERROR
+# The arguments passed to `ignoringFields` are inconsistent with the proto which is the subject of the assertion.
+
+#ProtocolBufferOrdinal=ERROR
+# To get the tag number of a protocol buffer enum, use getNumber() instead.
+
+#ProvidesMethodOutsideOfModule=ERROR
+# @Provides methods need to be declared in a Module to have any effect.
+
+#RandomCast=ERROR
+# Casting a random number in the range [0.0, 1.0) to an integer or long always results in 0.
+
+#RandomModInteger=ERROR
+# Use Random.nextInt(int). Random.nextInt() % n can have negative results
+
+#RectIntersectReturnValueIgnored=ERROR
+# Return value of android.graphics.Rect.intersect() must be checked
+
+#RequiredModifiers=ERROR
+# This annotation is missing required modifiers as specified by its @RequiredModifiers annotation
+
+#RestrictedApiChecker=ERROR
+# Check for non-allowlisted callers to RestrictedApiChecker.
+
+#ReturnValueIgnored=ERROR
+# Return value of this method must be used
+
+#SelfAssignment=ERROR
+# Variable assigned to itself
+
+#SelfComparison=ERROR
+# An object is compared to itself
+
+#SelfEquals=ERROR
+# Testing an object for equality with itself will always be true.
+
+#ShouldHaveEvenArgs=ERROR
+# This method must be called with an even number of arguments.
+
+#SizeGreaterThanOrEqualsZero=ERROR
+# Comparison of a size >= 0 is always true, did you intend to check for non-emptiness?
+
+#StreamToString=ERROR
+# Calling toString on a Stream does not provide useful information
+
+#StringBuilderInitWithChar=ERROR
+# StringBuilder does not have a char constructor; this invokes the int constructor.
+
+#SubstringOfZero=ERROR
+# String.substring(0) returns the original String
+
+#SuppressWarningsDeprecated=ERROR
+# Suppressing "deprecated" is probably a typo for "deprecation"
+
+#TemporalAccessorGetChronoField=ERROR
+# TemporalAccessor.get() only works for certain values of ChronoField.
+
+#TestParametersNotInitialized=ERROR
+# This test has @TestParameter fields but is using the default JUnit4 runner. The parameters will not be initialised beyond their default value.
+
+#TheoryButNoTheories=ERROR
+# This test has members annotated with @Theory, @DataPoint, or @DataPoints but is using the default JUnit4 runner.
+
+#ThrowIfUncheckedKnownChecked=ERROR
+# throwIfUnchecked(knownCheckedException) is a no-op.
+
+#ThrowNull=ERROR
+# Throwing 'null' always results in a NullPointerException being thrown.
+
+#TreeToString=ERROR
+# Tree#toString shouldn't be used for Trees deriving from the code being compiled, as it discards whitespace and comments.
+
+#TruthSelfEquals=ERROR
+# isEqualTo should not be used to test an object for equality with itself; the assertion will never fail.
+
+#TryFailThrowable=ERROR
+# Catching Throwable/Error masks failures from fail() or assert*() in the try block
+
+#TypeParameterQualifier=ERROR
+# Type parameter used as type qualifier
+
+#UnicodeDirectionalityCharacters=ERROR
+# Unicode directionality modifiers can be used to conceal code in many editors.
+
+UnicodeInCode=OFF
+# Avoid using non-ASCII Unicode characters outside of comments and literals, as they can be confusing.
+
+#UnnecessaryCheckNotNull=ERROR
+# This null check is unnecessary; the expression can never be null
+
+#UnnecessaryTypeArgument=ERROR
+# Non-generic methods should not be invoked with type arguments
+
+#UnsafeWildcard=ERROR
+# Certain wildcard types can confuse the compiler.
+
+#UnusedAnonymousClass=ERROR
+# Instance created but never used
+
+#UnusedCollectionModifiedInPlace=ERROR
+# Collection is modified in place, but the result is not used
+
+#VarTypeName=ERROR
+# `var` should not be used as a type name.
+
+#WrongOneof=ERROR
+# This field is guaranteed not to be set given it's within a switch over a one_of.
+
+#XorPower=ERROR
+# The `^` operator is binary XOR, not a power operator.
+
+#ZoneIdOfZ=ERROR
+# Use ZoneOffset.UTC instead of ZoneId.of("Z").
+
+####################################################################################################
+# On by default : WARNING
+# See https://errorprone.info/bugpatterns
+####################################################################################################
+
+#AlmostJavadoc=WARN
+# This comment contains Javadoc or HTML tags, but isn't started with a double asterisk (/**); is it meant to be Javadoc?
+
+#AlreadyChecked=WARN
+# This condition has already been checked.
+
+#AmbiguousMethodReference=WARN
+# Method reference is ambiguous
+
+AnnotateFormatMethod=ERROR
+# This method passes a pair of parameters through to String.format, but the enclosing method wasn't annotated @FormatMethod. Doing so gives compile-time rather than run-time protection against malformed format strings.
+
+#ArgumentSelectionDefectChecker=WARN
+# Arguments are in the wrong order or could be commented for clarity.
+
+ArrayAsKeyOfSetOrMap=ERROR
+# Arrays do not override equals() or hashCode, so comparisons will be done on reference equality only. If neither deduplication nor lookup are needed, consider using a List instead. Otherwise, use IdentityHashMap/Set, a Map from a library that handles object arrays, or an Iterable/List of pairs.
+
+AssertEqualsArgumentOrderChecker=ERROR
+# Arguments are swapped in assertEquals-like call
+
+AssertThrowsMultipleStatements=ERROR
+# The lambda passed to assertThrows should contain exactly one statement
+
+AssertionFailureIgnored=ERROR
+# This assertion throws an AssertionError if it fails, which will be caught by an enclosing try block.
+
+#AssistedInjectAndInjectOnSameConstructor=WARN
+# @AssistedInject and @Inject cannot be used on the same constructor.
+
+#AutoValueFinalMethods=WARN
+# Make toString(), hashCode() and equals() final in AutoValue classes, so it is clear to readers that AutoValue is not overriding them
+
+#AutoValueImmutableFields=WARN
+# AutoValue recommends using immutable collections
+
+#AutoValueSubclassLeaked=WARN
+# Do not refer to the autogenerated AutoValue_ class outside the file containing the corresponding @AutoValue base class.
+
+#BadComparable=WARN
+# Possible sign flip from narrowing conversion
+
+BadImport=ERROR
+# Importing nested classes/static methods/static fields with commonly-used names can make code harder to read, because it may not be clear from the context exactly which type is being referred to. Qualifying the name with that of the containing class can make the code clearer.
+
+BadInstanceof=ERROR
+# instanceof used in a way that is equivalent to a null check.
+
+BareDotMetacharacter=ERROR
+# "." is rarely useful as a regex, as it matches any character. To match a literal '.' character, instead write "\.".
+
+BigDecimalEquals=ERROR
+# BigDecimal#equals has surprising behavior: it also compares scale.
+
+BigDecimalLiteralDouble=ERROR
+# new BigDecimal(double) loses precision in this case.
+
+BoxedPrimitiveConstructor=ERROR
+# valueOf or autoboxing provides better time and space performance
+
+#BugPatternNaming=WARN
+# Giving BugPatterns a name different to the enclosing class can be confusing
+
+#BuilderReturnThis=WARN
+# Builder instance method does not return 'this'
+
+ByteBufferBackingArray=ERROR
+# ByteBuffer.array() shouldn't be called unless ByteBuffer.arrayOffset() is used or if the ByteBuffer was initialized using ByteBuffer.wrap() or ByteBuffer.allocate().
+
+#CacheLoaderNull=WARN
+# The result of CacheLoader#load must be non-null.
+
+CanIgnoreReturnValueSuggester=OFF
+# Methods that always 'return this' should be annotated with @CanIgnoreReturnValue
+
+#CannotMockFinalClass=WARN
+# Mockito cannot mock final classes
+
+#CanonicalDuration=WARN
+# Duration can be expressed more clearly with different units
+
+CatchAndPrintStackTrace=ERROR
+# Logging or rethrowing exceptions should usually be preferred to catching and calling printStackTrace
+
+#CatchFail=WARN
+# Ignoring exceptions and calling fail() is unnecessary, and makes test output less useful
+
+#ChainedAssertionLosesContext=WARN
+# Inside a Subject, use check(…) instead of assert*() to preserve user-supplied messages and other settings.
+
+#CharacterGetNumericValue=WARN
+# getNumericValue has unexpected behaviour: it interprets A-Z as base-36 digits with values 10-35, but also supports non-arabic numerals and miscellaneous numeric unicode characters like ㊷; consider using Character.digit or UCharacter.getUnicodeNumericValue instead
+
+#ClassCanBeStatic=WARN
+# Inner class is non-static but does not reference enclosing class
+
+ClassNewInstance=ERROR
+# Class.newInstance() bypasses exception checking; prefer getDeclaredConstructor().newInstance()
+
+#CloseableProvides=WARN
+# Providing Closeable resources makes their lifecycle unclear
+
+#CollectionUndefinedEquality=WARN
+# This type does not have well-defined equals behavior.
+
+#CollectorShouldNotUseState=WARN
+# Collector.of() should not use state
+
+#ComparableAndComparator=WARN
+# Class should not implement both `Comparable` and `Comparator`
+
+#CompareToZero=WARN
+# The result of #compareTo or #compare should only be compared to 0. It is an implementation detail whether a given type returns strictly the values {-1, 0, +1} or others.
+
+#ComplexBooleanConstant=WARN
+# Non-trivial compile time constant boolean expressions shouldn't be used.
+
+#DateChecker=WARN
+# Warns against suspect looking calls to java.util.Date APIs
+
+DateFormatConstant=ERROR
+# DateFormat is not thread-safe, and should not be used as a constant field.
+
+DefaultCharset=ERROR
+# Implicit use of the platform default charset, which can result in differing behaviour between JVM executions or incorrect behavior if the encoding of the data source doesn't match expectations.
+
+#DefaultPackage=WARN
+# Java classes shouldn't use default package
+
+#DeprecatedVariable=WARN
+# Applying the @Deprecated annotation to local variables or parameters has no effect
+
+#DirectInvocationOnMock=WARN
+# Methods should not be directly invoked on mocks. Should this be part of a verify(..) call?
+
+DistinctVarargsChecker=ERROR
+# Method expects distinct arguments at some/all positions
+
+#DoNotCallSuggester=WARN
+# Consider annotating methods that always throw with @DoNotCall. Read more at https://errorprone.info/bugpattern/DoNotCall
+
+#DoNotClaimAnnotations=WARN
+# Don't 'claim' annotations in annotation processors; Processor#process should unconditionally return `false`
+
+#DoNotMockAutoValue=WARN
+# AutoValue classes represent pure data classes, so mocking them should not be necessary. Construct a real instance of the class instead.
+
+DoubleCheckedLocking=ERROR
+# Double-checked locking on non-volatile fields is unsafe
+
+#EmptyBlockTag=WARN
+# A block tag (@param, @return, @throws, @deprecated) has an empty description. Block tags without descriptions don't add much value for future readers of the code; consider removing the tag entirely or adding a description.
+
+#EmptyCatch=WARN
+# Caught exceptions should not be ignored
+
+#EmptySetMultibindingContributions=WARN
+# @Multibinds is a more efficient and declarative mechanism for ensuring that a set multibinding is present in the graph.
+
+EqualsGetClass=ERROR
+# Prefer instanceof to getClass when implementing Object#equals.
+
+EqualsIncompatibleType=ERROR
+# An equality test between objects with incompatible types always returns false
+
+EqualsUnsafeCast=ERROR
+# The contract of #equals states that it should return false for incompatible types, while this implementation may throw ClassCastException.
+
+EqualsUsingHashCode=ERROR
+# Implementing #equals by just comparing hashCodes is fragile. Hashes collide frequently, and this will lead to false positives in #equals.
+
+ErroneousBitwiseExpression=ERROR
+# This expression evaluates to 0. If this isn't an error, consider expressing it as a literal 0.
+
+ErroneousThreadPoolConstructorChecker=ERROR
+# Thread pool size will never go beyond corePoolSize if an unbounded queue is used
+
+EscapedEntity=ERROR
+# HTML entities in @code/@literal tags will appear literally in the rendered javadoc.
+
+#ExtendingJUnitAssert=WARN
+# When only using JUnit Assert's static methods, you should import statically instead of extending.
+
+#ExtendsObject=WARN
+# `T extends Object` is redundant (unless you are using the Checker Framework).
+
+#FallThrough=WARN
+# Switch case may fall through
+
+#Finally=WARN
+# If you return or throw from a finally, then values returned or thrown from the try-catch block will be ignored. Consider using try-with-resources instead.
+
+FloatCast=ERROR
+# Use parentheses to make the precedence explicit
+
+FloatingPointAssertionWithinEpsilon=ERROR
+# This fuzzy equality check is using a tolerance less than the gap to the next number. You may want a less restrictive tolerance, or to assert equality.
+
+FloatingPointLiteralPrecision=ERROR
+# Floating point literal loses precision
+
+#FloggerArgumentToString=WARN
+# Use Flogger's printf-style formatting instead of explicitly converting arguments to strings
+
+#FloggerStringConcatenation=WARN
+# Prefer string formatting using printf placeholders (e.g. %s) instead of string concatenation
+
+#FragmentInjection=WARN
+# Classes extending PreferenceActivity must implement isValidFragment such that it does not unconditionally return true to prevent vulnerability to fragment injection attacks.
+
+#FragmentNotInstantiable=WARN
+# Subclasses of Fragment must be instantiable via Class#newInstance(): the class must be public, static and have a public nullary constructor
+
+FutureReturnValueIgnored=ERROR
+# Return value of methods returning Future must be checked. Ignoring returned Futures suppresses exceptions thrown from the code that completes the Future.
+
+GetClassOnEnum=ERROR
+# Calling getClass() on an enum may return a subclass of the enum type
+
+#HidingField=WARN
+# Hiding fields of superclasses may cause confusion and errors
+
+#IdentityHashMapUsage=WARN
+# IdentityHashMap usage shouldn't be intermingled with Map
+
+#ImmutableAnnotationChecker=WARN
+# Annotations should always be immutable
+
+#ImmutableEnumChecker=WARN
+# Enums should always be immutable
+
+#InconsistentCapitalization=WARN
+# It is confusing to have a field and a parameter under the same scope that differ only in capitalization.
+
+InconsistentHashCode=ERROR
+# Including fields in hashCode which are not compared in equals violates the contract of hashCode.
+
+#IncorrectMainMethod=WARN
+# 'main' methods must be public, static, and void
+
+#IncrementInForLoopAndHeader=WARN
+# This for loop increments the same variable in the header and in the body
+
+#InheritDoc=WARN
+# Invalid use of @inheritDoc.
+
+#InjectInvalidTargetingOnScopingAnnotation=WARN
+# A scoping annotation's Target should include TYPE and METHOD.
+
+#InjectOnConstructorOfAbstractClass=WARN
+# Constructors on abstract classes are never directly @Inject'ed, only the constructors of their subclasses can be @Inject'ed.
+
+#InjectScopeAnnotationOnInterfaceOrAbstractClass=WARN
+# Scope annotation on an interface or abstract class is not allowed
+
+#InjectedConstructorAnnotations=WARN
+# Injected constructors cannot be optional nor have binding annotations
+
+#InlineFormatString=WARN
+# Prefer to create format strings inline, instead of extracting them to a single-use constant
+
+#InlineMeInliner=WARN
+# Callers of this API should be inlined.
+
+#InlineMeSuggester=WARN
+# This deprecated API looks inlineable. If you'd like the body of the API to be inlined to its callers, please annotate it with @InlineMe.
+
+#InputStreamSlowMultibyteRead=WARN
+# Please also override int read(byte[], int, int), otherwise multi-byte reads from this input stream are likely to be slow.
+
+#InstanceOfAndCastMatchWrongType=WARN
+# Casting inside an if block should be plausibly consistent with the instanceof type
+
+IntLongMath=ERROR
+# Expression of type int may overflow before being assigned to a long
+
+#InvalidBlockTag=WARN
+# This tag is invalid.
+
+#InvalidInlineTag=WARN
+# This tag is invalid.
+
+#InvalidLink=WARN
+# This @link tag looks wrong.
+
+#InvalidParam=WARN
+# This @param tag doesn't refer to a parameter of the method.
+
+#InvalidThrows=WARN
+# The documented method doesn't actually throw this checked exception.
+
+#InvalidThrowsLink=WARN
+# Javadoc links to exceptions in @throws without a @link tag (@throws Exception, not @throws {@link Exception}).
+
+#IterableAndIterator=WARN
+# Class should not implement both `Iterable` and `Iterator`
+
+#JUnit3FloatingPointComparisonWithoutDelta=WARN
+# Floating-point comparison without error tolerance
+
+#JUnit4ClassUsedInJUnit3=WARN
+# Some JUnit4 construct cannot be used in a JUnit3 context. Convert your class to JUnit4 style to use them.
+
+#JUnitAmbiguousTestClass=WARN
+# Test class inherits from JUnit 3's TestCase but has JUnit 4 @Test or @RunWith annotations.
+
+#JavaDurationGetSecondsGetNano=WARN
+# duration.getNano() only accesses the underlying nanosecond adjustment from the whole second.
+
+#JavaDurationWithNanos=WARN
+# Use of java.time.Duration.withNanos(int) is not allowed.
+
+#JavaDurationWithSeconds=WARN
+# Use of java.time.Duration.withSeconds(long) is not allowed.
+
+#JavaInstantGetSecondsGetNano=WARN
+# instant.getNano() only accesses the underlying nanosecond adjustment from the whole second.
+
+JavaLangClash=ERROR
+# Never reuse class names from java.lang
+
+#JavaLocalDateTimeGetNano=WARN
+# localDateTime.getNano() only accesss the nanos-of-second field. It's rare to only use getNano() without a nearby getSecond() call.
+
+#JavaLocalTimeGetNano=WARN
+# localTime.getNano() only accesses the nanos-of-second field. It's rare to only use getNano() without a nearby getSecond() call.
+
+#JavaPeriodGetDays=WARN
+# period.getDays() only accesses the "days" portion of the Period, and doesn't represent the total span of time of the period. Consider using org.threeten.extra.Days to extract the difference between two civil dates if you want the whole time.
+
+#JavaTimeDefaultTimeZone=WARN
+# java.time APIs that silently use the default system time-zone are not allowed.
+
+#JavaUtilDate=WARN
+# Date has a bad API that leads to bugs; prefer java.time.Instant or LocalDate.
+
+#JavaxInjectOnFinalField=WARN
+# @javax.inject.Inject cannot be put on a final field.
+
+JdkObsolete=ERROR
+# Suggests alternatives to obsolete JDK classes.
+
+#JodaConstructors=WARN
+# Use of certain JodaTime constructors are not allowed.
+
+#JodaDateTimeConstants=WARN
+# Using the `PER` constants in `DateTimeConstants` is problematic because they encourage manual date/time math.
+
+#JodaDurationWithMillis=WARN
+# Use of duration.withMillis(long) is not allowed. Please use Duration.millis(long) instead.
+
+#JodaInstantWithMillis=WARN
+# Use of instant.withMillis(long) is not allowed. Please use new Instant(long) instead.
+
+#JodaNewPeriod=WARN
+# This may have surprising semantics, e.g. new Period(LocalDate.parse("1970-01-01"), LocalDate.parse("1970-02-02")).getDays() == 1, not 32.
+
+#JodaPlusMinusLong=WARN
+# Use of JodaTime's type.plus(long) or type.minus(long) is not allowed (where = {Duration,Instant,DateTime,DateMidnight}). Please use type.plus(Duration.millis(long)) or type.minus(Duration.millis(long)) instead.
+
+#JodaTimeConverterManager=WARN
+# Joda-Time's ConverterManager makes the semantics of DateTime/Instant/etc construction subject to global static state. If you need to define your own converters, use a helper.
+
+#JodaWithDurationAddedLong=WARN
+# Use of JodaTime's type.withDurationAdded(long, int) (where = {Duration,Instant,DateTime}). Please use type.withDurationAdded(Duration.millis(long), int) instead.
+
+#LiteEnumValueOf=WARN
+# Instead of converting enums to string and back, its numeric value should be used instead as it is the stable part of the protocol defined by the enum.
+
+#LiteProtoToString=WARN
+# toString() on lite protos will not generate a useful representation of the proto from optimized builds. Consider whether using some subset of fields instead would provide useful information.
+
+LockNotBeforeTry=ERROR
+# Calls to Lock#lock should be immediately followed by a try block which releases the lock.
+
+#LogicalAssignment=WARN
+# Assignment where a boolean expression was expected; use == if this assignment wasn't expected or add parentheses for clarity.
+
+LongDoubleConversion=ERROR
+# Conversion from long to double may lose precision; use an explicit cast to double if this was intentional
+
+LongFloatConversion=ERROR
+# Conversion from long to float may lose precision; use an explicit cast to float if this was intentional
+
+#LoopOverCharArray=WARN
+# toCharArray allocates a new array, using charAt is more efficient
+
+#MalformedInlineTag=WARN
+# This Javadoc tag is malformed. The correct syntax is {@tag and not @{tag.
+
+#MathAbsoluteRandom=WARN
+# Math.abs does not always give a positive result. Please consider other methods for positive random numbers.
+
+#MemoizeConstantVisitorStateLookups=WARN
+# Anytime you need to look up a constant value from VisitorState, improve performance by creating a cache for it with VisitorState.memoize
+
+#MissingCasesInEnumSwitch=WARN
+# Switches on enum types should either handle all values, or have a default case.
+
+#MissingFail=WARN
+# Not calling fail() when expecting an exception masks bugs
+
+#MissingImplementsComparable=WARN
+# Classes implementing valid compareTo function should implement Comparable interface
+
+MissingOverride=ERROR
+# method overrides method in supertype; expected @Override
+
+#MissingSummary=WARN
+# A summary line is required on public/protected Javadocs.
+
+#MixedMutabilityReturnType=WARN
+# This method returns both mutable and immutable collections or maps from different paths. This may be confusing for users of the method.
+
+#ModifiedButNotUsed=WARN
+# A collection or proto builder was created, but its values were never accessed.
+
+#MockNotUsedInProduction=WARN
+# This mock is instantiated and configured, but is never passed to production code. It should be
+# either removed or used.
+
+#ModifyCollectionInEnhancedForLoop=WARN
+# Modifying a collection while iterating over it in a loop may cause a ConcurrentModificationException to be thrown or lead to undefined behavior.
+
+#ModifySourceCollectionInStream=WARN
+# Modifying the backing source during stream operations may cause unintended results.
+
+#MultipleParallelOrSequentialCalls=WARN
+# Multiple calls to either parallel or sequential are unnecessary and cause confusion.
+
+#MultipleUnaryOperatorsInMethodCall=WARN
+# Avoid having multiple unary operators acting on the same variable in a method call
+
+#MutablePublicArray=WARN
+# Non-empty arrays are mutable, so this `public static final` array is not a constant and can be modified by clients of this class. Prefer an ImmutableList, or provide an accessor method that returns a defensive copy.
+
+#NarrowCalculation=WARN
+# This calculation may lose precision compared to its target type.
+
+#NarrowingCompoundAssignment=WARN
+# Compound assignments may hide dangerous casts
+
+#NegativeCharLiteral=WARN
+# Casting a negative signed literal to an (unsigned) char might be misleading.
+
+#NestedInstanceOfConditions=WARN
+# Nested instanceOf conditions of disjoint types create blocks of code that never execute
+
+#NonAtomicVolatileUpdate=WARN
+# This update of a volatile variable is non-atomic
+
+#NonCanonicalType=WARN
+# This type is referred to by a non-canonical name, which may be misleading.
+
+#NonOverridingEquals=WARN
+# equals method doesn't override Object.equals
+
+#NullOptional=WARN
+# Passing a literal null to an Optional parameter is almost certainly a mistake. Did you mean to provide an empty Optional?
+
+#NullableConstructor=WARN
+# Constructors should not be annotated with @Nullable since they cannot return null
+
+#NullablePrimitive=WARN
+# @Nullable should not be used for primitive types since they cannot be null
+
+#NullablePrimitiveArray=WARN
+# @Nullable type annotations should not be used for primitive types since they cannot be null
+
+#NullableVoid=WARN
+# void-returning methods should not be annotated with @Nullable, since they cannot return null
+
+ObjectEqualsForPrimitives=ERROR
+# Avoid unnecessary boxing by using plain == for primitive types.
+
+#ObjectToString=WARN
+# Calling toString on Objects that don't override toString() doesn't provide useful information
+
+#ObjectsHashCodePrimitive=WARN
+# Objects.hashCode(Object o) should not be passed a primitive value
+
+OperatorPrecedence=ERROR
+# Use grouping parenthesis to make the operator precedence explicit
+
+#OptionalMapToOptional=WARN
+# Mapping to another Optional will yield a nested Optional. Did you mean flatMap?
+
+#OptionalNotPresent=WARN
+# This Optional has been confirmed to be empty at this point, so the call to `get` will throw.
+
+OrphanedFormatString=ERROR
+# String literal contains format specifiers, but is not passed to a format method
+
+#OutlineNone=WARN
+# Setting CSS outline style to none or 0 (while not otherwise providing visual focus indicators) is inaccessible for users navigating a web page without a mouse.
+
+#OverrideThrowableToString=WARN
+# To return a custom message with a Throwable class, one should override getMessage() instead of toString().
+
+Overrides=ERROR
+# Varargs doesn't agree for overridden method
+
+#OverridesGuiceInjectableMethod=WARN
+# This method is not annotated with @Inject, but it overrides a method that is annotated with @com.google.inject.Inject. Guice will inject this method, and it is recommended to annotate it explicitly.
+
+#ParameterName=WARN
+# Detects `/* name= */`-style comments on actual parameters where the name doesn't match the formal parameter
+
+#PreconditionsCheckNotNullRepeated=WARN
+# Including the first argument of checkNotNull in the failure message is not useful, as it will always be `null`.
+
+#PrimitiveAtomicReference=WARN
+# Using compareAndSet with boxed primitives is dangerous, as reference rather than value equality is used. Consider using AtomicInteger, AtomicLong, AtomicBoolean from JDK or AtomicDouble from Guava instead.
+
+#ProtectedMembersInFinalClass=WARN
+# Protected members in final classes can be package-private
+
+#ProtoDurationGetSecondsGetNano=WARN
+# getNanos() only accesses the underlying nanosecond-adjustment of the duration.
+
+#ProtoRedundantSet=WARN
+# A field on a protocol buffer was set twice in the same chained expression.
+
+#ProtoTimestampGetSecondsGetNano=WARN
+# getNanos() only accesses the underlying nanosecond-adjustment of the instant.
+
+#QualifierOrScopeOnInjectMethod=WARN
+# Qualifiers/Scope annotations on @Inject methods don't have any effect. Move the qualifier annotation to the binding location.
+
+#ReachabilityFenceUsage=WARN
+# reachabilityFence should always be called inside a finally block
+
+#ReferenceEquality=WARN
+# Comparison using reference equality instead of value equality
+
+#RethrowReflectiveOperationExceptionAsLinkageError=WARN
+# Prefer LinkageError for rethrowing ReflectiveOperationException as unchecked
+
+#ReturnFromVoid=WARN
+# Void methods should not have a @return tag.
+
+#RobolectricShadowDirectlyOn=WARN
+# Migrate off a deprecated overload of org.robolectric.shadow.api.Shadow#directlyOn
+
+#RxReturnValueIgnored=WARN
+# Returned Rx objects must be checked. Ignoring a returned Rx value means it is never scheduled for execution
+
+#SameNameButDifferent=WARN
+# This type name shadows another in a way that may be confusing.
+
+#SelfAlwaysReturnsThis=WARN
+# Non-abstract instance methods named 'self()' that return the enclosing class must always 'return this'.
+
+#ShortCircuitBoolean=WARN
+# Prefer the short-circuiting boolean operators && and || to & and |.
+
+StaticAssignmentInConstructor=ERROR
+# This assignment is to a static field. Mutating static state from a constructor is highly error-prone.
+
+#StaticAssignmentOfThrowable=WARN
+# Saving instances of Throwable in static fields is discouraged, prefer to create them on-demand when an exception is thrown
+
+#StaticGuardedByInstance=WARN
+# Writes to static fields should not be guarded by instance locks
+
+#StaticMockMember=WARN
+# @Mock members of test classes shouldn't share state between tests and preferably be non-static
+
+#StreamResourceLeak=WARN
+# Streams that encapsulate a closeable resource should be closed using try-with-resources
+
+StreamToIterable=ERROR
+# Using stream::iterator creates a one-shot Iterable, which may cause surprising failures.
+
+#StringSplitter=WARN
+# String.split(String) has surprising behavior
+
+#SwigMemoryLeak=WARN
+# SWIG generated code that can't call a C++ destructor will leak memory
+
+SynchronizeOnNonFinalField=ERROR
+# Synchronizing on non-final fields is not safe: if the field is ever updated, different threads may end up locking on different objects.
+
+#ThreadJoinLoop=WARN
+# Thread.join needs to be immediately surrounded by a loop until it succeeds. Consider using Uninterruptibles.joinUninterruptibly.
+
+ThreadLocalUsage=ERROR
+# ThreadLocals should be stored in static fields
+
+#ThreadPriorityCheck=WARN
+# Relying on the thread scheduler is discouraged.
+
+#ThreeLetterTimeZoneID=WARN
+# Three-letter time zone identifiers are deprecated, may be ambiguous, and might not do what you intend; the full IANA time zone ID should be used instead.
+
+#TimeUnitConversionChecker=WARN
+# This TimeUnit conversion looks buggy: converting from a smaller unit to a larger unit (and passing a constant), converting to/from the same TimeUnit, or converting TimeUnits where the result is statically known to be 0 or 1 are all buggy patterns.
+
+#ToStringReturnsNull=WARN
+# An implementation of Object.toString() should never return null.
+
+#TruthAssertExpected=WARN
+# The actual and expected values appear to be swapped, which results in poor assertion failure messages. The actual value should come first.
+
+#TruthConstantAsserts=WARN
+# Truth Library assert is called on a constant.
+
+#TruthGetOrDefault=WARN
+# Asserting on getOrDefault is unclear; prefer containsEntry or doesNotContainKey
+
+#TruthIncompatibleType=WARN
+# Argument is not compatible with the subject's type.
+
+#TypeEquals=WARN
+# com.sun.tools.javac.code.Type doesn't override Object.equals and instances are not interned by javac, so testing types for equality should be done with Types#isSameType instead
+
+#TypeNameShadowing=WARN
+# Type parameter declaration shadows another named type
+
+#TypeParameterShadowing=WARN
+# Type parameter declaration overrides another type parameter already declared
+
+#TypeParameterUnusedInFormals=WARN
+# Declaring a type parameter that is only used in the return type is a misuse of generics: operations on the type parameter are unchecked, it hides unsafe casts at invocations of the method, and it interacts badly with method overload resolution.
+
+URLEqualsHashCode=ERROR
+# Avoid hash-based containers of java.net.URL–the containers rely on equals() and hashCode(), which cause java.net.URL to make blocking internet connections.
+
+#UndefinedEquals=WARN
+# This type is not guaranteed to implement a useful #equals method.
+
+#UnescapedEntity=WARN
+# Javadoc is interpreted as HTML, so HTML entities such as &, <, > must be escaped. If this finding seems wrong (e.g. is within a @code or @literal tag), check whether the tag could be malformed and not recognised by the compiler.
+
+#UnicodeEscape=WARN
+# Using unicode escape sequences for printable ASCII characters is obfuscated, and potentially dangerous.
+
+#UnnecessaryAssignment=WARN
+# Fields annotated with @Inject/@Mock should not be manually assigned to, as they should be initialized by a framework. Remove the assignment if a framework is being used, or the annotation if one isn't.
+
+UnnecessaryLambda=ERROR
+# Returning a lambda from a helper method or saving it in a constant is unnecessary; prefer to implement the functional interface method directly and use a method reference instead.
+
+#UnnecessaryLongToIntConversion=WARN
+# Converting a long or Long to an int to pass as a long parameter is usually not necessary. If this conversion is intentional, consider `Longs.constrainToRange()` instead.
+
+#UnnecessaryMethodInvocationMatcher=WARN
+# It is not necessary to wrap a MethodMatcher with methodInvocation().
+
+#UnnecessaryMethodReference=WARN
+# This method reference is unnecessary, and can be replaced with the variable itself.
+
+#UnnecessaryParentheses=WARN
+# These grouping parentheses are unnecessary; it is unlikely the code will be misinterpreted without them
+
+#UnrecognisedJavadocTag=WARN
+# This Javadoc tag wasn't recognised by the parser. Is it malformed somehow, perhaps with mismatched braces?
+
+#UnsafeFinalization=WARN
+# Finalizer may run before native code finishes execution
+
+#UnsafeReflectiveConstructionCast=WARN
+# Prefer `asSubclass` instead of casting the result of `newInstance`, to detect classes of incorrect type before invoking their constructors.This way, if the class is of the incorrect type,it will throw an exception before invoking its constructor.
+
+#UnsynchronizedOverridesSynchronized=WARN
+# Unsynchronized method overrides a synchronized method.
+
+UnusedMethod=ERROR
+# Unused.
+
+#UnusedNestedClass=WARN
+# This nested class is unused, and can be removed.
+
+#UnusedTypeParameter=WARN
+# This type parameter is unused and can be removed.
+
+#UnusedVariable=WARN
+# Unused.
+
+#UseBinds=WARN
+# @Binds is a more efficient and declarative mechanism for delegating a binding.
+
+UseCorrectAssertInTests=ERROR
+# Java assert is used in test. For testing purposes Assert.* matchers should be used.
+
+#VariableNameSameAsType=WARN
+# variableName and type with the same name would refer to the static field instead of the class
+
+#WaitNotInLoop=WARN
+# Because of spurious wakeups, Object.wait() and Condition.await() must always be called in a loop
+
+#WakelockReleasedDangerously=WARN
+# A wakelock acquired with a timeout may be released by the system before calling `release`, even after checking `isHeld()`. If so, it will throw a RuntimeException. Please wrap in a try/catch block.
+
+#WithSignatureDiscouraged=WARN
+# withSignature is discouraged. Prefer .named and/or .withParameters where possible.
+
+####################################################################################################
+# Experimental : ERROR
+# See https://errorprone.info/bugpatterns
+####################################################################################################
+
+#AndroidJdkLibsChecker=ERROR
+# Use of class, field, or method that is not compatible with legacy Android devices
+
+#AutoFactoryAtInject=ERROR
+# @AutoFactory and @Inject should not be used in the same type.
+
+#BanSerializableRead=ERROR
+# Deserializing user input via the `Serializable` API is extremely dangerous
+
+#ClassName=ERROR
+# The source file name should match the name of the top-level class it contains
+
+#ComparisonContractViolated=ERROR
+# This comparison method violates the contract
+
+#DeduplicateConstants=ERROR
+# This expression was previously declared as a constant; consider replacing this occurrence.
+
+#DepAnn=ERROR
+# Item documented with a @deprecated javadoc note is not annotated with @Deprecated
+
+#EmptyIf=ERROR
+# Empty statement after if
+
+#ExtendsAutoValue=ERROR
+# Do not extend an @AutoValue/@AutoOneOf class in non-generated code.
+
+#InjectMoreThanOneQualifier=ERROR
+# Using more than one qualifier annotation on the same element is not allowed.
+
+#InjectScopeOrQualifierAnnotationRetention=ERROR
+# Scoping and qualifier annotations must have runtime retention.
+
+#InsecureCryptoUsage=ERROR
+# A standard cryptographic operation is used in a mode that is prone to vulnerabilities
+
+#IterablePathParameter=ERROR
+# Path implements Iterable; prefer Collection for clarity
+
+#Java7ApiChecker=ERROR
+# Use of class, field, or method that is not compatible with JDK 7
+
+#Java8ApiChecker=ERROR
+# Use of class, field, or method that is not compatible with JDK 8
+
+#LockMethodChecker=ERROR
+# This method does not acquire the locks specified by its @LockMethod annotation
+
+#LongLiteralLowerCaseSuffix=ERROR
+# Prefer 'L' to 'l' for the suffix to long literals
+
+#NoAllocation=ERROR
+# @NoAllocation was specified on this method, but something was found that would trigger an allocation
+
+#RefersToDaggerCodegen=ERROR
+# Don't refer to Dagger's internal or generated code
+
+#StaticOrDefaultInterfaceMethod=ERROR
+# Static and default interface methods are not natively supported on older Android devices.
+
+#StaticQualifiedUsingExpression=ERROR
+# A static variable or method should be qualified with a class name, not expression
+
+#UnlockMethod=ERROR
+# This method does not acquire the locks specified by its @UnlockMethod annotation
+
+####################################################################################################
+# Experimental : WARNING
+# See https://errorprone.info/bugpatterns
+####################################################################################################
+
+#AnnotationPosition=WARN
+# Annotations should be positioned after Javadocs, but before modifiers.
+
+#AssertFalse=WARN
+# Assertions may be disabled at runtime and do not guarantee that execution will halt here; consider throwing an exception instead
+
+#AssistedInjectAndInjectOnConstructors=WARN
+# @AssistedInject and @Inject should not be used on different constructors in the same class.
+
+#BinderIdentityRestoredDangerously=WARN
+# A call to Binder.clearCallingIdentity() should be followed by Binder.restoreCallingIdentity() in a finally block. Otherwise the wrong Binder identity may be used by subsequent code.
+
+#BindingToUnqualifiedCommonType=WARN
+# This code declares a binding for a common value type without a Qualifier annotation.
+
+#CannotMockFinalClass=WARN
+# Mockito cannot mock final classes
+
+#CatchingUnchecked=WARN
+# This catch block catches `Exception`, but can only catch unchecked exceptions. Consider catching RuntimeException (or something more specific) instead so it is more apparent that no checked exceptions are being handled.
+
+#CheckedExceptionNotThrown=WARN
+# This method cannot throw a checked exception that it claims to. This may cause consumers of the API to incorrectly attempt to handle, or propagate, this exception.
+
+ConstantPatternCompile=WARN
+# Variables initialized with Pattern#compile calls on constants can be constants
+
+#DifferentNameButSame=WARN
+# This type is referred to in different ways within this file, which may be confusing.
+
+#EqualsBrokenForNull=WARN
+# equals() implementation may throw NullPointerException when given null
+
+#ExpectedExceptionChecker=WARN
+# Prefer assertThrows to ExpectedException
+
+#FloggerLogWithCause=WARN
+# Setting the caught exception as the cause of the log message may provide more context for anyone debugging errors.
+
+#FloggerMessageFormat=WARN
+# Invalid message format-style format specifier ({0}), expected printf-style (%s)
+
+#FloggerPassedAround=WARN
+# There is no advantage to passing around a logger rather than declaring one in the class that needs it.
+
+#FloggerRedundantIsEnabled=WARN
+# Logger level check is already implied in the log() call. An explicit atLevel.isEnabled() check is redundant.
+
+#FloggerRequiredModifiers=WARN
+# FluentLogger.forEnclosingClass should always be saved to a private static final field.
+
+#FloggerWithCause=WARN
+# Calling withCause(Throwable) with an inline allocated Throwable is discouraged. Consider using withStackTrace(StackSize) instead, and specifying a reduced stack size (e.g. SMALL, MEDIUM or LARGE) instead of FULL, to improve performance.
+
+#FloggerWithoutCause=WARN
+# Use withCause to associate Exceptions with log statements
+
+#FunctionalInterfaceClash=WARN
+# Overloads will be ambiguous when passing lambda arguments.
+
+#HardCodedSdCardPath=WARN
+# Hardcoded reference to /sdcard
+
+#InconsistentOverloads=WARN
+# The ordering of parameters in overloaded methods should be as consistent as possible (when viewed from left to right)
+
+#InitializeInline=WARN
+# Initializing variables in their declaring statement is clearer, where possible.
+
+#InterfaceWithOnlyStatics=WARN
+# This interface only contains static fields and methods; consider making it a final class instead to prevent subclassing.
+
+#InterruptedExceptionSwallowed=WARN
+# This catch block appears to be catching an explicitly declared InterruptedException as an Exception/Throwable and not handling the interruption separately.
+
+#MemberName=WARN
+# Methods and non-static variables should be named in lowerCamelCase.
+
+#MissingDefault=WARN
+# The Google Java Style Guide requires that each switch statement includes a default statement group, even if it contains no code. (This requirement is lifted for any switch statement that covers all values of an enum.)
+
+#NonCanonicalStaticMemberImport=WARN
+# Static import of member uses non-canonical name
+
+#PreferJavaTimeOverload=WARN
+# Prefer using java.time-based APIs when available. Note that this checker does not and cannot guarantee that the overloads have equivalent semantics, but that is generally the case with overloaded methods.
+
+#PreferredInterfaceType=WARN
+# This type can be more specific.
+
+PrimitiveArrayPassedToVarargsMethod=WARN
+# Passing a primitive array to a varargs method is usually wrong
+
+#QualifierWithTypeUse=WARN
+# Injection frameworks currently don't understand Qualifiers in TYPE_PARAMETER or TYPE_USE contexts.
+
+RedundantOverride=ERROR
+# This overriding method is redundant, and can be removed.
+
+RedundantThrows=ERROR
+# Thrown exception is a subtype of another
+
+StronglyTypeByteString=WARN
+# This primitive byte array is only used to construct ByteStrings. It would be clearer to strongly type the field instead.
+
+StronglyTypeTime=WARN
+# This primitive integral type is only used to construct time types. It would be clearer to strongly type the field instead.
+
+#SuppressWarningsWithoutExplanation=WARN
+# Use of @SuppressWarnings should be accompanied by a comment describing why the warning is safe to ignore.
+
+#SystemExitOutsideMain=WARN
+# Code that contains System.exit() is untestable.
+
+#SystemOut=WARN
+# Printing to standard output should only be used for debugging, not in production code
+
+#TestExceptionChecker=WARN
+# Using @Test(expected=…) is discouraged, since the test will pass if any statement in the test method throws the expected exception
+
+#ThrowSpecificExceptions=WARN
+# Base exception classes should be treated as abstract. If the exception is intended to be caught, throw a domain-specific exception. Otherwise, prefer a more specific exception for clarity. Common alternatives include: AssertionError, IllegalArgumentException, IllegalStateException, and (Guava's) VerifyException.
+
+#TimeUnitMismatch=WARN
+# An value that appears to be represented in one unit is used where another appears to be required (e.g., seconds where nanos are needed)
+
+#TooManyParameters=WARN
+# A large number of parameters on public APIs should be avoided.
+
+#TransientMisuse=WARN
+# Static fields are implicitly transient, so the explicit modifier is unnecessary
+
+#TryWithResourcesVariable=WARN
+# This variable is unnecessary, the try-with-resources resource can be a reference to a final or effectively final variable
+
+#UnnecessarilyFullyQualified=WARN
+# This fully qualified name is unambiguous to the compiler if imported.
+
+#UnnecessarilyVisible=WARN
+# Some methods (such as those annotated with @Inject or @Provides) are only intended to be called by a framework, and so should have default visibility.
+
+#UnnecessaryAnonymousClass=WARN
+# Implementing a functional interface is unnecessary; prefer to implement the functional interface method directly and use a method reference instead.
+
+#UnnecessaryDefaultInEnumSwitch=WARN
+# Switch handles all enum values: an explicit default case is unnecessary and defeats error checking for non-exhaustive switches.
+
+#UnnecessaryFinal=WARN
+# Since Java 8, it's been unnecessary to make local variables and parameters `final` for use in lambdas or anonymous classes. Marking them as `final` is weakly discouraged, as it adds a fair amount of noise for minimal benefit.
+
+#UnnecessaryOptionalGet=WARN
+# This code can be simplified by directly using the lambda parameters instead of calling get..() on optional.
+
+#UnsafeLocaleUsage=WARN
+# Possible unsafe operation related to the java.util.Locale library.
+
+#UnusedException=WARN
+# This catch block catches an exception and re-throws another, but swallows the caught exception rather than setting it as a cause. This can make debugging harder.
+
+#UrlInSee=WARN
+# URLs should not be used in @see tags; they are designed for Java elements which could be used with @link.
+
+#UsingJsr305CheckReturnValue=WARN
+# Prefer ErrorProne's @CheckReturnValue over JSR305's version.
+
+#Var=WARN
+# Non-constant variable missing @Var annotation
+
+#Varifier=WARN
+# Consider using `var` here to avoid boilerplate.
+
+####################################################################################################
+# Experimental : SUGGESTION
+# See https://errorprone.info/bugpatterns
+####################################################################################################
+
+#AnnotationMirrorToString=WARN
+# AnnotationMirror#toString doesn't use fully qualified type names, prefer auto-common's AnnotationMirrors#toString
+
+#AnnotationValueToString=WARN
+# AnnotationValue#toString doesn't use fully qualified type names, prefer auto-common's AnnotationValues#toString
+
+#BooleanParameter=WARN
+# Use parameter comments to document ambiguous literals
+
+#ClassNamedLikeTypeParameter=WARN
+# This class's name looks like a Type Parameter.
+
+#ConstantField=WARN
+# Fields with CONSTANT_CASE names should be both static and final
+
+#EqualsMissingNullable=WARN
+# Method overrides Object.equals but does not have @Nullable on its parameter
+
+#FieldCanBeFinal=WARN
+# This field is only assigned during initialization; consider making it final
+
+#FieldCanBeLocal=WARN
+# This field can be replaced with a local variable in the methods that use it.
+
+#FieldCanBeStatic=WARN
+# A final field initialized at compile-time with an instance of an immutable type can be static.
+
+#FieldMissingNullable=WARN
+# Field is assigned (or compared against) a definitely null value but is not annotated @Nullable
+
+#ForEachIterable=WARN
+# This loop can be replaced with an enhanced for loop.
+
+#ImmutableMemberCollection=WARN
+# If you don't intend to mutate a member collection prefer using Immutable types.
+
+#ImmutableRefactoring=WARN
+# Refactors uses of the JSR 305 @Immutable to Error Prone's annotation
+
+#ImmutableSetForContains=WARN
+# This private static ImmutableList is only used for contains, containsAll or isEmpty checks; prefer ImmutableSet.
+
+#ImplementAssertionWithChaining=WARN
+# Prefer check(…), which usually generates more readable failure messages.
+
+#LambdaFunctionalInterface=WARN
+# Use Java's utility functional interfaces instead of Function for primitive types.
+
+#MethodCanBeStatic=WARN
+# A private method that does not reference the enclosing instance can be static
+
+#MissingBraces=WARN
+# The Google Java Style Guide requires braces to be used with if, else, for, do and while statements, even when the body is empty or contains only a single statement.
+
+#MixedArrayDimensions=WARN
+# C-style array declarations should not be used
+
+#MultiVariableDeclaration=WARN
+# Variable declarations should declare only one variable
+
+#MultipleTopLevelClasses=WARN
+# Source files should not contain multiple top-level class declarations
+
+#PackageLocation=WARN
+# Package names should match the directory they are declared in
+
+#ParameterComment=WARN
+# Non-standard parameter comment; prefer `/* paramName= */ arg`
+
+#ParameterMissingNullable=WARN
+# Parameter has handling for null but is not annotated @Nullable
+
+#PrivateConstructorForNoninstantiableModule=WARN
+# Add a private constructor to modules that will not be instantiated by Dagger.
+
+#PrivateConstructorForUtilityClass=WARN
+# Classes which are not intended to be instantiated should be made non-instantiable with a private constructor. This includes utility classes (classes with only static members), and the main class.
+
+#PublicApiNamedStreamShouldReturnStream=WARN
+# Public methods named stream() are generally expected to return a type whose name ends with Stream. Consider choosing a different method name instead.
+
+#RemoveUnusedImports=WARN
+# Unused imports
+
+#ReturnMissingNullable=WARN
+# Method returns a definitely null value but is not annotated @Nullable
+
+#ReturnsNullCollection=WARN
+# Method has a collection return type and returns {@code null} in some cases but does not annotate the method as @Nullable. See Effective Java 3rd Edition Item 54.
+
+#ScopeOnModule=WARN
+# Scopes on modules have no function and will soon be an error.
+
+#SwitchDefault=WARN
+# The default case of a switch should appear at the end of the last statement group
+
+#SymbolToString=WARN
+# Symbol#toString shouldn't be used for comparison as it is expensive and fragile.
+
+#ThrowsUncheckedException=WARN
+# Unchecked exceptions do not need to be declared in the method signature.
+
+TryFailRefactoring=WARN
+# Prefer assertThrows to try/fail
+
+#TypeParameterNaming=WARN
+# Type parameters must be a single letter with an optional numeric suffix, or an UpperCamelCase name followed by the letter 'T'.
+
+#TypeToString=WARN
+# Type#toString shouldn't be used for comparison as it is expensive and fragile.
+
+#UngroupedOverloads=WARN
+# Constructors and methods with the same name should appear sequentially with no other code in between, even when modifiers such as static or private differ between the methods. Please re-order or re-name methods.
+
+UnnecessaryBoxedAssignment=WARN
+# This expression can be implicitly boxed.
+
+UnnecessaryBoxedVariable=ERROR
+# It is unnecessary for this variable to be boxed. Use the primitive instead.
+
+#UnnecessarySetDefault=WARN
+# Unnecessary call to NullPointerTester#setDefault
+
+#UnnecessaryStaticImport=WARN
+# Using static imports for types is unnecessary
+
+UseEnumSwitch=WARN
+# Prefer using a switch instead of a chained if-else for enums
+
+#VoidMissingNullable=WARN
+# The type Void is not annotated @Nullable
+
+#WildcardImport=WARN
+# Wildcard imports, static or otherwise, should not be used
+
+####################################################################################################
+# SLF4j
+# See https://github.com/KengoTODA/errorprone-slf4j
+####################################################################################################
+
+Slf4jPlaceholderMismatch=ERROR
+Slf4jFormatShouldBeConst=ERROR
+Slf4jLoggerShouldBePrivate=ERROR
+Slf4jLoggerShouldBeFinal=ERROR
+Slf4jLoggerShouldBeNonStatic=OFF
+Slf4jIllegalPassedClass=ERROR
+#Slf4jSignOnlyFormat=OFF
+Slf4jDoNotLogMessageOfExceptionExplicitly=OFF
+
diff --git a/iceberg-catalog-migrator/codestyle/intellij-java-google-style.xml b/iceberg-catalog-migrator/codestyle/intellij-java-google-style.xml
new file mode 100644
index 00000000..f3a6743e
--- /dev/null
+++ b/iceberg-catalog-migrator/codestyle/intellij-java-google-style.xml
@@ -0,0 +1,598 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ xmlns:android
+ ^$
+
+
+
+
+
+
+
+
+ xmlns:.*
+ ^$
+
+
+ BY_NAME
+
+
+
+
+
+
+ .*:id
+ http://schemas.android.com/apk/res/android
+
+
+
+
+
+
+
+
+ style
+ ^$
+
+
+
+
+
+
+
+
+ .*
+ ^$
+
+
+ BY_NAME
+
+
+
+
+
+
+ .*:.*Style
+ http://schemas.android.com/apk/res/android
+
+
+ BY_NAME
+
+
+
+
+
+
+ .*:layout_width
+ http://schemas.android.com/apk/res/android
+
+
+
+
+
+
+
+
+ .*:layout_height
+ http://schemas.android.com/apk/res/android
+
+
+
+
+
+
+
+
+ .*:layout_weight
+ http://schemas.android.com/apk/res/android
+
+
+
+
+
+
+
+
+ .*:layout_margin
+ http://schemas.android.com/apk/res/android
+
+
+
+
+
+
+
+
+ .*:layout_marginTop
+ http://schemas.android.com/apk/res/android
+
+
+
+
+
+
+
+
+ .*:layout_marginBottom
+ http://schemas.android.com/apk/res/android
+
+
+
+
+
+
+
+
+ .*:layout_marginStart
+ http://schemas.android.com/apk/res/android
+
+
+
+
+
+
+
+
+ .*:layout_marginEnd
+ http://schemas.android.com/apk/res/android
+
+
+
+
+
+
+
+
+ .*:layout_marginLeft
+ http://schemas.android.com/apk/res/android
+
+
+
+
+
+
+
+
+ .*:layout_marginRight
+ http://schemas.android.com/apk/res/android
+
+
+
+
+
+
+
+
+ .*:layout_.*
+ http://schemas.android.com/apk/res/android
+
+
+ BY_NAME
+
+
+
+
+
+
+ .*:padding
+ http://schemas.android.com/apk/res/android
+
+
+
+
+
+
+
+
+ .*:paddingTop
+ http://schemas.android.com/apk/res/android
+
+
+
+
+
+
+
+
+ .*:paddingBottom
+ http://schemas.android.com/apk/res/android
+
+
+
+
+
+
+
+
+ .*:paddingStart
+ http://schemas.android.com/apk/res/android
+
+
+
+
+
+
+
+
+ .*:paddingEnd
+ http://schemas.android.com/apk/res/android
+
+
+
+
+
+
+
+
+ .*:paddingLeft
+ http://schemas.android.com/apk/res/android
+
+
+
+
+
+
+
+
+ .*:paddingRight
+ http://schemas.android.com/apk/res/android
+
+
+
+
+
+
+
+
+ .*
+ http://schemas.android.com/apk/res/android
+
+
+ BY_NAME
+
+
+
+
+
+
+ .*
+ http://schemas.android.com/apk/res-auto
+
+
+ BY_NAME
+
+
+
+
+
+
+ .*
+ http://schemas.android.com/tools
+
+
+ BY_NAME
+
+
+
+
+
+
+ .*
+ .*
+
+
+ BY_NAME
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/iceberg-catalog-migrator/codestyle/org.eclipse.wst.xml.core.prefs b/iceberg-catalog-migrator/codestyle/org.eclipse.wst.xml.core.prefs
new file mode 100644
index 00000000..9fe6a375
--- /dev/null
+++ b/iceberg-catalog-migrator/codestyle/org.eclipse.wst.xml.core.prefs
@@ -0,0 +1,7 @@
+eclipse.preferences.version=1
+formatCommentJoinLines=false
+formatCommentText=false
+indentationChar=space
+indentationSize=2
+lineWidth=100
+spaceBeforeEmptyCloseTag=false
diff --git a/iceberg-catalog-migrator/gradle.properties b/iceberg-catalog-migrator/gradle.properties
new file mode 100644
index 00000000..8f3a93cf
--- /dev/null
+++ b/iceberg-catalog-migrator/gradle.properties
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# enable the Gradle build cache
+org.gradle.caching=true
+# enable Gradle parallel builds
+org.gradle.parallel=true
+# configure only necessary Gradle tasks
+org.gradle.configureondemand=true
+# explicitly disable the configuration cache
+org.gradle.configuration-cache=false
+#org.gradle.configuration-cache-problems=warn
+# bump the Gradle daemon heap size (you can set bigger heap sizes as well)
+org.gradle.jvmargs=-Xms2g -Xmx4g -XX:MaxMetaspaceSize=768m
\ No newline at end of file
diff --git a/iceberg-catalog-migrator/gradle/baselibs.versions.toml b/iceberg-catalog-migrator/gradle/baselibs.versions.toml
new file mode 100644
index 00000000..18d8384a
--- /dev/null
+++ b/iceberg-catalog-migrator/gradle/baselibs.versions.toml
@@ -0,0 +1,15 @@
+# Dependencies needed by buildSrc/
+
+[versions]
+errorpronePlugin = "4.1.0"
+ideaExt = "1.1.10"
+jandexPlugin = "1.90"
+shadowPlugin = "8.1.1"
+spotlessPlugin = "7.0.2"
+
+[libraries]
+errorprone = { module = "net.ltgt.gradle:gradle-errorprone-plugin", version.ref = "errorpronePlugin" }
+idea-ext = { module = "gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext", version.ref = "ideaExt" }
+jandex = { module = "com.github.vlsi.gradle:jandex-plugin", version.ref = "jandexPlugin" }
+shadow = { module = "com.github.johnrengelman:shadow", version.ref = "shadowPlugin" }
+spotless = { module = "com.diffplug.spotless:spotless-plugin-gradle", version.ref = "spotlessPlugin" }
diff --git a/iceberg-catalog-migrator/gradle/gradlew-include.sh b/iceberg-catalog-migrator/gradle/gradlew-include.sh
new file mode 100644
index 00000000..38251e36
--- /dev/null
+++ b/iceberg-catalog-migrator/gradle/gradlew-include.sh
@@ -0,0 +1,65 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Downloads the gradle-wrapper.jar if necessary and verifies its integrity.
+# Included from /.gradlew
+
+# Extract the Gradle version from gradle-wrapper.properties.
+GRADLE_DIST_VERSION="$(grep distributionUrl= "$APP_HOME/gradle/wrapper/gradle-wrapper.properties" | sed 's/^.*gradle-\([0-9.]*\)-[a-z]*.zip$/\1/')"
+GRADLE_WRAPPER_SHA256="$APP_HOME/gradle/wrapper/gradle-wrapper-${GRADLE_DIST_VERSION}.jar.sha256"
+GRADLE_WRAPPER_JAR="$APP_HOME/gradle/wrapper/gradle-wrapper.jar"
+if [ -x "$(command -v sha256sum)" ] ; then
+ SHASUM="sha256sum"
+else
+ if [ -x "$(command -v shasum)" ] ; then
+ SHASUM="shasum -a 256"
+ else
+ echo "Neither sha256sum nor shasum are available, install either." > /dev/stderr
+ exit 1
+ fi
+fi
+if [ ! -e "${GRADLE_WRAPPER_SHA256}" ]; then
+ # Delete the wrapper jar, if the checksum file does not exist.
+ rm -f "${GRADLE_WRAPPER_JAR}"
+fi
+if [ -e "${GRADLE_WRAPPER_JAR}" ]; then
+ # Verify the wrapper jar, if it exists, delete wrapper jar and checksum file, if the checksums
+ # do not match.
+ JAR_CHECKSUM="$(${SHASUM} "${GRADLE_WRAPPER_JAR}" | cut -d\ -f1)"
+ EXPECTED="$(cat "${GRADLE_WRAPPER_SHA256}")"
+ if [ "${JAR_CHECKSUM}" != "${EXPECTED}" ]; then
+ rm -f "${GRADLE_WRAPPER_JAR}" "${GRADLE_WRAPPER_SHA256}"
+ fi
+fi
+if [ ! -e "${GRADLE_WRAPPER_SHA256}" ]; then
+ curl --location --output "${GRADLE_WRAPPER_SHA256}" https://services.gradle.org/distributions/gradle-${GRADLE_DIST_VERSION}-wrapper.jar.sha256 || exit 1
+fi
+if [ ! -e "${GRADLE_WRAPPER_JAR}" ]; then
+ # The Gradle version extracted from the `distributionUrl` property does not contain ".0" patch
+ # versions. Need to append a ".0" in that case to download the wrapper jar.
+ GRADLE_VERSION="$(echo "$GRADLE_DIST_VERSION" | sed 's/^\([0-9]*[.][0-9]*\)$/\1.0/')"
+ curl --location --output "${GRADLE_WRAPPER_JAR}" https://raw.githubusercontent.com/gradle/gradle/v${GRADLE_VERSION}/gradle/wrapper/gradle-wrapper.jar || exit 1
+ JAR_CHECKSUM="$(${SHASUM} "${GRADLE_WRAPPER_JAR}" | cut -d\ -f1)"
+ EXPECTED="$(cat "${GRADLE_WRAPPER_SHA256}")"
+ if [ "${JAR_CHECKSUM}" != "${EXPECTED}" ]; then
+ # If the (just downloaded) checksum and the downloaded wrapper jar do not match, something
+ # really bad is going on.
+ echo "Expected sha256 of the downloaded gradle-wrapper.jar does not match the downloaded sha256!" > /dev/stderr
+ exit 1
+ fi
+fi
diff --git a/iceberg-catalog-migrator/gradle/libs.versions.toml b/iceberg-catalog-migrator/gradle/libs.versions.toml
new file mode 100644
index 00000000..5482ba8a
--- /dev/null
+++ b/iceberg-catalog-migrator/gradle/libs.versions.toml
@@ -0,0 +1,64 @@
+[versions]
+assertj = "3.27.3"
+aws = "2.30.3" # this is in mapping with iceberg repo.
+checkstyle = "10.21.3"
+errorprone = "2.36.0"
+errorproneSlf4j = "0.1.28"
+googleJavaFormat = "1.25.2"
+guava = "33.4.0-jre"
+hadoop = "2.7.3" # this is in mapping with iceberg repo.
+hive = "2.3.9" # this is in mapping with iceberg repo.
+iceberg = "1.7.1"
+immutables = "2.10.1"
+jacoco = "0.8.12"
+jandex = "3.2.3"
+junit = "5.12.0"
+logback = "1.5.17"
+logcaptor = "2.10.1"
+nessie = "0.101.3"
+nessieRunner = "0.32.2"
+picocli = "4.7.6"
+shadowPlugin = "8.1.1"
+slf4j = "1.7.36"
+
+[libraries]
+assertj = { module = "org.assertj:assertj-core", version.ref = "assertj" }
+aws-sdk-apache-client = { module = "software.amazon.awssdk:apache-client", version.ref = "aws" }
+aws-sdk-auth = { module = "software.amazon.awssdk:auth", version.ref = "aws" }
+aws-sdk-dynamo = { module = "software.amazon.awssdk:dynamodb", version.ref = "aws" }
+aws-sdk-glue = { module = "software.amazon.awssdk:glue", version.ref = "aws" }
+aws-sdk-kms = { module = "software.amazon.awssdk:kms", version.ref = "aws" }
+aws-sdk-lakeformation = { module = "software.amazon.awssdk:lakeformation", version.ref = "aws" }
+aws-sdk-sts = { module = "software.amazon.awssdk:sts", version.ref = "aws" }
+aws-sdk-s3 = { module = "software.amazon.awssdk:s3", version.ref = "aws" }
+aws-sdk-url-connection-client = { module = "software.amazon.awssdk:url-connection-client", version.ref = "aws" }
+checkstyle = { module = "com.puppycrawl.tools:checkstyle", version.ref = "checkstyle" }
+errorprone-annotations = { module = "com.google.errorprone:error_prone_annotations", version.ref = "errorprone" }
+errorprone-core = { module = "com.google.errorprone:error_prone_core", version.ref = "errorprone" }
+errorprone-slf4j = { module = "jp.skypencil.errorprone.slf4j:errorprone-slf4j", version.ref = "errorproneSlf4j" }
+findbugs-annotations = { module = "com.google.code.findbugs:annotations", version = "3.0.1" }
+findbugs-jsr305 = { module = "com.google.code.findbugs:jsr305", version = "3.0.2" }
+google-java-format = { module = "com.google.googlejavaformat:google-java-format", version.ref = "googleJavaFormat" }
+guava = { module = "com.google.guava:guava", version.ref = "guava" }
+hadoop-aws = { module = "org.apache.hadoop:hadoop-aws", version.ref = "hadoop" }
+hadoop-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "hadoop" }
+iceberg-dell = { module = "org.apache.iceberg:iceberg-dell", version.ref = "iceberg" }
+iceberg-spark-runtime = { module = "org.apache.iceberg:iceberg-spark-runtime-3.3_2.12", version.ref = "iceberg" }
+immutables-builder = { module = "org.immutables:builder", version.ref = "immutables" }
+immutables-value-annotations = { module = "org.immutables:value-annotations", version.ref = "immutables" }
+immutables-value-processor = { module = "org.immutables:value-processor", version.ref = "immutables" }
+jacoco-ant = { module = "org.jacoco:org.jacoco.ant", version.ref = "jacoco" }
+jacoco-report = { module = "org.jacoco:org.jacoco.report", version.ref = "jacoco" }
+jacoco-maven-plugin = { module = "org.jacoco:jacoco-maven-plugin", version.ref = "jacoco" }
+jandex = { module = "org.jboss:jandex", version.ref = "jandex" }
+junit-bom = { module = "org.junit:junit-bom", version.ref = "junit" }
+logback-classic = { module = "ch.qos.logback:logback-classic", version.ref = "logback" }
+logcaptor = { module = "io.github.hakky54:logcaptor", version.ref = "logcaptor" }
+picocli = { module = "info.picocli:picocli", version.ref = "picocli" }
+slf4j = { module = "org.slf4j:log4j-over-slf4j", version.ref = "slf4j" }
+
+[plugins]
+errorprone = { id = "net.ltgt.errorprone", version = "4.1.0" }
+nessie-run = { id = "org.projectnessie", version.ref = "nessieRunner" }
+nexus-publish-plugin = { id = "io.github.gradle-nexus.publish-plugin", version = "2.0.0" }
+shadow = { id = "com.github.johnrengelman.shadow", version.ref = "shadowPlugin" }
diff --git a/iceberg-catalog-migrator/gradle/wrapper/gradle-wrapper.properties b/iceberg-catalog-migrator/gradle/wrapper/gradle-wrapper.properties
new file mode 100644
index 00000000..36e4933e
--- /dev/null
+++ b/iceberg-catalog-migrator/gradle/wrapper/gradle-wrapper.properties
@@ -0,0 +1,8 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionSha256Sum=20f1b1176237254a6fc204d8434196fa11a4cfb387567519c61556e8710aed78
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-bin.zip
+networkTimeout=10000
+validateDistributionUrl=true
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
diff --git a/iceberg-catalog-migrator/gradlew b/iceberg-catalog-migrator/gradlew
new file mode 100755
index 00000000..2741145b
--- /dev/null
+++ b/iceberg-catalog-migrator/gradlew
@@ -0,0 +1,253 @@
+#!/bin/sh
+
+#
+# Copyright © 2015-2021 the original authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+##############################################################################
+#
+# Gradle start up script for POSIX generated by Gradle.
+#
+# Important for running:
+#
+# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
+# noncompliant, but you have some other compliant shell such as ksh or
+# bash, then to run this script, type that shell name before the whole
+# command line, like:
+#
+# ksh Gradle
+#
+# Busybox and similar reduced shells will NOT work, because this script
+# requires all of these POSIX shell features:
+# * functions;
+# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
+# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
+# * compound commands having a testable exit status, especially «case»;
+# * various built-in commands including «command», «set», and «ulimit».
+#
+# Important for patching:
+#
+# (2) This script targets any POSIX shell, so it avoids extensions provided
+# by Bash, Ksh, etc; in particular arrays are avoided.
+#
+# The "traditional" practice of packing multiple parameters into a
+# space-separated string is a well documented source of bugs and security
+# problems, so this is (mostly) avoided, by progressively accumulating
+# options in "$@", and eventually passing that to Java.
+#
+# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
+# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
+# see the in-line comments for details.
+#
+# There are tweaks for specific operating systems such as AIX, CygWin,
+# Darwin, MinGW, and NonStop.
+#
+# (3) This script is generated from the Groovy template
+# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+# within the Gradle project.
+#
+# You can find Gradle at https://github.com/gradle/gradle/.
+#
+##############################################################################
+
+# Attempt to set APP_HOME
+
+# Resolve links: $0 may be a link
+app_path=$0
+
+# Need this for daisy-chained symlinks.
+while
+ APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
+ [ -h "$app_path" ]
+do
+ ls=$( ls -ld "$app_path" )
+ link=${ls#*' -> '}
+ case $link in #(
+ /*) app_path=$link ;; #(
+ *) app_path=$APP_HOME$link ;;
+ esac
+done
+
+# This is normally unused
+# shellcheck disable=SC2034
+APP_BASE_NAME=${0##*/}
+# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
+
+. ${APP_HOME}/gradle/gradlew-include.sh
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD=maximum
+
+warn () {
+ echo "$*"
+} >&2
+
+die () {
+ echo
+ echo "$*"
+ echo
+ exit 1
+} >&2
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "$( uname )" in #(
+ CYGWIN* ) cygwin=true ;; #(
+ Darwin* ) darwin=true ;; #(
+ MSYS* | MINGW* ) msys=true ;; #(
+ NONSTOP* ) nonstop=true ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ JAVACMD=$JAVA_HOME/jre/sh/java
+ else
+ JAVACMD=$JAVA_HOME/bin/java
+ fi
+ if [ ! -x "$JAVACMD" ] ; then
+ die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+ fi
+else
+ JAVACMD=java
+ if ! command -v java >/dev/null 2>&1
+ then
+ die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+ fi
+fi
+
+# Increase the maximum file descriptors if we can.
+if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
+ case $MAX_FD in #(
+ max*)
+ # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
+ # shellcheck disable=SC2039,SC3045
+ MAX_FD=$( ulimit -H -n ) ||
+ warn "Could not query maximum file descriptor limit"
+ esac
+ case $MAX_FD in #(
+ '' | soft) :;; #(
+ *)
+ # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
+ # shellcheck disable=SC2039,SC3045
+ ulimit -n "$MAX_FD" ||
+ warn "Could not set maximum file descriptor limit to $MAX_FD"
+ esac
+fi
+
+# Collect all arguments for the java command, stacking in reverse order:
+# * args from the command line
+# * the main class name
+# * -classpath
+# * -D...appname settings
+# * --module-path (only if needed)
+# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
+
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if "$cygwin" || "$msys" ; then
+ APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
+ CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
+
+ JAVACMD=$( cygpath --unix "$JAVACMD" )
+
+ # Now convert the arguments - kludge to limit ourselves to /bin/sh
+ for arg do
+ if
+ case $arg in #(
+ -*) false ;; # don't mess with options #(
+ /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
+ [ -e "$t" ] ;; #(
+ *) false ;;
+ esac
+ then
+ arg=$( cygpath --path --ignore --mixed "$arg" )
+ fi
+ # Roll the args list around exactly as many times as the number of
+ # args, so each arg winds up back in the position where it started, but
+ # possibly modified.
+ #
+ # NB: a `for` loop captures its iteration list before it begins, so
+ # changing the positional parameters here affects neither the number of
+ # iterations, nor the values presented in `arg`.
+ shift # remove old arg
+ set -- "$@" "$arg" # push replacement arg
+ done
+fi
+
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Collect all arguments for the java command:
+# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+# and any embedded shellness will be escaped.
+# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
+# treated as '${Hostname}' itself on the command line.
+
+set -- \
+ "-Dorg.gradle.appname=$APP_BASE_NAME" \
+ -classpath "$CLASSPATH" \
+ org.gradle.wrapper.GradleWrapperMain \
+ "$@"
+
+# Stop when "xargs" is not available.
+if ! command -v xargs >/dev/null 2>&1
+then
+ die "xargs is not available"
+fi
+
+# Use "xargs" to parse quoted args.
+#
+# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
+#
+# In Bash we could simply go:
+#
+# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
+# set -- "${ARGS[@]}" "$@"
+#
+# but POSIX shell has neither arrays nor command substitution, so instead we
+# post-process each arg (as a line of input to sed) to backslash-escape any
+# character that might be a shell metacharacter, then use eval to reverse
+# that process (while maintaining the separation between arguments), and wrap
+# the whole thing up as a single "set" statement.
+#
+# This will of course break if any of these variables contains a newline or
+# an unmatched quote.
+#
+
+eval "set -- $(
+ printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
+ xargs -n1 |
+ sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
+ tr '\n' ' '
+ )" '"$@"'
+
+exec "$JAVACMD" "$@"
\ No newline at end of file
diff --git a/iceberg-catalog-migrator/ide-name.txt b/iceberg-catalog-migrator/ide-name.txt
new file mode 100644
index 00000000..1e77781d
--- /dev/null
+++ b/iceberg-catalog-migrator/ide-name.txt
@@ -0,0 +1 @@
+Iceberg Catalog Migrator
diff --git a/iceberg-catalog-migrator/settings.gradle.kts b/iceberg-catalog-migrator/settings.gradle.kts
new file mode 100644
index 00000000..72a71b06
--- /dev/null
+++ b/iceberg-catalog-migrator/settings.gradle.kts
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+val baseVersion = file("version.txt").readText().trim()
+
+rootProject.name = "polaris-iceberg-catalog-migrator"
+
+gradle.beforeProject {
+ group = "org.apache.polaris.tools"
+ version = baseVersion
+ description =
+ when (name) {
+ "api" -> "Iceberg catalog migrator - api implementation"
+ "api-test" -> "Iceberg catalog migrator - common test implementation"
+ "cli" -> "Iceberg catalog migrator - CLI implementation"
+ else -> name
+ }
+}
+
+fun catalogMigratorProject(name: String) {
+ include("iceberg-catalog-migrator-$name")
+ project(":iceberg-catalog-migrator-$name").projectDir = file(name)
+}
+
+catalogMigratorProject("api")
+
+catalogMigratorProject("api-test")
+
+catalogMigratorProject("cli")
+
+catalogMigratorProject("bom")
diff --git a/iceberg-catalog-migrator/version.txt b/iceberg-catalog-migrator/version.txt
new file mode 100644
index 00000000..4adba7ed
--- /dev/null
+++ b/iceberg-catalog-migrator/version.txt
@@ -0,0 +1 @@
+0.0.1-SNAPSHOT
\ No newline at end of file